mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-05 19:55:37 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 overtrue
|
9f87867495 | ||
|
overtrue
|
8386263b7a |
@@ -48,7 +48,6 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
|
||||
### S3 object actions, copy, multipart, and upload policy validation
|
||||
|
||||
- `GHSA-g8w9-qw9q-fghr`: a valid presigned `PutObject` accepted extra `x-amz-tagging`, website redirect, and storage-class headers omitted from `SignedHeaders`. Lesson: a presigned URL is a bounded capability; reject `x-amz-*` headers that are not cryptographically bound by the signature so unsigned metadata cannot change authorization, lifecycle, redirect, cost, or durability semantics.
|
||||
- `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
|
||||
- `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
|
||||
- `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
|
||||
@@ -120,7 +119,7 @@ Use these targeted searches when a diff touches security-sensitive code:
|
||||
```bash
|
||||
rg -n "validate_admin_request|check_permissions|AdminAction::|deny_only|is_allowed" rustfs crates
|
||||
rg -n "authorize_operation|FtpsDriver|SftpDriver|RETR|MKD|SIZE|MDTM|CreateBucket|GetObject|HeadObject" crates/protocols rustfs
|
||||
rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|content-length-range|starts-with" rustfs crates
|
||||
rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|content-length-range|starts-with" rustfs crates
|
||||
rg -n "ListBucketVersions|GetObjectVersion|versionId|VersionId|ExistingObjectTag|ForAllValues|ForAnyValue|POLICY_PLUGIN|opa" rustfs crates
|
||||
rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonicalize|\\.\\.|x-forwarded-for|x-real-ip|SourceIp" rustfs crates
|
||||
rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
|
||||
@@ -137,7 +136,6 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
- Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
|
||||
- IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
|
||||
- Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
|
||||
- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, and storage-class headers; require rejection before storage access, and verify explicitly signed equivalents still work.
|
||||
- Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
|
||||
- Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
|
||||
- Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=d6aa36cfaae2c4d8590482c7e47138c5965b335b34a75f50d11ffc3366e9021e
|
||||
sha256-linux=e3eb4ab7fc72224abf58c546ac0706d6605d3bd26bac7d8ce338829fd3daecc2
|
||||
sha256-linux=c8315465f50c194faee36141cdbb1e15e59271e524d948564a69e2d5eb408f2a
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=9c2b958035a038ffd5ab98cac5f59a1b8e6a16e141f109ec7fb956afc0f11105
|
||||
sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=8d5517f5f2fc32d561782dfccd51b7f746f5e25b2835e37e100c883f7f18777d
|
||||
sha256=655a3f3c1d042e694339d15caba7580518320322d1bac0f09450b37e6c09e2e7
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=dbebfbab9b9efd4eff31211e69dd32235dc00e207f2ab0dd919a1b2ac9e724c2
|
||||
sha256=294350518743cac8d7c41880a2835216e4b697908d7b0b1bc92b62816d94c59d
|
||||
|
||||
+11
-40
@@ -46,11 +46,6 @@ e2e-reliability = { max-threads = 1 }
|
||||
e2e-inline-boundaries = { max-threads = 1 }
|
||||
e2e-cluster-nightly = { max-threads = 1 }
|
||||
|
||||
# Deep async storage futures are composed into tests across several crates.
|
||||
# Keep the test stack bounded but above libtest's 2 MiB default.
|
||||
[scripts.setup.ecstore-base-stack]
|
||||
command = ['sh', '-c', 'echo RUST_MIN_STACK=4194304 >> "$NEXTEST_ENV"']
|
||||
|
||||
# These exact regression scenarios build deep async storage futures that exceed
|
||||
# libtest's 2 MiB spawned-thread stack on Linux. Give only their test processes
|
||||
# the same 32 MiB stack already used by the crate's dedicated large-stack tests.
|
||||
@@ -68,10 +63,6 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -109,29 +100,12 @@ test-group = 'embedded-test-ports'
|
||||
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the transition matrix tests. They build a 4-disk hermetic erasure
|
||||
# set, populate the get_object_metadata_cache, and assert generation lifecycle
|
||||
# semantics. serial_test's #[serial] has no effect across nextest's process
|
||||
# boundary, so concurrent execution races the shared metadata-cache generation
|
||||
# counter and causes spurious "metadata read should publish the generation"
|
||||
# panics. Preventive serialization, no retries.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# The durable ILM decommission regressions build isolated multi-pool stores and
|
||||
# deliberately take source or target disks offline while checking fencing.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Decommission entry and marker/barrier tests share process-wide fault hooks and
|
||||
# deterministic commit barriers. Keep the whole init decommission family in one
|
||||
# nextest group; serial_test alone cannot isolate separate test processes.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests. They drive
|
||||
# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
|
||||
# OnceLock state that serial_test's #[serial] cannot protect across nextest's
|
||||
@@ -186,10 +160,6 @@ path = "junit.xml"
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -262,20 +232,10 @@ test-group = 'embedded-test-ports'
|
||||
filter = 'package(rustfs-ecstore) & test(manual_transition_page_checkpoint_persists_durable_job_progress)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the transition matrix tests under the ci profile too (see the
|
||||
# matching default-profile override near the top). No retries.
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(set_disk::transition_matrix_tests::)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
|
||||
# too (see the matching default-profile override near the top). No retries.
|
||||
[[profile.ci.overrides]]
|
||||
@@ -492,12 +452,23 @@ path = "junit.xml"
|
||||
# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
|
||||
# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
|
||||
# Vault tests, both serialized below.
|
||||
# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
|
||||
# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
|
||||
# product failures cannot be quarantined away with retries, so each family is
|
||||
# excluded here with its tracking issue, under the same discipline as the
|
||||
# ci-profile quarantine (docs/testing/README.md): every entry MUST cite one
|
||||
# OPEN issue, and the fixing PR MUST delete the exclusion. The passing
|
||||
# negative-path siblings of each family stay in as regression guards.
|
||||
# * rustfs#4843 — over-limit archive entry paths hard-reject the whole
|
||||
# archive even under ignore-errors semantics.
|
||||
[profile.e2e-full]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
& !test(/^protocols::/)
|
||||
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||
& !test(/^replication_extension_test::/)
|
||||
& !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
|
||||
& !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
|
||||
@@ -6,4 +6,3 @@ self-hosted-runner:
|
||||
- sm-standard-4
|
||||
- dind-sm-standard-2
|
||||
- smoke-testing
|
||||
- pf-testing
|
||||
|
||||
@@ -7,11 +7,6 @@
|
||||
{ "workflow": ".github/workflows/e2e-s3tests.yml", "max_age_hours": 192 },
|
||||
{ "workflow": ".github/workflows/fuzz.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/mint.yml", "max_age_hours": 192 },
|
||||
{
|
||||
"workflow": ".github/workflows/minio-interop.yml",
|
||||
"max_age_hours": 36,
|
||||
"never_ran_grace_until": "2026-09-08T00:00:00Z"
|
||||
},
|
||||
{ "workflow": ".github/workflows/nightly-gnu.yml", "max_age_hours": 36 },
|
||||
{ "workflow": ".github/workflows/performance-ab.yml", "max_age_hours": 36 },
|
||||
{
|
||||
|
||||
@@ -244,7 +244,7 @@ jobs:
|
||||
needs: [ build-check, prepare-platform-matrix ]
|
||||
if: needs.build-check.outputs.should_build == 'true' && needs.prepare-platform-matrix.result == 'success'
|
||||
runs-on: ${{ matrix.os }}
|
||||
timeout-minutes: 180
|
||||
timeout-minutes: 150
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
# Release binaries ship without dial9 telemetry and therefore do not need
|
||||
|
||||
@@ -20,27 +20,27 @@
|
||||
# each run with Docker and then runs the `#[ignore]` reader tests in
|
||||
# rustfs/src/storage/minio_generated_read_test.rs.
|
||||
#
|
||||
# Scope: MinIO-to-RustFS SSE read interop is implemented behind the `rio-v2`
|
||||
# feature for MinIO's builtin static-KMS deployments — SSE-S3 and SSE-KMS
|
||||
# (single- and multipart) since rustfs/rustfs#6191, SSE-C detection since the
|
||||
# rustfs/backlog#1638 D2 close-out. This job is the standing evidence: it
|
||||
# regenerates real MinIO backend trees and proves byte-identical plaintext
|
||||
# reconstruction. KES/MinKMS-backed MinIO objects remain unreadable by design
|
||||
# (their envelopes are sealed by the KES service, not by a key RustFS can
|
||||
# hold), and default RustFS builds do not include the read path — it is a
|
||||
# special-purpose migration capability, not a default-build feature.
|
||||
# Scope: end-to-end MinIO-to-RustFS SSE interop is NOT implemented yet. Both
|
||||
# envelope parsers reject MinIO's own wrapped-DEK shape — see
|
||||
# `is_data_key_envelope` in crates/kms/src/encryption/dek.rs and the
|
||||
# `deny_unknown_fields` `LocalSseDekEnvelope` in rustfs/src/storage/sse.rs — and
|
||||
# closing that gap is tracked in rustfs/backlog#1638. Treat this job as the
|
||||
# harness for #1638, not as standing evidence that a MinIO migration reads back.
|
||||
#
|
||||
# Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python,
|
||||
# unlike the self-hosted fleet, whose pods drift in Docker/pip availability
|
||||
# (see the infra note in e2e-s3tests.yml). Nightly + manual only.
|
||||
# DISABLED. This workflow is switched off in the repository's Actions settings
|
||||
# (state: disabled_manually) and does not run on any trigger, including its cron
|
||||
# and workflow_dispatch. That state lives in GitHub's UI and is invisible when
|
||||
# reading this file, which has already misled at least one audit — hence this
|
||||
# banner. Re-enabling is a UI action; anyone doing so should first check that the
|
||||
# workflow still matches the current CI layout. See rustfs/backlog#1603.
|
||||
#
|
||||
# Enablement: this workflow was long disabled in the repository's Actions
|
||||
# settings (state: disabled_manually — a state that lives in GitHub's UI and is
|
||||
# invisible in this file). The change that updated this banner also re-added
|
||||
# the .github/scheduled-validations.json entry; both only make sense together
|
||||
# with re-enabling the workflow in the Actions settings. If it is ever disabled
|
||||
# again, remove the scheduled-validations entry in the same change — a disabled
|
||||
# workflow can never satisfy the freshness check. See rustfs/backlog#1603.
|
||||
# While disabled, this workflow is deliberately absent from
|
||||
# .github/scheduled-validations.json — a disabled workflow can never satisfy the
|
||||
# freshness check. Whoever re-enables it must re-add the entry in the same
|
||||
# change so the freshness gate covers it again.
|
||||
#
|
||||
name: minio-interop
|
||||
|
||||
|
||||
@@ -27,7 +27,6 @@ on:
|
||||
paths:
|
||||
- 'flake.nix'
|
||||
- 'flake.lock'
|
||||
- 'nix/**'
|
||||
- 'Cargo.toml'
|
||||
- 'Cargo.lock'
|
||||
- '.github/workflows/nix.yml'
|
||||
@@ -37,7 +36,6 @@ on:
|
||||
paths:
|
||||
- 'flake.nix'
|
||||
- 'flake.lock'
|
||||
- 'nix/**'
|
||||
- 'Cargo.toml'
|
||||
- 'Cargo.lock'
|
||||
- '.github/workflows/nix.yml'
|
||||
|
||||
@@ -224,7 +224,7 @@ jobs:
|
||||
ZIP_FILE=$(find ./binary-artifact -name "*.zip" -type f | head -1)
|
||||
if [[ -z "$ZIP_FILE" ]]; then
|
||||
echo "❌ No binary artifact found"
|
||||
find ./binary-artifact -mindepth 1 -maxdepth 1 -print 2>/dev/null || true
|
||||
ls -la ./binary-artifact/ || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -239,7 +239,7 @@ jobs:
|
||||
fi
|
||||
|
||||
chmod +x ./bin/rustfs
|
||||
stat --printf='%n %s bytes\n' ./bin/rustfs
|
||||
ls -lh ./bin/rustfs
|
||||
echo "✅ Binary extracted"
|
||||
|
||||
- name: Build DEB package
|
||||
@@ -336,7 +336,7 @@ jobs:
|
||||
fakeroot dpkg-deb --build "${PKG_DIR}"
|
||||
|
||||
DEB_FILE="${PKG_DIR}.deb"
|
||||
stat --printf='%n %s bytes\n' "$DEB_FILE"
|
||||
ls -lh "$DEB_FILE"
|
||||
echo "deb_file=$DEB_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ DEB built: $DEB_FILE"
|
||||
|
||||
@@ -410,14 +410,13 @@ jobs:
|
||||
LICENSE=/usr/share/doc/rustfs/LICENSE \
|
||||
README.md=/usr/share/doc/rustfs/README.md
|
||||
|
||||
RPM_FILE=$(find . -maxdepth 1 -type f -name 'rustfs-*.rpm' -print | head -1)
|
||||
RPM_FILE="${RPM_FILE#./}"
|
||||
RPM_FILE=$(ls -1 rustfs-*.rpm 2>/dev/null | head -1)
|
||||
if [[ -z "$RPM_FILE" ]]; then
|
||||
echo "❌ RPM build failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
stat --printf='%n %s bytes\n' "$RPM_FILE"
|
||||
ls -lh "$RPM_FILE"
|
||||
echo "rpm_file=$RPM_FILE" >> "$GITHUB_OUTPUT"
|
||||
echo "✅ RPM built: $RPM_FILE"
|
||||
|
||||
@@ -553,13 +552,11 @@ jobs:
|
||||
- name: Print summary
|
||||
shell: bash
|
||||
run: |
|
||||
{
|
||||
echo "## 📦 Package Summary"
|
||||
echo ""
|
||||
echo "| Item | Value |"
|
||||
echo "|------|-------|"
|
||||
echo "| Version | \`${{ needs.resolve.outputs.version }}\` |"
|
||||
echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |"
|
||||
echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |"
|
||||
echo "| Package Status | ${{ needs.package.result }} |"
|
||||
} >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "## 📦 Package Summary" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Item | Value |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "|------|-------|" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Version | \`${{ needs.resolve.outputs.version }}\` |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Build Type | ${{ needs.resolve.outputs.build_type }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Build Run | #${{ needs.resolve.outputs.build_run_id }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
echo "| Package Status | ${{ needs.package.result }} |" >> "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
@@ -30,7 +30,7 @@ permissions:
|
||||
# Only one test at a time: both this and the pool-expansion workflow mutate
|
||||
# the same test environment, so they share one concurrency group.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
group: rustfs-pool-expansion-test
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
@@ -43,25 +43,17 @@ env:
|
||||
RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
|
||||
jobs:
|
||||
heal-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 480
|
||||
# Manual-only standalone run. Nightly chain already runs heal in
|
||||
# rustfs-pool-expand-test.yml to avoid duplicate heal executions.
|
||||
if: ${{ github.event_name == 'workflow_dispatch' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
@@ -71,24 +63,11 @@ jobs:
|
||||
warp --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
- name: Reset test environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
chmod +x scripts/test/rustfs_heal_test.sh
|
||||
./scripts/test/rustfs_heal_test.sh --reset -y
|
||||
|
||||
- name: Install RustFS package & start cluster
|
||||
run: |
|
||||
@@ -98,7 +77,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||
./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Preflight checks
|
||||
run: |
|
||||
@@ -108,223 +87,17 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||
./scripts/test/rustfs_heal_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Run heal test (write -> outage -> heal -> verify)
|
||||
id: test
|
||||
run: |
|
||||
./auto-testing/rustfs_heal_test.sh \
|
||||
./scripts/test/rustfs_heal_test.sh \
|
||||
--steps "3,4,5,6,7" -y \
|
||||
--endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \
|
||||
--stop-node-gb "${{ inputs.stop_node_gb }}" \
|
||||
--warp-stop-gb "${{ inputs.warp_stop_gb }}" \
|
||||
--log-file /tmp/rustfs-heal-test.log
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-heal-test.log
|
||||
REPORT_FILE: /tmp/rustfs-heal-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
{
|
||||
echo "# RustFS heal test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-heal-report.md
|
||||
SUITE: heal
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -335,24 +108,10 @@ jobs:
|
||||
/tmp/rustfs-warp.*.log
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
- name: Reset test environment (after)
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
./scripts/test/rustfs_heal_test.sh --reset -y
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -1,376 +0,0 @@
|
||||
name: RustFS KMS Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
enforce_sse_key_policy:
|
||||
description: 'Enable RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY (runs KMS-401/402)'
|
||||
type: boolean
|
||||
default: false
|
||||
frame_v2:
|
||||
description: 'Enable RUSTFS_ENCRYPTION_FRAME_V2 (runs KMS-318)'
|
||||
type: boolean
|
||||
default: false
|
||||
config_secret:
|
||||
description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
# Strict shared-environment order: run after S3 compatibility test completes.
|
||||
workflows: ["RustFS S3 Compatibility Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
kms-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
docker --version || true
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Ensure docker (Vault container)
|
||||
run: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y docker.io
|
||||
fi
|
||||
sudo systemctl enable --now docker
|
||||
docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
|
||||
|
||||
- name: Run KMS suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-kms.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-kms-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
ARGS=(--all-topologies --backends "local,vault-kv2" -y --log-file "${LOG_FILE}")
|
||||
EXTRA_ENV=""
|
||||
if [ "${{ inputs.enforce_sse_key_policy }}" = "true" ]; then
|
||||
EXTRA_ENV+="RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true"$'\n'
|
||||
fi
|
||||
if [ "${{ inputs.frame_v2 }}" = "true" ]; then
|
||||
EXTRA_ENV+="RUSTFS_ENCRYPTION_FRAME_V2=true"$'\n'
|
||||
fi
|
||||
if [ -n "${{ inputs.config_secret }}" ]; then
|
||||
EXTRA_ENV+="RUSTFS_KMS_CONFIG_SECRET=${{ inputs.config_secret }}"$'\n'
|
||||
fi
|
||||
if [ -n "${EXTRA_ENV}" ]; then
|
||||
ARGS+=(--extra-env "${EXTRA_ENV}")
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-kms-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-kms.log
|
||||
REPORT_FILE: /tmp/rustfs-kms-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-kms-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS KMS test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-kms-report.md
|
||||
SUITE: kms
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-kms-test-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-kms.log
|
||||
/tmp/rustfs-kms-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS KMS suite failed"
|
||||
echo "See the uploaded report and log artifacts for details."
|
||||
@@ -78,7 +78,7 @@ env:
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
# Fixed benchmark result directory so later steps can read summary.md
|
||||
RUSTFS_RESULT_DIR: /tmp/rustfs-perf-results
|
||||
# Cross-repo token for uploading reports to rustfs/dashboard (set in repo settings)
|
||||
# Cross-repo token for writing to rustfs/backlog (set in repo settings)
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
@@ -89,14 +89,11 @@ jobs:
|
||||
# Skipped when nightly failed.
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
- name: Checkout
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
ref: ${{ github.event.workflow_run.head_sha || github.ref }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
@@ -108,8 +105,8 @@ jobs:
|
||||
- name: Reset test environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
chmod +x auto-testing/rustfs_performance_test.sh
|
||||
./auto-testing/rustfs_performance_test.sh --step 1 -y
|
||||
chmod +x scripts/test/rustfs_performance_test.sh
|
||||
./scripts/test/rustfs_performance_test.sh --step 1 -y
|
||||
|
||||
- name: Install RustFS package & start cluster (4x4)
|
||||
run: |
|
||||
@@ -119,7 +116,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
|
||||
./scripts/test/rustfs_performance_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Preflight checks
|
||||
run: |
|
||||
@@ -129,7 +126,7 @@ jobs:
|
||||
else
|
||||
ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}")
|
||||
fi
|
||||
./auto-testing/rustfs_performance_test.sh "${ARGS[@]}"
|
||||
./scripts/test/rustfs_performance_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Run benchmark (GET/PUT/MIXED)
|
||||
id: benchmark
|
||||
@@ -138,7 +135,7 @@ jobs:
|
||||
# Manual dispatch can restrict method(s)/size(s).
|
||||
export WARP_METHODS="${{ inputs.test_method }}"
|
||||
export WARP_SIZES="${{ inputs.object_size }}"
|
||||
./auto-testing/rustfs_performance_test.sh \
|
||||
./scripts/test/rustfs_performance_test.sh \
|
||||
--step 5 -y \
|
||||
--warp-duration "${{ inputs.warp_duration || '5m' }}" \
|
||||
--warp-concurrency "${{ inputs.warp_concurrency || '64' }}" \
|
||||
@@ -147,69 +144,40 @@ jobs:
|
||||
- name: Analyze results
|
||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||
run: |
|
||||
./auto-testing/rustfs_performance_test.sh --step 6 -y
|
||||
./scripts/test/rustfs_performance_test.sh --step 6 -y
|
||||
|
||||
- name: Collect RustFS version info
|
||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||
env:
|
||||
VERSION_FILE: /tmp/rustfs-version.txt
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES}"
|
||||
[ "${#NODES[@]}" -gt 0 ] || { echo "RUSTFS_NODES is empty"; exit 1; }
|
||||
NODE="${NODES[0]}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
{
|
||||
echo "Node: ${NODE}"
|
||||
echo "Command: rustfs --version"
|
||||
echo ""
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
||||
"${SSH_USER}@${NODE}" 'rustfs --version'
|
||||
} > "${VERSION_FILE}"
|
||||
|
||||
- name: Upload report to dashboard (reports/YYYY-MM-DD.md)
|
||||
- name: Post results to backlog issue
|
||||
if: ${{ steps.benchmark.conclusion == 'success' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
RESULT_DIR: ${{ env.RUSTFS_RESULT_DIR }}
|
||||
VERSION_FILE: /tmp/rustfs-version.txt
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping report upload"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping issue post"
|
||||
exit 0
|
||||
fi
|
||||
SUMMARY="${RESULT_DIR}/summary.md"
|
||||
[ -f "${SUMMARY}" ] || { echo "summary.md not found at ${SUMMARY}"; exit 1; }
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="reports/${DATE}.md"
|
||||
{
|
||||
echo "# RustFS nightly build performance testing report"
|
||||
echo "## RustFS nightly build performance testing report"
|
||||
echo ""
|
||||
echo "- **Date**: ${DATE}"
|
||||
echo "- **日期**: ${DATE}"
|
||||
echo "- **Run**: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- **Trigger**: ${{ github.event_name }}"
|
||||
echo "- **Package**: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "- **触发方式**: ${{ github.event_name }}"
|
||||
echo ""
|
||||
cat "${SUMMARY}"
|
||||
echo ""
|
||||
echo "## RustFS version"
|
||||
echo '```text'
|
||||
cat "${VERSION_FILE}"
|
||||
echo '```'
|
||||
} > /tmp/rustfs-perf-report.md
|
||||
CONTENT="$(python3 -c 'import base64; print(base64.b64encode(open("/tmp/rustfs-perf-report.md","rb").read()).decode())')"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
echo "updated ${REPORT_PATH} in rustfs/dashboard"
|
||||
} > /tmp/rustfs-perf-issue-body.md
|
||||
TITLE="RustFS nightly build performance testing report"
|
||||
EXISTING="$(gh issue list --repo rustfs/backlog \
|
||||
--search "in:title \"${TITLE}\"" --state all --limit 5 \
|
||||
--json number --jq '.[0].number // empty')"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
gh issue comment "${EXISTING}" --repo rustfs/backlog --body-file /tmp/rustfs-perf-issue-body.md
|
||||
echo "commented on existing issue #${EXISTING}"
|
||||
else
|
||||
jq -n --arg msg "report: ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
echo "created ${REPORT_PATH} in rustfs/dashboard"
|
||||
gh issue create --repo rustfs/backlog --title "${TITLE}" --body-file /tmp/rustfs-perf-issue-body.md
|
||||
fi
|
||||
|
||||
- name: Upload test logs & results
|
||||
@@ -220,13 +188,12 @@ jobs:
|
||||
path: |
|
||||
/tmp/rustfs-perf-test*.log
|
||||
/tmp/rustfs-perf-results/**
|
||||
/tmp/rustfs-version.txt
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Reset test environment (after)
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs_performance_test.sh --step 7 -y
|
||||
./scripts/test/rustfs_performance_test.sh --step 7 -y
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,406 +0,0 @@
|
||||
name: RustFS S3 Compatibility Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
# Run after upgrade compatibility completes; the nightly deb is what the test installs.
|
||||
workflows: ["RustFS Upgrade Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
s3-compat-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs
|
||||
'
|
||||
done
|
||||
|
||||
- name: Run S3 compatibility suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-s3-compat.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-s3-compat-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-s3-compat-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-s3-compat.log
|
||||
REPORT_FILE: /tmp/rustfs-s3-compat-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
RUSTFS_VERSION_INFO="N/A"
|
||||
if [ "${#NODES[@]}" -gt 0 ]; then
|
||||
DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
||||
"${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
|
||||
if [ -n "${DETECTED_VERSION}" ]; then
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
current = None
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
current = case_id
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
current = None
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS S3 compatibility test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-s3-compat-report.md
|
||||
SUITE: s3
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-s3-compat-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-s3-compat.log
|
||||
/tmp/rustfs-s3-compat-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS S3 compatibility suite failed"
|
||||
echo "See the uploaded report and log artifacts for details."
|
||||
@@ -1,381 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Security Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
topology:
|
||||
description: 'Topology to run (all = SNSD, SNMD, MNMD)'
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- single-single
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
oidc_live:
|
||||
description: 'Run the live Keycloak OIDC/SSO gate as part of the suite'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_before:
|
||||
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_after:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
workflow_run:
|
||||
# Runs last in the functional chain, after pool/heal, on the shared VMs.
|
||||
workflows: ["RustFS Pool Expansion / Heal Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The security suite uses the same shared VMs as the other functional tests,
|
||||
# so it must serialize with them instead of running in parallel.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
security-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Checkout repository (for the OIDC live gate script)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
aws --version || true
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
||||
'
|
||||
done
|
||||
|
||||
- name: Run security suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
REPORT_FILE: /tmp/rustfs-security-report.md
|
||||
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-security-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
TOPOLOGY='${{ inputs.topology }}'
|
||||
ARGS=(-y)
|
||||
if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
|
||||
ARGS+=(--all-topologies)
|
||||
else
|
||||
ARGS+=(--topology "${TOPOLOGY}")
|
||||
fi
|
||||
if [ "${{ inputs.oidc_live }}" = "true" ] || [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
|
||||
ARGS+=(--oidc-live)
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ ! -f /tmp/rustfs-security-report.md ]; then
|
||||
{
|
||||
echo "# RustFS security test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Test Step Outcome: failure (suite did not produce a report)"
|
||||
} > /tmp/rustfs-security-report.md
|
||||
fi
|
||||
cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-security-report.md
|
||||
SUITE: security
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-security-test-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-security-report.md
|
||||
/tmp/rustfs-security.*/*
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS security test failed"
|
||||
echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "See the uploaded report and logs for details."
|
||||
@@ -1,422 +0,0 @@
|
||||
name: RustFS Storage Engine Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
topology:
|
||||
description: 'Topology to run (all = SNSD, SNMD, MNMD)'
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- single-single
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
workflow_run:
|
||||
# Strict shared-environment order: run after tier test completes.
|
||||
workflows: ["RustFS Tier Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
storage-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Run storage engine suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-storage.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-storage-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
TOPOLOGY='${{ inputs.topology }}'
|
||||
ARGS=(-y --log-file "${LOG_FILE}")
|
||||
if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
|
||||
ARGS+=(--all-topologies)
|
||||
else
|
||||
ARGS+=(--topology "${TOPOLOGY}")
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-storage.log
|
||||
REPORT_FILE: /tmp/rustfs-storage-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
RUSTFS_VERSION_INFO="N/A"
|
||||
if [ "${#NODES[@]}" -gt 0 ]; then
|
||||
DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
||||
"${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
|
||||
if [ -n "${DETECTED_VERSION}" ]; then
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-storage-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
current = None
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
current = case_id
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
current = None
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS storage engine test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-storage-report.md
|
||||
SUITE: storage
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'storage', label: 'Storage Engine' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-storage-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-storage.log
|
||||
/tmp/rustfs-storage-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS storage engine suite failed"
|
||||
echo "See the uploaded report and log artifacts for details."
|
||||
@@ -1,374 +0,0 @@
|
||||
name: RustFS Tier Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
# Strict shared-environment order: run after KMS test completes.
|
||||
workflows: ["RustFS KMS Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
tier-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
sudo rm -f /tmp/rustfs-mosquitto.conf
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
||||
'
|
||||
done
|
||||
|
||||
- name: Ensure MQTT broker + clients
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! command -v mosquitto_sub >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y mosquitto-clients
|
||||
fi
|
||||
command -v docker >/dev/null 2>&1 || { echo 'docker not found on runner'; exit 1; }
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
cat <<'EOF' | sudo tee /tmp/rustfs-mosquitto.conf >/dev/null
|
||||
listener 1883 0.0.0.0
|
||||
allow_anonymous true
|
||||
EOF
|
||||
sudo docker run -d --name rustfs-test-mqtt -p 1883:1883 \
|
||||
-v /tmp/rustfs-mosquitto.conf:/mosquitto/config/mosquitto.conf:ro \
|
||||
eclipse-mosquitto:2 >/dev/null
|
||||
for _ in {1..10}; do
|
||||
if ss -tln 2>/dev/null | grep -q ':1883'; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
ss -tln 2>/dev/null | grep -q ':1883' || {
|
||||
echo 'mosquitto container is not listening on 1883'
|
||||
sudo docker logs rustfs-test-mqtt || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Run tier suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-tier.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-tier-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
ARGS=(--all-topologies -y --log-file "${LOG_FILE}")
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-tier-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-tier.log
|
||||
REPORT_FILE: /tmp/rustfs-tier-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-tier-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS tier test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-tier-report.md
|
||||
SUITE: tier
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-tier-test-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-tier.log
|
||||
/tmp/rustfs-tier-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
sudo rm -f /tmp/rustfs-mosquitto.conf
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS tier suite failed"
|
||||
echo "See the uploaded report and log artifacts for details."
|
||||
@@ -1,477 +0,0 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Upgrade Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
from_version:
|
||||
description: 'OLD RustFS release tag (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
from_url:
|
||||
description: 'OLD .deb URL. Overrides from_version.'
|
||||
required: false
|
||||
type: string
|
||||
to_version:
|
||||
description: 'NEW RustFS release tag (leave empty for latest nightly)'
|
||||
required: false
|
||||
to_url:
|
||||
description: 'NEW .deb URL. Overrides to_version / nightly default.'
|
||||
required: false
|
||||
type: string
|
||||
topology:
|
||||
description: 'Topology to run (all = SNSD, SNMD, MNMD)'
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- single-single
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
backends:
|
||||
description: 'KMS backends to run (local,vault-kv2)'
|
||||
required: false
|
||||
default: 'local,vault-kv2'
|
||||
cleanup_before:
|
||||
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_after:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
workflow_run:
|
||||
# Runs first in the functional chain: upgrade compatibility gates the
|
||||
# nightly suites that follow (S3 -> KMS -> Tier -> Pool/Heal -> Security).
|
||||
workflows: ["Nightly GNU Build"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
upgrade-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
aws --version || true
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Ensure docker (Vault container)
|
||||
run: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y docker.io
|
||||
fi
|
||||
sudo systemctl enable --now docker
|
||||
docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
|
||||
|
||||
- name: Run upgrade compatibility suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-upgrade-test.sh
|
||||
FROM_URL='${{ inputs.from_url }}'
|
||||
FROM_VERSION='${{ inputs.from_version }}'
|
||||
TO_URL='${{ inputs.to_url }}'
|
||||
TO_VERSION='${{ inputs.to_version }}'
|
||||
TOPOLOGY='${{ inputs.topology }}'
|
||||
BACKENDS='${{ inputs.backends }}'
|
||||
ARGS=(-y --log-file "${LOG_FILE}")
|
||||
if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
|
||||
ARGS+=(--all-topologies)
|
||||
else
|
||||
ARGS+=(--topology "${TOPOLOGY}")
|
||||
fi
|
||||
if [ -n "${BACKENDS}" ] && [ "${BACKENDS}" != "null" ]; then
|
||||
ARGS+=(--backends "${BACKENDS}")
|
||||
fi
|
||||
if [ -n "${FROM_URL}" ]; then
|
||||
ARGS+=(--from-url "${FROM_URL}")
|
||||
elif [ -n "${FROM_VERSION}" ] && [ "${FROM_VERSION}" != "null" ]; then
|
||||
ARGS+=(--from-version "${FROM_VERSION}")
|
||||
fi
|
||||
if [ -n "${TO_URL}" ]; then
|
||||
ARGS+=(--to-url "${TO_URL}")
|
||||
elif [ -n "${TO_VERSION}" ] && [ "${TO_VERSION}" != "null" ]; then
|
||||
ARGS+=(--to-version "${TO_VERSION}")
|
||||
else
|
||||
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
REPORT_FILE: /tmp/rustfs-upgrade-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
FROM_URL='${{ inputs.from_url }}'
|
||||
FROM_VERSION='${{ inputs.from_version }}'
|
||||
TO_URL='${{ inputs.to_url }}'
|
||||
TO_VERSION='${{ inputs.to_version }}'
|
||||
if [ -n "${FROM_URL}" ]; then
|
||||
FROM_SOURCE="${FROM_URL}"
|
||||
elif [ -n "${FROM_VERSION}" ]; then
|
||||
FROM_SOURCE="version ${FROM_VERSION}"
|
||||
else
|
||||
FROM_SOURCE="release (default)"
|
||||
fi
|
||||
if [ -n "${TO_URL}" ]; then
|
||||
TO_SOURCE="${TO_URL}"
|
||||
elif [ -n "${TO_VERSION}" ]; then
|
||||
TO_SOURCE="version ${TO_VERSION}"
|
||||
else
|
||||
TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS upgrade compatibility report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- From: ${FROM_SOURCE}"
|
||||
echo "- To: ${TO_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-upgrade-report.md
|
||||
SUITE: upgrade
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('upgrade');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-upgrade-test-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-upgrade-report.md
|
||||
/tmp/rustfs-upgrade.*/*
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS upgrade compatibility test failed"
|
||||
echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
|
||||
echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
|
||||
echo "See the uploaded report and logs for details."
|
||||
@@ -31,13 +31,8 @@ This file contains repository-wide rules. Use the nearest subdirectory
|
||||
- An existing clean, isolated task worktree is sufficient. Create another
|
||||
worktree only when the current checkout is shared, dirty with unrelated work,
|
||||
or belongs to another task.
|
||||
- Never commit from a shared checkout.
|
||||
- Use a task-specific branch named `<type>/<topic>`, such as `fix/...`,
|
||||
`feat/...`, `test/...`, or `docs/...`, unless the user specifies a name.
|
||||
- Do not include agent, tool, contributor, account, or organization names in
|
||||
branch names.
|
||||
- Push to the user-requested remote or the repository's configured push remote.
|
||||
Do not hard-code or infer a remote from an account name.
|
||||
- Never commit from a shared checkout. Use an `overtrue/` feature branch unless
|
||||
the user requests another name.
|
||||
- Check free space before artifact-heavy builds, tests, coverage, or downloads.
|
||||
Re-check before a broad gate when space is tight.
|
||||
- Remove only task-owned temporary/build artifacts. Never delete another task's
|
||||
|
||||
Generated
+148
-207
File diff suppressed because it is too large
Load Diff
+57
-58
@@ -72,7 +72,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.5"
|
||||
version = "1.0.0-rc.4"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -89,55 +89,55 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.4" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.4" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.4" }
|
||||
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.4" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.4" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.4" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.4" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.4" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.4" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.4" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.4" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.4" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.4" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.4" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.4" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.4" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.4" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.4" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.4" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.4" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.4" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.4" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.4" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.4" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.4", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.4" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.4" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.4" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.4" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.4" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.4" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.4" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.4" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.4" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.4" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.4" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.4" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.4" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.4" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.4" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.4" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.4" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.4" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.4" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.4" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.4" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.4" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.4" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.4" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -155,7 +155,7 @@ futures-util = "0.3.34"
|
||||
pollster = "1.0.1"
|
||||
pulsar = { default-features = false, version = "6.9.0" }
|
||||
lapin = { default-features = false, version = "4.10.0" }
|
||||
hyper = { version = "1.11.1" }
|
||||
hyper = { version = "1.11.0" }
|
||||
hyper-rustls = { default-features = false, version = "0.27.9" }
|
||||
hyper-util = { version = "0.1.20" }
|
||||
http = "1.5.0"
|
||||
@@ -198,7 +198,7 @@ serde_urlencoded = "0.7.1"
|
||||
# have incompatible APIs. Keep them exact-pinned and monitor upstream for stable
|
||||
# releases.
|
||||
aes-gcm = { version = "=0.11.1" }
|
||||
argon2 = { version = "=0.6.0" }
|
||||
argon2 = { version = "=0.6.0-rc.8" }
|
||||
blake2 = "=0.11.0"
|
||||
chacha20poly1305 = { version = "=0.11.0" }
|
||||
crc-fast = "1.10.0"
|
||||
@@ -232,8 +232,7 @@ tokio-postgres-rustls = "0.14.0"
|
||||
# Utilities and Tools
|
||||
anyhow = "1.0.104"
|
||||
arc-swap = "1.9.2"
|
||||
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until bounded extension parsing is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published tokio-tar release exposes the extension limits used here.
|
||||
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
|
||||
astral-tokio-tar = "0.6.4"
|
||||
atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.11.0" }
|
||||
@@ -248,7 +247,7 @@ base64-simd = "0.8.0"
|
||||
brotli = "8.0.4"
|
||||
clap = { version = "4.6.6" }
|
||||
const-str = { version = "1.1.0" }
|
||||
convert_case = "0.12.0"
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8" }
|
||||
crossbeam-queue = "0.3.13"
|
||||
crossbeam-channel = "0.5.16"
|
||||
@@ -258,7 +257,7 @@ datafusion = { default-features = false, version = "55.0.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
faster-hex = "0.10.0"
|
||||
flate2 = "1.1.10"
|
||||
flate2 = "1.1.9"
|
||||
glob = "0.3.4"
|
||||
google-cloud-storage = "1.18.0"
|
||||
google-cloud-auth = "1.16.0"
|
||||
@@ -305,7 +304,7 @@ rustify = { version = "0.7", default-features = false }
|
||||
rustix = { version = "1.1.4" }
|
||||
rust-embed = { version = "8.12.0" }
|
||||
rustc-hash = { version = "2.1.3" }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "9c4690d8e73fc8d184031a19b2c4539ebc77d180", version = "0.15.0", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/rustfs/s3s.git", rev = "0f6f83d98b37fd9edcaa3be573db4aa8f568e088", version = "0.15.0", features = ["minio"] }
|
||||
serial_test = "4.0.1"
|
||||
shadow-rs = { default-features = false, version = "2.0.0" }
|
||||
siphasher = "1.0.3"
|
||||
@@ -356,7 +355,7 @@ pyroscope = { version = "2.1.1" }
|
||||
libunftp = { version = "0.23.0" }
|
||||
unftp-core = "0.1.0"
|
||||
suppaftp = { version = "10.0.2" }
|
||||
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
rcgen = { version = "0.14.9", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
|
||||
russh = { version = "0.63.1" }
|
||||
russh-sftp = "2.4.0"
|
||||
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://docs.rustfs.com/en/installation">Getting Started</a>
|
||||
<a href="https://docs.rustfs.com/installation/">Getting Started</a>
|
||||
· <a href="https://docs.rustfs.com/">Docs</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/issues">Bug reports</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/discussions">Discussions</a>
|
||||
@@ -115,7 +115,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
@@ -245,26 +245,6 @@ nix build
|
||||
nix run
|
||||
```
|
||||
|
||||
The flake also exports a NixOS module and the RustFS `rc` client. Add the
|
||||
module to your system and provide credentials through runtime files (for
|
||||
example, sops-nix or agenix) so secrets are never stored in the Nix store:
|
||||
|
||||
```nix
|
||||
imports = [ inputs.rustfs.nixosModules.rustfs ];
|
||||
|
||||
services.rustfs = {
|
||||
enable = true;
|
||||
accessKeyFile = "/run/secrets/rustfs-access-key";
|
||||
secretKeyFile = "/run/secrets/rustfs-secret-key";
|
||||
volumes = [ "/var/lib/rustfs" ];
|
||||
};
|
||||
```
|
||||
|
||||
Install the S3-compatible client with
|
||||
`nix profile install github:rustfs/rustfs#rustfs-client` (the executable is named
|
||||
`rc`), or use `inputs.rustfs.packages.${pkgs.system}.rustfs-client` in a system
|
||||
configuration.
|
||||
|
||||
### 6\. X-CMD (Option 6)
|
||||
|
||||
If you are an [x-cmd](https://www.x-cmd.com/install/rustfs) user:
|
||||
|
||||
+2
-8
@@ -16,7 +16,7 @@
|
||||
</p>
|
||||
|
||||
<p align="center">
|
||||
<a href="https://docs.rustfs.com/zh/installation">快速开始</a>
|
||||
<a href="https://docs.rustfs.com/installation/">快速开始</a>
|
||||
· <a href="https://docs.rustfs.com/">文档</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/issues">报告 Bug</a>
|
||||
· <a href="https://github.com/rustfs/rustfs/discussions">社区讨论</a>
|
||||
@@ -112,7 +112,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录，也请用同样方式准备该目录：
|
||||
@@ -191,12 +191,6 @@ nix build
|
||||
nix run
|
||||
```
|
||||
|
||||
该 Flake 同时提供 NixOS 模块和 RustFS `rc` 客户端。将
|
||||
`inputs.rustfs.nixosModules.rustfs` 加入 `imports`，并通过运行时密钥文件
|
||||
（例如 sops-nix 或 agenix）配置 `accessKeyFile` 与 `secretKeyFile`，避免密钥
|
||||
进入 Nix store。客户端包为
|
||||
`inputs.rustfs.packages.${pkgs.system}.rustfs-client`，安装后的命令名为 `rc`。
|
||||
|
||||
### 6\. X-CMD (Option 6)
|
||||
|
||||
如果你是 [x-cmd](https://www.x-cmd.com/install/rustfs) 用户：
|
||||
|
||||
@@ -178,76 +178,6 @@ pub trait WorkloadAdmissionSnapshotProvider {
|
||||
fn workload_admission_snapshot(&self) -> WorkloadAdmissionRegistrySnapshot;
|
||||
}
|
||||
|
||||
/// Foreground workload pressure observed against a configured utilization threshold.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct ForegroundPressure {
|
||||
/// Foreground workload class whose utilization reached its threshold.
|
||||
pub class: WorkloadClass,
|
||||
/// Observed utilization percentage for the class.
|
||||
pub usage_pct: usize,
|
||||
/// Configured threshold percentage that the observed utilization reached.
|
||||
pub threshold_pct: usize,
|
||||
}
|
||||
|
||||
impl ForegroundPressure {
|
||||
/// Return a stable reason label for logs and metrics.
|
||||
pub const fn reason(self) -> &'static str {
|
||||
match self.class {
|
||||
WorkloadClass::ForegroundRead => "foreground_read_pressure",
|
||||
WorkloadClass::ForegroundWrite => "foreground_write_pressure",
|
||||
_ => "foreground_pressure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Return the strongest foreground pressure in `snapshot`, if any.
|
||||
///
|
||||
/// A zero threshold disables its class. `Saturated` counts as full utilization
|
||||
/// regardless of the reported limit; otherwise a class contributes only when it
|
||||
/// reports a non-zero limit, with a missing active count read as zero. When both
|
||||
/// classes are above their threshold the higher utilization wins.
|
||||
///
|
||||
/// Callers own the enable switch: this function evaluates thresholds only.
|
||||
pub fn foreground_pressure(
|
||||
snapshot: &WorkloadAdmissionRegistrySnapshot,
|
||||
read_threshold_pct: usize,
|
||||
write_threshold_pct: usize,
|
||||
) -> Option<ForegroundPressure> {
|
||||
[
|
||||
(WorkloadClass::ForegroundRead, read_threshold_pct),
|
||||
(WorkloadClass::ForegroundWrite, write_threshold_pct),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(class, threshold_pct)| {
|
||||
if threshold_pct == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let entry = snapshot.get(class)?;
|
||||
let usage_pct = if matches!(entry.state, AdmissionState::Saturated) {
|
||||
100
|
||||
} else {
|
||||
let limit = entry.limit?;
|
||||
if limit == 0 {
|
||||
return None;
|
||||
}
|
||||
entry
|
||||
.active
|
||||
.unwrap_or(0)
|
||||
.saturating_mul(100)
|
||||
.checked_div(limit)
|
||||
.unwrap_or(100)
|
||||
};
|
||||
|
||||
(usage_pct >= threshold_pct).then_some(ForegroundPressure {
|
||||
class,
|
||||
usage_pct,
|
||||
threshold_pct,
|
||||
})
|
||||
})
|
||||
.max_by_key(|pressure| pressure.usage_pct)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -384,205 +314,4 @@ mod tests {
|
||||
|
||||
assert!(err.to_string().contains("unexpected"));
|
||||
}
|
||||
|
||||
fn counted(
|
||||
class: WorkloadClass,
|
||||
state: AdmissionState,
|
||||
active: Option<usize>,
|
||||
limit: Option<usize>,
|
||||
) -> WorkloadAdmissionSnapshot {
|
||||
WorkloadAdmissionSnapshot::new(class, state).with_counts(active, None, limit)
|
||||
}
|
||||
|
||||
fn registry(entries: Vec<WorkloadAdmissionSnapshot>) -> WorkloadAdmissionRegistrySnapshot {
|
||||
WorkloadAdmissionRegistrySnapshot::new(entries)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_reason_labels_cover_non_foreground_classes() {
|
||||
let read = ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundRead,
|
||||
usage_pct: 90,
|
||||
threshold_pct: 80,
|
||||
};
|
||||
let write = ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundWrite,
|
||||
usage_pct: 90,
|
||||
threshold_pct: 80,
|
||||
};
|
||||
let repair = ForegroundPressure {
|
||||
class: WorkloadClass::Repair,
|
||||
usage_pct: 90,
|
||||
threshold_pct: 80,
|
||||
};
|
||||
|
||||
assert_eq!(read.reason(), "foreground_read_pressure");
|
||||
assert_eq!(write.reason(), "foreground_write_pressure");
|
||||
assert_eq!(repair.reason(), "foreground_pressure");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_is_disabled_when_both_thresholds_are_zero() {
|
||||
let snapshot = registry(vec![
|
||||
counted(WorkloadClass::ForegroundRead, AdmissionState::Saturated, Some(8), Some(8)),
|
||||
counted(WorkloadClass::ForegroundWrite, AdmissionState::Saturated, Some(8), Some(8)),
|
||||
]);
|
||||
|
||||
assert_eq!(foreground_pressure(&snapshot, 0, 0), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_skips_only_the_class_whose_threshold_is_zero() {
|
||||
let snapshot = registry(vec![
|
||||
counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(10), Some(10)),
|
||||
counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(9), Some(10)),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
foreground_pressure(&snapshot, 0, 80),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundWrite,
|
||||
usage_pct: 90,
|
||||
threshold_pct: 80,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
foreground_pressure(&snapshot, 80, 0),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundRead,
|
||||
usage_pct: 100,
|
||||
threshold_pct: 80,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_ignores_missing_entries() {
|
||||
let snapshot = registry(vec![counted(WorkloadClass::Scanner, AdmissionState::Saturated, Some(8), Some(8))]);
|
||||
|
||||
assert_eq!(foreground_pressure(&snapshot, 1, 1), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_ignores_missing_and_zero_limits() {
|
||||
let missing_limit = registry(vec![counted(
|
||||
WorkloadClass::ForegroundRead,
|
||||
AdmissionState::Throttled,
|
||||
Some(8),
|
||||
None,
|
||||
)]);
|
||||
let zero_limit = registry(vec![counted(
|
||||
WorkloadClass::ForegroundWrite,
|
||||
AdmissionState::Throttled,
|
||||
Some(8),
|
||||
Some(0),
|
||||
)]);
|
||||
|
||||
assert_eq!(foreground_pressure(&missing_limit, 1, 1), None);
|
||||
assert_eq!(foreground_pressure(&zero_limit, 1, 1), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_treats_saturated_as_full_without_reading_limit() {
|
||||
let snapshot = registry(vec![
|
||||
counted(WorkloadClass::ForegroundRead, AdmissionState::Saturated, None, None),
|
||||
counted(WorkloadClass::ForegroundWrite, AdmissionState::Saturated, Some(0), Some(0)),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
foreground_pressure(&snapshot, 100, 0),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundRead,
|
||||
usage_pct: 100,
|
||||
threshold_pct: 100,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
foreground_pressure(&snapshot, 0, 100),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundWrite,
|
||||
usage_pct: 100,
|
||||
threshold_pct: 100,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_reads_missing_active_as_zero() {
|
||||
let snapshot = registry(vec![counted(WorkloadClass::ForegroundRead, AdmissionState::Open, None, Some(8))]);
|
||||
|
||||
assert_eq!(foreground_pressure(&snapshot, 1, 1), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_returns_the_higher_utilization_when_both_classes_exceed() {
|
||||
let read_higher = registry(vec![
|
||||
counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(19), Some(20)),
|
||||
counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(17), Some(20)),
|
||||
]);
|
||||
let write_higher = registry(vec![
|
||||
counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(17), Some(20)),
|
||||
counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(19), Some(20)),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
foreground_pressure(&read_higher, 80, 80),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundRead,
|
||||
usage_pct: 95,
|
||||
threshold_pct: 80,
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
foreground_pressure(&write_higher, 80, 80),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundWrite,
|
||||
usage_pct: 95,
|
||||
threshold_pct: 80,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_breaks_utilization_ties_toward_the_write_class() {
|
||||
let snapshot = registry(vec![
|
||||
counted(WorkloadClass::ForegroundRead, AdmissionState::Open, Some(18), Some(20)),
|
||||
counted(WorkloadClass::ForegroundWrite, AdmissionState::Open, Some(18), Some(20)),
|
||||
]);
|
||||
|
||||
assert_eq!(
|
||||
foreground_pressure(&snapshot, 80, 80),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundWrite,
|
||||
usage_pct: 90,
|
||||
threshold_pct: 80,
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn foreground_pressure_triggers_exactly_at_the_threshold_and_not_below() {
|
||||
let at_threshold = registry(vec![counted(
|
||||
WorkloadClass::ForegroundRead,
|
||||
AdmissionState::Open,
|
||||
Some(8),
|
||||
Some(10),
|
||||
)]);
|
||||
let below_threshold = registry(vec![counted(
|
||||
WorkloadClass::ForegroundRead,
|
||||
AdmissionState::Open,
|
||||
Some(7),
|
||||
Some(10),
|
||||
)]);
|
||||
|
||||
assert_eq!(
|
||||
foreground_pressure(&at_threshold, 80, 80),
|
||||
Some(ForegroundPressure {
|
||||
class: WorkloadClass::ForegroundRead,
|
||||
usage_pct: 80,
|
||||
threshold_pct: 80,
|
||||
})
|
||||
);
|
||||
assert_eq!(foreground_pressure(&below_threshold, 80, 80), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,20 +59,20 @@ pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
|
||||
// ============================================================================
|
||||
|
||||
/// Scheduled update interval in seconds
|
||||
/// Default: 600 seconds (10 minutes)
|
||||
pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 600;
|
||||
/// Default: 120 seconds (2 minutes)
|
||||
pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 120;
|
||||
|
||||
/// Write trigger delay in seconds
|
||||
/// Default: 30 seconds
|
||||
pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 30;
|
||||
/// Default: 5 seconds
|
||||
pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 5;
|
||||
|
||||
/// Write frequency threshold (writes per minute)
|
||||
/// Default: 20 writes/minute
|
||||
pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 20;
|
||||
/// Default: 5 writes/minute
|
||||
pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 5;
|
||||
|
||||
/// Fast update threshold in seconds
|
||||
/// Default: 120 seconds
|
||||
pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 120;
|
||||
/// Default: 30 seconds
|
||||
pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 30;
|
||||
|
||||
/// Maximum files threshold for sampling
|
||||
/// Default: 200,000 files
|
||||
@@ -129,16 +129,4 @@ mod tests {
|
||||
assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
|
||||
assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_capacity_default_values() {
|
||||
assert_eq!(DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, 600);
|
||||
assert_eq!(DEFAULT_WRITE_TRIGGER_DELAY_SECS, 30);
|
||||
assert_eq!(DEFAULT_WRITE_FREQUENCY_THRESHOLD, 20);
|
||||
assert_eq!(DEFAULT_FAST_UPDATE_THRESHOLD_SECS, 120);
|
||||
assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
|
||||
assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
|
||||
assert_eq!(DEFAULT_SAMPLE_RATE, 200);
|
||||
assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -297,7 +297,7 @@ const _: () = assert!(!DEFAULT_PUT_FOREGROUND_ADMISSION_ENABLE);
|
||||
pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE";
|
||||
pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE: bool = true;
|
||||
|
||||
/// Maximum automatic foreground write requests admitted concurrently per process.
|
||||
/// Maximum large foreground PutObject requests admitted concurrently per process.
|
||||
///
|
||||
/// `0` derives a conservative default from the local disk-read scheduler cap,
|
||||
/// currently clamped to protect the commit path without making ordinary high
|
||||
@@ -305,24 +305,14 @@ pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_ENABLE: bool = true;
|
||||
pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT";
|
||||
pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_LIMIT: usize = 0;
|
||||
|
||||
/// Minimum direct PutObject size that enters automatic foreground write admission.
|
||||
/// Minimum object size that enters automatic large PutObject admission.
|
||||
///
|
||||
/// Requests with an unknown size are treated as large because the write pressure
|
||||
/// cannot be bounded from headers.
|
||||
pub const ENV_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: &str = "RUSTFS_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES";
|
||||
pub const DEFAULT_PUT_LARGE_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: usize = 32 * 1024 * 1024;
|
||||
|
||||
/// Minimum UploadPart size that enters automatic foreground write admission.
|
||||
///
|
||||
/// Multipart pressure is often many moderate-sized parts rather than one very
|
||||
/// large request. The default gates every multipart part through the same permit
|
||||
/// pool as large/unknown-size PutObject while keeping small direct PUTs on the
|
||||
/// legacy path.
|
||||
pub const ENV_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: &str =
|
||||
"RUSTFS_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES";
|
||||
pub const DEFAULT_PUT_MULTIPART_FOREGROUND_ADMISSION_MIN_SIZE_BYTES: usize = 0;
|
||||
|
||||
/// Time in milliseconds an automatic foreground write waits for a permit.
|
||||
/// Time in milliseconds a large foreground PutObject waits for a permit.
|
||||
///
|
||||
/// A short wait smooths transient bursts while still returning S3
|
||||
/// `SlowDown`/503 before body ingest when the node is already saturated.
|
||||
|
||||
@@ -198,11 +198,11 @@ pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
|
||||
/// Environment variable that controls scanner cache save timeout in seconds.
|
||||
/// The scanner enforces a minimum value of `1`.
|
||||
/// - Unit: seconds (u64).
|
||||
/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=14`
|
||||
/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
|
||||
pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
|
||||
|
||||
/// Default scanner cache save timeout in seconds.
|
||||
pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14;
|
||||
pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 30;
|
||||
|
||||
/// Environment variable that caps concurrent scanner set tasks.
|
||||
/// A value of `0` keeps the existing topology-based concurrency.
|
||||
|
||||
@@ -100,8 +100,7 @@ aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a",
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
|
||||
async-trait = { workspace = true }
|
||||
flate2.workspace = true
|
||||
http.workspace = true
|
||||
@@ -115,7 +114,6 @@ rustfs-signer.workspace = true
|
||||
# server's implementation: a shared helper could agree with a bug on both sides.
|
||||
data-encoding = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
minlz.workspace = true
|
||||
sha1 = { workspace = true }
|
||||
serde_urlencoded = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -31,9 +31,14 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path};
|
||||
use aws_sdk_s3::Client;
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client, rustfs_binary_path};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use http::header::HOST;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use std::io::Read;
|
||||
use std::process::{Command, Stdio};
|
||||
@@ -82,10 +87,10 @@ mod tests {
|
||||
}
|
||||
|
||||
/// Send a SigV4-signed request to `path` (optionally with a JSON `body`) and
|
||||
/// return `(status, body)`.
|
||||
///
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the
|
||||
/// call sites below keep their `Option<&str>` body shape.
|
||||
/// return `(status, body)`. Uses the `UNSIGNED_PAYLOAD` content hash so a
|
||||
/// request body can be attached without the caller pre-hashing it — the
|
||||
/// server verifies the signature against the same sentinel, exactly as the
|
||||
/// AWS SDKs / MinIO client do for streaming/unsigned payloads.
|
||||
async fn signed_request(
|
||||
base_url: &str,
|
||||
method: http::Method,
|
||||
@@ -94,13 +99,47 @@ mod tests {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("missing authority")?.to_string();
|
||||
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
// The signature is computed over `UNSIGNED_PAYLOAD`, so the body bytes do
|
||||
// not participate in the SigV4 hash — sign over an empty body and attach
|
||||
// the real payload to the wire request below.
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut rb = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
rb = rb.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
rb = rb.body(body_bytes);
|
||||
}
|
||||
let resp = rb.send().await?;
|
||||
let status = resp.status();
|
||||
let text = resp.text().await?;
|
||||
Ok((status, text))
|
||||
}
|
||||
|
||||
/// Build an S3 client bound to explicit credentials (used to exercise the S3
|
||||
/// data plane with rotated / stale root credentials).
|
||||
fn s3_client_with(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "sec4-admin-auth");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
/// Create a non-admin IAM user via the admin `add-user` API using the root
|
||||
@@ -112,7 +151,12 @@ mod tests {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
crate::common::admin_create_user(env, access_key, secret_key).await
|
||||
let path = format!("/rustfs/admin/v3/add-user?accessKey={access_key}");
|
||||
let body = serde_json::json!({ "secretKey": secret_key, "status": "enabled" }).to_string();
|
||||
let (status, resp) =
|
||||
signed_request(&env.url, http::Method::PUT, &path, Some(&body), &env.access_key, &env.secret_key).await?;
|
||||
assert!(status.is_success(), "add-user should succeed (status={status}, body={resp})");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A fully authenticated but non-admin credential must be rejected with
|
||||
|
||||
@@ -59,8 +59,8 @@ mod tests {
|
||||
|
||||
/// One signed admin request, returning the status and the raw body.
|
||||
///
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the
|
||||
/// call sites below keep their `Option<&str>` body shape.
|
||||
/// Signs with `UNSIGNED_PAYLOAD` so the body does not participate in the
|
||||
/// hash, matching how the other admin e2e tests drive these routes.
|
||||
async fn signed_request(
|
||||
base_url: &str,
|
||||
method: http::Method,
|
||||
@@ -69,7 +69,30 @@ mod tests {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(reqwest::StatusCode, String), Box<dyn Error + Send + Sync>> {
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("missing authority")?.to_string();
|
||||
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut builder = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
builder = builder.body(body_bytes);
|
||||
}
|
||||
let response = builder.send().await?;
|
||||
let status = response.status();
|
||||
let text = response.text().await?;
|
||||
Ok((status, text))
|
||||
}
|
||||
|
||||
/// A SigV4-signed `AssumeRole` form POST, optionally carrying a second factor.
|
||||
|
||||
@@ -15,23 +15,39 @@
|
||||
//! Regression test for Issue #1423
|
||||
//! Verifies that Bucket Policies are honored for Authenticated Users.
|
||||
|
||||
use crate::common::{AdminTransport, RustFSTestEnvironment, admin_create_user_via, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use tracing::info;
|
||||
|
||||
/// This suite deliberately drives the admin API through the external `awscurl`
|
||||
/// binary, so user creation pins `AdminTransport::Awscurl`.
|
||||
async fn create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
username: &str,
|
||||
password: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
|
||||
let create_user_body = serde_json::json!({
|
||||
"secretKey": password,
|
||||
"status": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
crate::common::awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn create_user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "test-user");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -27,10 +27,8 @@
|
||||
//! Readiness is established by the harness's `start()` handshake (TCP reachability
|
||||
//! plus an S3 `ListBuckets` poll) — there are no fixed sleeps.
|
||||
//!
|
||||
//! The volume-proxy smoke below also proves that the socket-level fault proxy
|
||||
//! can be installed before startup without changing the client-facing node URL.
|
||||
//! A full lock-plane partition matrix and 5GiB large-object budget remain
|
||||
//! tracked separately.
|
||||
//! Out of scope for this block (tracked separately): network fault injection
|
||||
//! (toxiproxy / socket proxy) and 5GiB large-object budgets.
|
||||
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
|
||||
@@ -78,28 +76,6 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 4 nodes x 4 drives, single pool: exercise the maximum local erasure layout
|
||||
/// supported by the cluster harness. This remains in the nightly lane because
|
||||
/// it starts four real server processes and sixteen data directories.
|
||||
#[tokio::test]
|
||||
async fn cluster_four_node_four_drive_single_pool_smoke() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?;
|
||||
|
||||
let volumes = cluster.rustfs_volumes_arg();
|
||||
assert_eq!(volumes.split(' ').count(), 16, "expected 16 explicit endpoints, got: {volumes}");
|
||||
assert!(!volumes.contains('{'), "single-pool layout must not use ellipses: {volumes}");
|
||||
assert!(cluster.nodes.iter().all(|node| node.data_dirs.len() == 4));
|
||||
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
|
||||
let payload = vec![0x3Cu8; 1024 * 1024];
|
||||
put_get_roundtrip(&cluster, "multidrive-4/object", &payload).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Two single-node pools, 2 drives each: the multi-pool layout boots and
|
||||
/// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1).
|
||||
#[tokio::test]
|
||||
@@ -127,27 +103,3 @@ async fn cluster_two_pool_smoke() -> TestResult {
|
||||
put_get_roundtrip(&cluster, "twopool/object", &payload).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A real cluster smoke for the volume FaultProxy wiring. The proxy target is
|
||||
/// not listening yet when it is created; cluster startup must still converge
|
||||
/// once the target node starts, and peer disk/RPC traffic must traverse it.
|
||||
#[tokio::test]
|
||||
async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(2, 2)).await?;
|
||||
let proxy = cluster.start_volume_proxy_for_node(0).await?;
|
||||
let proxied = proxy.local_addr().to_string();
|
||||
assert!(cluster.rustfs_volumes_arg().contains(&proxied));
|
||||
|
||||
let result: TestResult = async {
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
let payload = vec![0x6Du8; 256 * 1024];
|
||||
put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await
|
||||
}
|
||||
.await;
|
||||
|
||||
proxy.shutdown().await;
|
||||
result
|
||||
}
|
||||
|
||||
+32
-238
@@ -34,7 +34,6 @@ use serde_json;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -218,37 +217,7 @@ pub(crate) async fn signed_s3_request(
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||
signed_s3_request_with_headers(method, url, body, content_type, access_key, secret_key, &http::HeaderMap::new()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn signed_s3_request_with_headers(
|
||||
method: http::Method,
|
||||
url: &str,
|
||||
body: Option<String>,
|
||||
content_type: Option<&str>,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
extra_headers: &http::HeaderMap,
|
||||
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||
signed_s3_request_with_session_token(
|
||||
method,
|
||||
url,
|
||||
body,
|
||||
content_type,
|
||||
SigningCredentials {
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token: None,
|
||||
},
|
||||
extra_headers,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
struct SigningCredentials<'a> {
|
||||
access_key: &'a str,
|
||||
secret_key: &'a str,
|
||||
session_token: Option<&'a str>,
|
||||
signed_s3_request_with_session_token(method, url, body, content_type, access_key, secret_key, None).await
|
||||
}
|
||||
|
||||
async fn signed_s3_request_with_session_token(
|
||||
@@ -256,8 +225,9 @@ async fn signed_s3_request_with_session_token(
|
||||
url: &str,
|
||||
body: Option<String>,
|
||||
content_type: Option<&str>,
|
||||
credentials: SigningCredentials<'_>,
|
||||
extra_headers: &http::HeaderMap,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
session_token: Option<&str>,
|
||||
) -> Result<reqwest::Response, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("S3 URL missing authority")?.to_string();
|
||||
@@ -269,17 +239,14 @@ async fn signed_s3_request_with_session_token(
|
||||
if let Some(content_type) = content_type {
|
||||
request = request.header(CONTENT_TYPE, content_type);
|
||||
}
|
||||
for (name, value) in extra_headers {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
|
||||
let content_length = i64::try_from(body.as_ref().map_or(0, String::len)).map_err(|_| "S3 request body is too large")?;
|
||||
let signed = sign_v4(
|
||||
request.body(Body::empty())?,
|
||||
content_length,
|
||||
credentials.access_key,
|
||||
credentials.secret_key,
|
||||
credentials.session_token.unwrap_or_default(),
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token.unwrap_or_default(),
|
||||
"us-east-1",
|
||||
);
|
||||
|
||||
@@ -316,19 +283,8 @@ pub(crate) async fn admin_request_with_session_token(
|
||||
) -> Result<(StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path_and_query}");
|
||||
let content_type = body.as_ref().map(|_| "application/json");
|
||||
let response = signed_s3_request_with_session_token(
|
||||
method,
|
||||
&url,
|
||||
body,
|
||||
content_type,
|
||||
SigningCredentials {
|
||||
access_key,
|
||||
secret_key,
|
||||
session_token,
|
||||
},
|
||||
&http::HeaderMap::new(),
|
||||
)
|
||||
.await?;
|
||||
let response =
|
||||
signed_s3_request_with_session_token(method, &url, body, content_type, access_key, secret_key, session_token).await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
Ok((status, body))
|
||||
@@ -1215,9 +1171,6 @@ pub struct RustFSTestClusterEnvironment {
|
||||
pub node_extra_env: Vec<Vec<(String, String)>>,
|
||||
pub node_capture_log_paths: Vec<Option<String>>,
|
||||
pub topology: ClusterTopology,
|
||||
/// Optional socket proxies used for the corresponding node's volume
|
||||
/// endpoints. Proxies must be installed before [`Self::start`].
|
||||
volume_proxy_addresses: Vec<Option<SocketAddr>>,
|
||||
}
|
||||
|
||||
impl RustFSTestClusterEnvironment {
|
||||
@@ -1309,7 +1262,6 @@ impl RustFSTestClusterEnvironment {
|
||||
extra_env.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
|
||||
}
|
||||
|
||||
let node_count = topology.node_count;
|
||||
Ok(Self {
|
||||
nodes,
|
||||
temp_dir,
|
||||
@@ -1319,7 +1271,6 @@ impl RustFSTestClusterEnvironment {
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
volume_proxy_addresses: vec![None; node_count],
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1387,34 +1338,6 @@ impl RustFSTestClusterEnvironment {
|
||||
self.build_volumes_arg()
|
||||
}
|
||||
|
||||
/// Start a socket proxy for one node's volume endpoints and route all
|
||||
/// subsequent `RUSTFS_VOLUMES` references for that node through it.
|
||||
///
|
||||
/// Call this before [`Self::start`], then use the returned proxy's
|
||||
/// [`crate::fault_proxy::FaultProxy::set_mode`] to inject latency,
|
||||
/// blackhole, or one-way partition faults. The node's own listen address
|
||||
/// remains direct, so S3 clients can still reach it while peer disk/RPC
|
||||
/// traffic is steered through the proxy.
|
||||
pub async fn start_volume_proxy_for_node(
|
||||
&mut self,
|
||||
node_idx: usize,
|
||||
) -> Result<crate::fault_proxy::FaultProxy, Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.ensure_node_index(node_idx)?;
|
||||
if self.volume_proxy_addresses[node_idx].is_some() {
|
||||
return Err(format!("a volume proxy is already configured for node {node_idx}").into());
|
||||
}
|
||||
let target = self.nodes[node_idx].address.parse::<SocketAddr>()?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(target).await?;
|
||||
self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
|
||||
Ok(proxy)
|
||||
}
|
||||
|
||||
fn volume_address(&self, node_idx: usize) -> String {
|
||||
self.volume_proxy_addresses[node_idx]
|
||||
.map(|address| address.to_string())
|
||||
.unwrap_or_else(|| self.nodes[node_idx].address.clone())
|
||||
}
|
||||
|
||||
fn build_volumes_arg(&self) -> String {
|
||||
let pools = self.topology.normalized_pools();
|
||||
|
||||
@@ -1423,11 +1346,7 @@ impl RustFSTestClusterEnvironment {
|
||||
return self
|
||||
.nodes
|
||||
.iter()
|
||||
.enumerate()
|
||||
.flat_map(|(node_idx, n)| {
|
||||
let address = self.volume_address(node_idx);
|
||||
n.data_dirs.iter().map(move |dir| format!("http://{}{}", address, dir))
|
||||
})
|
||||
.flat_map(|n| n.data_dirs.iter().map(move |dir| format!("http://{}{}", n.address, dir)))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
}
|
||||
@@ -1438,19 +1357,13 @@ impl RustFSTestClusterEnvironment {
|
||||
pools
|
||||
.iter()
|
||||
.map(|nodes| {
|
||||
let node_idx = nodes[0];
|
||||
let node = &self.nodes[node_idx];
|
||||
let node = &self.nodes[nodes[0]];
|
||||
let base = node
|
||||
.data_dirs
|
||||
.first()
|
||||
.and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent))
|
||||
.unwrap_or(&node.data_dir);
|
||||
format!(
|
||||
"http://{}{}/drive{{0...{}}}",
|
||||
self.volume_address(node_idx),
|
||||
base,
|
||||
self.topology.drives_per_node - 1
|
||||
)
|
||||
format!("http://{}{}/drive{{0...{}}}", node.address, base, self.topology.drives_per_node - 1)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
@@ -1831,128 +1744,30 @@ pub(crate) async fn admin_create_user(
|
||||
username: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
admin_create_user_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, username, secret_key).await
|
||||
}
|
||||
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
let body = serde_json::json!({
|
||||
"secretKey": secret_key,
|
||||
"status": "enabled"
|
||||
});
|
||||
let response = signed_request(
|
||||
http::Method::PUT,
|
||||
&url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
Some(body.to_string().into_bytes()),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await?;
|
||||
|
||||
/// Transport used by the shared admin-API helpers: in-process SigV4 signing
|
||||
/// via [`signed_request`], or the external `awscurl` binary (an independent
|
||||
/// SigV4 implementation exercised by the awscurl-gated suites).
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub(crate) enum AdminTransport {
|
||||
Signed,
|
||||
Awscurl,
|
||||
}
|
||||
|
||||
/// Execute an admin-API request against `base_url` with admin credentials over
|
||||
/// the chosen transport, failing on any non-success response.
|
||||
pub(crate) async fn admin_execute_at(
|
||||
transport: AdminTransport,
|
||||
method: http::Method,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
path_and_query: &str,
|
||||
body: Option<&str>,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let url = format!("{base_url}{path_and_query}");
|
||||
match transport {
|
||||
AdminTransport::Signed => {
|
||||
let content_type = match body {
|
||||
Some(body) if !body.is_empty() => Some("application/json"),
|
||||
_ => None,
|
||||
};
|
||||
let response = signed_request(
|
||||
method.clone(),
|
||||
&url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
body.map(|body| body.as_bytes().to_vec()),
|
||||
content_type,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
let status = response.status();
|
||||
let text = response.text().await.unwrap_or_default();
|
||||
return Err(format!("{method} {path_and_query} failed: {status} {text}").into());
|
||||
}
|
||||
}
|
||||
AdminTransport::Awscurl => {
|
||||
execute_awscurl(&url, method.as_str(), body, admin_access_key, admin_secret_key).await?;
|
||||
}
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("create user failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Create a new IAM user via the admin API over the chosen transport.
|
||||
pub(crate) async fn admin_create_user_via(
|
||||
transport: AdminTransport,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
username: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = format!("/rustfs/admin/v3/add-user?accessKey={username}");
|
||||
let body = serde_json::json!({"secretKey": secret_key, "status": "enabled"}).to_string();
|
||||
admin_execute_at(
|
||||
transport,
|
||||
http::Method::PUT,
|
||||
base_url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
&path,
|
||||
Some(&body),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Install a canned policy via the admin API over the chosen transport.
|
||||
pub(crate) async fn admin_add_canned_policy_via(
|
||||
transport: AdminTransport,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
policy_name: &str,
|
||||
policy_json: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = format!("/rustfs/admin/v3/add-canned-policy?name={policy_name}");
|
||||
admin_execute_at(
|
||||
transport,
|
||||
http::Method::PUT,
|
||||
base_url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
&path,
|
||||
Some(policy_json),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Attach a canned policy to a user via the admin API over the chosen transport.
|
||||
pub(crate) async fn admin_attach_user_policy_via(
|
||||
transport: AdminTransport,
|
||||
base_url: &str,
|
||||
admin_access_key: &str,
|
||||
admin_secret_key: &str,
|
||||
policy_name: &str,
|
||||
username: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
let path = format!("/rustfs/admin/v3/set-user-or-group-policy?policyName={policy_name}&userOrGroup={username}&isGroup=false");
|
||||
// `Some("")` preserves the historical wire shape on both transports: awscurl
|
||||
// keeps sending `-d ''` and the signed path attaches an empty body with no
|
||||
// content type.
|
||||
admin_execute_at(
|
||||
transport,
|
||||
http::Method::PUT,
|
||||
base_url,
|
||||
admin_access_key,
|
||||
admin_secret_key,
|
||||
&path,
|
||||
Some(""),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -2044,7 +1859,7 @@ mod tests {
|
||||
}
|
||||
let multidrive = topology.drives_per_node > 1;
|
||||
|
||||
let nodes: Vec<ClusterNode> = (0..topology.node_count)
|
||||
let nodes = (0..topology.node_count)
|
||||
.map(|i| {
|
||||
let address = format!("127.0.0.1:{}", 9000 + i);
|
||||
let data_dirs: Vec<String> = if multidrive {
|
||||
@@ -2065,7 +1880,6 @@ mod tests {
|
||||
})
|
||||
.collect();
|
||||
|
||||
let node_count = nodes.len();
|
||||
RustFSTestClusterEnvironment {
|
||||
nodes,
|
||||
temp_dir,
|
||||
@@ -2075,7 +1889,6 @@ mod tests {
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
volume_proxy_addresses: vec![None; node_count],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2160,25 +1973,6 @@ mod tests {
|
||||
assert!(ClusterTopology::single_pool_multidrive(1, 1).validate().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn volume_proxy_rewrites_cluster_volume_endpoint() {
|
||||
let mut env = RustFSTestClusterEnvironment::new(1)
|
||||
.await
|
||||
.expect("cluster environment should allocate a node");
|
||||
let direct = env.nodes[0].address.clone();
|
||||
let proxy = env
|
||||
.start_volume_proxy_for_node(0)
|
||||
.await
|
||||
.expect("volume proxy should bind before the target server starts");
|
||||
let proxied = proxy.local_addr().to_string();
|
||||
let volumes = env.rustfs_volumes_arg();
|
||||
|
||||
assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}");
|
||||
assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}");
|
||||
|
||||
proxy.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_env_supports_per_node_overrides() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(4));
|
||||
|
||||
@@ -16,29 +16,37 @@
|
||||
//! session policy** (`Policy` parameter) via `awscurl --service sts` with explicit
|
||||
//! `Content-Type: application/x-www-form-urlencoded` on `POST /`.
|
||||
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
|
||||
awscurl_delete, awscurl_post_sts_form_urlencoded, build_test_s3_config, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use crate::common::{RustFSTestEnvironment, awscurl_delete, awscurl_post_sts_form_urlencoded, awscurl_put, init_logging};
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{Delete, ObjectIdentifier, Tag, Tagging};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use tracing::info;
|
||||
use uuid::Uuid;
|
||||
|
||||
fn user_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-existing-tag");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
fn sts_session_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str, session_token: &str) -> Client {
|
||||
Client::from_conf(build_test_s3_config(
|
||||
&env.url,
|
||||
access_key,
|
||||
secret_key,
|
||||
Some(session_token),
|
||||
"e2e-sts-session",
|
||||
))
|
||||
let credentials = Credentials::new(access_key, secret_key, Some(session_token.into()), None, "e2e-sts-session");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
fn extract_xml_tag(xml: &str, tag: &str) -> Option<String> {
|
||||
@@ -69,16 +77,15 @@ async fn assume_role_with_session_policy(
|
||||
parse_assume_role_credentials(&xml)
|
||||
}
|
||||
|
||||
// This suite deliberately drives the admin API through the external `awscurl`
|
||||
// binary (an independent SigV4 implementation), so the wrappers below pin
|
||||
// `AdminTransport::Awscurl`.
|
||||
|
||||
async fn admin_create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
username: &str,
|
||||
password: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
|
||||
let body = serde_json::json!({ "secretKey": password, "status": "enabled" }).to_string();
|
||||
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
awscurl_put(&url, &body, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admin_add_canned_policy(
|
||||
@@ -86,15 +93,9 @@ async fn admin_add_canned_policy(
|
||||
policy_name: &str,
|
||||
policy_json: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Awscurl,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
policy_name,
|
||||
policy_json,
|
||||
)
|
||||
.await
|
||||
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
|
||||
awscurl_put(&url, policy_json, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admin_attach_policy_to_user(
|
||||
@@ -102,7 +103,12 @@ async fn admin_attach_policy_to_user(
|
||||
policy_name: &str,
|
||||
username: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
env.url, policy_name, username
|
||||
);
|
||||
awscurl_put(&url, "", &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admin_remove_user(env: &RustFSTestEnvironment, username: &str) {
|
||||
|
||||
@@ -15,11 +15,20 @@
|
||||
//! E2E tests for group management (fixes #2028).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_ok, admin_request, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use tracing::info;
|
||||
|
||||
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-group-test");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
|
||||
@@ -16,14 +16,13 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging};
|
||||
use crate::chaos::signed_admin_post;
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use tracing::info;
|
||||
|
||||
fn has_file_under(path: &Path) -> bool {
|
||||
@@ -49,110 +48,6 @@ mod tests {
|
||||
disk.join(bucket).join(key).join("xl.meta").is_file()
|
||||
}
|
||||
|
||||
// Healing may rewrite non-identity bookkeeping in xl.meta. The census
|
||||
// therefore compares the canonical selected metadata fields plus every
|
||||
// physical shard, while the payload seed makes object mix-ups observable.
|
||||
#[derive(Debug)]
|
||||
struct PhysicalObjectManifest {
|
||||
key: String,
|
||||
payload_seed: u8,
|
||||
shard_census: VersionShardCensus,
|
||||
}
|
||||
|
||||
fn deterministic_object_body(len: usize, seed: u8) -> Vec<u8> {
|
||||
let mut value = seed;
|
||||
std::iter::repeat_with(|| {
|
||||
value = value.wrapping_mul(31).wrapping_add(17);
|
||||
value
|
||||
})
|
||||
.take(len)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn matching_manifest_count(
|
||||
disk: &Path,
|
||||
bucket: &str,
|
||||
expected_manifests: &[PhysicalObjectManifest],
|
||||
) -> Result<usize, Box<dyn Error + Send + Sync>> {
|
||||
let mut matching = 0;
|
||||
for expected in expected_manifests {
|
||||
let actual = census_object_version_on_disk(disk, bucket, &expected.key, None)?;
|
||||
if actual.matches_manifest(&expected.shard_census) {
|
||||
matching += 1;
|
||||
}
|
||||
}
|
||||
Ok(matching)
|
||||
}
|
||||
|
||||
fn metadata_count(disk: &Path, bucket: &str, expected_manifests: &[PhysicalObjectManifest]) -> usize {
|
||||
expected_manifests
|
||||
.iter()
|
||||
.filter(|expected| object_metadata_exists_on_disk(disk, bucket, &expected.key))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
let Ok(status) = serde_json::from_str::<serde_json::Value>(body) else {
|
||||
return body.to_string();
|
||||
};
|
||||
let items = status["items"].as_array();
|
||||
let mut unresolved_states = HashSet::new();
|
||||
for item in items.into_iter().flatten() {
|
||||
for drive in item["after"]["drives"].as_array().into_iter().flatten() {
|
||||
if let Some(state) = drive["state"].as_str()
|
||||
&& state != "ok"
|
||||
{
|
||||
unresolved_states.insert(state.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut unresolved_states = unresolved_states.into_iter().collect::<Vec<_>>();
|
||||
unresolved_states.sort();
|
||||
format!(
|
||||
"summary={:?}, detail={:?}, item_count={}, unresolved_drive_states={unresolved_states:?}",
|
||||
status["summary"].as_str(),
|
||||
status["detail"].as_str(),
|
||||
items.map_or(0, Vec::len)
|
||||
)
|
||||
}
|
||||
|
||||
fn cluster_heal_is_idle(status: &serde_json::Value) -> bool {
|
||||
let operations = &status["healOperations"];
|
||||
status["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
&& status["state"].as_str() == Some("idle")
|
||||
&& operations["queueLength"].as_u64() == Some(0)
|
||||
&& operations["activeTasks"].as_u64() == Some(0)
|
||||
&& operations["retryingTasks"].as_u64() == Some(0)
|
||||
}
|
||||
|
||||
fn only_admin_heal_is_active(status: &serde_json::Value) -> bool {
|
||||
let operations = &status["healOperations"];
|
||||
status["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
&& status["state"].as_str() == Some("active")
|
||||
&& operations["queueLength"].as_u64() == Some(0)
|
||||
&& operations["activeTasks"].as_u64() == Some(1)
|
||||
&& operations["retryingTasks"].as_u64() == Some(0)
|
||||
&& operations["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
}
|
||||
|
||||
async fn replacement_recovery_status(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
let (status, body) = admin_request(
|
||||
&cluster.nodes[0].url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v4/heal/replacement-recovery",
|
||||
None,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("replacement recovery status failed: {status} {body}").into());
|
||||
}
|
||||
serde_json::from_str(&body).map_err(|err| format!("replacement recovery status is not JSON ({err}): {body}").into())
|
||||
}
|
||||
|
||||
async fn assert_object_body(env: &RustFSTestEnvironment, bucket: &str, key: &str, expected: &[u8]) {
|
||||
let client = env.create_s3_client();
|
||||
let response = client
|
||||
@@ -547,380 +442,6 @@ mod tests {
|
||||
.into())
|
||||
}
|
||||
|
||||
// Keep the original unformatted-disk scenario above. This case retains the
|
||||
// format identity so only the explicit admin task can rebuild missing data.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_resumes_missing_remote_shards_after_node_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
info!(
|
||||
event = "heal_restart_started",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
"Starting root-heal restart test"
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_PER_SET", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_PARALLEL_ENABLE", "false");
|
||||
// Keep all storage nodes' Heal runtimes enabled so their disk services
|
||||
// complete normal registration after restart. Scanner, auto-heal and
|
||||
// MRF are disabled; the pre-root idle barrier below drains the direct
|
||||
// outage-object repair before the explicit admin task starts.
|
||||
let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
|
||||
.unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
|
||||
cluster.set_env("RUST_LOG", server_rust_log);
|
||||
if let Ok(log_dir) = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR") {
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
}
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
clients[0].create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let replaced_disk = PathBuf::from(&cluster.nodes[1].data_dir);
|
||||
let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
|
||||
let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
|
||||
format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}")
|
||||
})?;
|
||||
let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(24)
|
||||
.clamp(8, 64);
|
||||
let object_size_bytes = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(4 * 1024 * 1024)
|
||||
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
for index in 0..online_object_count {
|
||||
let key = format!("cluster/online/object-{index:04}.bin");
|
||||
let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?;
|
||||
assert!(
|
||||
shard_census.is_complete(),
|
||||
"node 1 should hold a complete baseline shard for {key}: {shard_census:?}"
|
||||
);
|
||||
assert!(
|
||||
!shard_census.expected_part_numbers.is_empty(),
|
||||
"chaos objects must use physical part shards rather than inline data: {shard_census:?}"
|
||||
);
|
||||
expected_manifests.push(PhysicalObjectManifest {
|
||||
key,
|
||||
payload_seed,
|
||||
shard_census,
|
||||
});
|
||||
}
|
||||
|
||||
cluster.stop_node(1)?;
|
||||
std::fs::remove_dir_all(&replaced_disk)?;
|
||||
std::fs::create_dir_all(
|
||||
replacement_format_path
|
||||
.parent()
|
||||
.ok_or("replacement format path has no parent")?,
|
||||
)?;
|
||||
std::fs::write(&replacement_format_path, replacement_format)?;
|
||||
assert!(
|
||||
replacement_format_path.is_file(),
|
||||
"replacement target must retain only its preformatted topology identity"
|
||||
);
|
||||
|
||||
let outage_key = "cluster/written-while-node-down.bin";
|
||||
let outage_payload_seed = 0xf1;
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[2]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(outage_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let mut outage_peer_erasure_indices = HashSet::new();
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
let census = census_object_version_on_disk(Path::new(&node.data_dir), bucket, outage_key, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census
|
||||
.erasure_index
|
||||
.ok_or_else(|| format!("online node {node_index} outage-object shard has no erasure index: {census:?}"))?;
|
||||
assert!(
|
||||
(1..=cluster.nodes.len()).contains(&erasure_index),
|
||||
"online node {node_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online nodes"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
outage_peer_erasure_indices.len(),
|
||||
cluster.nodes.len().saturating_sub(1),
|
||||
"every online node must contribute one unique outage-object erasure index"
|
||||
);
|
||||
let expected_outage_target_erasure_index = (1..=cluster.nodes.len())
|
||||
.find(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.ok_or("online outage-object shards leave no erasure index for the replacement target")?;
|
||||
|
||||
// The PUT path may have admitted a direct Internal object repair while
|
||||
// node 1 was offline. Cancel the isolated bucket path before the target
|
||||
// returns; otherwise it could rebuild the outage object and invalidate
|
||||
// the explicit-root ownership assertion below.
|
||||
let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
|
||||
let (cancel_status, cancel_body) = admin_request(
|
||||
&cluster.nodes[0].url,
|
||||
Method::POST,
|
||||
&cancel_outage_heal_path,
|
||||
Some(
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#
|
||||
.to_string(),
|
||||
),
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !cancel_status.is_success() {
|
||||
return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
|
||||
}
|
||||
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
assert!(
|
||||
!status_body.contains("MissingContentLength"),
|
||||
"background heal status should not fail without an explicit Content-Length: {status_body}"
|
||||
);
|
||||
let recovered: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if cluster_heal_is_idle(&recovered) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= recovery_deadline {
|
||||
return Err(format!("cluster heal operations did not become idle before root heal: {recovered}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?,
|
||||
0,
|
||||
"non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
|
||||
);
|
||||
assert!(
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
|
||||
"the object written during the outage must be absent before the explicit root heal"
|
||||
);
|
||||
let pre_heal_replacement = replacement_recovery_status(&cluster).await?;
|
||||
assert_eq!(
|
||||
pre_heal_replacement["cluster"]["records"].as_array().map(Vec::len),
|
||||
Some(0),
|
||||
"isolated target must not retain an automatic replacement generation: {pre_heal_replacement}"
|
||||
);
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
|
||||
let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
let client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?;
|
||||
let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
|
||||
let partial_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(60);
|
||||
let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs);
|
||||
let pre_interrupt_status = loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let active_status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if only_admin_heal_is_active(&active_status) {
|
||||
break active_status;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
let partial_count = loop {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
if matching > 0 && matching < expected_manifests.len() {
|
||||
break matching;
|
||||
}
|
||||
if matching == expected_manifests.len() {
|
||||
return Err(format!(
|
||||
"root heal rebuilt all {} baseline objects before the target could be interrupted",
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!(
|
||||
"root heal made no observable partial progress on the replacement target within {partial_timeout_secs}s"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
info!(
|
||||
event = "heal_restart_checkpoint",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
partial_count,
|
||||
"Verified unique admin owner before target interruption"
|
||||
);
|
||||
|
||||
cluster.stop_node(1)?;
|
||||
let stopped_count = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
assert!(
|
||||
stopped_count > 0 && stopped_count < expected_manifests.len(),
|
||||
"the target must stop after a partial rebuild, observed before stop={partial_count}, after stop={stopped_count}, total={}",
|
||||
expected_manifests.len()
|
||||
);
|
||||
let unclean_shutdown_marker = replaced_disk.join(".rustfs.sys").join("unclean-shutdown");
|
||||
match std::fs::remove_file(&unclean_shutdown_marker) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => {
|
||||
return Err(format!("failed to isolate unclean recovery marker {unclean_shutdown_marker:?}: {error}").into());
|
||||
}
|
||||
}
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let heal_timeout_secs = std::env::var("RUSTFS_HEAL_REPLACED_DISK_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(180);
|
||||
let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
|
||||
loop {
|
||||
if metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len()
|
||||
&& object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key)
|
||||
{
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
if matching == expected_manifests.len() && outage_census.is_complete() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if Instant::now() >= heal_deadline {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key)
|
||||
.await
|
||||
.unwrap_or_else(|err| format!("status request failed: {err}"));
|
||||
let task_status = match timeout(
|
||||
Duration::from_secs(5),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => heal_task_status_diagnostic(&body),
|
||||
Ok(Err(err)) => format!("task status request failed: {err}"),
|
||||
Err(_) => "task status request exceeded 5s diagnostic budget".to_string(),
|
||||
};
|
||||
let replacement_status = match timeout(Duration::from_secs(5), replacement_recovery_status(&cluster)).await {
|
||||
Ok(Ok(status)) => status.to_string(),
|
||||
Ok(Err(err)) => format!("replacement status request failed: {err}"),
|
||||
Err(_) => "replacement status request exceeded 5s diagnostic budget".to_string(),
|
||||
};
|
||||
return Err(format!(
|
||||
"root heal did not resume after target restart within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, replacement_status={replacement_status}",
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
for expected in &expected_manifests {
|
||||
let actual = census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?;
|
||||
assert!(
|
||||
actual.matches_manifest(&expected.shard_census),
|
||||
"rebuilt target shard differs from its baseline for {}: {actual:?}",
|
||||
expected.key
|
||||
);
|
||||
}
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
assert!(
|
||||
outage_census.is_complete(),
|
||||
"outage object must have a complete target shard: {outage_census:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_census.erasure_index,
|
||||
Some(expected_outage_target_erasure_index),
|
||||
"the outage object must be rebuilt into its own missing erasure slot"
|
||||
);
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
|
||||
let terminal_deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if cluster_heal_is_idle(&status) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= terminal_deadline {
|
||||
return Err(format!("heal data rebuilt but operations did not converge to terminal idle: {status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Issue #5850: `background-heal/status` must answer while a peer is down.
|
||||
///
|
||||
/// Exercises the production path in `read_cluster_heal_status` end to end,
|
||||
|
||||
@@ -15,13 +15,13 @@
|
||||
//! Four-node EC regression gate for inline storage and the inline GET reader.
|
||||
//!
|
||||
//! The storage decision is based on shard bytes (256 KiB / 32 KiB objects for
|
||||
//! the default EC 2+2 geometry), and the GET fast path follows the persisted
|
||||
//! inline marker. A local OTLP/HTTP collector observes the existing reader-path
|
||||
//! counter without adding a scrape endpoint or production logging.
|
||||
//! the default EC 2+2 geometry), while the GET fast path has its own object-size
|
||||
//! limits (128 KiB / 16 KiB). A local OTLP/HTTP collector observes the existing
|
||||
//! reader-path counter without adding a scrape endpoint or production logging.
|
||||
//! One S3 GET can select readers on multiple EC nodes, so the counter tracks
|
||||
//! distributed reader selection rather than HTTP request count.
|
||||
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{
|
||||
@@ -30,7 +30,7 @@ use aws_sdk_s3::types::{
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use flate2::read::GzDecoder;
|
||||
use http::header::CONTENT_ENCODING;
|
||||
use http::header::{CONTENT_ENCODING, HOST};
|
||||
use http::{Method, Request, Response, StatusCode};
|
||||
use http_body_util::{BodyExt, Full};
|
||||
use hyper::body::Incoming;
|
||||
@@ -42,6 +42,9 @@ use opentelemetry_proto::tonic::metrics::v1::{
|
||||
Metric, NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum, metric, number_data_point,
|
||||
};
|
||||
use prost::Message;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use std::collections::BTreeMap;
|
||||
use std::convert::Infallible;
|
||||
use std::error::Error;
|
||||
@@ -89,7 +92,6 @@ const MPU_PART_1_SIZE: usize = 5 * 1024 * 1024;
|
||||
const MPU_PART_2_SIZE: usize = 16 * KIB;
|
||||
const TIER_BUCKET: &str = "inline-fallback-cold-tier";
|
||||
const TIER_PREFIX: &str = "tiered";
|
||||
const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: &str = "RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT";
|
||||
const MSGPACK_FALLBACK_CONTROL_SERIES: [(&str, &str); 4] = [
|
||||
(FALLBACK_REQUEST_DIRECTION, "ReadMultipleReq"),
|
||||
(FALLBACK_RESPONSE_DIRECTION, "ReadMultipleResp"),
|
||||
@@ -792,12 +794,12 @@ fn metric_attribute(key: &str, value: &str) -> KeyValue {
|
||||
}
|
||||
|
||||
fn boundary_cases(state: VersionState) -> Vec<BoundaryCase> {
|
||||
let storage_limit = match state {
|
||||
VersionState::Enabled => 32 * KIB,
|
||||
VersionState::Unversioned => 256 * KIB,
|
||||
let (fast_limit, storage_limit) = match state {
|
||||
VersionState::Enabled => (16 * KIB, 32 * KIB),
|
||||
VersionState::Unversioned => (128 * KIB, 256 * KIB),
|
||||
// A suspended bucket stores its null version using the unversioned
|
||||
// shard threshold, while ObjectInfo keeps version-aware GET semantics.
|
||||
VersionState::Suspended => 256 * KIB,
|
||||
VersionState::Suspended => (16 * KIB, 256 * KIB),
|
||||
};
|
||||
let mut sizes = vec![0, 16 * KIB - 1, 16 * KIB, 16 * KIB + 1, 32 * KIB - 1, 32 * KIB, 32 * KIB + 1];
|
||||
if !matches!(state, VersionState::Enabled) {
|
||||
@@ -818,7 +820,7 @@ fn boundary_cases(state: VersionState) -> Vec<BoundaryCase> {
|
||||
stored_inline: size <= storage_limit,
|
||||
expected_reader_path: if size == 0 {
|
||||
EMPTY
|
||||
} else if size <= storage_limit {
|
||||
} else if size <= fast_limit {
|
||||
INLINE_DIRECT
|
||||
} else {
|
||||
LEGACY_DUPLEX
|
||||
@@ -1260,8 +1262,6 @@ async fn put_two_part_multipart(client: &Client, bucket: &str, key: &str) -> Tes
|
||||
Ok((body, part2, complete.e_tag().map(str::to_owned)))
|
||||
}
|
||||
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the call
|
||||
/// sites below keep their `Option<&str>` body shape.
|
||||
async fn signed_admin_request(
|
||||
base_url: &str,
|
||||
method: Method,
|
||||
@@ -1270,7 +1270,30 @@ async fn signed_admin_request(
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> TestResult<(reqwest::StatusCode, String)> {
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
|
||||
let body_bytes = body.map(|value| value.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut request_builder = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
request_builder = request_builder.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
request_builder = request_builder.body(body_bytes);
|
||||
}
|
||||
let response = request_builder.send().await?;
|
||||
let status = response.status();
|
||||
let text = response.text().await?;
|
||||
Ok((status, text))
|
||||
}
|
||||
|
||||
fn unique_tier_name() -> String {
|
||||
@@ -2101,7 +2124,6 @@ async fn four_node_add_tier_converges() -> TestResult {
|
||||
cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestClusterEnvironment::new(4).await?;
|
||||
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
|
||||
hot.start().await?;
|
||||
|
||||
let tier_name = unique_tier_name();
|
||||
@@ -2120,7 +2142,6 @@ async fn four_node_add_tier_converges_after_offline_node_restart_without_second_
|
||||
cold.create_s3_client().create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestClusterEnvironment::new(4).await?;
|
||||
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
|
||||
hot.start().await?;
|
||||
|
||||
let tier_name = unique_tier_name();
|
||||
@@ -2217,7 +2238,6 @@ async fn four_node_manual_transition_distributed_admission_conflict_reports_stat
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestClusterEnvironment::new(4).await?;
|
||||
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
|
||||
hot.set_env("RUSTFS_SCANNER_ENABLED", "false");
|
||||
hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
|
||||
hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "1");
|
||||
@@ -2360,7 +2380,6 @@ async fn four_node_manual_transition_rollout_non_empty_restart_readback() -> Tes
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestClusterEnvironment::new(4).await?;
|
||||
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
|
||||
hot.set_env("RUSTFS_SCANNER_ENABLED", "false");
|
||||
hot.set_env("RUSTFS_SCANNER_CYCLE", "3600");
|
||||
hot.set_env("RUSTFS_MAX_TRANSITION_WORKERS", "2");
|
||||
@@ -2465,7 +2484,6 @@ async fn four_node_mixed_msgpack_compat_mode_preserves_fallback_controls_during_
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut hot = RustFSTestClusterEnvironment::new(4).await?;
|
||||
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
|
||||
configure_mixed_msgpack_cluster(&mut hot, &collector)?;
|
||||
hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
|
||||
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
|
||||
@@ -2577,7 +2595,6 @@ async fn four_node_transitioned_inline_fallback() -> TestResult {
|
||||
|
||||
let collector = OtlpMetricCollector::start().await?;
|
||||
let mut hot = RustFSTestClusterEnvironment::new(4).await?;
|
||||
hot.set_env(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV, "true");
|
||||
configure_reader_metric_cluster(&mut hot, &collector);
|
||||
hot.set_env("RUSTFS_SCANNER_CYCLE", "1");
|
||||
hot.set_env("RUSTFS_ILM_PROCESS_TIME", "1");
|
||||
|
||||
@@ -17,7 +17,6 @@
|
||||
|
||||
use super::common::{
|
||||
LocalKMSTestEnvironment, VAULT_KEY_NAME, VaultTestEnvironment, configure_kms, get_kms_status, kms_admin_request, start_kms,
|
||||
test_sse_kms_encryption,
|
||||
};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, ServerSideEncryption, VersioningConfiguration};
|
||||
@@ -432,38 +431,6 @@ async fn test_configured_local_kms_admin_and_versioned_cleanup() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_configured_local_kms_is_restored_after_restart() -> TestResult {
|
||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||
env.base_env.start_rustfs_server(Vec::new()).await?;
|
||||
|
||||
let default_key_id = env.configure_local_kms().await?;
|
||||
start_kms(&env.base_env.url, &env.base_env.access_key, &env.base_env.secret_key).await?;
|
||||
|
||||
env.base_env.restart_server_preserving_data(Vec::new(), &[]).await?;
|
||||
assert_configured_status(
|
||||
&env.base_env.url,
|
||||
&env.base_env.access_key,
|
||||
&env.base_env.secret_key,
|
||||
"local",
|
||||
&default_key_id,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let bucket = format!("kms-restart-{}", Uuid::new_v4());
|
||||
env.base_env.create_test_bucket(&bucket).await?;
|
||||
let client = env.base_env.create_s3_client();
|
||||
test_sse_kms_encryption(&client, &bucket).await?;
|
||||
client
|
||||
.delete_object()
|
||||
.bucket(&bucket)
|
||||
.key("test-sse-kms-object")
|
||||
.send()
|
||||
.await?;
|
||||
env.base_env.delete_test_bucket(&bucket).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_configured_vault_kms_admin_and_versioned_cleanup() -> TestResult {
|
||||
let mut env = VaultTestEnvironment::new().await?;
|
||||
|
||||
@@ -66,7 +66,6 @@ const SURVIVOR_KEY: &str = "keep/object.bin";
|
||||
const TIER_NAME: &str = "KMSCOLD";
|
||||
const TIER_BUCKET: &str = "kms-ilm-cold-tier";
|
||||
const TIER_PREFIX: &str = "tiered";
|
||||
const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: (&str, &str) = ("RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT", "true");
|
||||
const TRANSITION_BUCKET: &str = "kms-ilm-transition";
|
||||
const TRANSITION_KEY: &str = "tier/object.bin";
|
||||
|
||||
@@ -81,7 +80,7 @@ const ILM_DEADLINE: StdDuration = StdDuration::from_secs(90);
|
||||
/// `--kms-default-key-id`, insecure dev defaults). The lifecycle env matches
|
||||
/// `reliant/lifecycle.rs::fast_lifecycle_env` plus `RUSTFS_ILM_DEBUG_DAY_SECS=2`,
|
||||
/// so a `Days=1` rule is due about two seconds after the write.
|
||||
async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
|
||||
async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment) -> TestResult {
|
||||
create_key_with_specific_id(&env.kms_keys_dir, SSE_KEY).await?;
|
||||
|
||||
let key_dir = env.kms_keys_dir.clone();
|
||||
@@ -95,14 +94,13 @@ async fn start_enforcing_ilm_server(env: &mut LocalKMSTestEnvironment, extra_env
|
||||
SSE_KEY,
|
||||
];
|
||||
|
||||
let mut envs = vec![
|
||||
let envs = [
|
||||
("RUSTFS_KMS_ALLOW_INSECURE_DEV_DEFAULTS", "true"),
|
||||
("RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY", "true"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_ILM_PROCESS_TIME", "1"),
|
||||
("RUSTFS_ILM_DEBUG_DAY_SECS", "2"),
|
||||
];
|
||||
envs.extend_from_slice(extra_env);
|
||||
|
||||
env.base_env.start_rustfs_server_with_env(args, &envs).await?;
|
||||
Ok(())
|
||||
@@ -429,7 +427,7 @@ async fn ilm_expiration_on_sse_kms_bucket_under_enforcement() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||
start_enforcing_ilm_server(&mut env, &[]).await?;
|
||||
start_enforcing_ilm_server(&mut env).await?;
|
||||
env.base_env.create_test_bucket(EXPIRY_BUCKET).await?;
|
||||
|
||||
let client = env.base_env.create_s3_client();
|
||||
@@ -501,7 +499,7 @@ async fn ilm_transition_on_sse_kms_bucket_under_enforcement_reads_back() -> Test
|
||||
|
||||
// Hot server: Local KMS + enforcement + accelerated lifecycle clock.
|
||||
let mut env = LocalKMSTestEnvironment::new().await?;
|
||||
start_enforcing_ilm_server(&mut env, &[ALLOW_LOOPBACK_TIER_ENDPOINT_ENV]).await?;
|
||||
start_enforcing_ilm_server(&mut env).await?;
|
||||
let hot_client = env.base_env.create_s3_client();
|
||||
|
||||
add_rustfs_tier(&env.base_env, &cold.base_env).await?;
|
||||
|
||||
@@ -57,9 +57,6 @@ mod copy_object_version_restore_sse_test;
|
||||
#[cfg(test)]
|
||||
mod configured_roundtrip_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod select_sse_response_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod kms_anonymous_enforcement_test;
|
||||
|
||||
|
||||
@@ -1,241 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SelectObjectContent SSE response-header compatibility (backlog#1625).
|
||||
|
||||
use super::common::{LocalKMSTestEnvironment, sse_customer_key_md5_base64, start_kms};
|
||||
use crate::common::signed_s3_request_with_headers;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::ServerSideEncryption;
|
||||
use base64_simd::STANDARD as BASE64;
|
||||
use http::{HeaderMap, Method};
|
||||
use std::error::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
|
||||
const CSV_BODY: &[u8] = b"name\nalice\n";
|
||||
const SELECT_BODY: &str = r#"<SelectObjectContentRequest xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Expression>SELECT * FROM S3Object</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
|
||||
<OutputSerialization><CSV/></OutputSerialization>
|
||||
</SelectObjectContentRequest>"#;
|
||||
const KMS_CONTEXT: &str = "eyJ0ZW5hbnQiOiJzMy1zZWxlY3QifQ==";
|
||||
const SSE_ALGORITHM: &str = "x-amz-server-side-encryption";
|
||||
const SSE_KMS_KEY_ID: &str = "x-amz-server-side-encryption-aws-kms-key-id";
|
||||
const SSE_KMS_CONTEXT: &str = "x-amz-server-side-encryption-context";
|
||||
const SSE_C_ALGORITHM: &str = "x-amz-server-side-encryption-customer-algorithm";
|
||||
const SSE_C_KEY: &str = "x-amz-server-side-encryption-customer-key";
|
||||
const SSE_C_KEY_MD5: &str = "x-amz-server-side-encryption-customer-key-md5";
|
||||
const LOG_FLUSH_SENTINEL: &str = "select-sse-log-flush-sentinel.csv";
|
||||
|
||||
async fn raw_select(
|
||||
env: &crate::common::RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
request_headers: &HeaderMap,
|
||||
) -> TestResult<reqwest::Response> {
|
||||
let url = format!("{}/{bucket}/{object}?select&select-type=2", env.url);
|
||||
signed_s3_request_with_headers(
|
||||
Method::POST,
|
||||
&url,
|
||||
Some(SELECT_BODY.to_string()),
|
||||
Some("application/xml"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
request_headers,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn assert_success_headers(response: reqwest::Response, expected: &[(&str, &str)], absent: &[&str]) -> TestResult {
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let url = response.url().clone();
|
||||
let body = response.text().await?;
|
||||
panic!("Select request to {url} failed with {status}: {body}");
|
||||
}
|
||||
for (name, value) in expected {
|
||||
assert_eq!(response.headers().get(*name).and_then(|header| header.to_str().ok()), Some(*value));
|
||||
}
|
||||
for name in absent {
|
||||
assert!(response.headers().get(*name).is_none(), "successful Select response must omit {name}");
|
||||
}
|
||||
let body = response.bytes().await?;
|
||||
assert!(
|
||||
body.windows(b"alice".len()).any(|window| window == b"alice"),
|
||||
"successful Select response must contain a Records event with the selected row"
|
||||
);
|
||||
assert!(
|
||||
body.windows(b"End".len()).any(|window| window == b"End"),
|
||||
"successful Select response must contain the terminal End event"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn assert_pre_stream_failure(response: reqwest::Response) -> TestResult {
|
||||
assert_eq!(response.status(), reqwest::StatusCode::BAD_REQUEST);
|
||||
let body = response.text().await?;
|
||||
assert!(body.contains("<Error>"), "pre-stream failure must return an S3 XML error: {body}");
|
||||
assert!(
|
||||
body.contains("<Code>InvalidRequest</Code>"),
|
||||
"invalid SSE-C parameters must preserve the S3 error code: {body}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn put_object(
|
||||
client: &aws_sdk_s3::Client,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
) -> aws_sdk_s3::operation::put_object::builders::PutObjectFluentBuilder {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.body(ByteStream::from_static(CSV_BODY))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn select_projects_encryption_headers_and_rejects_invalid_sse_c_before_streaming() -> TestResult {
|
||||
let mut kms = LocalKMSTestEnvironment::new().await?;
|
||||
let log_path = format!("{}/server.log", kms.base_env.temp_dir);
|
||||
kms.base_env.capture_log_path = Some(log_path.clone());
|
||||
kms.base_env
|
||||
.start_rustfs_server_with_env(Vec::new(), &[("RUST_LOG", "s3s=debug,rustfs=info")])
|
||||
.await?;
|
||||
let key_id = kms.configure_local_kms().await?;
|
||||
start_kms(&kms.base_env.url, &kms.base_env.access_key, &kms.base_env.secret_key).await?;
|
||||
|
||||
let client = kms.base_env.create_s3_client();
|
||||
let bucket = format!("select-sse-{}", Uuid::new_v4().simple());
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
|
||||
put_object(&client, &bucket, "plain.csv").send().await?;
|
||||
put_object(&client, &bucket, "sse-s3.csv")
|
||||
.server_side_encryption(ServerSideEncryption::Aes256)
|
||||
.send()
|
||||
.await?;
|
||||
put_object(&client, &bucket, "sse-kms.csv")
|
||||
.server_side_encryption(ServerSideEncryption::AwsKms)
|
||||
.ssekms_key_id(&key_id)
|
||||
.ssekms_encryption_context(KMS_CONTEXT)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let customer_key = "01234567890123456789012345678901";
|
||||
let customer_key_b64 = BASE64.encode_to_string(customer_key);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(customer_key);
|
||||
put_object(&client, &bucket, "sse-c.csv")
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key_b64)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
assert_success_headers(
|
||||
raw_select(&kms.base_env, &bucket, "plain.csv", &HeaderMap::new()).await?,
|
||||
&[],
|
||||
&[
|
||||
SSE_ALGORITHM,
|
||||
SSE_KMS_KEY_ID,
|
||||
SSE_KMS_CONTEXT,
|
||||
SSE_C_ALGORITHM,
|
||||
SSE_C_KEY,
|
||||
SSE_C_KEY_MD5,
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert_success_headers(
|
||||
raw_select(&kms.base_env, &bucket, "sse-s3.csv", &HeaderMap::new()).await?,
|
||||
&[(SSE_ALGORITHM, "AES256")],
|
||||
&[SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
|
||||
)
|
||||
.await?;
|
||||
assert_success_headers(
|
||||
raw_select(&kms.base_env, &bucket, "sse-kms.csv", &HeaderMap::new()).await?,
|
||||
&[
|
||||
(SSE_ALGORITHM, "aws:kms"),
|
||||
(SSE_KMS_KEY_ID, &key_id),
|
||||
(SSE_KMS_CONTEXT, KMS_CONTEXT),
|
||||
],
|
||||
&[SSE_C_ALGORITHM, SSE_C_KEY, SSE_C_KEY_MD5],
|
||||
)
|
||||
.await?;
|
||||
|
||||
let mut sse_c_headers = HeaderMap::new();
|
||||
sse_c_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
|
||||
sse_c_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
|
||||
sse_c_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
|
||||
assert_success_headers(
|
||||
raw_select(&kms.base_env, &bucket, "sse-c.csv", &sse_c_headers).await?,
|
||||
&[(SSE_C_ALGORITHM, "AES256"), (SSE_C_KEY_MD5, &customer_key_md5)],
|
||||
&[SSE_ALGORITHM, SSE_KMS_KEY_ID, SSE_KMS_CONTEXT, SSE_C_KEY],
|
||||
)
|
||||
.await?;
|
||||
|
||||
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &HeaderMap::new()).await?).await?;
|
||||
|
||||
let mut missing_algorithm_headers = HeaderMap::new();
|
||||
missing_algorithm_headers.insert(SSE_C_KEY, customer_key_b64.parse()?);
|
||||
missing_algorithm_headers.insert(SSE_C_KEY_MD5, customer_key_md5.parse()?);
|
||||
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &missing_algorithm_headers).await?).await?;
|
||||
|
||||
let mut wrong_algorithm_headers = sse_c_headers.clone();
|
||||
wrong_algorithm_headers.insert(SSE_C_ALGORITHM, "AES128".parse()?);
|
||||
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_algorithm_headers).await?).await?;
|
||||
|
||||
let wrong_md5 = sse_customer_key_md5_base64("99999999999999999999999999999999");
|
||||
let mut wrong_md5_headers = sse_c_headers.clone();
|
||||
wrong_md5_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
|
||||
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_md5_headers).await?).await?;
|
||||
|
||||
let wrong_key = "99999999999999999999999999999999";
|
||||
let wrong_key_b64 = BASE64.encode_to_string(wrong_key);
|
||||
let mut wrong_key_headers = HeaderMap::new();
|
||||
wrong_key_headers.insert(SSE_C_ALGORITHM, "AES256".parse()?);
|
||||
wrong_key_headers.insert(SSE_C_KEY, wrong_key_b64.parse()?);
|
||||
wrong_key_headers.insert(SSE_C_KEY_MD5, wrong_md5.parse()?);
|
||||
assert_pre_stream_failure(raw_select(&kms.base_env, &bucket, "sse-c.csv", &wrong_key_headers).await?).await?;
|
||||
|
||||
put_object(&client, &bucket, LOG_FLUSH_SENTINEL).send().await?;
|
||||
assert_success_headers(
|
||||
raw_select(&kms.base_env, &bucket, LOG_FLUSH_SENTINEL, &HeaderMap::new()).await?,
|
||||
&[],
|
||||
&[
|
||||
SSE_ALGORITHM,
|
||||
SSE_KMS_KEY_ID,
|
||||
SSE_KMS_CONTEXT,
|
||||
SSE_C_ALGORITHM,
|
||||
SSE_C_KEY,
|
||||
SSE_C_KEY_MD5,
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let mut logs = String::new();
|
||||
for _ in 0..100 {
|
||||
logs = tokio::fs::read_to_string(&log_path).await?;
|
||||
if logs.contains(LOG_FLUSH_SENTINEL) {
|
||||
break;
|
||||
}
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
}
|
||||
assert!(logs.contains(LOG_FLUSH_SENTINEL), "timed out waiting for the log sink to flush");
|
||||
for secret in [customer_key, customer_key_b64.as_str(), wrong_key, wrong_key_b64.as_str()] {
|
||||
assert!(!logs.contains(secret), "Select request logging leaked SSE-C customer key material");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -61,9 +61,6 @@ mod get_codec_streaming_compat_test;
|
||||
#[cfg(test)]
|
||||
mod version_id_regression_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod select_request_root_alias_test;
|
||||
|
||||
// Pinned previous-release -> current-build on-disk compatibility.
|
||||
#[cfg(test)]
|
||||
mod upgrade_compatibility_test;
|
||||
@@ -167,10 +164,6 @@ mod delete_objects_versioning_test;
|
||||
#[cfg(test)]
|
||||
mod delete_object_no_content_length_test;
|
||||
|
||||
// Regression test for signed empty PutObject requests without Content-Length.
|
||||
#[cfg(test)]
|
||||
mod put_object_no_content_length_test;
|
||||
|
||||
// Delete-marker visibility baseline for data-movement migration proof.
|
||||
#[cfg(test)]
|
||||
mod delete_marker_migration_semantics_test;
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Regression coverage for anonymous access on multipart control APIs.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
|
||||
use async_compression::tokio::write::{BzEncoder, XzEncoder};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -23,10 +23,7 @@ use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
@@ -190,12 +187,6 @@ fn gzip_bytes(data: &[u8]) -> Vec<u8> {
|
||||
encoder.finish().expect("gzip encoder should finish")
|
||||
}
|
||||
|
||||
fn zlib_bytes(data: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(data).expect("zlib encoder should accept input");
|
||||
encoder.finish().expect("zlib encoder should finish")
|
||||
}
|
||||
|
||||
fn zstd_bytes(data: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = zstd::Encoder::new(Vec::new(), 0).expect("zstd encoder should initialize");
|
||||
encoder.write_all(data).expect("zstd encoder should accept input");
|
||||
@@ -218,45 +209,6 @@ async fn xz_bytes(data: &[u8]) -> Vec<u8> {
|
||||
encoder.into_inner().into_inner()
|
||||
}
|
||||
|
||||
async fn lz4_bytes(data: &[u8]) -> Vec<u8> {
|
||||
let cursor = Cursor::new(Vec::new());
|
||||
let mut encoder = Lz4Encoder::new(cursor);
|
||||
encoder.write_all(data).await.expect("LZ4 encoder should accept input");
|
||||
encoder.shutdown().await.expect("LZ4 encoder should finish");
|
||||
encoder.into_inner().into_inner()
|
||||
}
|
||||
|
||||
/// Encode the S2 framed stream shape emitted by minio-go PutObjectsSnowball
|
||||
/// with `Compress: true`: 1 MiB independent blocks, better compression,
|
||||
/// masked CRC-32C, and the `S2sTwO` stream identifier.
|
||||
fn minio_go_snowball_s2_bytes(data: &[u8]) -> Vec<u8> {
|
||||
const BLOCK_SIZE: usize = 1 << 20;
|
||||
const CHECKSUM_SIZE: usize = 4;
|
||||
|
||||
let mut output = b"\xff\x06\x00\x00S2sTwO".to_vec();
|
||||
let mut encoder = minlz::Encoder::new();
|
||||
for block in data.chunks(BLOCK_SIZE) {
|
||||
let compressed = encoder.encode_better(block);
|
||||
let compressed_limit = block.len().saturating_sub(block.len() / 32).saturating_sub(5);
|
||||
let (chunk_type, payload) = if compressed.len() <= compressed_limit {
|
||||
(0x00, compressed.as_slice())
|
||||
} else {
|
||||
(0x01, block)
|
||||
};
|
||||
let chunk_len = payload.len() + CHECKSUM_SIZE;
|
||||
assert!(chunk_len < 1 << 24, "S2 fixture chunk must fit the 24-bit frame length");
|
||||
output.extend_from_slice(&[
|
||||
chunk_type,
|
||||
(chunk_len & 0xff) as u8,
|
||||
((chunk_len >> 8) & 0xff) as u8,
|
||||
((chunk_len >> 16) & 0xff) as u8,
|
||||
]);
|
||||
output.extend_from_slice(&minlz::crc::crc(block).to_le_bytes());
|
||||
output.extend_from_slice(payload);
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, code: &str)
|
||||
where
|
||||
T: std::fmt::Debug,
|
||||
@@ -3504,62 +3456,6 @@ async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_ignore_dirs_skips_unauthorized_directory()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "signed-extract-ignore-dirs-auth";
|
||||
let archive_key = "bundle.tar";
|
||||
let allowed_member = "allowed/member.txt";
|
||||
let denied_directory = "denied/";
|
||||
let username = "snowball-ignore-dirs";
|
||||
let secret_key = "snowball-ignore-dirs-secret";
|
||||
let expected_body = b"allowed-body";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
create_restricted_user(&env, username, secret_key).await?;
|
||||
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [username] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{bucket}/{archive_key}"),
|
||||
format!("arn:aws:s3:::{bucket}/{allowed_member}")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
|
||||
|
||||
let restricted_client = restricted_user_client(&env, username, secret_key);
|
||||
let tar_bytes = make_tar(&[(allowed_member, expected_body)], &[denied_directory]).await;
|
||||
restricted_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.body(ByteStream::from(tar_bytes))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let stored = admin_client.get_object().bucket(bucket).key(allowed_member).send().await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -4289,60 +4185,6 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_expands_s2_and_lz4_by_magic_with_raw_etags()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "signed-extract-magic-codecs";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let s2_tar = make_tar(&[("s2/object.txt", b"s2-body")], &[]).await;
|
||||
let s2_archive = minio_go_snowball_s2_bytes(&s2_tar);
|
||||
let expected_s2_etag = format!("\"{}\"", md5_hex(&s2_archive));
|
||||
let s2_response = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
// minio-go intentionally uploads a compressed S2 stream with a .tar key.
|
||||
.key("snowball-upload-0123456789abcdef.tar")
|
||||
.body(ByteStream::from(s2_archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(s2_response.e_tag(), Some(expected_s2_etag.as_str()));
|
||||
|
||||
let s2_object = client.get_object().bucket(bucket).key("s2/object.txt").send().await?;
|
||||
assert_eq!(s2_object.body.collect().await?.into_bytes().as_ref(), b"s2-body");
|
||||
|
||||
let lz4_tar = make_tar(&[("lz4/object.txt", b"lz4-body")], &[]).await;
|
||||
let lz4_archive = lz4_bytes(&lz4_tar).await;
|
||||
let expected_lz4_etag = format!("\"{}\"", md5_hex(&lz4_archive));
|
||||
let lz4_response = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("also-looks-like-a-plain.tar")
|
||||
.body(ByteStream::from(lz4_archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(lz4_response.e_tag(), Some(expected_lz4_etag.as_str()));
|
||||
|
||||
let lz4_object = client.get_object().bucket(bucket).key("lz4/object.txt").send().await?;
|
||||
assert_eq!(lz4_object.body.collect().await?.into_bytes().as_ref(), b"lz4-body");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -4467,15 +4309,9 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
let context_archive_resources = [
|
||||
format!("arn:aws:s3:::{bucket}/tag-context.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/lock-context.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/legal-hold-context.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/user-agent-bypass.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/sse-bypass.tar"),
|
||||
];
|
||||
let tag_entry_resource = format!("arn:aws:s3:::{bucket}/tag-context-entry.txt");
|
||||
let lock_entry_resource = format!("arn:aws:s3:::{bucket}/lock-context-entry.txt");
|
||||
let legal_hold_entry_resource = format!("arn:aws:s3:::{bucket}/legal-hold-context-entry.txt");
|
||||
let user_agent_entry_resource = format!("arn:aws:s3:::{bucket}/user-agent-bypass-entry.txt");
|
||||
let sse_entry_resource = format!("arn:aws:s3:::{bucket}/sse-bypass-entry.txt");
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
@@ -4535,7 +4371,7 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
"Sid": "PaxContextArchives",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectLegalHold", "s3:PutObjectTagging"],
|
||||
"Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging"],
|
||||
"Resource": context_archive_resources
|
||||
},
|
||||
{
|
||||
@@ -4575,49 +4411,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObjectRetention"],
|
||||
"Resource": [lock_entry_resource]
|
||||
},
|
||||
{
|
||||
"Sid": "PaxLegalHoldContextPut",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [legal_hold_entry_resource.clone()]
|
||||
},
|
||||
{
|
||||
"Sid": "PaxLegalHoldContextAction",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObjectLegalHold"],
|
||||
"Resource": [legal_hold_entry_resource],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:object-lock-legal-hold": "OFF"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "MemberUserAgentCondition",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [user_agent_entry_resource],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"aws:UserAgent": "trusted"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "MemberSseCondition",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [sse_entry_resource],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:x-amz-server-side-encryption": "AES256"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -4630,13 +4423,8 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
let cases = [
|
||||
(
|
||||
"legal-hold.tar",
|
||||
put_only_client.clone(),
|
||||
HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]),
|
||||
),
|
||||
(
|
||||
"tagging.tar",
|
||||
put_only_client,
|
||||
HashMap::from([("minio.metadata.x-amz-tagging", "classification=restricted".to_string())]),
|
||||
HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]),
|
||||
),
|
||||
(
|
||||
"retention-condition.tar",
|
||||
@@ -4724,57 +4512,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body");
|
||||
|
||||
let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret);
|
||||
for (archive_key, entry_key, pax_key, injected_value, outer_user_agent) in [
|
||||
(
|
||||
"user-agent-bypass.tar",
|
||||
"user-agent-bypass-entry.txt",
|
||||
"minio.metadata.user-agent",
|
||||
"trusted",
|
||||
Some("untrusted"),
|
||||
),
|
||||
(
|
||||
"sse-bypass.tar",
|
||||
"sse-bypass-entry.txt",
|
||||
"minio.metadata.x-amz-server-side-encryption",
|
||||
"AES256",
|
||||
None,
|
||||
),
|
||||
] {
|
||||
let pax = HashMap::from([(pax_key, injected_value.to_string())]);
|
||||
let archive = make_tar_with_pax_entry(entry_key, b"must-not-write", None, &pax).await;
|
||||
let err = pax_context_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(move |req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
if let Some(user_agent) = outer_user_agent {
|
||||
req.headers_mut().insert("user-agent", user_agent);
|
||||
}
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("PAX metadata must not satisfy unrelated IAM request conditions");
|
||||
assert_eq!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("AccessDenied"),
|
||||
"{archive_key}"
|
||||
);
|
||||
let err = admin_client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(entry_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied PAX member must not be written");
|
||||
assert!(matches!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("NoSuchKey" | "NotFound")
|
||||
));
|
||||
}
|
||||
|
||||
let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]);
|
||||
let archive = make_tar_with_pax_entry("tag-context-entry.txt", b"tag-context-body", None, &tag_pax).await;
|
||||
pax_context_client
|
||||
@@ -4838,34 +4575,6 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
pax_retain_until
|
||||
);
|
||||
|
||||
let legal_hold_pax = HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]);
|
||||
let archive = make_tar_with_pax_entry("legal-hold-context-entry.txt", b"must-not-write", None, &legal_hold_pax).await;
|
||||
let err = pax_context_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("legal-hold-context.tar")
|
||||
.object_lock_legal_hold_status(aws_sdk_s3::types::ObjectLockLegalHoldStatus::Off)
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("PAX legal hold must replace the outer value in the member IAM condition context");
|
||||
assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied"));
|
||||
let err = admin_client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key("legal-hold-context-entry.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied PAX legal-hold member must not be written");
|
||||
assert!(matches!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("NoSuchKey" | "NotFound")
|
||||
));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -5341,8 +5050,8 @@ async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting_extension()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
@@ -5355,7 +5064,8 @@ async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let tar_bytes = make_tar(&[("plain.txt", b"plain-body")], &[]).await;
|
||||
admin_client
|
||||
|
||||
let result = admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
@@ -5365,80 +5075,15 @@ async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
.await;
|
||||
|
||||
let plain = admin_client.get_object().bucket(bucket).key("plain.txt").send().await?;
|
||||
assert_eq!(plain.body.collect().await?.into_bytes().as_ref(), b"plain-body");
|
||||
|
||||
let raw_with_gzip_suffix = make_tar(&[("raw-with-wrong-suffix.txt", b"raw-body")], &[]).await;
|
||||
admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("raw-but-named.tar.gz")
|
||||
.body(ByteStream::from(raw_with_gzip_suffix))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let raw = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("raw-with-wrong-suffix.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(raw.body.collect().await?.into_bytes().as_ref(), b"raw-body");
|
||||
|
||||
let gzip_with_tar_suffix = gzip_bytes(&make_tar(&[("gzip-with-wrong-suffix.txt", b"gzip-body")], &[]).await);
|
||||
admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("gzip-but-named.tar")
|
||||
.body(ByteStream::from(gzip_with_tar_suffix))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let gzip = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("gzip-with-wrong-suffix.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(gzip.body.collect().await?.into_bytes().as_ref(), b"gzip-body");
|
||||
|
||||
let zlib_archive = zlib_bytes(&make_tar(&[("zlib-extension.txt", b"zlib-body")], &[]).await);
|
||||
admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("bundle.zlib")
|
||||
.body(ByteStream::from(zlib_archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let zlib = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("zlib-extension.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(zlib.body.collect().await?.into_bytes().as_ref(), b"zlib-body");
|
||||
assert_s3_error_code(result, "InvalidArgument");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_rejects_invalid_archive_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
async fn test_signed_put_object_extract_rejects_invalid_tar_gz_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
|
||||
@@ -36,10 +36,9 @@
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use rustfs_signer::constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER};
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key};
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Cursor;
|
||||
use time::macros::format_description;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use tracing::info;
|
||||
@@ -99,37 +98,15 @@ impl SigV4 {
|
||||
/// header AND folded into the canonical request — pass the hash of the
|
||||
/// body you *claim* to send, which may differ from what you actually send.
|
||||
fn sign(&self, method: &str, path: &str, canonical_query: &str, content_sha256: &str) -> SignedHeaders {
|
||||
self.sign_with_extra_headers(method, path, canonical_query, content_sha256, &[])
|
||||
}
|
||||
|
||||
/// Sign additional request headers while preserving SigV4's lowercase,
|
||||
/// lexicographically sorted canonical-header representation.
|
||||
fn sign_with_extra_headers(
|
||||
&self,
|
||||
method: &str,
|
||||
path: &str,
|
||||
canonical_query: &str,
|
||||
content_sha256: &str,
|
||||
extra_signed_headers: &[(&str, &str)],
|
||||
) -> SignedHeaders {
|
||||
let amz_date = amz_datetime(self.time);
|
||||
let mut canonical_header_values = vec![
|
||||
("host", self.host.as_str()),
|
||||
("x-amz-content-sha256", content_sha256),
|
||||
("x-amz-date", amz_date.as_str()),
|
||||
];
|
||||
canonical_header_values.extend(extra_signed_headers.iter().copied());
|
||||
canonical_header_values.sort_unstable_by(|left, right| left.0.cmp(right.0));
|
||||
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
|
||||
|
||||
let signed_headers = canonical_header_values
|
||||
.iter()
|
||||
.map(|(name, _)| *name)
|
||||
.collect::<Vec<_>>()
|
||||
.join(";");
|
||||
let mut canonical_headers = String::new();
|
||||
for (name, value) in canonical_header_values {
|
||||
let _ = writeln!(canonical_headers, "{name}:{value}");
|
||||
}
|
||||
let canonical_headers = format!(
|
||||
"host:{host}\nx-amz-content-sha256:{sha}\nx-amz-date:{date}\n",
|
||||
host = self.host,
|
||||
sha = content_sha256,
|
||||
date = amz_date,
|
||||
);
|
||||
let canonical_request =
|
||||
format!("{method}\n{path}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{content_sha256}");
|
||||
|
||||
@@ -202,34 +179,6 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_single_member_archive(
|
||||
member_key: &str,
|
||||
member_body: &[u8],
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(member_body.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, member_key, Cursor::new(member_body)).await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn sha256_base64(data: &[u8]) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
base64_simd::STANDARD.encode_to_string(Sha256::digest(data))
|
||||
}
|
||||
|
||||
fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
|
||||
let checksum = sha256_base64(decoded);
|
||||
let mut encoded = format!("{:x}\r\n", decoded.len()).into_bytes();
|
||||
encoded.extend_from_slice(decoded);
|
||||
encoded.extend_from_slice(b"\r\n0\r\n\r\n");
|
||||
encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}").as_bytes());
|
||||
encoded
|
||||
}
|
||||
|
||||
/// Positive control: a correctly hand-signed request must succeed. Without
|
||||
/// this, every negative assertion below could pass for the wrong reason (a
|
||||
/// broken signer that never produces a valid signature).
|
||||
@@ -300,128 +249,6 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `STREAMING-UNSIGNED-PAYLOAD-TRAILER` disables per-chunk signatures, not the
|
||||
/// seed/header SigV4 signature. A forged request must be rejected before the
|
||||
/// Snowball handler can publish any archive member.
|
||||
#[tokio::test]
|
||||
async fn snowball_streaming_unsigned_trailer_rejects_forged_signature() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
setup(&mut env).await?;
|
||||
|
||||
let archive_key = "forged-streaming-snowball.tar";
|
||||
let member_key = "must-not-be-published.txt";
|
||||
let archive = build_single_member_archive(member_key, b"forged request payload").await?;
|
||||
let decoded_content_length = archive.len().to_string();
|
||||
let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
|
||||
let path = format!("/{BUCKET}/{archive_key}");
|
||||
|
||||
let mut signer = SigV4::new(&env);
|
||||
signer.secret_key = "wrong-secret-for-forged-streaming-request".to_string();
|
||||
let extra_signed_headers = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", decoded_content_length.as_str()),
|
||||
("x-amz-meta-snowball-auto-extract", "true"),
|
||||
("x-amz-trailer", "x-amz-checksum-sha256"),
|
||||
];
|
||||
let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
|
||||
|
||||
let response = local_http_client()
|
||||
.put(format!("{}{}", env.url, path))
|
||||
.header("authorization", &headers.authorization)
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("x-amz-content-sha256", &headers.content_sha256)
|
||||
.header("x-amz-date", &headers.amz_date)
|
||||
.header("x-amz-decoded-content-length", &decoded_content_length)
|
||||
.header("x-amz-meta-snowball-auto-extract", "true")
|
||||
.header("x-amz-trailer", "x-amz-checksum-sha256")
|
||||
.body(encoded_body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
assert_eq!(status.as_u16(), 403, "forged streaming signature must be 403, body:\n{body}");
|
||||
assert_error_code(&body, "SignatureDoesNotMatch");
|
||||
|
||||
let absent = env
|
||||
.create_s3_client()
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a forged streaming request must not publish a Snowball member");
|
||||
assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Snowball must consume the complete aws-chunked body before reading the
|
||||
/// trailing checksum exported by s3s into the PutObject response.
|
||||
#[tokio::test]
|
||||
async fn snowball_streaming_unsigned_trailer_returns_sha256_checksum() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
setup(&mut env).await?;
|
||||
|
||||
let archive_key = "valid-streaming-snowball.tar";
|
||||
let member_key = "streaming-checksum-member.txt";
|
||||
let member_body = b"valid streaming Snowball payload";
|
||||
let archive = build_single_member_archive(member_key, member_body).await?;
|
||||
let expected_checksum = sha256_base64(&archive);
|
||||
let decoded_content_length = archive.len().to_string();
|
||||
let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
|
||||
let path = format!("/{BUCKET}/{archive_key}");
|
||||
|
||||
let signer = SigV4::new(&env);
|
||||
let extra_signed_headers = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", decoded_content_length.as_str()),
|
||||
("x-amz-meta-snowball-auto-extract", "true"),
|
||||
("x-amz-sdk-checksum-algorithm", "SHA256"),
|
||||
("x-amz-trailer", "x-amz-checksum-sha256"),
|
||||
];
|
||||
let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
|
||||
|
||||
let response = local_http_client()
|
||||
.put(format!("{}{}", env.url, path))
|
||||
.header("authorization", &headers.authorization)
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("x-amz-content-sha256", &headers.content_sha256)
|
||||
.header("x-amz-date", &headers.amz_date)
|
||||
.header("x-amz-decoded-content-length", &decoded_content_length)
|
||||
.header("x-amz-meta-snowball-auto-extract", "true")
|
||||
.header("x-amz-sdk-checksum-algorithm", "SHA256")
|
||||
.header("x-amz-trailer", "x-amz-checksum-sha256")
|
||||
.body(encoded_body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let response_checksum = response
|
||||
.headers()
|
||||
.get("x-amz-checksum-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let response_body = response.text().await?;
|
||||
assert_eq!(status.as_u16(), 200, "valid streaming Snowball PUT failed, body:\n{response_body}");
|
||||
assert_eq!(response_checksum.as_deref(), Some(expected_checksum.as_str()));
|
||||
|
||||
let member = env
|
||||
.create_s3_client()
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await?;
|
||||
let stored = member.body.collect().await?.into_bytes();
|
||||
assert_eq!(stored.as_ref(), member_body);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// (b) A valid AccessKeyId paired with the wrong secret key must be rejected
|
||||
/// with SignatureDoesNotMatch / 403.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -38,10 +38,14 @@ use aws_sdk_s3::types::{
|
||||
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use local_ip_address::local_ip;
|
||||
use reqwest::StatusCode;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
|
||||
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
|
||||
use s3s::Body;
|
||||
use serde_json::Value;
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
@@ -411,16 +415,42 @@ async fn collect_until(
|
||||
// Admin target configuration (signed admin HTTP)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Thin wrapper over [`crate::common::signed_request`] with this suite's
|
||||
/// root credentials; a `Some` body is always JSON here.
|
||||
async fn signed_admin_request(
|
||||
env: &RustFSTestEnvironment,
|
||||
method: http::Method,
|
||||
url: &str,
|
||||
body: Option<Vec<u8>>,
|
||||
) -> Result<reqwest::Response, BoxError> {
|
||||
let content_type = body.is_some().then_some("application/json");
|
||||
crate::common::signed_request(method, url, &env.access_key, &env.secret_key, body, content_type).await
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("admin URL missing authority")?.to_string();
|
||||
let mut builder = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if body.is_some() {
|
||||
builder = builder.header(CONTENT_TYPE, "application/json");
|
||||
}
|
||||
|
||||
let content_len = body.as_ref().map(|b| b.len() as i64).unwrap_or_default();
|
||||
let signed = sign_v4(
|
||||
builder.body(Body::empty())?,
|
||||
content_len,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
"",
|
||||
"us-east-1",
|
||||
);
|
||||
|
||||
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
|
||||
let mut request = crate::common::local_http_client().request(reqwest_method, url);
|
||||
for (name, value) in signed.headers() {
|
||||
request = request.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request = request.body(body);
|
||||
}
|
||||
Ok(request.send().await?)
|
||||
}
|
||||
|
||||
async fn enable_notify_module(env: &RustFSTestEnvironment) -> TestResult {
|
||||
|
||||
@@ -15,24 +15,28 @@
|
||||
//! Tests for AWS IAM policy variables with single-value, multi-value, and nested scenarios
|
||||
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via,
|
||||
awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
|
||||
RustFSTestEnvironment, awscurl_delete, awscurl_put, build_test_s3_config, build_test_sts_client, init_logging,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use tracing::info;
|
||||
|
||||
/// Helper function to create a regular user with given credentials.
|
||||
///
|
||||
/// This suite deliberately drives the admin API through the external `awscurl`
|
||||
/// binary, so the shared helpers are pinned to `AdminTransport::Awscurl`.
|
||||
/// Helper function to create a regular user with given credentials
|
||||
async fn create_user(
|
||||
env: &RustFSTestEnvironment,
|
||||
username: &str,
|
||||
password: &str,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
admin_create_user_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, username, password).await
|
||||
let create_user_body = serde_json::json!({
|
||||
"secretKey": password,
|
||||
"status": "enabled"
|
||||
})
|
||||
.to_string();
|
||||
|
||||
let create_user_url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", env.url, username);
|
||||
awscurl_put(&create_user_url, &create_user_body, &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper function to create and attach a policy
|
||||
@@ -42,17 +46,18 @@ async fn create_and_attach_policy(
|
||||
username: &str,
|
||||
policy_document: serde_json::Value,
|
||||
) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Awscurl,
|
||||
&env.url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
policy_name,
|
||||
&policy_document.to_string(),
|
||||
)
|
||||
.await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Awscurl, &env.url, &env.access_key, &env.secret_key, policy_name, username)
|
||||
.await?;
|
||||
let policy_string = policy_document.to_string();
|
||||
|
||||
// Create policy
|
||||
let add_policy_url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
|
||||
awscurl_put(&add_policy_url, &policy_string, &env.access_key, &env.secret_key).await?;
|
||||
|
||||
// Attach policy to user
|
||||
let attach_policy_url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
env.url, policy_name, username
|
||||
);
|
||||
awscurl_put(&attach_policy_url, "", &env.access_key, &env.secret_key).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -31,11 +31,15 @@
|
||||
//!
|
||||
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-3p3x-734c-h5vx>
|
||||
|
||||
use crate::common::local_http_client;
|
||||
use crate::common::rustfs_binary_path_with_features;
|
||||
use crate::common::{AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via};
|
||||
use crate::protocols::test_env::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, ProtocolTestEnvironment};
|
||||
use anyhow::Result;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use reqwest::Client;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use tokio::process::Command;
|
||||
use tracing::info;
|
||||
|
||||
@@ -63,43 +67,92 @@ fn basic_auth_header_for(access_key: &str, secret_key: &str) -> String {
|
||||
format!("Basic {}", encoded)
|
||||
}
|
||||
|
||||
async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
|
||||
admin_create_user_via(
|
||||
AdminTransport::Signed,
|
||||
base_url,
|
||||
async fn signed_admin_request(
|
||||
method: http::Method,
|
||||
url: &str,
|
||||
body: Option<Vec<u8>>,
|
||||
content_type: Option<&str>,
|
||||
) -> Result<reqwest::Response> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri
|
||||
.authority()
|
||||
.ok_or_else(|| anyhow::anyhow!("request URL missing authority"))?
|
||||
.to_string();
|
||||
let mut request = http::Request::builder().method(method.clone()).uri(uri);
|
||||
request = request.header(HOST, authority);
|
||||
request = request.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
if let Some(content_type) = content_type {
|
||||
request = request.header(CONTENT_TYPE, content_type);
|
||||
}
|
||||
|
||||
let content_len = body.as_ref().map(|body| body.len() as i64).unwrap_or_default();
|
||||
let signed = sign_v4(
|
||||
request.body(Body::empty())?,
|
||||
content_len,
|
||||
DEFAULT_ACCESS_KEY,
|
||||
DEFAULT_SECRET_KEY,
|
||||
username,
|
||||
secret_key,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
"",
|
||||
"us-east-1",
|
||||
);
|
||||
|
||||
let reqwest_method = reqwest::Method::from_bytes(method.as_str().as_bytes())?;
|
||||
let mut request_builder = local_http_client().request(reqwest_method, url);
|
||||
for (name, value) in signed.headers() {
|
||||
request_builder = request_builder.header(name, value);
|
||||
}
|
||||
if let Some(body) = body {
|
||||
request_builder = request_builder.body(body);
|
||||
}
|
||||
|
||||
Ok(request_builder.send().await?)
|
||||
}
|
||||
|
||||
async fn admin_create_user(base_url: &str, username: &str, secret_key: &str) -> Result<()> {
|
||||
let url = format!("{}/rustfs/admin/v3/add-user?accessKey={}", base_url, username);
|
||||
let body = serde_json::json!({
|
||||
"secretKey": secret_key,
|
||||
"status": "enabled"
|
||||
});
|
||||
let response =
|
||||
signed_admin_request(http::Method::PUT, &url, Some(body.to_string().into_bytes()), Some("application/json")).await?;
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
anyhow::bail!("create user failed: {status} {body}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admin_add_canned_policy(base_url: &str, policy_name: &str, policy: &serde_json::Value) -> Result<()> {
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Signed,
|
||||
base_url,
|
||||
DEFAULT_ACCESS_KEY,
|
||||
DEFAULT_SECRET_KEY,
|
||||
policy_name,
|
||||
&policy.to_string(),
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", base_url, policy_name);
|
||||
let response =
|
||||
signed_admin_request(http::Method::PUT, &url, Some(policy.to_string().into_bytes()), Some("application/json")).await?;
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
anyhow::bail!("add canned policy failed: {status} {body}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admin_attach_policy_to_user(base_url: &str, policy_name: &str, username: &str) -> Result<()> {
|
||||
admin_attach_user_policy_via(
|
||||
AdminTransport::Signed,
|
||||
base_url,
|
||||
DEFAULT_ACCESS_KEY,
|
||||
DEFAULT_SECRET_KEY,
|
||||
policy_name,
|
||||
username,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| anyhow::anyhow!(e))
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
base_url, policy_name, username
|
||||
);
|
||||
let response = signed_admin_request(http::Method::PUT, &url, Some(Vec::new()), None).await?;
|
||||
|
||||
if response.status() != reqwest::StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
anyhow::bail!("attach policy failed: {status} {body}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Test WebDAV: MKCOL (create bucket), PUT, GET, DELETE, PROPFIND operations
|
||||
|
||||
@@ -1,152 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Regression coverage for rustfs#6830: a signed empty `PutObject` request
|
||||
//! without `Content-Length` and without `Transfer-Encoding` is still a
|
||||
//! zero-length object upload.
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use http::header::{CONTENT_LENGTH, HOST, TRANSFER_ENCODING};
|
||||
use rustfs_signer::sign_v4;
|
||||
use rustfs_utils::hash::EMPTY_STRING_SHA256_HASH;
|
||||
use s3s::Body;
|
||||
use std::error::Error;
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::time::{Duration, timeout};
|
||||
use tracing::info;
|
||||
|
||||
const RAW_RESPONSE_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
|
||||
fn parse_status(raw_response: &str) -> Option<u16> {
|
||||
raw_response.lines().next()?.split_whitespace().nth(1)?.parse().ok()
|
||||
}
|
||||
|
||||
async fn send_raw_signed_put(
|
||||
url: &str,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
transfer_encoding: Option<&str>,
|
||||
raw_body: &[u8],
|
||||
) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
|
||||
let path_and_query = uri.path_and_query().ok_or("request URL missing path")?.as_str().to_string();
|
||||
|
||||
let mut request = http::Request::builder()
|
||||
.method(http::Method::PUT)
|
||||
.uri(uri)
|
||||
.header(HOST, authority.clone())
|
||||
.header("x-amz-content-sha256", EMPTY_STRING_SHA256_HASH);
|
||||
if let Some(value) = transfer_encoding {
|
||||
request = request.header(TRANSFER_ENCODING, value);
|
||||
}
|
||||
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let mut raw_request = format!("PUT {path_and_query} HTTP/1.1\r\nHost: {authority}\r\nConnection: close\r\n");
|
||||
for (name, value) in signed.headers() {
|
||||
if name == HOST || name == CONTENT_LENGTH {
|
||||
continue;
|
||||
}
|
||||
raw_request.push_str(name.as_str());
|
||||
raw_request.push_str(": ");
|
||||
raw_request.push_str(value.to_str()?);
|
||||
raw_request.push_str("\r\n");
|
||||
}
|
||||
raw_request.push_str("\r\n");
|
||||
|
||||
assert!(
|
||||
!raw_request.to_ascii_lowercase().contains("\r\ncontent-length:"),
|
||||
"raw regression request must omit Content-Length; request was:\n{raw_request}"
|
||||
);
|
||||
|
||||
let mut stream = TcpStream::connect(&authority).await?;
|
||||
stream.write_all(raw_request.as_bytes()).await?;
|
||||
stream.write_all(raw_body).await?;
|
||||
stream.flush().await?;
|
||||
|
||||
let mut response = Vec::new();
|
||||
timeout(RAW_RESPONSE_TIMEOUT, stream.read_to_end(&mut response))
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "timed out reading raw PUT response"))??;
|
||||
Ok(String::from_utf8_lossy(&response).into_owned())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_put_object_without_content_length_boundaries() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
info!("TEST: PutObject without Content-Length boundaries");
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let empty_bucket = "put-no-content-length";
|
||||
let empty_key = "empty.bin";
|
||||
let chunked_bucket = "put-chunked-no-length";
|
||||
let chunked_key = "chunked.bin";
|
||||
|
||||
client.create_bucket().bucket(empty_bucket).send().await?;
|
||||
client.create_bucket().bucket(chunked_bucket).send().await?;
|
||||
|
||||
let url = format!("{}/{}/{}", env.url, empty_bucket, empty_key);
|
||||
let raw_response = send_raw_signed_put(&url, &env.access_key, &env.secret_key, None, b"").await?;
|
||||
info!("raw empty PUT response:\n{}", raw_response);
|
||||
|
||||
assert_eq!(
|
||||
parse_status(&raw_response),
|
||||
Some(200),
|
||||
"empty PutObject without Content-Length should succeed, got:\n{raw_response}"
|
||||
);
|
||||
assert!(
|
||||
raw_response.to_ascii_lowercase().contains("\r\netag:"),
|
||||
"successful PutObject should return an ETag header: {raw_response}"
|
||||
);
|
||||
|
||||
let head = client.head_object().bucket(empty_bucket).key(empty_key).send().await?;
|
||||
assert_eq!(head.content_length(), Some(0), "stored object must be zero length");
|
||||
|
||||
let url = format!("{}/{}/{}", env.url, chunked_bucket, chunked_key);
|
||||
let raw_response = send_raw_signed_put(&url, &env.access_key, &env.secret_key, Some("chunked"), b"0\r\n\r\n").await?;
|
||||
info!("raw chunked PUT response:\n{}", raw_response);
|
||||
|
||||
assert_eq!(
|
||||
parse_status(&raw_response),
|
||||
Some(411),
|
||||
"unknown-length chunked PutObject must stay rejected, got:\n{raw_response}"
|
||||
);
|
||||
assert!(
|
||||
raw_response.contains("<Code>MissingContentLength</Code>"),
|
||||
"expected MissingContentLength, got:\n{raw_response}"
|
||||
);
|
||||
|
||||
let missing = client
|
||||
.head_object()
|
||||
.bucket(chunked_bucket)
|
||||
.key(chunked_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected unknown-length PUT must not create an object");
|
||||
assert_eq!(
|
||||
missing.raw_response().map(|response| response.status().as_u16()),
|
||||
Some(404),
|
||||
"rejected unknown-length PUT absence probe must return HTTP 404, got {missing:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -21,6 +21,5 @@ mod head_tls_bodyless_test;
|
||||
mod lifecycle;
|
||||
mod lock;
|
||||
mod node_interact_test;
|
||||
mod s3_select_compression;
|
||||
mod sql;
|
||||
mod tiering;
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
#![cfg(test)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use async_compression::tokio::write::BzEncoder;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::ProvideErrorMetadata,
|
||||
operation::select_object_content::{SelectObjectContentOutput, builders::SelectObjectContentFluentBuilder},
|
||||
types::{
|
||||
CompressionType, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput,
|
||||
JsonType, OutputSerialization, SelectObjectContentEventStream,
|
||||
},
|
||||
};
|
||||
use aws_smithy_types::event_stream::RawMessage;
|
||||
use bytes::Bytes;
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use std::{error::Error, io::Cursor, time::Duration};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
const BUCKET: &str = "s3-select-compression";
|
||||
const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
|
||||
async fn create_test_environment(extra_env: &[(&str, &str)]) -> TestResult<(RustFSTestEnvironment, Client)> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], extra_env).await?;
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(BUCKET).send().await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
async fn put_object(client: &Client, key: &str, body: &[u8]) -> TestResult<()> {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn gzip(input: &[u8]) -> TestResult<Vec<u8>> {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
std::io::Write::write_all(&mut encoder, input)?;
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
|
||||
async fn bzip2(input: &[u8]) -> TestResult<Vec<u8>> {
|
||||
let mut encoder = BzEncoder::new(Cursor::new(Vec::new()));
|
||||
encoder.write_all(input).await?;
|
||||
encoder.shutdown().await?;
|
||||
Ok(encoder.into_inner().into_inner())
|
||||
}
|
||||
|
||||
fn csv_select_request(
|
||||
client: &Client,
|
||||
key: &str,
|
||||
compression: CompressionType,
|
||||
expression: &str,
|
||||
) -> SelectObjectContentFluentBuilder {
|
||||
client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression(expression)
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.compression_type(compression)
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
|
||||
}
|
||||
|
||||
fn json_select_request(
|
||||
client: &Client,
|
||||
key: &str,
|
||||
compression: CompressionType,
|
||||
json_type: JsonType,
|
||||
) -> SelectObjectContentFluentBuilder {
|
||||
client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression("SELECT name FROM S3Object")
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.compression_type(compression)
|
||||
.json(JsonInput::builder().set_type(Some(json_type)).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
|
||||
}
|
||||
|
||||
async fn collect_success(
|
||||
mut response: SelectObjectContentOutput,
|
||||
compressed_bytes: usize,
|
||||
processed_bytes: usize,
|
||||
) -> TestResult<Vec<u8>> {
|
||||
tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
|
||||
let mut records = Vec::new();
|
||||
let mut stats = None;
|
||||
let mut saw_end = false;
|
||||
|
||||
while let Some(event) = response.payload.recv().await? {
|
||||
assert!(!saw_end, "Select emitted an event after End");
|
||||
match event {
|
||||
SelectObjectContentEventStream::Records(event) => {
|
||||
assert!(stats.is_none(), "Select emitted Records after Stats");
|
||||
if let Some(payload) = event.payload {
|
||||
records.extend_from_slice(payload.as_ref());
|
||||
}
|
||||
}
|
||||
SelectObjectContentEventStream::Stats(event) => {
|
||||
assert!(stats.is_none(), "Select emitted more than one Stats event");
|
||||
stats = event.details;
|
||||
}
|
||||
SelectObjectContentEventStream::End(_) => {
|
||||
assert!(stats.is_some(), "Select emitted End before Stats");
|
||||
saw_end = true;
|
||||
}
|
||||
_ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
|
||||
}
|
||||
}
|
||||
|
||||
let stats = stats.ok_or("Select response ended without a Stats event")?;
|
||||
assert_eq!(stats.bytes_scanned(), Some(i64::try_from(compressed_bytes)?));
|
||||
assert_eq!(stats.bytes_processed(), Some(i64::try_from(processed_bytes)?));
|
||||
assert_eq!(stats.bytes_returned(), Some(i64::try_from(records.len())?));
|
||||
assert!(saw_end, "Select response ended without an End event");
|
||||
Ok::<_, Box<dyn Error + Send + Sync>>(records)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
|
||||
}
|
||||
|
||||
async fn assert_truncated_stream_failure(mut response: SelectObjectContentOutput) -> TestResult<()> {
|
||||
tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
|
||||
loop {
|
||||
match response.payload.recv().await {
|
||||
Err(error) => {
|
||||
// S3 Select request-level errors use `error` frames, which this SDK version exposes as raw response errors.
|
||||
if let Some(code) = error.code() {
|
||||
assert_eq!(code, "TruncatedInput", "unexpected modeled event-stream error: {error:?}");
|
||||
} else if let aws_sdk_s3::error::SdkError::ResponseError(context) = &error
|
||||
&& let RawMessage::Decoded(message) = context.raw()
|
||||
{
|
||||
let header = |name: &str| {
|
||||
message
|
||||
.headers()
|
||||
.iter()
|
||||
.find(|header| header.name().as_str() == name)
|
||||
.and_then(|header| header.value().as_string().ok())
|
||||
.map(|value| value.as_str())
|
||||
};
|
||||
assert_eq!(header(":message-type"), Some("error"));
|
||||
assert_eq!(header(":error-code"), Some("TruncatedInput"));
|
||||
} else {
|
||||
panic!("unexpected event-stream error: {error:?}");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Ok(Some(SelectObjectContentEventStream::Stats(_))) | Ok(Some(SelectObjectContentEventStream::End(_))) => {
|
||||
return Err("truncated compressed input reached a success terminal event".into());
|
||||
}
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => return Err("truncated compressed input ended without an error event".into()),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "truncated Select response timed out".into() })?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_compressed_csv_and_json() -> TestResult<()> {
|
||||
const CSV: &[u8] = b"name,age\nAlice,30\nBob,25\n";
|
||||
const JSON_LINES: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
|
||||
const JSON_DOCUMENT: &[u8] = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
|
||||
|
||||
let (_env, client) = create_test_environment(&[]).await?;
|
||||
|
||||
let gzip_csv = gzip(CSV)?;
|
||||
put_object(&client, "records.csv.gz", &gzip_csv).await?;
|
||||
let gzip_csv_records = collect_success(
|
||||
csv_select_request(&client, "records.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?,
|
||||
gzip_csv.len(),
|
||||
CSV.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(gzip_csv_records, b"Alice,30\nBob,25\n");
|
||||
|
||||
let bzip_csv = bzip2(CSV).await?;
|
||||
put_object(&client, "records.csv.bz2", &bzip_csv).await?;
|
||||
let bzip_csv_records = collect_success(
|
||||
csv_select_request(&client, "records.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?,
|
||||
bzip_csv.len(),
|
||||
CSV.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(bzip_csv_records, gzip_csv_records);
|
||||
|
||||
let gzip_json_lines = gzip(JSON_LINES)?;
|
||||
put_object(&client, "json-lines", &gzip_json_lines).await?;
|
||||
let gzip_json_records = collect_success(
|
||||
json_select_request(&client, "json-lines", CompressionType::Gzip, JsonType::Lines)
|
||||
.send()
|
||||
.await?,
|
||||
gzip_json_lines.len(),
|
||||
JSON_LINES.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(gzip_json_records, JSON_LINES);
|
||||
|
||||
let bzip_json_lines = bzip2(JSON_LINES).await?;
|
||||
put_object(&client, "records.jsonl.bz2", &bzip_json_lines).await?;
|
||||
let bzip_json_records = collect_success(
|
||||
json_select_request(&client, "records.jsonl.bz2", CompressionType::Bzip2, JsonType::Lines)
|
||||
.send()
|
||||
.await?,
|
||||
bzip_json_lines.len(),
|
||||
JSON_LINES.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(bzip_json_records, gzip_json_records);
|
||||
|
||||
let gzip_json_document = gzip(JSON_DOCUMENT)?;
|
||||
put_object(&client, "document.json.gz", &gzip_json_document).await?;
|
||||
let document_records = collect_success(
|
||||
json_select_request(&client, "document.json.gz", CompressionType::Gzip, JsonType::Document)
|
||||
.send()
|
||||
.await?,
|
||||
gzip_json_document.len(),
|
||||
JSON_DOCUMENT.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(document_records, JSON_LINES);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_invalid_compressed_stream_fails() -> TestResult<()> {
|
||||
const CSV: &[u8] = b"name\nAlice\n";
|
||||
|
||||
let (_env, client) = create_test_environment(&[]).await?;
|
||||
|
||||
put_object(&client, "invalid.csv.gz", CSV).await?;
|
||||
let invalid = csv_select_request(&client, "invalid.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("invalid GZIP header must fail before streaming");
|
||||
assert_eq!(
|
||||
invalid.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("InvalidCompressionFormat")
|
||||
);
|
||||
|
||||
put_object(&client, "empty.csv.gz", b"").await?;
|
||||
let empty = csv_select_request(&client, "empty.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("empty GZIP input must fail as truncated");
|
||||
assert_eq!(empty.as_service_error().and_then(ProvideErrorMetadata::code), Some("TruncatedInput"));
|
||||
|
||||
let mut truncated = bzip2(CSV).await?;
|
||||
truncated.pop();
|
||||
put_object(&client, "truncated.csv.bz2", &truncated).await?;
|
||||
let truncated = csv_select_request(&client, "truncated.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?;
|
||||
assert_truncated_stream_failure(truncated).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_compressed_disconnect_releases_query() -> TestResult<()> {
|
||||
const OBJECT: &str = "disconnect.csv.gz";
|
||||
const ROWS: usize = 16 * 1024;
|
||||
const RELEASE_ATTEMPTS: usize = 20;
|
||||
const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
|
||||
|
||||
let (_env, client) = create_test_environment(&[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")]).await?;
|
||||
let row = format!("{}\n", "x".repeat(1023));
|
||||
let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
|
||||
body.extend_from_slice(b"value\n");
|
||||
for _ in 0..ROWS {
|
||||
body.extend_from_slice(row.as_bytes());
|
||||
}
|
||||
let compressed = gzip(&body)?;
|
||||
put_object(&client, OBJECT, &compressed).await?;
|
||||
|
||||
let first = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?;
|
||||
let saturated = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the unread compressed response should retain the only query permit");
|
||||
assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
|
||||
|
||||
drop(first);
|
||||
let second = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
for attempt in 0..RELEASE_ATTEMPTS {
|
||||
match csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
|
||||
Err(error)
|
||||
if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown")
|
||||
&& attempt + 1 < RELEASE_ATTEMPTS =>
|
||||
{
|
||||
tokio::time::sleep(RELEASE_BACKOFF).await;
|
||||
}
|
||||
Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
|
||||
return Err("disconnected compressed Select retained its query permit".into());
|
||||
}
|
||||
Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
|
||||
}
|
||||
}
|
||||
Err("query permit release retry loop ended unexpectedly".into())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "compressed Select did not release its query permit".into() })??;
|
||||
drop(second);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -17,8 +17,7 @@ use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType,
|
||||
OutputSerialization, RequestProgress,
|
||||
CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use std::error::Error;
|
||||
@@ -27,9 +26,6 @@ use std::time::Duration;
|
||||
const BUCKET: &str = "test-sql-bucket";
|
||||
const CSV_OBJECT: &str = "test-data.csv";
|
||||
const JSON_OBJECT: &str = "test-data.json";
|
||||
const JSON_DOCUMENT_OBJECT: &str = "nested-data.json";
|
||||
const JSON_ROOT_ARRAY_OBJECT: &str = "root-array.json";
|
||||
const JSON_ROOT_SCALAR_ARRAY_OBJECT: &str = "root-scalars.json";
|
||||
const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
@@ -77,69 +73,6 @@ async fn upload_test_json(client: &Client) -> TestResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upload_nested_json_document(client: &Client) -> TestResult<()> {
|
||||
let json_data = r#"{"departments":[{"employees":[{"name":"Alice","active":true},{"name":"Bob","active":false}]},{"employees":[{"name":"Charlie","active":true}]}]}"#;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(JSON_DOCUMENT_OBJECT)
|
||||
.body(Bytes::from_static(json_data.as_bytes()).into())
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(JSON_ROOT_ARRAY_OBJECT)
|
||||
.body(Bytes::from_static(br#"[{"name":"Alice"},{"name":"Bob"}]"#).into())
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(JSON_ROOT_SCALAR_ARRAY_OBJECT)
|
||||
.body(Bytes::from_static(b"[1,2]").into())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn select_json_document(client: &Client, key: &str, expression: &str) -> TestResult<String> {
|
||||
let response = client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression(expression)
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
|
||||
.send()
|
||||
.await?;
|
||||
process_select_response(response).await
|
||||
}
|
||||
|
||||
fn csv_select_request(
|
||||
client: &Client,
|
||||
key: &str,
|
||||
) -> aws_sdk_s3::operation::select_object_content::builders::SelectObjectContentFluentBuilder {
|
||||
client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression("SELECT * FROM S3Object")
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
|
||||
}
|
||||
|
||||
async fn process_select_response(
|
||||
mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput,
|
||||
) -> TestResult<String> {
|
||||
@@ -171,209 +104,6 @@ async fn process_select_response(
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
|
||||
}
|
||||
|
||||
async fn assert_input_byte_stats(
|
||||
client: &Client,
|
||||
object: &str,
|
||||
body: &[u8],
|
||||
expression: &str,
|
||||
input_serialization: InputSerialization,
|
||||
output_serialization: OutputSerialization,
|
||||
progress_enabled: bool,
|
||||
) -> TestResult<()> {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(object)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut request = client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(object)
|
||||
.expression(expression)
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(input_serialization)
|
||||
.output_serialization(output_serialization);
|
||||
if progress_enabled {
|
||||
request = request.request_progress(RequestProgress::builder().enabled(true).build());
|
||||
}
|
||||
let response = request.send().await?;
|
||||
|
||||
let mut payload = response.payload;
|
||||
let mut records_len = 0_u64;
|
||||
let mut last_progress: Option<aws_sdk_s3::types::Progress> = None;
|
||||
let mut stats = None;
|
||||
let mut saw_end = false;
|
||||
tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async {
|
||||
// The AWS SDK validates both event-stream CRCs before yielding an event.
|
||||
while let Some(event) = payload.recv().await? {
|
||||
assert!(!saw_end, "Select emitted an event after End");
|
||||
match event {
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::Records(records) => {
|
||||
assert!(stats.is_none(), "Select emitted Records after Stats");
|
||||
if let Some(bytes) = records.payload {
|
||||
records_len = records_len.saturating_add(u64::try_from(bytes.as_ref().len())?);
|
||||
}
|
||||
}
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::Progress(event) => {
|
||||
assert!(stats.is_none(), "Select emitted Progress after Stats");
|
||||
let details = event.details.ok_or("Progress event did not contain details")?;
|
||||
if let Some(previous) = last_progress.as_ref() {
|
||||
assert!(details.bytes_scanned() >= previous.bytes_scanned());
|
||||
assert!(details.bytes_processed() >= previous.bytes_processed());
|
||||
assert!(details.bytes_returned() >= previous.bytes_returned());
|
||||
}
|
||||
last_progress = Some(details);
|
||||
}
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::Stats(event) => {
|
||||
assert!(stats.is_none(), "Select emitted more than one Stats event");
|
||||
stats = event.details;
|
||||
}
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
|
||||
assert!(stats.is_some(), "Select emitted End before Stats");
|
||||
saw_end = true;
|
||||
}
|
||||
_ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
|
||||
}
|
||||
}
|
||||
Ok::<(), Box<dyn Error + Send + Sync>>(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })??;
|
||||
|
||||
let stats = stats.ok_or("Select response ended without a Stats event")?;
|
||||
let input_len = i64::try_from(body.len())?;
|
||||
assert_eq!(stats.bytes_scanned(), Some(input_len));
|
||||
assert_eq!(stats.bytes_processed(), Some(input_len));
|
||||
assert_eq!(stats.bytes_returned(), Some(i64::try_from(records_len)?));
|
||||
if progress_enabled {
|
||||
if let Some(progress) = last_progress {
|
||||
assert!(stats.bytes_scanned() >= progress.bytes_scanned());
|
||||
assert!(stats.bytes_processed() >= progress.bytes_processed());
|
||||
assert!(stats.bytes_returned() >= progress.bytes_returned());
|
||||
}
|
||||
} else {
|
||||
assert!(last_progress.is_none(), "disabled request progress emitted a Progress event");
|
||||
}
|
||||
assert!(saw_end, "Select response ended without an End event");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_http_event_order_crc_and_input_byte_stats() -> TestResult<()> {
|
||||
const CSV_BODY: &[u8] = b"name,age\nAlice,30\nBob,25\n";
|
||||
const JSON_LINES_BODY: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
|
||||
const JSON_DOCUMENT_BODY: &[u8] = b"[{\"name\":\"Alice\"},{\"name\":\"Bob\"}]";
|
||||
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
setup_test_bucket(&client).await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics.csv",
|
||||
CSV_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics.jsonl",
|
||||
JSON_LINES_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.json(JsonInput::builder().set_type(Some(JsonType::Lines)).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics.json",
|
||||
JSON_DOCUMENT_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics-without-progress.csv",
|
||||
CSV_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_http_disconnect_releases_query() -> TestResult<()> {
|
||||
const OBJECT: &str = "disconnect.csv";
|
||||
const ROWS: usize = 16 * 1024;
|
||||
const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
|
||||
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")])
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
setup_test_bucket(&client).await?;
|
||||
|
||||
let row = format!("{}\n", "x".repeat(1023));
|
||||
let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
|
||||
body.extend_from_slice(b"value\n");
|
||||
for _ in 0..ROWS {
|
||||
body.extend_from_slice(row.as_bytes());
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(OBJECT)
|
||||
.body(Bytes::from(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Leaving this response body unread fills the bounded HTTP/event channels before the query can finish.
|
||||
let first = csv_select_request(&client, OBJECT).send().await?;
|
||||
let saturated = csv_select_request(&client, OBJECT)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the first HTTP stream should retain the only query permit");
|
||||
assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
|
||||
|
||||
drop(first);
|
||||
let second = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match csv_select_request(&client, OBJECT).send().await {
|
||||
Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
|
||||
Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
|
||||
tokio::time::sleep(RELEASE_BACKOFF).await;
|
||||
}
|
||||
Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "disconnected Select did not release its query permit".into() })??;
|
||||
drop(second);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_csv_basic() -> TestResult<()> {
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
@@ -498,107 +228,6 @@ async fn test_select_object_content_json_basic() -> TestResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_nested_json_source_path() -> TestResult<()> {
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
setup_test_bucket(&client).await?;
|
||||
upload_nested_json_document(&client).await?;
|
||||
|
||||
let result = select_json_document(
|
||||
&client,
|
||||
JSON_DOCUMENT_OBJECT,
|
||||
"SELECT e.name FROM S3Object[*].departments[*].employees[*] AS e WHERE e.active = true",
|
||||
)
|
||||
.await?;
|
||||
let names: Vec<String> = result
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["name"].as_str().ok_or("missing name field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
|
||||
assert_eq!(names, vec!["Alice", "Charlie"]);
|
||||
|
||||
let terminal_scalars = select_json_document(
|
||||
&client,
|
||||
JSON_DOCUMENT_OBJECT,
|
||||
"SELECT NAME FROM S3Object[*].DEPARTMENTS[*].employees[*].NAME",
|
||||
)
|
||||
.await?;
|
||||
let scalar_names: Vec<String> = terminal_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["name"].as_str().ok_or("missing scalar name field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(scalar_names, vec!["Alice", "Bob", "Charlie"]);
|
||||
|
||||
let aliased_scalars = select_json_document(
|
||||
&client,
|
||||
JSON_DOCUMENT_OBJECT,
|
||||
"SELECT v FROM S3Object[*].departments[*].employees[*].name AS v",
|
||||
)
|
||||
.await?;
|
||||
let aliased_names: Vec<String> = aliased_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["v"].as_str().ok_or("missing aliased scalar field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(aliased_names, vec!["Alice", "Bob", "Charlie"]);
|
||||
|
||||
let root_array = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][*] AS c").await?;
|
||||
let root_names: Vec<String> = root_array
|
||||
.lines()
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["name"].as_str().ok_or("missing root-array name field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(root_names, vec!["Alice", "Bob"]);
|
||||
|
||||
let root_index = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][0] AS c").await?;
|
||||
let root_index_value: serde_json::Value = serde_json::from_str(root_index.trim())?;
|
||||
assert_eq!(root_index_value["name"], "Alice");
|
||||
|
||||
let root_scalars = select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT V FROM S3Object AS V").await?;
|
||||
let scalar_values: Vec<i64> = root_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<i64> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["v"].as_i64().ok_or("missing root scalar value")?)
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(scalar_values, vec![1, 2]);
|
||||
|
||||
let implicit_root_scalars =
|
||||
select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT S3Object FROM S3Object").await?;
|
||||
let implicit_scalar_values: Vec<i64> = implicit_root_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<i64> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["s3object"].as_i64().ok_or("missing implicit root scalar value")?)
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(implicit_scalar_values, vec![1, 2]);
|
||||
|
||||
let quoted_root_scalars =
|
||||
select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT \"S3Object\" FROM \"S3Object\"").await?;
|
||||
let quoted_scalar_values: Vec<i64> = quoted_root_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<i64> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["S3Object"].as_i64().ok_or("missing quoted root scalar value")?)
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(quoted_scalar_values, vec![1, 2]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_csv_limit() -> TestResult<()> {
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
|
||||
@@ -23,9 +23,14 @@
|
||||
//!
|
||||
//! There are no containers, no external S3 backend and no `awscurl`: the
|
||||
//! `AddTier` admin call is signed in-process with `rustfs_signer`, exactly like
|
||||
//! the other admin-API e2e suites in this crate. The source server uses the
|
||||
//! explicit test-only loopback opt-in to tier to `cold` over
|
||||
//! `http://127.0.0.1:<port>` while production keeps the SSRF guard enabled.
|
||||
//! the other admin-API e2e suites in this crate. Every warm backend's endpoint
|
||||
//! (including RustFS) runs through the shared outbound policy
|
||||
//! (crates/utils/src/egress.rs), which rejects loopback hosts by default, so
|
||||
//! `hot` is started with `RUSTFS_OUTBOUND_ALLOW_ORIGINS` set to `cold`'s exact
|
||||
//! origin (see `hot_env_for_tier`) to allow this hermetic suite's real
|
||||
//! `http://127.0.0.1:<port>` connectivity — the same operator escape hatch
|
||||
//! already used for webhook targets and OIDC discovery URLs, not a relaxation
|
||||
//! of the check itself.
|
||||
//!
|
||||
//! The hermetic tests drive the transition and restore paths and pin the
|
||||
//! chains required by ilm-7 and the restore follow-up:
|
||||
@@ -46,7 +51,7 @@
|
||||
//! retry serves the object locally until expiry, and expiry leaves the
|
||||
//! remote object available for a second restore.
|
||||
|
||||
use crate::common::RustFSTestEnvironment;
|
||||
use crate::common::{RustFSTestEnvironment, local_http_client};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -56,6 +61,10 @@ use aws_sdk_s3::types::{
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use http::Method;
|
||||
use http::header::HOST;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::sign_v4;
|
||||
use s3s::Body;
|
||||
use serde::Deserialize;
|
||||
use std::time::{Duration as StdDuration, Instant};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
@@ -96,7 +105,6 @@ const MANUAL_ACTIVE_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_RESTART_CANCEL_OBJECTS: usize = 512;
|
||||
const MANUAL_ACTIVE_CANCEL_RUNNING_TIMEOUT: StdDuration = StdDuration::from_secs(15);
|
||||
const MANUAL_TRANSITION_CANCEL_BARRIER_ENV: &str = "RUSTFS_E2E_MANUAL_TRANSITION_CANCEL_BARRIER";
|
||||
const ALLOW_LOOPBACK_TIER_ENDPOINT_ENV: (&str, &str) = ("RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT", "true");
|
||||
const MANUAL_ASYNC_CONFLICT_TERMINAL_TIMEOUT: StdDuration = StdDuration::from_secs(90);
|
||||
const MANUAL_RESTART_RECOVERY_TIMEOUT: StdDuration = StdDuration::from_secs(80);
|
||||
const OBJECT_KEY: &str = "tier/鲁A12345/report.bin";
|
||||
@@ -113,20 +121,6 @@ const HDR_SOURCE_REPLICATION_REQUEST: &str = "x-rustfs-source-replication-reques
|
||||
const HDR_SOURCE_MTIME: &str = "x-rustfs-source-mtime";
|
||||
const TIER_MUTATION_RECOVERY_CHANGED: &str = "Remote tier mutation recovery changed before publish";
|
||||
|
||||
async fn start_tier_source(hot: &mut RustFSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
|
||||
let mut env = Vec::with_capacity(extra_env.len() + 1);
|
||||
env.push(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV);
|
||||
env.extend_from_slice(extra_env);
|
||||
hot.start_rustfs_server_with_env(vec![], &env).await
|
||||
}
|
||||
|
||||
async fn restart_tier_source(hot: &mut RustFSTestEnvironment, extra_env: &[(&str, &str)]) -> TestResult {
|
||||
let mut env = Vec::with_capacity(extra_env.len() + 1);
|
||||
env.push(ALLOW_LOOPBACK_TIER_ENDPOINT_ENV);
|
||||
env.extend_from_slice(extra_env);
|
||||
hot.restart_server_preserving_data(vec![], &env).await
|
||||
}
|
||||
|
||||
/// 5 MiB — the S3 minimum size for a non-final multipart part; the object's only
|
||||
/// internal part boundary sits at this offset.
|
||||
const PART0_SIZE: usize = 5 * 1024 * 1024;
|
||||
@@ -142,8 +136,9 @@ fn payload() -> Vec<u8> {
|
||||
|
||||
/// Sign and send an admin request in-process (no `awscurl`).
|
||||
///
|
||||
/// Thin wrapper over [`crate::common::admin_request`], kept local so the call
|
||||
/// sites below keep their `Option<&str>` body shape.
|
||||
/// Mirrors the shared admin-API e2e pattern: the SigV4 signature is computed
|
||||
/// over `UNSIGNED_PAYLOAD`, so the JSON body rides on the wire without being
|
||||
/// pre-hashed. Returns the response status and body text.
|
||||
async fn signed_admin_request(
|
||||
base_url: &str,
|
||||
method: Method,
|
||||
@@ -152,7 +147,52 @@ async fn signed_admin_request(
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(reqwest::StatusCode, String), Box<dyn std::error::Error + Send + Sync>> {
|
||||
crate::common::admin_request(base_url, method, path, body.map(str::to_string), access_key, secret_key).await
|
||||
let url = format!("{base_url}{path}");
|
||||
let uri = url.parse::<http::Uri>()?;
|
||||
let authority = uri.authority().ok_or("request URL missing authority")?.to_string();
|
||||
let body_bytes = body.map(|b| b.as_bytes().to_vec()).unwrap_or_default();
|
||||
|
||||
let request = http::Request::builder()
|
||||
.method(method.clone())
|
||||
.uri(uri)
|
||||
.header(HOST, authority)
|
||||
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD);
|
||||
let signed = sign_v4(request.body(Body::empty())?, 0, access_key, secret_key, "", "us-east-1");
|
||||
|
||||
let client = local_http_client();
|
||||
let mut request_builder = client.request(method, url.as_str());
|
||||
for (name, value) in signed.headers() {
|
||||
request_builder = request_builder.header(name, value);
|
||||
}
|
||||
if !body_bytes.is_empty() {
|
||||
request_builder = request_builder.body(body_bytes);
|
||||
}
|
||||
let response = request_builder.send().await?;
|
||||
let status = response.status();
|
||||
let text = response.text().await?;
|
||||
Ok((status, text))
|
||||
}
|
||||
|
||||
/// Extra child-process env for `hot` when it will be wired to a `cold` tier
|
||||
/// target over loopback.
|
||||
///
|
||||
/// `WarmBackendRustFS::new` now runs every tier endpoint through the shared
|
||||
/// outbound policy (crates/utils/src/egress.rs), which rejects loopback hosts
|
||||
/// by default just like the S3/Wasabi tier types already did. This hermetic
|
||||
/// suite's `cold` target is a second embedded server on `127.0.0.1`, so `hot`
|
||||
/// needs an explicit, exact-origin allowlist entry to reach it — the same
|
||||
/// operator escape hatch already used for webhook targets and OIDC discovery
|
||||
/// URLs, not a relaxation of the check itself (metadata/link-local/unspecified
|
||||
/// hosts stay forbidden even with this set).
|
||||
///
|
||||
/// Takes `cold`'s origin as a plain `&str` (rather than `&RustFSTestEnvironment`)
|
||||
/// so building this env list never holds a live borrow of `cold` itself — tests
|
||||
/// that later call a `&mut cold` method (e.g. `stop_server`) can pass an owned
|
||||
/// clone of `cold.url` instead.
|
||||
fn hot_env_for_tier<'a>(cold_origin: &'a str, extra: &[(&'a str, &'a str)]) -> Vec<(&'a str, &'a str)> {
|
||||
let mut env = vec![("RUSTFS_OUTBOUND_ALLOW_ORIGINS", cold_origin)];
|
||||
env.extend_from_slice(extra);
|
||||
env
|
||||
}
|
||||
|
||||
/// Wire `hot` -> `cold` as a `TierType::RustFS` remote tier via `AddTier`.
|
||||
@@ -210,27 +250,19 @@ async fn add_rustfs_tier(hot: &RustFSTestEnvironment, cold: &RustFSTestEnvironme
|
||||
}
|
||||
}
|
||||
|
||||
fn clear_tiers_confirmation_token(now: OffsetDateTime) -> String {
|
||||
let mut rand = "AGD1R25GI3I1GJGUGJFD7FBS4DFAASDF".to_string();
|
||||
rand.insert_str(3, &now.day().to_string());
|
||||
rand.insert_str(17, &now.month().to_string());
|
||||
rand.insert_str(23, &now.year().to_string());
|
||||
rand
|
||||
}
|
||||
|
||||
async fn clear_rustfs_tiers_force(hot: &RustFSTestEnvironment) -> TestResult {
|
||||
async fn remove_rustfs_tier_force(hot: &RustFSTestEnvironment) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/tier/{TIER_NAME}?force=true");
|
||||
let deadline = Instant::now() + StdDuration::from_secs(30);
|
||||
loop {
|
||||
let rand = clear_tiers_confirmation_token(OffsetDateTime::now_utc());
|
||||
let path = format!("/rustfs/admin/v3/tier/clear?rand={rand}&force=true");
|
||||
let (status, resp) = signed_admin_request(&hot.url, Method::POST, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
let (status, resp) =
|
||||
signed_admin_request(&hot.url, Method::DELETE, &path, None, &hot.access_key, &hot.secret_key).await?;
|
||||
if status.is_success() {
|
||||
return Ok(());
|
||||
}
|
||||
if (!resp.contains("TierNameBackendInUse") && !resp.contains(TIER_MUTATION_RECOVERY_CHANGED))
|
||||
|| Instant::now() >= deadline
|
||||
{
|
||||
return Err(format!("ClearTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
return Err(format!("RemoveTier(RustFS) failed: status={status}, body={resp}").into());
|
||||
}
|
||||
// Tier mutation cleanup and startup recovery are asynchronous.
|
||||
tokio::time::sleep(StdDuration::from_millis(100)).await;
|
||||
@@ -883,7 +915,8 @@ async fn test_hermetic_transition_main_path() -> TestResult {
|
||||
// Hot/source server. A 1s scanner cycle is a backstop; transition is
|
||||
// primarily driven immediately by the multipart completion path.
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_CYCLE", "1")]).await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &hot_env_for_tier(cold.url.as_str(), &[("RUSTFS_SCANNER_CYCLE", "1")]))
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
|
||||
// Wire the RustFS remote tier (real connectivity probe, no force).
|
||||
@@ -981,7 +1014,11 @@ async fn test_hermetic_transition_restore_failure_expiry_and_retry() -> TestResu
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(cold.url.as_str(), &[("RUSTFS_SCANNER_CYCLE", "1"), ("RUSTFS_ILM_DEBUG_DAY_SECS", "5")]),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1109,7 +1146,14 @@ async fn test_manual_transition_run_black_box_semantics() -> TestResult {
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
|
||||
@@ -1214,7 +1258,14 @@ async fn test_manual_transition_async_job_status_polling() -> TestResult {
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1312,7 +1363,14 @@ async fn test_manual_transition_async_limit_reports_terminal_partial() -> TestRe
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1473,13 +1531,16 @@ async fn test_manual_transition_async_scope_conflicts_report_active_job() -> Tes
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(
|
||||
&mut hot,
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
@@ -1583,7 +1644,14 @@ async fn test_manual_transition_async_different_buckets_admit_concurrently() ->
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1703,7 +1771,14 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -1716,7 +1791,7 @@ async fn test_manual_transition_async_tier_failure_reports_terminal_partial() ->
|
||||
0,
|
||||
)
|
||||
.await?;
|
||||
clear_rustfs_tiers_force(&hot).await?;
|
||||
remove_rustfs_tier_force(&hot).await?;
|
||||
|
||||
let due_mtime = OffsetDateTime::now_utc() - time::Duration::hours(25);
|
||||
put_backdated_single_part_object(
|
||||
@@ -1795,7 +1870,14 @@ async fn test_manual_transition_async_worker_failure_reports_terminal_partial()
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
cold.stop_server();
|
||||
@@ -1887,13 +1969,16 @@ async fn test_manual_transition_async_active_cancel_reports_terminal_cancelled()
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(
|
||||
&mut hot,
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
(MANUAL_TRANSITION_CANCEL_BARRIER_ENV, "1"),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
@@ -1986,14 +2071,20 @@ async fn test_manual_transition_async_cancel_after_process_restart_recovers_term
|
||||
let cold_client = cold.create_s3_client();
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let restart_env = [
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "512"),
|
||||
];
|
||||
// Owned copy: `cold` is stopped (a `&mut cold` call) below, and
|
||||
// `restart_env` must stay valid past that point for the later restart.
|
||||
let cold_origin = cold.url.clone();
|
||||
let restart_env = hot_env_for_tier(
|
||||
&cold_origin,
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "512"),
|
||||
],
|
||||
);
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &restart_env).await?;
|
||||
hot.start_rustfs_server_with_env(vec![], &restart_env).await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -2021,7 +2112,7 @@ async fn test_manual_transition_async_cancel_after_process_restart_recovers_term
|
||||
.ok_or("async response must include status_endpoint")?;
|
||||
assert_eq!(accepted.cancel_endpoint.as_deref(), Some(status_endpoint));
|
||||
|
||||
restart_tier_source(&mut hot, &restart_env).await?;
|
||||
hot.restart_server_preserving_data(vec![], &restart_env).await?;
|
||||
|
||||
let restarted = manual_transition_job_status(&hot, status_endpoint).await?;
|
||||
assert_eq!(restarted.job_id, job_id);
|
||||
@@ -2145,7 +2236,14 @@ async fn test_manual_transition_run_contract_no_status_cancel_fields() -> TestRe
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -2183,7 +2281,14 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
add_rustfs_tier(&hot, &cold).await?;
|
||||
|
||||
@@ -2223,7 +2328,14 @@ async fn test_manual_transition_run_continuation_token_resumes_without_raw_marke
|
||||
"continuation token must not expose the raw object prefix: {continuation}"
|
||||
);
|
||||
|
||||
restart_tier_source(&mut hot, &[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")]).await?;
|
||||
hot.restart_server_preserving_data(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[("RUSTFS_SCANNER_ENABLED", "false"), ("RUSTFS_SCANNER_CYCLE", "3600")],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let second = manual_transition_run_with_max_and_continuation(
|
||||
&hot,
|
||||
@@ -2256,14 +2368,17 @@ async fn test_manual_transition_run_queue_pressure_partial() -> TestResult {
|
||||
cold_client.create_bucket().bucket(TIER_BUCKET).send().await?;
|
||||
|
||||
let mut hot = RustFSTestEnvironment::new().await?;
|
||||
start_tier_source(
|
||||
&mut hot,
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1"),
|
||||
],
|
||||
hot.start_rustfs_server_with_env(
|
||||
vec![],
|
||||
&hot_env_for_tier(
|
||||
cold.url.as_str(),
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_SCANNER_CYCLE", "3600"),
|
||||
("RUSTFS_MAX_TRANSITION_WORKERS", "1"),
|
||||
("RUSTFS_TRANSITION_QUEUE_CAPACITY", "1"),
|
||||
],
|
||||
),
|
||||
)
|
||||
.await?;
|
||||
let hot_client = hot.create_s3_client();
|
||||
|
||||
@@ -13,9 +13,8 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user,
|
||||
awscurl_post_sts_form_urlencoded, init_logging, local_http_client, replication_fast_env, rustfs_binary_path, signed_request,
|
||||
signed_request_with_client, signed_request_with_session_token,
|
||||
RustFSTestEnvironment, admin_create_user, awscurl_post_sts_form_urlencoded, init_logging, local_http_client,
|
||||
replication_fast_env, rustfs_binary_path, signed_request, signed_request_with_client, signed_request_with_session_token,
|
||||
};
|
||||
use crate::fake_s3_target::{
|
||||
FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation,
|
||||
@@ -26,7 +25,7 @@ use crate::kms::common::{
|
||||
sse_customer_key_md5_base64,
|
||||
};
|
||||
use crate::storage_api::replication_extension::BucketTargetSys;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::config::{Credentials, Region};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::operation::list_object_versions::ListObjectVersionsOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -34,6 +33,7 @@ use aws_sdk_s3::types::{
|
||||
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, DeleteMarkerEntry, ObjectVersion, ServerSideEncryption,
|
||||
VersioningConfiguration,
|
||||
};
|
||||
use aws_sdk_s3::{Client, Config};
|
||||
use base64_simd::STANDARD as BASE64_STANDARD;
|
||||
use bytes::Bytes;
|
||||
use flate2::read::GzDecoder;
|
||||
@@ -895,7 +895,15 @@ async fn wait_for_replicated_object_over_https(
|
||||
}
|
||||
|
||||
fn create_user_s3_client(env: &RustFSTestEnvironment, access_key: &str, secret_key: &str) -> Client {
|
||||
env.create_s3_client_with_credentials(access_key, secret_key)
|
||||
let credentials = Credentials::new(access_key, secret_key, None, None, "e2e-site-replication");
|
||||
let config = Config::builder()
|
||||
.credentials_provider(credentials)
|
||||
.region(Region::new("us-east-1"))
|
||||
.endpoint_url(&env.url)
|
||||
.force_path_style(true)
|
||||
.behavior_version_latest()
|
||||
.build();
|
||||
Client::from_conf(config)
|
||||
}
|
||||
|
||||
async fn admin_add_canned_policy(
|
||||
@@ -903,15 +911,24 @@ async fn admin_add_canned_policy(
|
||||
policy_name: &str,
|
||||
policy: &serde_json::Value,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
admin_add_canned_policy_via(
|
||||
AdminTransport::Signed,
|
||||
&env.url,
|
||||
let url = format!("{}/rustfs/admin/v3/add-canned-policy?name={}", env.url, policy_name);
|
||||
let response = signed_request(
|
||||
http::Method::PUT,
|
||||
&url,
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
policy_name,
|
||||
&policy.to_string(),
|
||||
Some(policy.to_string().into_bytes()),
|
||||
Some("application/json"),
|
||||
)
|
||||
.await
|
||||
.await?;
|
||||
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("add canned policy failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admin_attach_policy_to_user(
|
||||
@@ -919,7 +936,19 @@ async fn admin_attach_policy_to_user(
|
||||
policy_name: &str,
|
||||
username: &str,
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, policy_name, username).await
|
||||
let url = format!(
|
||||
"{}/rustfs/admin/v3/set-user-or-group-policy?policyName={}&userOrGroup={}&isGroup=false",
|
||||
env.url, policy_name, username
|
||||
);
|
||||
let response = signed_request(http::Method::PUT, &url, &env.access_key, &env.secret_key, Some(Vec::new()), None).await?;
|
||||
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("attach policy to user failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn admin_update_group_members(
|
||||
@@ -1912,21 +1941,6 @@ async fn site_replication_info(env: &RustFSTestEnvironment) -> Result<SiteReplic
|
||||
Ok(serde_json::from_slice(&response.bytes().await?)?)
|
||||
}
|
||||
|
||||
async fn site_replication_rotate_svc_acct(
|
||||
env: &RustFSTestEnvironment,
|
||||
) -> Result<ReplicateEditStatus, Box<dyn Error + Send + Sync>> {
|
||||
let url = format!("{}/rustfs/admin/v3/site-replication/rotate-svc-acct", env.url);
|
||||
let response = signed_request(http::Method::POST, &url, &env.access_key, &env.secret_key, None, None).await?;
|
||||
|
||||
if response.status() != StatusCode::OK {
|
||||
let status = response.status();
|
||||
let body = response.text().await.unwrap_or_default();
|
||||
return Err(format!("site replication rotate-svc-acct failed: {status} {body}").into());
|
||||
}
|
||||
|
||||
Ok(serde_json::from_slice(&response.bytes().await?)?)
|
||||
}
|
||||
|
||||
async fn site_replication_resync_op(
|
||||
env: &RustFSTestEnvironment,
|
||||
operation: &str,
|
||||
@@ -6325,112 +6339,6 @@ async fn test_site_replication_remove_all_real_dual_node() -> Result<(), Box<dyn
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_site_replication_rotate_svc_acct_completes_and_replication_survives_real_dual_node()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
source_env
|
||||
.start_rustfs_server_with_env(vec![], LOOPBACK_REPLICATION_TARGET_ENV)
|
||||
.await?;
|
||||
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
target_env
|
||||
.start_rustfs_server_without_cleanup_with_env(LOOPBACK_REPLICATION_TARGET_ENV)
|
||||
.await?;
|
||||
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
let bucket = "site-repl-rotate-svc-acct";
|
||||
|
||||
let add_status = site_replication_add(
|
||||
&source_env,
|
||||
&[
|
||||
PeerSite {
|
||||
name: "source-site".to_string(),
|
||||
endpoint: source_env.url.clone(),
|
||||
access_key: source_env.access_key.clone(),
|
||||
secret_key: source_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
PeerSite {
|
||||
name: "target-site".to_string(),
|
||||
endpoint: target_env.url.clone(),
|
||||
access_key: target_env.access_key.clone(),
|
||||
secret_key: target_env.secret_key.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
assert!(add_status.success, "unexpected site add result: {add_status:?}");
|
||||
|
||||
let _source_info = wait_for_site_replication_enabled(&source_env, 2).await?;
|
||||
let _target_info = wait_for_site_replication_enabled(&target_env, 2).await?;
|
||||
|
||||
source_client.create_bucket().bucket(bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, bucket).await?;
|
||||
wait_for_bucket_on_target(&target_client, bucket).await?;
|
||||
let baseline_payload = b"before rotation".to_vec();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("before-rotate.txt")
|
||||
.body(ByteStream::from(baseline_payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let replicated_baseline = wait_for_object_on_target(&target_client, bucket, "before-rotate.txt").await?;
|
||||
assert_eq!(replicated_baseline, baseline_payload);
|
||||
|
||||
// A single rotation call must finish the whole hand-over. Before the fix
|
||||
// the join push could only sign with the freshly installed secret, every
|
||||
// peer rejected it, the rotation stayed pending forever, and both
|
||||
// replication directions were dead until an operator retried.
|
||||
let rotate_status = site_replication_rotate_svc_acct(&source_env).await?;
|
||||
assert!(rotate_status.success, "rotation did not complete in one call: {rotate_status:?}");
|
||||
|
||||
for env in [&source_env, &target_env] {
|
||||
let deadline = std::time::Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let info = site_replication_info(env).await?;
|
||||
if info.enabled && info.pending_operation.is_none() {
|
||||
break;
|
||||
}
|
||||
if std::time::Instant::now() > deadline {
|
||||
return Err(format!("rotation left {} with a pending operation: {:?}", env.url, info.pending_operation).into());
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
|
||||
// Replication must actually flow again in both directions with the
|
||||
// rotated service-account secret.
|
||||
let forward_payload = b"after rotation from source".to_vec();
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("after-rotate-forward.txt")
|
||||
.body(ByteStream::from(forward_payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let replicated_forward = wait_for_object_on_target(&target_client, bucket, "after-rotate-forward.txt").await?;
|
||||
assert_eq!(replicated_forward, forward_payload);
|
||||
|
||||
let reverse_payload = b"after rotation from target".to_vec();
|
||||
target_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("after-rotate-reverse.txt")
|
||||
.body(ByteStream::from(reverse_payload.clone()))
|
||||
.send()
|
||||
.await?;
|
||||
let replicated_reverse = wait_for_object_on_target(&source_client, bucket, "after-rotate-reverse.txt").await?;
|
||||
assert_eq!(replicated_reverse, reverse_payload);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_site_replication_state_edit_fresh_and_stale_real_dual_node() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Raw HTTP regression coverage for the Select request root alias (backlog#1626).
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, signed_s3_request};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use std::error::Error;
|
||||
use uuid::Uuid;
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
|
||||
const CSV_BODY: &[u8] = b"name\nGatewayJ-root-alias\nignored\n";
|
||||
const EXPECTED_RECORD: &[u8] = b"GatewayJ-root-alias";
|
||||
|
||||
fn select_request(root: &str) -> String {
|
||||
format!(
|
||||
r#"<{root} xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
|
||||
<Expression>SELECT s.name FROM S3Object s WHERE s.name = 'GatewayJ-root-alias'</Expression>
|
||||
<ExpressionType>SQL</ExpressionType>
|
||||
<InputSerialization><CSV><FileHeaderInfo>USE</FileHeaderInfo></CSV></InputSerialization>
|
||||
<OutputSerialization><CSV/></OutputSerialization>
|
||||
</{root}>"#
|
||||
)
|
||||
}
|
||||
|
||||
async fn raw_select(env: &RustFSTestEnvironment, bucket: &str, object: &str, root: &str) -> TestResult {
|
||||
let response = signed_s3_request(
|
||||
Method::POST,
|
||||
&format!("{}/{bucket}/{object}?select&select-type=2", env.url),
|
||||
Some(select_request(root)),
|
||||
Some("application/xml"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
)
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let body = response.bytes().await?.to_vec();
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::OK,
|
||||
"{root} root was rejected: {}",
|
||||
String::from_utf8_lossy(&body)
|
||||
);
|
||||
assert!(
|
||||
body.windows(EXPECTED_RECORD.len()).any(|window| window == EXPECTED_RECORD),
|
||||
"{root} root did not return the projected record"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn select_request_root_alias_reaches_select_endpoint() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(Vec::new()).await?;
|
||||
let client = env.create_s3_client();
|
||||
let bucket = format!("select-root-{}", Uuid::new_v4().simple());
|
||||
let object = "input.csv";
|
||||
|
||||
client.create_bucket().bucket(&bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(&bucket)
|
||||
.key(object)
|
||||
.body(ByteStream::from_static(CSV_BODY))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
raw_select(&env, &bucket, object, "SelectObjectContentRequest").await?;
|
||||
raw_select(&env, &bucket, object, "SelectRequest").await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -17,56 +17,8 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use std::error::Error;
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
fn pax_record(key: &str, value: &str) -> Vec<u8> {
|
||||
let payload = format!("{key}={value}\n");
|
||||
let mut len = payload.len() + 3;
|
||||
loop {
|
||||
let record = format!("{len} {payload}");
|
||||
if record.len() == len {
|
||||
return record.into_bytes();
|
||||
}
|
||||
len = record.len();
|
||||
}
|
||||
}
|
||||
|
||||
async fn append_pax_header(
|
||||
builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
|
||||
entry_type: tokio_tar::EntryType,
|
||||
records: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut payload = Vec::new();
|
||||
for (key, value) in records {
|
||||
payload.extend(pax_record(key, value));
|
||||
}
|
||||
let mut header = tokio_tar::Header::new_ustar();
|
||||
header.set_entry_type(entry_type);
|
||||
header.set_size(u64::try_from(payload.len()).expect("PAX payload length should fit in u64"));
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_typed_entry(
|
||||
builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
|
||||
path: &str,
|
||||
entry_type: tokio_tar::EntryType,
|
||||
body: &[u8],
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_entry_type(entry_type);
|
||||
header.set_size(u64::try_from(body.len()).expect("TAR member length should fit in u64"));
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, path, Cursor::new(body)).await?;
|
||||
Ok(())
|
||||
}
|
||||
use std::io::Cursor;
|
||||
|
||||
async fn build_test_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
@@ -117,50 +69,12 @@ mod tests {
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
async fn build_archive_with_invalid_checksum() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut archive = build_test_archive().await?;
|
||||
archive[0] ^= 1;
|
||||
Ok(archive)
|
||||
}
|
||||
|
||||
async fn build_archive_with_negative_gnu_mtime() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(b"negative-mtime-body".len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.as_old_mut().mtime.fill(0xff);
|
||||
builder
|
||||
.append_data(&mut header, "negative-mtime.txt", Cursor::new(b"negative-mtime-body".as_slice()))
|
||||
.await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn gzip_member(payload: &[u8]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(payload)?;
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
|
||||
async fn build_concatenated_gzip_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let archive = build_test_archive().await?;
|
||||
let split_at = archive.len() / 2;
|
||||
let mut encoded = gzip_member(&archive[..split_at])?;
|
||||
encoded.extend(gzip_member(&archive[split_at..])?);
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
async fn build_gzip_archive_with_invalid_crc() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut encoded = gzip_member(&build_test_archive().await?)?;
|
||||
let crc_offset = encoded.len().checked_sub(8).expect("gzip fixture must contain a trailer");
|
||||
encoded[crc_offset] ^= 1;
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
fn append_raw_tar_entry_with_type(archive: &mut Vec<u8>, path: &[u8], data: &[u8], entry_type: u8) {
|
||||
assert!(path.len() <= 100, "raw TAR fixture path must fit in the name field");
|
||||
fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
|
||||
let path = format!("../{victim_bucket}/evil-injected.txt");
|
||||
let data = b"injected-body";
|
||||
let mut header = [0u8; 512];
|
||||
|
||||
header[..path.len()].copy_from_slice(path);
|
||||
header[..path.len()].copy_from_slice(path.as_bytes());
|
||||
header[100..108].copy_from_slice(b"0000644\0");
|
||||
header[108..116].copy_from_slice(b"0000000\0");
|
||||
header[116..124].copy_from_slice(b"0000000\0");
|
||||
@@ -168,7 +82,7 @@ mod tests {
|
||||
header[124..136].copy_from_slice(size.as_bytes());
|
||||
header[136..148].copy_from_slice(b"00000000000\0");
|
||||
header[148..156].fill(b' ');
|
||||
header[156] = entry_type;
|
||||
header[156] = b'0';
|
||||
header[257..263].copy_from_slice(b"ustar\0");
|
||||
header[263..265].copy_from_slice(b"00");
|
||||
|
||||
@@ -176,87 +90,11 @@ mod tests {
|
||||
let checksum = format!("{:06o}\0 ", checksum);
|
||||
header[148..156].copy_from_slice(checksum.as_bytes());
|
||||
|
||||
let mut archive = Vec::new();
|
||||
archive.extend_from_slice(&header);
|
||||
archive.extend_from_slice(data);
|
||||
let padding = (512 - (data.len() % 512)) % 512;
|
||||
archive.extend(std::iter::repeat_n(0, padding));
|
||||
}
|
||||
|
||||
fn append_raw_tar_entry(archive: &mut Vec<u8>, path: &[u8], data: &[u8]) {
|
||||
append_raw_tar_entry_with_type(archive, path, data, b'0');
|
||||
}
|
||||
|
||||
fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
|
||||
let path = format!("../{victim_bucket}/evil-injected.txt");
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry(&mut archive, path.as_bytes(), b"injected-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
|
||||
async fn build_member_semantics_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_pax_header(
|
||||
&mut builder,
|
||||
tokio_tar::EntryType::XGlobalHeader,
|
||||
&[
|
||||
("minio.metadata.x-amz-meta-owner", "global"),
|
||||
("minio.metadata.x-amz-meta-snowball-auto-extract", "true"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
append_pax_header(
|
||||
&mut builder,
|
||||
tokio_tar::EntryType::XHeader,
|
||||
&[("minio.metadata.x-amz-meta-owner", "local")],
|
||||
)
|
||||
.await?;
|
||||
append_typed_entry(&mut builder, "regular.txt", tokio_tar::EntryType::Regular, b"regular-body").await?;
|
||||
for (path, entry_type) in [
|
||||
("char", tokio_tar::EntryType::Char),
|
||||
("block", tokio_tar::EntryType::Block),
|
||||
("fifo", tokio_tar::EntryType::Fifo),
|
||||
] {
|
||||
append_typed_entry(&mut builder, path, entry_type, b"").await?;
|
||||
}
|
||||
let mut directory = tokio_tar::Header::new_gnu();
|
||||
directory.set_entry_type(tokio_tar::EntryType::Directory);
|
||||
directory.set_size(0);
|
||||
directory.set_mode(0o755);
|
||||
directory.set_cksum();
|
||||
builder
|
||||
.append_data(&mut directory, "directory/", Cursor::new(Vec::new()))
|
||||
.await?;
|
||||
for (path, entry_type) in [
|
||||
("hard-link", tokio_tar::EntryType::Link),
|
||||
("symlink", tokio_tar::EntryType::Symlink),
|
||||
("continuous", tokio_tar::EntryType::Continuous),
|
||||
("unknown", tokio_tar::EntryType::Other(b'9')),
|
||||
] {
|
||||
append_typed_entry(&mut builder, path, entry_type, b"").await?;
|
||||
}
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
async fn build_versioned_member_archive(path: &str, version_id: &str) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, &[("minio.versionId", version_id)]).await?;
|
||||
append_typed_entry(&mut builder, path, tokio_tar::EntryType::Regular, b"versioned-body").await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn build_archive_with_invalid_utf8_entry() -> Vec<u8> {
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry(&mut archive, b"invalid-\xff.txt", b"ignored-body");
|
||||
append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
|
||||
fn build_archive_with_invalid_utf8_symlink() -> Vec<u8> {
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry_with_type(&mut archive, b"invalid-\xff-link", b"", b'2');
|
||||
append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
@@ -297,147 +135,6 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_applies_member_semantics_and_metadata_precedence() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-member-semantics";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Prefix", "members")
|
||||
.metadata("owner", "outer")
|
||||
.body(ByteStream::from(build_member_semantics_archive().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let regular = client.head_object().bucket(bucket).key("members/regular.txt").send().await?;
|
||||
let regular_metadata = regular.metadata().expect("regular member should expose metadata");
|
||||
assert_eq!(regular_metadata.get("owner").map(String::as_str), Some("local"));
|
||||
assert!(!regular_metadata.contains_key("snowball-auto-extract"));
|
||||
assert!(!regular_metadata.contains_key("minio-snowball-prefix"));
|
||||
|
||||
for key in ["char", "block", "fifo"] {
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("members/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(head.content_length(), Some(0), "{key} should be materialized as an empty object");
|
||||
assert_eq!(
|
||||
head.metadata().and_then(|metadata| metadata.get("owner")).map(String::as_str),
|
||||
Some("outer"),
|
||||
"{key} should not inherit global PAX metadata"
|
||||
);
|
||||
}
|
||||
let directory = client.head_object().bucket(bucket).key("members/directory/").send().await?;
|
||||
assert_eq!(directory.content_length(), Some(0));
|
||||
|
||||
for key in ["hard-link", "symlink", "continuous", "unknown"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("members/{key}"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported TAR entry type must be skipped");
|
||||
assert_eq!(error.into_service_error().code(), Some("NotFound"), "{key}");
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_validates_pax_version_id_against_bucket_state() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-version-semantics";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("null.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_versioned_member_archive("null.txt", "null").await?))
|
||||
.send()
|
||||
.await?;
|
||||
let null_member = client.get_object().bucket(bucket).key("null.txt").send().await?;
|
||||
assert_eq!(null_member.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
|
||||
|
||||
for (archive_key, member_key, version_id) in [
|
||||
("uuid.tar", "uuid.txt", uuid::Uuid::new_v4().to_string()),
|
||||
("uppercase-null.tar", "uppercase-null.txt", "NULL".to_string()),
|
||||
] {
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_versioned_member_archive(member_key, &version_id).await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("invalid or unversioned UUID import must be rejected");
|
||||
assert_eq!(error.into_service_error().code(), Some("InvalidArgument"), "{archive_key}");
|
||||
let missing = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected version import must not create an object");
|
||||
assert_eq!(missing.into_service_error().code(), Some("NotFound"), "{member_key}");
|
||||
}
|
||||
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
aws_sdk_s3::types::VersioningConfiguration::builder()
|
||||
.status(aws_sdk_s3::types::BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let imported_version_id = uuid::Uuid::new_v4().to_string();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versioned-uuid.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(
|
||||
build_versioned_member_archive("versioned-uuid.txt", &imported_version_id).await?,
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
let imported = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("versioned-uuid.txt")
|
||||
.version_id(&imported_version_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(imported.version_id(), Some(imported_version_id.as_str()));
|
||||
assert_eq!(imported.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
@@ -566,113 +263,6 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_accepts_negative_gnu_mtime() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-negative-mtime";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_archive_with_negative_gnu_mtime().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object = client.get_object().bucket(bucket).key("negative-mtime.txt").send().await?;
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"negative-mtime-body");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_consumes_concatenated_gzip_members() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-concatenated-gzip";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar.gz")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_concatenated_gzip_archive().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object = client.get_object().bucket(bucket).key("root.txt").send().await?;
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_gzip_crc_error_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-gzip-crc-ignore-errors";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar.gz")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(build_gzip_archive_with_invalid_crc().await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("gzip integrity failures must remain fatal under ignore-errors");
|
||||
|
||||
assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_mismatched_content_md5() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-content-md5";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.content_md5("AAAAAAAAAAAAAAAAAAAAAA==")
|
||||
.body(ByteStream::from(build_test_archive().await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("mismatched Content-MD5 must fail after the raw body reaches EOF");
|
||||
|
||||
assert_eq!(err.into_service_error().code(), Some("BadDigest"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -709,100 +299,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_skips_non_utf8_symlink_without_ignore_errors() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-invalid-utf8-link";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_archive_with_invalid_utf8_symlink()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
|
||||
assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
|
||||
assert_eq!(keys, vec!["valid.txt"]);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_skips_non_utf8_member_without_lossy_key_collision() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-invalid-utf8";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(build_archive_with_invalid_utf8_entry()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
|
||||
assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
|
||||
assert_eq!(keys, vec!["valid.txt"]);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_corrupt_tar_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-corrupt-ignore-errors";
|
||||
let archive = build_archive_with_invalid_checksum().await?;
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("corrupt TAR structure must remain fatal under ignore-errors");
|
||||
assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
|
||||
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
assert!(listed.contents().is_empty(), "corrupt archive must not produce objects");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_parent_dir_entry_even_when_ignore_errors_enabled()
|
||||
async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -822,7 +319,6 @@ mod tests {
|
||||
.bucket(attacker_bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await
|
||||
|
||||
@@ -194,10 +194,9 @@ pub mod bucket {
|
||||
BucketReplicationResyncStatus, BucketReplicationStat, BucketReplicationStats, BucketStats,
|
||||
DeleteReplicationConfigSnapshot, DeletedObjectReplicationInfo, DurableMrfBacklog, DynReplicationPool, InQueueMetric,
|
||||
MrfOpKind, MrfReplicateEntry, MustReplicateOptions, ObjectOpts, OperatorRuleContract,
|
||||
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATE_INCOMING_DELETE,
|
||||
REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS,
|
||||
ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
||||
REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS,
|
||||
REPLICATE_INCOMING_DELETE, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REPLICATION_WRITABLE_FIELDS, ReplicateDecision, ReplicateObjectInfo, ReplicationBatchAdmission, ReplicationConfig,
|
||||
ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationDeleteScheduleInput,
|
||||
ReplicationDeleteStateSource, ReplicationHealQueueResult, ReplicationObjectBridge, ReplicationObjectIO,
|
||||
ReplicationOperation, ReplicationPoolTrait, ReplicationPriority, ReplicationQueueAdmission, ReplicationScannerBridge,
|
||||
@@ -406,7 +405,7 @@ pub mod notification {
|
||||
pub use crate::services::notification_sys::{
|
||||
CrossPoolFenceFleetProofToken, NotificationPeerErr, NotificationSys, ScannerPublicationLeaseGrant,
|
||||
acquire_cross_pool_fence_fleet_proof, cross_pool_fence_fleet_proof_matches, get_global_notification_sys,
|
||||
new_global_notification_sys, scanner_peer_transport_error_message_is_retryable, start_remote_version_state_fleet_probe,
|
||||
new_global_notification_sys, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -416,10 +415,9 @@ pub mod object {
|
||||
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReader, NamespaceLockFence, ObjectEncryptionResolver,
|
||||
ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions, PutObjReader, QuotaAdmission,
|
||||
RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
|
||||
ScannerPublicationCommitState, StreamConsumer, get_object_body_cache_plaintext_len, lookup_get_object_body_cache_hook,
|
||||
register_get_object_body_cache_hook, register_object_mutation_hook, unregister_get_object_body_cache_hook,
|
||||
unregister_object_mutation_hook,
|
||||
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, StreamConsumer, get_object_body_cache_plaintext_len,
|
||||
lookup_get_object_body_cache_hook, register_get_object_body_cache_hook, register_object_mutation_hook,
|
||||
unregister_get_object_body_cache_hook, unregister_object_mutation_hook,
|
||||
};
|
||||
pub use crate::store::{
|
||||
PrepareSelectObjectSnapshotError, PreparedGetObjectReader, SelectObjectSnapshot, SelectObjectSnapshotReadError,
|
||||
@@ -461,8 +459,8 @@ pub mod rpc {
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -489,9 +487,9 @@ pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
|
||||
find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -22,9 +22,7 @@ use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use aws_credential_types::Credentials as SdkCredentials;
|
||||
use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
|
||||
use aws_sdk_s3::config::Region as SdkRegion;
|
||||
use aws_sdk_s3::config::RequestChecksumCalculation;
|
||||
use aws_sdk_s3::config::SharedHttpClient;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
@@ -40,7 +38,6 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::Tagging as SdkTagging;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ServerSideEncryption,
|
||||
};
|
||||
use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
|
||||
use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
|
||||
@@ -80,7 +77,7 @@ use std::str::FromStr as _;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use std::sync::Weak;
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
use std::time::{Duration, Instant};
|
||||
use time::{OffsetDateTime, format_description::well_known::Rfc3339};
|
||||
use tokio::sync::Mutex;
|
||||
use tokio::sync::RwLock;
|
||||
@@ -92,71 +89,6 @@ use uuid::Uuid;
|
||||
|
||||
const MAX_CONCURRENT_TARGET_HEALTH_CHECKS: usize = 16;
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
const EXPIRED_REMOTE_TARGET_CREDENTIALS: &str = "remote target credentials have expired";
|
||||
|
||||
#[derive(Clone)]
|
||||
struct RemoteTargetCredentialsProvider {
|
||||
credentials: SdkCredentials,
|
||||
}
|
||||
|
||||
impl RemoteTargetCredentialsProvider {
|
||||
fn resolve_at(&self, now: SystemTime) -> aws_credential_types::provider::Result {
|
||||
if self.credentials.expiry().is_some_and(|expiration| expiration <= now) {
|
||||
return Err(CredentialsError::provider_error(std::io::Error::other(EXPIRED_REMOTE_TARGET_CREDENTIALS)));
|
||||
}
|
||||
Ok(self.credentials.clone())
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Debug for RemoteTargetCredentialsProvider {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RemoteTargetCredentialsProvider")
|
||||
.field("temporary", &self.credentials.session_token().is_some())
|
||||
.field("expiration", &self.credentials.expiry())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl ProvideCredentials for RemoteTargetCredentialsProvider {
|
||||
fn provide_credentials<'a>(&'a self) -> future::ProvideCredentials<'a>
|
||||
where
|
||||
Self: 'a,
|
||||
{
|
||||
future::ProvideCredentials::ready(self.resolve_at(SystemTime::now()))
|
||||
}
|
||||
|
||||
fn fallback_on_interrupt(&self) -> Option<SdkCredentials> {
|
||||
self.resolve_at(SystemTime::now()).ok()
|
||||
}
|
||||
}
|
||||
|
||||
fn remote_target_sdk_credentials(
|
||||
credentials: &Credentials,
|
||||
account_id: &str,
|
||||
now: SystemTime,
|
||||
) -> Result<SdkCredentials, &'static str> {
|
||||
let session_token = credentials.effective_session_token();
|
||||
let expiration = credentials.effective_expiration().map(SystemTime::from);
|
||||
if expiration.is_some() && session_token.is_none() {
|
||||
return Err("remote target credential expiration requires a session token");
|
||||
}
|
||||
if expiration.is_some_and(|expiration| expiration <= now) {
|
||||
return Err(EXPIRED_REMOTE_TARGET_CREDENTIALS);
|
||||
}
|
||||
|
||||
let mut builder = SdkCredentials::builder()
|
||||
.access_key_id(credentials.access_key.clone())
|
||||
.secret_access_key(credentials.secret_key.clone())
|
||||
.account_id(account_id.to_string())
|
||||
.provider_name("bucket_target_sys");
|
||||
if let Some(session_token) = session_token {
|
||||
builder = builder.session_token(session_token.to_string());
|
||||
}
|
||||
if let Some(expiration) = expiration {
|
||||
builder = builder.expiry(expiration);
|
||||
}
|
||||
Ok(builder.build())
|
||||
}
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
pub type GetObjectSdkError = Box<SdkError<GetObjectError>>;
|
||||
@@ -420,28 +352,6 @@ pub struct BucketTargetSys {
|
||||
heartbeat_started: OnceLock<()>,
|
||||
}
|
||||
|
||||
/// Build the bucket-target health-check HTTP client without panicking when
|
||||
/// the host has no system CA bundle (issue #6734).
|
||||
///
|
||||
/// `BucketTargetSys::get()` initializes lazily on the startup path (bucket
|
||||
/// metadata install calls it on the main thread), and `reqwest::Client::new()`
|
||||
/// panics when the TLS backend cannot load any system trust root — the state
|
||||
/// of a minimal container image. Fall back to a client with an explicit empty
|
||||
/// trust store: HTTP health checks keep working, and HTTPS targets fail closed
|
||||
/// at the TLS handshake with a clear certificate error instead of aborting
|
||||
/// the whole process at startup.
|
||||
fn build_health_check_client() -> HttpClient {
|
||||
HttpClient::builder().build().unwrap_or_else(|error| {
|
||||
warn!(
|
||||
"bucket target health-check HTTP client could not load system TLS roots ({error}); continuing with an empty trust store — HTTPS target health checks will fail until a CA bundle is installed"
|
||||
);
|
||||
HttpClient::builder()
|
||||
.tls_certs_only(std::iter::empty::<reqwest::Certificate>())
|
||||
.build()
|
||||
.expect("HTTP client construction must succeed with an explicit empty trust store")
|
||||
})
|
||||
}
|
||||
|
||||
impl BucketTargetSys {
|
||||
pub fn get() -> &'static Self {
|
||||
GLOBAL_BUCKET_TARGET_SYS.get_or_init(Self::new)
|
||||
@@ -454,7 +364,7 @@ impl BucketTargetSys {
|
||||
targets_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_h_mutex: Arc::new(RwLock::new(HashMap::new())),
|
||||
hc_client: Arc::new(build_health_check_client()),
|
||||
hc_client: Arc::new(HttpClient::new()),
|
||||
a_mutex: Arc::new(Mutex::new(HashMap::new())),
|
||||
arn_errs_map: Arc::new(RwLock::new(HashMap::new())),
|
||||
target_update_mutexes: Arc::new(Mutex::new(HashMap::new())),
|
||||
@@ -913,26 +823,13 @@ impl BucketTargetSys {
|
||||
Ok(BucketTargets { targets: new_targets })
|
||||
}
|
||||
|
||||
async fn mark_refresh_attempt(&self, arn: &str) {
|
||||
// Rate-limit a failed config fetch as well as a failed client build.
|
||||
// A successful rebuild replaces this timestamp during publication.
|
||||
self.arn_remotes_map
|
||||
.write()
|
||||
.await
|
||||
.entry(arn.to_string())
|
||||
.or_default()
|
||||
.last_refresh = OffsetDateTime::now_utc();
|
||||
}
|
||||
|
||||
pub async fn mark_refresh_in_progress(&self, bucket: &str, arn: &str) {
|
||||
let mut arn_errs = self.arn_errs_map.write().await;
|
||||
let err = arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
|
||||
count: 1,
|
||||
arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
|
||||
bucket: bucket.to_string(),
|
||||
..Default::default()
|
||||
update_in_progress: true,
|
||||
count: 1,
|
||||
});
|
||||
err.update_in_progress = true;
|
||||
err.bucket = bucket.to_string();
|
||||
}
|
||||
|
||||
pub async fn mark_refresh_done(&self, bucket: &str, arn: &str) {
|
||||
@@ -944,21 +841,15 @@ impl BucketTargetSys {
|
||||
}
|
||||
|
||||
pub async fn is_reloading_target(&self, _bucket: &str, arn: &str) -> bool {
|
||||
self.arn_errs_map
|
||||
.read()
|
||||
.await
|
||||
.get(arn)
|
||||
.is_some_and(|err| err.update_in_progress)
|
||||
let arn_errs = self.arn_errs_map.read().await;
|
||||
arn_errs.get(arn).map(|err| err.update_in_progress).unwrap_or(false)
|
||||
}
|
||||
|
||||
pub async fn inc_arn_errs(&self, bucket: &str, arn: &str) {
|
||||
pub async fn inc_arn_errs(&self, _bucket: &str, arn: &str) {
|
||||
let mut arn_errs = self.arn_errs_map.write().await;
|
||||
let err = arn_errs.entry(arn.to_string()).or_insert_with(|| ArnErrs {
|
||||
bucket: bucket.to_string(),
|
||||
..Default::default()
|
||||
});
|
||||
err.count += 1;
|
||||
err.bucket = bucket.to_string();
|
||||
if let Some(err) = arn_errs.get_mut(arn) {
|
||||
err.count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_remote_target_client(&self, bucket: &str, arn: &str) -> Option<Arc<TargetClient>> {
|
||||
@@ -971,15 +862,15 @@ impl BucketTargetSys {
|
||||
.unwrap_or((None, None))
|
||||
};
|
||||
|
||||
let credentials_expired = cli
|
||||
.as_ref()
|
||||
.is_some_and(|client| client.credentials_expired_at(jiff::Timestamp::now()));
|
||||
if let Some(cli) = cli
|
||||
&& !credentials_expired
|
||||
{
|
||||
if let Some(cli) = cli {
|
||||
return Some(cli);
|
||||
}
|
||||
|
||||
// TODO(backlog): spawn an async task to proactively reload the replication target
|
||||
if self.is_reloading_target(bucket, arn).await {
|
||||
return None;
|
||||
}
|
||||
|
||||
if let Some(last_refresh) = last_refresh {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
if now - last_refresh < Duration::from_secs(60 * 5) {
|
||||
@@ -987,24 +878,16 @@ impl BucketTargetSys {
|
||||
}
|
||||
}
|
||||
|
||||
// The existing per-bucket publication lock is also the reload claim:
|
||||
// try-locking keeps the request path non-blocking, is cancellation-safe,
|
||||
// and prevents a stale reload from publishing after a credential update.
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let Ok(update_guard) = update_mutex.try_lock() else {
|
||||
return None;
|
||||
};
|
||||
self.mark_refresh_attempt(arn).await;
|
||||
|
||||
match get_bucket_targets_config(bucket).await {
|
||||
Ok(bucket_targets) => {
|
||||
self.update_all_targets_locked(bucket, Some(&bucket_targets)).await;
|
||||
self.mark_refresh_in_progress(bucket, arn).await;
|
||||
self.update_all_targets(bucket, Some(&bucket_targets)).await;
|
||||
self.mark_refresh_done(bucket, arn).await;
|
||||
}
|
||||
Err(e) => {
|
||||
error!("get bucket targets config error:{}", e);
|
||||
}
|
||||
};
|
||||
drop(update_guard);
|
||||
|
||||
let cli = self
|
||||
.arn_remotes_map
|
||||
@@ -1012,10 +895,8 @@ impl BucketTargetSys {
|
||||
.await
|
||||
.get(arn)
|
||||
.and_then(|target| target.client.clone());
|
||||
if let Some(cli) = cli
|
||||
&& !cli.credentials_expired_at(jiff::Timestamp::now())
|
||||
{
|
||||
return Some(cli);
|
||||
if cli.is_some() {
|
||||
return cli;
|
||||
}
|
||||
|
||||
self.inc_arn_errs(bucket, arn).await;
|
||||
@@ -1045,13 +926,12 @@ impl BucketTargetSys {
|
||||
});
|
||||
};
|
||||
|
||||
let creds = remote_target_sdk_credentials(credentials, &target.reset_id, SystemTime::now()).map_err(|error| {
|
||||
BucketTargetError::RemoteTargetConnectionErr {
|
||||
bucket: target.target_bucket.clone(),
|
||||
access_key: credentials.access_key.clone(),
|
||||
error: error.to_string(),
|
||||
}
|
||||
})?;
|
||||
let creds = SdkCredentials::builder()
|
||||
.access_key_id(credentials.access_key.clone())
|
||||
.secret_access_key(credentials.secret_key.clone())
|
||||
.account_id(target.reset_id.clone())
|
||||
.provider_name("bucket_target_sys")
|
||||
.build();
|
||||
|
||||
let endpoint = if target.secure {
|
||||
format!("https://{}", target.endpoint)
|
||||
@@ -1071,10 +951,9 @@ impl BucketTargetSys {
|
||||
|
||||
let mut config_builder = S3Config::builder()
|
||||
.endpoint_url(endpoint.clone())
|
||||
.credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
|
||||
.credentials_provider(SharedCredentialsProvider::new(creds))
|
||||
.region(SdkRegion::new(target.region.clone()))
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
|
||||
|
||||
if should_force_path_style(target) {
|
||||
config_builder = config_builder.force_path_style(true);
|
||||
@@ -1146,13 +1025,6 @@ impl BucketTargetSys {
|
||||
let update_mutex = self.target_update_mutex(bucket).await;
|
||||
let _update_guard = update_mutex.lock().await;
|
||||
|
||||
self.update_all_targets_locked(bucket, targets).await;
|
||||
}
|
||||
|
||||
/// Builds and publishes one bucket snapshot while its update mutex is held.
|
||||
/// Keeping persisted-config reads under the same mutex prevents a stale
|
||||
/// reload from overwriting a concurrent credential rotation.
|
||||
async fn update_all_targets_locked(&self, bucket: &str, targets: Option<&BucketTargets>) {
|
||||
let mut clients = Vec::new();
|
||||
if let Some(new_targets) = targets {
|
||||
for target in &new_targets.targets {
|
||||
@@ -1184,17 +1056,6 @@ impl BucketTargetSys {
|
||||
&& !new_targets.is_empty()
|
||||
{
|
||||
for (target, client) in clients {
|
||||
// Keep a timestamped placeholder for configured targets whose
|
||||
// client cannot be built. Replication records these attempts as
|
||||
// failed, while the placeholder prevents every object from
|
||||
// triggering another metadata reload/client build for five minutes.
|
||||
arn_remotes_map.insert(
|
||||
target.arn.clone(),
|
||||
ArnTarget {
|
||||
client: None,
|
||||
last_refresh: OffsetDateTime::now_utc(),
|
||||
},
|
||||
);
|
||||
match client {
|
||||
Ok(client) => {
|
||||
arn_remotes_map.insert(
|
||||
@@ -1207,6 +1068,11 @@ impl BucketTargetSys {
|
||||
health_map.insert(client.arn.clone(), target_health(&client));
|
||||
self.update_bandwidth_limit(bucket, &target.arn, target.bandwidth_limit);
|
||||
}
|
||||
// The target stays in `targets_map`, so it keeps showing up in
|
||||
// `bucket remote ls` while no client exists to replicate through it —
|
||||
// replication then drops every object for this ARN. Without this the
|
||||
// rejection (loopback endpoint, bad CA, unparseable URL) left no trace
|
||||
// anywhere.
|
||||
Err(err) => warn!(
|
||||
bucket = %bucket,
|
||||
arn = %target.arn,
|
||||
@@ -1370,25 +1236,6 @@ fn loopback_replication_targets_allowed() -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
|
||||
|
||||
/// Streaming trailer checksums make the SDK frame request bodies as
|
||||
/// `aws-chunked`; a target that does not decode that framing stores the frames
|
||||
/// verbatim, silently corrupting every replica while the transfer itself
|
||||
/// succeeds (#6853). Plain signed payloads are the compatible default; the env
|
||||
/// knob restores trailer checksums for fleets whose targets are all known to
|
||||
/// decode them.
|
||||
fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
|
||||
if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
|
||||
.map(|v| v.eq_ignore_ascii_case("true") || v == "1")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
RequestChecksumCalculation::WhenSupported
|
||||
} else {
|
||||
RequestChecksumCalculation::WhenRequired
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_replication_target_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
|
||||
validate_replication_target_endpoint_inner(url, loopback_replication_targets_allowed())
|
||||
}
|
||||
@@ -1768,17 +1615,6 @@ impl Default for AdvancedPutOptions {
|
||||
}
|
||||
}
|
||||
|
||||
/// The subset of the target's PutObject response replication audits.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RemotePutObjectResponse {
|
||||
/// Version id the target assigned (`x-amz-version-id`).
|
||||
pub version_id: Option<String>,
|
||||
/// ETag of what the target stored; `None` when the target withheld it or
|
||||
/// when its encryption mode (SSE-KMS / SSE-C) makes it incomparable to
|
||||
/// the source ETag. `None` is therefore "not decidable", never evidence.
|
||||
pub etag: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PutObjectOptions {
|
||||
pub user_metadata: HashMap<String, String>,
|
||||
@@ -2104,13 +1940,6 @@ pub struct TargetClient {
|
||||
}
|
||||
|
||||
impl TargetClient {
|
||||
fn credentials_expired_at(&self, now: jiff::Timestamp) -> bool {
|
||||
self.credentials
|
||||
.as_ref()
|
||||
.and_then(Credentials::effective_expiration)
|
||||
.is_some_and(|expiration| expiration <= now)
|
||||
}
|
||||
|
||||
pub fn to_url(&self) -> Url {
|
||||
Url::parse(&self.endpoint).unwrap()
|
||||
}
|
||||
@@ -2324,9 +2153,7 @@ impl TargetClient {
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||
/// contract — a target that adopts the source version echoes it back —
|
||||
/// together with the ETag of what the target actually stored, so callers
|
||||
/// can detect a target that persisted transformed bytes (#6853).
|
||||
/// contract — a target that adopts the source version echoes it back.
|
||||
pub async fn put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2334,7 +2161,7 @@ impl TargetClient {
|
||||
size: i64,
|
||||
body: ByteStream,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<RemotePutObjectResponse, S3ClientError> {
|
||||
) -> Result<Option<String>, S3ClientError> {
|
||||
let mut headers = opts.header();
|
||||
|
||||
let builder = self.client.put_object();
|
||||
@@ -2369,25 +2196,7 @@ impl TargetClient {
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
// of the stored plaintext, so it cannot be compared against the
|
||||
// source ETag; withhold it rather than let a caller conclude
|
||||
// corruption from an opaque value.
|
||||
let etag_comparable = output.sse_customer_algorithm().is_none()
|
||||
&& !matches!(
|
||||
output.server_side_encryption(),
|
||||
Some(ServerSideEncryption::AwsKms) | Some(ServerSideEncryption::AwsKmsDsse)
|
||||
);
|
||||
Ok(RemotePutObjectResponse {
|
||||
version_id: output.version_id().map(ToOwned::to_owned),
|
||||
etag: if etag_comparable {
|
||||
output.e_tag().map(ToOwned::to_owned)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
})
|
||||
}
|
||||
Ok(output) => Ok(output.version_id().map(ToOwned::to_owned)),
|
||||
Err(e) => match e {
|
||||
SdkError::ServiceError(service_err) => {
|
||||
let err = service_err.into_err();
|
||||
@@ -2535,21 +2344,6 @@ impl TargetClient {
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn abort_multipart_upload(&self, bucket: &str, object: &str, upload_id: &str) -> Result<(), S3ClientError> {
|
||||
match self
|
||||
.client
|
||||
.abort_multipart_upload()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.upload_id(upload_id)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(_) => Ok(()),
|
||||
Err(e) => Err(e.into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn remove_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2696,18 +2490,6 @@ mod tests {
|
||||
use super::*;
|
||||
use rcgen::generate_simple_self_signed;
|
||||
|
||||
// The startup panic fix for hosts without a CA bundle (issue #6734) rests
|
||||
// on two properties: the health-check client constructor never panics, and
|
||||
// its degraded fallback — an explicit empty trust store — always builds.
|
||||
#[test]
|
||||
fn health_check_client_construction_never_panics() {
|
||||
let _ = build_health_check_client();
|
||||
HttpClient::builder()
|
||||
.tls_certs_only(std::iter::empty::<reqwest::Certificate>())
|
||||
.build()
|
||||
.expect("empty-trust-store client build must succeed without touching system roots");
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RecordingHttpConnector {
|
||||
request_uris: Arc<std::sync::Mutex<Vec<String>>>,
|
||||
@@ -2726,165 +2508,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
type RecordedHeaders = Arc<std::sync::Mutex<Vec<Vec<(String, String)>>>>;
|
||||
|
||||
/// Records full request headers and answers with canned response headers,
|
||||
/// for asserting wire framing and response parsing.
|
||||
#[derive(Clone, Debug)]
|
||||
struct RecordingHeaderConnector {
|
||||
request_headers: RecordedHeaders,
|
||||
response_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for RecordingHeaderConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
self.request_headers
|
||||
.lock()
|
||||
.expect("recorded header lock should not be poisoned")
|
||||
.push(
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let mut response = HttpResponse::new(
|
||||
aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
|
||||
SdkBody::empty(),
|
||||
);
|
||||
for (name, value) in &self.response_headers {
|
||||
response.headers_mut().insert(name.clone(), value.clone());
|
||||
}
|
||||
HttpConnectorFuture::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
fn header_recording_target_client(response_headers: Vec<(String, String)>) -> (TargetClient, RecordedHeaders) {
|
||||
let request_headers: RecordedHeaders = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHeaderConnector {
|
||||
request_headers: Arc::clone(&request_headers),
|
||||
response_headers,
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let client = s3_client_for_test(443, Some(http_client));
|
||||
(
|
||||
TargetClient {
|
||||
endpoint: "https://localhost:443".to_string(),
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
storage_class: String::new(),
|
||||
disable_proxy: false,
|
||||
arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
|
||||
reset_id: String::new(),
|
||||
secure: true,
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
ByteStream::new(SdkBody::from_body_1_x(body))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_checksums_default_to_plain_payloads() {
|
||||
assert!(matches!(
|
||||
replication_request_checksum_calculation(),
|
||||
RequestChecksumCalculation::WhenRequired
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replication_put_object_sends_plain_signed_payloads_by_default() {
|
||||
let (client, recorded) = header_recording_target_client(Vec::new());
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
|
||||
let recorded = recorded.lock().expect("recorded header lock should not be poisoned");
|
||||
let headers = &recorded[0];
|
||||
let header = |name: &str| {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case(name))
|
||||
.map(|(_, v)| v.as_str())
|
||||
};
|
||||
// The #6853 regression shape: trailer checksums force aws-chunked
|
||||
// framing, which a non-decoding target stores verbatim as the object.
|
||||
assert_eq!(header("x-amz-trailer"), None, "streaming uploads must not carry a trailer checksum");
|
||||
assert!(
|
||||
header("content-encoding").is_none_or(|v| !v.contains("aws-chunked")),
|
||||
"streaming uploads must not be aws-chunked framed"
|
||||
);
|
||||
assert_eq!(header("x-amz-decoded-content-length"), None);
|
||||
assert_eq!(header("content-length"), Some("4"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_returns_the_etag_the_target_stored() {
|
||||
let (client, _) =
|
||||
header_recording_target_client(vec![("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string())]);
|
||||
let response = client
|
||||
.put_object(
|
||||
"target-bucket",
|
||||
"object",
|
||||
4,
|
||||
ByteStream::from_static(b"data"),
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
assert_eq!(response.etag.as_deref(), Some("\"9a0364b9e99bb480dd25e1f0284c8555\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_withholds_the_etag_under_target_side_kms() {
|
||||
let (client, _) = header_recording_target_client(vec![
|
||||
("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string()),
|
||||
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
|
||||
]);
|
||||
let response = client
|
||||
.put_object(
|
||||
"target-bucket",
|
||||
"object",
|
||||
4,
|
||||
ByteStream::from_static(b"data"),
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
assert!(
|
||||
response.etag.is_none(),
|
||||
"a KMS-encrypted replica's etag is not the content MD5 and must be withheld"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RecordingAuthConnector {
|
||||
signed_requests: Arc<std::sync::Mutex<Vec<(bool, bool)>>>,
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for RecordingAuthConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
let has_expected_token = request.headers().get("x-amz-security-token") == Some("temporary-session-token");
|
||||
let has_authorization = request.headers().contains_key("authorization");
|
||||
self.signed_requests
|
||||
.lock()
|
||||
.expect("recorded auth request lock should not be poisoned")
|
||||
.push((has_expected_token, has_authorization));
|
||||
HttpConnectorFuture::ready(Ok(HttpResponse::new(
|
||||
aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
|
||||
SdkBody::empty(),
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
fn recording_target_client() -> (TargetClient, Arc<std::sync::Mutex<Vec<String>>>) {
|
||||
let request_uris = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHttpConnector {
|
||||
@@ -2910,150 +2533,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_sdk_credentials_preserve_temporary_credential_fields() {
|
||||
let now = SystemTime::UNIX_EPOCH + Duration::from_secs(1_000);
|
||||
let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
|
||||
let credentials = Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: Some("temporary-session-token".to_string()),
|
||||
expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
|
||||
};
|
||||
|
||||
let sdk_credentials =
|
||||
remote_target_sdk_credentials(&credentials, "account", now).expect("unexpired temporary credentials should build");
|
||||
|
||||
assert_eq!(sdk_credentials.session_token(), Some("temporary-session-token"));
|
||||
assert_eq!(sdk_credentials.expiry(), Some(expiration));
|
||||
assert_eq!(sdk_credentials.account_id().map(|id| id.as_str()), Some("account"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_sdk_credentials_normalize_go_zero_expiration() {
|
||||
let credentials = Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: None,
|
||||
expiration: Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse")),
|
||||
};
|
||||
|
||||
let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
|
||||
.expect("Go zero expiration should remain compatible with static credentials");
|
||||
|
||||
assert!(sdk_credentials.session_token().is_none());
|
||||
assert!(sdk_credentials.expiry().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_sdk_credentials_reject_invalid_expiration_boundaries() {
|
||||
let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
|
||||
let mut credentials = Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: None,
|
||||
expiration: Some(jiff::Timestamp::try_from(expiration).expect("test expiration should convert")),
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
remote_target_sdk_credentials(&credentials, "", SystemTime::UNIX_EPOCH + Duration::from_secs(1_000))
|
||||
.expect_err("expiration without a session token must fail"),
|
||||
"remote target credential expiration requires a session token"
|
||||
);
|
||||
|
||||
credentials.session_token = Some("temporary-session-token".to_string());
|
||||
assert_eq!(
|
||||
remote_target_sdk_credentials(&credentials, "", expiration)
|
||||
.expect_err("credentials expire at the exact expiration boundary"),
|
||||
EXPIRED_REMOTE_TARGET_CREDENTIALS
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_target_credentials_provider_fails_closed_after_expiration() {
|
||||
let expiration = SystemTime::UNIX_EPOCH + Duration::from_secs(2_000);
|
||||
let provider = RemoteTargetCredentialsProvider {
|
||||
credentials: SdkCredentials::new(
|
||||
"access",
|
||||
"secret",
|
||||
Some("temporary-session-token".to_string()),
|
||||
Some(expiration),
|
||||
"test",
|
||||
),
|
||||
};
|
||||
|
||||
assert!(provider.resolve_at(expiration - Duration::from_nanos(1)).is_ok());
|
||||
let err = provider
|
||||
.resolve_at(expiration)
|
||||
.expect_err("expired credentials must not be returned");
|
||||
assert_eq!(err.source().map(ToString::to_string).as_deref(), Some(EXPIRED_REMOTE_TARGET_CREDENTIALS));
|
||||
assert!(!format!("{provider:?}").contains("temporary-session-token"));
|
||||
assert!(!format!("{provider:?}").contains("secret"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn target_client_detects_expiration_for_cache_refresh() {
|
||||
let expiration: jiff::Timestamp = "2099-01-01T00:00:00Z".parse().expect("expiration should parse");
|
||||
let (mut client, _) = recording_target_client();
|
||||
client.credentials = Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: Some("temporary-session-token".to_string()),
|
||||
expiration: Some(expiration),
|
||||
});
|
||||
|
||||
assert!(!client.credentials_expired_at("2098-12-31T23:59:59Z".parse().expect("pre-expiration timestamp should parse")));
|
||||
assert!(client.credentials_expired_at(expiration));
|
||||
|
||||
client.credentials.as_mut().expect("credentials should exist").expiration =
|
||||
Some("0001-01-01T00:00:00Z".parse().expect("Go zero time should parse"));
|
||||
assert!(!client.credentials_expired_at(jiff::Timestamp::now()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn temporary_credentials_add_security_token_to_sigv4_requests() {
|
||||
let signed_requests = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingAuthConnector {
|
||||
signed_requests: Arc::clone(&signed_requests),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let credentials = Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: Some("temporary-session-token".to_string()),
|
||||
expiration: Some("2099-01-01T00:00:00Z".parse().expect("future expiration should parse")),
|
||||
};
|
||||
let sdk_credentials = remote_target_sdk_credentials(&credentials, "", SystemTime::now())
|
||||
.expect("unexpired temporary credentials should build");
|
||||
let client = S3Client::from_conf(
|
||||
S3Config::builder()
|
||||
.endpoint_url("https://target.example")
|
||||
.credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider {
|
||||
credentials: sdk_credentials,
|
||||
}))
|
||||
.region(SdkRegion::new("us-east-1"))
|
||||
.http_client(http_client)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.build(),
|
||||
);
|
||||
|
||||
client
|
||||
.head_bucket()
|
||||
.bucket("target-bucket")
|
||||
.send()
|
||||
.await
|
||||
.expect("recording connector should accept the signed request");
|
||||
|
||||
assert_eq!(
|
||||
signed_requests
|
||||
.lock()
|
||||
.expect("recorded auth request lock should not be poisoned")
|
||||
.as_slice(),
|
||||
&[(true, true)],
|
||||
"SigV4 request must include both authorization and the session-token header"
|
||||
);
|
||||
}
|
||||
|
||||
fn spawn_https_server(cert: &rcgen::CertifiedKey<rcgen::KeyPair>, requests: usize) -> (u16, std::thread::JoinHandle<()>) {
|
||||
use std::io::{Read, Write};
|
||||
|
||||
@@ -3161,10 +2640,7 @@ mod tests {
|
||||
.credentials_provider(SharedCredentialsProvider::new(credentials))
|
||||
.region(SdkRegion::new("us-east-1"))
|
||||
.force_path_style(true)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
// Mirror the production remote-target builder so recorded requests
|
||||
// exercise the same checksum/framing behavior (#6853).
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
|
||||
if let Some(http_client) = http_client {
|
||||
config = config.http_client(http_client);
|
||||
}
|
||||
@@ -3988,29 +3464,6 @@ mod tests {
|
||||
assert!(mutexes.contains_key("second"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_refresh_attempt_updates_retry_timestamp_and_error_count() {
|
||||
let sys = BucketTargetSys::default();
|
||||
|
||||
sys.mark_refresh_attempt("arn:reload").await;
|
||||
let last_refresh = sys.arn_remotes_map.read().await["arn:reload"].last_refresh;
|
||||
assert!(OffsetDateTime::now_utc() - last_refresh < Duration::from_secs(5));
|
||||
|
||||
sys.inc_arn_errs("bucket", "arn:reload").await;
|
||||
sys.inc_arn_errs("bucket", "arn:reload").await;
|
||||
let errors = sys.arn_errs_map.read().await;
|
||||
assert_eq!(errors["arn:reload"].count, 2);
|
||||
assert_eq!(errors["arn:reload"].bucket, "bucket");
|
||||
drop(errors);
|
||||
|
||||
sys.mark_refresh_in_progress("bucket", "arn:reload").await;
|
||||
assert!(sys.is_reloading_target("bucket", "arn:reload").await);
|
||||
sys.mark_refresh_done("bucket", "arn:reload").await;
|
||||
assert!(!sys.is_reloading_target("bucket", "arn:reload").await);
|
||||
sys.mark_refresh_in_progress("bucket", "arn:reload").await;
|
||||
assert!(sys.is_reloading_target("bucket", "arn:reload").await);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_all_targets_publishes_disable_proxy_on_target_client() {
|
||||
// The read-proxy selector (replication_proxy::get_proxy_targets) skips
|
||||
@@ -4049,88 +3502,6 @@ mod tests {
|
||||
assert!(opted_out.disable_proxy, "disable_proxy must reach the published TargetClient");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn update_all_targets_keeps_failed_client_placeholder() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let target = BucketTarget {
|
||||
arn: "arn:expired".to_string(),
|
||||
endpoint: "192.168.1.10:9000".to_string(),
|
||||
target_bucket: "target-bucket".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: Some("temporary-session-token".to_string()),
|
||||
expiration: Some("2000-01-01T00:00:00Z".parse().expect("expired timestamp should parse")),
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let targets = BucketTargets { targets: vec![target] };
|
||||
|
||||
sys.update_all_targets("bucket", Some(&targets)).await;
|
||||
|
||||
let remotes = sys.arn_remotes_map.read().await;
|
||||
let placeholder = remotes
|
||||
.get("arn:expired")
|
||||
.expect("configured target should retain a cache entry");
|
||||
assert!(placeholder.client.is_none());
|
||||
assert!(OffsetDateTime::now_utc() - placeholder.last_refresh < Duration::from_secs(5));
|
||||
drop(remotes);
|
||||
assert!(sys.get_remote_target_client("bucket", "arn:expired").await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn credential_rotation_atomically_replaces_published_client() {
|
||||
let sys = BucketTargetSys::default();
|
||||
let target = |session_token: &str| BucketTarget {
|
||||
arn: "arn:rotating".to_string(),
|
||||
endpoint: "192.168.1.10:9000".to_string(),
|
||||
target_bucket: "target-bucket".to_string(),
|
||||
region: "us-east-1".to_string(),
|
||||
credentials: Some(Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: Some(session_token.to_string()),
|
||||
expiration: None,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
sys.update_all_targets(
|
||||
"bucket",
|
||||
Some(&BucketTargets {
|
||||
targets: vec![target("old-session-token")],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let old_client = sys
|
||||
.get_remote_target_client("bucket", "arn:rotating")
|
||||
.await
|
||||
.expect("initial client should be published");
|
||||
|
||||
sys.update_all_targets(
|
||||
"bucket",
|
||||
Some(&BucketTargets {
|
||||
targets: vec![target("new-session-token")],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let new_client = sys
|
||||
.get_remote_target_client("bucket", "arn:rotating")
|
||||
.await
|
||||
.expect("rotated client should be published");
|
||||
|
||||
assert!(!Arc::ptr_eq(&old_client, &new_client));
|
||||
assert_eq!(
|
||||
old_client.credentials.as_ref().and_then(Credentials::effective_session_token),
|
||||
Some("old-session-token")
|
||||
);
|
||||
assert_eq!(
|
||||
new_client.credentials.as_ref().and_then(Credentials::effective_session_token),
|
||||
Some("new-session-token")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn target_updates_serialize_client_build_through_publication_per_bucket() {
|
||||
let sys = Arc::new(BucketTargetSys::default());
|
||||
|
||||
@@ -490,7 +490,7 @@ impl ExpiryStats {
|
||||
}
|
||||
|
||||
fn add_nonnegative(counter: &AtomicI64, delta: i64) {
|
||||
let _ = counter.try_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(delta).max(0)));
|
||||
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(delta).max(0)));
|
||||
}
|
||||
|
||||
fn increment_missed_expiry_tasks(&self) {
|
||||
@@ -12195,7 +12195,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_continues_after_deleted_marker_bucket() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let suffix = Uuid::new_v4().simple();
|
||||
let earlier_bucket = format!("zzzz-recovery-{suffix}-a");
|
||||
let deleted_marker = format!("zzzz-recovery-{suffix}-m");
|
||||
@@ -12203,7 +12203,11 @@ mod tests {
|
||||
let later_object = "a-before-stale-marker";
|
||||
create_test_bucket(&ecstore, &earlier_bucket).await;
|
||||
create_test_bucket(&ecstore, &later_bucket).await;
|
||||
seed_recoverable_free_version(&disk_paths, &later_bucket, later_object, None, None).await;
|
||||
let mut reader = PutObjReader::from_vec(b"cursor reset probe".to_vec());
|
||||
ecstore
|
||||
.put_object(&later_bucket, later_object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("successor bucket object should be created");
|
||||
|
||||
let page = list_tier_free_versions(
|
||||
Arc::clone(&ecstore),
|
||||
@@ -12216,10 +12220,14 @@ mod tests {
|
||||
.expect("recovery should resume at the first bucket after a deleted marker bucket");
|
||||
|
||||
assert_eq!(page.buckets_scanned, 1, "the later bucket must not be skipped");
|
||||
assert_eq!(page.items.len(), 1, "the successor bucket's recoverable object must be returned");
|
||||
assert_eq!(page.items[0].bucket, later_bucket);
|
||||
assert_eq!(page.items[0].name, later_object);
|
||||
remove_seeded_free_version(&disk_paths, &later_bucket, later_object).await;
|
||||
assert_eq!(
|
||||
page.scanned_entries, 1,
|
||||
"the deleted bucket's object marker must not skip objects in the successor bucket"
|
||||
);
|
||||
ecstore
|
||||
.delete_object(&later_bucket, later_object, ObjectOptions::default())
|
||||
.await
|
||||
.expect("successor bucket object should be removed");
|
||||
for bucket in [&earlier_bucket, &later_bucket] {
|
||||
ecstore
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
|
||||
@@ -20,13 +20,14 @@ mod durable_namespace;
|
||||
pub mod evaluator;
|
||||
pub mod manual_transition_job;
|
||||
mod metadata_boundary;
|
||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs, get_lifecycle_config};
|
||||
pub(crate) use metadata_boundary::{LifecycleExpiryConfigs, get_expiry_configs};
|
||||
mod object_handlers_common;
|
||||
mod object_lock_boundary;
|
||||
pub use self::core as lifecycle;
|
||||
mod replication_sink;
|
||||
pub mod rule;
|
||||
mod runtime_boundary;
|
||||
mod tagging_boundary;
|
||||
pub mod tier_delete_journal;
|
||||
pub mod tier_free_version_recovery;
|
||||
pub mod tier_last_day_stats;
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
reason = "declared boundary surface for the ECStore replication split plan; no caller in this port (backlog#1823)"
|
||||
)]
|
||||
pub(crate) fn decode_tags_to_map(tags: &str) -> HashMap<String, String> {
|
||||
crate::bucket::tagging::decode_tags_to_map(tags)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::decode_tags_to_map;
|
||||
|
||||
#[test]
|
||||
fn decode_tags_to_map_preserves_bucket_tagging_parser_behavior() {
|
||||
let tags = decode_tags_to_map("env=prod&encoded=a%2Fb&=ignored");
|
||||
|
||||
assert_eq!(tags.get("env").map(String::as_str), Some("prod"));
|
||||
assert_eq!(tags.get("encoded").map(String::as_str), Some("a/b"));
|
||||
assert!(!tags.contains_key(""));
|
||||
}
|
||||
}
|
||||
@@ -412,14 +412,8 @@ pub(crate) fn require_bucket_metadata_sys_in(
|
||||
}
|
||||
|
||||
pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result<Arc<ECStore>> {
|
||||
object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance"))
|
||||
}
|
||||
|
||||
pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option<Arc<ECStore>> {
|
||||
let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?;
|
||||
Some(sys.read().await.api.clone())
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
Ok(sys.read().await.api.clone())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
@@ -2518,169 +2512,11 @@ pub(crate) mod test_support {
|
||||
mod tests {
|
||||
use super::test_support::isolated_store_over_temp_disks;
|
||||
use super::*;
|
||||
use crate::bucket::metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
|
||||
BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
|
||||
};
|
||||
use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
|
||||
use crate::config::com::read_config;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use byteorder::{ByteOrder as _, LittleEndian};
|
||||
use serial_test::serial;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const NEW_WRITER_REPLICATION_XML: &[u8] = br#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::111122223333:role/replication-role</Role><Rule><ID>rollback</ID><Priority>1</Priority><Filter><Prefix>documents/</Prefix></Filter><Status>Enabled</Status><Destination><Bucket>arn:aws:s3:::replica-bucket</Bucket></Destination><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication></Rule></ReplicationConfiguration>"#;
|
||||
|
||||
const NEW_WRITER_CONFIGS: [(&str, &[u8]); 14] = [
|
||||
(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#),
|
||||
(BUCKET_NOTIFICATION_CONFIG, br#"<NotificationConfiguration/>"#),
|
||||
(
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#,
|
||||
),
|
||||
(
|
||||
OBJECT_LOCK_CONFIG,
|
||||
br#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_SSECONFIG,
|
||||
br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
BUCKET_CORS_CONFIG,
|
||||
br#"<CORSConfiguration><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>https://example.test</AllowedOrigin></CORSRule></CORSConfiguration>"#,
|
||||
),
|
||||
(BUCKET_LOGGING_CONFIG, br#"<BucketLoggingStatus/>"#),
|
||||
(
|
||||
BUCKET_WEBSITE_CONFIG,
|
||||
br#"<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument></WebsiteConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_ACCELERATE_CONFIG,
|
||||
br#"<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
br#"<RequestPaymentConfiguration><Payer>Requester</Payer></RequestPaymentConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG,
|
||||
br#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#,
|
||||
),
|
||||
];
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "rollback-new-replication";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
|
||||
}
|
||||
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
metadata
|
||||
.update_config(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML.to_vec())
|
||||
.expect("new-writer replication XML should be accepted before persistence");
|
||||
writer
|
||||
.persist_new_and_set(metadata)
|
||||
.await
|
||||
.expect("new-writer replication metadata should persist");
|
||||
|
||||
let old_reader = BucketMetadataSys::new(store);
|
||||
let (loaded, _) = old_reader
|
||||
.get_replication_config(bucket)
|
||||
.await
|
||||
.expect("old metadata_sys must not classify new-writer replication XML as invalid");
|
||||
assert_eq!(loaded.role, "arn:aws:iam::111122223333:role/replication-role");
|
||||
assert_eq!(loaded.rules.len(), 1);
|
||||
assert_eq!(loaded.rules[0].id.as_deref(), Some("rollback"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_004_new_writer_metadata_blob_keeps_legacy_header_and_configs() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "rollback-new-metadata";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
|
||||
}
|
||||
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
for (config_file, bytes) in NEW_WRITER_CONFIGS {
|
||||
metadata
|
||||
.update_config(config_file, bytes.to_vec())
|
||||
.unwrap_or_else(|err| panic!("new-writer {config_file} fixture must be valid: {err}"));
|
||||
}
|
||||
writer
|
||||
.persist_new_and_set(metadata)
|
||||
.await
|
||||
.expect("new-writer metadata should persist");
|
||||
|
||||
let path = BucketMetadata::new(bucket).save_file_path();
|
||||
let blob = read_config(store.clone(), &path)
|
||||
.await
|
||||
.expect("persisted .metadata.bin should be readable");
|
||||
assert_eq!(
|
||||
LittleEndian::read_u16(&blob[0..2]),
|
||||
1,
|
||||
"bucket metadata format must stay rollback-readable"
|
||||
);
|
||||
assert_eq!(
|
||||
LittleEndian::read_u16(&blob[2..4]),
|
||||
1,
|
||||
"bucket metadata version must stay rollback-readable"
|
||||
);
|
||||
|
||||
let loaded = load_bucket_metadata(store, bucket)
|
||||
.await
|
||||
.expect("old read_bucket_metadata path must load the new-writer blob");
|
||||
let loaded_configs: [(&str, &[u8]); 14] = [
|
||||
(BUCKET_POLICY_CONFIG, &loaded.policy_config_json),
|
||||
(BUCKET_NOTIFICATION_CONFIG, &loaded.notification_config_xml),
|
||||
(BUCKET_LIFECYCLE_CONFIG, &loaded.lifecycle_config_xml),
|
||||
(OBJECT_LOCK_CONFIG, &loaded.object_lock_config_xml),
|
||||
(BUCKET_VERSIONING_CONFIG, &loaded.versioning_config_xml),
|
||||
(BUCKET_SSECONFIG, &loaded.encryption_config_xml),
|
||||
(BUCKET_TAGGING_CONFIG, &loaded.tagging_config_xml),
|
||||
(BUCKET_REPLICATION_CONFIG, &loaded.replication_config_xml),
|
||||
(BUCKET_CORS_CONFIG, &loaded.cors_config_xml),
|
||||
(BUCKET_LOGGING_CONFIG, &loaded.logging_config_xml),
|
||||
(BUCKET_WEBSITE_CONFIG, &loaded.website_config_xml),
|
||||
(BUCKET_ACCELERATE_CONFIG, &loaded.accelerate_config_xml),
|
||||
(BUCKET_REQUEST_PAYMENT_CONFIG, &loaded.request_payment_config_xml),
|
||||
(BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, &loaded.public_access_block_config_xml),
|
||||
];
|
||||
for ((expected_name, expected), (loaded_name, actual)) in NEW_WRITER_CONFIGS.into_iter().zip(loaded_configs) {
|
||||
assert_eq!(loaded_name, expected_name);
|
||||
assert_eq!(actual, expected, "old read_bucket_metadata changed {expected_name} bytes");
|
||||
}
|
||||
assert!(loaded.policy_config.is_some());
|
||||
assert!(loaded.notification_config.is_some());
|
||||
assert!(loaded.lifecycle_config.is_some());
|
||||
assert!(loaded.object_lock_config.is_some());
|
||||
assert!(loaded.versioning_config.is_some());
|
||||
assert!(loaded.sse_config.is_some());
|
||||
assert!(loaded.tagging_config.is_some());
|
||||
assert!(loaded.replication_config.is_some());
|
||||
assert!(loaded.cors_config.is_some());
|
||||
assert!(loaded.logging_config.is_some());
|
||||
assert!(loaded.website_config.is_some());
|
||||
assert!(loaded.accelerate_config.is_some());
|
||||
assert!(loaded.request_payment_config.is_some());
|
||||
assert!(loaded.public_access_block_config.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_delete_configs_are_not_treated_as_absent() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -177,28 +177,6 @@ pub fn replication_write_may_pass_worm_gate(
|
||||
Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
|
||||
}
|
||||
|
||||
/// Whether an authorized replication delete (`ObjectOptions::replication_request`)
|
||||
/// addressed to an explicit version may bypass GOVERNANCE retention on the
|
||||
/// local replica, exactly as an `x-amz-bypass-governance-retention` caller
|
||||
/// with the bypass permission would.
|
||||
///
|
||||
/// The source is authoritative for a replicated version purge (issue #6850):
|
||||
/// the same WORM deletion gate already ran there, and GOVERNANCE retention
|
||||
/// with an authorized bypass is the only lock state it can purge through.
|
||||
/// Requiring the bypass header again here makes the purge permanently
|
||||
/// undeliverable — replication senders never carry it — and the sites diverge
|
||||
/// forever. COMPLIANCE retention and legal hold stay blocking: the source
|
||||
/// gate can never purge through them, so a replication purge that meets one
|
||||
/// here is divergence or forgery and fails closed.
|
||||
///
|
||||
/// The trust judgment is the same one the write-path exemption uses:
|
||||
/// `replication_request` is only set once the receiving handler has
|
||||
/// authorized the caller for the replication action
|
||||
/// (`ReplicateDeleteAction`), never straight from request headers.
|
||||
pub fn replication_delete_may_bypass_governance(opts: &ObjectOptions) -> bool {
|
||||
opts.replication_request && opts.version_id.is_some()
|
||||
}
|
||||
|
||||
/// Check if an object is locked based on its metadata.
|
||||
/// This is a common function used by both lifecycle evaluation and deletion checks.
|
||||
///
|
||||
@@ -702,32 +680,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("modification time"));
|
||||
}
|
||||
|
||||
/// The replicated-purge GOVERNANCE bypass (#6850) applies only to an
|
||||
/// authorized replication delete addressed to an explicit version: a
|
||||
/// local delete never gets it, and a replicated delete without a version
|
||||
/// id creates a delete marker rather than purging anything.
|
||||
#[test]
|
||||
fn replication_delete_bypasses_governance_only_for_authorized_version_purges() {
|
||||
let version_purge = ObjectOptions {
|
||||
replication_request: true,
|
||||
version_id: Some("6b6ffbc0-b0d3-4a86-8f6c-fe19163b8dcd".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(replication_delete_may_bypass_governance(&version_purge));
|
||||
|
||||
let local_version_delete = ObjectOptions {
|
||||
replication_request: false,
|
||||
..version_purge.clone()
|
||||
};
|
||||
assert!(!replication_delete_may_bypass_governance(&local_version_delete));
|
||||
|
||||
let replicated_marker_creation = ObjectOptions {
|
||||
version_id: None,
|
||||
..version_purge
|
||||
};
|
||||
assert!(!replication_delete_may_bypass_governance(&replicated_marker_creation));
|
||||
}
|
||||
|
||||
/// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
|
||||
/// lock keys as empty strings (the MinIO on-disk shape, see
|
||||
/// `parse_object_lock_retention`); that is "no lock", not corruption, and
|
||||
|
||||
@@ -44,14 +44,14 @@ mod replication_versioning_boundary;
|
||||
mod runtime_boundary;
|
||||
|
||||
pub use replication_config_boundary::{
|
||||
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError,
|
||||
ReplicationConfigurationExt, ReplicationTargetValidationError, assign_site_replication_rule_priorities,
|
||||
invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
|
||||
merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
|
||||
replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationTargetValidationError,
|
||||
assign_site_replication_rule_priorities, invalid_replication_config_status_field, is_site_replication_role,
|
||||
is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
|
||||
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
|
||||
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
pub(crate) use replication_filemeta_boundary::version_purge_statuses_map;
|
||||
pub use replication_filemeta_boundary::{
|
||||
|
||||
@@ -13,12 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
pub use rustfs_replication::{
|
||||
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REMOTE_TARGET_UNSUPPORTED_FIELDS, REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION,
|
||||
REPLICATION_READ_ONLY_HISTORICAL_FIELDS, REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError,
|
||||
ReplicationConfigurationExt, ReplicationRuleExt, ReplicationTargetValidationError, assign_site_replication_rule_priorities,
|
||||
invalid_replication_config_status_field, is_site_replication_role, is_site_replication_rule,
|
||||
merge_incoming_replication_config, merge_user_replication_config, replication_target_arn_deployment_id,
|
||||
replication_target_arns, should_remove_replication_target, site_replication_rule_deployment_id,
|
||||
unsupported_replication_config_field, validate_replication_config_structure, validate_replication_config_target_arns,
|
||||
ObjectOpts, OperatorRuleContract, REMOTE_TARGET_CAPABILITY_CONTRACT_VERSION, REMOTE_TARGET_UNSUPPORTED_FIELDS,
|
||||
REMOTE_TARGET_WRITABLE_FIELDS, REPLICATION_CAPABILITY_CONTRACT_VERSION, REPLICATION_READ_ONLY_HISTORICAL_FIELDS,
|
||||
REPLICATION_WRITABLE_FIELDS, ReplicationConfigStructureError, ReplicationConfigurationExt, ReplicationRuleExt,
|
||||
ReplicationTargetValidationError, assign_site_replication_rule_priorities, invalid_replication_config_status_field,
|
||||
is_site_replication_role, is_site_replication_rule, merge_incoming_replication_config, merge_user_replication_config,
|
||||
replication_target_arn_deployment_id, replication_target_arns, should_remove_replication_target,
|
||||
site_replication_rule_deployment_id, unsupported_replication_config_field, validate_replication_config_structure,
|
||||
validate_replication_config_target_arns,
|
||||
};
|
||||
|
||||
@@ -436,21 +436,16 @@ pub(crate) async fn check_replicate_delete_strict(
|
||||
}
|
||||
|
||||
for target in decision.targets_map.values_mut() {
|
||||
let replicate_sync = ReplicationTargetStore::remote_target_client(bucket, &target.arn)
|
||||
.await
|
||||
.map(|client| client.replicate_sync);
|
||||
apply_target_delivery_mode(target, replicate_sync);
|
||||
if let Some(client) = ReplicationTargetStore::remote_target_client(bucket, &target.arn).await {
|
||||
target.synchronous = client.replicate_sync;
|
||||
} else {
|
||||
target.replicate = false;
|
||||
target.synchronous = false;
|
||||
}
|
||||
}
|
||||
Ok(decision)
|
||||
}
|
||||
|
||||
fn apply_target_delivery_mode(target: &mut ReplicateTargetDecision, replicate_sync: Option<bool>) {
|
||||
// A missing runtime client is a delivery failure, not a rule mismatch.
|
||||
// Preserve admission and fall back to the asynchronous worker, which can
|
||||
// persist FAILED state for the heal/retry path.
|
||||
target.synchronous = replicate_sync.unwrap_or(false);
|
||||
}
|
||||
|
||||
pub(crate) fn check_replicate_delete_with_snapshot(
|
||||
dobj: &ObjectToDelete,
|
||||
oi: &ObjectInfo,
|
||||
@@ -634,23 +629,6 @@ mod tests {
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn missing_target_client_preserves_delete_admission_as_async() {
|
||||
let mut target = ReplicateTargetDecision::new("arn:target".to_string(), true, true);
|
||||
|
||||
apply_target_delivery_mode(&mut target, None);
|
||||
|
||||
assert!(target.replicate, "a runtime client miss must not erase the replication rule decision");
|
||||
assert!(
|
||||
!target.synchronous,
|
||||
"unavailable synchronous targets must fall back to the async retry path"
|
||||
);
|
||||
|
||||
apply_target_delivery_mode(&mut target, Some(true));
|
||||
assert!(target.replicate);
|
||||
assert!(target.synchronous);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn must_replicate_options_preserve_request_flag() {
|
||||
let user_defined = HashMap::new();
|
||||
|
||||
@@ -19,9 +19,9 @@ pub use rustfs_replication::{
|
||||
};
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
|
||||
delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
|
||||
resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
delete_marker_purge_version_id, delete_replication_missing_source_decision, delete_replication_object_opts,
|
||||
heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
|
||||
should_retry_delete_marker_purge, target_delete_version_id,
|
||||
};
|
||||
|
||||
@@ -1048,6 +1048,7 @@ pub fn resync_start_conflict_id(error: &EcstoreError) -> Option<&str> {
|
||||
}
|
||||
|
||||
/// Main replication pool structure
|
||||
#[derive(Debug)]
|
||||
pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
// Atomic counters for active workers
|
||||
active_workers: Arc<AtomicI32>,
|
||||
@@ -1093,16 +1094,6 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
resyncer: Arc<ReplicationResyncer>,
|
||||
}
|
||||
|
||||
impl<S: ReplicationStorage> std::fmt::Debug for ReplicationPool<S> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ReplicationPool")
|
||||
.field("active_workers", &self.active_workers.load(Ordering::Relaxed))
|
||||
.field("active_lrg_workers", &self.active_lrg_workers.load(Ordering::Relaxed))
|
||||
.field("active_mrf_workers", &self.active_mrf_workers.load(Ordering::Relaxed))
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
/// Creates a new replication pool with specified options
|
||||
pub async fn new(opts: ReplicationPoolOpts, stats: Arc<ReplicationStats>, storage: Arc<S>) -> Arc<Self> {
|
||||
@@ -2141,7 +2132,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
|
||||
/// Load bucket replication resync statuses into memory
|
||||
#[instrument(skip(self, buckets, _cancellation_token), fields(bucket_count = buckets.len()))]
|
||||
#[instrument(skip(_cancellation_token))]
|
||||
async fn load_resync(
|
||||
self: Arc<Self>,
|
||||
buckets: &[String],
|
||||
@@ -3177,19 +3168,6 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
}
|
||||
}
|
||||
ReplicationHealQueueAction::QueueDelete(dv) => {
|
||||
// A purge the peer denied under object lock cannot succeed until
|
||||
// the lock lapses (#6850); requeuing it every heal cycle only
|
||||
// burns bandwidth and failure counters. The backoff expires on
|
||||
// its own, so the purge is probed again — and converges — once
|
||||
// the retention window has a chance of being over.
|
||||
if super::replication_object_decision_boundary::is_version_delete_replication(&dv.delete_object)
|
||||
&& super::replication_resyncer::object_lock_denied_purge_backoff_active(&dv)
|
||||
{
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
};
|
||||
}
|
||||
let admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_delete_task(dv).await
|
||||
} else {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -36,8 +36,8 @@ use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions,
|
||||
S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
|
||||
@@ -25,8 +25,6 @@ use time::OffsetDateTime;
|
||||
use url::Url;
|
||||
|
||||
const REDACTED_CREDENTIAL: &str = "<redacted>";
|
||||
const GO_YEAR_ONE_START_UNIX_SECONDS: i64 = -62_135_596_800;
|
||||
const GO_YEAR_TWO_START_UNIX_SECONDS: i64 = -62_104_060_800;
|
||||
|
||||
#[derive(Deserialize, Serialize, Default, Clone)]
|
||||
pub struct Credentials {
|
||||
@@ -43,26 +41,6 @@ pub struct Credentials {
|
||||
}
|
||||
|
||||
impl Credentials {
|
||||
/// Returns the session token used for request signing.
|
||||
///
|
||||
/// MinIO-compatible payloads may carry an empty token. Treat whitespace-only
|
||||
/// values as absent without rewriting a real token, whose bytes are opaque.
|
||||
pub fn effective_session_token(&self) -> Option<&str> {
|
||||
self.session_token.as_deref().filter(|token| !token.trim().is_empty())
|
||||
}
|
||||
|
||||
/// Returns the credential expiry after normalizing Go's zero `time.Time`.
|
||||
///
|
||||
/// Go JSON encoders emit year 1 for an unset `time.Time`; persisted MinIO
|
||||
/// target metadata can therefore contain that sentinel even for static
|
||||
/// credentials.
|
||||
pub fn effective_expiration(&self) -> Option<Timestamp> {
|
||||
self.expiration.filter(|expiration| {
|
||||
let unix_seconds = expiration.as_second();
|
||||
!(GO_YEAR_ONE_START_UNIX_SECONDS..GO_YEAR_TWO_START_UNIX_SECONDS).contains(&unix_seconds)
|
||||
})
|
||||
}
|
||||
|
||||
pub fn redacted(&self) -> Self {
|
||||
Self {
|
||||
access_key: self.access_key.clone(),
|
||||
@@ -377,24 +355,6 @@ mod tests {
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
#[test]
|
||||
fn credential_effective_values_normalize_only_compatibility_sentinels() {
|
||||
let mut credentials = Credentials {
|
||||
access_key: "access".to_string(),
|
||||
secret_key: "secret".to_string(),
|
||||
session_token: Some(" ".to_string()),
|
||||
expiration: Some("0001-01-01T08:00:00+08:00".parse().expect("Go zero time should parse")),
|
||||
};
|
||||
|
||||
assert!(credentials.effective_session_token().is_none());
|
||||
assert!(credentials.effective_expiration().is_none());
|
||||
|
||||
credentials.session_token = Some(" opaque token ".to_string());
|
||||
credentials.expiration = Some("2099-01-01T00:00:00Z".parse().expect("future timestamp should parse"));
|
||||
assert_eq!(credentials.effective_session_token(), Some(" opaque token "));
|
||||
assert_eq!(credentials.effective_expiration(), credentials.expiration);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bucket_target_json_deserialize() {
|
||||
let json = r#"
|
||||
|
||||
@@ -73,7 +73,6 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
|
||||
@@ -1307,32 +1307,6 @@ pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonic
|
||||
verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, internode_rpc_body_digest_strict())
|
||||
}
|
||||
|
||||
/// Verify a non-disk mutation without accepting a newly-generated unsigned v2 body.
|
||||
///
|
||||
/// The disk mutation lane has a rolling-upgrade exception for `UNSIGNED-PAYLOAD`
|
||||
/// while peer replay-cache capability is being discovered. Historical v2 peers
|
||||
/// used the fixed `unsigned` nonce before body-digest rollout; preserve that
|
||||
/// exact marker for mixed-version compatibility, but reject unsigned v2
|
||||
/// requests that omit it or present a different nonce.
|
||||
pub fn verify_tonic_mutation_body_digest_reject_unsigned<T>(
|
||||
request: &tonic::Request<T>,
|
||||
canonical_body: &[u8],
|
||||
) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let nonce = request.metadata().get(RPC_NONCE_HEADER).and_then(|value| value.to_str().ok());
|
||||
if version == Some(RPC_AUTH_VERSION_V2) && digest == Some(UNSIGNED_PAYLOAD) && nonce != Some("unsigned") {
|
||||
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
|
||||
}
|
||||
verify_tonic_mutation_body_digest(request, canonical_body)
|
||||
}
|
||||
|
||||
/// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
|
||||
/// rollout postures are unit-testable without racing on process-global environment variables.
|
||||
fn verify_tonic_mutation_body_digest_with_strictness<T>(
|
||||
|
||||
@@ -36,7 +36,6 @@ use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, LazyLock, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
@@ -106,13 +105,9 @@ struct PutFileCapabilityCacheState {
|
||||
cached: Option<PutFileCapabilityState>,
|
||||
generation: u64,
|
||||
in_flight: Option<PutFileCapabilityFlight>,
|
||||
rejected_server_epoch: Option<Uuid>,
|
||||
}
|
||||
|
||||
// The registry lock is released before taking an entry lock. Entry guards cover
|
||||
// only cache transitions, never a probe or await; poll-based writers must be
|
||||
// able to reject an epoch atomically with those transitions.
|
||||
type PutFileCapabilityCacheEntry = Arc<parking_lot::RwLock<PutFileCapabilityCacheState>>;
|
||||
type PutFileCapabilityCacheEntry = Arc<tokio::sync::RwLock<PutFileCapabilityCacheState>>;
|
||||
|
||||
static PUT_FILE_CAPABILITY_CACHE: LazyLock<parking_lot::RwLock<HashMap<String, PutFileCapabilityCacheEntry>>> =
|
||||
LazyLock::new(|| parking_lot::RwLock::new(HashMap::new()));
|
||||
@@ -124,7 +119,7 @@ fn put_file_capability_cache_entry(endpoint: &str) -> PutFileCapabilityCacheEntr
|
||||
PUT_FILE_CAPABILITY_CACHE
|
||||
.write()
|
||||
.entry(endpoint.to_owned())
|
||||
.or_insert_with(|| Arc::new(parking_lot::RwLock::new(PutFileCapabilityCacheState::default())))
|
||||
.or_insert_with(|| Arc::new(tokio::sync::RwLock::new(PutFileCapabilityCacheState::default())))
|
||||
.clone()
|
||||
}
|
||||
|
||||
@@ -139,23 +134,6 @@ fn fresh_put_file_capability(state: Option<PutFileCapabilityState>, now: Instant
|
||||
}
|
||||
}
|
||||
|
||||
fn reject_put_file_server_epoch(endpoint: &str, server_epoch: Uuid) {
|
||||
let entry = PUT_FILE_CAPABILITY_CACHE.read().get(endpoint).cloned();
|
||||
if let Some(entry) = entry {
|
||||
let mut state = entry.write();
|
||||
if matches!(state.cached, Some(PutFileCapabilityState::V1 { server_epoch: cached, .. }) if cached == server_epoch) {
|
||||
state.rejected_server_epoch = Some(server_epoch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn usable_put_file_capability(state: &PutFileCapabilityCacheState, now: Instant) -> Option<Option<Uuid>> {
|
||||
match fresh_put_file_capability(state.cached, now)? {
|
||||
Some(server_epoch) if state.rejected_server_epoch == Some(server_epoch) => None,
|
||||
capability => Some(capability),
|
||||
}
|
||||
}
|
||||
|
||||
fn put_file_capability_status_is_legacy(status: u16) -> bool {
|
||||
status == 404
|
||||
}
|
||||
@@ -344,14 +322,13 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
|
||||
let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
|
||||
let url = build_put_file_stream_url(&request, auth_scope);
|
||||
let endpoint = request.endpoint;
|
||||
let nonce = server_epoch.map(|_| Uuid::new_v4());
|
||||
let url = build_put_file_stream_url(&request, nonce.zip(server_epoch));
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::PUT, &mut headers)?;
|
||||
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
|
||||
match auth_scope {
|
||||
Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
|
||||
match nonce {
|
||||
Some(nonce) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce))),
|
||||
None => Ok(Box::new(writer)),
|
||||
}
|
||||
}
|
||||
@@ -521,15 +498,15 @@ where
|
||||
{
|
||||
let entry = put_file_capability_cache_entry(endpoint);
|
||||
{
|
||||
let state = entry.read();
|
||||
if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
|
||||
let state = entry.read().await;
|
||||
if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
|
||||
return Ok(cached);
|
||||
}
|
||||
}
|
||||
|
||||
let flight = {
|
||||
let mut state = entry.write();
|
||||
if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
|
||||
let mut state = entry.write().await;
|
||||
if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
|
||||
return Ok(cached);
|
||||
}
|
||||
if let Some(flight) = state.in_flight.clone() {
|
||||
@@ -555,7 +532,7 @@ where
|
||||
.await;
|
||||
|
||||
{
|
||||
let mut state = entry.write();
|
||||
let mut state = entry.write().await;
|
||||
let is_current_flight = state
|
||||
.in_flight
|
||||
.as_ref()
|
||||
@@ -563,9 +540,6 @@ where
|
||||
if is_current_flight {
|
||||
match outcome {
|
||||
Ok(Some(server_epoch)) => {
|
||||
if state.rejected_server_epoch != Some(*server_epoch) {
|
||||
state.rejected_server_epoch = None;
|
||||
}
|
||||
state.cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch: *server_epoch,
|
||||
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
|
||||
@@ -656,23 +630,17 @@ struct PutFileAuthWriter<W> {
|
||||
inner: W,
|
||||
url: String,
|
||||
nonce: Uuid,
|
||||
endpoint: String,
|
||||
server_epoch: Uuid,
|
||||
server_epoch_rejected: bool,
|
||||
hasher: Sha256,
|
||||
trailer: Option<Vec<u8>>,
|
||||
trailer_offset: usize,
|
||||
}
|
||||
|
||||
impl<W> PutFileAuthWriter<W> {
|
||||
fn new(inner: W, url: String, nonce: Uuid, endpoint: String, server_epoch: Uuid) -> Self {
|
||||
fn new(inner: W, url: String, nonce: Uuid) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
url,
|
||||
nonce,
|
||||
endpoint,
|
||||
server_epoch,
|
||||
server_epoch_rejected: false,
|
||||
hasher: Sha256::new(),
|
||||
trailer: None,
|
||||
trailer_offset: 0,
|
||||
@@ -688,14 +656,6 @@ impl<W> PutFileAuthWriter<W> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reject_server_epoch_on_conflict(&mut self, error: &io::Error) {
|
||||
if self.server_epoch_rejected || !io_error_has_put_file_epoch_conflict(error) {
|
||||
return;
|
||||
}
|
||||
reject_put_file_server_epoch(&self.endpoint, self.server_epoch);
|
||||
self.server_epoch_rejected = true;
|
||||
}
|
||||
|
||||
fn poll_write_trailer(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
@@ -713,10 +673,7 @@ impl<W> PutFileAuthWriter<W> {
|
||||
)));
|
||||
}
|
||||
Poll::Ready(Ok(written)) => written,
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
return Poll::Ready(Err(err));
|
||||
}
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
};
|
||||
self.trailer_offset += written;
|
||||
@@ -725,15 +682,6 @@ impl<W> PutFileAuthWriter<W> {
|
||||
}
|
||||
}
|
||||
|
||||
fn io_error_has_put_file_epoch_conflict(error: &io::Error) -> bool {
|
||||
error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
|
||||
.is_some_and(
|
||||
|error| matches!(error.kind(), rustfs_rio::InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409),
|
||||
)
|
||||
}
|
||||
|
||||
impl<W> AsyncWrite for PutFileAuthWriter<W>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
@@ -750,22 +698,12 @@ where
|
||||
self.hasher.update(&buf[..written]);
|
||||
Poll::Ready(Ok(written))
|
||||
}
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
Poll::Ready(Err(err))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
match Pin::new(&mut self.inner).poll_flush(cx) {
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
Poll::Ready(Err(err))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
@@ -774,13 +712,7 @@ where
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
match Pin::new(&mut self.inner).poll_shutdown(cx) {
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
Poll::Ready(Err(err))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -908,6 +840,7 @@ mod tests {
|
||||
loop {
|
||||
let strong_count = entry
|
||||
.read()
|
||||
.await
|
||||
.in_flight
|
||||
.as_ref()
|
||||
.map(|flight| Arc::strong_count(&flight.outcome))
|
||||
@@ -925,50 +858,6 @@ mod tests {
|
||||
#[derive(Debug)]
|
||||
struct LegacyTestTransport;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum PutFileFailurePhase {
|
||||
Write,
|
||||
Flush,
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
struct PutFileFailureWriter {
|
||||
phase: PutFileFailurePhase,
|
||||
status: reqwest::StatusCode,
|
||||
}
|
||||
|
||||
impl PutFileFailureWriter {
|
||||
fn error(&self) -> io::Error {
|
||||
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
|
||||
}
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncWrite for PutFileFailureWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Write) {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(buf.len())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Flush) {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Shutdown) {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl InternodeDataTransport for LegacyTestTransport {
|
||||
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
|
||||
@@ -1159,7 +1048,7 @@ mod tests {
|
||||
let v1_endpoint = format!("http://v1-{}.invalid", Uuid::new_v4());
|
||||
let v1_entry = put_file_capability_cache_entry(&v1_endpoint);
|
||||
let server_epoch = Uuid::new_v4();
|
||||
v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
|
||||
v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch,
|
||||
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
|
||||
});
|
||||
@@ -1178,7 +1067,7 @@ mod tests {
|
||||
Some(server_epoch)
|
||||
);
|
||||
assert!(!cache_probe_called.load(Ordering::SeqCst));
|
||||
v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
|
||||
v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch,
|
||||
revalidate_after: Instant::now(),
|
||||
});
|
||||
@@ -1197,7 +1086,8 @@ mod tests {
|
||||
|
||||
let legacy_endpoint = format!("http://legacy-{}.invalid", Uuid::new_v4());
|
||||
let legacy_entry = put_file_capability_cache_entry(&legacy_endpoint);
|
||||
legacy_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
|
||||
legacy_entry.write().await.cached =
|
||||
Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
|
||||
assert!(
|
||||
transport
|
||||
.put_file_auth_capability(&legacy_endpoint)
|
||||
@@ -1208,7 +1098,7 @@ mod tests {
|
||||
|
||||
let expired_endpoint = format!("http://expired-legacy-{}.invalid", Uuid::new_v4());
|
||||
let expired_entry = put_file_capability_cache_entry(&expired_endpoint);
|
||||
expired_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
|
||||
expired_entry.write().await.cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
|
||||
let reprobed = std::sync::atomic::AtomicBool::new(false);
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&expired_endpoint, || async {
|
||||
@@ -1459,7 +1349,7 @@ mod tests {
|
||||
};
|
||||
probe_started.notified().await;
|
||||
{
|
||||
let mut state = entry.write();
|
||||
let mut state = entry.write().await;
|
||||
state.generation = state.generation.checked_add(1).expect("test generation should advance");
|
||||
state.cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch: newer_epoch,
|
||||
@@ -1472,7 +1362,10 @@ mod tests {
|
||||
task.await.expect("stale task should finish").expect("stale probe result"),
|
||||
Some(stale_epoch)
|
||||
);
|
||||
assert_eq!(fresh_put_file_capability(entry.read().cached, Instant::now()), Some(Some(newer_epoch)));
|
||||
assert_eq!(
|
||||
fresh_put_file_capability(entry.read().await.cached, Instant::now()),
|
||||
Some(Some(newer_epoch))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1505,8 +1398,6 @@ mod tests {
|
||||
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-writer-test-secret".to_string());
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let server_epoch = Uuid::parse_str("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee").expect("server epoch");
|
||||
let endpoint = "http://node1:9000".to_string();
|
||||
let url = concat!(
|
||||
"http://node1:9000/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
@@ -1515,7 +1406,7 @@ mod tests {
|
||||
let mut sink = Vec::new();
|
||||
|
||||
{
|
||||
let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce, endpoint, server_epoch);
|
||||
let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce);
|
||||
writer.write_all(b"hello world").await.expect("body write should succeed");
|
||||
writer.shutdown().await.expect("shutdown should append auth trailer");
|
||||
let err = writer
|
||||
@@ -1533,143 +1424,6 @@ mod tests {
|
||||
assert_eq!(verified, expected_digest);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_writer_reprobes_after_server_epoch_conflict() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-epoch-conflict-test-secret".to_string());
|
||||
for status in [reqwest::StatusCode::CONFLICT, reqwest::StatusCode::BAD_REQUEST] {
|
||||
for (phase, trailer_write) in [
|
||||
(PutFileFailurePhase::Write, false),
|
||||
(PutFileFailurePhase::Write, true),
|
||||
(PutFileFailurePhase::Flush, false),
|
||||
(PutFileFailurePhase::Shutdown, false),
|
||||
] {
|
||||
let endpoint = format!("http://epoch-conflict-{}.invalid", Uuid::new_v4());
|
||||
let stale_epoch = Uuid::new_v4();
|
||||
let replacement_epoch = Uuid::new_v4();
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(stale_epoch)) })
|
||||
.await
|
||||
.expect("initial capability should resolve");
|
||||
let mut writer = PutFileAuthWriter::new(
|
||||
PutFileFailureWriter { phase, status },
|
||||
format!("{endpoint}{PUT_FILE_AUTH_STREAM_PATH}"),
|
||||
Uuid::new_v4(),
|
||||
endpoint.clone(),
|
||||
stale_epoch,
|
||||
);
|
||||
let error = match (phase, trailer_write) {
|
||||
(PutFileFailurePhase::Write, false) => writer.write_all(b"body").await,
|
||||
(PutFileFailurePhase::Flush, _) => writer.flush().await,
|
||||
_ => writer.shutdown().await,
|
||||
}
|
||||
.expect_err("injected writer error must reach the caller");
|
||||
let conflict = status == reqwest::StatusCode::CONFLICT;
|
||||
assert_eq!(io_error_has_put_file_epoch_conflict(&error), conflict);
|
||||
|
||||
let probe_called = AtomicBool::new(false);
|
||||
let resolved = resolve_put_file_auth_capability(&endpoint, || async {
|
||||
probe_called.store(true, Ordering::SeqCst);
|
||||
Ok(Some(replacement_epoch))
|
||||
})
|
||||
.await
|
||||
.expect("capability should remain usable or be reprobed");
|
||||
assert_eq!(probe_called.load(Ordering::SeqCst), conflict, "phase={phase:?}, trailer={trailer_write}");
|
||||
assert_eq!(resolved, Some(if conflict { replacement_epoch } else { stale_epoch }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_put_file_epoch_rejection_preserves_current_rejection() {
|
||||
let endpoint = format!("http://late-epoch-conflict-{}.invalid", Uuid::new_v4());
|
||||
let old_epoch = Uuid::new_v4();
|
||||
let current_epoch = Uuid::new_v4();
|
||||
let replacement_epoch = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(old_epoch)) })
|
||||
.await
|
||||
.expect("initial epoch should be cached"),
|
||||
Some(old_epoch)
|
||||
);
|
||||
reject_put_file_server_epoch(&endpoint, old_epoch);
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(current_epoch)) })
|
||||
.await
|
||||
.expect("first restart should install a new epoch"),
|
||||
Some(current_epoch)
|
||||
);
|
||||
|
||||
reject_put_file_server_epoch(&endpoint, current_epoch);
|
||||
// A writer opened before the first restart can report its 409 after
|
||||
// a newer writer has already rejected the second server incarnation.
|
||||
reject_put_file_server_epoch(&endpoint, old_epoch);
|
||||
let probe_called = AtomicBool::new(false);
|
||||
let resolved = resolve_put_file_auth_capability(&endpoint, || async {
|
||||
probe_called.store(true, Ordering::SeqCst);
|
||||
Ok(Some(replacement_epoch))
|
||||
})
|
||||
.await
|
||||
.expect("late old-epoch rejection must preserve the current rejection");
|
||||
|
||||
assert!(probe_called.load(Ordering::SeqCst), "known-rejected current epoch must be reprobed");
|
||||
assert_eq!(resolved, Some(replacement_epoch));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_epoch_rejection_is_endpoint_and_epoch_scoped() {
|
||||
let endpoint = format!("http://scoped-epoch-{}.invalid", Uuid::new_v4());
|
||||
let other_endpoint = format!("http://other-epoch-{}.invalid", Uuid::new_v4());
|
||||
let current_epoch = Uuid::new_v4();
|
||||
for endpoint in [&endpoint, &other_endpoint] {
|
||||
resolve_put_file_auth_capability(endpoint, || async { Ok(Some(current_epoch)) })
|
||||
.await
|
||||
.expect("initial epoch should resolve");
|
||||
}
|
||||
reject_put_file_server_epoch(&endpoint, Uuid::new_v4());
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { panic!("old writer must not invalidate a new epoch") })
|
||||
.await
|
||||
.expect("new epoch must remain cached"),
|
||||
Some(current_epoch)
|
||||
);
|
||||
reject_put_file_server_epoch(&endpoint, current_epoch);
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&other_endpoint, || async { panic!("another endpoint must stay cached") })
|
||||
.await
|
||||
.expect("other endpoint must remain cached"),
|
||||
Some(current_epoch)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_rejected_epoch_survives_failed_stale_and_downgrade_probes() {
|
||||
let endpoint = format!("http://rejected-probe-{}.invalid", Uuid::new_v4());
|
||||
let rejected_epoch = Uuid::new_v4();
|
||||
let replacement_epoch = Uuid::new_v4();
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
|
||||
.await
|
||||
.expect("initial epoch should resolve");
|
||||
reject_put_file_server_epoch(&endpoint, rejected_epoch);
|
||||
let failure = resolve_put_file_auth_capability(&endpoint, || async { Err(Error::other("injected probe failure")) })
|
||||
.await
|
||||
.expect_err("probe failure must be returned");
|
||||
assert!(failure.to_string().contains("injected probe failure"));
|
||||
let downgrade = resolve_put_file_auth_capability(&endpoint, || async { Ok(None) })
|
||||
.await
|
||||
.expect_err("rejection must not unpin authenticated v1");
|
||||
assert!(downgrade.to_string().contains("downgrade rejected"));
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
|
||||
.await
|
||||
.expect("a probe racing a restart can still return the old epoch");
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(replacement_epoch)) })
|
||||
.await
|
||||
.expect("same-epoch probe must not clear known rejection"),
|
||||
Some(replacement_epoch)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn walk_dir_url_encodes_disk_ref() {
|
||||
let url = build_walk_dir_url(&WalkDirStreamRequest {
|
||||
|
||||
@@ -39,8 +39,8 @@ pub use http_auth::{
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
@@ -122,14 +122,6 @@ fn control_plane_failure(op: &str, bucket: Option<&str>, error_code: Option<i32>
|
||||
if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32) {
|
||||
return Error::RemoteNotInitialized;
|
||||
}
|
||||
if error_code == Some(rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32)
|
||||
{
|
||||
return Error::InvalidArgument(
|
||||
"control-plane".to_string(),
|
||||
op.to_string(),
|
||||
error_info.unwrap_or_else(|| format!("{op}: peer rejected invalid argument without details")),
|
||||
);
|
||||
}
|
||||
match error_info {
|
||||
Some(msg) => Error::other(msg),
|
||||
None => peer_failure_without_details(op, bucket),
|
||||
@@ -733,7 +725,7 @@ impl PeerRestClient {
|
||||
/// never take it offline no matter what its message says. The substring
|
||||
/// fallback only covers failures that exist purely as text, such as the
|
||||
/// dial errors `get_client` wraps.
|
||||
pub(crate) fn is_network_like_error(err: &Error) -> bool {
|
||||
fn is_network_like_error(err: &Error) -> bool {
|
||||
if let Error::Io(io_err) = err
|
||||
&& let Some(status) = embedded_tonic_status(io_err)
|
||||
{
|
||||
@@ -2343,29 +2335,6 @@ mod tests {
|
||||
use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
|
||||
assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorUnspecified as i32, 0);
|
||||
assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorNotInitialized as i32, 1);
|
||||
assert_eq!(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn control_plane_failure_preserves_typed_invalid_argument_reason() {
|
||||
use rustfs_protos::proto_gen::node_service::ControlPlaneErrorCode;
|
||||
|
||||
let reason = "durable unresolved-entry recovery requires pool metadata V2 or V3";
|
||||
let err = control_plane_failure(
|
||||
"start_decommission",
|
||||
None,
|
||||
Some(ControlPlaneErrorCode::ControlPlaneErrorInvalidArgument as i32),
|
||||
Some(reason.to_string()),
|
||||
);
|
||||
|
||||
assert!(
|
||||
matches!(
|
||||
err,
|
||||
Error::InvalidArgument(ref scope, ref operation, ref actual_reason)
|
||||
if scope == "control-plane" && operation == "start_decommission" && actual_reason == reason
|
||||
),
|
||||
"forwarded validation failures must remain typed and actionable"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -12,28 +12,35 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::target_defaults::{amqp_kvs, kafka_kvs, mysql_kvs, nats_kvs, postgres_kvs, pulsar_kvs, redis_kvs};
|
||||
use rustfs_config::audit::AUDIT_REDIS_DEFAULT_CHANNEL;
|
||||
use rustfs_config::server_config::{KV, KVS};
|
||||
use rustfs_config::{
|
||||
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
|
||||
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
|
||||
WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT,
|
||||
WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
|
||||
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
|
||||
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
|
||||
KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
|
||||
KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
|
||||
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
|
||||
MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
|
||||
MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
|
||||
MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
|
||||
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
|
||||
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
|
||||
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
|
||||
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
|
||||
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
|
||||
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
|
||||
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
|
||||
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
|
||||
REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY,
|
||||
WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
|
||||
WEBHOOK_SKIP_TLS_VERIFY,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit webhook settings.
|
||||
///
|
||||
/// `WEBHOOK_BATCH_SIZE`/`WEBHOOK_MAX_RETRY`/`WEBHOOK_RETRY_INTERVAL`/`WEBHOOK_HTTP_TIMEOUT`
|
||||
/// exist here but not in [`crate::config::notify::DEFAULT_NOTIFY_WEBHOOK_KVS`]. This mirrors
|
||||
/// MinIO upstream: `internal/logger/config.go`'s `DefaultAuditWebhookKVS` carries the same
|
||||
/// four keys with the same defaults (`"1"`/`"0"`/`"3s"`/`"5s"`), while
|
||||
/// `internal/config/notify/parse.go`'s `DefaultWebhookKVS` (bucket event notifications) does
|
||||
/// not — the notify webhook delivery path never supported them. Not a copy/paste gap
|
||||
/// (backlog#2054).
|
||||
pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
@@ -49,7 +56,7 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KV {
|
||||
key: WEBHOOK_AUTH_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true, // Sensitive field; matches notify's webhook auth_token (backlog#2054)
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: WEBHOOK_CLIENT_CERT.to_owned(),
|
||||
@@ -111,15 +118,6 @@ pub static DEFAULT_AUDIT_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
|
||||
#[allow(clippy::declare_interior_mutable_const)]
|
||||
/// Default KVS for audit MQTT settings.
|
||||
///
|
||||
/// `MQTT_QOS`/`MQTT_KEEP_ALIVE_INTERVAL`/`MQTT_RECONNECT_INTERVAL` default to a stronger
|
||||
/// delivery posture here (`"1"`/`"60s"`/`"5s"`) than
|
||||
/// [`crate::config::notify::DEFAULT_NOTIFY_MQTT_KVS`] (`"0"`/`"0s"`/`"0s"`, which matches
|
||||
/// MinIO's own `DefaultMQTTKVS` in `internal/config/notify/parse.go` byte-for-byte). MinIO has
|
||||
/// no MQTT audit target to compare against — audit-over-MQTT is a RustFS-original addition —
|
||||
/// so this divergence cannot be checked against upstream; it is intentional (audit favors
|
||||
/// at-least-once delivery and faster reconnect over notify's opt-in defaults), not a
|
||||
/// copy/paste gap (backlog#2054).
|
||||
pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
@@ -210,18 +208,542 @@ pub static DEFAULT_AUDIT_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
// The remaining targets declare the same defaults as notify, so both sides build them from
|
||||
// `target_defaults`. Redis and mysql pass in the single default that audit and notify disagree on.
|
||||
pub static DEFAULT_AUDIT_AMQP_KVS: LazyLock<KVS> = LazyLock::new(amqp_kvs);
|
||||
pub static DEFAULT_AUDIT_AMQP_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_URL.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_EXCHANGE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_ROUTING_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_MANDATORY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_PERSISTENT.to_owned(),
|
||||
value: EnableState::On.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(nats_kvs);
|
||||
pub static DEFAULT_AUDIT_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_ADDRESS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_SUBJECT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_CREDENTIALS_FILE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_REQUIRED.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_JETSTREAM_ENABLE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
|
||||
value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(pulsar_kvs);
|
||||
pub static DEFAULT_AUDIT_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_BROKER.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TOPIC.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_AUTH_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
|
||||
value: EnableState::On.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_AUDIT_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| redis_kvs(AUDIT_REDIS_DEFAULT_CHANNEL));
|
||||
pub static DEFAULT_AUDIT_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_URL.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_CHANNEL.to_owned(),
|
||||
value: AUDIT_REDIS_DEFAULT_CHANNEL.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(),
|
||||
value: "15".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(),
|
||||
value: "3".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_MIN_RETRY_DELAY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_MAX_RETRY_DELAY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_CONNECTION_TIMEOUT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_RESPONSE_TIMEOUT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_POLICY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_ALLOW_INSECURE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_AUDIT_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(postgres_kvs);
|
||||
pub static DEFAULT_AUDIT_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_DSN_STRING.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TABLE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_FORMAT.to_owned(),
|
||||
value: "namespace".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_REQUIRED.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(kafka_kvs);
|
||||
pub static DEFAULT_AUDIT_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_BROKERS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TOPIC.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_ACKS.to_owned(),
|
||||
value: "1".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_ENABLE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_ENABLE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_MECHANISM.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_AUDIT_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| mysql_kvs("rustfs_audit_logs"));
|
||||
pub static DEFAULT_AUDIT_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_DSN_STRING.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TABLE.to_owned(),
|
||||
value: "rustfs_audit_logs".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_FORMAT.to_owned(),
|
||||
value: "access".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(),
|
||||
value: "2".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
@@ -800,22 +800,11 @@ where
|
||||
if log_error {
|
||||
error!("save_config_with_opts: err: {:?}, file: {}", err, file);
|
||||
}
|
||||
Err(map_system_metadata_write_error(err, file))
|
||||
Err(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A system metadata volume outage must remain retryable instead of being
|
||||
/// exposed as the user-facing bucket-not-found response.
|
||||
pub(crate) fn map_system_metadata_write_error(err: Error, file: &str) -> Error {
|
||||
match err {
|
||||
Error::BucketNotFound(_) | Error::VolumeNotFound => {
|
||||
Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), file.to_string())
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn new_server_config() -> Config {
|
||||
Config::new()
|
||||
}
|
||||
@@ -2372,7 +2361,6 @@ where
|
||||
scan_mode: HealScanMode::Deep,
|
||||
update_parity: false,
|
||||
no_lock: false,
|
||||
read_repair: false,
|
||||
pool: None,
|
||||
set: None,
|
||||
};
|
||||
@@ -2807,14 +2795,14 @@ mod tests {
|
||||
use super::{
|
||||
SERVER_CONFIG_LOCK, ServerConfigSnapshot, apply_dynamic_config_for_sub_sys_with, build_scalar_config_object,
|
||||
config_task_join_error, configs_semantically_equal, decode_server_config_blob, encode_server_config_blob,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, map_system_metadata_write_error,
|
||||
new_and_save_server_config, read_config, read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty,
|
||||
read_config_with_metadata, read_config_without_migrate, read_server_config_snapshot, save_config_with_opts_inner,
|
||||
save_server_config, save_server_config_snapshot, save_server_config_snapshot_with_generation,
|
||||
server_config_transaction_lock_path, should_warn_ignored_scalar_section, storage_class_kvs_mut,
|
||||
heal_config_descriptor, is_standard_object_server_config, lookup_configs, new_and_save_server_config, read_config,
|
||||
read_config_no_lock_preserve_empty_with_metadata, read_config_preserve_empty, read_config_with_metadata,
|
||||
read_config_without_migrate, read_server_config_snapshot, save_server_config, save_server_config_snapshot,
|
||||
save_server_config_snapshot_with_generation, server_config_transaction_lock_path, should_warn_ignored_scalar_section,
|
||||
storage_class_kvs_mut,
|
||||
};
|
||||
use crate::config::{audit, heal, notify, oidc, scanner};
|
||||
use crate::disk::{RUSTFS_META_BUCKET, endpoint::Endpoint};
|
||||
use crate::disk::endpoint::Endpoint;
|
||||
use crate::error::{Error, Result};
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
@@ -2846,72 +2834,6 @@ mod tests {
|
||||
assert!(rendered.contains("panicked"));
|
||||
assert!(!rendered.contains("do-not-expose-payload"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn system_metadata_volume_failures_map_to_retryable_write_errors() {
|
||||
for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
|
||||
assert_eq!(
|
||||
map_system_metadata_write_error(error, "buckets/example/.metadata.bin"),
|
||||
Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
let other = Error::other("metadata encoding failed");
|
||||
assert_eq!(map_system_metadata_write_error(other.clone(), "buckets/example/.metadata.bin"), other);
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct MetadataWriteStore {
|
||||
error: Option<Error>,
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl crate::storage_api_contracts::object::ObjectIO for MetadataWriteStore {
|
||||
type Error = Error;
|
||||
type RangeSpec = HTTPRangeSpec;
|
||||
type HeaderMap = HeaderMap;
|
||||
type ObjectOptions = ObjectOptions;
|
||||
type ObjectInfo = ObjectInfo;
|
||||
type GetObjectReader = GetObjectReader;
|
||||
type PutObjectReader = PutObjReader;
|
||||
|
||||
async fn get_object_reader(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_range: Option<Self::RangeSpec>,
|
||||
_headers: Self::HeaderMap,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> core::result::Result<Self::GetObjectReader, Self::Error> {
|
||||
Err(Error::FileNotFound)
|
||||
}
|
||||
|
||||
async fn put_object(
|
||||
&self,
|
||||
_bucket: &str,
|
||||
_object: &str,
|
||||
_data: &mut Self::PutObjectReader,
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> core::result::Result<Self::ObjectInfo, Self::Error> {
|
||||
Err(self.error.clone().expect("test store error should be configured"))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn save_config_preserves_retryable_system_volume_errors() {
|
||||
let store = Arc::new(MetadataWriteStore {
|
||||
error: Some(Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())),
|
||||
});
|
||||
let error =
|
||||
save_config_with_opts_inner(store, "buckets/example/.metadata.bin", Vec::new(), &ObjectOptions::default(), false)
|
||||
.await
|
||||
.expect_err("missing metadata volume must fail");
|
||||
|
||||
assert_eq!(
|
||||
error,
|
||||
Error::InsufficientWriteQuorum(RUSTFS_META_BUCKET.to_string(), "buckets/example/.metadata.bin".to_string())
|
||||
);
|
||||
}
|
||||
use rustfs_lock::client::LockClient;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
use rustfs_lock::{LockError, LockInfo, LockResponse, LockStats};
|
||||
|
||||
@@ -21,7 +21,6 @@ mod notify;
|
||||
mod oidc;
|
||||
mod scanner;
|
||||
pub mod storageclass;
|
||||
mod target_defaults;
|
||||
|
||||
use crate::error::Result;
|
||||
use crate::store::ECStore;
|
||||
|
||||
@@ -12,26 +12,34 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::target_defaults::{amqp_kvs, kafka_kvs, mysql_kvs, nats_kvs, postgres_kvs, pulsar_kvs, redis_kvs};
|
||||
use rustfs_config::notify::NOTIFY_REDIS_DEFAULT_CHANNEL;
|
||||
use rustfs_config::server_config::{KV, KVS};
|
||||
use rustfs_config::{
|
||||
COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY, EVENT_DEFAULT_DIR, EnableState, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD,
|
||||
MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY,
|
||||
MQTT_TLS_POLICY, MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, WEBHOOK_AUTH_TOKEN,
|
||||
WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT,
|
||||
WEBHOOK_SKIP_TLS_VERIFY,
|
||||
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
|
||||
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
|
||||
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
|
||||
KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
|
||||
KAFKA_TLS_ENABLE, KAFKA_TOPIC, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
|
||||
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TLS_CA, MQTT_TLS_CLIENT_CERT, MQTT_TLS_CLIENT_KEY, MQTT_TLS_POLICY,
|
||||
MQTT_TLS_TRUST_LEAF_AS_CA, MQTT_TOPIC, MQTT_USERNAME, MQTT_WS_PATH_ALLOWLIST, MYSQL_DSN_STRING, MYSQL_FORMAT,
|
||||
MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR, MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT,
|
||||
MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS, NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE, NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR,
|
||||
NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT, NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN,
|
||||
NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR, POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE,
|
||||
POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY, POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER,
|
||||
PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT, PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA,
|
||||
PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL, REDIS_CONNECTION_TIMEOUT,
|
||||
REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY, REDIS_PASSWORD,
|
||||
REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS, REDIS_RESPONSE_TIMEOUT,
|
||||
REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY, REDIS_TLS_POLICY, REDIS_URL,
|
||||
REDIS_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_CLIENT_CA, WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT,
|
||||
WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, WEBHOOK_SKIP_TLS_VERIFY,
|
||||
};
|
||||
use std::sync::LazyLock;
|
||||
|
||||
/// The default configuration collection of webhooks，
|
||||
/// Initialized only once during the program life cycle, enabling high-performance lazy loading.
|
||||
///
|
||||
/// This table has no `batch_size`/`max_retry`/`retry_interval`/`http_timeout` keys, unlike
|
||||
/// [`crate::config::audit::DEFAULT_AUDIT_WEBHOOK_KVS`] — matching MinIO upstream, whose
|
||||
/// `internal/config/notify/parse.go` `DefaultWebhookKVS` (bucket event notifications) also
|
||||
/// omits them while `internal/logger/config.go`'s `DefaultAuditWebhookKVS` carries them.
|
||||
/// Intentional, not a copy/paste gap (backlog#2054).
|
||||
pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
@@ -89,12 +97,6 @@ pub static DEFAULT_NOTIFY_WEBHOOK_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
});
|
||||
|
||||
/// MQTT's default configuration collection
|
||||
///
|
||||
/// `MQTT_QOS`/`MQTT_KEEP_ALIVE_INTERVAL`/`MQTT_RECONNECT_INTERVAL` default to `"0"`/`"0s"`/`"0s"`
|
||||
/// here, matching MinIO's `DefaultMQTTKVS` in `internal/config/notify/parse.go`
|
||||
/// byte-for-byte — this table is a faithful port. [`crate::config::audit::DEFAULT_AUDIT_MQTT_KVS`]
|
||||
/// uses stronger, RustFS-original defaults instead (MinIO has no MQTT audit target to compare
|
||||
/// against); that divergence is intentional, not a copy/paste gap (backlog#2054).
|
||||
pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
@@ -186,17 +188,543 @@ pub static DEFAULT_NOTIFY_MQTT_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_AMQP_KVS: LazyLock<KVS> = LazyLock::new(amqp_kvs);
|
||||
pub static DEFAULT_NOTIFY_AMQP_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_URL.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_EXCHANGE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_ROUTING_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_MANDATORY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_PERSISTENT.to_owned(),
|
||||
value: EnableState::On.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: AMQP_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(nats_kvs);
|
||||
pub static DEFAULT_NOTIFY_NATS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_ADDRESS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_SUBJECT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_CREDENTIALS_FILE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: NATS_TLS_REQUIRED.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_JETSTREAM_ENABLE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_JETSTREAM_STREAM_NAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: NATS_JETSTREAM_ACK_TIMEOUT_SECS.to_owned(),
|
||||
value: NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(pulsar_kvs);
|
||||
pub static DEFAULT_NOTIFY_PULSAR_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_BROKER.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TOPIC.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_AUTH_TOKEN.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_ALLOW_INSECURE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_TLS_HOSTNAME_VERIFICATION.to_owned(),
|
||||
value: EnableState::On.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: PULSAR_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| redis_kvs(NOTIFY_REDIS_DEFAULT_CHANNEL));
|
||||
pub static DEFAULT_NOTIFY_REDIS_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_URL.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_CHANNEL.to_owned(),
|
||||
value: NOTIFY_REDIS_DEFAULT_CHANNEL.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_KEEP_ALIVE_INTERVAL.to_owned(),
|
||||
value: "15".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_MAX_RETRY_ATTEMPTS.to_owned(),
|
||||
value: "3".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_RECONNECT_RETRY_ATTEMPTS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_MIN_RETRY_DELAY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_MAX_RETRY_DELAY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_CONNECTION_TIMEOUT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_RESPONSE_TIMEOUT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_PIPELINE_BUFFER_SIZE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_POLICY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: REDIS_TLS_ALLOW_INSECURE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(postgres_kvs);
|
||||
pub static DEFAULT_NOTIFY_POSTGRES_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_DSN_STRING.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TABLE.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_FORMAT.to_owned(),
|
||||
value: "namespace".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_REQUIRED.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: POSTGRES_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(kafka_kvs);
|
||||
pub static DEFAULT_NOTIFY_KAFKA_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_BROKERS.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TOPIC.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_ACKS.to_owned(),
|
||||
value: "1".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_ENABLE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_ENABLE.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_MECHANISM.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_USERNAME.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_SASL_PASSWORD.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: KAFKA_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
/// MySQL notification target default configuration
|
||||
pub static DEFAULT_NOTIFY_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| mysql_kvs("rustfs_events"));
|
||||
pub static DEFAULT_NOTIFY_MYSQL_KVS: LazyLock<KVS> = LazyLock::new(|| {
|
||||
KVS(vec![
|
||||
KV {
|
||||
key: ENABLE_KEY.to_owned(),
|
||||
value: EnableState::Off.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_DSN_STRING.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TABLE.to_owned(),
|
||||
value: "rustfs_events".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_FORMAT.to_owned(),
|
||||
value: "access".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TLS_CA.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TLS_CLIENT_CERT.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_TLS_CLIENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: true,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_QUEUE_DIR.to_owned(),
|
||||
value: EVENT_DEFAULT_DIR.to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_QUEUE_LIMIT.to_owned(),
|
||||
value: DEFAULT_LIMIT.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: MYSQL_MAX_OPEN_CONNECTIONS.to_owned(),
|
||||
value: "2".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
key: COMMENT_KEY.to_owned(),
|
||||
value: "".to_owned(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
])
|
||||
});
|
||||
|
||||
@@ -246,7 +246,16 @@ impl Config {
|
||||
}
|
||||
|
||||
let shard_size = shard_size as usize;
|
||||
let inline_block = self.effective_inline_block(data_shards);
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
// Use div_ceil to match the shard_file_size calculation (which also uses
|
||||
// div_ceil), avoiding a 1-byte rounding discrepancy that prevents inline
|
||||
// for objects right at the threshold.
|
||||
let inline_block = if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
};
|
||||
|
||||
if versioned {
|
||||
shard_size <= inline_block / 8
|
||||
@@ -255,27 +264,6 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the per-shard inline budget used by both write admission and
|
||||
/// legacy read fallback.
|
||||
///
|
||||
/// The default budget is scaled by the number of data shards so a wider EC
|
||||
/// layout does not silently increase the maximum inline object size. An
|
||||
/// explicitly configured `inline_block` remains a fixed per-shard limit for
|
||||
/// compatibility with deployments that opted into the historical policy.
|
||||
pub(crate) fn effective_inline_block(&self, data_shards: usize) -> usize {
|
||||
if data_shards == 0 {
|
||||
return 0;
|
||||
}
|
||||
|
||||
if self.initialized && self.inline_block_explicit {
|
||||
self.inline_block
|
||||
} else {
|
||||
// Keep the historical two-data-shard object budget while preventing
|
||||
// wider EC layouts from multiplying the maximum inline object size.
|
||||
DEFAULT_INLINE_OBJECT_BUDGET.div_ceil(data_shards).min(DEFAULT_INLINE_BLOCK)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn inline_block(&self) -> usize {
|
||||
if !self.initialized {
|
||||
DEFAULT_INLINE_BLOCK
|
||||
@@ -614,51 +602,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn should_inline_keeps_ec8_and_ec12_object_boundaries_consistent() {
|
||||
let config = Config::default();
|
||||
let object_sizes = [128 * 1024, 256 * 1024, 512 * 1024, 1024 * 1024, 4 * 1024 * 1024];
|
||||
|
||||
for (data_shards, parity_shards) in [(8, 4), (12, 4)] {
|
||||
let erasure = crate::erasure::coding::Erasure::new(data_shards, parity_shards, 1024 * 1024);
|
||||
let mut previous = true;
|
||||
for object_size in object_sizes {
|
||||
let shard_size = erasure.shard_file_size(object_size);
|
||||
let inline = config.should_inline(shard_size, data_shards, false);
|
||||
|
||||
// The effective policy is monotonic across object sizes. This
|
||||
// table covers the boundaries that previously exposed the
|
||||
// fixed-shard read-ahead mismatch, including the 1 MiB case.
|
||||
assert!(!inline || previous, "inline decision must not re-enable at {object_size} bytes");
|
||||
previous = inline;
|
||||
}
|
||||
|
||||
assert!(
|
||||
!config.should_inline(erasure.shard_file_size(1024 * 1024), data_shards, false),
|
||||
"1 MiB must use the non-inline path for EC{data_shards}+{parity_shards}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn effective_inline_block_scales_default_budget_and_preserves_explicit_limit() {
|
||||
let config = Config::default();
|
||||
assert_eq!(config.effective_inline_block(8), 32 * 1024);
|
||||
assert_eq!(config.effective_inline_block(12), 21_846);
|
||||
assert_eq!(config.effective_inline_block(0), 0);
|
||||
|
||||
let explicit = lookup_config_for_pools_with_env(
|
||||
&KVS::new(),
|
||||
&[12],
|
||||
StorageClassEnvOverrides {
|
||||
inline_block: Some("128KiB".to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("explicit inline block should resolve");
|
||||
assert_eq!(explicit.effective_inline_block(12), 128 * 1024);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_inline_block_preserves_fixed_per_shard_rollback() {
|
||||
let overrides = StorageClassEnvOverrides {
|
||||
|
||||
@@ -1,414 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Shared default KVS tables for delivery targets that audit and notify declare identically.
|
||||
//!
|
||||
//! The audit and notify subsystems register one default KVS per delivery target. For amqp, nats,
|
||||
//! pulsar, postgres and kafka both sides declare byte-identical tables; for redis and mysql they
|
||||
//! differ only in a single default literal, which the caller passes in.
|
||||
//!
|
||||
//! Webhook and mqtt are deliberately absent: audit's webhook table carries extra batching/retry
|
||||
//! keys and both tables disagree on key order and on several defaults (mqtt qos, keep-alive and
|
||||
//! reconnect intervals), so they are real behavioral forks, not duplication.
|
||||
//!
|
||||
//! Key order is part of the contract: it drives the order admin config output lists the keys in,
|
||||
//! so every constructor reproduces the existing order exactly.
|
||||
|
||||
use rustfs_config::server_config::{KV, KVS};
|
||||
use rustfs_config::{
|
||||
AMQP_EXCHANGE, AMQP_MANDATORY, AMQP_PASSWORD, AMQP_PERSISTENT, AMQP_QUEUE_DIR, AMQP_QUEUE_LIMIT, AMQP_ROUTING_KEY,
|
||||
AMQP_TLS_CA, AMQP_TLS_CLIENT_CERT, AMQP_TLS_CLIENT_KEY, AMQP_URL, AMQP_USERNAME, COMMENT_KEY, DEFAULT_LIMIT, ENABLE_KEY,
|
||||
EVENT_DEFAULT_DIR, EnableState, KAFKA_ACKS, KAFKA_BROKERS, KAFKA_QUEUE_DIR, KAFKA_QUEUE_LIMIT, KAFKA_SASL_ENABLE,
|
||||
KAFKA_SASL_MECHANISM, KAFKA_SASL_PASSWORD, KAFKA_SASL_USERNAME, KAFKA_TLS_CA, KAFKA_TLS_CLIENT_CERT, KAFKA_TLS_CLIENT_KEY,
|
||||
KAFKA_TLS_ENABLE, KAFKA_TOPIC, MYSQL_DSN_STRING, MYSQL_FORMAT, MYSQL_MAX_OPEN_CONNECTIONS, MYSQL_QUEUE_DIR,
|
||||
MYSQL_QUEUE_LIMIT, MYSQL_TABLE, MYSQL_TLS_CA, MYSQL_TLS_CLIENT_CERT, MYSQL_TLS_CLIENT_KEY, NATS_ADDRESS,
|
||||
NATS_CREDENTIALS_FILE, NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS, NATS_JETSTREAM_ACK_TIMEOUT_SECS, NATS_JETSTREAM_ENABLE,
|
||||
NATS_JETSTREAM_STREAM_NAME, NATS_PASSWORD, NATS_QUEUE_DIR, NATS_QUEUE_LIMIT, NATS_SUBJECT, NATS_TLS_CA, NATS_TLS_CLIENT_CERT,
|
||||
NATS_TLS_CLIENT_KEY, NATS_TLS_REQUIRED, NATS_TOKEN, NATS_USERNAME, POSTGRES_DSN_STRING, POSTGRES_FORMAT, POSTGRES_QUEUE_DIR,
|
||||
POSTGRES_QUEUE_LIMIT, POSTGRES_TABLE, POSTGRES_TLS_CA, POSTGRES_TLS_CLIENT_CERT, POSTGRES_TLS_CLIENT_KEY,
|
||||
POSTGRES_TLS_REQUIRED, PULSAR_AUTH_TOKEN, PULSAR_BROKER, PULSAR_PASSWORD, PULSAR_QUEUE_DIR, PULSAR_QUEUE_LIMIT,
|
||||
PULSAR_TLS_ALLOW_INSECURE, PULSAR_TLS_CA, PULSAR_TLS_HOSTNAME_VERIFICATION, PULSAR_TOPIC, PULSAR_USERNAME, REDIS_CHANNEL,
|
||||
REDIS_CONNECTION_TIMEOUT, REDIS_KEEP_ALIVE_INTERVAL, REDIS_MAX_RETRY_ATTEMPTS, REDIS_MAX_RETRY_DELAY, REDIS_MIN_RETRY_DELAY,
|
||||
REDIS_PASSWORD, REDIS_PIPELINE_BUFFER_SIZE, REDIS_QUEUE_DIR, REDIS_QUEUE_LIMIT, REDIS_RECONNECT_RETRY_ATTEMPTS,
|
||||
REDIS_RESPONSE_TIMEOUT, REDIS_TLS_ALLOW_INSECURE, REDIS_TLS_CA, REDIS_TLS_CLIENT_CERT, REDIS_TLS_CLIENT_KEY,
|
||||
REDIS_TLS_POLICY, REDIS_URL, REDIS_USERNAME,
|
||||
};
|
||||
|
||||
/// Builds one default entry. `hidden_if_empty` marks values the admin API elides when unset.
|
||||
fn kv(key: &str, value: impl Into<String>, hidden_if_empty: bool) -> KV {
|
||||
KV {
|
||||
key: key.to_owned(),
|
||||
value: value.into(),
|
||||
hidden_if_empty,
|
||||
}
|
||||
}
|
||||
|
||||
/// Default KVS for the amqp delivery target.
|
||||
// Unused until the audit/notify tables are migrated onto these constructors.
|
||||
#[allow(dead_code)]
|
||||
pub fn amqp_kvs() -> KVS {
|
||||
KVS(vec![
|
||||
kv(ENABLE_KEY, EnableState::Off.to_string(), false),
|
||||
kv(AMQP_URL, "", false),
|
||||
kv(AMQP_EXCHANGE, "", false),
|
||||
kv(AMQP_ROUTING_KEY, "", false),
|
||||
kv(AMQP_MANDATORY, EnableState::Off.to_string(), false),
|
||||
kv(AMQP_PERSISTENT, EnableState::On.to_string(), false),
|
||||
kv(AMQP_USERNAME, "", false),
|
||||
kv(AMQP_PASSWORD, "", true),
|
||||
kv(AMQP_TLS_CA, "", true),
|
||||
kv(AMQP_TLS_CLIENT_CERT, "", true),
|
||||
kv(AMQP_TLS_CLIENT_KEY, "", true),
|
||||
kv(AMQP_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
|
||||
kv(AMQP_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
|
||||
kv(COMMENT_KEY, "", false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Default KVS for the nats delivery target.
|
||||
// Unused until the audit/notify tables are migrated onto these constructors.
|
||||
#[allow(dead_code)]
|
||||
pub fn nats_kvs() -> KVS {
|
||||
KVS(vec![
|
||||
kv(ENABLE_KEY, EnableState::Off.to_string(), false),
|
||||
kv(NATS_ADDRESS, "", false),
|
||||
kv(NATS_SUBJECT, "", false),
|
||||
kv(NATS_USERNAME, "", false),
|
||||
kv(NATS_PASSWORD, "", true),
|
||||
kv(NATS_TOKEN, "", true),
|
||||
kv(NATS_CREDENTIALS_FILE, "", true),
|
||||
kv(NATS_TLS_CA, "", true),
|
||||
kv(NATS_TLS_CLIENT_CERT, "", true),
|
||||
kv(NATS_TLS_CLIENT_KEY, "", true),
|
||||
kv(NATS_TLS_REQUIRED, EnableState::Off.to_string(), false),
|
||||
kv(NATS_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
|
||||
kv(NATS_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
|
||||
kv(NATS_JETSTREAM_ENABLE, EnableState::Off.to_string(), false),
|
||||
kv(NATS_JETSTREAM_STREAM_NAME, "", false),
|
||||
kv(
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_SECS,
|
||||
NATS_JETSTREAM_ACK_TIMEOUT_DEFAULT_SECS.to_string(),
|
||||
false,
|
||||
),
|
||||
kv(COMMENT_KEY, "", false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Default KVS for the pulsar delivery target.
|
||||
// Unused until the audit/notify tables are migrated onto these constructors.
|
||||
#[allow(dead_code)]
|
||||
pub fn pulsar_kvs() -> KVS {
|
||||
KVS(vec![
|
||||
kv(ENABLE_KEY, EnableState::Off.to_string(), false),
|
||||
kv(PULSAR_BROKER, "", false),
|
||||
kv(PULSAR_TOPIC, "", false),
|
||||
kv(PULSAR_AUTH_TOKEN, "", true),
|
||||
kv(PULSAR_USERNAME, "", false),
|
||||
kv(PULSAR_PASSWORD, "", true),
|
||||
kv(PULSAR_TLS_CA, "", true),
|
||||
kv(PULSAR_TLS_ALLOW_INSECURE, EnableState::Off.to_string(), false),
|
||||
kv(PULSAR_TLS_HOSTNAME_VERIFICATION, EnableState::On.to_string(), false),
|
||||
kv(PULSAR_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
|
||||
kv(PULSAR_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
|
||||
kv(COMMENT_KEY, "", false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Default KVS for the postgres delivery target.
|
||||
// Unused until the audit/notify tables are migrated onto these constructors.
|
||||
#[allow(dead_code)]
|
||||
pub fn postgres_kvs() -> KVS {
|
||||
KVS(vec![
|
||||
kv(ENABLE_KEY, EnableState::Off.to_string(), false),
|
||||
kv(POSTGRES_DSN_STRING, "", true),
|
||||
kv(POSTGRES_TABLE, "", false),
|
||||
kv(POSTGRES_FORMAT, "namespace", false),
|
||||
kv(POSTGRES_TLS_REQUIRED, EnableState::Off.to_string(), false),
|
||||
kv(POSTGRES_TLS_CA, "", true),
|
||||
kv(POSTGRES_TLS_CLIENT_CERT, "", true),
|
||||
kv(POSTGRES_TLS_CLIENT_KEY, "", true),
|
||||
kv(POSTGRES_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
|
||||
kv(POSTGRES_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
|
||||
kv(COMMENT_KEY, "", false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Default KVS for the kafka delivery target.
|
||||
// Unused until the audit/notify tables are migrated onto these constructors.
|
||||
#[allow(dead_code)]
|
||||
pub fn kafka_kvs() -> KVS {
|
||||
KVS(vec![
|
||||
kv(ENABLE_KEY, EnableState::Off.to_string(), false),
|
||||
kv(KAFKA_BROKERS, "", false),
|
||||
kv(KAFKA_TOPIC, "", false),
|
||||
kv(KAFKA_ACKS, "1", false),
|
||||
kv(KAFKA_TLS_ENABLE, EnableState::Off.to_string(), false),
|
||||
kv(KAFKA_TLS_CA, "", true),
|
||||
kv(KAFKA_TLS_CLIENT_CERT, "", true),
|
||||
kv(KAFKA_TLS_CLIENT_KEY, "", true),
|
||||
kv(KAFKA_SASL_ENABLE, EnableState::Off.to_string(), false),
|
||||
kv(KAFKA_SASL_MECHANISM, "", false),
|
||||
kv(KAFKA_SASL_USERNAME, "", false),
|
||||
kv(KAFKA_SASL_PASSWORD, "", true),
|
||||
kv(KAFKA_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
|
||||
kv(KAFKA_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
|
||||
kv(COMMENT_KEY, "", false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Default KVS for the redis delivery target. `channel` is the subsystem's default pub/sub channel,
|
||||
/// which is the only value audit and notify disagree on.
|
||||
// Unused until the audit/notify tables are migrated onto these constructors.
|
||||
#[allow(dead_code)]
|
||||
pub fn redis_kvs(channel: &str) -> KVS {
|
||||
KVS(vec![
|
||||
kv(ENABLE_KEY, EnableState::Off.to_string(), false),
|
||||
kv(REDIS_URL, "", false),
|
||||
kv(REDIS_CHANNEL, channel, false),
|
||||
kv(REDIS_USERNAME, "", false),
|
||||
kv(REDIS_PASSWORD, "", true),
|
||||
kv(REDIS_KEEP_ALIVE_INTERVAL, "15", false),
|
||||
kv(REDIS_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
|
||||
kv(REDIS_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
|
||||
kv(REDIS_MAX_RETRY_ATTEMPTS, "3", false),
|
||||
kv(REDIS_RECONNECT_RETRY_ATTEMPTS, "", false),
|
||||
kv(REDIS_MIN_RETRY_DELAY, "", false),
|
||||
kv(REDIS_MAX_RETRY_DELAY, "", false),
|
||||
kv(REDIS_CONNECTION_TIMEOUT, "", false),
|
||||
kv(REDIS_RESPONSE_TIMEOUT, "", false),
|
||||
kv(REDIS_PIPELINE_BUFFER_SIZE, "", false),
|
||||
kv(REDIS_TLS_POLICY, "", true),
|
||||
kv(REDIS_TLS_CA, "", true),
|
||||
kv(REDIS_TLS_CLIENT_CERT, "", true),
|
||||
kv(REDIS_TLS_CLIENT_KEY, "", true),
|
||||
kv(REDIS_TLS_ALLOW_INSECURE, EnableState::Off.to_string(), false),
|
||||
kv(COMMENT_KEY, "", false),
|
||||
])
|
||||
}
|
||||
|
||||
/// Default KVS for the mysql delivery target. `table` is the subsystem's default destination table,
|
||||
/// which is the only value audit and notify disagree on.
|
||||
// Unused until the audit/notify tables are migrated onto these constructors.
|
||||
#[allow(dead_code)]
|
||||
pub fn mysql_kvs(table: &str) -> KVS {
|
||||
KVS(vec![
|
||||
kv(ENABLE_KEY, EnableState::Off.to_string(), false),
|
||||
kv(MYSQL_DSN_STRING, "", true),
|
||||
kv(MYSQL_TABLE, table, false),
|
||||
kv(MYSQL_FORMAT, "access", false),
|
||||
kv(MYSQL_TLS_CA, "", true),
|
||||
kv(MYSQL_TLS_CLIENT_CERT, "", true),
|
||||
kv(MYSQL_TLS_CLIENT_KEY, "", true),
|
||||
kv(MYSQL_QUEUE_DIR, EVENT_DEFAULT_DIR, false),
|
||||
kv(MYSQL_QUEUE_LIMIT, DEFAULT_LIMIT.to_string(), false),
|
||||
kv(MYSQL_MAX_OPEN_CONNECTIONS, "2", false),
|
||||
kv(COMMENT_KEY, "", false),
|
||||
])
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
/// Expected values are spelled out as literals on purpose: they mirror the tables currently
|
||||
/// declared in `audit.rs` and `notify.rs`, so a drift in key order or in any default breaks
|
||||
/// the test instead of silently changing admin config output.
|
||||
fn assert_table(actual: &KVS, expected: &[(&str, &str, bool)]) {
|
||||
let actual: Vec<(&str, &str, bool)> = actual
|
||||
.0
|
||||
.iter()
|
||||
.map(|kv| (kv.key.as_str(), kv.value.as_str(), kv.hidden_if_empty))
|
||||
.collect();
|
||||
assert_eq!(actual, expected);
|
||||
}
|
||||
|
||||
const QUEUE_DIR: &str = "/opt/rustfs/events";
|
||||
const QUEUE_LIMIT: &str = "100000";
|
||||
|
||||
#[test]
|
||||
fn amqp_table_matches_audit_and_notify() {
|
||||
assert_table(
|
||||
&amqp_kvs(),
|
||||
&[
|
||||
("enable", "off", false),
|
||||
("url", "", false),
|
||||
("exchange", "", false),
|
||||
("routing_key", "", false),
|
||||
("mandatory", "off", false),
|
||||
("persistent", "on", false),
|
||||
("username", "", false),
|
||||
("password", "", true),
|
||||
("tls_ca", "", true),
|
||||
("tls_client_cert", "", true),
|
||||
("tls_client_key", "", true),
|
||||
("queue_dir", QUEUE_DIR, false),
|
||||
("queue_limit", QUEUE_LIMIT, false),
|
||||
("comment", "", false),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nats_table_matches_audit_and_notify() {
|
||||
assert_table(
|
||||
&nats_kvs(),
|
||||
&[
|
||||
("enable", "off", false),
|
||||
("address", "", false),
|
||||
("subject", "", false),
|
||||
("username", "", false),
|
||||
("password", "", true),
|
||||
("token", "", true),
|
||||
("credentials_file", "", true),
|
||||
("tls_ca", "", true),
|
||||
("tls_client_cert", "", true),
|
||||
("tls_client_key", "", true),
|
||||
("tls_required", "off", false),
|
||||
("queue_dir", QUEUE_DIR, false),
|
||||
("queue_limit", QUEUE_LIMIT, false),
|
||||
("jetstream_enable", "off", false),
|
||||
("jetstream_stream_name", "", false),
|
||||
("jetstream_ack_timeout_secs", "30", false),
|
||||
("comment", "", false),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pulsar_table_matches_audit_and_notify() {
|
||||
assert_table(
|
||||
&pulsar_kvs(),
|
||||
&[
|
||||
("enable", "off", false),
|
||||
("broker", "", false),
|
||||
("topic", "", false),
|
||||
("auth_token", "", true),
|
||||
("username", "", false),
|
||||
("password", "", true),
|
||||
("tls_ca", "", true),
|
||||
("tls_allow_insecure", "off", false),
|
||||
("tls_hostname_verification", "on", false),
|
||||
("queue_dir", QUEUE_DIR, false),
|
||||
("queue_limit", QUEUE_LIMIT, false),
|
||||
("comment", "", false),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn postgres_table_matches_audit_and_notify() {
|
||||
assert_table(
|
||||
&postgres_kvs(),
|
||||
&[
|
||||
("enable", "off", false),
|
||||
("dsn_string", "", true),
|
||||
("table", "", false),
|
||||
("format", "namespace", false),
|
||||
("tls_required", "off", false),
|
||||
("tls_ca", "", true),
|
||||
("tls_client_cert", "", true),
|
||||
("tls_client_key", "", true),
|
||||
("queue_dir", QUEUE_DIR, false),
|
||||
("queue_limit", QUEUE_LIMIT, false),
|
||||
("comment", "", false),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn kafka_table_matches_audit_and_notify() {
|
||||
assert_table(
|
||||
&kafka_kvs(),
|
||||
&[
|
||||
("enable", "off", false),
|
||||
("brokers", "", false),
|
||||
("topic", "", false),
|
||||
("acks", "1", false),
|
||||
("tls_enable", "off", false),
|
||||
("tls_ca", "", true),
|
||||
("tls_client_cert", "", true),
|
||||
("tls_client_key", "", true),
|
||||
("sasl_enable", "off", false),
|
||||
("sasl_mechanism", "", false),
|
||||
("sasl_username", "", false),
|
||||
("sasl_password", "", true),
|
||||
("queue_dir", QUEUE_DIR, false),
|
||||
("queue_limit", QUEUE_LIMIT, false),
|
||||
("comment", "", false),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
fn expected_redis(channel: &str) -> Vec<(&str, &str, bool)> {
|
||||
vec![
|
||||
("enable", "off", false),
|
||||
("url", "", false),
|
||||
("channel", channel, false),
|
||||
("username", "", false),
|
||||
("password", "", true),
|
||||
("keep_alive_interval", "15", false),
|
||||
("queue_dir", QUEUE_DIR, false),
|
||||
("queue_limit", QUEUE_LIMIT, false),
|
||||
("max_retry_attempts", "3", false),
|
||||
("reconnect_retry_attempts", "", false),
|
||||
("min_retry_delay", "", false),
|
||||
("max_retry_delay", "", false),
|
||||
("connection_timeout", "", false),
|
||||
("response_timeout", "", false),
|
||||
("pipeline_buffer_size", "", false),
|
||||
("tls_policy", "", true),
|
||||
("tls_ca", "", true),
|
||||
("tls_client_cert", "", true),
|
||||
("tls_client_key", "", true),
|
||||
("tls_allow_insecure", "off", false),
|
||||
("comment", "", false),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redis_table_matches_audit() {
|
||||
assert_table(&redis_kvs("rustfs_audit_channel"), &expected_redis("rustfs_audit_channel"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn redis_table_matches_notify() {
|
||||
assert_table(&redis_kvs("rustfs_notify_channel"), &expected_redis("rustfs_notify_channel"));
|
||||
}
|
||||
|
||||
fn expected_mysql(table: &str) -> Vec<(&str, &str, bool)> {
|
||||
vec![
|
||||
("enable", "off", false),
|
||||
("dsn_string", "", true),
|
||||
("table", table, false),
|
||||
("format", "access", false),
|
||||
("tls_ca", "", true),
|
||||
("tls_client_cert", "", true),
|
||||
("tls_client_key", "", true),
|
||||
("queue_dir", QUEUE_DIR, false),
|
||||
("queue_limit", QUEUE_LIMIT, false),
|
||||
("max_open_connections", "2", false),
|
||||
("comment", "", false),
|
||||
]
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_table_matches_audit() {
|
||||
assert_table(&mysql_kvs("rustfs_audit_logs"), &expected_mysql("rustfs_audit_logs"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mysql_table_matches_notify() {
|
||||
assert_table(&mysql_kvs("rustfs_events"), &expected_mysql("rustfs_events"));
|
||||
}
|
||||
}
|
||||
+402
-5118
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -781,7 +781,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for Sets {
|
||||
.await
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self, opts))]
|
||||
#[tracing::instrument(skip(self))]
|
||||
async fn delete_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> Result<ObjectInfo> {
|
||||
if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
self.delete_prefix(bucket, object, &opts).await?;
|
||||
@@ -1351,20 +1351,11 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>)
|
||||
pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
|
||||
ctx: Arc<InstanceContext>,
|
||||
pool_idx: usize,
|
||||
) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
make_local_two_set_sets_for_pool_with_drive_count_and_ctx(ctx, pool_idx, 2).await
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
ctx: Arc<InstanceContext>,
|
||||
pool_idx: usize,
|
||||
set_drive_count: usize,
|
||||
) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
|
||||
let format = FormatV3::new(2, set_drive_count);
|
||||
let format = FormatV3::new(2, 2);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut all_endpoints = Vec::new();
|
||||
let mut disk_sets = Vec::new();
|
||||
@@ -1372,7 +1363,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
for set_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..set_drive_count {
|
||||
for disk_index in 0..2 {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
|
||||
.expect("endpoint should parse");
|
||||
@@ -1398,7 +1389,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let lockers = (0..set_drive_count)
|
||||
let lockers = (0..2)
|
||||
.map(|_| {
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
@@ -1409,7 +1400,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
set_drive_count,
|
||||
2,
|
||||
1,
|
||||
set_index,
|
||||
pool_idx,
|
||||
@@ -1429,7 +1420,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
endpoints: PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: set_drive_count,
|
||||
drives_per_set: 2,
|
||||
endpoints: Endpoints::from(all_endpoints),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
@@ -1437,7 +1428,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
format,
|
||||
parity_count: 1,
|
||||
set_count: 2,
|
||||
set_drive_count,
|
||||
set_drive_count: 2,
|
||||
default_parity_count: 1,
|
||||
distribution_algo: DistributionAlgoVersion::V1,
|
||||
exit_signal: None,
|
||||
|
||||
@@ -15,8 +15,7 @@
|
||||
use crate::error::{Error, Result};
|
||||
use crate::runtime::sources::{self as runtime_sources, WorkloadSnapshotProviderRef};
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_concurrency::WorkloadAdmissionSnapshotProvider;
|
||||
use rustfs_concurrency::workload::ForegroundPressure;
|
||||
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionSnapshotProvider, WorkloadClass};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::time::sleep;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
@@ -138,6 +137,23 @@ async fn wait_for_data_movement_admission_with_provider(
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
struct ForegroundPressure {
|
||||
class: WorkloadClass,
|
||||
usage_pct: usize,
|
||||
threshold_pct: usize,
|
||||
}
|
||||
|
||||
impl ForegroundPressure {
|
||||
const fn reason(self) -> &'static str {
|
||||
match self.class {
|
||||
WorkloadClass::ForegroundRead => "foreground_read_pressure",
|
||||
WorkloadClass::ForegroundWrite => "foreground_write_pressure",
|
||||
_ => "foreground_pressure",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn foreground_pressure(
|
||||
config: &DataMovementBackpressureConfig,
|
||||
provider: Option<&(dyn WorkloadAdmissionSnapshotProvider + Send + Sync)>,
|
||||
@@ -147,11 +163,39 @@ fn foreground_pressure(
|
||||
}
|
||||
|
||||
let snapshot = provider?.workload_admission_snapshot();
|
||||
rustfs_concurrency::workload::foreground_pressure(
|
||||
&snapshot,
|
||||
config.foreground_read_high_percent,
|
||||
config.foreground_write_high_percent,
|
||||
)
|
||||
[
|
||||
(WorkloadClass::ForegroundRead, config.foreground_read_high_percent),
|
||||
(WorkloadClass::ForegroundWrite, config.foreground_write_high_percent),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(class, threshold_pct)| {
|
||||
if threshold_pct == 0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let entry = snapshot.get(class)?;
|
||||
let usage_pct = if matches!(entry.state, AdmissionState::Saturated) {
|
||||
100
|
||||
} else {
|
||||
let limit = entry.limit?;
|
||||
if limit == 0 {
|
||||
return None;
|
||||
}
|
||||
entry
|
||||
.active
|
||||
.unwrap_or(0)
|
||||
.saturating_mul(100)
|
||||
.checked_div(limit)
|
||||
.unwrap_or(100)
|
||||
};
|
||||
|
||||
(usage_pct >= threshold_pct).then_some(ForegroundPressure {
|
||||
class,
|
||||
usage_pct,
|
||||
threshold_pct,
|
||||
})
|
||||
})
|
||||
.max_by_key(|pressure| pressure.usage_pct)
|
||||
}
|
||||
|
||||
fn record_delay_start(
|
||||
@@ -232,7 +276,7 @@ fn record_delay_completion(
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_concurrency::{AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadClass};
|
||||
use rustfs_concurrency::{WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot};
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -16,14 +16,13 @@
|
||||
|
||||
pub(crate) mod backpressure;
|
||||
|
||||
use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id};
|
||||
use crate::error::{
|
||||
Error, Result, is_err_data_movement_overwrite, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::{
|
||||
multipart::CompletePart,
|
||||
multipart::{CompletePart, MultipartOperations as _},
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
@@ -161,99 +160,6 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
|
||||
flag.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DataMovementMultipartAbortBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct DataMovementMultipartAbortBarrier {
|
||||
state: Arc<DataMovementMultipartAbortBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl DataMovementMultipartAbortBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(DataMovementMultipartAbortBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("data movement multipart abort barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "data movement multipart abort barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("data movement multipart failure should reach abort cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for DataMovementMultipartAbortBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("data movement multipart abort barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
|
||||
let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("data movement multipart abort barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn data_movement_abort_opts(
|
||||
src_pool_idx: usize,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
lock_lost_signal: Option<&Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
) -> ObjectOptions {
|
||||
let mut opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx,
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut opts);
|
||||
}
|
||||
if let Some(signal) = lock_lost_signal {
|
||||
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
opts
|
||||
}
|
||||
|
||||
fn insert_data_movement_checksum(user_defined: &mut HashMap<String, String>, object_info: &ObjectInfo) {
|
||||
rustfs_utils::http::remove_header_map(user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC);
|
||||
if let Some(checksum) = object_info.checksum.as_ref().filter(|checksum| !checksum.is_empty()) {
|
||||
@@ -286,7 +192,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -457,7 +363,7 @@ fn data_movement_complete_multipart_opts(
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
user_defined,
|
||||
src_pool_idx,
|
||||
..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
|
||||
..Default::default()
|
||||
})
|
||||
}
|
||||
|
||||
@@ -627,7 +533,6 @@ fn schedule_data_movement_multipart_abort_cleanup(
|
||||
bucket: String,
|
||||
object: String,
|
||||
upload_id: String,
|
||||
opts: ObjectOptions,
|
||||
op_label: &str,
|
||||
) {
|
||||
let op_label = op_label.to_string();
|
||||
@@ -635,32 +540,23 @@ fn schedule_data_movement_multipart_abort_cleanup(
|
||||
for attempt in 1..=DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS {
|
||||
tokio::time::sleep(StdDuration::from_secs(DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS)).await;
|
||||
|
||||
if store.pools.get(target_pool_idx).is_none() {
|
||||
let Some(pool) = store.pools.get(target_pool_idx).cloned() else {
|
||||
error!(
|
||||
"{op_label}: background abort_multipart_upload cleanup skipped for {bucket}/{object} upload {upload_id}: target pool {target_pool_idx} is out of range"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut cleanup_opts = opts.clone();
|
||||
let _multipart_mutation_fence = match DecommissionCapacityOwner::from_options(&cleanup_opts) {
|
||||
Some(owner) => match store.acquire_decommission_multipart_mutation_fence(owner).await {
|
||||
Ok(fence) => {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
Some(fence)
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
"{op_label}: background abort_multipart_upload cleanup could not fence {bucket}/{object} upload {upload_id} on attempt {attempt}: {err:?}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
|
||||
match store
|
||||
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object, &upload_id, &cleanup_opts)
|
||||
match pool
|
||||
.abort_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload_id,
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
@@ -1438,43 +1334,27 @@ fn resolve_data_movement_overwrite_resume_result_for(
|
||||
Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct DataMovementOverwriteCapacity {
|
||||
owner: Option<DecommissionCapacityOwner>,
|
||||
expected_data_bytes: Option<usize>,
|
||||
}
|
||||
|
||||
async fn should_treat_data_movement_overwrite_as_complete(
|
||||
store: &ECStore,
|
||||
pool_indices: (usize, usize),
|
||||
src_pool_idx: usize,
|
||||
target_pool_idx: usize,
|
||||
bucket: &str,
|
||||
object_info: &ObjectInfo,
|
||||
err: &Error,
|
||||
compare_part_checksums: bool,
|
||||
capacity: DataMovementOverwriteCapacity,
|
||||
) -> Result<bool> {
|
||||
if !should_check_data_movement_overwrite_resume(err) {
|
||||
return Ok(false);
|
||||
}
|
||||
let (src_pool_idx, target_pool_idx) = pool_indices;
|
||||
|
||||
let equivalent = resolve_data_movement_overwrite_resume_result_for(
|
||||
resolve_data_movement_overwrite_resume_result_for(
|
||||
err,
|
||||
find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await,
|
||||
object_info,
|
||||
src_pool_idx,
|
||||
target_pool_idx,
|
||||
compare_part_checksums,
|
||||
)?;
|
||||
if equivalent && let Some(owner) = capacity.owner {
|
||||
let expected_data_bytes = capacity
|
||||
.expected_data_bytes
|
||||
.ok_or_else(|| Error::other("equivalent data-movement target cannot reconcile unknown committed data size"))?;
|
||||
store
|
||||
.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
|
||||
.await?;
|
||||
}
|
||||
Ok(equivalent)
|
||||
)
|
||||
}
|
||||
|
||||
fn data_movement_part_stage_error(
|
||||
@@ -1515,7 +1395,6 @@ pub(crate) async fn migrate_decommission_object(
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
) -> Result<()> {
|
||||
let source = rd.object_info.clone();
|
||||
let _mutation_fence = store
|
||||
@@ -1536,7 +1415,6 @@ pub(crate) async fn migrate_decommission_object(
|
||||
source_bucket_incarnation_id,
|
||||
op_label,
|
||||
None,
|
||||
capacity_owner,
|
||||
Some(&_mutation_fence),
|
||||
)
|
||||
.await
|
||||
@@ -1573,7 +1451,6 @@ pub(crate) async fn migrate_object_with_lock_lost_signal(
|
||||
op_label,
|
||||
lock_lost_signal,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1587,102 +1464,22 @@ async fn migrate_object_inner(
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<()> {
|
||||
let object_info = rd.object_info.clone();
|
||||
let capacity_owner = capacity_owner.map(|owner| {
|
||||
let version_id = object_info.version_id.map(|version_id| version_id.to_string());
|
||||
let mutation_id = owner.mutation_id.unwrap_or_else(|| {
|
||||
decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
version_id.as_deref(),
|
||||
object_info.delete_marker,
|
||||
object_info.mod_time,
|
||||
)
|
||||
});
|
||||
owner.with_mutation_id(mutation_id)
|
||||
});
|
||||
let has_part_checksums = object_info
|
||||
.parts
|
||||
.iter()
|
||||
.any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty()));
|
||||
|
||||
let preserve_part_checksums = data_movement_part_checksum_writer_enabled();
|
||||
let capacity_expected_data_bytes = usize::try_from(object_info.size).ok();
|
||||
|
||||
if should_use_multipart_data_movement(&object_info, has_part_checksums) {
|
||||
// The decommission object fence already covers the source/target
|
||||
// namespace for this migration. Acquiring the synthetic multipart
|
||||
// fence while holding that read lock deadlocks local lock domains;
|
||||
// retain the extra fence only for callers without the outer fence.
|
||||
let multipart_mutation_fence = match (capacity_owner, mutation_fence.is_some()) {
|
||||
(Some(owner), false) => Some(store.acquire_decommission_multipart_mutation_fence(owner).await?),
|
||||
_ => None,
|
||||
};
|
||||
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut new_multipart_opts);
|
||||
}
|
||||
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
new_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut new_multipart_opts);
|
||||
}
|
||||
if let Some(owner) = capacity_owner {
|
||||
let existing_target_pool_idx = store
|
||||
.select_data_movement_pool_idx(&bucket, &object_info.name, -1, &new_multipart_opts, false)
|
||||
.await?;
|
||||
if existing_target_pool_idx != pool_idx
|
||||
&& let Some(target) =
|
||||
find_data_movement_target_info(store.as_ref(), existing_target_pool_idx, &bucket, &object_info).await?
|
||||
&& is_equivalent_data_movement_object_identity(&object_info, &target, true, preserve_part_checksums)
|
||||
{
|
||||
let expected_data_bytes = capacity_expected_data_bytes
|
||||
.ok_or_else(|| Error::other("equivalent multipart target cannot reconcile unknown committed data size"))?;
|
||||
store
|
||||
.reconcile_decommission_capacity_after_equivalent_target(owner, existing_target_pool_idx, expected_data_bytes)
|
||||
.await?;
|
||||
info!(
|
||||
"{op_label}: multipart upload restart reconciled equivalent target for {}/{}",
|
||||
bucket.as_str(),
|
||||
object_info.name.as_str()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let mut cleanup_opts =
|
||||
data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
}
|
||||
for target_pool_idx in store.decommission_capacity_cleanup_target_indices(owner).await? {
|
||||
store
|
||||
.reconcile_multipart_uploads_for_data_movement(
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&data_movement_upload_identity(&object_info),
|
||||
&cleanup_opts,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
data_movement_stage_error(
|
||||
op_label,
|
||||
"reconcile_multipart_upload",
|
||||
bucket.as_str(),
|
||||
object_info.name.as_str(),
|
||||
err,
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
|
||||
.await
|
||||
@@ -1735,15 +1532,9 @@ async fn migrate_object_inner(
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut part_opts);
|
||||
}
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
part_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut part_opts);
|
||||
}
|
||||
let pi = match store
|
||||
.put_object_part_for_data_movement(
|
||||
target_pool_idx,
|
||||
@@ -1787,16 +1578,10 @@ async fn migrate_object_inner(
|
||||
err,
|
||||
)
|
||||
})?;
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut complete_multipart_opts);
|
||||
}
|
||||
complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id;
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
complete_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut complete_multipart_opts);
|
||||
}
|
||||
if let Err(err) = store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement(
|
||||
@@ -1811,15 +1596,12 @@ async fn migrate_object_inner(
|
||||
{
|
||||
if should_treat_data_movement_overwrite_as_complete(
|
||||
store.as_ref(),
|
||||
(pool_idx, target_pool_idx),
|
||||
pool_idx,
|
||||
target_pool_idx,
|
||||
bucket.as_str(),
|
||||
&object_info,
|
||||
&err,
|
||||
preserve_part_checksums,
|
||||
DataMovementOverwriteCapacity {
|
||||
owner: capacity_owner,
|
||||
expected_data_bytes: capacity_expected_data_bytes,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1847,37 +1629,31 @@ async fn migrate_object_inner(
|
||||
.await;
|
||||
|
||||
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
let mut abort_opts =
|
||||
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
let abort_result = store
|
||||
.abort_multipart_upload_for_data_movement(
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&res.upload_id,
|
||||
&abort_opts,
|
||||
)
|
||||
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
|
||||
let mut opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: pool_idx,
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
opts
|
||||
})
|
||||
.await;
|
||||
match abort_result {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(abort_err) if is_err_invalid_upload_id(&abort_err) => {
|
||||
if should_treat_data_movement_overwrite_as_complete(
|
||||
store.as_ref(),
|
||||
(pool_idx, target_pool_idx),
|
||||
pool_idx,
|
||||
target_pool_idx,
|
||||
bucket.as_str(),
|
||||
&object_info,
|
||||
&abort_err,
|
||||
preserve_part_checksums,
|
||||
DataMovementOverwriteCapacity {
|
||||
owner: capacity_owner,
|
||||
expected_data_bytes: capacity_expected_data_bytes,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1907,7 +1683,6 @@ async fn migrate_object_inner(
|
||||
bucket.clone(),
|
||||
object_info.name.clone(),
|
||||
res.upload_id.clone(),
|
||||
abort_opts,
|
||||
op_label,
|
||||
);
|
||||
return Err(data_movement_stage_error(
|
||||
@@ -1923,24 +1698,19 @@ async fn migrate_object_inner(
|
||||
|
||||
if let Err(primary_err) = multipart_result {
|
||||
if should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
#[cfg(test)]
|
||||
pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
|
||||
let mut abort_opts =
|
||||
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
return match store
|
||||
.abort_multipart_upload_for_data_movement(
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&res.upload_id,
|
||||
&abort_opts,
|
||||
)
|
||||
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
|
||||
let mut opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: pool_idx,
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
opts
|
||||
})
|
||||
.await
|
||||
{
|
||||
Ok(()) => Err(primary_err),
|
||||
@@ -1952,7 +1722,6 @@ async fn migrate_object_inner(
|
||||
bucket.clone(),
|
||||
object_info.name.clone(),
|
||||
res.upload_id.clone(),
|
||||
abort_opts,
|
||||
op_label,
|
||||
);
|
||||
Err(resolve_data_movement_abort_result(
|
||||
@@ -1975,9 +1744,6 @@ async fn migrate_object_inner(
|
||||
let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?;
|
||||
|
||||
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut put_opts);
|
||||
}
|
||||
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
if let Some(signal) = lock_lost_signal {
|
||||
put_opts.add_namespace_lock_lost_signal(signal);
|
||||
@@ -1989,15 +1755,12 @@ async fn migrate_object_inner(
|
||||
if let Err(err) = put_result {
|
||||
if should_treat_data_movement_overwrite_as_complete(
|
||||
store.as_ref(),
|
||||
(pool_idx, target_pool_idx),
|
||||
pool_idx,
|
||||
target_pool_idx,
|
||||
bucket.as_str(),
|
||||
&object_info,
|
||||
&err,
|
||||
preserve_part_checksums,
|
||||
DataMovementOverwriteCapacity {
|
||||
owner: capacity_owner,
|
||||
expected_data_bytes: capacity_expected_data_bytes,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -109,10 +109,7 @@ static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
/// strictly tighter than beta.11 (usage treated as 0) and strictly more
|
||||
/// available than a blanket 503. The fallback applies to any window without
|
||||
/// authoritative usage, not only pre-v2 upgrades; the values always come from
|
||||
/// the last persisted scanner output — pre-discard sizes of the
|
||||
/// authoritative snapshot first, backfilled per bucket from the observed
|
||||
/// (nonconverged) snapshot for buckets no authoritative cycle has covered
|
||||
/// yet (issue #6852). Loads go through the TTL-bounded
|
||||
/// the last persisted scanner output. Loads go through the TTL-bounded
|
||||
/// snapshot cache, so the quota path adds at most one backend read per
|
||||
/// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent
|
||||
/// from every persisted snapshot — those still fail closed.
|
||||
@@ -171,7 +168,7 @@ fn fresh_cached_data_usage_snapshot(
|
||||
|
||||
fn cache_data_usage_snapshot_result(
|
||||
cache: &mut Option<CachedDataUsageSnapshot>,
|
||||
result: Result<LoadedUsageBaseline, Error>,
|
||||
result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
refresh_generation: u64,
|
||||
current_generation: u64,
|
||||
@@ -181,19 +178,7 @@ fn cache_data_usage_snapshot_result(
|
||||
}
|
||||
|
||||
Some(match result {
|
||||
Ok(LoadedUsageBaseline {
|
||||
info,
|
||||
mut degraded_baseline,
|
||||
observed_unavailable,
|
||||
}) => {
|
||||
// A flaky observed read must not shrink quota coverage for a TTL
|
||||
// window: carry the previous refresh's baseline entries forward,
|
||||
// letting the fresh (authoritative) values win where they exist.
|
||||
if observed_unavailable && let Some(previous) = cache.as_ref() {
|
||||
for (bucket, size) in &previous.degraded_baseline {
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(*size);
|
||||
}
|
||||
}
|
||||
Ok((info, degraded_baseline)) => {
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info.clone()),
|
||||
loaded_at,
|
||||
@@ -437,7 +422,9 @@ async fn save_data_usage_in_backend(
|
||||
if publication_epoch != expected_publication_epoch {
|
||||
return Err(Error::other("data usage publication epoch changed before save"));
|
||||
}
|
||||
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data).await?;
|
||||
crate::config::com::save_config(store.clone(), &DATA_USAGE_OBJ_NAME_PATH, data)
|
||||
.await
|
||||
.map_err(Error::other)?;
|
||||
drop(publication_guard);
|
||||
|
||||
cleanup_observed_data_usage_after_authoritative_save_with_publication(store.as_ref(), &data_usage_info, Some(store.as_ref()))
|
||||
@@ -591,7 +578,7 @@ pub(crate) async fn prepare_bucket_usage_for_namespace_change(
|
||||
guard: Option<&rustfs_lock::NamespaceLockGuard>,
|
||||
) -> Result<(), Error> {
|
||||
ensure_bucket_namespace_guard(guard, bucket, "data usage cache cleanup")?;
|
||||
let _ = USAGE_MEMORY_GENERATION.try_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
|
||||
let _ = USAGE_MEMORY_GENERATION.fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| Some(current.saturating_add(1)));
|
||||
live_bucket_usage_cache().invalidate(bucket).await;
|
||||
clear_bucket_usage_memory(bucket, guard).await?;
|
||||
|
||||
@@ -654,7 +641,7 @@ where
|
||||
{
|
||||
Ok(reader) => reader,
|
||||
Err(Error::FileNotFound | Error::ObjectNotFound(_, _) | Error::ConfigNotFound) => return Ok(None),
|
||||
Err(err) => return Err(map_data_usage_metadata_read_error(err, object)),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
let revision = reader
|
||||
.object_info
|
||||
@@ -669,18 +656,6 @@ where
|
||||
Ok(Some((data_usage_info, revision)))
|
||||
}
|
||||
|
||||
/// A missing usage object is harmless during bucket creation, but a missing
|
||||
/// system metadata volume is a storage outage. Keep the latter retryable and
|
||||
/// distinguishable from the user bucket not existing.
|
||||
fn map_data_usage_metadata_read_error(err: Error, object: &str) -> Error {
|
||||
match err {
|
||||
Error::BucketNotFound(_) | Error::VolumeNotFound => {
|
||||
Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), object.to_string())
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn data_usage_contains_bucket(data_usage_info: &DataUsageInfo, bucket: &str) -> bool {
|
||||
data_usage_info.buckets_usage.contains_key(bucket) || data_usage_info.bucket_sizes.contains_key(bucket)
|
||||
}
|
||||
@@ -937,7 +912,7 @@ where
|
||||
)
|
||||
.await;
|
||||
drop(publication_guard);
|
||||
match save_result.map_err(|err| crate::config::com::map_system_metadata_write_error(err, object)) {
|
||||
match save_result {
|
||||
Ok(_) => return Ok(()),
|
||||
Err(err) => {
|
||||
if let Some((observed, observed_revision)) = load_data_usage_for_bucket_removal(store, object).await? {
|
||||
@@ -1128,78 +1103,24 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
|
||||
/// Load data usage info from backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
Ok(load_data_usage_from_backend_with_baseline(store).await?.info)
|
||||
}
|
||||
|
||||
/// One refresh of the persisted usage snapshot plus the quota-admission
|
||||
/// baseline derived from it.
|
||||
struct LoadedUsageBaseline {
|
||||
info: DataUsageInfo,
|
||||
degraded_baseline: HashMap<String, u64>,
|
||||
/// True when the observed snapshot could not be read (a transport error,
|
||||
/// not absence): the cached loader then carries the previous refresh's
|
||||
/// baseline entries forward instead of shrinking quota coverage for a
|
||||
/// whole TTL window over one flaky read.
|
||||
observed_unavailable: bool,
|
||||
Ok(load_data_usage_from_backend_with_baseline(store).await?.0)
|
||||
}
|
||||
|
||||
/// Like [`load_data_usage_from_backend`], but also returns the pre-discard
|
||||
/// per-bucket sizes so the cached loader can retain them as the degraded
|
||||
/// quota-admission baseline (issue #5716).
|
||||
async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<LoadedUsageBaseline, Error> {
|
||||
let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
// The observed-newness gate below compares against the snapshot as
|
||||
// persisted, before normalization demotes or discards anything.
|
||||
let authoritative_as_persisted = loaded_snapshot.clone();
|
||||
let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
|
||||
|
||||
// A bucket without a converged scanner cycle behind it — a freshly joined
|
||||
// replica whose every cycle is superseded by the sustained replication
|
||||
// write stream, or a bucket created after the last converged cycle on a
|
||||
// busy site (#6852) — has no authoritative size, and quota admission
|
||||
// fails its writes closed indefinitely. The observed (nonconverged)
|
||||
// snapshot those superseded cycles still publish is the only grounded
|
||||
// usage in that window, so it backfills buckets the loaded baseline does
|
||||
// not cover; a value already in the baseline always wins. The newness
|
||||
// gate ties the observation to this exact authoritative snapshot, so a
|
||||
// stale observed object left behind by an earlier incarnation (e.g. a
|
||||
// deleted and recreated bucket) cannot inject ghost usage. Loads sit
|
||||
// behind the same TTL cache as the snapshot itself, so this adds at most
|
||||
// one backend read per TTL window.
|
||||
let mut observed_unavailable = false;
|
||||
match load_observed_data_usage_snapshot(store).await {
|
||||
Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
|
||||
backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(_) => observed_unavailable = true,
|
||||
}
|
||||
|
||||
Ok(LoadedUsageBaseline {
|
||||
info,
|
||||
degraded_baseline,
|
||||
observed_unavailable,
|
||||
})
|
||||
async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<(DataUsageInfo, HashMap<String, u64>), Error> {
|
||||
let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
|
||||
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
|
||||
}
|
||||
|
||||
/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without
|
||||
/// overriding any bucket the authoritative baseline already covers.
|
||||
fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap<String, u64>, observed: &DataUsageInfo) {
|
||||
for (bucket, usage) in &observed.buckets_usage {
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(usage.size);
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok(None)` means the observed snapshot is absent or invalid (a settled
|
||||
/// answer); `Err` means it could not be read at all, so the caller may keep
|
||||
/// using what it learned from a previous read.
|
||||
async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option<DataUsageInfo>, Error> {
|
||||
async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
|
||||
let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
|
||||
Ok(data) => data,
|
||||
Err(Error::ConfigNotFound) => return Ok(None),
|
||||
Err(Error::ConfigNotFound) => return None,
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
return Err(err);
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1208,7 +1129,7 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option
|
||||
if info.usage_snapshot_converged == Some(false)
|
||||
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
|
||||
{
|
||||
Ok(Some(info))
|
||||
Some(info)
|
||||
}
|
||||
Ok(_) => {
|
||||
error!(
|
||||
@@ -1219,11 +1140,11 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option
|
||||
object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
"observed data usage snapshot was not a structurally complete nonconverged view"
|
||||
);
|
||||
Ok(None)
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
Ok(None)
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1240,50 +1161,14 @@ fn select_admin_data_usage_snapshot(
|
||||
authoritative.usage_snapshot_converged = Some(true);
|
||||
}
|
||||
match observed {
|
||||
Some(observed)
|
||||
if observed.usage_snapshot_partial
|
||||
&& authoritative.is_complete_bucket_usage_snapshot()
|
||||
&& observed_data_usage_is_newer(&observed, &authoritative) =>
|
||||
{
|
||||
(merge_partial_observation_for_admin(authoritative, observed), true)
|
||||
}
|
||||
Some(observed) if observed_data_usage_is_newer(&observed, &authoritative) => (observed, true),
|
||||
_ => (authoritative, authoritative_format),
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observed: DataUsageInfo) -> DataUsageInfo {
|
||||
for (bucket, usage) in observed.buckets_usage {
|
||||
authoritative.buckets_usage.insert(bucket, usage);
|
||||
}
|
||||
|
||||
authoritative.last_update = observed.last_update;
|
||||
authoritative.scanner_cycle = observed.scanner_cycle;
|
||||
authoritative.scanner_epoch = observed.scanner_epoch;
|
||||
authoritative.usage_snapshot_complete = false;
|
||||
authoritative.usage_snapshot_partial = true;
|
||||
authoritative.usage_snapshot_converged = Some(false);
|
||||
authoritative.usage_snapshot_authoritative_baseline = observed.usage_snapshot_authoritative_baseline;
|
||||
authoritative.usage_snapshot_set_states = observed.usage_snapshot_set_states;
|
||||
authoritative.usage_snapshot_bootstrap_pending = false;
|
||||
authoritative.buckets_count = authoritative.buckets_usage.len() as u64;
|
||||
authoritative.bucket_sizes = authoritative
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
authoritative.replication_info.clear();
|
||||
authoritative.tier_stats = None;
|
||||
authoritative.unknown_tier_stats = None;
|
||||
authoritative.calculate_totals();
|
||||
authoritative
|
||||
}
|
||||
|
||||
async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
// For the one-shot admin view a failed observed read degrades to "no
|
||||
// observation", same as before the read was fallible.
|
||||
let observed = load_observed_data_usage_snapshot(store).await.ok().flatten();
|
||||
let observed = load_observed_data_usage_snapshot(store).await;
|
||||
let (selected, selected_is_current_format) =
|
||||
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
|
||||
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
|
||||
@@ -1446,11 +1331,7 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
let refresh_generation = admin_data_usage_snapshot_generation();
|
||||
let result = load_admin_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.map(|info| LoadedUsageBaseline {
|
||||
info,
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
});
|
||||
.map(|info| (info, HashMap::new()));
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = admin_data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) = cache_data_usage_snapshot_result(
|
||||
@@ -2601,37 +2482,6 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
use tokio::{io::AsyncReadExt, sync::Mutex};
|
||||
|
||||
#[test]
|
||||
fn observed_snapshot_only_backfills_baseline_gaps() {
|
||||
let mut baseline = HashMap::from([("covered".to_string(), 111_u64)]);
|
||||
let observed = DataUsageInfo {
|
||||
buckets_usage: HashMap::from([
|
||||
(
|
||||
"covered".to_string(),
|
||||
BucketUsageInfo {
|
||||
size: 999,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"replica-only".to_string(),
|
||||
BucketUsageInfo {
|
||||
size: 42,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
backfill_degraded_baseline_from_observed(&mut baseline, &observed);
|
||||
|
||||
// The authoritative value must win; only the uncovered bucket (#6852:
|
||||
// a replica that never landed a converged cycle) is filled in.
|
||||
assert_eq!(baseline.get("covered"), Some(&111));
|
||||
assert_eq!(baseline.get("replica-only"), Some(&42));
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct UsageCasState {
|
||||
object: Option<(Vec<u8>, u64)>,
|
||||
@@ -2875,7 +2725,6 @@ mod tests {
|
||||
struct UsageCacheReadStore {
|
||||
transient_failures: Mutex<usize>,
|
||||
reads: Mutex<Vec<String>>,
|
||||
terminal_error: Mutex<Option<Error>>,
|
||||
}
|
||||
|
||||
impl UsageCacheReadStore {
|
||||
@@ -2883,15 +2732,6 @@ mod tests {
|
||||
Self {
|
||||
transient_failures: Mutex::new(n),
|
||||
reads: Mutex::new(Vec::new()),
|
||||
terminal_error: Mutex::new(None),
|
||||
}
|
||||
}
|
||||
|
||||
fn with_terminal_error(error: Error) -> Self {
|
||||
Self {
|
||||
transient_failures: Mutex::new(0),
|
||||
reads: Mutex::new(Vec::new()),
|
||||
terminal_error: Mutex::new(Some(error)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2919,9 +2759,6 @@ mod tests {
|
||||
_opts: &Self::ObjectOptions,
|
||||
) -> Result<Self::GetObjectReader, Self::Error> {
|
||||
self.reads.lock().await.push(object.to_string());
|
||||
if let Some(error) = self.terminal_error.lock().await.clone() {
|
||||
return Err(error);
|
||||
}
|
||||
let mut remaining = self.transient_failures.lock().await;
|
||||
if *remaining > 0 {
|
||||
*remaining -= 1;
|
||||
@@ -2964,7 +2801,6 @@ mod tests {
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: TokioMutex::new(()),
|
||||
pool_meta_save_gate: TokioMutex::default(),
|
||||
decommission_capacity_entry_gate: TokioMutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
@@ -2987,22 +2823,6 @@ mod tests {
|
||||
assert!(!is_data_usage_cache_absent(&Error::DiskNotFound));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn data_usage_removal_maps_missing_system_volume_to_read_quorum() {
|
||||
for error in [Error::VolumeNotFound, Error::BucketNotFound(RUSTFS_META_BUCKET.to_string())] {
|
||||
assert_eq!(
|
||||
map_data_usage_metadata_read_error(error, "bucket-metadata/.usage.json"),
|
||||
Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
let missing_object = Error::ObjectNotFound(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string());
|
||||
assert_eq!(
|
||||
map_data_usage_metadata_read_error(missing_object.clone(), "bucket-metadata/.usage.json"),
|
||||
missing_object
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_data_usage_cache_treats_absence_as_an_empty_cache_without_retrying() {
|
||||
let name = "usage-cache";
|
||||
@@ -3018,22 +2838,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn data_usage_removal_surfaces_missing_system_volume_as_read_quorum() {
|
||||
for cause in [Error::BucketNotFound(RUSTFS_META_BUCKET.to_string()), Error::VolumeNotFound] {
|
||||
let store = UsageCacheReadStore::with_terminal_error(cause);
|
||||
|
||||
let error = load_data_usage_for_bucket_removal(&store, "bucket-metadata/.usage.json")
|
||||
.await
|
||||
.expect_err("missing system metadata volume must not be treated as an absent usage object");
|
||||
|
||||
assert_eq!(
|
||||
error,
|
||||
Error::InsufficientReadQuorum(RUSTFS_META_BUCKET.to_string(), "bucket-metadata/.usage.json".to_string())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn load_data_usage_cache_retries_a_transient_failure() {
|
||||
let name = "usage-cache";
|
||||
@@ -3413,108 +3217,6 @@ mod tests {
|
||||
assert_eq!(selected.buckets_usage.get("bucket").map(|usage| usage.size), Some(100));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_admin_observation_preserves_authoritative_cold_buckets() {
|
||||
let baseline_time = SystemTime::UNIX_EPOCH + Duration::from_secs(10);
|
||||
let mut authoritative = data_usage_info_for_test("cold", 152_318, 80 * 1024 * 1024 * 1024, baseline_time);
|
||||
authoritative.scanner_epoch = Some(4);
|
||||
authoritative.scanner_cycle = Some(10);
|
||||
authoritative.buckets_usage.insert(
|
||||
"hot".to_string(),
|
||||
BucketUsageInfo {
|
||||
objects_count: 3_000,
|
||||
versions_count: 3_000,
|
||||
size: 400 * 1024 * 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
authoritative.buckets_count = 2;
|
||||
authoritative.bucket_sizes = authoritative
|
||||
.buckets_usage
|
||||
.iter()
|
||||
.map(|(bucket, usage)| (bucket.clone(), usage.size))
|
||||
.collect();
|
||||
authoritative.calculate_totals();
|
||||
authoritative.replication_info.insert(
|
||||
"stale-target".to_string(),
|
||||
BucketTargetUsageInfo {
|
||||
replicated_size: 400 * 1024 * 1024,
|
||||
replicated_count: 3_000,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
authoritative.tier_stats = Some(rustfs_data_usage::AllTierStats {
|
||||
tiers: HashMap::from([(
|
||||
"WARM".to_string(),
|
||||
rustfs_data_usage::TierStats {
|
||||
total_size: 80 * 1024 * 1024 * 1024,
|
||||
num_versions: 152_318,
|
||||
num_objects: 152_318,
|
||||
},
|
||||
)]),
|
||||
});
|
||||
|
||||
let mut observed = DataUsageInfo {
|
||||
last_update: Some(baseline_time + Duration::from_secs(1)),
|
||||
scanner_epoch: Some(4),
|
||||
scanner_cycle: Some(11),
|
||||
usage_snapshot_complete: false,
|
||||
usage_snapshot_partial: true,
|
||||
usage_snapshot_converged: Some(false),
|
||||
usage_snapshot_authoritative_baseline: Some(authoritative.snapshot_identity()),
|
||||
usage_snapshot_set_states: vec![rustfs_data_usage::DataUsageSnapshotSetState {
|
||||
pool_index: 0,
|
||||
set_index: 0,
|
||||
scanner_cycle: Some(11),
|
||||
scanner_epoch: Some(4),
|
||||
scan_plan_digest: Some([1; 32]),
|
||||
complete: true,
|
||||
tombstone: false,
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
observed.buckets_usage.insert(
|
||||
"hot".to_string(),
|
||||
BucketUsageInfo {
|
||||
objects_count: 34,
|
||||
versions_count: 34,
|
||||
size: 8 * 1024 * 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
observed.buckets_count = 1;
|
||||
observed.bucket_sizes.insert("hot".to_string(), 8 * 1024 * 1024);
|
||||
observed.calculate_totals();
|
||||
|
||||
let (selected, current_format) = select_admin_data_usage_snapshot(authoritative, true, Some(observed));
|
||||
|
||||
assert!(current_format);
|
||||
assert!(!selected.usage_snapshot_complete);
|
||||
assert!(selected.usage_snapshot_partial);
|
||||
assert!(selected.is_valid_partial_snapshot());
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(false));
|
||||
assert_eq!(selected.buckets_count, 2);
|
||||
assert_eq!(
|
||||
selected
|
||||
.buckets_usage
|
||||
.get("cold")
|
||||
.map(|usage| (usage.objects_count, usage.size)),
|
||||
Some((152_318, 80 * 1024 * 1024 * 1024))
|
||||
);
|
||||
assert_eq!(
|
||||
selected
|
||||
.buckets_usage
|
||||
.get("hot")
|
||||
.map(|usage| (usage.objects_count, usage.size)),
|
||||
Some((34, 8 * 1024 * 1024))
|
||||
);
|
||||
assert_eq!(selected.objects_total_count, 152_352);
|
||||
assert_eq!(selected.objects_total_size, 80 * 1024 * 1024 * 1024 + 8 * 1024 * 1024);
|
||||
assert!(selected.replication_info.is_empty());
|
||||
assert!(selected.tier_stats.is_none());
|
||||
assert!(selected.unknown_tier_stats.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn authoritative_save_cleanup_removes_observed_snapshot_best_effort() {
|
||||
let store = UsageCasStore::default();
|
||||
@@ -3586,11 +3288,7 @@ mod tests {
|
||||
|
||||
let first = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: expected,
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
Ok((expected, HashMap::new())),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
@@ -3605,38 +3303,6 @@ mod tests {
|
||||
assert_snapshot_bucket(&cached, "bucket");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn unavailable_observed_read_keeps_previous_baseline_coverage() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
|
||||
});
|
||||
|
||||
cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH),
|
||||
degraded_baseline: HashMap::from([("covered".to_string(), 2_u64)]),
|
||||
observed_unavailable: true,
|
||||
}),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
|
||||
let baseline = &cache.as_ref().expect("cache must be populated").degraded_baseline;
|
||||
// The bucket only the (now unreadable) observed snapshot covered must
|
||||
// survive the refresh; the freshly loaded value wins where it exists.
|
||||
assert_eq!(baseline.get("observed-only"), Some(&7));
|
||||
assert_eq!(baseline.get("covered"), Some(&2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() {
|
||||
@@ -3651,11 +3317,7 @@ mod tests {
|
||||
|
||||
let stale_result = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH),
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
|
||||
@@ -250,40 +250,6 @@ pub(crate) trait DiskStoreRenameDataExt {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
) -> Result<RenameDataResp>;
|
||||
|
||||
async fn rename_data_borrowed_with_guard(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<RenameDataResp> {
|
||||
let _ = external_guard;
|
||||
self.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a mutation in an owned task when a caller supplied publication guard.
|
||||
/// RPC cancellation drops only the waiter; the mutation owner keeps the guard
|
||||
/// until its operation has returned, including any detached blocking syscall.
|
||||
async fn run_owned_mutation<T, F, Fut>(external_guard: Option<Arc<dyn Send + Sync>>, operation: F) -> Result<T>
|
||||
where
|
||||
T: Send + 'static,
|
||||
F: FnOnce() -> Fut + Send + 'static,
|
||||
Fut: std::future::Future<Output = Result<T>> + Send + 'static,
|
||||
{
|
||||
if external_guard.is_none() {
|
||||
return operation().await;
|
||||
}
|
||||
tokio::spawn(async move {
|
||||
let _external_guard = external_guard;
|
||||
operation().await
|
||||
})
|
||||
.await
|
||||
.map_err(|_| Error::other("owned mutation task failed"))?
|
||||
}
|
||||
|
||||
impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
@@ -307,49 +273,6 @@ impl DiskStoreRenameDataExt for LocalDiskWrapper {
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn rename_data_borrowed_with_guard(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<RenameDataResp> {
|
||||
let operation = self.clone();
|
||||
let src_volume = src_volume.to_owned();
|
||||
let src_path = src_path.to_owned();
|
||||
let fi = fi.clone();
|
||||
let dst_volume = dst_volume.to_owned();
|
||||
let dst_path = dst_path.to_owned();
|
||||
let timeout_duration = if external_guard.is_some() {
|
||||
// A fenced mutation owns the publication guard until the storage
|
||||
// operation returns. Timing out this waiter would cancel the
|
||||
// LocalDisk future while a spawn_blocking namespace syscall could
|
||||
// still be committing, reopening the movement window. The caller
|
||||
// may drop its waiter; the owned task drains the mutation.
|
||||
Duration::ZERO
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
run_owned_mutation(external_guard, move || async move {
|
||||
operation
|
||||
.track_disk_health_mutation(
|
||||
"rename_data",
|
||||
DiskMetricMutation::Write,
|
||||
|| async {
|
||||
operation
|
||||
.disk
|
||||
.rename_data_borrowed(&src_volume, &src_path, &fi, &dst_volume, &dst_path)
|
||||
.await
|
||||
},
|
||||
timeout_duration,
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_drive_walkdir_timeout() -> Duration {
|
||||
@@ -755,20 +678,17 @@ impl DiskOperationMetrics {
|
||||
let elapsed_nanos = u64::try_from(elapsed.as_nanos()).unwrap_or(u64::MAX);
|
||||
let slot = &self.last_minute[(now_sec % 60) as usize];
|
||||
loop {
|
||||
// The successful CAS below is AcqRel, so it is the publication
|
||||
// fence for the writer that owns this slot. The initial parity
|
||||
// check does not need to acquire the slot payload.
|
||||
let version = slot.version.load(Ordering::Relaxed);
|
||||
let version = slot.version.load(Ordering::Acquire);
|
||||
if !version.is_multiple_of(2) {
|
||||
std::hint::spin_loop();
|
||||
continue;
|
||||
}
|
||||
if slot
|
||||
.version
|
||||
.compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Relaxed)
|
||||
.compare_exchange(version, version.wrapping_add(1), Ordering::AcqRel, Ordering::Acquire)
|
||||
.is_ok()
|
||||
{
|
||||
if slot.unix_sec.load(Ordering::Relaxed) != now_sec {
|
||||
if slot.unix_sec.load(Ordering::Acquire) != now_sec {
|
||||
slot.count.store(0, Ordering::Relaxed);
|
||||
slot.acc_time.store(0, Ordering::Relaxed);
|
||||
slot.unix_sec.store(now_sec, Ordering::Release);
|
||||
@@ -784,10 +704,14 @@ impl DiskOperationMetrics {
|
||||
fn last_minute_snapshot(&self, now_sec: u64) -> TimedAction {
|
||||
let mut snapshot = TimedAction::default();
|
||||
for slot in &self.last_minute {
|
||||
let Some((slot_sec, count, acc_time)) = slot.snapshot() else {
|
||||
let version = slot.version.load(Ordering::Acquire);
|
||||
if !version.is_multiple_of(2) {
|
||||
continue;
|
||||
};
|
||||
if slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
|
||||
}
|
||||
let slot_sec = slot.unix_sec.load(Ordering::Acquire);
|
||||
let count = slot.count.load(Ordering::Acquire);
|
||||
let acc_time = slot.acc_time.load(Ordering::Acquire);
|
||||
if slot.version.load(Ordering::Acquire) == version && slot_sec <= now_sec && now_sec.saturating_sub(slot_sec) < 60 {
|
||||
snapshot.count = snapshot.count.saturating_add(count);
|
||||
snapshot.acc_time = snapshot.acc_time.saturating_add(acc_time);
|
||||
}
|
||||
@@ -796,23 +720,6 @@ impl DiskOperationMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
impl TimedActionSlot {
|
||||
fn snapshot(&self) -> Option<(u64, u64, u64)> {
|
||||
let version = self.version.load(Ordering::Acquire);
|
||||
if !version.is_multiple_of(2) {
|
||||
return None;
|
||||
}
|
||||
|
||||
// The first Acquire load publishes the payload written before the
|
||||
// matching Release store. Relaxed payload loads are sufficient while
|
||||
// the final Acquire version load validates that no writer intervened.
|
||||
let slot_sec = self.unix_sec.load(Ordering::Relaxed);
|
||||
let count = self.count.load(Ordering::Relaxed);
|
||||
let acc_time = self.acc_time.load(Ordering::Relaxed);
|
||||
(self.version.load(Ordering::Acquire) == version).then_some((slot_sec, count, acc_time))
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct DiskHealthWaitingGuard<'a> {
|
||||
health: &'a DiskHealthTracker,
|
||||
}
|
||||
@@ -1190,37 +1097,6 @@ impl LocalDiskWrapper {
|
||||
)
|
||||
}
|
||||
|
||||
/// Run a delete under an owned coordinator task when a publication guard
|
||||
/// is present. This keeps the guard alive if the RPC waiter is cancelled
|
||||
/// while the local namespace mutation is still in progress.
|
||||
pub(crate) async fn delete_with_publication_guard(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
options: DeleteOptions,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
let operation = self.clone();
|
||||
let volume = volume.to_owned();
|
||||
let path = path.to_owned();
|
||||
let timeout_duration = if external_guard.is_some() {
|
||||
Duration::ZERO
|
||||
} else {
|
||||
get_max_timeout_duration()
|
||||
};
|
||||
run_owned_mutation(external_guard, move || async move {
|
||||
operation
|
||||
.track_disk_health_mutation(
|
||||
"delete",
|
||||
DiskMetricMutation::Delete,
|
||||
|| async { operation.disk.delete(&volume, &path, options).await },
|
||||
timeout_duration,
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) fn new_with_reconnect_state(
|
||||
disk: Arc<LocalDisk>,
|
||||
health_check: bool,
|
||||
@@ -2371,44 +2247,6 @@ mod tests {
|
||||
};
|
||||
use tokio::io::AsyncWrite;
|
||||
|
||||
struct DropProbe(Arc<std::sync::atomic::AtomicUsize>);
|
||||
|
||||
impl Drop for DropProbe {
|
||||
fn drop(&mut self) {
|
||||
self.0.fetch_add(1, std::sync::atomic::Ordering::SeqCst);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
|
||||
let drops = Arc::new(std::sync::atomic::AtomicUsize::new(0));
|
||||
let guard: Arc<dyn Send + Sync> = Arc::new(DropProbe(Arc::clone(&drops)));
|
||||
let (started_tx, started_rx) = tokio::sync::oneshot::channel();
|
||||
let (release_tx, release_rx) = tokio::sync::oneshot::channel();
|
||||
let (finished_tx, finished_rx) = tokio::sync::oneshot::channel();
|
||||
|
||||
let waiter = tokio::spawn(run_owned_mutation(Some(guard), move || async move {
|
||||
started_tx.send(()).expect("mutation should signal start");
|
||||
release_rx.await.expect("mutation should be released");
|
||||
finished_tx.send(()).expect("mutation should signal completion");
|
||||
Ok::<_, Error>(())
|
||||
}));
|
||||
|
||||
started_rx.await.expect("mutation owner should start");
|
||||
waiter.abort();
|
||||
assert_eq!(drops.load(std::sync::atomic::Ordering::SeqCst), 0);
|
||||
|
||||
release_tx.send(()).expect("mutation owner should still be alive");
|
||||
finished_rx.await.expect("mutation owner should finish");
|
||||
tokio::time::timeout(Duration::from_secs(1), async {
|
||||
while drops.load(std::sync::atomic::Ordering::SeqCst) == 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("publication guard should be released after mutation completion");
|
||||
}
|
||||
|
||||
struct PendingWriter;
|
||||
|
||||
#[test]
|
||||
@@ -2435,25 +2273,6 @@ mod tests {
|
||||
assert_eq!(window.acc_time, 18_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn timed_action_slot_snapshot_skips_writer_owned_slot() {
|
||||
let slot = TimedActionSlot::default();
|
||||
slot.unix_sec.store(70, Ordering::Relaxed);
|
||||
slot.count.store(2, Ordering::Relaxed);
|
||||
slot.acc_time.store(18_000, Ordering::Relaxed);
|
||||
slot.version.store(2, Ordering::Release);
|
||||
assert_eq!(slot.snapshot(), Some((70, 2, 18_000)));
|
||||
|
||||
assert_eq!(slot.version.compare_exchange(2, 3, Ordering::AcqRel, Ordering::Relaxed), Ok(2));
|
||||
slot.unix_sec.store(71, Ordering::Relaxed);
|
||||
slot.count.store(1, Ordering::Relaxed);
|
||||
slot.acc_time.store(11_000, Ordering::Relaxed);
|
||||
assert_eq!(slot.snapshot(), None);
|
||||
|
||||
slot.version.store(4, Ordering::Release);
|
||||
assert_eq!(slot.snapshot(), Some((71, 1, 11_000)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disk_health_metrics_snapshot_exports_waiting_errors_and_operation_windows() {
|
||||
let metrics = DiskHealthMetricEpoch::default();
|
||||
|
||||
@@ -12,7 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM;
|
||||
use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
|
||||
use std::error::Error as StdError;
|
||||
use std::hash::{Hash, Hasher};
|
||||
@@ -23,7 +22,6 @@ pub type Error = DiskError;
|
||||
pub type Result<T> = core::result::Result<T, Error>;
|
||||
|
||||
const METACACHE_OUTPUT_STREAM_CLOSED: &str = "metacache output stream closed";
|
||||
pub(crate) const HEAL_DANGLING_DELETE_GRACE_MESSAGE: &str = "dangling object deletion deferred by heal grace window";
|
||||
|
||||
/// Marker carried by a shard-read `io::Error` when the underlying reader can
|
||||
/// no longer be realigned after a fresh remote open failed. The marker is
|
||||
@@ -35,12 +33,6 @@ pub(crate) struct TerminalReadError {
|
||||
source: DiskError,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct DanglingDeleteGraceError {
|
||||
retry_after_secs: i64,
|
||||
grace_secs: i64,
|
||||
}
|
||||
|
||||
// DiskError == StorageErr
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum DiskError {
|
||||
@@ -208,18 +200,6 @@ impl StdError for TerminalReadError {
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for DanglingDeleteGraceError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
f,
|
||||
"{HEAL_DANGLING_DELETE_GRACE_MESSAGE}; retry_after_secs={}; grace_secs={}",
|
||||
self.retry_after_secs, self.grace_secs
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl StdError for DanglingDeleteGraceError {}
|
||||
|
||||
fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskError> {
|
||||
if error.is_remote_file_not_found() {
|
||||
return Some(DiskError::FileNotFound);
|
||||
@@ -230,19 +210,6 @@ fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskEr
|
||||
None
|
||||
}
|
||||
|
||||
fn internode_write_error_is_retryable(error: &InternodeHttpError) -> bool {
|
||||
error.kind().is_retryable()
|
||||
|| (matches!(error.kind(), InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409)
|
||||
&& error.context().operation() == Some(INTERNODE_OPERATION_PUT_FILE_STREAM))
|
||||
}
|
||||
|
||||
fn io_error_contains_retryable_internode_write(error: &io::Error) -> bool {
|
||||
error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.is_some_and(internode_write_error_is_retryable)
|
||||
}
|
||||
|
||||
/// Wrap a terminal shard-read failure without changing its typed
|
||||
/// classification. Timeout-like disk errors retain `TimedOut`; other errors
|
||||
/// retain their inner I/O kind or use `Other` when no more specific kind exists.
|
||||
@@ -286,24 +253,6 @@ impl DiskError {
|
||||
DiskError::Io(std::io::Error::other(error))
|
||||
}
|
||||
|
||||
pub(crate) fn dangling_delete_grace(retry_after_secs: i64, grace_secs: i64) -> Self {
|
||||
DiskError::other(DanglingDeleteGraceError {
|
||||
retry_after_secs,
|
||||
grace_secs,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn is_dangling_delete_grace(&self) -> bool {
|
||||
matches!(self, DiskError::Io(io_error) if Self::io_error_is_dangling_delete_grace(io_error))
|
||||
}
|
||||
|
||||
pub fn io_error_is_dangling_delete_grace(io_error: &io::Error) -> bool {
|
||||
io_error
|
||||
.get_ref()
|
||||
.is_some_and(|source| source.downcast_ref::<DanglingDeleteGraceError>().is_some())
|
||||
|| io_error.to_string().contains(HEAL_DANGLING_DELETE_GRACE_MESSAGE)
|
||||
}
|
||||
|
||||
pub(crate) fn metacache_output_stream_closed() -> Self {
|
||||
DiskError::Io(std::io::Error::new(std::io::ErrorKind::BrokenPipe, METACACHE_OUTPUT_STREAM_CLOSED))
|
||||
}
|
||||
@@ -350,7 +299,10 @@ impl DiskError {
|
||||
|
||||
pub fn is_retryable_internode_write_failure(&self) -> bool {
|
||||
match self {
|
||||
DiskError::Io(io_error) => io_error_contains_retryable_internode_write(io_error),
|
||||
DiskError::Io(io_error) => io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.is_some_and(|err| err.kind().is_retryable()),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -1251,68 +1203,6 @@ mod tests {
|
||||
assert!(!DiskError::FileNotFound.is_internode_http_status(429));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_file_server_epoch_conflict_is_retryable_write_failure() {
|
||||
let conflict = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::CONFLICT),
|
||||
));
|
||||
let bad_request = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::BAD_REQUEST),
|
||||
));
|
||||
|
||||
assert!(conflict.is_retryable_internode_write_failure());
|
||||
assert!(!bad_request.is_retryable_internode_write_failure());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind isolated HTTP fixture");
|
||||
let address = listener.local_addr().expect("fixture address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept read request");
|
||||
let mut request = [0_u8; 4096];
|
||||
let mut read = 0;
|
||||
loop {
|
||||
let count = stream.read(&mut request[read..]).await.expect("read HTTP request");
|
||||
assert!(count > 0, "request ended before its complete headers");
|
||||
read += count;
|
||||
if request[..read].windows(4).any(|bytes| bytes == b"\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
assert!(read < request.len(), "fixture request headers exceed their budget");
|
||||
}
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("send typed conflict response");
|
||||
});
|
||||
let error = match rustfs_rio::HttpReader::new(
|
||||
format!("http://{address}/rustfs/rpc/read_file_stream"),
|
||||
http::Method::GET,
|
||||
http::HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("HTTP 409 must fail the read"),
|
||||
Err(error) => DiskError::from(error),
|
||||
};
|
||||
server.await.expect("fixture task should complete");
|
||||
assert!(error.is_internode_http_status(409));
|
||||
assert!(
|
||||
!error.is_retryable_internode_write_failure(),
|
||||
"read-operation 409 must not trigger put-file retry"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("isolated read-conflict test must finish within its budget");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_missing_errors_preserve_disk_error_types() {
|
||||
let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::config::storageclass::DEFAULT_INLINE_BLOCK;
|
||||
use crate::crash_inject::{self, CrashPoint};
|
||||
use crate::data_usage::local_snapshot::ensure_data_usage_layout;
|
||||
use crate::diagnostics::get::{
|
||||
@@ -3148,10 +3149,9 @@ impl LocalIoBackend for StdBackend {
|
||||
direct_read_copy_fault_delta: MmapPageFaultDelta,
|
||||
blocking_task_duration: StdDuration,
|
||||
used_direct_io: bool,
|
||||
/// The descriptor and size snapshot opened by THIS call (None on a
|
||||
/// cache hit), handed back so the async caller can index it in the
|
||||
/// fd cache.
|
||||
opened_fd: Option<Arc<FdCacheEntry>>,
|
||||
/// The descriptor opened by THIS call (None on a cache hit), handed
|
||||
/// back so the async caller can index it in the fd cache.
|
||||
opened_fd: Option<Arc<std::fs::File>>,
|
||||
}
|
||||
|
||||
enum MmapCopyReadError {
|
||||
@@ -3198,12 +3198,12 @@ impl LocalIoBackend for StdBackend {
|
||||
(cache, key, gen_at_open)
|
||||
});
|
||||
#[cfg(target_os = "linux")]
|
||||
let cached_fd: Option<Arc<FdCacheEntry>> = match &fd_lookup {
|
||||
let cached_fd: Option<Arc<std::fs::File>> = match &fd_lookup {
|
||||
Some((cache, key, _)) => cache.get(key).await,
|
||||
None => None,
|
||||
};
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let cached_fd: Option<Arc<FdCacheEntry>> = None;
|
||||
let cached_fd: Option<Arc<std::fs::File>> = None;
|
||||
|
||||
let blocking_wait_start = metrics_enabled.then(std::time::Instant::now);
|
||||
let read_result = tokio::task::spawn_blocking(move || {
|
||||
@@ -3225,15 +3225,8 @@ impl LocalIoBackend for StdBackend {
|
||||
// the read below is positioned (mmap offset argument / `read_exact_at`)
|
||||
// and never depends on the descriptor's current offset. `cached_fd` being
|
||||
// None also marks this call as a miss for the cache-insert side-channel.
|
||||
// The cached length is the metadata snapshot captured at open time;
|
||||
// all in-place/replacement writers invalidate this entry before
|
||||
// publishing a mutation, so cache hits avoid a redundant fstat.
|
||||
let (file, cached_len, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
|
||||
(
|
||||
cached.file.as_ref().try_clone().map_err(DiskError::from)?,
|
||||
Some(cached.len),
|
||||
StdDuration::ZERO,
|
||||
)
|
||||
let (file, access_check_duration) = if let Some(cached) = cached_fd.as_ref() {
|
||||
(cached.as_ref().try_clone().map_err(DiskError::from)?, StdDuration::ZERO)
|
||||
} else {
|
||||
// Measure the volume access probe only — the part-path resolution
|
||||
// above is accounted in `path_resolve_duration` (rustfs/backlog#1801).
|
||||
@@ -3244,27 +3237,20 @@ impl LocalIoBackend for StdBackend {
|
||||
.map_err(|e| DiskError::from(to_access_error(e, DiskError::VolumeAccessDenied)))?;
|
||||
}
|
||||
let access_check_duration = access_check_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
(std::fs::File::open(&file_path).map_err(DiskError::from)?, None, access_check_duration)
|
||||
(std::fs::File::open(&file_path).map_err(DiskError::from)?, access_check_duration)
|
||||
};
|
||||
let file_open_duration = file_open_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
let (metadata_len, metadata_lookup_duration) = if let Some(len) = cached_len {
|
||||
// Reuse the open-time metadata snapshot on a cache hit. The
|
||||
// generation fence and mutation invalidation keep this value
|
||||
// tied to the inode held by `file`.
|
||||
(len, StdDuration::ZERO)
|
||||
} else {
|
||||
let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
|
||||
let meta = file.metadata().map_err(DiskError::from)?;
|
||||
let duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
(meta.len(), duration)
|
||||
};
|
||||
let metadata_lookup_start = metrics_enabled.then(StdInstant::now);
|
||||
// On a cache hit this fstats the cached descriptor — the inode it was
|
||||
// opened against, which invalidation keeps current for live entries. EC
|
||||
// shards are fixed-length, so a still-cached pre-heal length is benign.
|
||||
let meta = file.metadata().map_err(DiskError::from)?;
|
||||
let metadata_lookup_duration = metadata_lookup_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
|
||||
let metadata_validate_start = metrics_enabled.then(StdInstant::now);
|
||||
if metadata_len < end_offset_u64 {
|
||||
return Err(MmapCopyReadError::OutOfBounds {
|
||||
actual_size: metadata_len,
|
||||
});
|
||||
if meta.len() < end_offset_u64 {
|
||||
return Err(MmapCopyReadError::OutOfBounds { actual_size: meta.len() });
|
||||
}
|
||||
let metadata_validate_duration =
|
||||
metadata_validate_start.map_or(StdDuration::ZERO, |started_at| started_at.elapsed());
|
||||
@@ -3410,14 +3396,9 @@ impl LocalIoBackend for StdBackend {
|
||||
// Arc; `cached_fd.is_none()` is true exactly when this call did the open.
|
||||
// Non-Linux has no fd cache, so skip the Arc allocation there.
|
||||
#[cfg(target_os = "linux")]
|
||||
let opened_fd: Option<Arc<FdCacheEntry>> = cached_fd.is_none().then(|| {
|
||||
Arc::new(FdCacheEntry {
|
||||
file: Arc::new(file),
|
||||
len: metadata_len,
|
||||
})
|
||||
});
|
||||
let opened_fd: Option<Arc<std::fs::File>> = cached_fd.is_none().then(|| Arc::new(file));
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let opened_fd: Option<Arc<FdCacheEntry>> = None;
|
||||
let opened_fd: Option<Arc<std::fs::File>> = None;
|
||||
|
||||
Ok::<MmapCopyReadResult, MmapCopyReadError>(MmapCopyReadResult {
|
||||
bytes,
|
||||
@@ -3540,7 +3521,7 @@ impl LocalIoBackend for StdBackend {
|
||||
}
|
||||
}
|
||||
}
|
||||
// Index the freshly opened descriptor and metadata snapshot for future cache hits
|
||||
// Index the freshly opened descriptor for future cache hits
|
||||
// (rustfs/backlog#1801). `insert_if_fresh` refuses to cache if an
|
||||
// invalidation (heal/delete/rename) bumped the generation between the
|
||||
// open snapshot and now, so a stale pre-mutation inode is never served
|
||||
@@ -3892,18 +3873,6 @@ struct FdKey {
|
||||
direct: bool,
|
||||
}
|
||||
|
||||
/// Descriptor and immutable size snapshot retained for one cached shard inode.
|
||||
///
|
||||
/// The generation fence and explicit mutation invalidation keep the snapshot
|
||||
/// tied to the inode held by `file`, allowing cache hits to avoid a repeated
|
||||
/// metadata syscall without weakening replacement/heal semantics.
|
||||
struct FdCacheEntry {
|
||||
/// An independently cloneable descriptor for the immutable shard inode.
|
||||
file: Arc<std::fs::File>,
|
||||
/// File length captured together with the descriptor.
|
||||
len: u64,
|
||||
}
|
||||
|
||||
/// Per-disk cache of open descriptors for io_uring reads (backlog#1145).
|
||||
///
|
||||
/// Why this exists: `pread_uring` opened the file on the blocking pool for every
|
||||
@@ -3933,7 +3902,7 @@ struct FdCacheEntry {
|
||||
/// the descriptor once no in-flight read still holds it.
|
||||
#[cfg(target_os = "linux")]
|
||||
struct FdCache {
|
||||
cache: moka::future::Cache<FdKey, Arc<FdCacheEntry>>,
|
||||
cache: moka::future::Cache<FdKey, Arc<std::fs::File>>,
|
||||
/// Bumped by every invalidation. A miss-path open snapshots this before it
|
||||
/// opens and refuses to insert if it moved, so an fd opened before a
|
||||
/// heal/delete commit can never be resurrected into the cache after the
|
||||
@@ -3963,7 +3932,7 @@ impl FdCache {
|
||||
}
|
||||
}
|
||||
|
||||
async fn get(&self, key: &FdKey) -> Option<Arc<FdCacheEntry>> {
|
||||
async fn get(&self, key: &FdKey) -> Option<Arc<std::fs::File>> {
|
||||
self.cache.get(key).await
|
||||
}
|
||||
|
||||
@@ -3978,11 +3947,11 @@ impl FdCache {
|
||||
/// open bumped the generation, so a stale pre-heal/pre-delete inode is never
|
||||
/// cached. The post-insert re-check closes the tiny window where an
|
||||
/// invalidate races the insert itself, by removing the entry we just added.
|
||||
async fn insert_if_fresh(&self, key: FdKey, entry: Arc<FdCacheEntry>, gen_at_open: u64) {
|
||||
async fn insert_if_fresh(&self, key: FdKey, file: Arc<std::fs::File>, gen_at_open: u64) {
|
||||
if self.generation.load(Ordering::Acquire) != gen_at_open {
|
||||
return;
|
||||
}
|
||||
self.cache.insert(key.clone(), entry).await;
|
||||
self.cache.insert(key.clone(), file).await;
|
||||
if self.generation.load(Ordering::Acquire) != gen_at_open {
|
||||
self.cache.invalidate(&key).await;
|
||||
}
|
||||
@@ -4018,7 +3987,7 @@ impl FdCache {
|
||||
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||
let volume = volume.to_owned();
|
||||
let prefix = prefix.trim_end_matches('/').to_owned();
|
||||
let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| {
|
||||
let matches = move |k: &FdKey, _: &Arc<std::fs::File>| {
|
||||
k.volume == volume && (k.path == prefix || k.path.strip_prefix(&prefix).is_some_and(|r| r.starts_with('/')))
|
||||
};
|
||||
if self.cache.invalidate_entries_if(matches).is_err() {
|
||||
@@ -4034,7 +4003,7 @@ impl FdCache {
|
||||
fn invalidate_volume(&self, volume: &str) {
|
||||
self.generation.fetch_add(1, Ordering::AcqRel);
|
||||
let volume = volume.to_owned();
|
||||
let matches = move |k: &FdKey, _: &Arc<FdCacheEntry>| k.volume == volume;
|
||||
let matches = move |k: &FdKey, _: &Arc<std::fs::File>| k.volume == volume;
|
||||
if self.cache.invalidate_entries_if(matches).is_err() {
|
||||
self.cache.invalidate_all();
|
||||
}
|
||||
@@ -4052,8 +4021,7 @@ impl FdCache {
|
||||
/// tests that drive the cache directly.
|
||||
#[cfg(test)]
|
||||
async fn insert(&self, key: FdKey, file: Arc<std::fs::File>) {
|
||||
let len = file.metadata().map(|metadata| metadata.len()).unwrap_or_default();
|
||||
self.cache.insert(key, Arc::new(FdCacheEntry { file, len })).await;
|
||||
self.cache.insert(key, file).await;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -4432,12 +4400,7 @@ impl UringBackend {
|
||||
};
|
||||
|
||||
let file = match cached {
|
||||
Some(entry) => {
|
||||
if entry.len < u64::try_from(end_offset).map_err(|_| DiskError::FileCorrupt)? {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Arc::clone(&entry.file)
|
||||
}
|
||||
Some(file) => file,
|
||||
None => {
|
||||
// Snapshot the cache generation BEFORE opening (rustfs/backlog#1176):
|
||||
// if a heal/delete invalidation runs while this open is in flight,
|
||||
@@ -4447,7 +4410,7 @@ impl UringBackend {
|
||||
let root = self.root.clone();
|
||||
let volume_owned = volume.to_owned();
|
||||
let path_owned = path.to_owned();
|
||||
let (file, len) = tokio::task::spawn_blocking(move || -> Result<(std::fs::File, u64)> {
|
||||
let file = tokio::task::spawn_blocking(move || -> Result<std::fs::File> {
|
||||
let file_path = resolve_uring_object_path(&root, &volume_owned, &path_owned)?;
|
||||
let file = std::fs::File::open(&file_path).map_err(DiskError::from)?;
|
||||
let meta = file.metadata().map_err(DiskError::from)?;
|
||||
@@ -4455,22 +4418,30 @@ impl UringBackend {
|
||||
if meta.len() < end_offset_u64 {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
Ok((file, meta.len()))
|
||||
Ok(file)
|
||||
})
|
||||
.await
|
||||
.map_err(|e| DiskError::other(format!("uring pread join error: {e}")))??;
|
||||
let file = Arc::new(FdCacheEntry {
|
||||
file: Arc::new(file),
|
||||
len,
|
||||
});
|
||||
let file = Arc::new(file);
|
||||
if let (Some((cache, key)), Some(gen_at_open)) = (cache_entry, gen_at_open) {
|
||||
cache.insert_if_fresh(key, Arc::clone(&file), gen_at_open).await;
|
||||
}
|
||||
file.file.clone()
|
||||
file
|
||||
}
|
||||
};
|
||||
|
||||
if length == 0 {
|
||||
// Parity with StdBackend and the miss path (rustfs/backlog#1173): a
|
||||
// zero-length read still rejects an offset past EOF. The miss path
|
||||
// validated `meta.len() < end_offset` (end_offset == offset here), but
|
||||
// a cache hit skipped it — so fstat the descriptor and match. This is
|
||||
// a rare path (callers do not issue zero-length reads), so the one
|
||||
// extra fstat is negligible.
|
||||
match file.metadata() {
|
||||
Ok(meta) if offset_u64 > meta.len() => return Err(DiskError::FileCorrupt),
|
||||
Ok(_) => {}
|
||||
Err(e) => return Err(DiskError::from(e)),
|
||||
}
|
||||
return Ok(Bytes::new());
|
||||
}
|
||||
|
||||
@@ -10439,14 +10410,8 @@ impl DiskAPI for LocalDisk {
|
||||
fi.data = None;
|
||||
}
|
||||
|
||||
// Keep this compatibility read-ahead decision on the same policy
|
||||
// as PUT's inline admission. In particular, do not use the old
|
||||
// fixed 128 KiB shard limit: a non-inline object in a wider EC
|
||||
// layout can have a smaller shard and would otherwise be copied
|
||||
// out of part.1 during every metadata read. Such objects remain
|
||||
// fully readable through the normal EC reader below.
|
||||
let storage_class_config = runtime_sources::storage_class_config_snapshot();
|
||||
if should_read_legacy_inline_part(&fi, storage_class_config.as_ref()) {
|
||||
let inline = fi.transition_status.is_empty() && fi.data_dir.is_some() && fi.parts.len() == 1;
|
||||
if inline && fi.shard_file_size(fi.parts[0].actual_size) < DEFAULT_INLINE_BLOCK as i64 {
|
||||
let part_path = path_join_buf(&[
|
||||
path,
|
||||
fi.data_dir.map_or_else(|| "".to_string(), |dir| dir.to_string()).as_str(),
|
||||
@@ -10948,21 +10913,6 @@ impl DiskAPI for LocalDisk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether a legacy object without the inline marker should have its external
|
||||
/// part materialized into `FileInfo.data` for compatibility with the old GET
|
||||
/// fast path. The marker-bearing path is handled by `read_raw`/`get_file_info`;
|
||||
/// this is only a conservative fallback for old metadata.
|
||||
fn should_read_legacy_inline_part(fi: &FileInfo, storage_class_config: &crate::config::storageclass::Config) -> bool {
|
||||
if !fi.transition_status.is_empty() || fi.data_dir.is_none() || fi.parts.len() != 1 || fi.inline_data() {
|
||||
return false;
|
||||
}
|
||||
|
||||
let part = &fi.parts[0];
|
||||
let shard_size = fi.shard_file_size(part.actual_size);
|
||||
let versioned = fi.versioned || fi.version_id.is_some_and(|version_id| !version_id.is_nil());
|
||||
storage_class_config.should_inline(shard_size, fi.erasure.data_blocks, versioned)
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
pub(crate) async fn rename_data_borrowed(
|
||||
&self,
|
||||
@@ -11099,46 +11049,6 @@ mod test {
|
||||
file_info
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_inline_read_ahead_matches_writer_policy_for_ec_layouts() {
|
||||
let config = crate::config::storageclass::Config::default();
|
||||
let object_sizes = [128 * 1024_i64, 256 * 1024, 512 * 1024, 1024 * 1024, 4 * 1024 * 1024];
|
||||
|
||||
for (data_shards, parity_shards) in [(8, 4), (12, 4)] {
|
||||
for object_size in object_sizes {
|
||||
let mut fi = FileInfo::new("object", data_shards, parity_shards);
|
||||
fi.data_dir = Some(Uuid::from_u128(1));
|
||||
fi.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: usize::try_from(object_size).expect("test object size should fit usize"),
|
||||
actual_size: object_size,
|
||||
..Default::default()
|
||||
}];
|
||||
|
||||
let writer_decision = config.should_inline(fi.shard_file_size(object_size), data_shards, false);
|
||||
assert_eq!(
|
||||
should_read_legacy_inline_part(&fi, &config),
|
||||
writer_decision,
|
||||
"legacy read-ahead must match PUT for EC{data_shards}+{parity_shards}, size={object_size}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let mut ec12 = FileInfo::new("object", 12, 4);
|
||||
ec12.data_dir = Some(Uuid::from_u128(1));
|
||||
ec12.parts = vec![ObjectPartInfo {
|
||||
number: 1,
|
||||
size: 1024 * 1024,
|
||||
actual_size: 1024 * 1024,
|
||||
..Default::default()
|
||||
}];
|
||||
assert!(
|
||||
ec12.shard_file_size(1024 * 1024) < crate::config::storageclass::DEFAULT_INLINE_BLOCK as i64,
|
||||
"the regression guard must exercise the old fixed 128 KiB read-ahead boundary"
|
||||
);
|
||||
assert!(!should_read_legacy_inline_part(&ec12, &config));
|
||||
}
|
||||
|
||||
fn test_meta(fi: FileInfo) -> Vec<u8> {
|
||||
let mut meta = FileMeta::default();
|
||||
meta.add_version(fi).expect("test metadata should accept file info");
|
||||
@@ -21437,10 +21347,11 @@ mod test {
|
||||
|
||||
/// Zero-length read bounds parity on the cache-HIT path (backlog#1173/#1180).
|
||||
/// A `length == 0` read past EOF must be rejected identically whether the
|
||||
/// descriptor is freshly opened (miss path) or served from the cache. Seeds
|
||||
/// the cache with a normal read so the zero-length reads reuse the same
|
||||
/// open-time size snapshot, then pins that UringBackend and StdBackend agree
|
||||
/// on every case.
|
||||
/// descriptor is freshly opened (miss path) or served from the cache: the
|
||||
/// cache-hit branch fstats the descriptor to reproduce the miss path's
|
||||
/// `offset > len` check instead of returning empty unconditionally. Seeds
|
||||
/// the cache with a normal read so the zero-length reads are hits, then pins
|
||||
/// that UringBackend and StdBackend agree on every case.
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn uring_zero_length_read_bounds_match_std_on_cache_hit() {
|
||||
|
||||
@@ -677,20 +677,15 @@ impl DiskAPI for Disk {
|
||||
}
|
||||
|
||||
impl Disk {
|
||||
pub async fn delete_with_scanner_publication_lease_and_guard(
|
||||
pub(crate) async fn delete_with_scanner_publication_lease(
|
||||
&self,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
opts: DeleteOptions,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<()> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.delete_with_publication_guard(volume, path, opts, external_guard)
|
||||
.await
|
||||
}
|
||||
Disk::Local(local_disk) => local_disk.delete(volume, path, opts).await,
|
||||
Disk::Remote(remote_disk) => {
|
||||
remote_disk
|
||||
.delete_with_scanner_publication_lease(volume, path, opts, scanner_publication_lease_token)
|
||||
@@ -719,34 +714,11 @@ impl Disk {
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
) -> Result<RenameDataResp> {
|
||||
self.rename_data_borrowed_with_fence_and_guard(
|
||||
src_volume,
|
||||
src_path,
|
||||
fi,
|
||||
dst_volume,
|
||||
dst_path,
|
||||
scanner_publication_lease_token,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn rename_data_borrowed_with_fence_and_guard(
|
||||
&self,
|
||||
src_volume: &str,
|
||||
src_path: &str,
|
||||
fi: &FileInfo,
|
||||
dst_volume: &str,
|
||||
dst_path: &str,
|
||||
scanner_publication_lease_token: Option<Uuid>,
|
||||
external_guard: Option<Arc<dyn Send + Sync>>,
|
||||
) -> Result<RenameDataResp> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => {
|
||||
local_disk
|
||||
.rename_data_borrowed_with_guard(src_volume, src_path, fi, dst_volume, dst_path, external_guard)
|
||||
.rename_data_borrowed(src_volume, src_path, fi, dst_volume, dst_path)
|
||||
.await
|
||||
}
|
||||
Disk::Remote(remote_disk) => {
|
||||
|
||||
@@ -16,143 +16,10 @@ use rustfs_filemeta::{MetacacheReader, MetacacheWriter};
|
||||
use std::io::Cursor;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Test-only lock client whose refresh path can be rejected independently of
|
||||
/// every other lock operation. The observed event is awaitable so lock-loss
|
||||
/// tests do not depend on sleeps or scheduler timing.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct RefreshLossLockClient {
|
||||
inner: rustfs_lock::LocalClient,
|
||||
reject_refresh: AtomicBool,
|
||||
rejected_refresh: AtomicBool,
|
||||
rejected_refresh_notify: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
impl RefreshLossLockClient {
|
||||
pub(crate) fn with_manager(manager: Arc<rustfs_lock::GlobalLockManager>) -> Self {
|
||||
Self {
|
||||
inner: rustfs_lock::LocalClient::with_manager(manager),
|
||||
reject_refresh: AtomicBool::new(false),
|
||||
rejected_refresh: AtomicBool::new(false),
|
||||
rejected_refresh_notify: tokio::sync::Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reject_refreshes(&self) {
|
||||
self.reject_refresh.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn refreshes_rejected(&self) -> bool {
|
||||
self.rejected_refresh.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_rejected_refresh(
|
||||
&self,
|
||||
timeout: std::time::Duration,
|
||||
) -> std::result::Result<(), tokio::time::error::Elapsed> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notified = self.rejected_refresh_notify.notified();
|
||||
if self.refreshes_rejected() {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl rustfs_lock::LockClient for RefreshLossLockClient {
|
||||
async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<rustfs_lock::LockResponse> {
|
||||
rustfs_lock::LockClient::acquire_lock(&self.inner, request).await
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
rustfs_lock::LockClient::release(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
if self.reject_refresh.load(Ordering::Acquire) {
|
||||
self.rejected_refresh.store(true, Ordering::Release);
|
||||
self.rejected_refresh_notify.notify_waiters();
|
||||
return Ok(false);
|
||||
}
|
||||
rustfs_lock::LockClient::refresh(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
rustfs_lock::LockClient::force_release(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<Option<rustfs_lock::LockInfo>> {
|
||||
rustfs_lock::LockClient::check_status(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn list_lock_leases(&self) -> Vec<rustfs_lock::LockLeaseInfo> {
|
||||
rustfs_lock::LockClient::list_lock_leases(&self.inner).await
|
||||
}
|
||||
|
||||
async fn get_stats(&self) -> rustfs_lock::Result<rustfs_lock::LockStats> {
|
||||
rustfs_lock::LockClient::get_stats(&self.inner).await
|
||||
}
|
||||
|
||||
async fn close(&self) -> rustfs_lock::Result<()> {
|
||||
rustfs_lock::LockClient::close(&self.inner).await
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
rustfs_lock::LockClient::is_online(&self.inner).await
|
||||
}
|
||||
|
||||
async fn is_local(&self) -> bool {
|
||||
rustfs_lock::LockClient::is_local(&self.inner).await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)));
|
||||
let client = RefreshLossLockClient::with_manager(manager);
|
||||
let resource = rustfs_lock::ObjectKey::new("bucket", "object");
|
||||
let response = rustfs_lock::LockClient::acquire_lock(
|
||||
&client,
|
||||
&rustfs_lock::LockRequest::new(resource, rustfs_lock::LockType::Shared, "refresh-loss-harness"),
|
||||
)
|
||||
.await
|
||||
.expect("acquire should reach the inner local client");
|
||||
let lock_id = response.lock_info.expect("the inner local client should acquire the lock").id;
|
||||
assert_eq!(
|
||||
rustfs_lock::LockClient::list_lock_leases(&client).await.len(),
|
||||
1,
|
||||
"lease diagnostics must remain transparent through the refresh wrapper"
|
||||
);
|
||||
|
||||
client.reject_refreshes();
|
||||
assert!(
|
||||
!rustfs_lock::LockClient::refresh(&client, &lock_id)
|
||||
.await
|
||||
.expect("refresh should return a response")
|
||||
);
|
||||
client
|
||||
.wait_for_rejected_refresh(std::time::Duration::from_millis(50))
|
||||
.await
|
||||
.expect("a waiter registered after rejection must still observe the event");
|
||||
assert!(client.refreshes_rejected());
|
||||
assert!(
|
||||
rustfs_lock::LockClient::release(&client, &lock_id)
|
||||
.await
|
||||
.expect("release should reach the inner local client")
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
|
||||
/// them alive for the test's duration and the directories are removed on drop.
|
||||
pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
|
||||
|
||||
@@ -46,7 +46,6 @@ type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Re
|
||||
type OwnedShardReadFuture<'a, R> =
|
||||
Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
|
||||
pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
|
||||
pub(crate) type DecodeOutcome = (usize, Option<std::io::Error>, bool);
|
||||
|
||||
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
|
||||
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
|
||||
@@ -575,7 +574,6 @@ pub(crate) struct ParallelReader<R> {
|
||||
read_timeout: Duration,
|
||||
verify_reconstruction: bool,
|
||||
locality_preference_enabled: bool,
|
||||
demand_bound_lockstep: bool,
|
||||
// Request-scoped shard buffers keyed by shard index. Keeping ownership in
|
||||
// `ParallelReader` avoids dropping unused parity/backup slot buffers between stripes.
|
||||
buffers: ShardBufferPool,
|
||||
@@ -587,8 +585,10 @@ pub(crate) struct ParallelReader<R> {
|
||||
// it to the current stripe when it is engaged mid-object (backlog#923).
|
||||
engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
// Demand-bound hedges use a fresh deferred reader so cancelling a hedge
|
||||
// never consumes the unopened reader reserved for a later stripe.
|
||||
// Copy-source hedges use a fresh deferred reader so cancelling a hedge
|
||||
// never consumes the unopened reader reserved for a later stripe. The
|
||||
// vector is empty for callers that do not provide a reopen factory (tests
|
||||
// and the ordinary GET path retain the handle-based behavior).
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
stripe_index: usize,
|
||||
}
|
||||
@@ -777,9 +777,9 @@ where
|
||||
// reads all live readers on every stripe — the pre-backlog#923
|
||||
// behavior. With the gate on, only data slots start engaged; parity is
|
||||
// engaged on demand, stripe-aligned through its deferred handle.
|
||||
let demand_bound_lockstep = get_lockstep_data_shards_only_enabled();
|
||||
let data_shards_only = get_lockstep_data_shards_only_enabled();
|
||||
let engaged: SmallVec<_> = (0..readers.len())
|
||||
.map(|index| !demand_bound_lockstep || index < e.data_shards)
|
||||
.map(|index| !data_shards_only || index < e.data_shards)
|
||||
.collect();
|
||||
ParallelReader {
|
||||
readers,
|
||||
@@ -793,7 +793,6 @@ where
|
||||
read_timeout,
|
||||
verify_reconstruction,
|
||||
locality_preference_enabled: get_shard_locality_preference_enabled(),
|
||||
demand_bound_lockstep,
|
||||
buffers: ShardBufferPool::new(e.data_shards + e.parity_shards),
|
||||
stripe_state: None,
|
||||
engaged,
|
||||
@@ -1276,7 +1275,7 @@ where
|
||||
/// realigned (no pending deferred handle) is likewise retired instead of
|
||||
/// being read out of position.
|
||||
async fn read_lockstep(&mut self, state: &mut StripeReadState) {
|
||||
if self.demand_bound_lockstep {
|
||||
if matches!(decode_read_policy(), DecodeReadPolicy::DemandBound) {
|
||||
self.read_lockstep_demand_bound(state).await;
|
||||
return;
|
||||
}
|
||||
@@ -1532,18 +1531,17 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Demand-bound data-shards-only lockstep stripe read.
|
||||
/// Demand-bound lockstep stripe read used by server-side copy sources.
|
||||
///
|
||||
/// The ordinary lockstep path can cancel every in-flight reader once it
|
||||
/// has a quorum because all of its parity readers are already engaged.
|
||||
/// Copy sources and the data-shards-only rollout gate keep parity unopened
|
||||
/// until a data reader is missing. A hedge therefore has to race the
|
||||
/// deferred parity reads against the original data reads and may retire the
|
||||
/// latter only after parity has produced an actual decode-plus-verification
|
||||
/// quorum. The futures own their readers so disjoint data/parity slots can
|
||||
/// be admitted while the other group is still pending; dropping an
|
||||
/// abandoned future retires its stream without leaving a borrowed slot
|
||||
/// behind.
|
||||
/// Copy sources keep parity unopened until a data reader is missing. A
|
||||
/// hedge therefore has to race the deferred parity reads against the
|
||||
/// original data reads and may retire the latter only after the parity has
|
||||
/// produced an actual decode-plus-verification quorum. The futures own
|
||||
/// their readers so disjoint data/parity slots can be admitted while the
|
||||
/// other group is still pending; dropping an abandoned future retires its
|
||||
/// stream without leaving a borrowed slot behind.
|
||||
async fn read_lockstep_demand_bound(&mut self, state: &mut StripeReadState) {
|
||||
let num_readers = self.readers.len();
|
||||
state.reset(num_readers, self.data_shards);
|
||||
@@ -1578,14 +1576,14 @@ where
|
||||
let mut completed = 0usize;
|
||||
let mut failed = 0usize;
|
||||
let mut first_shard_recorded = false;
|
||||
let mut active: ActiveReaders = smallvec![false; num_readers];
|
||||
let mut temporary_parity: ActiveReaders = smallvec![false; num_readers];
|
||||
let mut active = vec![false; num_readers];
|
||||
let mut temporary_parity = vec![false; num_readers];
|
||||
// A deferred parity slot is attempted at most once per stripe. A
|
||||
// failed disposable hedge keeps its unopened reserve for the next
|
||||
// stripe, but must not be relaunched in a tight same-stripe retry
|
||||
// loop (which would defeat the bounded fan-out and amplify a remote
|
||||
// outage).
|
||||
let mut attempted_parity: ActiveReaders = smallvec![false; num_readers];
|
||||
let mut attempted_parity = vec![false; num_readers];
|
||||
// Once a data reader has returned an error (or was already missing at
|
||||
// setup), the loss is permanent for lockstep alignment. Use the
|
||||
// deferred handle and keep parity engaged across subsequent stripes;
|
||||
@@ -2191,10 +2189,8 @@ impl Erasure {
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let (written, error, _) = self
|
||||
.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
|
||||
.await;
|
||||
(written, error)
|
||||
self.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
|
||||
.await
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
|
||||
@@ -2211,10 +2207,8 @@ impl Erasure {
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let (written, error, _) = self
|
||||
.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
|
||||
.await;
|
||||
(written, error)
|
||||
self.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
|
||||
.await
|
||||
}
|
||||
|
||||
/// GET decode entry point that also carries the deferred-parity stripe
|
||||
@@ -2267,37 +2261,6 @@ impl Erasure {
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
) -> (usize, Option<std::io::Error>)
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let (written, error, _) = self
|
||||
.decode_inner(
|
||||
writer,
|
||||
readers,
|
||||
offset,
|
||||
length,
|
||||
total_length,
|
||||
read_costs,
|
||||
deferred_handles,
|
||||
deferred_reopeners,
|
||||
)
|
||||
.await;
|
||||
(written, error)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn decode_with_stripe_handles_and_reopeners_with_diagnostics<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
total_length: usize,
|
||||
read_costs: Option<Vec<ShardReadCost>>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
) -> DecodeOutcome
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
@@ -2335,7 +2298,6 @@ impl Erasure {
|
||||
written: &mut usize,
|
||||
ret_err: &mut Option<std::io::Error>,
|
||||
stage_metrics_enabled: bool,
|
||||
require_surplus_source: bool,
|
||||
) -> StripeFlow
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
@@ -2373,12 +2335,7 @@ impl Erasure {
|
||||
// missing data shard and an extra source shard was available, verify
|
||||
// the reconstructed data against that source before streaming bytes.
|
||||
let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let decode_result = if require_surplus_source {
|
||||
self.decode_data_with_reconstruction_verification_for_lockstep(shards)
|
||||
} else {
|
||||
self.decode_data_with_reconstruction_verification(shards)
|
||||
};
|
||||
if let Err(e) = decode_result {
|
||||
if let Err(e) = self.decode_data_with_reconstruction_verification(shards) {
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
|
||||
let reason = GetObjectFailureReason::DecodeError;
|
||||
error!(
|
||||
@@ -2447,48 +2404,36 @@ impl Erasure {
|
||||
read_costs: Option<Vec<ShardReadCost>>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
) -> DecodeOutcome
|
||||
) -> (usize, Option<std::io::Error>)
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
if readers.len() != self.data_shards + self.parity_shards {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")), false);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
|
||||
}
|
||||
|
||||
// block_size/data_shards come from on-disk metadata; a corrupt FileInfo with a
|
||||
// zero here must surface as an error, not a divide-by-zero panic on every GET.
|
||||
if self.block_size == 0 || self.data_shards == 0 {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
|
||||
false,
|
||||
);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")));
|
||||
}
|
||||
|
||||
let Some(end_offset) = offset.checked_add(length) else {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
false,
|
||||
);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
|
||||
};
|
||||
if end_offset > total_length {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
false,
|
||||
);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
|
||||
}
|
||||
|
||||
let mut ret_err = None;
|
||||
|
||||
if length == 0 {
|
||||
return (0, ret_err, false);
|
||||
return (0, ret_err);
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
@@ -2528,7 +2473,6 @@ impl Erasure {
|
||||
}
|
||||
};
|
||||
|
||||
let mut exact_quorum = false;
|
||||
if legacy_stripe_prefetch_enabled() {
|
||||
// Depth-1 stripe prefetch (backlog#930 HP-9 step 2): while the current
|
||||
// stripe is reconstructed and emitted, the next stripe's shard reads
|
||||
@@ -2571,7 +2515,6 @@ impl Erasure {
|
||||
let Some((mut shards, errs)) = current.take() else {
|
||||
break;
|
||||
};
|
||||
exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
|
||||
|
||||
if idx + 1 < blocks.len() {
|
||||
// Overlap: read stripe idx+1 while reconstructing/emitting idx.
|
||||
@@ -2603,7 +2546,6 @@ impl Erasure {
|
||||
// `shards` are borrowed again below. In the `Stop` case that
|
||||
// drop is what cancels the still-in-flight prefetch read.
|
||||
let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
|
||||
let require_surplus_source = reader.demand_bound_lockstep;
|
||||
let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
|
||||
let emit_fut = self.emit_decoded_stripe(
|
||||
writer,
|
||||
@@ -2614,7 +2556,6 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
);
|
||||
tokio::pin!(read_fut);
|
||||
tokio::pin!(emit_fut);
|
||||
@@ -2662,7 +2603,6 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
reader.demand_bound_lockstep,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2686,7 +2626,6 @@ impl Erasure {
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let (mut shards, errs) = reader.read().await;
|
||||
exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_STAGE_STRIPE_READ,
|
||||
@@ -2703,7 +2642,6 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
reader.demand_bound_lockstep,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2716,14 +2654,14 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if ret_err.is_some() {
|
||||
return (written, ret_err, exact_quorum);
|
||||
return (written, ret_err);
|
||||
}
|
||||
|
||||
if written < length {
|
||||
ret_err = Some(Error::LessData.into());
|
||||
}
|
||||
|
||||
(written, ret_err, exact_quorum)
|
||||
(written, ret_err)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2928,7 +2866,6 @@ mod tests {
|
||||
cursor: Cursor<Vec<u8>>,
|
||||
stall: Duration,
|
||||
sleep: Option<Pin<Box<Sleep>>>,
|
||||
stall_polls: Arc<AtomicUsize>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2967,12 +2904,7 @@ mod tests {
|
||||
TestShardReader::TerminalFileNotFound => {
|
||||
Poll::Ready(Err(crate::disk::error::terminal_read_error_to_io(Error::FileNotFound)))
|
||||
}
|
||||
TestShardReader::PrefixThenSlow {
|
||||
cursor,
|
||||
stall,
|
||||
sleep,
|
||||
stall_polls,
|
||||
} => {
|
||||
TestShardReader::PrefixThenSlow { cursor, stall, sleep } => {
|
||||
let before = buf.filled().len();
|
||||
match Pin::new(cursor).poll_read(cx, buf) {
|
||||
// Cursor still has bytes for the current stripe: serve them.
|
||||
@@ -2982,7 +2914,6 @@ mod tests {
|
||||
// the task cleanly (no busy `wake_by_ref` spin), letting the
|
||||
// `#[tokio::test(start_paused = true)]` clock auto-advance.
|
||||
Poll::Ready(Ok(())) => {
|
||||
stall_polls.fetch_add(1, Ordering::SeqCst);
|
||||
let stall = *stall;
|
||||
let sleeper = sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep(stall)));
|
||||
let _ = sleeper.as_mut().poll(cx);
|
||||
@@ -3011,29 +2942,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct YieldOnceThenFailWriter {
|
||||
yielded: bool,
|
||||
}
|
||||
|
||||
impl AsyncWrite for YieldOnceThenFailWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
if !self.yielded {
|
||||
self.yielded = true;
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "injected emit failure after prefetch poll")))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
struct DownstreamClosedWriter;
|
||||
|
||||
impl AsyncWrite for DownstreamClosedWriter {
|
||||
@@ -3970,7 +3878,6 @@ mod tests {
|
||||
(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some(READ_TIMEOUT_SECS)),
|
||||
];
|
||||
temp_env::async_with_vars(vars, async {
|
||||
let stall_polls = Arc::new(AtomicUsize::new(0));
|
||||
let readers: Vec<Option<BitrotReader<TestShardReader>>> = shard_bufs
|
||||
.iter()
|
||||
.map(|buf| {
|
||||
@@ -3980,13 +3887,12 @@ mod tests {
|
||||
cursor: Cursor::new(prefix),
|
||||
stall: STALL,
|
||||
sleep: None,
|
||||
stall_polls: Arc::clone(&stall_polls),
|
||||
};
|
||||
Some(BitrotReader::new(reader, shard_size, hash_algo.clone(), false))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut writer = YieldOnceThenFailWriter { yielded: false };
|
||||
let mut writer = FailingEmitWriter;
|
||||
let start = TokioInstant::now();
|
||||
let (written, err) = erasure.decode(&mut writer, readers, 0, total_len, total_len).await;
|
||||
let elapsed = start.elapsed();
|
||||
@@ -3994,10 +3900,6 @@ mod tests {
|
||||
// Emit failed on stripe 0, so the GET fails with no bytes emitted.
|
||||
assert!(err.is_some(), "emit failure must surface as an error");
|
||||
assert_eq!(written, 0, "the failing writer accepts no bytes");
|
||||
assert!(
|
||||
stall_polls.load(Ordering::SeqCst) > 0,
|
||||
"the speculative next-stripe read must be in flight before emit fails"
|
||||
);
|
||||
// The decisive assertion: the prefetch read was cancelled rather than
|
||||
// awaited. Without cancel-safety this would take READ_TIMEOUT_SECS.
|
||||
assert!(
|
||||
@@ -5009,24 +4911,6 @@ mod tests {
|
||||
/// read timeout even though both parity readers were available to engage.
|
||||
#[tokio::test]
|
||||
async fn test_demand_bound_lockstep_hedges_to_deferred_parity_quorum() {
|
||||
with_decode_read_policy(DecodeReadPolicy::DemandBound, assert_deferred_parity_hedges_slow_data()).await;
|
||||
}
|
||||
|
||||
/// The ordinary GET rollout gate must use the same bounded parity race as
|
||||
/// CopySource. Leaving it on the legacy lockstep loop deadlocks the hedge:
|
||||
/// that loop waits for a parity success before cancelling the slow data
|
||||
/// read, but does not admit deferred parity until after the data read ends.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_data_shards_only_gate_hedges_to_deferred_parity_quorum() {
|
||||
temp_env::async_with_vars(
|
||||
[(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
|
||||
assert_deferred_parity_hedges_slow_data(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn assert_deferred_parity_hedges_slow_data() {
|
||||
const NUM_SHARDS: usize = 1;
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 2;
|
||||
@@ -5067,27 +4951,33 @@ mod tests {
|
||||
];
|
||||
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
erasure,
|
||||
0,
|
||||
NUM_SHARDS * BLOCK_SIZE,
|
||||
None,
|
||||
vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS],
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
);
|
||||
let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read())
|
||||
.await
|
||||
.expect("deferred parity must cover a hedged data shard without waiting for read_timeout");
|
||||
let (bufs, errs, engaged, readers_remaining) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
erasure,
|
||||
0,
|
||||
NUM_SHARDS * BLOCK_SIZE,
|
||||
None,
|
||||
vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS],
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
);
|
||||
let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read())
|
||||
.await
|
||||
.expect("deferred parity must cover a hedged data shard without waiting for read_timeout");
|
||||
(
|
||||
bufs,
|
||||
errs,
|
||||
parallel_reader.engaged.clone(),
|
||||
parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
|
||||
assert_eq!(bufs.iter().filter(|buf| buf.is_some()).count(), DATA_SHARDS + 1);
|
||||
assert_eq!(parallel_reader.engaged.as_slice(), &[true, true, true, true]);
|
||||
assert_eq!(
|
||||
parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
|
||||
vec![false, true, true, true]
|
||||
);
|
||||
assert_eq!(engaged.as_slice(), &[true, true, true, true]);
|
||||
assert_eq!(readers_remaining, vec![false, true, true, true]);
|
||||
}
|
||||
|
||||
/// A fast data failure must admit deferred parity immediately. There is
|
||||
@@ -5156,24 +5046,6 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_demand_bound_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
|
||||
with_decode_read_policy(
|
||||
DecodeReadPolicy::DemandBound,
|
||||
assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_data_shards_only_gate_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
|
||||
temp_env::async_with_vars(
|
||||
[(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
|
||||
assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn assert_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
@@ -5222,7 +5094,7 @@ mod tests {
|
||||
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo, false)),
|
||||
];
|
||||
|
||||
let (first_parity_reserved, second_result) = {
|
||||
let (first_parity_reserved, second_result) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
@@ -5283,7 +5155,8 @@ mod tests {
|
||||
parallel_reader.readers[2].is_some() && parallel_reader.readers[3].is_some(),
|
||||
(third_buffers, third_errors),
|
||||
)
|
||||
};
|
||||
})
|
||||
.await;
|
||||
|
||||
assert!(first_parity_reserved);
|
||||
assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS * 2);
|
||||
@@ -5367,58 +5240,6 @@ mod tests {
|
||||
assert!(error.is_none(), "a failed disposable hedge must not fail a recovered stripe: {error:?}");
|
||||
}
|
||||
|
||||
/// Rollout guard for backlog#1308: when a data shard and the first parity
|
||||
/// hedge both fail, the gate-on path must not settle at decode quorum and
|
||||
/// emit an unverified body. The second parity can restore decode quorum but
|
||||
/// cannot provide the extra source required for reconstruction verification,
|
||||
/// so the stripe must fail before exposing bytes.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_data_shards_only_gate_data_and_parity_failure_fails_before_output() {
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], async {
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let payload = (0..BLOCK_SIZE).map(|value| value as u8).collect::<Vec<_>>();
|
||||
let shards = erasure.encode_data(&payload).expect("test payload should encode");
|
||||
let shard_size = erasure.shard_size();
|
||||
|
||||
let readers = vec![
|
||||
Some(BitrotReader::new(TestShardReader::TimedOut, shard_size, HashAlgorithm::None, false)),
|
||||
Some(BitrotReader::new(
|
||||
TestShardReader::Ready(Cursor::new(shards[1].to_vec())),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
)),
|
||||
Some(BitrotReader::new(
|
||||
TestShardReader::TerminalFileNotFound,
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
)),
|
||||
Some(BitrotReader::new(
|
||||
TestShardReader::Ready(Cursor::new(shards[3].to_vec())),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
)),
|
||||
];
|
||||
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = erasure.decode(&mut output, readers, 0, payload.len(), payload.len()).await;
|
||||
|
||||
assert_eq!(written, 0, "an unverified stripe must not report body bytes");
|
||||
assert!(output.is_empty(), "an unverified stripe must not expose a clean short body");
|
||||
let error = error.expect("data plus parity loss must fail closed");
|
||||
assert_eq!(error.kind(), ErrorKind::InvalidData);
|
||||
assert!(error.to_string().contains("insufficient source shards"));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Lockstep verification-quorum regression (backlog#1156). When a data shard is
|
||||
/// missing, the hedge must settle only at `data_shards + 1` (decode quorum plus
|
||||
/// a reconstruction-verification source), never at exactly `data_shards` — that
|
||||
|
||||
@@ -27,8 +27,6 @@ use std::io;
|
||||
use std::io::ErrorKind;
|
||||
use std::pin::Pin;
|
||||
use std::sync::Mutex;
|
||||
#[cfg(test)]
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::task::{Context, Poll, ready};
|
||||
use std::time::Instant;
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
@@ -40,14 +38,6 @@ const DEFAULT_RUSTFS_GET_CODEC_STREAMING_MAX_INFLIGHT: usize = 2;
|
||||
const FILL_POLICY_SINGLE_INFLIGHT: &str = "single_inflight";
|
||||
const FILL_POLICY_DUAL_INFLIGHT: &str = "dual_inflight";
|
||||
|
||||
#[cfg(test)]
|
||||
static SINGLE_INFLIGHT_CONSTRUCTIONS: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn test_single_inflight_construction_count() -> u64 {
|
||||
SINGLE_INFLIGHT_CONSTRUCTIONS.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
type FillTask = oneshot::Receiver<FillResult>;
|
||||
|
||||
struct FillWorker {
|
||||
@@ -165,23 +155,6 @@ where
|
||||
Self::new_with_fill_policy_inner(source, engine, total_length, metrics_path, FillPolicy::from_env())
|
||||
}
|
||||
|
||||
/// Construct the bounded reader without lookahead.
|
||||
///
|
||||
/// Mid-size GETs are latency-sensitive and are already gated to a single
|
||||
/// plain part. Keeping one stripe in flight avoids retaining a second
|
||||
/// decoded output buffer while preserving the same source, reconstruction,
|
||||
/// bitrot and cancellation semantics as the general streaming reader.
|
||||
pub(crate) fn new_single_inflight_with_metrics_path(
|
||||
source: S,
|
||||
engine: E,
|
||||
total_length: usize,
|
||||
metrics_path: &'static str,
|
||||
) -> io::Result<Self> {
|
||||
#[cfg(test)]
|
||||
SINGLE_INFLIGHT_CONSTRUCTIONS.fetch_add(1, Ordering::Relaxed);
|
||||
Self::new_with_fill_policy_inner(source, engine, total_length, metrics_path, FillPolicy::SingleInFlight)
|
||||
}
|
||||
|
||||
fn new_with_fill_policy_inner(
|
||||
source: S,
|
||||
engine: E,
|
||||
@@ -629,8 +602,7 @@ where
|
||||
|
||||
loop {
|
||||
if self.output_pos < self.output_buf.len() {
|
||||
if self.fill_policy == FillPolicy::DualInFlight
|
||||
&& self.prefetched_bufs.len() < self.fill_policy.max_inflight()
|
||||
if self.prefetched_bufs.len() < self.fill_policy.max_inflight()
|
||||
&& self.prefetch_error.is_none()
|
||||
&& self.remaining > 0
|
||||
&& let Poll::Ready(result) = self.poll_prefetch(cx)
|
||||
@@ -1648,149 +1620,6 @@ mod tests {
|
||||
assert_eq!(decoded, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_inflight_reader_reads_full_body_without_lookahead() {
|
||||
let erasure = Erasure::new(4, 2, 32);
|
||||
let data = (0..96u8).collect::<Vec<_>>();
|
||||
let read_count = Arc::new(AtomicUsize::new(0));
|
||||
let mut source = source_from_data(&erasure, &data, &[]);
|
||||
source.read_count = Some(Arc::clone(&read_count));
|
||||
let engine = LegacyEcDecodeEngine::new(erasure);
|
||||
let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
|
||||
source,
|
||||
engine,
|
||||
data.len(),
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
)
|
||||
.expect("single-inflight reader should be constructed");
|
||||
let mut decoded = Vec::new();
|
||||
|
||||
reader
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect("single-inflight reader should decode the complete body");
|
||||
|
||||
assert_eq!(decoded, data);
|
||||
assert_eq!(
|
||||
read_count.load(Ordering::SeqCst),
|
||||
3,
|
||||
"single-inflight must not read ahead after the final stripe"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_inflight_reader_preserves_partial_reads() {
|
||||
let erasure = Erasure::new(4, 2, 32);
|
||||
let data = (0..83u8).collect::<Vec<_>>();
|
||||
let engine = LegacyEcDecodeEngine::new(erasure.clone());
|
||||
let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
|
||||
source_from_data(&erasure, &data, &[]),
|
||||
engine,
|
||||
data.len(),
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
)
|
||||
.expect("single-inflight reader should be constructed");
|
||||
let mut decoded = Vec::with_capacity(data.len());
|
||||
let mut chunk = [0u8; 3];
|
||||
|
||||
loop {
|
||||
let read = reader.read(&mut chunk).await.expect("partial read should succeed");
|
||||
if read == 0 {
|
||||
break;
|
||||
}
|
||||
decoded.extend_from_slice(&chunk[..read]);
|
||||
}
|
||||
|
||||
assert_eq!(decoded, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_inflight_reader_does_not_prefetch_before_output_is_drained() {
|
||||
let erasure = Erasure::new(4, 2, 32);
|
||||
let data = (0..96u8).collect::<Vec<_>>();
|
||||
let read_count = Arc::new(AtomicUsize::new(0));
|
||||
let mut source = source_from_data(&erasure, &data, &[]);
|
||||
source.read_count = Some(Arc::clone(&read_count));
|
||||
let engine = LegacyEcDecodeEngine::new(erasure);
|
||||
let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
|
||||
source,
|
||||
engine,
|
||||
data.len(),
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
)
|
||||
.expect("single-inflight reader should be constructed");
|
||||
let mut first = [0u8; 3];
|
||||
|
||||
reader
|
||||
.read_exact(&mut first)
|
||||
.await
|
||||
.expect("first partial read should succeed");
|
||||
|
||||
assert_eq!(
|
||||
read_count.load(Ordering::SeqCst),
|
||||
1,
|
||||
"single-inflight must not prefetch while output remains"
|
||||
);
|
||||
assert_eq!(&first, &data[..3]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_inflight_reader_reconstructs_degraded_body() {
|
||||
let erasure = Erasure::new(4, 2, 32);
|
||||
let data = (0..97u16).map(|value| value as u8).collect::<Vec<_>>();
|
||||
let engine = LegacyEcDecodeEngine::new(erasure.clone());
|
||||
let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
|
||||
source_from_data(&erasure, &data, &[1]),
|
||||
engine,
|
||||
data.len(),
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
)
|
||||
.expect("single-inflight reader should be constructed");
|
||||
let mut decoded = Vec::new();
|
||||
|
||||
reader
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect("a readable degraded stripe should be reconstructed");
|
||||
|
||||
assert_eq!(decoded, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_inflight_reader_surfaces_error_after_buffered_body() {
|
||||
let erasure = Erasure::new(4, 2, 32);
|
||||
let first_stripe = (0..32u8).collect::<Vec<_>>();
|
||||
let first_state = source_from_data(&erasure, &first_stripe, &[])
|
||||
.stripes
|
||||
.pop_front()
|
||||
.expect("first stripe should exist");
|
||||
let source = VecStripeSource {
|
||||
stripes: VecDeque::from([
|
||||
first_state,
|
||||
StripeReadState::from_parts(Vec::new(), Vec::new(), erasure.data_shards),
|
||||
]),
|
||||
read_quorum: erasure.data_shards,
|
||||
read_count: None,
|
||||
};
|
||||
let engine = LegacyEcDecodeEngine::new(erasure);
|
||||
let mut reader = ErasureDecodeReader::new_single_inflight_with_metrics_path(
|
||||
source,
|
||||
engine,
|
||||
first_stripe.len() + 1,
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
)
|
||||
.expect("single-inflight reader should be constructed");
|
||||
let mut decoded = Vec::new();
|
||||
|
||||
let error = reader
|
||||
.read_to_end(&mut decoded)
|
||||
.await
|
||||
.expect_err("short source error should be returned after buffered bytes");
|
||||
|
||||
assert_eq!(error.kind(), ErrorKind::Other);
|
||||
assert_eq!(decoded, first_stripe);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_decode_reader_stops_at_eof_for_empty_object() {
|
||||
let erasure = Erasure::new(4, 2, 32);
|
||||
@@ -2053,9 +1882,14 @@ mod tests {
|
||||
};
|
||||
let engine = LegacyEcDecodeEngine::new(Erasure::new(1, 0, 32));
|
||||
let task = tokio::spawn(async move {
|
||||
let mut reader =
|
||||
ErasureDecodeReader::new_single_inflight_with_metrics_path(source, engine, 1, GET_OBJECT_PATH_CODEC_STREAMING)
|
||||
.expect("reader should be constructed");
|
||||
let mut reader = ErasureDecodeReader::new_with_fill_policy(
|
||||
source,
|
||||
engine,
|
||||
1,
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
FillPolicy::SingleInFlight,
|
||||
)
|
||||
.expect("reader should be constructed");
|
||||
let mut first_read = [0u8; 1];
|
||||
let _ = reader.read(&mut first_read).await;
|
||||
});
|
||||
@@ -2392,7 +2226,7 @@ mod tests {
|
||||
engine,
|
||||
data.len(),
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
FillPolicy::DualInFlight,
|
||||
FillPolicy::SingleInFlight,
|
||||
)
|
||||
.expect("reader should be constructed");
|
||||
let mut first_read = [0u8; 1];
|
||||
@@ -2401,11 +2235,13 @@ mod tests {
|
||||
|
||||
assert_eq!(read, first_read.len());
|
||||
assert_eq!(first_read[0], data[0]);
|
||||
assert_eq!(
|
||||
read_count.load(Ordering::SeqCst),
|
||||
2,
|
||||
"dual-inflight reader should prefetch the next stripe before returning the first byte"
|
||||
);
|
||||
timeout(Duration::from_secs(1), async {
|
||||
while read_count.load(Ordering::SeqCst) < 2 {
|
||||
yield_now().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("reader should start reading the next stripe before the current output buffer is fully consumed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -321,13 +321,6 @@ impl<'a> MultiWriter<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn take_retryable_internode_write_failure(&mut self) -> Option<Error> {
|
||||
self.errs
|
||||
.iter_mut()
|
||||
.find(|error| error.as_ref().is_some_and(Error::is_retryable_internode_write_failure))
|
||||
.and_then(Option::take)
|
||||
}
|
||||
|
||||
/// Effective budget for one shard operation: the smaller of the per-shard
|
||||
/// stall timeout and the time remaining until the object's absolute cap.
|
||||
/// Returns `None` when neither deadline is configured (wait indefinitely).
|
||||
|
||||
@@ -933,29 +933,8 @@ impl Erasure {
|
||||
}
|
||||
|
||||
pub(crate) fn decode_data_with_reconstruction_verification(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
||||
self.decode_data_with_reconstruction_verification_policy(shards, false)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_data_with_reconstruction_verification_for_lockstep(
|
||||
&self,
|
||||
shards: &mut [Option<Vec<u8>>],
|
||||
) -> io::Result<()> {
|
||||
self.decode_data_with_reconstruction_verification_policy(shards, true)
|
||||
}
|
||||
|
||||
fn decode_data_with_reconstruction_verification_policy(
|
||||
&self,
|
||||
shards: &mut [Option<Vec<u8>>],
|
||||
require_surplus_source: bool,
|
||||
) -> io::Result<()> {
|
||||
let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none());
|
||||
let available_shards = shards.iter().filter(|shard| shard.is_some()).count();
|
||||
if require_surplus_source && missing_data_source && available_shards == self.data_shards {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"insufficient source shards to verify reconstructed data",
|
||||
));
|
||||
}
|
||||
let source_parity = if missing_data_source && available_shards > self.data_shards {
|
||||
shards
|
||||
.iter()
|
||||
@@ -1889,31 +1868,6 @@ mod tests {
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_data_with_verification_scopes_exact_quorum_to_lockstep() {
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(3, 2, 128, uses_legacy);
|
||||
let data = b"verified reads must not accept reconstruction without a surplus source";
|
||||
let encoded = erasure.encode_data(data).expect("encode should succeed");
|
||||
let mut exact_quorum = optional_shards(&encoded);
|
||||
exact_quorum[0] = None;
|
||||
exact_quorum[erasure.total_shard_count() - 1] = None;
|
||||
|
||||
let mut default_shards = exact_quorum.clone();
|
||||
erasure
|
||||
.decode_data_with_reconstruction_verification(&mut default_shards)
|
||||
.expect("default decode must preserve exact-quorum reconstruction");
|
||||
assert_eq!(default_shards[0].as_deref(), Some(encoded[0].as_ref()));
|
||||
|
||||
let err = erasure
|
||||
.decode_data_with_reconstruction_verification_for_lockstep(&mut exact_quorum)
|
||||
.expect_err("data-shards-only lockstep must reject an exact decode quorum");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("insufficient source shards"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_data_and_parity_rejects_missing_and_mismatched_shards() {
|
||||
let erasure = Erasure::new(4, 2, 128);
|
||||
|
||||
@@ -108,13 +108,6 @@ where
|
||||
(shards, errs)
|
||||
}
|
||||
|
||||
fn heal_writer_failure(writers: &mut MultiWriter<'_>, error: io::Error) -> Error {
|
||||
writers
|
||||
.take_retryable_internode_write_failure()
|
||||
.map(|error| Error::RemoteClientUnavailable(error.to_string()))
|
||||
.unwrap_or_else(|| error.into())
|
||||
}
|
||||
|
||||
impl super::Erasure {
|
||||
pub async fn heal<R>(
|
||||
&self,
|
||||
@@ -209,14 +202,10 @@ impl super::Erasure {
|
||||
.map(|s| Bytes::from(s.unwrap_or_default()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if let Err(error) = writers.write(shards).await {
|
||||
return Err(heal_writer_failure(&mut writers, error));
|
||||
}
|
||||
writers.write(shards).await?;
|
||||
}
|
||||
|
||||
if let Err(error) = writers.shutdown().await {
|
||||
return Err(heal_writer_failure(&mut writers, error));
|
||||
}
|
||||
writers.shutdown().await?;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -257,35 +246,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct InternodeFailureWriter {
|
||||
fail_on_write: bool,
|
||||
status: http::StatusCode,
|
||||
}
|
||||
|
||||
impl InternodeFailureWriter {
|
||||
fn error(&self) -> io::Error {
|
||||
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for InternodeFailureWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
Poll::Ready(if self.fail_on_write {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(buf.len())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Err(self.error()))
|
||||
}
|
||||
}
|
||||
|
||||
struct PendingReader;
|
||||
|
||||
impl AsyncRead for PendingReader {
|
||||
@@ -371,94 +331,6 @@ mod tests {
|
||||
assert!(writers.iter().all(Option::is_some));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_maps_put_file_epoch_conflict_to_retryable_remote_unavailable() {
|
||||
for status in [http::StatusCode::CONFLICT, http::StatusCode::BAD_REQUEST] {
|
||||
for (fail_on_write, data) in [
|
||||
(false, b"".as_slice()),
|
||||
(false, b"payload".as_slice()),
|
||||
(true, b"payload".as_slice()),
|
||||
] {
|
||||
let erasure = Erasure::new(2, 1, 64);
|
||||
let encoded = erasure.encode_data(data).expect("source shards should encode");
|
||||
let readers = encoded
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, shard)| {
|
||||
(index < erasure.data_shards).then(|| {
|
||||
BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut writers = (0..erasure.total_shard_count())
|
||||
.map(|index| {
|
||||
(index == erasure.data_shards).then(|| {
|
||||
BitrotWriterWrapper::new(
|
||||
CustomWriter::new_tokio_writer(InternodeFailureWriter { fail_on_write, status }),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::None,
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let error = erasure
|
||||
.heal(&mut writers, readers, data.len(), &[])
|
||||
.await
|
||||
.expect_err("failed sole target must not satisfy heal write quorum");
|
||||
assert_eq!(
|
||||
matches!(error, Error::RemoteClientUnavailable(_)),
|
||||
status == http::StatusCode::CONFLICT,
|
||||
"status={status}, fail_on_write={fail_on_write}, len={}, error={error:?}",
|
||||
data.len()
|
||||
);
|
||||
assert!(writers.iter().all(Option::is_none), "failed target must not be committed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_epoch_conflict_does_not_abort_healthy_target() {
|
||||
for fail_on_write in [false, true] {
|
||||
let erasure = Erasure::new(2, 2, 64);
|
||||
let data = b"healthy target must retain exact reconstructed bytes";
|
||||
let encoded = erasure.encode_data(data).expect("source shards should encode");
|
||||
let readers = encoded
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, shard)| {
|
||||
(index < erasure.data_shards)
|
||||
.then(|| BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut writers = vec![
|
||||
None,
|
||||
None,
|
||||
Some(BitrotWriterWrapper::new(
|
||||
CustomWriter::new_tokio_writer(InternodeFailureWriter {
|
||||
fail_on_write,
|
||||
status: http::StatusCode::CONFLICT,
|
||||
}),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::None,
|
||||
)),
|
||||
Some(inline_writer(erasure.shard_size())),
|
||||
];
|
||||
erasure
|
||||
.heal(&mut writers, readers, data.len(), &[])
|
||||
.await
|
||||
.expect("one healthy target must still satisfy the existing heal quorum");
|
||||
assert!(writers[2].is_none(), "conflicting target must be dropped");
|
||||
assert_eq!(
|
||||
writers[3]
|
||||
.take()
|
||||
.expect("healthy target remains")
|
||||
.into_inline_data()
|
||||
.expect("inline target data"),
|
||||
encoded[3].to_vec()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_reconstructs_missing_parity_shard() {
|
||||
let erasure = Erasure::new(2, 2, 64);
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user