mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-05 19:55:37 +00:00
Compare commits
70 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| e3ca1ca54c | |||
| ec0a65703a | |||
| 0d1e40ee73 | |||
| 281e40f1cc | |||
| 7541bb2c5d | |||
| 25dd879cf4 | |||
| af1ebbfb8e | |||
| 3e3eb4d8d5 | |||
| 48b6548988 | |||
| 655f6ae452 | |||
| 61821a6f3e | |||
| 896781a52b | |||
| 612dd38fea | |||
| ff28b79088 | |||
| 35456bcede | |||
| 9d4ccb7884 | |||
| 9a22cb85f3 | |||
| 6c67086d0b | |||
| ea01cd339c | |||
| bb37841362 | |||
| 59a7194d7f | |||
| f647ada320 | |||
| 589a954478 | |||
| 1d606e1cf6 | |||
| dc2e25b48c | |||
| d690f5d60d | |||
| 3eca80e37d | |||
| 45a2ccb734 | |||
| c876df53f5 | |||
| 769da6d81f | |||
| ca46ae9e56 | |||
| 7df0920c80 | |||
| c4ac11d22e | |||
| 602ed2cbcd | |||
| b6c3108e53 | |||
| 8ecd8f2520 | |||
| e6234d3714 | |||
| 042a0c3014 | |||
| 87333f7b24 | |||
| 9945c67f7e | |||
| fca1514aac | |||
| 47ad69b691 | |||
| 489408c0b0 | |||
| 1b3744a1da | |||
| 9244eb36ed | |||
| 442298d5f7 | |||
| be7d35d441 | |||
| ec1cd606d3 | |||
| 16af688a7a | |||
| 37b23a16da | |||
| 006e9b7d28 | |||
| d214c27583 | |||
| 8fd364a99c | |||
| c2d8488728 | |||
| 1370434f3a | |||
| 5dde2c188c | |||
| 2f9c75d04f | |||
| 9ee7b1221d | |||
| fcc3c7fb6b | |||
| 01dc55ee5b | |||
| 3d24526704 | |||
| 51532e19fb | |||
| 931ff60182 | |||
| 07212c4e26 | |||
| 4932af080b | |||
| d6f9a7c462 | |||
| 7345b49cf6 | |||
| 4753e35035 | |||
| 96239fc034 | |||
| b428875bed |
@@ -48,6 +48,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect
|
||||
|
||||
### S3 object actions, copy, multipart, and upload policy validation
|
||||
|
||||
- `GHSA-g8w9-qw9q-fghr`: a valid presigned `PutObject` accepted extra `x-amz-tagging`, website redirect, and storage-class headers omitted from `SignedHeaders`. Lesson: a presigned URL is a bounded capability; reject `x-amz-*` headers that are not cryptographically bound by the signature so unsigned metadata cannot change authorization, lifecycle, redirect, cost, or durability semantics.
|
||||
- `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial.
|
||||
- `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies.
|
||||
- `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`.
|
||||
@@ -119,7 +120,7 @@ Use these targeted searches when a diff touches security-sensitive code:
|
||||
```bash
|
||||
rg -n "validate_admin_request|check_permissions|AdminAction::|deny_only|is_allowed" rustfs crates
|
||||
rg -n "authorize_operation|FtpsDriver|SftpDriver|RETR|MKD|SIZE|MDTM|CreateBucket|GetObject|HeadObject" crates/protocols rustfs
|
||||
rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|content-length-range|starts-with" rustfs crates
|
||||
rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|content-length-range|starts-with" rustfs crates
|
||||
rg -n "ListBucketVersions|GetObjectVersion|versionId|VersionId|ExistingObjectTag|ForAllValues|ForAnyValue|POLICY_PLUGIN|opa" rustfs crates
|
||||
rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonicalize|\\.\\.|x-forwarded-for|x-real-ip|SourceIp" rustfs crates
|
||||
rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates
|
||||
@@ -136,6 +137,7 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates
|
||||
- Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend.
|
||||
- IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy.
|
||||
- Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases.
|
||||
- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, and storage-class headers; require rejection before storage access, and verify explicitly signed equivalents still work.
|
||||
- Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases.
|
||||
- Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode.
|
||||
- Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks.
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=9b9bc336b43b70d0e06e0adb5455bf035bb18945d85d60936eb6fe4d48e0e680
|
||||
sha256=9c2b958035a038ffd5ab98cac5f59a1b8e6a16e141f109ec7fb956afc0f11105
|
||||
|
||||
@@ -1 +1 @@
|
||||
sha256=294350518743cac8d7c41880a2835216e4b697908d7b0b1bc92b62816d94c59d
|
||||
sha256=dbebfbab9b9efd4eff31211e69dd32235dc00e207f2ab0dd919a1b2ac9e724c2
|
||||
|
||||
+24
-11
@@ -46,6 +46,11 @@ e2e-reliability = { max-threads = 1 }
|
||||
e2e-inline-boundaries = { max-threads = 1 }
|
||||
e2e-cluster-nightly = { max-threads = 1 }
|
||||
|
||||
# Deep async storage futures are composed into tests across several crates.
|
||||
# Keep the test stack bounded but above libtest's 2 MiB default.
|
||||
[scripts.setup.ecstore-base-stack]
|
||||
command = ['sh', '-c', 'echo RUST_MIN_STACK=4194304 >> "$NEXTEST_ENV"']
|
||||
|
||||
# These exact regression scenarios build deep async storage futures that exceed
|
||||
# libtest's 2 MiB spawned-thread stack on Linux. Give only their test processes
|
||||
# the same 32 MiB stack already used by the crate's dedicated large-stack tests.
|
||||
@@ -63,6 +68,10 @@ command = ['sh', '-c', 'echo RUST_MIN_STACK=33554432 >> "$NEXTEST_ENV"']
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.default.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -116,6 +125,13 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Decommission entry and marker/barrier tests share process-wide fault hooks and
|
||||
# deterministic commit barriers. Keep the whole init decommission family in one
|
||||
# nextest group; serial_test alone cannot isolate separate test processes.
|
||||
[[profile.default.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests. They drive
|
||||
# init_bucket_metadata_sys and bucket_metadata_sys_of, i.e. process-global
|
||||
# OnceLock state that serial_test's #[serial] cannot protect across nextest's
|
||||
@@ -170,6 +186,10 @@ path = "junit.xml"
|
||||
filter = 'package(rustfs-ecstore) & test(/^(bucket::lifecycle::bucket_lifecycle_ops::tests::manual_transition_worker_result_recovery_marks_unknown_for_corrupt_marker|services::rebalance::entry::tests::real_rebalance_run_fence_loss_blocks_multipart_publication|store::init::tests::(decommission_entry_(allows_free_version_consumed_before_source_lock|rejects_subquorum_free_version_conflict_and_retains_source|skips_cleanup_only_marker_when_free_version_is_present)|prepared_tier_delete_recovery_(checks_later_pool_then_commits_after_source_removal|finds_directory_source_on_encoded_set|retains_journal_on_source_metadata_error)|tier_mutation_peer_handler_applies_prepare_commit_and_abort_idempotently|transition_response_loss_persists_unknown_outcome_for_provider_recovery|transition_transaction_recovery_(drops_record_after_confirmed_local_commit|keeps_cleanup_pending_local_commit)))$/)'
|
||||
setup = 'ecstore-large-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
|
||||
setup = 'ecstore-base-stack'
|
||||
|
||||
[[profile.ci.scripts]]
|
||||
filter = 'binary(lifecycle_integration_test) | (package(rustfs) & test(/^app::lifecycle_transition_api_test::/))'
|
||||
setup = 'lifecycle-large-stack'
|
||||
@@ -252,6 +272,10 @@ test-group = 'ecstore-serial-flaky'
|
||||
filter = 'package(rustfs-ecstore) & (test(decommission_migrates_and_verifies_registered_durable_ilm_records) | test(decommission_durable_ilm_target_read_error_is_not_masked_by_peer_success) | test(decommission_durable_ilm_terminal_receipt_recovers_failed_source_cleanup) | test(decommission_durable_ilm_receipt_pagination_fails_closed_on_second_page) | test(decommission_durable_ilm_recovery_keeps_multiple_active_sources))'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
[[profile.ci.overrides]]
|
||||
filter = 'package(rustfs-ecstore) & test(/^store::init::tests::(decommission_|suspended_.*decommission)$/)'
|
||||
test-group = 'ecstore-serial-flaky'
|
||||
|
||||
# Serialize the bucket-incarnation / lifecycle-fence tests under the ci profile
|
||||
# too (see the matching default-profile override near the top). No retries.
|
||||
[[profile.ci.overrides]]
|
||||
@@ -468,23 +492,12 @@ path = "junit.xml"
|
||||
# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
|
||||
# 4-disk reliability / degraded-read fault-injection tests and the fixed-port
|
||||
# Vault tests, both serialized below.
|
||||
# KNOWN-FAILURE EXCLUSIONS (characterization run 29381309848, 2026-07-15:
|
||||
# 341 ran / 32 failed on the suites' first automated run ever). Deterministic
|
||||
# product failures cannot be quarantined away with retries, so each family is
|
||||
# excluded here with its tracking issue, under the same discipline as the
|
||||
# ci-profile quarantine (docs/testing/README.md): every entry MUST cite one
|
||||
# OPEN issue, and the fixing PR MUST delete the exclusion. The passing
|
||||
# negative-path siblings of each family stay in as regression guards.
|
||||
# * rustfs#4843 — over-limit archive entry paths hard-reject the whole
|
||||
# archive even under ignore-errors semantics.
|
||||
[profile.e2e-full]
|
||||
default-filter = """
|
||||
package(e2e_test)
|
||||
& !test(/^protocols::/)
|
||||
& !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|heal_erasure_disk_rebuild_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/)
|
||||
& !test(/^replication_extension_test::/)
|
||||
& !test(/^multipart_auth_test::test_signed_put_object_extract_skips_invalid_entry_when_ignore_errors_enabled$/)
|
||||
& !test(/^snowball_auto_extract_test::tests::snowball_auto_extract_(ignores_invalid_entries_when_requested|supports_standard_headers_with_combined_extract_options)$/)
|
||||
"""
|
||||
fail-fast = false
|
||||
|
||||
|
||||
@@ -43,6 +43,7 @@ env:
|
||||
RUSTFS_API_ENDPOINT: ${{ secrets.RUSTFS_API_ENDPOINT || vars.RUSTFS_API_ENDPOINT || vars.RUSTFS_RC_ENDPOINT }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
|
||||
jobs:
|
||||
@@ -70,11 +71,24 @@ jobs:
|
||||
warp --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Reset test environment (before)
|
||||
- name: Cleanup environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
chmod +x auto-testing/rustfs_heal_test.sh
|
||||
./auto-testing/rustfs_heal_test.sh --reset -y
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Install RustFS package & start cluster
|
||||
run: |
|
||||
@@ -97,6 +111,7 @@ jobs:
|
||||
./auto-testing/rustfs_heal_test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Run heal test (write -> outage -> heal -> verify)
|
||||
id: test
|
||||
run: |
|
||||
./auto-testing/rustfs_heal_test.sh \
|
||||
--steps "3,4,5,6,7" -y \
|
||||
@@ -105,6 +120,211 @@ jobs:
|
||||
--warp-stop-gb "${{ inputs.warp_stop_gb }}" \
|
||||
--log-file /tmp/rustfs-heal-test.log
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-heal-test.log
|
||||
REPORT_FILE: /tmp/rustfs-heal-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
{
|
||||
echo "# RustFS heal test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-heal-report.md
|
||||
SUITE: heal
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -115,10 +335,24 @@ jobs:
|
||||
/tmp/rustfs-warp.*.log
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Reset test environment (after)
|
||||
- name: Cleanup environment (after)
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs_heal_test.sh --reset -y
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
|
||||
@@ -11,8 +11,20 @@ on:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
enforce_sse_key_policy:
|
||||
description: 'Enable RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY (runs KMS-401/402)'
|
||||
type: boolean
|
||||
default: false
|
||||
frame_v2:
|
||||
description: 'Enable RUSTFS_ENCRYPTION_FRAME_V2 (runs KMS-318)'
|
||||
type: boolean
|
||||
default: false
|
||||
config_secret:
|
||||
description: 'Set RUSTFS_KMS_CONFIG_SECRET (runs KMS-107 config sealing)'
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
# Strict shared-environment order: run after S3 compatibility test succeeds.
|
||||
# Strict shared-environment order: run after S3 compatibility test completes.
|
||||
workflows: ["RustFS S3 Compatibility Test"]
|
||||
types: [completed]
|
||||
|
||||
@@ -38,8 +50,9 @@ env:
|
||||
jobs:
|
||||
kms-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -86,6 +99,7 @@ jobs:
|
||||
|
||||
- name: Run KMS suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-kms.log
|
||||
run: |
|
||||
@@ -94,6 +108,19 @@ jobs:
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
ARGS=(--all-topologies --backends "local,vault-kv2" -y --log-file "${LOG_FILE}")
|
||||
EXTRA_ENV=""
|
||||
if [ "${{ inputs.enforce_sse_key_policy }}" = "true" ]; then
|
||||
EXTRA_ENV+="RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true"$'\n'
|
||||
fi
|
||||
if [ "${{ inputs.frame_v2 }}" = "true" ]; then
|
||||
EXTRA_ENV+="RUSTFS_ENCRYPTION_FRAME_V2=true"$'\n'
|
||||
fi
|
||||
if [ -n "${{ inputs.config_secret }}" ]; then
|
||||
EXTRA_ENV+="RUSTFS_KMS_CONFIG_SECRET=${{ inputs.config_secret }}"$'\n'
|
||||
fi
|
||||
if [ -n "${EXTRA_ENV}" ]; then
|
||||
ARGS+=(--extra-env "${EXTRA_ENV}")
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
@@ -119,12 +146,65 @@ jobs:
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-kms-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS KMS test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
@@ -133,6 +213,133 @@ jobs:
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-kms-report.md
|
||||
SUITE: kms
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -12,8 +12,8 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
# Run after the nightly build completes; the nightly deb is what the test installs.
|
||||
workflows: ["Nightly GNU Build"]
|
||||
# Run after upgrade compatibility completes; the nightly deb is what the test installs.
|
||||
workflows: ["RustFS Upgrade Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
@@ -33,10 +33,12 @@ env:
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
s3-compat-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
@@ -76,6 +78,7 @@ jobs:
|
||||
|
||||
- name: Run S3 compatibility suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-s3-compat.log
|
||||
run: |
|
||||
@@ -109,12 +112,79 @@ jobs:
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
RUSTFS_VERSION_INFO="N/A"
|
||||
if [ "${#NODES[@]}" -gt 0 ]; then
|
||||
DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
||||
"${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
|
||||
if [ -n "${DETECTED_VERSION}" ]; then
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-s3-compat-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
current = None
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
current = case_id
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
current = None
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS S3 compatibility test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
@@ -123,6 +193,183 @@ jobs:
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-s3-compat-report.md
|
||||
SUITE: s3
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
|
||||
@@ -0,0 +1,381 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Security Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
topology:
|
||||
description: 'Topology to run (all = SNSD, SNMD, MNMD)'
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- single-single
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
oidc_live:
|
||||
description: 'Run the live Keycloak OIDC/SSO gate as part of the suite'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_before:
|
||||
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_after:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
workflow_run:
|
||||
# Runs last in the functional chain, after pool/heal, on the shared VMs.
|
||||
workflows: ["RustFS Pool Expansion / Heal Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The security suite uses the same shared VMs as the other functional tests,
|
||||
# so it must serialize with them instead of running in parallel.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
security-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Checkout repository (for the OIDC live gate script)
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
aws --version || true
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
||||
'
|
||||
done
|
||||
|
||||
- name: Run security suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
REPORT_FILE: /tmp/rustfs-security-report.md
|
||||
RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-security-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
TOPOLOGY='${{ inputs.topology }}'
|
||||
ARGS=(-y)
|
||||
if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
|
||||
ARGS+=(--all-topologies)
|
||||
else
|
||||
ARGS+=(--topology "${TOPOLOGY}")
|
||||
fi
|
||||
if [ "${{ inputs.oidc_live }}" = "true" ] || [ "${{ github.event_name }}" != "workflow_dispatch" ]; then
|
||||
ARGS+=(--oidc-live)
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ] && [ "${RUSTFS_VERSION}" != "null" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-security-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ ! -f /tmp/rustfs-security-report.md ]; then
|
||||
{
|
||||
echo "# RustFS security test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Test Step Outcome: failure (suite did not produce a report)"
|
||||
} > /tmp/rustfs-security-report.md
|
||||
fi
|
||||
cat /tmp/rustfs-security-report.md >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-security-report.md
|
||||
SUITE: security
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-security-test-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-security-report.md
|
||||
/tmp/rustfs-security.*/*
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS security test failed"
|
||||
echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "See the uploaded report and logs for details."
|
||||
@@ -0,0 +1,422 @@
|
||||
name: RustFS Storage Engine Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
type: string
|
||||
topology:
|
||||
description: 'Topology to run (all = SNSD, SNMD, MNMD)'
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- single-single
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
workflow_run:
|
||||
# Strict shared-environment order: run after tier test completes.
|
||||
workflows: ["RustFS Tier Test"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
storage-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 360
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Run storage engine suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-storage.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-storage-test.sh
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
TOPOLOGY='${{ inputs.topology }}'
|
||||
ARGS=(-y --log-file "${LOG_FILE}")
|
||||
if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
|
||||
ARGS+=(--all-topologies)
|
||||
else
|
||||
ARGS+=(--topology "${TOPOLOGY}")
|
||||
fi
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
ARGS+=(--package-url "${PACKAGE_URL}")
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
ARGS+=(--version "${RUSTFS_VERSION}")
|
||||
else
|
||||
ARGS+=(--package-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-storage-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-storage.log
|
||||
REPORT_FILE: /tmp/rustfs-storage-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
RUSTFS_VERSION_INFO="N/A"
|
||||
if [ "${#NODES[@]}" -gt 0 ]; then
|
||||
DETECTED_VERSION="$(ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new \
|
||||
"${SSH_USER}@${NODES[0]}" 'rustfs --version' 2>/dev/null | tr -d '\r' | head -n 1 || true)"
|
||||
if [ -n "${DETECTED_VERSION}" ]; then
|
||||
RUSTFS_VERSION_INFO="${DETECTED_VERSION}"
|
||||
fi
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-storage-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z0-9]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z0-9]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
current = None
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
current = case_id
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
current = None
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS storage engine test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- RustFS Version: ${RUSTFS_VERSION_INFO}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-storage-report.md
|
||||
SUITE: storage
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'storage', label: 'Storage Engine' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-storage-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-storage.log
|
||||
/tmp/rustfs-storage-report.md
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS storage engine suite failed"
|
||||
echo "See the uploaded report and log artifacts for details."
|
||||
@@ -12,7 +12,7 @@ on:
|
||||
required: false
|
||||
type: string
|
||||
workflow_run:
|
||||
# Strict shared-environment order: run after KMS test succeeds.
|
||||
# Strict shared-environment order: run after KMS test completes.
|
||||
workflows: ["RustFS KMS Test"]
|
||||
types: [completed]
|
||||
|
||||
@@ -38,8 +38,9 @@ env:
|
||||
jobs:
|
||||
tier-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
@@ -60,6 +61,8 @@ jobs:
|
||||
- name: Cleanup environment (before)
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
sudo rm -f /tmp/rustfs-mosquitto.conf
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
@@ -77,18 +80,35 @@ jobs:
|
||||
|
||||
- name: Ensure MQTT broker + clients
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if ! command -v mosquitto_sub >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y mosquitto mosquitto-clients
|
||||
sudo apt-get install -y mosquitto-clients
|
||||
fi
|
||||
sudo mkdir -p /etc/mosquitto/conf.d
|
||||
printf 'listener 1883 0.0.0.0\nallow_anonymous true\n' | sudo tee /etc/mosquitto/conf.d/rustfs-test.conf >/dev/null
|
||||
sudo systemctl restart mosquitto
|
||||
sleep 2
|
||||
ss -tln 2>/dev/null | grep -q ':1883' || { echo 'mosquitto not listening on 1883'; exit 1; }
|
||||
command -v docker >/dev/null 2>&1 || { echo 'docker not found on runner'; exit 1; }
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
cat <<'EOF' | sudo tee /tmp/rustfs-mosquitto.conf >/dev/null
|
||||
listener 1883 0.0.0.0
|
||||
allow_anonymous true
|
||||
EOF
|
||||
sudo docker run -d --name rustfs-test-mqtt -p 1883:1883 \
|
||||
-v /tmp/rustfs-mosquitto.conf:/mosquitto/config/mosquitto.conf:ro \
|
||||
eclipse-mosquitto:2 >/dev/null
|
||||
for _ in {1..10}; do
|
||||
if ss -tln 2>/dev/null | grep -q ':1883'; then
|
||||
break
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
ss -tln 2>/dev/null | grep -q ':1883' || {
|
||||
echo 'mosquitto container is not listening on 1883'
|
||||
sudo docker logs rustfs-test-mqtt || true
|
||||
exit 1
|
||||
}
|
||||
|
||||
- name: Run tier suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-tier.log
|
||||
run: |
|
||||
@@ -122,12 +142,65 @@ jobs:
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-tier-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS tier test report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- Package: ${PACKAGE_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
@@ -136,6 +209,133 @@ jobs:
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-tier-report.md
|
||||
SUITE: tier
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>S3, KMS, Tier report tabs. Each tab lists reports by date.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = files.map(f => `<li><a href="${f.html_url}" target="_blank" rel="noreferrer">${f.name.replace('.md','')}</a></li>`).join('');
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('s3');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
@@ -150,6 +350,8 @@ jobs:
|
||||
if: always()
|
||||
run: |
|
||||
set -euo pipefail
|
||||
sudo docker rm -f rustfs-test-mqtt >/dev/null 2>&1 || true
|
||||
sudo rm -f /tmp/rustfs-mosquitto.conf
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
|
||||
@@ -0,0 +1,477 @@
|
||||
# Copyright 2024 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
name: RustFS Upgrade Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
from_version:
|
||||
description: 'OLD RustFS release tag (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
from_url:
|
||||
description: 'OLD .deb URL. Overrides from_version.'
|
||||
required: false
|
||||
type: string
|
||||
to_version:
|
||||
description: 'NEW RustFS release tag (leave empty for latest nightly)'
|
||||
required: false
|
||||
to_url:
|
||||
description: 'NEW .deb URL. Overrides to_version / nightly default.'
|
||||
required: false
|
||||
type: string
|
||||
topology:
|
||||
description: 'Topology to run (all = SNSD, SNMD, MNMD)'
|
||||
type: choice
|
||||
options:
|
||||
- all
|
||||
- single-single
|
||||
- single-multi
|
||||
- multi-multi
|
||||
default: all
|
||||
backends:
|
||||
description: 'KMS backends to run (local,vault-kv2)'
|
||||
required: false
|
||||
default: 'local,vault-kv2'
|
||||
cleanup_before:
|
||||
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_after:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
workflow_run:
|
||||
# Runs first in the functional chain: upgrade compatibility gates the
|
||||
# nightly suites that follow (S3 -> KMS -> Tier -> Pool/Heal -> Security).
|
||||
workflows: ["Nightly GNU Build"]
|
||||
types: [completed]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
jobs:
|
||||
upgrade-test:
|
||||
runs-on: smoke-testing
|
||||
continue-on-error: true
|
||||
timeout-minutes: 420
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }}
|
||||
steps:
|
||||
- name: Checkout auto-testing scripts
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
with:
|
||||
repository: rustfs/auto-testing
|
||||
ref: main
|
||||
path: auto-testing
|
||||
persist-credentials: false
|
||||
token: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
openssl version
|
||||
aws --version || true
|
||||
docker --version || true
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' || github.event_name != 'workflow_dispatch' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Ensure docker (Vault container)
|
||||
run: |
|
||||
if ! command -v docker >/dev/null 2>&1; then
|
||||
sudo apt-get update
|
||||
sudo apt-get install -y docker.io
|
||||
fi
|
||||
sudo systemctl enable --now docker
|
||||
docker info >/dev/null 2>&1 || sudo docker info >/dev/null 2>&1
|
||||
|
||||
- name: Run upgrade compatibility suite
|
||||
id: test
|
||||
continue-on-error: true
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
chmod +x auto-testing/rustfs-upgrade-test.sh
|
||||
FROM_URL='${{ inputs.from_url }}'
|
||||
FROM_VERSION='${{ inputs.from_version }}'
|
||||
TO_URL='${{ inputs.to_url }}'
|
||||
TO_VERSION='${{ inputs.to_version }}'
|
||||
TOPOLOGY='${{ inputs.topology }}'
|
||||
BACKENDS='${{ inputs.backends }}'
|
||||
ARGS=(-y --log-file "${LOG_FILE}")
|
||||
if [ "${TOPOLOGY}" = "all" ] || [ -z "${TOPOLOGY}" ] || [ "${TOPOLOGY}" = "null" ]; then
|
||||
ARGS+=(--all-topologies)
|
||||
else
|
||||
ARGS+=(--topology "${TOPOLOGY}")
|
||||
fi
|
||||
if [ -n "${BACKENDS}" ] && [ "${BACKENDS}" != "null" ]; then
|
||||
ARGS+=(--backends "${BACKENDS}")
|
||||
fi
|
||||
if [ -n "${FROM_URL}" ]; then
|
||||
ARGS+=(--from-url "${FROM_URL}")
|
||||
elif [ -n "${FROM_VERSION}" ] && [ "${FROM_VERSION}" != "null" ]; then
|
||||
ARGS+=(--from-version "${FROM_VERSION}")
|
||||
fi
|
||||
if [ -n "${TO_URL}" ]; then
|
||||
ARGS+=(--to-url "${TO_URL}")
|
||||
elif [ -n "${TO_VERSION}" ] && [ "${TO_VERSION}" != "null" ]; then
|
||||
ARGS+=(--to-version "${TO_VERSION}")
|
||||
else
|
||||
ARGS+=(--to-url "${RUSTFS_NIGHTLY_PACKAGE_URL}")
|
||||
fi
|
||||
./auto-testing/rustfs-upgrade-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: always()
|
||||
env:
|
||||
LOG_FILE: /tmp/rustfs-upgrade.log
|
||||
REPORT_FILE: /tmp/rustfs-upgrade-report.md
|
||||
run: |
|
||||
set -euo pipefail
|
||||
FROM_URL='${{ inputs.from_url }}'
|
||||
FROM_VERSION='${{ inputs.from_version }}'
|
||||
TO_URL='${{ inputs.to_url }}'
|
||||
TO_VERSION='${{ inputs.to_version }}'
|
||||
if [ -n "${FROM_URL}" ]; then
|
||||
FROM_SOURCE="${FROM_URL}"
|
||||
elif [ -n "${FROM_VERSION}" ]; then
|
||||
FROM_SOURCE="version ${FROM_VERSION}"
|
||||
else
|
||||
FROM_SOURCE="release (default)"
|
||||
fi
|
||||
if [ -n "${TO_URL}" ]; then
|
||||
TO_SOURCE="${TO_URL}"
|
||||
elif [ -n "${TO_VERSION}" ]; then
|
||||
TO_SOURCE="version ${TO_VERSION}"
|
||||
else
|
||||
TO_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
CASE_TABLE="/tmp/rustfs-upgrade-cases.md"
|
||||
python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
|
||||
log_file, out_file = sys.argv[1], sys.argv[2]
|
||||
ansi = re.compile(r'\x1b\[[0-9;]*m')
|
||||
start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$')
|
||||
done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b')
|
||||
|
||||
rows = []
|
||||
index = {}
|
||||
try:
|
||||
with open(log_file, 'r', encoding='utf-8', errors='replace') as fh:
|
||||
for raw in fh:
|
||||
line = ansi.sub('', raw).strip()
|
||||
m = start_re.match(line)
|
||||
if m:
|
||||
case_id, name = m.group(1), m.group(2)
|
||||
if case_id not in index:
|
||||
index[case_id] = len(rows)
|
||||
rows.append([case_id, name, 'RUNNING'])
|
||||
continue
|
||||
m = done_re.match(line)
|
||||
if m:
|
||||
status, case_id = m.group(1), m.group(2)
|
||||
if case_id in index:
|
||||
rows[index[case_id]][2] = status
|
||||
else:
|
||||
rows.append([case_id, case_id, status])
|
||||
index[case_id] = len(rows) - 1
|
||||
except FileNotFoundError:
|
||||
rows = []
|
||||
|
||||
counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0}
|
||||
for _, _, status in rows:
|
||||
counts[status] = counts.get(status, 0) + 1
|
||||
|
||||
with open(out_file, 'w', encoding='utf-8') as out:
|
||||
out.write('## Case Summary\n\n')
|
||||
out.write(f"- Total: {len(rows)}\\n")
|
||||
out.write(f"- PASS: {counts.get('PASS', 0)}\\n")
|
||||
out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n")
|
||||
out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n")
|
||||
out.write('\\n')
|
||||
out.write('| Case | Name | Status |\\n')
|
||||
out.write('| --- | --- | --- |\\n')
|
||||
for case_id, name, status in rows:
|
||||
out.write(f'| {case_id} | {name} | {status} |\\n')
|
||||
PY
|
||||
{
|
||||
echo "# RustFS upgrade compatibility report"
|
||||
echo ""
|
||||
echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
|
||||
echo "- Trigger: ${{ github.event_name }}"
|
||||
echo "- From: ${FROM_SOURCE}"
|
||||
echo "- To: ${TO_SOURCE}"
|
||||
echo "- Test Step Outcome: ${{ steps.test.outcome }}"
|
||||
echo ""
|
||||
cat "${CASE_TABLE}" || true
|
||||
echo ""
|
||||
echo "## Log tail"
|
||||
echo '```text'
|
||||
tail -n 200 "${LOG_FILE}" || true
|
||||
echo '```'
|
||||
} | tee "${REPORT_FILE}"
|
||||
cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}"
|
||||
|
||||
- name: Upload functional report to dashboard
|
||||
if: always()
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }}
|
||||
REPORT_FILE: /tmp/rustfs-upgrade-report.md
|
||||
SUITE: upgrade
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload"
|
||||
exit 0
|
||||
fi
|
||||
DATE="$(date -u +%Y-%m-%d)"
|
||||
REPORT_PATH="functional-reports/${SUITE}/${DATE}.md"
|
||||
CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")"
|
||||
SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${SHA}" ]; then
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
cat > /tmp/rustfs-functional-index.html <<'EOF'
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<title>RustFS Functional Test Reports</title>
|
||||
<style>
|
||||
:root { --bg:#f4f6fb; --card:#fff; --text:#1f2937; --muted:#6b7280; --line:#e5e7eb; --accent:#0f766e; }
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; font-family: ui-sans-serif, -apple-system, Segoe UI, Helvetica, Arial, sans-serif; background: var(--bg); color: var(--text); }
|
||||
.wrap { max-width: 980px; margin: 32px auto; padding: 0 16px; }
|
||||
.card { background: var(--card); border: 1px solid var(--line); border-radius: 14px; padding: 20px; }
|
||||
h1 { margin: 0 0 8px; font-size: 26px; }
|
||||
p { margin: 0 0 14px; color: var(--muted); }
|
||||
.tabs { display: flex; gap: 10px; margin: 14px 0 18px; flex-wrap: wrap; }
|
||||
button { border: 1px solid var(--line); background: #fff; color: var(--text); border-radius: 10px; padding: 8px 14px; cursor: pointer; }
|
||||
button.active { background: var(--accent); color: #fff; border-color: var(--accent); }
|
||||
.report-btn { border: 0; background: transparent; padding: 0; color: var(--accent); }
|
||||
ul { list-style: none; margin: 0; padding: 0; }
|
||||
li { padding: 10px 0; border-bottom: 1px solid var(--line); }
|
||||
a { color: var(--accent); text-decoration: none; }
|
||||
a:hover { text-decoration: underline; }
|
||||
.meta { margin-top: 16px; border-top: 1px solid var(--line); padding-top: 14px; }
|
||||
.kv { margin: 6px 0; color: var(--text); }
|
||||
.muted { color: var(--muted); }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<div class="wrap">
|
||||
<div class="card">
|
||||
<h1>RustFS Functional Test Reports</h1>
|
||||
<p>Select a suite and date to view the build version used in that run.</p>
|
||||
<div class="tabs" id="tabs"></div>
|
||||
<ul id="list"></ul>
|
||||
<div class="meta">
|
||||
<div class="kv"><strong>Date:</strong> <span id="report-date" class="muted">N/A</span></div>
|
||||
<div class="kv"><strong>RustFS Version:</strong> <span id="report-version" class="muted">N/A</span></div>
|
||||
<div class="kv"><a id="report-link" href="#" target="_blank" rel="noreferrer">Open report</a></div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<script>
|
||||
const suites = [
|
||||
{ key: 'upgrade', label: 'Upgrade' },
|
||||
{ key: 's3', label: 'S3 Compatibility' },
|
||||
{ key: 'kms', label: 'KMS' },
|
||||
{ key: 'tier', label: 'Tier' },
|
||||
{ key: 'heal', label: 'Heal' },
|
||||
{ key: 'pool', label: 'Pool Expansion' },
|
||||
{ key: 'security', label: 'Security' },
|
||||
];
|
||||
const tabs = document.getElementById('tabs');
|
||||
const list = document.getElementById('list');
|
||||
const reportDate = document.getElementById('report-date');
|
||||
const reportVersion = document.getElementById('report-version');
|
||||
const reportLink = document.getElementById('report-link');
|
||||
|
||||
function parseVersion(markdown) {
|
||||
const m = markdown.match(/^- RustFS Version:\s*(.+)$/m);
|
||||
return m ? m[1].trim() : 'N/A';
|
||||
}
|
||||
|
||||
async function showReport(report) {
|
||||
reportDate.textContent = report.name.replace('.md', '');
|
||||
reportVersion.textContent = 'Loading...';
|
||||
reportLink.href = report.html_url;
|
||||
try {
|
||||
const res = await fetch(report.download_url, { cache: 'no-store' });
|
||||
if (!res.ok) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
return;
|
||||
}
|
||||
const text = await res.text();
|
||||
reportVersion.textContent = parseVersion(text);
|
||||
} catch (_e) {
|
||||
reportVersion.textContent = 'N/A';
|
||||
}
|
||||
}
|
||||
|
||||
async function loadSuite(suite) {
|
||||
list.innerHTML = '<li>Loading...</li>';
|
||||
const api = `https://api.github.com/repos/rustfs/dashboard/contents/functional-reports/${suite}`;
|
||||
try {
|
||||
const res = await fetch(api);
|
||||
if (!res.ok) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
return;
|
||||
}
|
||||
const data = await res.json();
|
||||
const files = data.filter(f => f.type === 'file' && f.name.endsWith('.md')).sort((a,b) => b.name.localeCompare(a.name));
|
||||
if (!files.length) {
|
||||
list.innerHTML = '<li>No reports yet.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
return;
|
||||
}
|
||||
list.innerHTML = '';
|
||||
files.forEach((f) => {
|
||||
const li = document.createElement('li');
|
||||
const btn = document.createElement('button');
|
||||
btn.className = 'report-btn';
|
||||
btn.textContent = f.name.replace('.md', '');
|
||||
btn.addEventListener('click', () => showReport(f));
|
||||
li.appendChild(btn);
|
||||
list.appendChild(li);
|
||||
});
|
||||
showReport(files[0]);
|
||||
} catch (_e) {
|
||||
list.innerHTML = '<li>Failed to load reports.</li>';
|
||||
reportDate.textContent = 'N/A';
|
||||
reportVersion.textContent = 'N/A';
|
||||
reportLink.href = '#';
|
||||
}
|
||||
}
|
||||
|
||||
function setActive(key) {
|
||||
for (const btn of tabs.querySelectorAll('button')) {
|
||||
btn.classList.toggle('active', btn.dataset.key === key);
|
||||
}
|
||||
loadSuite(key);
|
||||
}
|
||||
|
||||
for (const suite of suites) {
|
||||
const btn = document.createElement('button');
|
||||
btn.textContent = suite.label;
|
||||
btn.dataset.key = suite.key;
|
||||
btn.addEventListener('click', () => setActive(suite.key));
|
||||
tabs.appendChild(btn);
|
||||
}
|
||||
setActive('upgrade');
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
EOF
|
||||
|
||||
INDEX_PATH="functional/index.html"
|
||||
INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')"
|
||||
INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)"
|
||||
if [ -n "${INDEX_SHA}" ]; then
|
||||
jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \
|
||||
'{message:$msg, content:$content, sha:$sha}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
else
|
||||
jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \
|
||||
'{message:$msg, content:$content}' \
|
||||
| gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null
|
||||
fi
|
||||
|
||||
- name: Upload report and logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-upgrade-test-${{ github.run_id }}
|
||||
path: |
|
||||
/tmp/rustfs-upgrade-report.md
|
||||
/tmp/rustfs-upgrade.*/*
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: ${{ always() && (inputs.cleanup_after != 'false' || github.event_name != 'workflow_dispatch') }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}"
|
||||
SSH_USER="${RUSTFS_SSH_USER:-azureuser}"
|
||||
for node in "${NODES[@]}"; do
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" '
|
||||
set -euo pipefail
|
||||
SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n"
|
||||
${SUDO} systemctl stop rustfs 2>/dev/null || true
|
||||
if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then
|
||||
${SUDO} dpkg -P rustfs
|
||||
fi
|
||||
for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done
|
||||
${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup
|
||||
'
|
||||
done
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS upgrade compatibility test failed"
|
||||
echo "From: ${{ inputs.from_url || inputs.from_version || 'release (default)' }}"
|
||||
echo "To: ${{ inputs.to_url || inputs.to_version || 'nightly (R2 latest)' }}"
|
||||
echo "See the uploaded report and logs for details."
|
||||
Generated
+61
-52
@@ -627,8 +627,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "astral-tokio-tar"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f2e989b33246fe9240d39accf4dd9a01e0b6c1f3ce9dd095e0a47fa02505523"
|
||||
source = "git+https://github.com/cxymds/tokio-tar.git?rev=603756478b7668436e464519c77ccac22a99ba96#603756478b7668436e464519c77ccac22a99ba96"
|
||||
dependencies = [
|
||||
"futures-core",
|
||||
"libc",
|
||||
@@ -2163,6 +2162,7 @@ dependencies = [
|
||||
"compression-core",
|
||||
"flate2",
|
||||
"liblzma",
|
||||
"lz4",
|
||||
"memchr",
|
||||
"zstd",
|
||||
"zstd-safe",
|
||||
@@ -3916,7 +3916,7 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
|
||||
|
||||
[[package]]
|
||||
name = "e2e_test"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"astral-tokio-tar",
|
||||
@@ -3926,6 +3926,7 @@ dependencies = [
|
||||
"aws-sdk-s3",
|
||||
"aws-sdk-sts",
|
||||
"aws-smithy-http-client",
|
||||
"aws-smithy-types",
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
"chrono",
|
||||
@@ -3943,6 +3944,7 @@ dependencies = [
|
||||
"hyper-util",
|
||||
"local-ip-address",
|
||||
"md-5 0.11.0",
|
||||
"minlz",
|
||||
"opentelemetry-proto",
|
||||
"prost 0.14.4",
|
||||
"rand 0.10.2",
|
||||
@@ -9384,7 +9386,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -9525,7 +9527,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-audit"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"futures",
|
||||
@@ -9547,7 +9549,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-checksums"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -9563,7 +9565,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"metrics",
|
||||
@@ -9576,7 +9578,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-concurrency"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"insta",
|
||||
@@ -9589,7 +9591,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"const-str",
|
||||
"hotpath",
|
||||
@@ -9599,7 +9601,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-credentials"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"hmac 0.13.0",
|
||||
@@ -9613,7 +9615,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
@@ -9634,7 +9636,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-data-usage"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rmp-serde",
|
||||
@@ -9644,7 +9646,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-channel",
|
||||
@@ -9779,7 +9781,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-extension-schema"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -9789,7 +9791,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"byteorder",
|
||||
@@ -9816,7 +9818,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -9852,7 +9854,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-heal-contracts"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -9862,7 +9864,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -9910,7 +9912,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-core"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"hotpath",
|
||||
@@ -9922,7 +9924,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-io-metrics"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"hotpath",
|
||||
@@ -9986,7 +9988,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-keystone"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -10013,7 +10015,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-kms"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"anyhow",
|
||||
@@ -10063,7 +10065,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lifecycle"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10086,7 +10088,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"compact_str",
|
||||
@@ -10109,7 +10111,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-log-analyzer"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"flate2",
|
||||
@@ -10128,7 +10130,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"http 1.5.0",
|
||||
@@ -10166,7 +10168,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
@@ -10201,7 +10203,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-capacity"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"criterion",
|
||||
"futures",
|
||||
@@ -10220,7 +10222,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-object-data-cache"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"criterion",
|
||||
@@ -10237,7 +10239,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-obs"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"crossbeam-channel",
|
||||
@@ -10295,7 +10297,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
@@ -10326,7 +10328,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protocols"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"astral-tokio-tar",
|
||||
"async-compression",
|
||||
@@ -10388,7 +10390,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
@@ -10413,7 +10415,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-replication"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"byteorder",
|
||||
"bytes",
|
||||
@@ -10431,7 +10433,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"arc-swap",
|
||||
@@ -10448,6 +10450,7 @@ dependencies = [
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"md-5 0.11.0",
|
||||
"minlz",
|
||||
"pin-project-lite",
|
||||
"rand 0.10.2",
|
||||
"reqwest",
|
||||
@@ -10471,7 +10474,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio-v2"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"bytes",
|
||||
@@ -10494,7 +10497,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-client"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10538,7 +10541,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-ops"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-s3-types",
|
||||
@@ -10546,7 +10549,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3-types"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"serde",
|
||||
@@ -10555,12 +10558,16 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-compression",
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"crc-fast",
|
||||
"datafusion",
|
||||
"flate2",
|
||||
"futures",
|
||||
"futures-core",
|
||||
"hotpath",
|
||||
@@ -10576,6 +10583,7 @@ dependencies = [
|
||||
"serial_test",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"transform-stream",
|
||||
@@ -10585,7 +10593,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
@@ -10604,7 +10612,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -10647,7 +10655,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-scanner-contracts"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"jiff",
|
||||
@@ -10662,7 +10670,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-security-governance"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"thiserror 2.0.20",
|
||||
@@ -10670,7 +10678,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"bytes",
|
||||
@@ -10688,7 +10696,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-storage-api"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"hotpath",
|
||||
@@ -10703,7 +10711,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-targets"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-nats",
|
||||
@@ -10757,7 +10765,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-test-utils"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"hotpath",
|
||||
"rustfs-data-usage",
|
||||
@@ -10773,7 +10781,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-tls-runtime"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"hotpath",
|
||||
@@ -10794,7 +10802,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-trusted-proxies"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
@@ -10831,7 +10839,7 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-utils"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"blake2",
|
||||
@@ -10873,10 +10881,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"hotpath",
|
||||
"rustfs-rio",
|
||||
"thiserror 2.0.20",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
+52
-51
@@ -72,7 +72,7 @@ edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-rc.4"
|
||||
version = "1.0.0-rc.5"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
keywords = ["RustFS", "Minio", "object-storage", "filesystem", "s3"]
|
||||
@@ -89,55 +89,55 @@ redundant_clone = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
# RustFS Internal Crates
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.4" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.4" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.4" }
|
||||
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.4" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.4" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.4" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.4" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.4" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.4" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.4" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.4" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.4" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.4" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.4" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.4" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.4" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.4" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.4" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.4" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.4" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.4" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.4" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.4" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.4" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.4", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.4" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.4" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.4" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.4" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.4" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.4" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.4" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.4" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.4" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.4" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.4" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.4" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.4" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.4" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.4" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.4" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.4" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.4" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.4" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.4" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.4" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.4" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.4" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.4" }
|
||||
rustfs = { path = "./rustfs", version = "1.0.0-rc.5" }
|
||||
rustfs-heal = { path = "crates/heal", version = "1.0.0-rc.5" }
|
||||
rustfs-heal-contracts = { path = "crates/heal-contracts", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner-contracts = { path = "crates/scanner-contracts", version = "1.0.0-rc.5" }
|
||||
rustfs-audit = { path = "crates/audit", version = "1.0.0-rc.5" }
|
||||
rustfs-checksums = { path = "crates/checksums", version = "1.0.0-rc.5" }
|
||||
rustfs-common = { path = "crates/common", version = "1.0.0-rc.5" }
|
||||
rustfs-data-usage = { path = "crates/data-usage", version = "1.0.0-rc.5" }
|
||||
rustfs-config = { path = "./crates/config", version = "1.0.0-rc.5" }
|
||||
rustfs-concurrency = { path = "./crates/concurrency", version = "1.0.0-rc.5" }
|
||||
rustfs-credentials = { path = "crates/credentials", version = "1.0.0-rc.5" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "1.0.0-rc.5" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "1.0.0-rc.5" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "1.0.0-rc.5" }
|
||||
rustfs-iam = { path = "crates/iam", version = "1.0.0-rc.5" }
|
||||
rustfs-keystone = { path = "crates/keystone", version = "1.0.0-rc.5" }
|
||||
rustfs-lifecycle = { path = "crates/lifecycle", version = "1.0.0-rc.5" }
|
||||
rustfs-kms = { path = "crates/kms", version = "1.0.0-rc.5" }
|
||||
rustfs-lock = { path = "crates/lock", version = "1.0.0-rc.5" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "1.0.0-rc.5" }
|
||||
rustfs-notify = { path = "crates/notify", version = "1.0.0-rc.5" }
|
||||
rustfs-io-metrics = { path = "crates/io-metrics", version = "1.0.0-rc.5" }
|
||||
rustfs-io-core = { path = "crates/io-core", version = "1.0.0-rc.5" }
|
||||
rustfs-object-capacity = { path = "crates/object-capacity", version = "1.0.0-rc.5" }
|
||||
rustfs-object-data-cache = { path = "crates/object-data-cache", version = "1.0.0-rc.5", default-features = false }
|
||||
rustfs-log-analyzer = { path = "crates/log-analyzer", version = "1.0.0-rc.5" }
|
||||
rustfs-obs = { path = "crates/obs", version = "1.0.0-rc.5" }
|
||||
rustfs-policy = { path = "crates/policy", version = "1.0.0-rc.5" }
|
||||
rustfs-protos = { path = "crates/protos", version = "1.0.0-rc.5" }
|
||||
rustfs-protocols = { path = "crates/protocols", version = "1.0.0-rc.5" }
|
||||
rustfs-replication = { path = "crates/replication", version = "1.0.0-rc.5" }
|
||||
rustfs-rio = { path = "crates/rio", version = "1.0.0-rc.5" }
|
||||
rustfs-rio-v2 = { path = "crates/rio-v2", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-client = { path = "crates/s3-client", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-types = { path = "crates/s3-types", version = "1.0.0-rc.5" }
|
||||
rustfs-s3-ops = { path = "crates/s3-ops", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "1.0.0-rc.5" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "1.0.0-rc.5" }
|
||||
rustfs-scanner = { path = "crates/scanner", version = "1.0.0-rc.5" }
|
||||
rustfs-security-governance = { path = "crates/security-governance", version = "1.0.0-rc.5" }
|
||||
rustfs-extension-schema = { path = "crates/extension-schema", version = "1.0.0-rc.5" }
|
||||
rustfs-signer = { path = "crates/signer", version = "1.0.0-rc.5" }
|
||||
rustfs-storage-api = { path = "crates/storage-api", version = "1.0.0-rc.5" }
|
||||
rustfs-trusted-proxies = { path = "crates/trusted-proxies", version = "1.0.0-rc.5" }
|
||||
rustfs-targets = { path = "crates/targets", version = "1.0.0-rc.5" }
|
||||
rustfs-test-utils = { path = "crates/test-utils", version = "1.0.0-rc.5" }
|
||||
rustfs-tls-runtime = { path = "crates/tls-runtime", version = "1.0.0-rc.5" }
|
||||
rustfs-utils = { path = "crates/utils", version = "1.0.0-rc.5" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "1.0.0-rc.5" }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -232,7 +232,8 @@ tokio-postgres-rustls = "0.14.0"
|
||||
# Utilities and Tools
|
||||
anyhow = "1.0.104"
|
||||
arc-swap = "1.9.2"
|
||||
astral-tokio-tar = "0.7.0"
|
||||
# RUSTFS_COMPAT_TODO(tokio-tar-extension-limits): keep the fork pin until bounded extension parsing is released upstream. Remove after astral-sh/tokio-tar#118 is merged and a published tokio-tar release exposes the extension limits used here.
|
||||
astral-tokio-tar = { git = "https://github.com/cxymds/tokio-tar.git", rev = "603756478b7668436e464519c77ccac22a99ba96" }
|
||||
atoi = "3.1.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.11.0" }
|
||||
|
||||
@@ -115,7 +115,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# Using specific version
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
```
|
||||
|
||||
If you use [podman](https://github.com/containers/podman) instead of docker, you can install the RustFS with the below command
|
||||
|
||||
+1
-1
@@ -112,7 +112,7 @@ chown -R 10001:10001 data logs
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:latest
|
||||
|
||||
# 使用指定版本运行
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.4
|
||||
docker run -d -p 9000:9000 -p 9001:9001 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0-rc.5
|
||||
```
|
||||
|
||||
如果您通过绑定挂载启用 TLS 证书目录,也请用同样方式准备该目录:
|
||||
|
||||
@@ -59,20 +59,20 @@ pub const ENV_CAPACITY_MAX_TIMEOUT: &str = "RUSTFS_CAPACITY_MAX_TIMEOUT";
|
||||
// ============================================================================
|
||||
|
||||
/// Scheduled update interval in seconds
|
||||
/// Default: 120 seconds (2 minutes)
|
||||
pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 120;
|
||||
/// Default: 600 seconds (10 minutes)
|
||||
pub const DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS: u64 = 600;
|
||||
|
||||
/// Write trigger delay in seconds
|
||||
/// Default: 5 seconds
|
||||
pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 5;
|
||||
/// Default: 30 seconds
|
||||
pub const DEFAULT_WRITE_TRIGGER_DELAY_SECS: u64 = 30;
|
||||
|
||||
/// Write frequency threshold (writes per minute)
|
||||
/// Default: 5 writes/minute
|
||||
pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 5;
|
||||
/// Default: 20 writes/minute
|
||||
pub const DEFAULT_WRITE_FREQUENCY_THRESHOLD: usize = 20;
|
||||
|
||||
/// Fast update threshold in seconds
|
||||
/// Default: 30 seconds
|
||||
pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 30;
|
||||
/// Default: 120 seconds
|
||||
pub const DEFAULT_FAST_UPDATE_THRESHOLD_SECS: u64 = 120;
|
||||
|
||||
/// Maximum files threshold for sampling
|
||||
/// Default: 200,000 files
|
||||
@@ -129,4 +129,16 @@ mod tests {
|
||||
assert_eq!(ENV_CAPACITY_MIN_TIMEOUT, "RUSTFS_CAPACITY_MIN_TIMEOUT");
|
||||
assert_eq!(ENV_CAPACITY_MAX_TIMEOUT, "RUSTFS_CAPACITY_MAX_TIMEOUT");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_capacity_default_values() {
|
||||
assert_eq!(DEFAULT_SCHEDULED_UPDATE_INTERVAL_SECS, 600);
|
||||
assert_eq!(DEFAULT_WRITE_TRIGGER_DELAY_SECS, 30);
|
||||
assert_eq!(DEFAULT_WRITE_FREQUENCY_THRESHOLD, 20);
|
||||
assert_eq!(DEFAULT_FAST_UPDATE_THRESHOLD_SECS, 120);
|
||||
assert_eq!(DEFAULT_MAX_FILES_THRESHOLD, 200_000);
|
||||
assert_eq!(DEFAULT_STAT_TIMEOUT_SECS, 3);
|
||||
assert_eq!(DEFAULT_SAMPLE_RATE, 200);
|
||||
assert_eq!(DEFAULT_CAPACITY_METRICS_INTERVAL_SECS, 600);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -198,11 +198,11 @@ pub const ENV_SCANNER_IDLE_MODE: &str = "RUSTFS_SCANNER_IDLE_MODE";
|
||||
/// Environment variable that controls scanner cache save timeout in seconds.
|
||||
/// The scanner enforces a minimum value of `1`.
|
||||
/// - Unit: seconds (u64).
|
||||
/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=30`
|
||||
/// - Example: `export RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS=14`
|
||||
pub const ENV_SCANNER_CACHE_SAVE_TIMEOUT_SECS: &str = "RUSTFS_SCANNER_CACHE_SAVE_TIMEOUT_SECS";
|
||||
|
||||
/// Default scanner cache save timeout in seconds.
|
||||
pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 30;
|
||||
pub const DEFAULT_SCANNER_CACHE_SAVE_TIMEOUT_SECS: u64 = 14;
|
||||
|
||||
/// Environment variable that caps concurrent scanner set tasks.
|
||||
/// A value of `0` keeps the existing topology-based concurrency.
|
||||
|
||||
@@ -100,7 +100,8 @@ aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a",
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] }
|
||||
aws-smithy-types.workspace = true
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
async-trait = { workspace = true }
|
||||
flate2.workspace = true
|
||||
http.workspace = true
|
||||
@@ -114,6 +115,7 @@ rustfs-signer.workspace = true
|
||||
# server's implementation: a shared helper could agree with a bug on both sides.
|
||||
data-encoding = { workspace = true }
|
||||
hmac = { workspace = true }
|
||||
minlz.workspace = true
|
||||
sha1 = { workspace = true }
|
||||
serde_urlencoded = { workspace = true }
|
||||
tracing = { workspace = true }
|
||||
|
||||
@@ -27,8 +27,10 @@
|
||||
//! Readiness is established by the harness's `start()` handshake (TCP reachability
|
||||
//! plus an S3 `ListBuckets` poll) — there are no fixed sleeps.
|
||||
//!
|
||||
//! Out of scope for this block (tracked separately): network fault injection
|
||||
//! (toxiproxy / socket proxy) and 5GiB large-object budgets.
|
||||
//! The volume-proxy smoke below also proves that the socket-level fault proxy
|
||||
//! can be installed before startup without changing the client-facing node URL.
|
||||
//! A full lock-plane partition matrix and 5GiB large-object budget remain
|
||||
//! tracked separately.
|
||||
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment};
|
||||
|
||||
@@ -76,6 +78,28 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// 4 nodes x 4 drives, single pool: exercise the maximum local erasure layout
|
||||
/// supported by the cluster harness. This remains in the nightly lane because
|
||||
/// it starts four real server processes and sixteen data directories.
|
||||
#[tokio::test]
|
||||
async fn cluster_four_node_four_drive_single_pool_smoke() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?;
|
||||
|
||||
let volumes = cluster.rustfs_volumes_arg();
|
||||
assert_eq!(volumes.split(' ').count(), 16, "expected 16 explicit endpoints, got: {volumes}");
|
||||
assert!(!volumes.contains('{'), "single-pool layout must not use ellipses: {volumes}");
|
||||
assert!(cluster.nodes.iter().all(|node| node.data_dirs.len() == 4));
|
||||
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
|
||||
let payload = vec![0x3Cu8; 1024 * 1024];
|
||||
put_get_roundtrip(&cluster, "multidrive-4/object", &payload).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Two single-node pools, 2 drives each: the multi-pool layout boots and
|
||||
/// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1).
|
||||
#[tokio::test]
|
||||
@@ -103,3 +127,27 @@ async fn cluster_two_pool_smoke() -> TestResult {
|
||||
put_get_roundtrip(&cluster, "twopool/object", &payload).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// A real cluster smoke for the volume FaultProxy wiring. The proxy target is
|
||||
/// not listening yet when it is created; cluster startup must still converge
|
||||
/// once the target node starts, and peer disk/RPC traffic must traverse it.
|
||||
#[tokio::test]
|
||||
async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult {
|
||||
crate::common::init_logging();
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(2, 2)).await?;
|
||||
let proxy = cluster.start_volume_proxy_for_node(0).await?;
|
||||
let proxied = proxy.local_addr().to_string();
|
||||
assert!(cluster.rustfs_volumes_arg().contains(&proxied));
|
||||
|
||||
let result: TestResult = async {
|
||||
cluster.start().await?;
|
||||
cluster.create_test_bucket(BUCKET).await?;
|
||||
let payload = vec![0x6Du8; 256 * 1024];
|
||||
put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await
|
||||
}
|
||||
.await;
|
||||
|
||||
proxy.shutdown().await;
|
||||
result
|
||||
}
|
||||
|
||||
@@ -34,6 +34,7 @@ use serde_json;
|
||||
use std::ffi::OsStr;
|
||||
use std::fs as stdfs;
|
||||
use std::io::ErrorKind;
|
||||
use std::net::SocketAddr;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::sync::Once;
|
||||
@@ -1214,6 +1215,9 @@ pub struct RustFSTestClusterEnvironment {
|
||||
pub node_extra_env: Vec<Vec<(String, String)>>,
|
||||
pub node_capture_log_paths: Vec<Option<String>>,
|
||||
pub topology: ClusterTopology,
|
||||
/// Optional socket proxies used for the corresponding node's volume
|
||||
/// endpoints. Proxies must be installed before [`Self::start`].
|
||||
volume_proxy_addresses: Vec<Option<SocketAddr>>,
|
||||
}
|
||||
|
||||
impl RustFSTestClusterEnvironment {
|
||||
@@ -1305,6 +1309,7 @@ impl RustFSTestClusterEnvironment {
|
||||
extra_env.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string()));
|
||||
}
|
||||
|
||||
let node_count = topology.node_count;
|
||||
Ok(Self {
|
||||
nodes,
|
||||
temp_dir,
|
||||
@@ -1314,6 +1319,7 @@ impl RustFSTestClusterEnvironment {
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
volume_proxy_addresses: vec![None; node_count],
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1381,6 +1387,34 @@ impl RustFSTestClusterEnvironment {
|
||||
self.build_volumes_arg()
|
||||
}
|
||||
|
||||
/// Start a socket proxy for one node's volume endpoints and route all
|
||||
/// subsequent `RUSTFS_VOLUMES` references for that node through it.
|
||||
///
|
||||
/// Call this before [`Self::start`], then use the returned proxy's
|
||||
/// [`crate::fault_proxy::FaultProxy::set_mode`] to inject latency,
|
||||
/// blackhole, or one-way partition faults. The node's own listen address
|
||||
/// remains direct, so S3 clients can still reach it while peer disk/RPC
|
||||
/// traffic is steered through the proxy.
|
||||
pub async fn start_volume_proxy_for_node(
|
||||
&mut self,
|
||||
node_idx: usize,
|
||||
) -> Result<crate::fault_proxy::FaultProxy, Box<dyn std::error::Error + Send + Sync>> {
|
||||
self.ensure_node_index(node_idx)?;
|
||||
if self.volume_proxy_addresses[node_idx].is_some() {
|
||||
return Err(format!("a volume proxy is already configured for node {node_idx}").into());
|
||||
}
|
||||
let target = self.nodes[node_idx].address.parse::<SocketAddr>()?;
|
||||
let proxy = crate::fault_proxy::FaultProxy::start(target).await?;
|
||||
self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr());
|
||||
Ok(proxy)
|
||||
}
|
||||
|
||||
fn volume_address(&self, node_idx: usize) -> String {
|
||||
self.volume_proxy_addresses[node_idx]
|
||||
.map(|address| address.to_string())
|
||||
.unwrap_or_else(|| self.nodes[node_idx].address.clone())
|
||||
}
|
||||
|
||||
fn build_volumes_arg(&self) -> String {
|
||||
let pools = self.topology.normalized_pools();
|
||||
|
||||
@@ -1389,7 +1423,11 @@ impl RustFSTestClusterEnvironment {
|
||||
return self
|
||||
.nodes
|
||||
.iter()
|
||||
.flat_map(|n| n.data_dirs.iter().map(move |dir| format!("http://{}{}", n.address, dir)))
|
||||
.enumerate()
|
||||
.flat_map(|(node_idx, n)| {
|
||||
let address = self.volume_address(node_idx);
|
||||
n.data_dirs.iter().map(move |dir| format!("http://{}{}", address, dir))
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ");
|
||||
}
|
||||
@@ -1400,13 +1438,19 @@ impl RustFSTestClusterEnvironment {
|
||||
pools
|
||||
.iter()
|
||||
.map(|nodes| {
|
||||
let node = &self.nodes[nodes[0]];
|
||||
let node_idx = nodes[0];
|
||||
let node = &self.nodes[node_idx];
|
||||
let base = node
|
||||
.data_dirs
|
||||
.first()
|
||||
.and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent))
|
||||
.unwrap_or(&node.data_dir);
|
||||
format!("http://{}{}/drive{{0...{}}}", node.address, base, self.topology.drives_per_node - 1)
|
||||
format!(
|
||||
"http://{}{}/drive{{0...{}}}",
|
||||
self.volume_address(node_idx),
|
||||
base,
|
||||
self.topology.drives_per_node - 1
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
@@ -2000,7 +2044,7 @@ mod tests {
|
||||
}
|
||||
let multidrive = topology.drives_per_node > 1;
|
||||
|
||||
let nodes = (0..topology.node_count)
|
||||
let nodes: Vec<ClusterNode> = (0..topology.node_count)
|
||||
.map(|i| {
|
||||
let address = format!("127.0.0.1:{}", 9000 + i);
|
||||
let data_dirs: Vec<String> = if multidrive {
|
||||
@@ -2021,6 +2065,7 @@ mod tests {
|
||||
})
|
||||
.collect();
|
||||
|
||||
let node_count = nodes.len();
|
||||
RustFSTestClusterEnvironment {
|
||||
nodes,
|
||||
temp_dir,
|
||||
@@ -2030,6 +2075,7 @@ mod tests {
|
||||
node_extra_env: vec![Vec::new(); topology.node_count],
|
||||
node_capture_log_paths: vec![None; topology.node_count],
|
||||
topology,
|
||||
volume_proxy_addresses: vec![None; node_count],
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2114,6 +2160,25 @@ mod tests {
|
||||
assert!(ClusterTopology::single_pool_multidrive(1, 1).validate().is_ok());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn volume_proxy_rewrites_cluster_volume_endpoint() {
|
||||
let mut env = RustFSTestClusterEnvironment::new(1)
|
||||
.await
|
||||
.expect("cluster environment should allocate a node");
|
||||
let direct = env.nodes[0].address.clone();
|
||||
let proxy = env
|
||||
.start_volume_proxy_for_node(0)
|
||||
.await
|
||||
.expect("volume proxy should bind before the target server starts");
|
||||
let proxied = proxy.local_addr().to_string();
|
||||
let volumes = env.rustfs_volumes_arg();
|
||||
|
||||
assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}");
|
||||
assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}");
|
||||
|
||||
proxy.shutdown().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_node_env_supports_per_node_overrides() {
|
||||
let mut env = fake_cluster(ClusterTopology::single_pool(4));
|
||||
|
||||
@@ -16,13 +16,14 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::chaos::signed_admin_post;
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, init_logging};
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk, signed_admin_post};
|
||||
use crate::common::{RustFSTestClusterEnvironment, RustFSTestEnvironment, admin_request, init_logging};
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use http::Method;
|
||||
use std::collections::HashSet;
|
||||
use std::error::Error;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::time::{Duration, sleep, timeout};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::info;
|
||||
|
||||
fn has_file_under(path: &Path) -> bool {
|
||||
@@ -48,6 +49,110 @@ mod tests {
|
||||
disk.join(bucket).join(key).join("xl.meta").is_file()
|
||||
}
|
||||
|
||||
// Healing may rewrite non-identity bookkeeping in xl.meta. The census
|
||||
// therefore compares the canonical selected metadata fields plus every
|
||||
// physical shard, while the payload seed makes object mix-ups observable.
|
||||
#[derive(Debug)]
|
||||
struct PhysicalObjectManifest {
|
||||
key: String,
|
||||
payload_seed: u8,
|
||||
shard_census: VersionShardCensus,
|
||||
}
|
||||
|
||||
fn deterministic_object_body(len: usize, seed: u8) -> Vec<u8> {
|
||||
let mut value = seed;
|
||||
std::iter::repeat_with(|| {
|
||||
value = value.wrapping_mul(31).wrapping_add(17);
|
||||
value
|
||||
})
|
||||
.take(len)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn matching_manifest_count(
|
||||
disk: &Path,
|
||||
bucket: &str,
|
||||
expected_manifests: &[PhysicalObjectManifest],
|
||||
) -> Result<usize, Box<dyn Error + Send + Sync>> {
|
||||
let mut matching = 0;
|
||||
for expected in expected_manifests {
|
||||
let actual = census_object_version_on_disk(disk, bucket, &expected.key, None)?;
|
||||
if actual.matches_manifest(&expected.shard_census) {
|
||||
matching += 1;
|
||||
}
|
||||
}
|
||||
Ok(matching)
|
||||
}
|
||||
|
||||
fn metadata_count(disk: &Path, bucket: &str, expected_manifests: &[PhysicalObjectManifest]) -> usize {
|
||||
expected_manifests
|
||||
.iter()
|
||||
.filter(|expected| object_metadata_exists_on_disk(disk, bucket, &expected.key))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn heal_task_status_diagnostic(body: &str) -> String {
|
||||
let Ok(status) = serde_json::from_str::<serde_json::Value>(body) else {
|
||||
return body.to_string();
|
||||
};
|
||||
let items = status["items"].as_array();
|
||||
let mut unresolved_states = HashSet::new();
|
||||
for item in items.into_iter().flatten() {
|
||||
for drive in item["after"]["drives"].as_array().into_iter().flatten() {
|
||||
if let Some(state) = drive["state"].as_str()
|
||||
&& state != "ok"
|
||||
{
|
||||
unresolved_states.insert(state.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut unresolved_states = unresolved_states.into_iter().collect::<Vec<_>>();
|
||||
unresolved_states.sort();
|
||||
format!(
|
||||
"summary={:?}, detail={:?}, item_count={}, unresolved_drive_states={unresolved_states:?}",
|
||||
status["summary"].as_str(),
|
||||
status["detail"].as_str(),
|
||||
items.map_or(0, Vec::len)
|
||||
)
|
||||
}
|
||||
|
||||
fn cluster_heal_is_idle(status: &serde_json::Value) -> bool {
|
||||
let operations = &status["healOperations"];
|
||||
status["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
&& status["state"].as_str() == Some("idle")
|
||||
&& operations["queueLength"].as_u64() == Some(0)
|
||||
&& operations["activeTasks"].as_u64() == Some(0)
|
||||
&& operations["retryingTasks"].as_u64() == Some(0)
|
||||
}
|
||||
|
||||
fn only_admin_heal_is_active(status: &serde_json::Value) -> bool {
|
||||
let operations = &status["healOperations"];
|
||||
status["clusterStatusComplete"] == serde_json::Value::Bool(true)
|
||||
&& status["state"].as_str() == Some("active")
|
||||
&& operations["queueLength"].as_u64() == Some(0)
|
||||
&& operations["activeTasks"].as_u64() == Some(1)
|
||||
&& operations["retryingTasks"].as_u64() == Some(0)
|
||||
&& operations["activeBySource"]["admin"].as_u64() == Some(1)
|
||||
}
|
||||
|
||||
async fn replacement_recovery_status(
|
||||
cluster: &RustFSTestClusterEnvironment,
|
||||
) -> Result<serde_json::Value, Box<dyn Error + Send + Sync>> {
|
||||
let (status, body) = admin_request(
|
||||
&cluster.nodes[0].url,
|
||||
Method::GET,
|
||||
"/rustfs/admin/v4/heal/replacement-recovery",
|
||||
None,
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("replacement recovery status failed: {status} {body}").into());
|
||||
}
|
||||
serde_json::from_str(&body).map_err(|err| format!("replacement recovery status is not JSON ({err}): {body}").into())
|
||||
}
|
||||
|
||||
async fn assert_object_body(env: &RustFSTestEnvironment, bucket: &str, key: &str, expected: &[u8]) {
|
||||
let client = env.create_s3_client();
|
||||
let response = client
|
||||
@@ -442,6 +547,380 @@ mod tests {
|
||||
.into())
|
||||
}
|
||||
|
||||
// Keep the original unformatted-disk scenario above. This case retains the
|
||||
// format identity so only the explicit admin task can rebuild missing data.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_resumes_missing_remote_shards_after_node_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
info!(
|
||||
event = "heal_restart_started",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
"Starting root-heal restart test"
|
||||
);
|
||||
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
cluster.set_env("RUSTFS_UNSAFE_BYPASS_DISK_CHECK", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_HEAL_MRF_ENABLE", "false");
|
||||
cluster.set_env("RUSTFS_SCANNER_ENABLED", "false");
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_HEALS", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_MAX_CONCURRENT_PER_SET", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_OBJECT_CONCURRENCY", "1");
|
||||
cluster.set_env("RUSTFS_HEAL_PAGE_PARALLEL_ENABLE", "false");
|
||||
// Keep all storage nodes' Heal runtimes enabled so their disk services
|
||||
// complete normal registration after restart. Scanner, auto-heal and
|
||||
// MRF are disabled; the pre-root idle barrier below drains the direct
|
||||
// outage-object repair before the explicit admin task starts.
|
||||
let server_rust_log = std::env::var("RUSTFS_HEAL_CHAOS_SERVER_RUST_LOG")
|
||||
.unwrap_or_else(|_| "rustfs::heal::task=info,rustfs=error".to_string());
|
||||
cluster.set_env("RUST_LOG", server_rust_log);
|
||||
if let Ok(log_dir) = std::env::var("RUSTFS_HEAL_CHAOS_LOG_DIR") {
|
||||
std::fs::create_dir_all(&log_dir)?;
|
||||
for node_index in 0..cluster.nodes.len() {
|
||||
cluster.set_node_capture_log_path(node_index, format!("{log_dir}/node{node_index}.log"))?;
|
||||
}
|
||||
}
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
|
||||
let bucket = "heal-restart-during-rebuild";
|
||||
clients[0].create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let replaced_disk = PathBuf::from(&cluster.nodes[1].data_dir);
|
||||
let replacement_format_path = replaced_disk.join(".rustfs.sys").join("format.json");
|
||||
let replacement_format = std::fs::read(&replacement_format_path).map_err(|err| {
|
||||
format!("failed to capture target format before replacement wipe at {replacement_format_path:?}: {err}")
|
||||
})?;
|
||||
let online_object_count = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_COUNT")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(24)
|
||||
.clamp(8, 64);
|
||||
let object_size_bytes = std::env::var("RUSTFS_HEAL_CHAOS_OBJECT_SIZE_BYTES")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<usize>().ok())
|
||||
.unwrap_or(4 * 1024 * 1024)
|
||||
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
for index in 0..online_object_count {
|
||||
let key = format!("cluster/online/object-{index:04}.bin");
|
||||
let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[0]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(&key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
let shard_census = census_object_version_on_disk(&replaced_disk, bucket, &key, None)?;
|
||||
assert!(
|
||||
shard_census.is_complete(),
|
||||
"node 1 should hold a complete baseline shard for {key}: {shard_census:?}"
|
||||
);
|
||||
assert!(
|
||||
!shard_census.expected_part_numbers.is_empty(),
|
||||
"chaos objects must use physical part shards rather than inline data: {shard_census:?}"
|
||||
);
|
||||
expected_manifests.push(PhysicalObjectManifest {
|
||||
key,
|
||||
payload_seed,
|
||||
shard_census,
|
||||
});
|
||||
}
|
||||
|
||||
cluster.stop_node(1)?;
|
||||
std::fs::remove_dir_all(&replaced_disk)?;
|
||||
std::fs::create_dir_all(
|
||||
replacement_format_path
|
||||
.parent()
|
||||
.ok_or("replacement format path has no parent")?,
|
||||
)?;
|
||||
std::fs::write(&replacement_format_path, replacement_format)?;
|
||||
assert!(
|
||||
replacement_format_path.is_file(),
|
||||
"replacement target must retain only its preformatted topology identity"
|
||||
);
|
||||
|
||||
let outage_key = "cluster/written-while-node-down.bin";
|
||||
let outage_payload_seed = 0xf1;
|
||||
timeout(
|
||||
Duration::from_secs(30),
|
||||
clients[2]
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(outage_key)
|
||||
.body(ByteStream::from(deterministic_object_body(object_size_bytes, outage_payload_seed)))
|
||||
.send(),
|
||||
)
|
||||
.await??;
|
||||
|
||||
let mut outage_peer_erasure_indices = HashSet::new();
|
||||
for (node_index, node) in cluster.nodes.iter().enumerate() {
|
||||
if node_index == 1 {
|
||||
continue;
|
||||
}
|
||||
let census = census_object_version_on_disk(Path::new(&node.data_dir), bucket, outage_key, None)?;
|
||||
assert!(
|
||||
census.is_complete(),
|
||||
"online node {node_index} must hold a complete outage-object shard: {census:?}"
|
||||
);
|
||||
let erasure_index = census
|
||||
.erasure_index
|
||||
.ok_or_else(|| format!("online node {node_index} outage-object shard has no erasure index: {census:?}"))?;
|
||||
assert!(
|
||||
(1..=cluster.nodes.len()).contains(&erasure_index),
|
||||
"online node {node_index} outage-object erasure index is out of range: {census:?}"
|
||||
);
|
||||
assert!(
|
||||
outage_peer_erasure_indices.insert(erasure_index),
|
||||
"outage-object erasure index {erasure_index} is duplicated across online nodes"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
outage_peer_erasure_indices.len(),
|
||||
cluster.nodes.len().saturating_sub(1),
|
||||
"every online node must contribute one unique outage-object erasure index"
|
||||
);
|
||||
let expected_outage_target_erasure_index = (1..=cluster.nodes.len())
|
||||
.find(|index| !outage_peer_erasure_indices.contains(index))
|
||||
.ok_or("online outage-object shards leave no erasure index for the replacement target")?;
|
||||
|
||||
// The PUT path may have admitted a direct Internal object repair while
|
||||
// node 1 was offline. Cancel the isolated bucket path before the target
|
||||
// returns; otherwise it could rebuild the outage object and invalidate
|
||||
// the explicit-root ownership assertion below.
|
||||
let cancel_outage_heal_path = format!("/rustfs/admin/v3/heal/{bucket}?forceStop=true");
|
||||
let (cancel_status, cancel_body) = admin_request(
|
||||
&cluster.nodes[0].url,
|
||||
Method::POST,
|
||||
&cancel_outage_heal_path,
|
||||
Some(
|
||||
r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#
|
||||
.to_string(),
|
||||
),
|
||||
&cluster.access_key,
|
||||
&cluster.secret_key,
|
||||
)
|
||||
.await?;
|
||||
if !cancel_status.is_success() {
|
||||
return Err(format!("cancel outage heal failed: {cancel_status} {cancel_body}").into());
|
||||
}
|
||||
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let status_url = format!("{}/rustfs/admin/v3/background-heal/status", cluster.nodes[0].url);
|
||||
let recovery_deadline = Instant::now() + Duration::from_secs(60);
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
assert!(
|
||||
!status_body.contains("MissingContentLength"),
|
||||
"background heal status should not fail without an explicit Content-Length: {status_body}"
|
||||
);
|
||||
let recovered: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if cluster_heal_is_idle(&recovered) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= recovery_deadline {
|
||||
return Err(format!("cluster heal operations did not become idle before root heal: {recovered}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
assert_eq!(
|
||||
matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?,
|
||||
0,
|
||||
"non-admin Heal is disabled, so the replacement target must remain empty before the explicit root heal"
|
||||
);
|
||||
assert!(
|
||||
!census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?.has_xl_meta,
|
||||
"the object written during the outage must be absent before the explicit root heal"
|
||||
);
|
||||
let pre_heal_replacement = replacement_recovery_status(&cluster).await?;
|
||||
assert_eq!(
|
||||
pre_heal_replacement["cluster"]["records"].as_array().map(Vec::len),
|
||||
Some(0),
|
||||
"isolated target must not retain an automatic replacement generation: {pre_heal_replacement}"
|
||||
);
|
||||
|
||||
let heal_body = r#"{"recursive":true,"dryRun":false,"remove":false,"recreate":true,"scanMode":2,"updateParity":false,"nolock":false}"#;
|
||||
let heal_url = format!("{}/rustfs/admin/v3/heal/?forceStart=true", cluster.nodes[0].url);
|
||||
let heal_start_body = signed_admin_post(&heal_url, Some(heal_body), &cluster.access_key, &cluster.secret_key).await?;
|
||||
let heal_start: serde_json::Value = serde_json::from_str(&heal_start_body)
|
||||
.map_err(|err| format!("heal start response is not JSON ({err}): {heal_start_body}"))?;
|
||||
let client_token = heal_start["clientToken"]
|
||||
.as_str()
|
||||
.filter(|token| !token.is_empty())
|
||||
.ok_or_else(|| format!("heal start response has no client token: {heal_start}"))?;
|
||||
let task_status_url = format!("{}/rustfs/admin/v3/heal/?clientToken={client_token}", cluster.nodes[0].url);
|
||||
|
||||
let partial_timeout_secs = std::env::var("RUSTFS_HEAL_CHAOS_PARTIAL_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(60);
|
||||
let partial_deadline = Instant::now() + Duration::from_secs(partial_timeout_secs);
|
||||
let pre_interrupt_status = loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let active_status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if only_admin_heal_is_active(&active_status) {
|
||||
break active_status;
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!("root heal never became active within {partial_timeout_secs}s: {active_status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
};
|
||||
let partial_count = loop {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
if matching > 0 && matching < expected_manifests.len() {
|
||||
break matching;
|
||||
}
|
||||
if matching == expected_manifests.len() {
|
||||
return Err(format!(
|
||||
"root heal rebuilt all {} baseline objects before the target could be interrupted",
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
if Instant::now() >= partial_deadline {
|
||||
return Err(format!(
|
||||
"root heal made no observable partial progress on the replacement target within {partial_timeout_secs}s"
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(10)).await;
|
||||
};
|
||||
info!(
|
||||
event = "heal_restart_checkpoint",
|
||||
component = "e2e_test",
|
||||
subsystem = "heal",
|
||||
partial_count,
|
||||
"Verified unique admin owner before target interruption"
|
||||
);
|
||||
|
||||
cluster.stop_node(1)?;
|
||||
let stopped_count = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
assert!(
|
||||
stopped_count > 0 && stopped_count < expected_manifests.len(),
|
||||
"the target must stop after a partial rebuild, observed before stop={partial_count}, after stop={stopped_count}, total={}",
|
||||
expected_manifests.len()
|
||||
);
|
||||
let unclean_shutdown_marker = replaced_disk.join(".rustfs.sys").join("unclean-shutdown");
|
||||
match std::fs::remove_file(&unclean_shutdown_marker) {
|
||||
Ok(()) => {}
|
||||
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
|
||||
Err(error) => {
|
||||
return Err(format!("failed to isolate unclean recovery marker {unclean_shutdown_marker:?}: {error}").into());
|
||||
}
|
||||
}
|
||||
cluster.start_node(1).await?;
|
||||
|
||||
let heal_timeout_secs = std::env::var("RUSTFS_HEAL_REPLACED_DISK_TIMEOUT_SECS")
|
||||
.ok()
|
||||
.and_then(|value| value.parse::<u64>().ok())
|
||||
.unwrap_or(180);
|
||||
let heal_deadline = Instant::now() + Duration::from_secs(heal_timeout_secs);
|
||||
loop {
|
||||
if metadata_count(&replaced_disk, bucket, &expected_manifests) == expected_manifests.len()
|
||||
&& object_metadata_exists_on_disk(&replaced_disk, bucket, outage_key)
|
||||
{
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
if matching == expected_manifests.len() && outage_census.is_complete() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
if Instant::now() >= heal_deadline {
|
||||
let matching = matching_manifest_count(&replaced_disk, bucket, &expected_manifests)?;
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
let final_status = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key)
|
||||
.await
|
||||
.unwrap_or_else(|err| format!("status request failed: {err}"));
|
||||
let task_status = match timeout(
|
||||
Duration::from_secs(5),
|
||||
signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(body)) => heal_task_status_diagnostic(&body),
|
||||
Ok(Err(err)) => format!("task status request failed: {err}"),
|
||||
Err(_) => "task status request exceeded 5s diagnostic budget".to_string(),
|
||||
};
|
||||
let replacement_status = match timeout(Duration::from_secs(5), replacement_recovery_status(&cluster)).await {
|
||||
Ok(Ok(status)) => status.to_string(),
|
||||
Ok(Err(err)) => format!("replacement status request failed: {err}"),
|
||||
Err(_) => "replacement status request exceeded 5s diagnostic budget".to_string(),
|
||||
};
|
||||
return Err(format!(
|
||||
"root heal did not resume after target restart within {heal_timeout_secs}s: baseline={matching}/{}, outage={outage_census:?}, status={final_status}, task_status={task_status}, pre_interrupt_status={pre_interrupt_status}, pre_heal_replacement={pre_heal_replacement}, replacement_status={replacement_status}",
|
||||
expected_manifests.len()
|
||||
)
|
||||
.into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
for expected in &expected_manifests {
|
||||
let actual = census_object_version_on_disk(&replaced_disk, bucket, &expected.key, None)?;
|
||||
assert!(
|
||||
actual.matches_manifest(&expected.shard_census),
|
||||
"rebuilt target shard differs from its baseline for {}: {actual:?}",
|
||||
expected.key
|
||||
);
|
||||
}
|
||||
let outage_census = census_object_version_on_disk(&replaced_disk, bucket, outage_key, None)?;
|
||||
assert!(
|
||||
outage_census.is_complete(),
|
||||
"outage object must have a complete target shard: {outage_census:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
outage_census.erasure_index,
|
||||
Some(expected_outage_target_erasure_index),
|
||||
"the outage object must be rebuilt into its own missing erasure slot"
|
||||
);
|
||||
|
||||
let target_client = cluster.create_s3_client(1)?;
|
||||
for expected in &expected_manifests {
|
||||
let response = target_client.get_object().bucket(bucket).key(&expected.key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_body = deterministic_object_body(object_size_bytes, expected.payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_body.as_slice(), "object body changed for {}", expected.key);
|
||||
}
|
||||
let response = target_client.get_object().bucket(bucket).key(outage_key).send().await?;
|
||||
let actual = response.body.collect().await?.into_bytes();
|
||||
let expected_outage_body = deterministic_object_body(object_size_bytes, outage_payload_seed);
|
||||
assert_eq!(actual.as_ref(), expected_outage_body.as_slice(), "object body changed for {outage_key}");
|
||||
|
||||
let terminal_deadline = Instant::now() + Duration::from_secs(30);
|
||||
loop {
|
||||
let status_body = signed_admin_post(&status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let status: serde_json::Value = serde_json::from_str(&status_body)
|
||||
.map_err(|err| format!("background heal status is not JSON ({err}): {status_body}"))?;
|
||||
if cluster_heal_is_idle(&status) {
|
||||
break;
|
||||
}
|
||||
if Instant::now() >= terminal_deadline {
|
||||
return Err(format!("heal data rebuilt but operations did not converge to terminal idle: {status}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
|
||||
let task_status_body = signed_admin_post(&task_status_url, None, &cluster.access_key, &cluster.secret_key).await?;
|
||||
let task_status: serde_json::Value = serde_json::from_str(&task_status_body)
|
||||
.map_err(|err| format!("heal task status is not JSON ({err}): {task_status_body}"))?;
|
||||
if task_status["summary"].as_str() != Some("finished") {
|
||||
return Err(format!("heal data rebuilt but task did not finish successfully: {task_status}").into());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Issue #5850: `background-heal/status` must answer while a peer is down.
|
||||
///
|
||||
/// Exercises the production path in `read_cluster_heal_status` end to end,
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Regression coverage for anonymous access on multipart control APIs.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use async_compression::tokio::write::{BzEncoder, XzEncoder};
|
||||
use async_compression::tokio::write::{BzEncoder, Lz4Encoder, XzEncoder};
|
||||
use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError};
|
||||
use aws_sdk_s3::operation::head_object::HeadObjectOutput;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
@@ -23,7 +23,10 @@ use aws_sdk_s3::types::{
|
||||
ServerSideEncryption, ServerSideEncryptionByDefault, ServerSideEncryptionConfiguration, ServerSideEncryptionRule,
|
||||
};
|
||||
use chrono::{Duration as ChronoDuration, Utc};
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use flate2::{
|
||||
Compression,
|
||||
write::{GzEncoder, ZlibEncoder},
|
||||
};
|
||||
use http::HeaderValue;
|
||||
use http::header::{CONTENT_TYPE, HOST};
|
||||
use md5::{Digest as Md5Digest, Md5};
|
||||
@@ -187,6 +190,12 @@ fn gzip_bytes(data: &[u8]) -> Vec<u8> {
|
||||
encoder.finish().expect("gzip encoder should finish")
|
||||
}
|
||||
|
||||
fn zlib_bytes(data: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = ZlibEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(data).expect("zlib encoder should accept input");
|
||||
encoder.finish().expect("zlib encoder should finish")
|
||||
}
|
||||
|
||||
fn zstd_bytes(data: &[u8]) -> Vec<u8> {
|
||||
let mut encoder = zstd::Encoder::new(Vec::new(), 0).expect("zstd encoder should initialize");
|
||||
encoder.write_all(data).expect("zstd encoder should accept input");
|
||||
@@ -209,6 +218,45 @@ async fn xz_bytes(data: &[u8]) -> Vec<u8> {
|
||||
encoder.into_inner().into_inner()
|
||||
}
|
||||
|
||||
async fn lz4_bytes(data: &[u8]) -> Vec<u8> {
|
||||
let cursor = Cursor::new(Vec::new());
|
||||
let mut encoder = Lz4Encoder::new(cursor);
|
||||
encoder.write_all(data).await.expect("LZ4 encoder should accept input");
|
||||
encoder.shutdown().await.expect("LZ4 encoder should finish");
|
||||
encoder.into_inner().into_inner()
|
||||
}
|
||||
|
||||
/// Encode the S2 framed stream shape emitted by minio-go PutObjectsSnowball
|
||||
/// with `Compress: true`: 1 MiB independent blocks, better compression,
|
||||
/// masked CRC-32C, and the `S2sTwO` stream identifier.
|
||||
fn minio_go_snowball_s2_bytes(data: &[u8]) -> Vec<u8> {
|
||||
const BLOCK_SIZE: usize = 1 << 20;
|
||||
const CHECKSUM_SIZE: usize = 4;
|
||||
|
||||
let mut output = b"\xff\x06\x00\x00S2sTwO".to_vec();
|
||||
let mut encoder = minlz::Encoder::new();
|
||||
for block in data.chunks(BLOCK_SIZE) {
|
||||
let compressed = encoder.encode_better(block);
|
||||
let compressed_limit = block.len().saturating_sub(block.len() / 32).saturating_sub(5);
|
||||
let (chunk_type, payload) = if compressed.len() <= compressed_limit {
|
||||
(0x00, compressed.as_slice())
|
||||
} else {
|
||||
(0x01, block)
|
||||
};
|
||||
let chunk_len = payload.len() + CHECKSUM_SIZE;
|
||||
assert!(chunk_len < 1 << 24, "S2 fixture chunk must fit the 24-bit frame length");
|
||||
output.extend_from_slice(&[
|
||||
chunk_type,
|
||||
(chunk_len & 0xff) as u8,
|
||||
((chunk_len >> 8) & 0xff) as u8,
|
||||
((chunk_len >> 16) & 0xff) as u8,
|
||||
]);
|
||||
output.extend_from_slice(&minlz::crc::crc(block).to_le_bytes());
|
||||
output.extend_from_slice(payload);
|
||||
}
|
||||
output
|
||||
}
|
||||
|
||||
fn assert_s3_error_code<T, E>(result: Result<T, SdkError<E>>, code: &str)
|
||||
where
|
||||
T: std::fmt::Debug,
|
||||
@@ -3456,6 +3504,62 @@ async fn test_signed_put_object_extract_expands_tar_entries_with_prefix_headers(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_ignore_dirs_skips_unauthorized_directory()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "signed-extract-ignore-dirs-auth";
|
||||
let archive_key = "bundle.tar";
|
||||
let allowed_member = "allowed/member.txt";
|
||||
let denied_directory = "denied/";
|
||||
let username = "snowball-ignore-dirs";
|
||||
let secret_key = "snowball-ignore-dirs-secret";
|
||||
let expected_body = b"allowed-body";
|
||||
|
||||
let admin_client = env.create_s3_client();
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
create_restricted_user(&env, username, secret_key).await?;
|
||||
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [username] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [
|
||||
format!("arn:aws:s3:::{bucket}/{archive_key}"),
|
||||
format!("arn:aws:s3:::{bucket}/{allowed_member}")
|
||||
]
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client.put_bucket_policy().bucket(bucket).policy(policy).send().await?;
|
||||
|
||||
let restricted_client = restricted_user_client(&env, username, secret_key);
|
||||
let tar_bytes = make_tar(&[(allowed_member, expected_body)], &[denied_directory]).await;
|
||||
restricted_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.body(ByteStream::from(tar_bytes))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
req.headers_mut().insert("x-amz-meta-snowball-ignore-dirs", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let stored = admin_client.get_object().bucket(bucket).key(allowed_member).send().await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_preserves_request_metadata_on_extracted_objects()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
@@ -4185,6 +4289,60 @@ async fn test_signed_put_object_extract_returns_archive_etag() -> Result<(), Box
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_expands_s2_and_lz4_by_magic_with_raw_etags()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let bucket = "signed-extract-magic-codecs";
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let s2_tar = make_tar(&[("s2/object.txt", b"s2-body")], &[]).await;
|
||||
let s2_archive = minio_go_snowball_s2_bytes(&s2_tar);
|
||||
let expected_s2_etag = format!("\"{}\"", md5_hex(&s2_archive));
|
||||
let s2_response = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
// minio-go intentionally uploads a compressed S2 stream with a .tar key.
|
||||
.key("snowball-upload-0123456789abcdef.tar")
|
||||
.body(ByteStream::from(s2_archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(s2_response.e_tag(), Some(expected_s2_etag.as_str()));
|
||||
|
||||
let s2_object = client.get_object().bucket(bucket).key("s2/object.txt").send().await?;
|
||||
assert_eq!(s2_object.body.collect().await?.into_bytes().as_ref(), b"s2-body");
|
||||
|
||||
let lz4_tar = make_tar(&[("lz4/object.txt", b"lz4-body")], &[]).await;
|
||||
let lz4_archive = lz4_bytes(&lz4_tar).await;
|
||||
let expected_lz4_etag = format!("\"{}\"", md5_hex(&lz4_archive));
|
||||
let lz4_response = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("also-looks-like-a-plain.tar")
|
||||
.body(ByteStream::from(lz4_archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(lz4_response.e_tag(), Some(expected_lz4_etag.as_str()));
|
||||
|
||||
let lz4_object = client.get_object().bucket(bucket).key("lz4/object.txt").send().await?;
|
||||
assert_eq!(lz4_object.body.collect().await?.into_bytes().as_ref(), b"lz4-body");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_preserves_entry_mtime() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -4309,9 +4467,15 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
let context_archive_resources = [
|
||||
format!("arn:aws:s3:::{bucket}/tag-context.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/lock-context.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/legal-hold-context.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/user-agent-bypass.tar"),
|
||||
format!("arn:aws:s3:::{bucket}/sse-bypass.tar"),
|
||||
];
|
||||
let tag_entry_resource = format!("arn:aws:s3:::{bucket}/tag-context-entry.txt");
|
||||
let lock_entry_resource = format!("arn:aws:s3:::{bucket}/lock-context-entry.txt");
|
||||
let legal_hold_entry_resource = format!("arn:aws:s3:::{bucket}/legal-hold-context-entry.txt");
|
||||
let user_agent_entry_resource = format!("arn:aws:s3:::{bucket}/user-agent-bypass-entry.txt");
|
||||
let sse_entry_resource = format!("arn:aws:s3:::{bucket}/sse-bypass-entry.txt");
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [
|
||||
@@ -4371,7 +4535,7 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
"Sid": "PaxContextArchives",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging"],
|
||||
"Action": ["s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectLegalHold", "s3:PutObjectTagging"],
|
||||
"Resource": context_archive_resources
|
||||
},
|
||||
{
|
||||
@@ -4411,6 +4575,49 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObjectRetention"],
|
||||
"Resource": [lock_entry_resource]
|
||||
},
|
||||
{
|
||||
"Sid": "PaxLegalHoldContextPut",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [legal_hold_entry_resource.clone()]
|
||||
},
|
||||
{
|
||||
"Sid": "PaxLegalHoldContextAction",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObjectLegalHold"],
|
||||
"Resource": [legal_hold_entry_resource],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:object-lock-legal-hold": "OFF"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "MemberUserAgentCondition",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [user_agent_entry_resource],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"aws:UserAgent": "trusted"
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"Sid": "MemberSseCondition",
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [pax_context_user] },
|
||||
"Action": ["s3:PutObject"],
|
||||
"Resource": [sse_entry_resource],
|
||||
"Condition": {
|
||||
"StringEquals": {
|
||||
"s3:x-amz-server-side-encryption": "AES256"
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
})
|
||||
@@ -4423,9 +4630,14 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
let cases = [
|
||||
(
|
||||
"legal-hold.tar",
|
||||
put_only_client,
|
||||
put_only_client.clone(),
|
||||
HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]),
|
||||
),
|
||||
(
|
||||
"tagging.tar",
|
||||
put_only_client,
|
||||
HashMap::from([("minio.metadata.x-amz-tagging", "classification=restricted".to_string())]),
|
||||
),
|
||||
(
|
||||
"retention-condition.tar",
|
||||
conditional_client,
|
||||
@@ -4512,6 +4724,57 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), b"condition-body");
|
||||
|
||||
let pax_context_client = restricted_user_client(&env, pax_context_user, pax_context_secret);
|
||||
for (archive_key, entry_key, pax_key, injected_value, outer_user_agent) in [
|
||||
(
|
||||
"user-agent-bypass.tar",
|
||||
"user-agent-bypass-entry.txt",
|
||||
"minio.metadata.user-agent",
|
||||
"trusted",
|
||||
Some("untrusted"),
|
||||
),
|
||||
(
|
||||
"sse-bypass.tar",
|
||||
"sse-bypass-entry.txt",
|
||||
"minio.metadata.x-amz-server-side-encryption",
|
||||
"AES256",
|
||||
None,
|
||||
),
|
||||
] {
|
||||
let pax = HashMap::from([(pax_key, injected_value.to_string())]);
|
||||
let archive = make_tar_with_pax_entry(entry_key, b"must-not-write", None, &pax).await;
|
||||
let err = pax_context_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(move |req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
if let Some(user_agent) = outer_user_agent {
|
||||
req.headers_mut().insert("user-agent", user_agent);
|
||||
}
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("PAX metadata must not satisfy unrelated IAM request conditions");
|
||||
assert_eq!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("AccessDenied"),
|
||||
"{archive_key}"
|
||||
);
|
||||
let err = admin_client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(entry_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied PAX member must not be written");
|
||||
assert!(matches!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("NoSuchKey" | "NotFound")
|
||||
));
|
||||
}
|
||||
|
||||
let tag_pax = HashMap::from([("minio.metadata.x-amz-tagging", "classification=public".to_string())]);
|
||||
let archive = make_tar_with_pax_entry("tag-context-entry.txt", b"tag-context-body", None, &tag_pax).await;
|
||||
pax_context_client
|
||||
@@ -4575,6 +4838,34 @@ async fn test_signed_put_object_extract_authorizes_each_pax_privilege_and_retent
|
||||
pax_retain_until
|
||||
);
|
||||
|
||||
let legal_hold_pax = HashMap::from([("minio.metadata.x-amz-object-lock-legal-hold", "ON".to_string())]);
|
||||
let archive = make_tar_with_pax_entry("legal-hold-context-entry.txt", b"must-not-write", None, &legal_hold_pax).await;
|
||||
let err = pax_context_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("legal-hold-context.tar")
|
||||
.object_lock_legal_hold_status(aws_sdk_s3::types::ObjectLockLegalHoldStatus::Off)
|
||||
.body(ByteStream::from(archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.expect_err("PAX legal hold must replace the outer value in the member IAM condition context");
|
||||
assert_eq!(err.as_service_error().and_then(|error| error.meta().code()), Some("AccessDenied"));
|
||||
let err = admin_client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key("legal-hold-context-entry.txt")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a denied PAX legal-hold member must not be written");
|
||||
assert!(matches!(
|
||||
err.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("NoSuchKey" | "NotFound")
|
||||
));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -5050,8 +5341,8 @@ async fn test_signed_put_object_extract_expands_tzst_archive() -> Result<(), Box
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
async fn test_signed_put_object_extract_uses_magic_without_requiring_or_trusting_extension()
|
||||
-> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
@@ -5064,8 +5355,7 @@ async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> R
|
||||
admin_client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let tar_bytes = make_tar(&[("plain.txt", b"plain-body")], &[]).await;
|
||||
|
||||
let result = admin_client
|
||||
admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
@@ -5075,15 +5365,80 @@ async fn test_signed_put_object_extract_rejects_missing_archive_extension() -> R
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await;
|
||||
.await?;
|
||||
|
||||
assert_s3_error_code(result, "InvalidArgument");
|
||||
let plain = admin_client.get_object().bucket(bucket).key("plain.txt").send().await?;
|
||||
assert_eq!(plain.body.collect().await?.into_bytes().as_ref(), b"plain-body");
|
||||
|
||||
let raw_with_gzip_suffix = make_tar(&[("raw-with-wrong-suffix.txt", b"raw-body")], &[]).await;
|
||||
admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("raw-but-named.tar.gz")
|
||||
.body(ByteStream::from(raw_with_gzip_suffix))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let raw = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("raw-with-wrong-suffix.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(raw.body.collect().await?.into_bytes().as_ref(), b"raw-body");
|
||||
|
||||
let gzip_with_tar_suffix = gzip_bytes(&make_tar(&[("gzip-with-wrong-suffix.txt", b"gzip-body")], &[]).await);
|
||||
admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("gzip-but-named.tar")
|
||||
.body(ByteStream::from(gzip_with_tar_suffix))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let gzip = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("gzip-with-wrong-suffix.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(gzip.body.collect().await?.into_bytes().as_ref(), b"gzip-body");
|
||||
|
||||
let zlib_archive = zlib_bytes(&make_tar(&[("zlib-extension.txt", b"zlib-body")], &[]).await);
|
||||
admin_client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("bundle.zlib")
|
||||
.body(ByteStream::from(zlib_archive))
|
||||
.customize()
|
||||
.mutate_request(|req| {
|
||||
req.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
|
||||
})
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let zlib = admin_client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("zlib-extension.txt")
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(zlib.body.collect().await?.into_bytes().as_ref(), b"zlib-body");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_signed_put_object_extract_rejects_invalid_tar_gz_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
async fn test_signed_put_object_extract_rejects_invalid_archive_payload() -> Result<(), Box<dyn std::error::Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
|
||||
@@ -36,9 +36,10 @@
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
use rustfs_signer::constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER};
|
||||
use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key};
|
||||
use std::fmt::Write as _;
|
||||
use std::io::Cursor;
|
||||
use time::macros::format_description;
|
||||
use time::{Duration, OffsetDateTime};
|
||||
use tracing::info;
|
||||
@@ -98,15 +99,37 @@ impl SigV4 {
|
||||
/// header AND folded into the canonical request — pass the hash of the
|
||||
/// body you *claim* to send, which may differ from what you actually send.
|
||||
fn sign(&self, method: &str, path: &str, canonical_query: &str, content_sha256: &str) -> SignedHeaders {
|
||||
let amz_date = amz_datetime(self.time);
|
||||
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
|
||||
self.sign_with_extra_headers(method, path, canonical_query, content_sha256, &[])
|
||||
}
|
||||
|
||||
let canonical_headers = format!(
|
||||
"host:{host}\nx-amz-content-sha256:{sha}\nx-amz-date:{date}\n",
|
||||
host = self.host,
|
||||
sha = content_sha256,
|
||||
date = amz_date,
|
||||
);
|
||||
/// Sign additional request headers while preserving SigV4's lowercase,
|
||||
/// lexicographically sorted canonical-header representation.
|
||||
fn sign_with_extra_headers(
|
||||
&self,
|
||||
method: &str,
|
||||
path: &str,
|
||||
canonical_query: &str,
|
||||
content_sha256: &str,
|
||||
extra_signed_headers: &[(&str, &str)],
|
||||
) -> SignedHeaders {
|
||||
let amz_date = amz_datetime(self.time);
|
||||
let mut canonical_header_values = vec![
|
||||
("host", self.host.as_str()),
|
||||
("x-amz-content-sha256", content_sha256),
|
||||
("x-amz-date", amz_date.as_str()),
|
||||
];
|
||||
canonical_header_values.extend(extra_signed_headers.iter().copied());
|
||||
canonical_header_values.sort_unstable_by(|left, right| left.0.cmp(right.0));
|
||||
|
||||
let signed_headers = canonical_header_values
|
||||
.iter()
|
||||
.map(|(name, _)| *name)
|
||||
.collect::<Vec<_>>()
|
||||
.join(";");
|
||||
let mut canonical_headers = String::new();
|
||||
for (name, value) in canonical_header_values {
|
||||
let _ = writeln!(canonical_headers, "{name}:{value}");
|
||||
}
|
||||
let canonical_request =
|
||||
format!("{method}\n{path}\n{canonical_query}\n{canonical_headers}\n{signed_headers}\n{content_sha256}");
|
||||
|
||||
@@ -179,6 +202,34 @@ async fn setup(env: &mut RustFSTestEnvironment) -> Result<(), Box<dyn std::error
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_single_member_archive(
|
||||
member_key: &str,
|
||||
member_body: &[u8],
|
||||
) -> Result<Vec<u8>, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(member_body.len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, member_key, Cursor::new(member_body)).await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn sha256_base64(data: &[u8]) -> String {
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
base64_simd::STANDARD.encode_to_string(Sha256::digest(data))
|
||||
}
|
||||
|
||||
fn encode_unsigned_aws_chunked_with_sha256_trailer(decoded: &[u8]) -> Vec<u8> {
|
||||
let checksum = sha256_base64(decoded);
|
||||
let mut encoded = format!("{:x}\r\n", decoded.len()).into_bytes();
|
||||
encoded.extend_from_slice(decoded);
|
||||
encoded.extend_from_slice(b"\r\n0\r\n\r\n");
|
||||
encoded.extend_from_slice(format!("x-amz-checksum-sha256:{checksum}").as_bytes());
|
||||
encoded
|
||||
}
|
||||
|
||||
/// Positive control: a correctly hand-signed request must succeed. Without
|
||||
/// this, every negative assertion below could pass for the wrong reason (a
|
||||
/// broken signer that never produces a valid signature).
|
||||
@@ -249,6 +300,128 @@ async fn tampered_signature_returns_signature_does_not_match() -> Result<(), Box
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// `STREAMING-UNSIGNED-PAYLOAD-TRAILER` disables per-chunk signatures, not the
|
||||
/// seed/header SigV4 signature. A forged request must be rejected before the
|
||||
/// Snowball handler can publish any archive member.
|
||||
#[tokio::test]
|
||||
async fn snowball_streaming_unsigned_trailer_rejects_forged_signature() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
setup(&mut env).await?;
|
||||
|
||||
let archive_key = "forged-streaming-snowball.tar";
|
||||
let member_key = "must-not-be-published.txt";
|
||||
let archive = build_single_member_archive(member_key, b"forged request payload").await?;
|
||||
let decoded_content_length = archive.len().to_string();
|
||||
let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
|
||||
let path = format!("/{BUCKET}/{archive_key}");
|
||||
|
||||
let mut signer = SigV4::new(&env);
|
||||
signer.secret_key = "wrong-secret-for-forged-streaming-request".to_string();
|
||||
let extra_signed_headers = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", decoded_content_length.as_str()),
|
||||
("x-amz-meta-snowball-auto-extract", "true"),
|
||||
("x-amz-trailer", "x-amz-checksum-sha256"),
|
||||
];
|
||||
let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
|
||||
|
||||
let response = local_http_client()
|
||||
.put(format!("{}{}", env.url, path))
|
||||
.header("authorization", &headers.authorization)
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("x-amz-content-sha256", &headers.content_sha256)
|
||||
.header("x-amz-date", &headers.amz_date)
|
||||
.header("x-amz-decoded-content-length", &decoded_content_length)
|
||||
.header("x-amz-meta-snowball-auto-extract", "true")
|
||||
.header("x-amz-trailer", "x-amz-checksum-sha256")
|
||||
.body(encoded_body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let body = response.text().await?;
|
||||
assert_eq!(status.as_u16(), 403, "forged streaming signature must be 403, body:\n{body}");
|
||||
assert_error_code(&body, "SignatureDoesNotMatch");
|
||||
|
||||
let absent = env
|
||||
.create_s3_client()
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("a forged streaming request must not publish a Snowball member");
|
||||
assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404));
|
||||
assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Snowball must consume the complete aws-chunked body before reading the
|
||||
/// trailing checksum exported by s3s into the PutObject response.
|
||||
#[tokio::test]
|
||||
async fn snowball_streaming_unsigned_trailer_returns_sha256_checksum() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
setup(&mut env).await?;
|
||||
|
||||
let archive_key = "valid-streaming-snowball.tar";
|
||||
let member_key = "streaming-checksum-member.txt";
|
||||
let member_body = b"valid streaming Snowball payload";
|
||||
let archive = build_single_member_archive(member_key, member_body).await?;
|
||||
let expected_checksum = sha256_base64(&archive);
|
||||
let decoded_content_length = archive.len().to_string();
|
||||
let encoded_body = encode_unsigned_aws_chunked_with_sha256_trailer(&archive);
|
||||
let path = format!("/{BUCKET}/{archive_key}");
|
||||
|
||||
let signer = SigV4::new(&env);
|
||||
let extra_signed_headers = [
|
||||
("content-encoding", "aws-chunked"),
|
||||
("x-amz-decoded-content-length", decoded_content_length.as_str()),
|
||||
("x-amz-meta-snowball-auto-extract", "true"),
|
||||
("x-amz-sdk-checksum-algorithm", "SHA256"),
|
||||
("x-amz-trailer", "x-amz-checksum-sha256"),
|
||||
];
|
||||
let headers = signer.sign_with_extra_headers("PUT", &path, "", UNSIGNED_PAYLOAD_TRAILER, &extra_signed_headers);
|
||||
|
||||
let response = local_http_client()
|
||||
.put(format!("{}{}", env.url, path))
|
||||
.header("authorization", &headers.authorization)
|
||||
.header("content-encoding", "aws-chunked")
|
||||
.header("x-amz-content-sha256", &headers.content_sha256)
|
||||
.header("x-amz-date", &headers.amz_date)
|
||||
.header("x-amz-decoded-content-length", &decoded_content_length)
|
||||
.header("x-amz-meta-snowball-auto-extract", "true")
|
||||
.header("x-amz-sdk-checksum-algorithm", "SHA256")
|
||||
.header("x-amz-trailer", "x-amz-checksum-sha256")
|
||||
.body(encoded_body)
|
||||
.send()
|
||||
.await?;
|
||||
let status = response.status();
|
||||
let response_checksum = response
|
||||
.headers()
|
||||
.get("x-amz-checksum-sha256")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned);
|
||||
let response_body = response.text().await?;
|
||||
assert_eq!(status.as_u16(), 200, "valid streaming Snowball PUT failed, body:\n{response_body}");
|
||||
assert_eq!(response_checksum.as_deref(), Some(expected_checksum.as_str()));
|
||||
|
||||
let member = env
|
||||
.create_s3_client()
|
||||
.get_object()
|
||||
.bucket(BUCKET)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await?;
|
||||
let stored = member.body.collect().await?.into_bytes();
|
||||
assert_eq!(stored.as_ref(), member_body);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// (b) A valid AccessKeyId paired with the wrong secret key must be rejected
|
||||
/// with SignatureDoesNotMatch / 403.
|
||||
#[tokio::test]
|
||||
|
||||
@@ -21,5 +21,6 @@ mod head_tls_bodyless_test;
|
||||
mod lifecycle;
|
||||
mod lock;
|
||||
mod node_interact_test;
|
||||
mod s3_select_compression;
|
||||
mod sql;
|
||||
mod tiering;
|
||||
|
||||
@@ -0,0 +1,351 @@
|
||||
#![cfg(test)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use async_compression::tokio::write::BzEncoder;
|
||||
use aws_sdk_s3::{
|
||||
Client,
|
||||
error::ProvideErrorMetadata,
|
||||
operation::select_object_content::{SelectObjectContentOutput, builders::SelectObjectContentFluentBuilder},
|
||||
types::{
|
||||
CompressionType, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput,
|
||||
JsonType, OutputSerialization, SelectObjectContentEventStream,
|
||||
},
|
||||
};
|
||||
use aws_smithy_types::event_stream::RawMessage;
|
||||
use bytes::Bytes;
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use std::{error::Error, io::Cursor, time::Duration};
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
const BUCKET: &str = "s3-select-compression";
|
||||
const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
|
||||
async fn create_test_environment(extra_env: &[(&str, &str)]) -> TestResult<(RustFSTestEnvironment, Client)> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], extra_env).await?;
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(BUCKET).send().await?;
|
||||
Ok((env, client))
|
||||
}
|
||||
|
||||
async fn put_object(client: &Client, key: &str, body: &[u8]) -> TestResult<()> {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn gzip(input: &[u8]) -> TestResult<Vec<u8>> {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
std::io::Write::write_all(&mut encoder, input)?;
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
|
||||
async fn bzip2(input: &[u8]) -> TestResult<Vec<u8>> {
|
||||
let mut encoder = BzEncoder::new(Cursor::new(Vec::new()));
|
||||
encoder.write_all(input).await?;
|
||||
encoder.shutdown().await?;
|
||||
Ok(encoder.into_inner().into_inner())
|
||||
}
|
||||
|
||||
fn csv_select_request(
|
||||
client: &Client,
|
||||
key: &str,
|
||||
compression: CompressionType,
|
||||
expression: &str,
|
||||
) -> SelectObjectContentFluentBuilder {
|
||||
client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression(expression)
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.compression_type(compression)
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
|
||||
}
|
||||
|
||||
fn json_select_request(
|
||||
client: &Client,
|
||||
key: &str,
|
||||
compression: CompressionType,
|
||||
json_type: JsonType,
|
||||
) -> SelectObjectContentFluentBuilder {
|
||||
client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression("SELECT name FROM S3Object")
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.compression_type(compression)
|
||||
.json(JsonInput::builder().set_type(Some(json_type)).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
|
||||
}
|
||||
|
||||
async fn collect_success(
|
||||
mut response: SelectObjectContentOutput,
|
||||
compressed_bytes: usize,
|
||||
processed_bytes: usize,
|
||||
) -> TestResult<Vec<u8>> {
|
||||
tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
|
||||
let mut records = Vec::new();
|
||||
let mut stats = None;
|
||||
let mut saw_end = false;
|
||||
|
||||
while let Some(event) = response.payload.recv().await? {
|
||||
assert!(!saw_end, "Select emitted an event after End");
|
||||
match event {
|
||||
SelectObjectContentEventStream::Records(event) => {
|
||||
assert!(stats.is_none(), "Select emitted Records after Stats");
|
||||
if let Some(payload) = event.payload {
|
||||
records.extend_from_slice(payload.as_ref());
|
||||
}
|
||||
}
|
||||
SelectObjectContentEventStream::Stats(event) => {
|
||||
assert!(stats.is_none(), "Select emitted more than one Stats event");
|
||||
stats = event.details;
|
||||
}
|
||||
SelectObjectContentEventStream::End(_) => {
|
||||
assert!(stats.is_some(), "Select emitted End before Stats");
|
||||
saw_end = true;
|
||||
}
|
||||
_ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
|
||||
}
|
||||
}
|
||||
|
||||
let stats = stats.ok_or("Select response ended without a Stats event")?;
|
||||
assert_eq!(stats.bytes_scanned(), Some(i64::try_from(compressed_bytes)?));
|
||||
assert_eq!(stats.bytes_processed(), Some(i64::try_from(processed_bytes)?));
|
||||
assert_eq!(stats.bytes_returned(), Some(i64::try_from(records.len())?));
|
||||
assert!(saw_end, "Select response ended without an End event");
|
||||
Ok::<_, Box<dyn Error + Send + Sync>>(records)
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
|
||||
}
|
||||
|
||||
async fn assert_truncated_stream_failure(mut response: SelectObjectContentOutput) -> TestResult<()> {
|
||||
tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move {
|
||||
loop {
|
||||
match response.payload.recv().await {
|
||||
Err(error) => {
|
||||
// S3 Select request-level errors use `error` frames, which this SDK version exposes as raw response errors.
|
||||
if let Some(code) = error.code() {
|
||||
assert_eq!(code, "TruncatedInput", "unexpected modeled event-stream error: {error:?}");
|
||||
} else if let aws_sdk_s3::error::SdkError::ResponseError(context) = &error
|
||||
&& let RawMessage::Decoded(message) = context.raw()
|
||||
{
|
||||
let header = |name: &str| {
|
||||
message
|
||||
.headers()
|
||||
.iter()
|
||||
.find(|header| header.name().as_str() == name)
|
||||
.and_then(|header| header.value().as_string().ok())
|
||||
.map(|value| value.as_str())
|
||||
};
|
||||
assert_eq!(header(":message-type"), Some("error"));
|
||||
assert_eq!(header(":error-code"), Some("TruncatedInput"));
|
||||
} else {
|
||||
panic!("unexpected event-stream error: {error:?}");
|
||||
}
|
||||
return Ok(());
|
||||
}
|
||||
Ok(Some(SelectObjectContentEventStream::Stats(_))) | Ok(Some(SelectObjectContentEventStream::End(_))) => {
|
||||
return Err("truncated compressed input reached a success terminal event".into());
|
||||
}
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => return Err("truncated compressed input ended without an error event".into()),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "truncated Select response timed out".into() })?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_compressed_csv_and_json() -> TestResult<()> {
|
||||
const CSV: &[u8] = b"name,age\nAlice,30\nBob,25\n";
|
||||
const JSON_LINES: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
|
||||
const JSON_DOCUMENT: &[u8] = br#"[{"name":"Alice"},{"name":"Bob"}]"#;
|
||||
|
||||
let (_env, client) = create_test_environment(&[]).await?;
|
||||
|
||||
let gzip_csv = gzip(CSV)?;
|
||||
put_object(&client, "records.csv.gz", &gzip_csv).await?;
|
||||
let gzip_csv_records = collect_success(
|
||||
csv_select_request(&client, "records.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?,
|
||||
gzip_csv.len(),
|
||||
CSV.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(gzip_csv_records, b"Alice,30\nBob,25\n");
|
||||
|
||||
let bzip_csv = bzip2(CSV).await?;
|
||||
put_object(&client, "records.csv.bz2", &bzip_csv).await?;
|
||||
let bzip_csv_records = collect_success(
|
||||
csv_select_request(&client, "records.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?,
|
||||
bzip_csv.len(),
|
||||
CSV.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(bzip_csv_records, gzip_csv_records);
|
||||
|
||||
let gzip_json_lines = gzip(JSON_LINES)?;
|
||||
put_object(&client, "json-lines", &gzip_json_lines).await?;
|
||||
let gzip_json_records = collect_success(
|
||||
json_select_request(&client, "json-lines", CompressionType::Gzip, JsonType::Lines)
|
||||
.send()
|
||||
.await?,
|
||||
gzip_json_lines.len(),
|
||||
JSON_LINES.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(gzip_json_records, JSON_LINES);
|
||||
|
||||
let bzip_json_lines = bzip2(JSON_LINES).await?;
|
||||
put_object(&client, "records.jsonl.bz2", &bzip_json_lines).await?;
|
||||
let bzip_json_records = collect_success(
|
||||
json_select_request(&client, "records.jsonl.bz2", CompressionType::Bzip2, JsonType::Lines)
|
||||
.send()
|
||||
.await?,
|
||||
bzip_json_lines.len(),
|
||||
JSON_LINES.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(bzip_json_records, gzip_json_records);
|
||||
|
||||
let gzip_json_document = gzip(JSON_DOCUMENT)?;
|
||||
put_object(&client, "document.json.gz", &gzip_json_document).await?;
|
||||
let document_records = collect_success(
|
||||
json_select_request(&client, "document.json.gz", CompressionType::Gzip, JsonType::Document)
|
||||
.send()
|
||||
.await?,
|
||||
gzip_json_document.len(),
|
||||
JSON_DOCUMENT.len(),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(document_records, JSON_LINES);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_invalid_compressed_stream_fails() -> TestResult<()> {
|
||||
const CSV: &[u8] = b"name\nAlice\n";
|
||||
|
||||
let (_env, client) = create_test_environment(&[]).await?;
|
||||
|
||||
put_object(&client, "invalid.csv.gz", CSV).await?;
|
||||
let invalid = csv_select_request(&client, "invalid.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("invalid GZIP header must fail before streaming");
|
||||
assert_eq!(
|
||||
invalid.as_service_error().and_then(ProvideErrorMetadata::code),
|
||||
Some("InvalidCompressionFormat")
|
||||
);
|
||||
|
||||
put_object(&client, "empty.csv.gz", b"").await?;
|
||||
let empty = csv_select_request(&client, "empty.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("empty GZIP input must fail as truncated");
|
||||
assert_eq!(empty.as_service_error().and_then(ProvideErrorMetadata::code), Some("TruncatedInput"));
|
||||
|
||||
let mut truncated = bzip2(CSV).await?;
|
||||
truncated.pop();
|
||||
put_object(&client, "truncated.csv.bz2", &truncated).await?;
|
||||
let truncated = csv_select_request(&client, "truncated.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?;
|
||||
assert_truncated_stream_failure(truncated).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_compressed_disconnect_releases_query() -> TestResult<()> {
|
||||
const OBJECT: &str = "disconnect.csv.gz";
|
||||
const ROWS: usize = 16 * 1024;
|
||||
const RELEASE_ATTEMPTS: usize = 20;
|
||||
const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
|
||||
|
||||
let (_env, client) = create_test_environment(&[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")]).await?;
|
||||
let row = format!("{}\n", "x".repeat(1023));
|
||||
let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
|
||||
body.extend_from_slice(b"value\n");
|
||||
for _ in 0..ROWS {
|
||||
body.extend_from_slice(row.as_bytes());
|
||||
}
|
||||
let compressed = gzip(&body)?;
|
||||
put_object(&client, OBJECT, &compressed).await?;
|
||||
|
||||
let first = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await?;
|
||||
let saturated = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the unread compressed response should retain the only query permit");
|
||||
assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
|
||||
|
||||
drop(first);
|
||||
let second = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
for attempt in 0..RELEASE_ATTEMPTS {
|
||||
match csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object")
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
|
||||
Err(error)
|
||||
if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown")
|
||||
&& attempt + 1 < RELEASE_ATTEMPTS =>
|
||||
{
|
||||
tokio::time::sleep(RELEASE_BACKOFF).await;
|
||||
}
|
||||
Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
|
||||
return Err("disconnected compressed Select retained its query permit".into());
|
||||
}
|
||||
Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
|
||||
}
|
||||
}
|
||||
Err("query permit release retry loop ended unexpectedly".into())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "compressed Select did not release its query permit".into() })??;
|
||||
drop(second);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -17,7 +17,8 @@ use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::types::{
|
||||
CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType, OutputSerialization,
|
||||
CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, JsonType,
|
||||
OutputSerialization, RequestProgress,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use std::error::Error;
|
||||
@@ -26,6 +27,9 @@ use std::time::Duration;
|
||||
const BUCKET: &str = "test-sql-bucket";
|
||||
const CSV_OBJECT: &str = "test-data.csv";
|
||||
const JSON_OBJECT: &str = "test-data.json";
|
||||
const JSON_DOCUMENT_OBJECT: &str = "nested-data.json";
|
||||
const JSON_ROOT_ARRAY_OBJECT: &str = "root-array.json";
|
||||
const JSON_ROOT_SCALAR_ARRAY_OBJECT: &str = "root-scalars.json";
|
||||
const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
|
||||
type TestResult<T> = Result<T, Box<dyn Error + Send + Sync>>;
|
||||
@@ -73,6 +77,69 @@ async fn upload_test_json(client: &Client) -> TestResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn upload_nested_json_document(client: &Client) -> TestResult<()> {
|
||||
let json_data = r#"{"departments":[{"employees":[{"name":"Alice","active":true},{"name":"Bob","active":false}]},{"employees":[{"name":"Charlie","active":true}]}]}"#;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(JSON_DOCUMENT_OBJECT)
|
||||
.body(Bytes::from_static(json_data.as_bytes()).into())
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(JSON_ROOT_ARRAY_OBJECT)
|
||||
.body(Bytes::from_static(br#"[{"name":"Alice"},{"name":"Bob"}]"#).into())
|
||||
.send()
|
||||
.await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(JSON_ROOT_SCALAR_ARRAY_OBJECT)
|
||||
.body(Bytes::from_static(b"[1,2]").into())
|
||||
.send()
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn select_json_document(client: &Client, key: &str, expression: &str) -> TestResult<String> {
|
||||
let response = client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression(expression)
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build())
|
||||
.send()
|
||||
.await?;
|
||||
process_select_response(response).await
|
||||
}
|
||||
|
||||
fn csv_select_request(
|
||||
client: &Client,
|
||||
key: &str,
|
||||
) -> aws_sdk_s3::operation::select_object_content::builders::SelectObjectContentFluentBuilder {
|
||||
client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(key)
|
||||
.expression("SELECT * FROM S3Object")
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(
|
||||
InputSerialization::builder()
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
)
|
||||
.output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build())
|
||||
}
|
||||
|
||||
async fn process_select_response(
|
||||
mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput,
|
||||
) -> TestResult<String> {
|
||||
@@ -104,6 +171,209 @@ async fn process_select_response(
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })?
|
||||
}
|
||||
|
||||
async fn assert_input_byte_stats(
|
||||
client: &Client,
|
||||
object: &str,
|
||||
body: &[u8],
|
||||
expression: &str,
|
||||
input_serialization: InputSerialization,
|
||||
output_serialization: OutputSerialization,
|
||||
progress_enabled: bool,
|
||||
) -> TestResult<()> {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(object)
|
||||
.body(Bytes::copy_from_slice(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut request = client
|
||||
.select_object_content()
|
||||
.bucket(BUCKET)
|
||||
.key(object)
|
||||
.expression(expression)
|
||||
.expression_type(ExpressionType::Sql)
|
||||
.input_serialization(input_serialization)
|
||||
.output_serialization(output_serialization);
|
||||
if progress_enabled {
|
||||
request = request.request_progress(RequestProgress::builder().enabled(true).build());
|
||||
}
|
||||
let response = request.send().await?;
|
||||
|
||||
let mut payload = response.payload;
|
||||
let mut records_len = 0_u64;
|
||||
let mut last_progress: Option<aws_sdk_s3::types::Progress> = None;
|
||||
let mut stats = None;
|
||||
let mut saw_end = false;
|
||||
tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async {
|
||||
// The AWS SDK validates both event-stream CRCs before yielding an event.
|
||||
while let Some(event) = payload.recv().await? {
|
||||
assert!(!saw_end, "Select emitted an event after End");
|
||||
match event {
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::Records(records) => {
|
||||
assert!(stats.is_none(), "Select emitted Records after Stats");
|
||||
if let Some(bytes) = records.payload {
|
||||
records_len = records_len.saturating_add(u64::try_from(bytes.as_ref().len())?);
|
||||
}
|
||||
}
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::Progress(event) => {
|
||||
assert!(stats.is_none(), "Select emitted Progress after Stats");
|
||||
let details = event.details.ok_or("Progress event did not contain details")?;
|
||||
if let Some(previous) = last_progress.as_ref() {
|
||||
assert!(details.bytes_scanned() >= previous.bytes_scanned());
|
||||
assert!(details.bytes_processed() >= previous.bytes_processed());
|
||||
assert!(details.bytes_returned() >= previous.bytes_returned());
|
||||
}
|
||||
last_progress = Some(details);
|
||||
}
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::Stats(event) => {
|
||||
assert!(stats.is_none(), "Select emitted more than one Stats event");
|
||||
stats = event.details;
|
||||
}
|
||||
aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => {
|
||||
assert!(stats.is_some(), "Select emitted End before Stats");
|
||||
saw_end = true;
|
||||
}
|
||||
_ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"),
|
||||
}
|
||||
}
|
||||
Ok::<(), Box<dyn Error + Send + Sync>>(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "Select response timed out".into() })??;
|
||||
|
||||
let stats = stats.ok_or("Select response ended without a Stats event")?;
|
||||
let input_len = i64::try_from(body.len())?;
|
||||
assert_eq!(stats.bytes_scanned(), Some(input_len));
|
||||
assert_eq!(stats.bytes_processed(), Some(input_len));
|
||||
assert_eq!(stats.bytes_returned(), Some(i64::try_from(records_len)?));
|
||||
if progress_enabled {
|
||||
if let Some(progress) = last_progress {
|
||||
assert!(stats.bytes_scanned() >= progress.bytes_scanned());
|
||||
assert!(stats.bytes_processed() >= progress.bytes_processed());
|
||||
assert!(stats.bytes_returned() >= progress.bytes_returned());
|
||||
}
|
||||
} else {
|
||||
assert!(last_progress.is_none(), "disabled request progress emitted a Progress event");
|
||||
}
|
||||
assert!(saw_end, "Select response ended without an End event");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_http_event_order_crc_and_input_byte_stats() -> TestResult<()> {
|
||||
const CSV_BODY: &[u8] = b"name,age\nAlice,30\nBob,25\n";
|
||||
const JSON_LINES_BODY: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n";
|
||||
const JSON_DOCUMENT_BODY: &[u8] = b"[{\"name\":\"Alice\"},{\"name\":\"Bob\"}]";
|
||||
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
setup_test_bucket(&client).await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics.csv",
|
||||
CSV_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics.jsonl",
|
||||
JSON_LINES_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.json(JsonInput::builder().set_type(Some(JsonType::Lines)).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics.json",
|
||||
JSON_DOCUMENT_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.json(JsonInput::builder().set_type(Some(JsonType::Document)).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().json(JsonOutput::builder().build()).build(),
|
||||
true,
|
||||
)
|
||||
.await?;
|
||||
assert_input_byte_stats(
|
||||
&client,
|
||||
"input-metrics-without-progress.csv",
|
||||
CSV_BODY,
|
||||
"SELECT name FROM S3Object",
|
||||
InputSerialization::builder()
|
||||
.csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build())
|
||||
.build(),
|
||||
OutputSerialization::builder().csv(CsvOutput::builder().build()).build(),
|
||||
false,
|
||||
)
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_http_disconnect_releases_query() -> TestResult<()> {
|
||||
const OBJECT: &str = "disconnect.csv";
|
||||
const ROWS: usize = 16 * 1024;
|
||||
const RELEASE_BACKOFF: Duration = Duration::from_millis(25);
|
||||
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")])
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
setup_test_bucket(&client).await?;
|
||||
|
||||
let row = format!("{}\n", "x".repeat(1023));
|
||||
let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len());
|
||||
body.extend_from_slice(b"value\n");
|
||||
for _ in 0..ROWS {
|
||||
body.extend_from_slice(row.as_bytes());
|
||||
}
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(OBJECT)
|
||||
.body(Bytes::from(body).into())
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
// Leaving this response body unread fills the bounded HTTP/event channels before the query can finish.
|
||||
let first = csv_select_request(&client, OBJECT).send().await?;
|
||||
let saturated = csv_select_request(&client, OBJECT)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("the first HTTP stream should retain the only query permit");
|
||||
assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown"));
|
||||
|
||||
drop(first);
|
||||
let second = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
loop {
|
||||
match csv_select_request(&client, OBJECT).send().await {
|
||||
Ok(response) => return Ok::<_, Box<dyn Error + Send + Sync>>(response),
|
||||
Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => {
|
||||
tokio::time::sleep(RELEASE_BACKOFF).await;
|
||||
}
|
||||
Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()),
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| -> Box<dyn Error + Send + Sync> { "disconnected Select did not release its query permit".into() })??;
|
||||
drop(second);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_csv_basic() -> TestResult<()> {
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
@@ -228,6 +498,107 @@ async fn test_select_object_content_json_basic() -> TestResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_nested_json_source_path() -> TestResult<()> {
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
setup_test_bucket(&client).await?;
|
||||
upload_nested_json_document(&client).await?;
|
||||
|
||||
let result = select_json_document(
|
||||
&client,
|
||||
JSON_DOCUMENT_OBJECT,
|
||||
"SELECT e.name FROM S3Object[*].departments[*].employees[*] AS e WHERE e.active = true",
|
||||
)
|
||||
.await?;
|
||||
let names: Vec<String> = result
|
||||
.lines()
|
||||
.filter(|line| !line.trim().is_empty())
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["name"].as_str().ok_or("missing name field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
|
||||
assert_eq!(names, vec!["Alice", "Charlie"]);
|
||||
|
||||
let terminal_scalars = select_json_document(
|
||||
&client,
|
||||
JSON_DOCUMENT_OBJECT,
|
||||
"SELECT NAME FROM S3Object[*].DEPARTMENTS[*].employees[*].NAME",
|
||||
)
|
||||
.await?;
|
||||
let scalar_names: Vec<String> = terminal_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["name"].as_str().ok_or("missing scalar name field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(scalar_names, vec!["Alice", "Bob", "Charlie"]);
|
||||
|
||||
let aliased_scalars = select_json_document(
|
||||
&client,
|
||||
JSON_DOCUMENT_OBJECT,
|
||||
"SELECT v FROM S3Object[*].departments[*].employees[*].name AS v",
|
||||
)
|
||||
.await?;
|
||||
let aliased_names: Vec<String> = aliased_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["v"].as_str().ok_or("missing aliased scalar field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(aliased_names, vec!["Alice", "Bob", "Charlie"]);
|
||||
|
||||
let root_array = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][*] AS c").await?;
|
||||
let root_names: Vec<String> = root_array
|
||||
.lines()
|
||||
.map(|line| -> TestResult<String> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["name"].as_str().ok_or("missing root-array name field")?.to_string())
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(root_names, vec!["Alice", "Bob"]);
|
||||
|
||||
let root_index = select_json_document(&client, JSON_ROOT_ARRAY_OBJECT, "SELECT c.name FROM S3Object[*][0] AS c").await?;
|
||||
let root_index_value: serde_json::Value = serde_json::from_str(root_index.trim())?;
|
||||
assert_eq!(root_index_value["name"], "Alice");
|
||||
|
||||
let root_scalars = select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT V FROM S3Object AS V").await?;
|
||||
let scalar_values: Vec<i64> = root_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<i64> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["v"].as_i64().ok_or("missing root scalar value")?)
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(scalar_values, vec![1, 2]);
|
||||
|
||||
let implicit_root_scalars =
|
||||
select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT S3Object FROM S3Object").await?;
|
||||
let implicit_scalar_values: Vec<i64> = implicit_root_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<i64> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["s3object"].as_i64().ok_or("missing implicit root scalar value")?)
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(implicit_scalar_values, vec![1, 2]);
|
||||
|
||||
let quoted_root_scalars =
|
||||
select_json_document(&client, JSON_ROOT_SCALAR_ARRAY_OBJECT, "SELECT \"S3Object\" FROM \"S3Object\"").await?;
|
||||
let quoted_scalar_values: Vec<i64> = quoted_root_scalars
|
||||
.lines()
|
||||
.map(|line| -> TestResult<i64> {
|
||||
let value: serde_json::Value = serde_json::from_str(line)?;
|
||||
Ok(value["S3Object"].as_i64().ok_or("missing quoted root scalar value")?)
|
||||
})
|
||||
.collect::<TestResult<_>>()?;
|
||||
assert_eq!(quoted_scalar_values, vec![1, 2]);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn test_select_object_content_csv_limit() -> TestResult<()> {
|
||||
let (_env, client) = create_test_environment().await?;
|
||||
|
||||
@@ -17,8 +17,56 @@ mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use flate2::{Compression, write::GzEncoder};
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
use std::io::{Cursor, Write};
|
||||
|
||||
fn pax_record(key: &str, value: &str) -> Vec<u8> {
|
||||
let payload = format!("{key}={value}\n");
|
||||
let mut len = payload.len() + 3;
|
||||
loop {
|
||||
let record = format!("{len} {payload}");
|
||||
if record.len() == len {
|
||||
return record.into_bytes();
|
||||
}
|
||||
len = record.len();
|
||||
}
|
||||
}
|
||||
|
||||
async fn append_pax_header(
|
||||
builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
|
||||
entry_type: tokio_tar::EntryType,
|
||||
records: &[(&str, &str)],
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut payload = Vec::new();
|
||||
for (key, value) in records {
|
||||
payload.extend(pax_record(key, value));
|
||||
}
|
||||
let mut header = tokio_tar::Header::new_ustar();
|
||||
header.set_entry_type(entry_type);
|
||||
header.set_size(u64::try_from(payload.len()).expect("PAX payload length should fit in u64"));
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder
|
||||
.append_data(&mut header, "PaxHeaders.X/snowball", Cursor::new(payload))
|
||||
.await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn append_typed_entry(
|
||||
builder: &mut tokio_tar::Builder<Cursor<Vec<u8>>>,
|
||||
path: &str,
|
||||
entry_type: tokio_tar::EntryType,
|
||||
body: &[u8],
|
||||
) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_entry_type(entry_type);
|
||||
header.set_size(u64::try_from(body.len()).expect("TAR member length should fit in u64"));
|
||||
header.set_mode(0o644);
|
||||
header.set_cksum();
|
||||
builder.append_data(&mut header, path, Cursor::new(body)).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn build_test_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
@@ -69,12 +117,50 @@ mod tests {
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
|
||||
let path = format!("../{victim_bucket}/evil-injected.txt");
|
||||
let data = b"injected-body";
|
||||
async fn build_archive_with_invalid_checksum() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut archive = build_test_archive().await?;
|
||||
archive[0] ^= 1;
|
||||
Ok(archive)
|
||||
}
|
||||
|
||||
async fn build_archive_with_negative_gnu_mtime() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
let mut header = tokio_tar::Header::new_gnu();
|
||||
header.set_size(b"negative-mtime-body".len() as u64);
|
||||
header.set_mode(0o644);
|
||||
header.as_old_mut().mtime.fill(0xff);
|
||||
builder
|
||||
.append_data(&mut header, "negative-mtime.txt", Cursor::new(b"negative-mtime-body".as_slice()))
|
||||
.await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn gzip_member(payload: &[u8]) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut encoder = GzEncoder::new(Vec::new(), Compression::default());
|
||||
encoder.write_all(payload)?;
|
||||
Ok(encoder.finish()?)
|
||||
}
|
||||
|
||||
async fn build_concatenated_gzip_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let archive = build_test_archive().await?;
|
||||
let split_at = archive.len() / 2;
|
||||
let mut encoded = gzip_member(&archive[..split_at])?;
|
||||
encoded.extend(gzip_member(&archive[split_at..])?);
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
async fn build_gzip_archive_with_invalid_crc() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut encoded = gzip_member(&build_test_archive().await?)?;
|
||||
let crc_offset = encoded.len().checked_sub(8).expect("gzip fixture must contain a trailer");
|
||||
encoded[crc_offset] ^= 1;
|
||||
Ok(encoded)
|
||||
}
|
||||
|
||||
fn append_raw_tar_entry_with_type(archive: &mut Vec<u8>, path: &[u8], data: &[u8], entry_type: u8) {
|
||||
assert!(path.len() <= 100, "raw TAR fixture path must fit in the name field");
|
||||
let mut header = [0u8; 512];
|
||||
|
||||
header[..path.len()].copy_from_slice(path.as_bytes());
|
||||
header[..path.len()].copy_from_slice(path);
|
||||
header[100..108].copy_from_slice(b"0000644\0");
|
||||
header[108..116].copy_from_slice(b"0000000\0");
|
||||
header[116..124].copy_from_slice(b"0000000\0");
|
||||
@@ -82,7 +168,7 @@ mod tests {
|
||||
header[124..136].copy_from_slice(size.as_bytes());
|
||||
header[136..148].copy_from_slice(b"00000000000\0");
|
||||
header[148..156].fill(b' ');
|
||||
header[156] = b'0';
|
||||
header[156] = entry_type;
|
||||
header[257..263].copy_from_slice(b"ustar\0");
|
||||
header[263..265].copy_from_slice(b"00");
|
||||
|
||||
@@ -90,11 +176,87 @@ mod tests {
|
||||
let checksum = format!("{:06o}\0 ", checksum);
|
||||
header[148..156].copy_from_slice(checksum.as_bytes());
|
||||
|
||||
let mut archive = Vec::new();
|
||||
archive.extend_from_slice(&header);
|
||||
archive.extend_from_slice(data);
|
||||
let padding = (512 - (data.len() % 512)) % 512;
|
||||
archive.extend(std::iter::repeat_n(0, padding));
|
||||
}
|
||||
|
||||
fn append_raw_tar_entry(archive: &mut Vec<u8>, path: &[u8], data: &[u8]) {
|
||||
append_raw_tar_entry_with_type(archive, path, data, b'0');
|
||||
}
|
||||
|
||||
fn build_archive_with_parent_dir_entry(victim_bucket: &str) -> Vec<u8> {
|
||||
let path = format!("../{victim_bucket}/evil-injected.txt");
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry(&mut archive, path.as_bytes(), b"injected-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
|
||||
async fn build_member_semantics_archive() -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_pax_header(
|
||||
&mut builder,
|
||||
tokio_tar::EntryType::XGlobalHeader,
|
||||
&[
|
||||
("minio.metadata.x-amz-meta-owner", "global"),
|
||||
("minio.metadata.x-amz-meta-snowball-auto-extract", "true"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
append_pax_header(
|
||||
&mut builder,
|
||||
tokio_tar::EntryType::XHeader,
|
||||
&[("minio.metadata.x-amz-meta-owner", "local")],
|
||||
)
|
||||
.await?;
|
||||
append_typed_entry(&mut builder, "regular.txt", tokio_tar::EntryType::Regular, b"regular-body").await?;
|
||||
for (path, entry_type) in [
|
||||
("char", tokio_tar::EntryType::Char),
|
||||
("block", tokio_tar::EntryType::Block),
|
||||
("fifo", tokio_tar::EntryType::Fifo),
|
||||
] {
|
||||
append_typed_entry(&mut builder, path, entry_type, b"").await?;
|
||||
}
|
||||
let mut directory = tokio_tar::Header::new_gnu();
|
||||
directory.set_entry_type(tokio_tar::EntryType::Directory);
|
||||
directory.set_size(0);
|
||||
directory.set_mode(0o755);
|
||||
directory.set_cksum();
|
||||
builder
|
||||
.append_data(&mut directory, "directory/", Cursor::new(Vec::new()))
|
||||
.await?;
|
||||
for (path, entry_type) in [
|
||||
("hard-link", tokio_tar::EntryType::Link),
|
||||
("symlink", tokio_tar::EntryType::Symlink),
|
||||
("continuous", tokio_tar::EntryType::Continuous),
|
||||
("unknown", tokio_tar::EntryType::Other(b'9')),
|
||||
] {
|
||||
append_typed_entry(&mut builder, path, entry_type, b"").await?;
|
||||
}
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
async fn build_versioned_member_archive(path: &str, version_id: &str) -> Result<Vec<u8>, Box<dyn Error + Send + Sync>> {
|
||||
let mut builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
|
||||
append_pax_header(&mut builder, tokio_tar::EntryType::XHeader, &[("minio.versionId", version_id)]).await?;
|
||||
append_typed_entry(&mut builder, path, tokio_tar::EntryType::Regular, b"versioned-body").await?;
|
||||
Ok(builder.into_inner().await?.into_inner())
|
||||
}
|
||||
|
||||
fn build_archive_with_invalid_utf8_entry() -> Vec<u8> {
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry(&mut archive, b"invalid-\xff.txt", b"ignored-body");
|
||||
append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
|
||||
fn build_archive_with_invalid_utf8_symlink() -> Vec<u8> {
|
||||
let mut archive = Vec::new();
|
||||
append_raw_tar_entry_with_type(&mut archive, b"invalid-\xff-link", b"", b'2');
|
||||
append_raw_tar_entry(&mut archive, b"valid.txt", b"valid-body");
|
||||
archive.extend_from_slice(&[0u8; 1024]);
|
||||
archive
|
||||
}
|
||||
@@ -135,6 +297,147 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_applies_member_semantics_and_metadata_precedence() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-member-semantics";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Prefix", "members")
|
||||
.metadata("owner", "outer")
|
||||
.body(ByteStream::from(build_member_semantics_archive().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let regular = client.head_object().bucket(bucket).key("members/regular.txt").send().await?;
|
||||
let regular_metadata = regular.metadata().expect("regular member should expose metadata");
|
||||
assert_eq!(regular_metadata.get("owner").map(String::as_str), Some("local"));
|
||||
assert!(!regular_metadata.contains_key("snowball-auto-extract"));
|
||||
assert!(!regular_metadata.contains_key("minio-snowball-prefix"));
|
||||
|
||||
for key in ["char", "block", "fifo"] {
|
||||
let head = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("members/{key}"))
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(head.content_length(), Some(0), "{key} should be materialized as an empty object");
|
||||
assert_eq!(
|
||||
head.metadata().and_then(|metadata| metadata.get("owner")).map(String::as_str),
|
||||
Some("outer"),
|
||||
"{key} should not inherit global PAX metadata"
|
||||
);
|
||||
}
|
||||
let directory = client.head_object().bucket(bucket).key("members/directory/").send().await?;
|
||||
assert_eq!(directory.content_length(), Some(0));
|
||||
|
||||
for key in ["hard-link", "symlink", "continuous", "unknown"] {
|
||||
let error = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(format!("members/{key}"))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("unsupported TAR entry type must be skipped");
|
||||
assert_eq!(error.into_service_error().code(), Some("NotFound"), "{key}");
|
||||
}
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_validates_pax_version_id_against_bucket_state() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-version-semantics";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("null.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_versioned_member_archive("null.txt", "null").await?))
|
||||
.send()
|
||||
.await?;
|
||||
let null_member = client.get_object().bucket(bucket).key("null.txt").send().await?;
|
||||
assert_eq!(null_member.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
|
||||
|
||||
for (archive_key, member_key, version_id) in [
|
||||
("uuid.tar", "uuid.txt", uuid::Uuid::new_v4().to_string()),
|
||||
("uppercase-null.tar", "uppercase-null.txt", "NULL".to_string()),
|
||||
] {
|
||||
let error = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key(archive_key)
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_versioned_member_archive(member_key, &version_id).await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("invalid or unversioned UUID import must be rejected");
|
||||
assert_eq!(error.into_service_error().code(), Some("InvalidArgument"), "{archive_key}");
|
||||
let missing = client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(member_key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("rejected version import must not create an object");
|
||||
assert_eq!(missing.into_service_error().code(), Some("NotFound"), "{member_key}");
|
||||
}
|
||||
|
||||
client
|
||||
.put_bucket_versioning()
|
||||
.bucket(bucket)
|
||||
.versioning_configuration(
|
||||
aws_sdk_s3::types::VersioningConfiguration::builder()
|
||||
.status(aws_sdk_s3::types::BucketVersioningStatus::Enabled)
|
||||
.build(),
|
||||
)
|
||||
.send()
|
||||
.await?;
|
||||
let imported_version_id = uuid::Uuid::new_v4().to_string();
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("versioned-uuid.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(
|
||||
build_versioned_member_archive("versioned-uuid.txt", &imported_version_id).await?,
|
||||
))
|
||||
.send()
|
||||
.await?;
|
||||
let imported = client
|
||||
.get_object()
|
||||
.bucket(bucket)
|
||||
.key("versioned-uuid.txt")
|
||||
.version_id(&imported_version_id)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(imported.version_id(), Some(imported_version_id.as_str()));
|
||||
assert_eq!(imported.body.collect().await?.into_bytes().as_ref(), b"versioned-body");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_supports_standard_headers_with_combined_extract_options()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
@@ -263,6 +566,113 @@ mod tests {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_accepts_negative_gnu_mtime() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-negative-mtime";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_archive_with_negative_gnu_mtime().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object = client.get_object().bucket(bucket).key("negative-mtime.txt").send().await?;
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"negative-mtime-body");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_consumes_concatenated_gzip_members() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-concatenated-gzip";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar.gz")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_concatenated_gzip_archive().await?))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let object = client.get_object().bucket(bucket).key("root.txt").send().await?;
|
||||
assert_eq!(object.body.collect().await?.into_bytes().as_ref(), b"root payload\n");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_gzip_crc_error_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-gzip-crc-ignore-errors";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar.gz")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(build_gzip_archive_with_invalid_crc().await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("gzip integrity failures must remain fatal under ignore-errors");
|
||||
|
||||
assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_mismatched_content_md5() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-content-md5";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.content_md5("AAAAAAAAAAAAAAAAAAAAAA==")
|
||||
.body(ByteStream::from(build_test_archive().await?))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("mismatched Content-MD5 must fail after the raw body reaches EOF");
|
||||
|
||||
assert_eq!(err.into_service_error().code(), Some("BadDigest"));
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_ignores_invalid_entries_when_requested() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
@@ -299,7 +709,100 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_parent_dir_entry_without_cross_bucket_write()
|
||||
async fn snowball_auto_extract_skips_non_utf8_symlink_without_ignore_errors() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-invalid-utf8-link";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.body(ByteStream::from(build_archive_with_invalid_utf8_symlink()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
|
||||
assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
|
||||
assert_eq!(keys, vec!["valid.txt"]);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_skips_non_utf8_member_without_lossy_key_collision() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-invalid-utf8";
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(build_archive_with_invalid_utf8_entry()))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let valid = client.get_object().bucket(bucket).key("valid.txt").send().await?;
|
||||
assert_eq!(valid.body.collect().await?.into_bytes().as_ref(), b"valid-body");
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
let keys: Vec<_> = listed.contents().iter().filter_map(|entry| entry.key()).collect();
|
||||
assert_eq!(keys, vec!["valid.txt"]);
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_corrupt_tar_when_ignore_errors_enabled() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let client = env.create_s3_client();
|
||||
let bucket = "snowball-corrupt-ignore-errors";
|
||||
let archive = build_archive_with_invalid_checksum().await?;
|
||||
client.create_bucket().bucket(bucket).send().await?;
|
||||
|
||||
let err = client
|
||||
.put_object()
|
||||
.bucket(bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await
|
||||
.expect_err("corrupt TAR structure must remain fatal under ignore-errors");
|
||||
assert_eq!(err.into_service_error().code(), Some("InvalidArgument"));
|
||||
|
||||
let listed = client.list_objects_v2().bucket(bucket).send().await?;
|
||||
assert!(listed.contents().is_empty(), "corrupt archive must not produce objects");
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snowball_auto_extract_rejects_parent_dir_entry_even_when_ignore_errors_enabled()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
|
||||
@@ -319,6 +822,7 @@ mod tests {
|
||||
.bucket(attacker_bucket)
|
||||
.key("fixture.tar")
|
||||
.metadata("Snowball-Auto-Extract", "true")
|
||||
.metadata("Minio-Snowball-Ignore-Errors", "true")
|
||||
.body(ByteStream::from(archive))
|
||||
.send()
|
||||
.await
|
||||
|
||||
@@ -461,8 +461,8 @@ pub mod rpc {
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -489,9 +489,9 @@ pub mod storage {
|
||||
pub use crate::core::pools::HealLifecycleExpiryContext;
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::{
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path,
|
||||
find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients,
|
||||
prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,7 @@ use crate::runtime::sources as runtime_sources;
|
||||
use aws_credential_types::Credentials as SdkCredentials;
|
||||
use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future};
|
||||
use aws_sdk_s3::config::Region as SdkRegion;
|
||||
use aws_sdk_s3::config::RequestChecksumCalculation;
|
||||
use aws_sdk_s3::config::SharedHttpClient;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
@@ -39,6 +40,7 @@ use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::Tagging as SdkTagging;
|
||||
use aws_sdk_s3::types::{
|
||||
ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode,
|
||||
ServerSideEncryption,
|
||||
};
|
||||
use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput};
|
||||
use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus};
|
||||
@@ -1071,7 +1073,8 @@ impl BucketTargetSys {
|
||||
.endpoint_url(endpoint.clone())
|
||||
.credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds }))
|
||||
.region(SdkRegion::new(target.region.clone()))
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
|
||||
if should_force_path_style(target) {
|
||||
config_builder = config_builder.force_path_style(true);
|
||||
@@ -1367,6 +1370,25 @@ fn loopback_replication_targets_allowed() -> bool {
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS";
|
||||
|
||||
/// Streaming trailer checksums make the SDK frame request bodies as
|
||||
/// `aws-chunked`; a target that does not decode that framing stores the frames
|
||||
/// verbatim, silently corrupting every replica while the transfer itself
|
||||
/// succeeds (#6853). Plain signed payloads are the compatible default; the env
|
||||
/// knob restores trailer checksums for fleets whose targets are all known to
|
||||
/// decode them.
|
||||
fn replication_request_checksum_calculation() -> RequestChecksumCalculation {
|
||||
if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV)
|
||||
.map(|v| v.eq_ignore_ascii_case("true") || v == "1")
|
||||
.unwrap_or(false)
|
||||
{
|
||||
RequestChecksumCalculation::WhenSupported
|
||||
} else {
|
||||
RequestChecksumCalculation::WhenRequired
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_replication_target_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
|
||||
validate_replication_target_endpoint_inner(url, loopback_replication_targets_allowed())
|
||||
}
|
||||
@@ -1746,6 +1768,17 @@ impl Default for AdvancedPutOptions {
|
||||
}
|
||||
}
|
||||
|
||||
/// The subset of the target's PutObject response replication audits.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RemotePutObjectResponse {
|
||||
/// Version id the target assigned (`x-amz-version-id`).
|
||||
pub version_id: Option<String>,
|
||||
/// ETag of what the target stored; `None` when the target withheld it or
|
||||
/// when its encryption mode (SSE-KMS / SSE-C) makes it incomparable to
|
||||
/// the source ETag. `None` is therefore "not decidable", never evidence.
|
||||
pub etag: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct PutObjectOptions {
|
||||
pub user_metadata: HashMap<String, String>,
|
||||
@@ -2291,7 +2324,9 @@ impl TargetClient {
|
||||
|
||||
/// On success returns the version id the target assigned (from
|
||||
/// `x-amz-version-id`), letting callers audit the version-identity
|
||||
/// contract — a target that adopts the source version echoes it back.
|
||||
/// contract — a target that adopts the source version echoes it back —
|
||||
/// together with the ETag of what the target actually stored, so callers
|
||||
/// can detect a target that persisted transformed bytes (#6853).
|
||||
pub async fn put_object(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -2299,7 +2334,7 @@ impl TargetClient {
|
||||
size: i64,
|
||||
body: ByteStream,
|
||||
opts: &PutObjectOptions,
|
||||
) -> Result<Option<String>, S3ClientError> {
|
||||
) -> Result<RemotePutObjectResponse, S3ClientError> {
|
||||
let mut headers = opts.header();
|
||||
|
||||
let builder = self.client.put_object();
|
||||
@@ -2334,7 +2369,25 @@ impl TargetClient {
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(output) => Ok(output.version_id().map(ToOwned::to_owned)),
|
||||
Ok(output) => {
|
||||
// Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5
|
||||
// of the stored plaintext, so it cannot be compared against the
|
||||
// source ETag; withhold it rather than let a caller conclude
|
||||
// corruption from an opaque value.
|
||||
let etag_comparable = output.sse_customer_algorithm().is_none()
|
||||
&& !matches!(
|
||||
output.server_side_encryption(),
|
||||
Some(ServerSideEncryption::AwsKms) | Some(ServerSideEncryption::AwsKmsDsse)
|
||||
);
|
||||
Ok(RemotePutObjectResponse {
|
||||
version_id: output.version_id().map(ToOwned::to_owned),
|
||||
etag: if etag_comparable {
|
||||
output.e_tag().map(ToOwned::to_owned)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
})
|
||||
}
|
||||
Err(e) => match e {
|
||||
SdkError::ServiceError(service_err) => {
|
||||
let err = service_err.into_err();
|
||||
@@ -2673,6 +2726,145 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
type RecordedHeaders = Arc<std::sync::Mutex<Vec<Vec<(String, String)>>>>;
|
||||
|
||||
/// Records full request headers and answers with canned response headers,
|
||||
/// for asserting wire framing and response parsing.
|
||||
#[derive(Clone, Debug)]
|
||||
struct RecordingHeaderConnector {
|
||||
request_headers: RecordedHeaders,
|
||||
response_headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for RecordingHeaderConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
self.request_headers
|
||||
.lock()
|
||||
.expect("recorded header lock should not be poisoned")
|
||||
.push(
|
||||
request
|
||||
.headers()
|
||||
.iter()
|
||||
.map(|(k, v)| (k.to_string(), v.to_string()))
|
||||
.collect(),
|
||||
);
|
||||
let mut response = HttpResponse::new(
|
||||
aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
|
||||
SdkBody::empty(),
|
||||
);
|
||||
for (name, value) in &self.response_headers {
|
||||
response.headers_mut().insert(name.clone(), value.clone());
|
||||
}
|
||||
HttpConnectorFuture::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
fn header_recording_target_client(response_headers: Vec<(String, String)>) -> (TargetClient, RecordedHeaders) {
|
||||
let request_headers: RecordedHeaders = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(RecordingHeaderConnector {
|
||||
request_headers: Arc::clone(&request_headers),
|
||||
response_headers,
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let client = s3_client_for_test(443, Some(http_client));
|
||||
(
|
||||
TargetClient {
|
||||
endpoint: "https://localhost:443".to_string(),
|
||||
credentials: None,
|
||||
bucket: "target-bucket".to_string(),
|
||||
storage_class: String::new(),
|
||||
disable_proxy: false,
|
||||
arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
|
||||
reset_id: String::new(),
|
||||
secure: true,
|
||||
health_check_duration: Duration::from_secs(5),
|
||||
replicate_sync: false,
|
||||
client: Arc::new(client),
|
||||
},
|
||||
request_headers,
|
||||
)
|
||||
}
|
||||
|
||||
fn streaming_test_body(payload: &'static [u8]) -> ByteStream {
|
||||
let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload));
|
||||
let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data)));
|
||||
ByteStream::new(SdkBody::from_body_1_x(body))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_checksums_default_to_plain_payloads() {
|
||||
assert!(matches!(
|
||||
replication_request_checksum_calculation(),
|
||||
RequestChecksumCalculation::WhenRequired
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn replication_put_object_sends_plain_signed_payloads_by_default() {
|
||||
let (client, recorded) = header_recording_target_client(Vec::new());
|
||||
client
|
||||
.put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &PutObjectOptions::default())
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
|
||||
let recorded = recorded.lock().expect("recorded header lock should not be poisoned");
|
||||
let headers = &recorded[0];
|
||||
let header = |name: &str| {
|
||||
headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case(name))
|
||||
.map(|(_, v)| v.as_str())
|
||||
};
|
||||
// The #6853 regression shape: trailer checksums force aws-chunked
|
||||
// framing, which a non-decoding target stores verbatim as the object.
|
||||
assert_eq!(header("x-amz-trailer"), None, "streaming uploads must not carry a trailer checksum");
|
||||
assert!(
|
||||
header("content-encoding").is_none_or(|v| !v.contains("aws-chunked")),
|
||||
"streaming uploads must not be aws-chunked framed"
|
||||
);
|
||||
assert_eq!(header("x-amz-decoded-content-length"), None);
|
||||
assert_eq!(header("content-length"), Some("4"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_returns_the_etag_the_target_stored() {
|
||||
let (client, _) =
|
||||
header_recording_target_client(vec![("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string())]);
|
||||
let response = client
|
||||
.put_object(
|
||||
"target-bucket",
|
||||
"object",
|
||||
4,
|
||||
ByteStream::from_static(b"data"),
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
assert_eq!(response.etag.as_deref(), Some("\"9a0364b9e99bb480dd25e1f0284c8555\""));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_withholds_the_etag_under_target_side_kms() {
|
||||
let (client, _) = header_recording_target_client(vec![
|
||||
("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string()),
|
||||
("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()),
|
||||
]);
|
||||
let response = client
|
||||
.put_object(
|
||||
"target-bucket",
|
||||
"object",
|
||||
4,
|
||||
ByteStream::from_static(b"data"),
|
||||
&PutObjectOptions::default(),
|
||||
)
|
||||
.await
|
||||
.expect("recorded put_object should succeed");
|
||||
assert!(
|
||||
response.etag.is_none(),
|
||||
"a KMS-encrypted replica's etag is not the content MD5 and must be withheld"
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct RecordingAuthConnector {
|
||||
signed_requests: Arc<std::sync::Mutex<Vec<(bool, bool)>>>,
|
||||
@@ -2969,7 +3161,10 @@ mod tests {
|
||||
.credentials_provider(SharedCredentialsProvider::new(credentials))
|
||||
.region(SdkRegion::new("us-east-1"))
|
||||
.force_path_style(true)
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest());
|
||||
.behavior_version(aws_sdk_s3::config::BehaviorVersion::latest())
|
||||
// Mirror the production remote-target builder so recorded requests
|
||||
// exercise the same checksum/framing behavior (#6853).
|
||||
.request_checksum_calculation(replication_request_checksum_calculation());
|
||||
if let Some(http_client) = http_client {
|
||||
config = config.http_client(http_client);
|
||||
}
|
||||
|
||||
@@ -12195,7 +12195,7 @@ mod tests {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_continues_after_deleted_marker_bucket() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let suffix = Uuid::new_v4().simple();
|
||||
let earlier_bucket = format!("zzzz-recovery-{suffix}-a");
|
||||
let deleted_marker = format!("zzzz-recovery-{suffix}-m");
|
||||
@@ -12203,11 +12203,7 @@ mod tests {
|
||||
let later_object = "a-before-stale-marker";
|
||||
create_test_bucket(&ecstore, &earlier_bucket).await;
|
||||
create_test_bucket(&ecstore, &later_bucket).await;
|
||||
let mut reader = PutObjReader::from_vec(b"cursor reset probe".to_vec());
|
||||
ecstore
|
||||
.put_object(&later_bucket, later_object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("successor bucket object should be created");
|
||||
seed_recoverable_free_version(&disk_paths, &later_bucket, later_object, None, None).await;
|
||||
|
||||
let page = list_tier_free_versions(
|
||||
Arc::clone(&ecstore),
|
||||
@@ -12220,14 +12216,10 @@ mod tests {
|
||||
.expect("recovery should resume at the first bucket after a deleted marker bucket");
|
||||
|
||||
assert_eq!(page.buckets_scanned, 1, "the later bucket must not be skipped");
|
||||
assert_eq!(
|
||||
page.scanned_entries, 1,
|
||||
"the deleted bucket's object marker must not skip objects in the successor bucket"
|
||||
);
|
||||
ecstore
|
||||
.delete_object(&later_bucket, later_object, ObjectOptions::default())
|
||||
.await
|
||||
.expect("successor bucket object should be removed");
|
||||
assert_eq!(page.items.len(), 1, "the successor bucket's recoverable object must be returned");
|
||||
assert_eq!(page.items[0].bucket, later_bucket);
|
||||
assert_eq!(page.items[0].name, later_object);
|
||||
remove_seeded_free_version(&disk_paths, &later_bucket, later_object).await;
|
||||
for bucket in [&earlier_bucket, &later_bucket] {
|
||||
ecstore
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
|
||||
@@ -412,8 +412,14 @@ pub(crate) fn require_bucket_metadata_sys_in(
|
||||
}
|
||||
|
||||
pub(crate) async fn object_store_in(ctx: &crate::runtime::instance::InstanceContext) -> Result<Arc<ECStore>> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?;
|
||||
Ok(sys.read().await.api.clone())
|
||||
object_store_if_initialized_in(ctx)
|
||||
.await
|
||||
.ok_or_else(|| Error::other("bucket metadata sys not initialized for this instance"))
|
||||
}
|
||||
|
||||
pub(crate) async fn object_store_if_initialized_in(ctx: &crate::runtime::instance::InstanceContext) -> Option<Arc<ECStore>> {
|
||||
let sys = ctx.bucket_metadata_sys().or_else(get_global_bucket_metadata_sys)?;
|
||||
Some(sys.read().await.api.clone())
|
||||
}
|
||||
|
||||
pub(crate) async fn get_in(ctx: &crate::runtime::instance::InstanceContext, bucket: &str) -> Result<Arc<BucketMetadata>> {
|
||||
@@ -2512,11 +2518,169 @@ pub(crate) mod test_support {
|
||||
mod tests {
|
||||
use super::test_support::isolated_store_over_temp_disks;
|
||||
use super::*;
|
||||
use crate::bucket::metadata::{
|
||||
BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG,
|
||||
BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG,
|
||||
};
|
||||
use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials};
|
||||
use crate::config::com::read_config;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use byteorder::{ByteOrder as _, LittleEndian};
|
||||
use serial_test::serial;
|
||||
use tokio::time::timeout;
|
||||
|
||||
const NEW_WRITER_REPLICATION_XML: &[u8] = br#"<ReplicationConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"><Role>arn:aws:iam::111122223333:role/replication-role</Role><Rule><ID>rollback</ID><Priority>1</Priority><Filter><Prefix>documents/</Prefix></Filter><Status>Enabled</Status><Destination><Bucket>arn:aws:s3:::replica-bucket</Bucket></Destination><DeleteMarkerReplication><Status>Disabled</Status></DeleteMarkerReplication></Rule></ReplicationConfiguration>"#;
|
||||
|
||||
const NEW_WRITER_CONFIGS: [(&str, &[u8]); 14] = [
|
||||
(BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#),
|
||||
(BUCKET_NOTIFICATION_CONFIG, br#"<NotificationConfiguration/>"#),
|
||||
(
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
br#"<LifecycleConfiguration><Rule><ID>expire</ID><Status>Enabled</Status><Filter><Prefix>logs/</Prefix></Filter><Expiration><Days>30</Days></Expiration></Rule></LifecycleConfiguration>"#,
|
||||
),
|
||||
(
|
||||
OBJECT_LOCK_CONFIG,
|
||||
br#"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled><Rule><DefaultRetention><Mode>GOVERNANCE</Mode><Days>7</Days></DefaultRetention></Rule></ObjectLockConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_SSECONFIG,
|
||||
br#"<ServerSideEncryptionConfiguration><Rule><ApplyServerSideEncryptionByDefault><SSEAlgorithm>AES256</SSEAlgorithm></ApplyServerSideEncryptionByDefault></Rule></ServerSideEncryptionConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_TAGGING_CONFIG,
|
||||
r#"<Tagging><TagSet><Tag><Key>environment</Key><Value>测试-🦀</Value></Tag></TagSet></Tagging>"#.as_bytes(),
|
||||
),
|
||||
(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML),
|
||||
(
|
||||
BUCKET_CORS_CONFIG,
|
||||
br#"<CORSConfiguration><CORSRule><AllowedMethod>GET</AllowedMethod><AllowedOrigin>https://example.test</AllowedOrigin></CORSRule></CORSConfiguration>"#,
|
||||
),
|
||||
(BUCKET_LOGGING_CONFIG, br#"<BucketLoggingStatus/>"#),
|
||||
(
|
||||
BUCKET_WEBSITE_CONFIG,
|
||||
br#"<WebsiteConfiguration><IndexDocument><Suffix>index.html</Suffix></IndexDocument></WebsiteConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_ACCELERATE_CONFIG,
|
||||
br#"<AccelerateConfiguration><Status>Enabled</Status></AccelerateConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_REQUEST_PAYMENT_CONFIG,
|
||||
br#"<RequestPaymentConfiguration><Payer>Requester</Payer></RequestPaymentConfiguration>"#,
|
||||
),
|
||||
(
|
||||
BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG,
|
||||
br#"<PublicAccessBlockConfiguration><BlockPublicAcls>true</BlockPublicAcls><IgnorePublicAcls>true</IgnorePublicAcls><BlockPublicPolicy>true</BlockPublicPolicy><RestrictPublicBuckets>false</RestrictPublicBuckets></PublicAccessBlockConfiguration>"#,
|
||||
),
|
||||
];
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "rollback-new-replication";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
|
||||
}
|
||||
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
metadata
|
||||
.update_config(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML.to_vec())
|
||||
.expect("new-writer replication XML should be accepted before persistence");
|
||||
writer
|
||||
.persist_new_and_set(metadata)
|
||||
.await
|
||||
.expect("new-writer replication metadata should persist");
|
||||
|
||||
let old_reader = BucketMetadataSys::new(store);
|
||||
let (loaded, _) = old_reader
|
||||
.get_replication_config(bucket)
|
||||
.await
|
||||
.expect("old metadata_sys must not classify new-writer replication XML as invalid");
|
||||
assert_eq!(loaded.role, "arn:aws:iam::111122223333:role/replication-role");
|
||||
assert_eq!(loaded.rules.len(), 1);
|
||||
assert_eq!(loaded.rules[0].id.as_deref(), Some("rollback"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn g_d3_004_new_writer_metadata_blob_keeps_legacy_header_and_configs() {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
let bucket = "rollback-new-metadata";
|
||||
for dir in &dirs {
|
||||
std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created");
|
||||
}
|
||||
|
||||
let writer = BucketMetadataSys::new(store.clone());
|
||||
let mut metadata = BucketMetadata::new(bucket);
|
||||
for (config_file, bytes) in NEW_WRITER_CONFIGS {
|
||||
metadata
|
||||
.update_config(config_file, bytes.to_vec())
|
||||
.unwrap_or_else(|err| panic!("new-writer {config_file} fixture must be valid: {err}"));
|
||||
}
|
||||
writer
|
||||
.persist_new_and_set(metadata)
|
||||
.await
|
||||
.expect("new-writer metadata should persist");
|
||||
|
||||
let path = BucketMetadata::new(bucket).save_file_path();
|
||||
let blob = read_config(store.clone(), &path)
|
||||
.await
|
||||
.expect("persisted .metadata.bin should be readable");
|
||||
assert_eq!(
|
||||
LittleEndian::read_u16(&blob[0..2]),
|
||||
1,
|
||||
"bucket metadata format must stay rollback-readable"
|
||||
);
|
||||
assert_eq!(
|
||||
LittleEndian::read_u16(&blob[2..4]),
|
||||
1,
|
||||
"bucket metadata version must stay rollback-readable"
|
||||
);
|
||||
|
||||
let loaded = load_bucket_metadata(store, bucket)
|
||||
.await
|
||||
.expect("old read_bucket_metadata path must load the new-writer blob");
|
||||
let loaded_configs: [(&str, &[u8]); 14] = [
|
||||
(BUCKET_POLICY_CONFIG, &loaded.policy_config_json),
|
||||
(BUCKET_NOTIFICATION_CONFIG, &loaded.notification_config_xml),
|
||||
(BUCKET_LIFECYCLE_CONFIG, &loaded.lifecycle_config_xml),
|
||||
(OBJECT_LOCK_CONFIG, &loaded.object_lock_config_xml),
|
||||
(BUCKET_VERSIONING_CONFIG, &loaded.versioning_config_xml),
|
||||
(BUCKET_SSECONFIG, &loaded.encryption_config_xml),
|
||||
(BUCKET_TAGGING_CONFIG, &loaded.tagging_config_xml),
|
||||
(BUCKET_REPLICATION_CONFIG, &loaded.replication_config_xml),
|
||||
(BUCKET_CORS_CONFIG, &loaded.cors_config_xml),
|
||||
(BUCKET_LOGGING_CONFIG, &loaded.logging_config_xml),
|
||||
(BUCKET_WEBSITE_CONFIG, &loaded.website_config_xml),
|
||||
(BUCKET_ACCELERATE_CONFIG, &loaded.accelerate_config_xml),
|
||||
(BUCKET_REQUEST_PAYMENT_CONFIG, &loaded.request_payment_config_xml),
|
||||
(BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, &loaded.public_access_block_config_xml),
|
||||
];
|
||||
for ((expected_name, expected), (loaded_name, actual)) in NEW_WRITER_CONFIGS.into_iter().zip(loaded_configs) {
|
||||
assert_eq!(loaded_name, expected_name);
|
||||
assert_eq!(actual, expected, "old read_bucket_metadata changed {expected_name} bytes");
|
||||
}
|
||||
assert!(loaded.policy_config.is_some());
|
||||
assert!(loaded.notification_config.is_some());
|
||||
assert!(loaded.lifecycle_config.is_some());
|
||||
assert!(loaded.object_lock_config.is_some());
|
||||
assert!(loaded.versioning_config.is_some());
|
||||
assert!(loaded.sse_config.is_some());
|
||||
assert!(loaded.tagging_config.is_some());
|
||||
assert!(loaded.replication_config.is_some());
|
||||
assert!(loaded.cors_config.is_some());
|
||||
assert!(loaded.logging_config.is_some());
|
||||
assert!(loaded.website_config.is_some());
|
||||
assert!(loaded.accelerate_config.is_some());
|
||||
assert!(loaded.request_payment_config.is_some());
|
||||
assert!(loaded.public_access_block_config.is_some());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn malformed_delete_configs_are_not_treated_as_absent() {
|
||||
let (_dirs, ecstore) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -177,6 +177,28 @@ pub fn replication_write_may_pass_worm_gate(
|
||||
Ok(!(retention_locked && opts.replication_retention_timestamp.is_none()))
|
||||
}
|
||||
|
||||
/// Whether an authorized replication delete (`ObjectOptions::replication_request`)
|
||||
/// addressed to an explicit version may bypass GOVERNANCE retention on the
|
||||
/// local replica, exactly as an `x-amz-bypass-governance-retention` caller
|
||||
/// with the bypass permission would.
|
||||
///
|
||||
/// The source is authoritative for a replicated version purge (issue #6850):
|
||||
/// the same WORM deletion gate already ran there, and GOVERNANCE retention
|
||||
/// with an authorized bypass is the only lock state it can purge through.
|
||||
/// Requiring the bypass header again here makes the purge permanently
|
||||
/// undeliverable — replication senders never carry it — and the sites diverge
|
||||
/// forever. COMPLIANCE retention and legal hold stay blocking: the source
|
||||
/// gate can never purge through them, so a replication purge that meets one
|
||||
/// here is divergence or forgery and fails closed.
|
||||
///
|
||||
/// The trust judgment is the same one the write-path exemption uses:
|
||||
/// `replication_request` is only set once the receiving handler has
|
||||
/// authorized the caller for the replication action
|
||||
/// (`ReplicateDeleteAction`), never straight from request headers.
|
||||
pub fn replication_delete_may_bypass_governance(opts: &ObjectOptions) -> bool {
|
||||
opts.replication_request && opts.version_id.is_some()
|
||||
}
|
||||
|
||||
/// Check if an object is locked based on its metadata.
|
||||
/// This is a common function used by both lifecycle evaluation and deletion checks.
|
||||
///
|
||||
@@ -680,6 +702,32 @@ mod tests {
|
||||
assert!(err.to_string().contains("modification time"));
|
||||
}
|
||||
|
||||
/// The replicated-purge GOVERNANCE bypass (#6850) applies only to an
|
||||
/// authorized replication delete addressed to an explicit version: a
|
||||
/// local delete never gets it, and a replicated delete without a version
|
||||
/// id creates a delete marker rather than purging anything.
|
||||
#[test]
|
||||
fn replication_delete_bypasses_governance_only_for_authorized_version_purges() {
|
||||
let version_purge = ObjectOptions {
|
||||
replication_request: true,
|
||||
version_id: Some("6b6ffbc0-b0d3-4a86-8f6c-fe19163b8dcd".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(replication_delete_may_bypass_governance(&version_purge));
|
||||
|
||||
let local_version_delete = ObjectOptions {
|
||||
replication_request: false,
|
||||
..version_purge.clone()
|
||||
};
|
||||
assert!(!replication_delete_may_bypass_governance(&local_version_delete));
|
||||
|
||||
let replicated_marker_creation = ObjectOptions {
|
||||
version_id: None,
|
||||
..version_purge
|
||||
};
|
||||
assert!(!replication_delete_may_bypass_governance(&replicated_marker_creation));
|
||||
}
|
||||
|
||||
/// A local PutObjectRetention / PutObjectLegalHold "clear" persists the
|
||||
/// lock keys as empty strings (the MinIO on-disk shape, see
|
||||
/// `parse_object_lock_retention`); that is "no lock", not corruption, and
|
||||
|
||||
@@ -20,8 +20,8 @@ pub use rustfs_replication::{
|
||||
pub(crate) use rustfs_replication::{
|
||||
ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry,
|
||||
delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, is_retryable_delete_replication_head_error,
|
||||
is_version_delete_replication, replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object,
|
||||
should_retry_delete_marker_purge, target_delete_version_id,
|
||||
delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
|
||||
resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
};
|
||||
|
||||
@@ -3177,6 +3177,19 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
}
|
||||
}
|
||||
ReplicationHealQueueAction::QueueDelete(dv) => {
|
||||
// A purge the peer denied under object lock cannot succeed until
|
||||
// the lock lapses (#6850); requeuing it every heal cycle only
|
||||
// burns bandwidth and failure counters. The backoff expires on
|
||||
// its own, so the purge is probed again — and converges — once
|
||||
// the retention window has a chance of being over.
|
||||
if super::replication_object_decision_boundary::is_version_delete_replication(&dv.delete_object)
|
||||
&& super::replication_resyncer::object_lock_denied_purge_backoff_active(&dv)
|
||||
{
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
};
|
||||
}
|
||||
let admission = if let Some(pool) = runtime_sources::replication_pool() {
|
||||
pool.queue_replica_delete_task(dv).await
|
||||
} else {
|
||||
|
||||
@@ -30,10 +30,10 @@ use super::replication_msgp_boundary::ReplicationMsgpCodec;
|
||||
use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate};
|
||||
use super::replication_object_decision_boundary::{
|
||||
MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
delete_replication_creates_marker, heal_uses_delete_replication_path, is_retryable_delete_replication_head_error,
|
||||
is_version_delete_replication, replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size,
|
||||
replication_multipart_part_plan, resync_existing_delete_replication_info, should_retry_delete_marker_purge,
|
||||
target_delete_version_id,
|
||||
delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete,
|
||||
is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match,
|
||||
replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info,
|
||||
should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id,
|
||||
};
|
||||
use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission};
|
||||
use super::replication_resync_boundary::ResyncStatusType;
|
||||
@@ -54,7 +54,7 @@ use super::replication_storage_boundary::{
|
||||
};
|
||||
use super::replication_target_boundary::{
|
||||
ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions,
|
||||
ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient,
|
||||
RemotePutObjectResponse, ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient,
|
||||
is_replication_target_offline_error, replication_action_for_target_head, replication_complete_multipart_options,
|
||||
replication_delete_marker_purge_remove_options, replication_delete_remove_options, replication_force_delete_remove_options,
|
||||
replication_object_is_ssec_encrypted, replication_put_object_header_size, replication_put_object_options,
|
||||
@@ -96,7 +96,7 @@ use tokio::task::{JoinHandle, JoinSet};
|
||||
use tokio::time::Duration as TokioDuration;
|
||||
use tokio_util::io::ReaderStream;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, error, instrument, trace, warn};
|
||||
use tracing::{debug, error, info, instrument, trace, warn};
|
||||
|
||||
const BACKGROUND_WALKDIR_TIMEOUT: TokioDuration = TokioDuration::from_secs(60);
|
||||
const ENV_REPL_RESYNC_MAX_JOBS: &str = "RUSTFS_REPL_RESYNC_MAX_JOBS";
|
||||
@@ -112,11 +112,13 @@ const EVENT_REPLICATION_DELETE_SKIPPED: &str = "replication_delete_skipped";
|
||||
const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_skipped";
|
||||
const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed";
|
||||
const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed";
|
||||
const EVENT_REPLICATION_ABORT_RETRY_RESOLVED: &str = "replication_abort_retry_resolved";
|
||||
const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed";
|
||||
const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_failed";
|
||||
const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf";
|
||||
const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total";
|
||||
const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift";
|
||||
const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied";
|
||||
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -194,6 +196,127 @@ const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_id
|
||||
/// after a restart is acceptable.
|
||||
static VERSION_IDENTITY_WARNED_ARNS: LazyLock<StdMutex<HashSet<String>>> = LazyLock::new(|| StdMutex::new(HashSet::new()));
|
||||
|
||||
/// Version purges the peer denied under object lock (#6850). A RustFS peer
|
||||
/// with the replicated-purge GOVERNANCE exemption
|
||||
/// (`replication_delete_may_bypass_governance`) no longer produces this for
|
||||
/// governance retention, but COMPLIANCE retention, legal hold, and targets
|
||||
/// without the exemption (older RustFS, MinIO, generic S3) still deny — and
|
||||
/// such a purge cannot succeed until the lock on the replica lapses, so
|
||||
/// retrying every heal cycle only burns bandwidth and failure counters.
|
||||
/// Entries suppress heal requeues for the backoff window; after it expires
|
||||
/// one probe runs again, so the purge still converges on its own once
|
||||
/// retention ends. In-process only: a restart costs at most one extra probe
|
||||
/// per entry.
|
||||
const OBJECT_LOCK_DENIED_PURGE_BACKOFF: std::time::Duration = std::time::Duration::from_secs(60 * 60);
|
||||
const OBJECT_LOCK_DENIED_PURGE_CACHE_MAX: usize = 4096;
|
||||
type ObjectLockDeniedPurgeKey = (String, String, String);
|
||||
|
||||
struct ObjectLockDeniedPurge {
|
||||
denied_at: std::time::Instant,
|
||||
denied_arns: HashSet<String>,
|
||||
}
|
||||
|
||||
static OBJECT_LOCK_DENIED_PURGES: LazyLock<StdMutex<HashMap<ObjectLockDeniedPurgeKey, ObjectLockDeniedPurge>>> =
|
||||
LazyLock::new(|| StdMutex::new(HashMap::new()));
|
||||
|
||||
fn object_lock_denied_purge_key(dobj: &DeletedObjectReplicationInfo) -> ObjectLockDeniedPurgeKey {
|
||||
let version_id = dobj
|
||||
.delete_object
|
||||
.delete_marker_version_id
|
||||
.or(dobj.delete_object.version_id)
|
||||
.unwrap_or_default();
|
||||
(dobj.bucket.clone(), dobj.delete_object.object_name.clone(), version_id.to_string())
|
||||
}
|
||||
|
||||
fn record_object_lock_denied_purge(dobj: &DeletedObjectReplicationInfo, arn: &str) {
|
||||
let mut denied = OBJECT_LOCK_DENIED_PURGES
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if denied.len() >= OBJECT_LOCK_DENIED_PURGE_CACHE_MAX {
|
||||
denied.retain(|_, entry| entry.denied_at.elapsed() < OBJECT_LOCK_DENIED_PURGE_BACKOFF);
|
||||
}
|
||||
let key = object_lock_denied_purge_key(dobj);
|
||||
if denied.len() < OBJECT_LOCK_DENIED_PURGE_CACHE_MAX || denied.contains_key(&key) {
|
||||
let entry = denied.entry(key).or_insert_with(|| ObjectLockDeniedPurge {
|
||||
denied_at: std::time::Instant::now(),
|
||||
denied_arns: HashSet::new(),
|
||||
});
|
||||
entry.denied_at = std::time::Instant::now();
|
||||
entry.denied_arns.insert(arn.to_string());
|
||||
}
|
||||
// Still full after dropping expired entries: skip recording — the purge
|
||||
// then simply keeps retrying, which is the pre-#6850 behavior.
|
||||
}
|
||||
|
||||
/// Whether a heal requeue of this delete can only reach targets that denied
|
||||
/// it under object lock within the backoff window. A target the entry does
|
||||
/// not cover (another peer, or one whose denial expired) keeps the requeue
|
||||
/// flowing — suppressing it would delay a purge that could succeed there.
|
||||
pub(crate) fn object_lock_denied_purge_backoff_active(dobj: &DeletedObjectReplicationInfo) -> bool {
|
||||
let key = object_lock_denied_purge_key(dobj);
|
||||
let mut denied = OBJECT_LOCK_DENIED_PURGES
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
match denied.get(&key) {
|
||||
Some(entry) if entry.denied_at.elapsed() < OBJECT_LOCK_DENIED_PURGE_BACKOFF => {
|
||||
let admitted = dobj.admitted_target_arns();
|
||||
!admitted.is_empty() && admitted.iter().all(|arn| entry.denied_arns.contains(arn))
|
||||
}
|
||||
Some(_) => {
|
||||
denied.remove(&key);
|
||||
false
|
||||
}
|
||||
None => false,
|
||||
}
|
||||
}
|
||||
|
||||
const REPLICA_ETAG_VERIFY_ENV: &str = "RUSTFS_REPLICATION_REPLICA_ETAG_VERIFY";
|
||||
|
||||
/// Escape hatch for a target whose 32-hex ETags are legitimately not the
|
||||
/// content MD5 (e.g. a gateway hashing its own ciphertext without announcing
|
||||
/// SSE in the response) — such a target would otherwise fail every object.
|
||||
fn replica_etag_verification_enabled() -> bool {
|
||||
std::env::var(REPLICA_ETAG_VERIFY_ENV)
|
||||
.map(|v| !(v.eq_ignore_ascii_case("false") || v == "0"))
|
||||
.unwrap_or(true)
|
||||
}
|
||||
|
||||
/// A 200 from the target is not proof the replica holds the source bytes: a
|
||||
/// target that stores a transformed payload (e.g. undecoded `aws-chunked`
|
||||
/// frames, #6853) returns the ETag of what it actually wrote. Reporting
|
||||
/// COMPLETED over such a replica is silent corruption, so a decidable
|
||||
/// mismatch fails the replication instead. An SSE-C ciphertext passthrough
|
||||
/// transfer is exempt: the wire bytes are ciphertext while the source ETag is
|
||||
/// the plaintext MD5, and that path has its own HEAD-back audit.
|
||||
fn verify_single_part_replica(
|
||||
object_info: &ObjectInfo,
|
||||
response: &RemotePutObjectResponse,
|
||||
ciphertext_passthrough: bool,
|
||||
) -> std::result::Result<(), std::io::Error> {
|
||||
if ciphertext_passthrough || !replica_etag_verification_enabled() {
|
||||
return Ok(());
|
||||
}
|
||||
if single_part_replica_etag_mismatch(object_info.etag.as_deref(), response.etag.as_deref()) {
|
||||
// The differing ETags go into the structured log; the error message
|
||||
// stays constant so same-cause failures bucket together downstream.
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %object_info.bucket,
|
||||
object = %object_info.name,
|
||||
source_etag = ?object_info.etag,
|
||||
replica_etag = ?response.etag,
|
||||
operation = "verify_replica_etag",
|
||||
"Replication target operation failed"
|
||||
);
|
||||
return Err(std::io::Error::other(REPLICA_ETAG_MISMATCH_ERROR));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
const REPLICA_ETAG_MISMATCH_ERROR: &str = "replica etag mismatch: the target persisted different bytes than were sent";
|
||||
|
||||
fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) {
|
||||
if !version_identity_drifted(source_version_id, assigned_version_id) {
|
||||
return;
|
||||
@@ -2708,19 +2831,43 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = tgt_client.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = ?version_id,
|
||||
delete_marker = dobj.delete_object.delete_marker,
|
||||
is_version_purge,
|
||||
error = %e,
|
||||
operation = "replicate_delete_to_target",
|
||||
"Replication target operation failed"
|
||||
);
|
||||
let object_lock_denied = is_version_purge && is_object_lock_denied_delete(e.code.as_deref(), e.message.as_deref());
|
||||
if object_lock_denied {
|
||||
// Terminal for as long as the lock holds: the peer retains
|
||||
// this version under COMPLIANCE retention or legal hold, or
|
||||
// is a target without the replicated-purge GOVERNANCE
|
||||
// exemption (#6850), so the sites stay diverged until the
|
||||
// lock on the replica lapses. Surface it loudly instead of
|
||||
// letting a silent failed counter and a hot heal-retry loop
|
||||
// stand in for the divergence.
|
||||
record_object_lock_denied_purge(dobj, &tgt_client.arn);
|
||||
error!(
|
||||
event = EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = tgt_client.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = ?version_id,
|
||||
arn = %tgt_client.arn,
|
||||
error = %e,
|
||||
operation = "replicate_delete_to_target",
|
||||
"Replicated version purge denied by object lock on the target; the sites stay diverged until the lock lapses"
|
||||
);
|
||||
} else {
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = tgt_client.bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = ?version_id,
|
||||
delete_marker = dobj.delete_object.delete_marker,
|
||||
is_version_purge,
|
||||
error = %e,
|
||||
operation = "replicate_delete_to_target",
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
rinfo.error = Some(e.to_string());
|
||||
if !is_version_purge {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
@@ -3274,14 +3421,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
let result = tgt_client
|
||||
.put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts)
|
||||
.await
|
||||
.map(|assigned_version_id| {
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))
|
||||
.and_then(|response| {
|
||||
audit_target_version_identity(
|
||||
&tgt_client,
|
||||
&put_opts.internal.source_version_id,
|
||||
assigned_version_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
response.version_id.as_deref(),
|
||||
);
|
||||
verify_single_part_replica(&object_info, &response, obj_opts.raw_data_movement_read)
|
||||
});
|
||||
result.err()
|
||||
} {
|
||||
rinfo.replication_status = ReplicationStatusType::Failed;
|
||||
@@ -3942,14 +4090,15 @@ async fn replicate_all_payload_to_target<S: ReplicationObjectIO>(
|
||||
.tgt_client
|
||||
.put_object(&ctx.tgt_client.bucket, ctx.object, ctx.transfer_size, byte_stream, &ctx.put_opts)
|
||||
.await
|
||||
.map(|assigned_version_id| {
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))
|
||||
.and_then(|response| {
|
||||
audit_target_version_identity(
|
||||
ctx.tgt_client,
|
||||
&ctx.put_opts.internal.source_version_id,
|
||||
assigned_version_id.as_deref(),
|
||||
)
|
||||
})
|
||||
.map_err(|e| std::io::Error::other(e.to_string()));
|
||||
response.version_id.as_deref(),
|
||||
);
|
||||
verify_single_part_replica(ctx.object_info, &response, ctx.obj_opts.raw_data_movement_read)
|
||||
});
|
||||
result.err()
|
||||
}
|
||||
}
|
||||
@@ -4036,28 +4185,132 @@ async fn replicate_object_with_multipart<S: ReplicationObjectIO>(ctx: MultipartR
|
||||
let arn = ctx.arn;
|
||||
|
||||
let result = replicate_multipart_parts_and_complete(ctx, &upload_id).await;
|
||||
abort_multipart_on_failure(result, dst_bucket, object, &upload_id, arn, || async {
|
||||
cli.abort_multipart_upload(dst_bucket, object, &upload_id).await
|
||||
})
|
||||
abort_multipart_on_failure(
|
||||
result,
|
||||
dst_bucket,
|
||||
object,
|
||||
&upload_id,
|
||||
arn,
|
||||
|| async { cli.abort_multipart_upload(dst_bucket, object, &upload_id).await },
|
||||
|| {
|
||||
schedule_replication_abort_retry(
|
||||
cli.clone(),
|
||||
dst_bucket.to_string(),
|
||||
object.to_string(),
|
||||
upload_id.clone(),
|
||||
arn.to_string(),
|
||||
)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
const REPLICATION_ABORT_RETRY_ATTEMPTS: u32 = 5;
|
||||
const REPLICATION_ABORT_RETRY_INITIAL_DELAY_SECS: u64 = 30;
|
||||
|
||||
/// The immediate abort usually fails for the same reason the transfer did —
|
||||
/// the target is unreachable — and MRF only retries the *object*: every replay
|
||||
/// mints a fresh upload id, so a failed abort would leak its upload on the
|
||||
/// target forever (#6854). Retry the abort on a detached, bounded backoff
|
||||
/// (~30s..8m) so it lands once the target comes back; an upload the target no
|
||||
/// longer knows counts as cleaned up.
|
||||
fn schedule_replication_abort_retry(cli: Arc<TargetClient>, dst_bucket: String, object: String, upload_id: String, arn: String) {
|
||||
tokio::spawn(async move {
|
||||
let mut delay_secs = REPLICATION_ABORT_RETRY_INITIAL_DELAY_SECS;
|
||||
for attempt in 1..=REPLICATION_ABORT_RETRY_ATTEMPTS {
|
||||
tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs)).await;
|
||||
delay_secs = delay_secs.saturating_mul(2);
|
||||
|
||||
match cli.abort_multipart_upload(&dst_bucket, &object, &upload_id).await {
|
||||
Ok(()) => {
|
||||
info!(
|
||||
event = EVENT_REPLICATION_ABORT_RETRY_RESOLVED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
target_bucket = %dst_bucket,
|
||||
object = %object,
|
||||
arn = %arn,
|
||||
upload_id = %upload_id,
|
||||
operation = "abort_multipart_upload_retry",
|
||||
attempt,
|
||||
"Replication abort retry cleaned up the orphaned upload"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) if target_upload_already_removed(&err) => {
|
||||
info!(
|
||||
event = EVENT_REPLICATION_ABORT_RETRY_RESOLVED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
target_bucket = %dst_bucket,
|
||||
object = %object,
|
||||
arn = %arn,
|
||||
upload_id = %upload_id,
|
||||
operation = "abort_multipart_upload_retry",
|
||||
attempt,
|
||||
"Replication abort retry found the upload already removed"
|
||||
);
|
||||
return;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
target_bucket = %dst_bucket,
|
||||
object = %object,
|
||||
arn = %arn,
|
||||
upload_id = %upload_id,
|
||||
operation = "abort_multipart_upload_retry",
|
||||
attempt,
|
||||
error = %err,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Terminal: the upload id stays in the log so an operator can reap it
|
||||
// with list-multipart-uploads/abort by hand (the #6840 contract).
|
||||
warn!(
|
||||
event = EVENT_RESYNC_TARGET_OPERATION_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
target_bucket = %dst_bucket,
|
||||
object = %object,
|
||||
arn = %arn,
|
||||
upload_id = %upload_id,
|
||||
operation = "abort_multipart_upload_retry",
|
||||
result = "gave_up",
|
||||
"Replication abort retries exhausted; the incomplete upload remains on the target"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/// AWS answers an abort for an unknown upload with `NoSuchUpload`; that means
|
||||
/// the orphan is gone (aborted elsewhere or expired), which is the goal state.
|
||||
fn target_upload_already_removed(err: &S3ClientError) -> bool {
|
||||
err.code.as_deref() == Some("NoSuchUpload")
|
||||
}
|
||||
|
||||
/// Best-effort abort of the target-side multipart upload once the transfer has
|
||||
/// failed past CreateMultipartUpload; without it every failed attempt leaves an
|
||||
/// invisible incomplete upload on the target that keeps billing for its parts.
|
||||
/// The abort outcome never replaces the transfer error: an abort failure is
|
||||
/// only logged and `result` is returned as-is.
|
||||
async fn abort_multipart_on_failure<F, Fut>(
|
||||
async fn abort_multipart_on_failure<F, Fut, R>(
|
||||
result: std::io::Result<()>,
|
||||
dst_bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
arn: &str,
|
||||
abort: F,
|
||||
schedule_abort_retry: R,
|
||||
) -> std::io::Result<()>
|
||||
where
|
||||
F: FnOnce() -> Fut,
|
||||
Fut: std::future::Future<Output = std::result::Result<(), S3ClientError>>,
|
||||
R: FnOnce(),
|
||||
{
|
||||
if result.is_ok() {
|
||||
return result;
|
||||
@@ -4075,6 +4328,9 @@ where
|
||||
error = %abort_err,
|
||||
"Replication target operation failed"
|
||||
);
|
||||
if !target_upload_already_removed(&abort_err) {
|
||||
schedule_abort_retry();
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -5354,27 +5610,72 @@ mod tests {
|
||||
assert!(!resync_state_accepts_update(¤t, &stale));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lock_denied_purge_backoff_tracks_version_and_target() {
|
||||
let denied = DeletedObjectReplicationInfo {
|
||||
bucket: "worm-backoff-test-bucket".to_string(),
|
||||
target_arn: "arn:rustfs:replication::worm-test:t1".to_string(),
|
||||
delete_object: ReplicationDeletedObject {
|
||||
object_name: "locked-object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert!(!object_lock_denied_purge_backoff_active(&denied));
|
||||
|
||||
record_object_lock_denied_purge(&denied, "arn:rustfs:replication::worm-test:t1");
|
||||
assert!(object_lock_denied_purge_backoff_active(&denied));
|
||||
|
||||
// A requeue that can also reach a target this denial does not cover
|
||||
// must keep flowing: the purge may succeed there.
|
||||
let mut other_target = denied.clone();
|
||||
other_target.target_arn = "arn:rustfs:replication::worm-test:t2".to_string();
|
||||
assert!(!object_lock_denied_purge_backoff_active(&other_target));
|
||||
|
||||
// A different version of the same object must not be suppressed.
|
||||
let mut other_version = denied;
|
||||
other_version.delete_object.version_id = Some(uuid::Uuid::new_v4());
|
||||
assert!(!object_lock_denied_purge_backoff_active(&other_version));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abort_multipart_on_failure_skips_abort_when_transfer_succeeded() {
|
||||
let aborted = Arc::new(AtomicBool::new(false));
|
||||
let flag = aborted.clone();
|
||||
let retry_scheduled = Arc::new(AtomicBool::new(false));
|
||||
let retry_flag = retry_scheduled.clone();
|
||||
|
||||
let result = abort_multipart_on_failure(Ok(()), "dst-bucket", "obj", "upload-1", "arn:dest", move || async move {
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
})
|
||||
let result = abort_multipart_on_failure(
|
||||
Ok(()),
|
||||
"dst-bucket",
|
||||
"obj",
|
||||
"upload-1",
|
||||
"arn:dest",
|
||||
move || async move {
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
Ok(())
|
||||
},
|
||||
move || retry_flag.store(true, Ordering::SeqCst),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(result.is_ok());
|
||||
assert!(!aborted.load(Ordering::SeqCst));
|
||||
assert!(!retry_scheduled.load(Ordering::SeqCst));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abort_multipart_on_failure_aborts_and_keeps_transfer_error() {
|
||||
let aborted = Arc::new(AtomicBool::new(false));
|
||||
let flag = aborted.clone();
|
||||
let retry_scheduled = Arc::new(AtomicBool::new(false));
|
||||
let retry_flag = retry_scheduled.clone();
|
||||
|
||||
// The abort itself failing must not mask the transfer error.
|
||||
// The abort itself failing must not mask the transfer error, and a
|
||||
// failed abort must hand the upload id to the retry schedule (#6854):
|
||||
// the object itself is re-replicated under a fresh upload id, so
|
||||
// nothing else will ever abort this one.
|
||||
let result = abort_multipart_on_failure(
|
||||
Err(std::io::Error::other("transfer failed")),
|
||||
"dst-bucket",
|
||||
@@ -5385,10 +5686,34 @@ mod tests {
|
||||
flag.store(true, Ordering::SeqCst);
|
||||
Err(S3ClientError::new("abort failed"))
|
||||
},
|
||||
move || retry_flag.store(true, Ordering::SeqCst),
|
||||
)
|
||||
.await;
|
||||
|
||||
assert!(aborted.load(Ordering::SeqCst));
|
||||
assert!(retry_scheduled.load(Ordering::SeqCst));
|
||||
assert_eq!(result.unwrap_err().to_string(), "transfer failed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn abort_multipart_on_failure_does_not_retry_a_gone_upload() {
|
||||
let retry_scheduled = Arc::new(AtomicBool::new(false));
|
||||
let retry_flag = retry_scheduled.clone();
|
||||
|
||||
let result = abort_multipart_on_failure(
|
||||
Err(std::io::Error::other("transfer failed")),
|
||||
"dst-bucket",
|
||||
"obj",
|
||||
"upload-1",
|
||||
"arn:dest",
|
||||
|| async { Err(S3ClientError::with_metadata("gone", None, Some("NoSuchUpload".to_string()), None)) },
|
||||
move || retry_flag.store(true, Ordering::SeqCst),
|
||||
)
|
||||
.await;
|
||||
|
||||
// NoSuchUpload means the orphan no longer exists; retrying would only
|
||||
// produce noise.
|
||||
assert!(!retry_scheduled.load(Ordering::SeqCst));
|
||||
assert_eq!(result.unwrap_err().to_string(), "transfer failed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,8 +36,8 @@ use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, S3ClientError,
|
||||
TargetClient, resolve_read_api_version_id,
|
||||
AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions,
|
||||
S3ClientError, TargetClient, resolve_read_api_version_id,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
|
||||
@@ -73,6 +73,7 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> {
|
||||
check_bucket_name_common(bucket_name, true)
|
||||
}
|
||||
|
||||
// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten.
|
||||
pub fn deserialize<T>(input: &[u8]) -> xml::DeResult<T>
|
||||
where
|
||||
T: for<'xml> xml::Deserialize<'xml>,
|
||||
|
||||
@@ -1307,6 +1307,32 @@ pub fn verify_tonic_mutation_body_digest<T>(request: &tonic::Request<T>, canonic
|
||||
verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, internode_rpc_body_digest_strict())
|
||||
}
|
||||
|
||||
/// Verify a non-disk mutation without accepting a newly-generated unsigned v2 body.
|
||||
///
|
||||
/// The disk mutation lane has a rolling-upgrade exception for `UNSIGNED-PAYLOAD`
|
||||
/// while peer replay-cache capability is being discovered. Historical v2 peers
|
||||
/// used the fixed `unsigned` nonce before body-digest rollout; preserve that
|
||||
/// exact marker for mixed-version compatibility, but reject unsigned v2
|
||||
/// requests that omit it or present a different nonce.
|
||||
pub fn verify_tonic_mutation_body_digest_reject_unsigned<T>(
|
||||
request: &tonic::Request<T>,
|
||||
canonical_body: &[u8],
|
||||
) -> std::io::Result<()> {
|
||||
let version = request
|
||||
.metadata()
|
||||
.get(RPC_AUTH_VERSION_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let digest = request
|
||||
.metadata()
|
||||
.get(RPC_CONTENT_SHA256_HEADER)
|
||||
.and_then(|value| value.to_str().ok());
|
||||
let nonce = request.metadata().get(RPC_NONCE_HEADER).and_then(|value| value.to_str().ok());
|
||||
if version == Some(RPC_AUTH_VERSION_V2) && digest == Some(UNSIGNED_PAYLOAD) && nonce != Some("unsigned") {
|
||||
return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature"));
|
||||
}
|
||||
verify_tonic_mutation_body_digest(request, canonical_body)
|
||||
}
|
||||
|
||||
/// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both
|
||||
/// rollout postures are unit-testable without racing on process-global environment variables.
|
||||
fn verify_tonic_mutation_body_digest_with_strictness<T>(
|
||||
|
||||
@@ -36,6 +36,7 @@ use rustfs_rio::{ChunkReaderBox, HttpChunkReader, HttpReader, HttpWriter};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Arc, LazyLock, OnceLock};
|
||||
use std::task::{Context, Poll};
|
||||
@@ -105,9 +106,13 @@ struct PutFileCapabilityCacheState {
|
||||
cached: Option<PutFileCapabilityState>,
|
||||
generation: u64,
|
||||
in_flight: Option<PutFileCapabilityFlight>,
|
||||
rejected_server_epoch: Option<Uuid>,
|
||||
}
|
||||
|
||||
type PutFileCapabilityCacheEntry = Arc<tokio::sync::RwLock<PutFileCapabilityCacheState>>;
|
||||
// The registry lock is released before taking an entry lock. Entry guards cover
|
||||
// only cache transitions, never a probe or await; poll-based writers must be
|
||||
// able to reject an epoch atomically with those transitions.
|
||||
type PutFileCapabilityCacheEntry = Arc<parking_lot::RwLock<PutFileCapabilityCacheState>>;
|
||||
|
||||
static PUT_FILE_CAPABILITY_CACHE: LazyLock<parking_lot::RwLock<HashMap<String, PutFileCapabilityCacheEntry>>> =
|
||||
LazyLock::new(|| parking_lot::RwLock::new(HashMap::new()));
|
||||
@@ -119,7 +124,7 @@ fn put_file_capability_cache_entry(endpoint: &str) -> PutFileCapabilityCacheEntr
|
||||
PUT_FILE_CAPABILITY_CACHE
|
||||
.write()
|
||||
.entry(endpoint.to_owned())
|
||||
.or_insert_with(|| Arc::new(tokio::sync::RwLock::new(PutFileCapabilityCacheState::default())))
|
||||
.or_insert_with(|| Arc::new(parking_lot::RwLock::new(PutFileCapabilityCacheState::default())))
|
||||
.clone()
|
||||
}
|
||||
|
||||
@@ -134,6 +139,23 @@ fn fresh_put_file_capability(state: Option<PutFileCapabilityState>, now: Instant
|
||||
}
|
||||
}
|
||||
|
||||
fn reject_put_file_server_epoch(endpoint: &str, server_epoch: Uuid) {
|
||||
let entry = PUT_FILE_CAPABILITY_CACHE.read().get(endpoint).cloned();
|
||||
if let Some(entry) = entry {
|
||||
let mut state = entry.write();
|
||||
if matches!(state.cached, Some(PutFileCapabilityState::V1 { server_epoch: cached, .. }) if cached == server_epoch) {
|
||||
state.rejected_server_epoch = Some(server_epoch);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn usable_put_file_capability(state: &PutFileCapabilityCacheState, now: Instant) -> Option<Option<Uuid>> {
|
||||
match fresh_put_file_capability(state.cached, now)? {
|
||||
Some(server_epoch) if state.rejected_server_epoch == Some(server_epoch) => None,
|
||||
capability => Some(capability),
|
||||
}
|
||||
}
|
||||
|
||||
fn put_file_capability_status_is_legacy(status: u16) -> bool {
|
||||
status == 404
|
||||
}
|
||||
@@ -322,13 +344,14 @@ impl InternodeDataTransport for TcpHttpInternodeDataTransport {
|
||||
|
||||
async fn open_write(&self, request: WriteStreamRequest) -> Result<FileWriter> {
|
||||
let server_epoch = self.put_file_auth_capability(&request.endpoint).await?;
|
||||
let nonce = server_epoch.map(|_| Uuid::new_v4());
|
||||
let url = build_put_file_stream_url(&request, nonce.zip(server_epoch));
|
||||
let auth_scope = server_epoch.map(|server_epoch| (Uuid::new_v4(), server_epoch));
|
||||
let url = build_put_file_stream_url(&request, auth_scope);
|
||||
let endpoint = request.endpoint;
|
||||
let mut headers = json_headers();
|
||||
build_auth_headers(&url, &Method::PUT, &mut headers)?;
|
||||
let writer = HttpWriter::new(url.clone(), Method::PUT, headers).await?;
|
||||
match nonce {
|
||||
Some(nonce) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce))),
|
||||
match auth_scope {
|
||||
Some((nonce, server_epoch)) => Ok(Box::new(PutFileAuthWriter::new(writer, url, nonce, endpoint, server_epoch))),
|
||||
None => Ok(Box::new(writer)),
|
||||
}
|
||||
}
|
||||
@@ -498,15 +521,15 @@ where
|
||||
{
|
||||
let entry = put_file_capability_cache_entry(endpoint);
|
||||
{
|
||||
let state = entry.read().await;
|
||||
if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
|
||||
let state = entry.read();
|
||||
if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
|
||||
return Ok(cached);
|
||||
}
|
||||
}
|
||||
|
||||
let flight = {
|
||||
let mut state = entry.write().await;
|
||||
if let Some(cached) = fresh_put_file_capability(state.cached, Instant::now()) {
|
||||
let mut state = entry.write();
|
||||
if let Some(cached) = usable_put_file_capability(&state, Instant::now()) {
|
||||
return Ok(cached);
|
||||
}
|
||||
if let Some(flight) = state.in_flight.clone() {
|
||||
@@ -532,7 +555,7 @@ where
|
||||
.await;
|
||||
|
||||
{
|
||||
let mut state = entry.write().await;
|
||||
let mut state = entry.write();
|
||||
let is_current_flight = state
|
||||
.in_flight
|
||||
.as_ref()
|
||||
@@ -540,6 +563,9 @@ where
|
||||
if is_current_flight {
|
||||
match outcome {
|
||||
Ok(Some(server_epoch)) => {
|
||||
if state.rejected_server_epoch != Some(*server_epoch) {
|
||||
state.rejected_server_epoch = None;
|
||||
}
|
||||
state.cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch: *server_epoch,
|
||||
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
|
||||
@@ -630,17 +656,23 @@ struct PutFileAuthWriter<W> {
|
||||
inner: W,
|
||||
url: String,
|
||||
nonce: Uuid,
|
||||
endpoint: String,
|
||||
server_epoch: Uuid,
|
||||
server_epoch_rejected: bool,
|
||||
hasher: Sha256,
|
||||
trailer: Option<Vec<u8>>,
|
||||
trailer_offset: usize,
|
||||
}
|
||||
|
||||
impl<W> PutFileAuthWriter<W> {
|
||||
fn new(inner: W, url: String, nonce: Uuid) -> Self {
|
||||
fn new(inner: W, url: String, nonce: Uuid, endpoint: String, server_epoch: Uuid) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
url,
|
||||
nonce,
|
||||
endpoint,
|
||||
server_epoch,
|
||||
server_epoch_rejected: false,
|
||||
hasher: Sha256::new(),
|
||||
trailer: None,
|
||||
trailer_offset: 0,
|
||||
@@ -656,6 +688,14 @@ impl<W> PutFileAuthWriter<W> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn reject_server_epoch_on_conflict(&mut self, error: &io::Error) {
|
||||
if self.server_epoch_rejected || !io_error_has_put_file_epoch_conflict(error) {
|
||||
return;
|
||||
}
|
||||
reject_put_file_server_epoch(&self.endpoint, self.server_epoch);
|
||||
self.server_epoch_rejected = true;
|
||||
}
|
||||
|
||||
fn poll_write_trailer(&mut self, cx: &mut Context<'_>) -> Poll<std::io::Result<()>>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
@@ -673,7 +713,10 @@ impl<W> PutFileAuthWriter<W> {
|
||||
)));
|
||||
}
|
||||
Poll::Ready(Ok(written)) => written,
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
return Poll::Ready(Err(err));
|
||||
}
|
||||
Poll::Pending => return Poll::Pending,
|
||||
};
|
||||
self.trailer_offset += written;
|
||||
@@ -682,6 +725,15 @@ impl<W> PutFileAuthWriter<W> {
|
||||
}
|
||||
}
|
||||
|
||||
fn io_error_has_put_file_epoch_conflict(error: &io::Error) -> bool {
|
||||
error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<rustfs_rio::InternodeHttpError>())
|
||||
.is_some_and(
|
||||
|error| matches!(error.kind(), rustfs_rio::InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409),
|
||||
)
|
||||
}
|
||||
|
||||
impl<W> AsyncWrite for PutFileAuthWriter<W>
|
||||
where
|
||||
W: AsyncWrite + Unpin,
|
||||
@@ -698,12 +750,22 @@ where
|
||||
self.hasher.update(&buf[..written]);
|
||||
Poll::Ready(Ok(written))
|
||||
}
|
||||
other => other,
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
Poll::Ready(Err(err))
|
||||
}
|
||||
Poll::Pending => Poll::Pending,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Pin::new(&mut self.inner).poll_flush(cx)
|
||||
match Pin::new(&mut self.inner).poll_flush(cx) {
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
Poll::Ready(Err(err))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
|
||||
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
@@ -712,7 +774,13 @@ where
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
Poll::Pending => return Poll::Pending,
|
||||
}
|
||||
Pin::new(&mut self.inner).poll_shutdown(cx)
|
||||
match Pin::new(&mut self.inner).poll_shutdown(cx) {
|
||||
Poll::Ready(Err(err)) => {
|
||||
self.reject_server_epoch_on_conflict(&err);
|
||||
Poll::Ready(Err(err))
|
||||
}
|
||||
other => other,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -840,7 +908,6 @@ mod tests {
|
||||
loop {
|
||||
let strong_count = entry
|
||||
.read()
|
||||
.await
|
||||
.in_flight
|
||||
.as_ref()
|
||||
.map(|flight| Arc::strong_count(&flight.outcome))
|
||||
@@ -858,6 +925,50 @@ mod tests {
|
||||
#[derive(Debug)]
|
||||
struct LegacyTestTransport;
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum PutFileFailurePhase {
|
||||
Write,
|
||||
Flush,
|
||||
Shutdown,
|
||||
}
|
||||
|
||||
struct PutFileFailureWriter {
|
||||
phase: PutFileFailurePhase,
|
||||
status: reqwest::StatusCode,
|
||||
}
|
||||
|
||||
impl PutFileFailureWriter {
|
||||
fn error(&self) -> io::Error {
|
||||
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
|
||||
}
|
||||
}
|
||||
|
||||
impl tokio::io::AsyncWrite for PutFileFailureWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<std::io::Result<usize>> {
|
||||
Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Write) {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(buf.len())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Flush) {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<std::io::Result<()>> {
|
||||
Poll::Ready(if matches!(self.phase, PutFileFailurePhase::Shutdown) {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl InternodeDataTransport for LegacyTestTransport {
|
||||
async fn open_read(&self, _request: ReadStreamRequest) -> Result<FileReader> {
|
||||
@@ -1048,7 +1159,7 @@ mod tests {
|
||||
let v1_endpoint = format!("http://v1-{}.invalid", Uuid::new_v4());
|
||||
let v1_entry = put_file_capability_cache_entry(&v1_endpoint);
|
||||
let server_epoch = Uuid::new_v4();
|
||||
v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
|
||||
v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch,
|
||||
revalidate_after: Instant::now() + PUT_FILE_V1_CAPABILITY_TTL,
|
||||
});
|
||||
@@ -1067,7 +1178,7 @@ mod tests {
|
||||
Some(server_epoch)
|
||||
);
|
||||
assert!(!cache_probe_called.load(Ordering::SeqCst));
|
||||
v1_entry.write().await.cached = Some(PutFileCapabilityState::V1 {
|
||||
v1_entry.write().cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch,
|
||||
revalidate_after: Instant::now(),
|
||||
});
|
||||
@@ -1086,8 +1197,7 @@ mod tests {
|
||||
|
||||
let legacy_endpoint = format!("http://legacy-{}.invalid", Uuid::new_v4());
|
||||
let legacy_entry = put_file_capability_cache_entry(&legacy_endpoint);
|
||||
legacy_entry.write().await.cached =
|
||||
Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
|
||||
legacy_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now() + PUT_FILE_LEGACY_CAPABILITY_TTL));
|
||||
assert!(
|
||||
transport
|
||||
.put_file_auth_capability(&legacy_endpoint)
|
||||
@@ -1098,7 +1208,7 @@ mod tests {
|
||||
|
||||
let expired_endpoint = format!("http://expired-legacy-{}.invalid", Uuid::new_v4());
|
||||
let expired_entry = put_file_capability_cache_entry(&expired_endpoint);
|
||||
expired_entry.write().await.cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
|
||||
expired_entry.write().cached = Some(PutFileCapabilityState::LegacyUntil(Instant::now()));
|
||||
let reprobed = std::sync::atomic::AtomicBool::new(false);
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&expired_endpoint, || async {
|
||||
@@ -1349,7 +1459,7 @@ mod tests {
|
||||
};
|
||||
probe_started.notified().await;
|
||||
{
|
||||
let mut state = entry.write().await;
|
||||
let mut state = entry.write();
|
||||
state.generation = state.generation.checked_add(1).expect("test generation should advance");
|
||||
state.cached = Some(PutFileCapabilityState::V1 {
|
||||
server_epoch: newer_epoch,
|
||||
@@ -1362,10 +1472,7 @@ mod tests {
|
||||
task.await.expect("stale task should finish").expect("stale probe result"),
|
||||
Some(stale_epoch)
|
||||
);
|
||||
assert_eq!(
|
||||
fresh_put_file_capability(entry.read().await.cached, Instant::now()),
|
||||
Some(Some(newer_epoch))
|
||||
);
|
||||
assert_eq!(fresh_put_file_capability(entry.read().cached, Instant::now()), Some(Some(newer_epoch)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1398,6 +1505,8 @@ mod tests {
|
||||
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-auth-writer-test-secret".to_string());
|
||||
let nonce = Uuid::parse_str("11111111-2222-4333-8444-555555555555").expect("nonce");
|
||||
let server_epoch = Uuid::parse_str("aaaaaaaa-bbbb-4ccc-8ddd-eeeeeeeeeeee").expect("server epoch");
|
||||
let endpoint = "http://node1:9000".to_string();
|
||||
let url = concat!(
|
||||
"http://node1:9000/rustfs/rpc/put_file_stream?disk=disk-a&volume=bucket&path=object%2Fpart.1",
|
||||
"&append=false&size=11&put_file_auth=digest-trailer-v1&put_file_nonce=11111111-2222-4333-8444-555555555555"
|
||||
@@ -1406,7 +1515,7 @@ mod tests {
|
||||
let mut sink = Vec::new();
|
||||
|
||||
{
|
||||
let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce);
|
||||
let mut writer = PutFileAuthWriter::new(&mut sink, url.clone(), nonce, endpoint, server_epoch);
|
||||
writer.write_all(b"hello world").await.expect("body write should succeed");
|
||||
writer.shutdown().await.expect("shutdown should append auth trailer");
|
||||
let err = writer
|
||||
@@ -1424,6 +1533,143 @@ mod tests {
|
||||
assert_eq!(verified, expected_digest);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_auth_writer_reprobes_after_server_epoch_conflict() {
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("put-file-epoch-conflict-test-secret".to_string());
|
||||
for status in [reqwest::StatusCode::CONFLICT, reqwest::StatusCode::BAD_REQUEST] {
|
||||
for (phase, trailer_write) in [
|
||||
(PutFileFailurePhase::Write, false),
|
||||
(PutFileFailurePhase::Write, true),
|
||||
(PutFileFailurePhase::Flush, false),
|
||||
(PutFileFailurePhase::Shutdown, false),
|
||||
] {
|
||||
let endpoint = format!("http://epoch-conflict-{}.invalid", Uuid::new_v4());
|
||||
let stale_epoch = Uuid::new_v4();
|
||||
let replacement_epoch = Uuid::new_v4();
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(stale_epoch)) })
|
||||
.await
|
||||
.expect("initial capability should resolve");
|
||||
let mut writer = PutFileAuthWriter::new(
|
||||
PutFileFailureWriter { phase, status },
|
||||
format!("{endpoint}{PUT_FILE_AUTH_STREAM_PATH}"),
|
||||
Uuid::new_v4(),
|
||||
endpoint.clone(),
|
||||
stale_epoch,
|
||||
);
|
||||
let error = match (phase, trailer_write) {
|
||||
(PutFileFailurePhase::Write, false) => writer.write_all(b"body").await,
|
||||
(PutFileFailurePhase::Flush, _) => writer.flush().await,
|
||||
_ => writer.shutdown().await,
|
||||
}
|
||||
.expect_err("injected writer error must reach the caller");
|
||||
let conflict = status == reqwest::StatusCode::CONFLICT;
|
||||
assert_eq!(io_error_has_put_file_epoch_conflict(&error), conflict);
|
||||
|
||||
let probe_called = AtomicBool::new(false);
|
||||
let resolved = resolve_put_file_auth_capability(&endpoint, || async {
|
||||
probe_called.store(true, Ordering::SeqCst);
|
||||
Ok(Some(replacement_epoch))
|
||||
})
|
||||
.await
|
||||
.expect("capability should remain usable or be reprobed");
|
||||
assert_eq!(probe_called.load(Ordering::SeqCst), conflict, "phase={phase:?}, trailer={trailer_write}");
|
||||
assert_eq!(resolved, Some(if conflict { replacement_epoch } else { stale_epoch }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn late_put_file_epoch_rejection_preserves_current_rejection() {
|
||||
let endpoint = format!("http://late-epoch-conflict-{}.invalid", Uuid::new_v4());
|
||||
let old_epoch = Uuid::new_v4();
|
||||
let current_epoch = Uuid::new_v4();
|
||||
let replacement_epoch = Uuid::new_v4();
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(old_epoch)) })
|
||||
.await
|
||||
.expect("initial epoch should be cached"),
|
||||
Some(old_epoch)
|
||||
);
|
||||
reject_put_file_server_epoch(&endpoint, old_epoch);
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(current_epoch)) })
|
||||
.await
|
||||
.expect("first restart should install a new epoch"),
|
||||
Some(current_epoch)
|
||||
);
|
||||
|
||||
reject_put_file_server_epoch(&endpoint, current_epoch);
|
||||
// A writer opened before the first restart can report its 409 after
|
||||
// a newer writer has already rejected the second server incarnation.
|
||||
reject_put_file_server_epoch(&endpoint, old_epoch);
|
||||
let probe_called = AtomicBool::new(false);
|
||||
let resolved = resolve_put_file_auth_capability(&endpoint, || async {
|
||||
probe_called.store(true, Ordering::SeqCst);
|
||||
Ok(Some(replacement_epoch))
|
||||
})
|
||||
.await
|
||||
.expect("late old-epoch rejection must preserve the current rejection");
|
||||
|
||||
assert!(probe_called.load(Ordering::SeqCst), "known-rejected current epoch must be reprobed");
|
||||
assert_eq!(resolved, Some(replacement_epoch));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_epoch_rejection_is_endpoint_and_epoch_scoped() {
|
||||
let endpoint = format!("http://scoped-epoch-{}.invalid", Uuid::new_v4());
|
||||
let other_endpoint = format!("http://other-epoch-{}.invalid", Uuid::new_v4());
|
||||
let current_epoch = Uuid::new_v4();
|
||||
for endpoint in [&endpoint, &other_endpoint] {
|
||||
resolve_put_file_auth_capability(endpoint, || async { Ok(Some(current_epoch)) })
|
||||
.await
|
||||
.expect("initial epoch should resolve");
|
||||
}
|
||||
reject_put_file_server_epoch(&endpoint, Uuid::new_v4());
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { panic!("old writer must not invalidate a new epoch") })
|
||||
.await
|
||||
.expect("new epoch must remain cached"),
|
||||
Some(current_epoch)
|
||||
);
|
||||
reject_put_file_server_epoch(&endpoint, current_epoch);
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&other_endpoint, || async { panic!("another endpoint must stay cached") })
|
||||
.await
|
||||
.expect("other endpoint must remain cached"),
|
||||
Some(current_epoch)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_file_rejected_epoch_survives_failed_stale_and_downgrade_probes() {
|
||||
let endpoint = format!("http://rejected-probe-{}.invalid", Uuid::new_v4());
|
||||
let rejected_epoch = Uuid::new_v4();
|
||||
let replacement_epoch = Uuid::new_v4();
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
|
||||
.await
|
||||
.expect("initial epoch should resolve");
|
||||
reject_put_file_server_epoch(&endpoint, rejected_epoch);
|
||||
let failure = resolve_put_file_auth_capability(&endpoint, || async { Err(Error::other("injected probe failure")) })
|
||||
.await
|
||||
.expect_err("probe failure must be returned");
|
||||
assert!(failure.to_string().contains("injected probe failure"));
|
||||
let downgrade = resolve_put_file_auth_capability(&endpoint, || async { Ok(None) })
|
||||
.await
|
||||
.expect_err("rejection must not unpin authenticated v1");
|
||||
assert!(downgrade.to_string().contains("downgrade rejected"));
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(rejected_epoch)) })
|
||||
.await
|
||||
.expect("a probe racing a restart can still return the old epoch");
|
||||
assert_eq!(
|
||||
resolve_put_file_auth_capability(&endpoint, || async { Ok(Some(replacement_epoch)) })
|
||||
.await
|
||||
.expect("same-epoch probe must not clear known rejection"),
|
||||
Some(replacement_epoch)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn walk_dir_url_encodes_disk_ref() {
|
||||
let url = build_walk_dir_url(&WalkDirStreamRequest {
|
||||
|
||||
@@ -39,8 +39,8 @@ pub use http_auth::{
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof,
|
||||
verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
|
||||
|
||||
+4840
-413
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -1351,11 +1351,20 @@ pub(crate) async fn make_local_two_set_sets_with_ctx(ctx: Arc<InstanceContext>)
|
||||
pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
|
||||
ctx: Arc<InstanceContext>,
|
||||
pool_idx: usize,
|
||||
) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
make_local_two_set_sets_for_pool_with_drive_count_and_ctx(ctx, pool_idx, 2).await
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pub(crate) async fn make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
ctx: Arc<InstanceContext>,
|
||||
pool_idx: usize,
|
||||
set_drive_count: usize,
|
||||
) -> (Vec<tempfile::TempDir>, Arc<Sets>) {
|
||||
use crate::layout::endpoint::Endpoint;
|
||||
use rustfs_lock::client::local::LocalClient;
|
||||
|
||||
let format = FormatV3::new(2, 2);
|
||||
let format = FormatV3::new(2, set_drive_count);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut all_endpoints = Vec::new();
|
||||
let mut disk_sets = Vec::new();
|
||||
@@ -1363,7 +1372,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
|
||||
for set_index in 0..2 {
|
||||
let mut endpoints = Vec::new();
|
||||
let mut disks = Vec::new();
|
||||
for disk_index in 0..2 {
|
||||
for disk_index in 0..set_drive_count {
|
||||
let temp_dir = tempfile::tempdir().expect("tempdir should be created");
|
||||
let mut endpoint = Endpoint::try_from(temp_dir.path().to_str().expect("tempdir path should be utf8"))
|
||||
.expect("endpoint should parse");
|
||||
@@ -1389,7 +1398,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
|
||||
endpoints.push(endpoint);
|
||||
disks.push(Some(disk));
|
||||
}
|
||||
let lockers = (0..2)
|
||||
let lockers = (0..set_drive_count)
|
||||
.map(|_| {
|
||||
Arc::new(LocalClient::with_manager(Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
@@ -1400,7 +1409,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
|
||||
SetDisks::new_with_instance_ctx(
|
||||
"test-owner".to_string(),
|
||||
Arc::new(RwLock::new(disks)),
|
||||
2,
|
||||
set_drive_count,
|
||||
1,
|
||||
set_index,
|
||||
pool_idx,
|
||||
@@ -1420,7 +1429,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
|
||||
endpoints: PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 2,
|
||||
drives_per_set: 2,
|
||||
drives_per_set: set_drive_count,
|
||||
endpoints: Endpoints::from(all_endpoints),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
@@ -1428,7 +1437,7 @@ pub(crate) async fn make_local_two_set_sets_for_pool_with_ctx(
|
||||
format,
|
||||
parity_count: 1,
|
||||
set_count: 2,
|
||||
set_drive_count: 2,
|
||||
set_drive_count,
|
||||
default_parity_count: 1,
|
||||
distribution_algo: DistributionAlgoVersion::V1,
|
||||
exit_signal: None,
|
||||
|
||||
@@ -16,13 +16,14 @@
|
||||
|
||||
pub(crate) mod backpressure;
|
||||
|
||||
use crate::core::pools::{DecommissionCapacityOwner, decommission_capacity_mutation_id};
|
||||
use crate::error::{
|
||||
Error, Result, is_err_data_movement_overwrite, is_err_invalid_upload_id, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions, PutObjReader};
|
||||
use crate::set_disk::{SetDisks, get_lock_acquire_timeout};
|
||||
use crate::storage_api_contracts::{
|
||||
multipart::{CompletePart, MultipartOperations as _},
|
||||
multipart::CompletePart,
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{HTTPPreconditions, ObjectOperations as _},
|
||||
};
|
||||
@@ -160,6 +161,99 @@ pub fn mark_multipart_upload_completed(flag: &Arc<AtomicBool>) {
|
||||
flag.store(false, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct DataMovementMultipartAbortBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) struct DataMovementMultipartAbortBarrier {
|
||||
state: Arc<DataMovementMultipartAbortBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static DATA_MOVEMENT_MULTIPART_ABORT_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<DataMovementMultipartAbortBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl DataMovementMultipartAbortBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
let state = Arc::new(DataMovementMultipartAbortBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("data movement multipart abort barrier mutex should not poison");
|
||||
assert!(slot.is_none(), "data movement multipart abort barrier must be unique");
|
||||
*slot = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_until_paused(&self) {
|
||||
tokio::time::timeout(StdDuration::from_secs(30), self.state.arrived.notified())
|
||||
.await
|
||||
.expect("data movement multipart failure should reach abort cleanup");
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for DataMovementMultipartAbortBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.release.notify_one();
|
||||
let mut slot = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("data movement multipart abort barrier mutex should not poison");
|
||||
if slot.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) {
|
||||
*slot = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_data_movement_multipart_before_abort(bucket: &str, object: &str) {
|
||||
let barrier = DATA_MOVEMENT_MULTIPART_ABORT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("data movement multipart abort barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
fn data_movement_abort_opts(
|
||||
src_pool_idx: usize,
|
||||
expected_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
lock_lost_signal: Option<&Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
) -> ObjectOptions {
|
||||
let mut opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx,
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut opts);
|
||||
}
|
||||
if let Some(signal) = lock_lost_signal {
|
||||
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
opts
|
||||
}
|
||||
|
||||
fn insert_data_movement_checksum(user_defined: &mut HashMap<String, String>, object_info: &ObjectInfo) {
|
||||
rustfs_utils::http::remove_header_map(user_defined, rustfs_utils::http::SUFFIX_REPLICATION_SSEC_CRC);
|
||||
if let Some(checksum) = object_info.checksum.as_ref().filter(|checksum| !checksum.is_empty()) {
|
||||
@@ -192,7 +286,7 @@ fn data_movement_new_multipart_opts(object_info: &ObjectInfo, src_pool_idx: usiz
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
src_pool_idx,
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -363,7 +457,7 @@ fn data_movement_complete_multipart_opts(
|
||||
preserve_etag: object_info.etag.clone(),
|
||||
user_defined,
|
||||
src_pool_idx,
|
||||
..Default::default()
|
||||
..ObjectOptions::with_capacity_expected_data_bytes(usize::try_from(object_info.size).ok())
|
||||
})
|
||||
}
|
||||
|
||||
@@ -533,6 +627,7 @@ fn schedule_data_movement_multipart_abort_cleanup(
|
||||
bucket: String,
|
||||
object: String,
|
||||
upload_id: String,
|
||||
opts: ObjectOptions,
|
||||
op_label: &str,
|
||||
) {
|
||||
let op_label = op_label.to_string();
|
||||
@@ -540,23 +635,32 @@ fn schedule_data_movement_multipart_abort_cleanup(
|
||||
for attempt in 1..=DATA_MOVEMENT_MULTIPART_ABORT_RETRY_ATTEMPTS {
|
||||
tokio::time::sleep(StdDuration::from_secs(DATA_MOVEMENT_MULTIPART_ABORT_RETRY_DELAY_SECS)).await;
|
||||
|
||||
let Some(pool) = store.pools.get(target_pool_idx).cloned() else {
|
||||
if store.pools.get(target_pool_idx).is_none() {
|
||||
error!(
|
||||
"{op_label}: background abort_multipart_upload cleanup skipped for {bucket}/{object} upload {upload_id}: target pool {target_pool_idx} is out of range"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
let mut cleanup_opts = opts.clone();
|
||||
let _multipart_mutation_fence = match DecommissionCapacityOwner::from_options(&cleanup_opts) {
|
||||
Some(owner) => match store.acquire_decommission_multipart_mutation_fence(owner).await {
|
||||
Ok(fence) => {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
Some(fence)
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
"{op_label}: background abort_multipart_upload cleanup could not fence {bucket}/{object} upload {upload_id} on attempt {attempt}: {err:?}"
|
||||
);
|
||||
continue;
|
||||
}
|
||||
},
|
||||
None => None,
|
||||
};
|
||||
|
||||
match pool
|
||||
.abort_multipart_upload(
|
||||
&bucket,
|
||||
&object,
|
||||
&upload_id,
|
||||
&ObjectOptions {
|
||||
data_movement: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
match store
|
||||
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object, &upload_id, &cleanup_opts)
|
||||
.await
|
||||
{
|
||||
Ok(()) => {
|
||||
@@ -1334,27 +1438,43 @@ fn resolve_data_movement_overwrite_resume_result_for(
|
||||
Ok(matches!(err, Error::PreconditionFailed) && is_superseding_unversioned_data_movement_object(source, &target))
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct DataMovementOverwriteCapacity {
|
||||
owner: Option<DecommissionCapacityOwner>,
|
||||
expected_data_bytes: Option<usize>,
|
||||
}
|
||||
|
||||
async fn should_treat_data_movement_overwrite_as_complete(
|
||||
store: &ECStore,
|
||||
src_pool_idx: usize,
|
||||
target_pool_idx: usize,
|
||||
pool_indices: (usize, usize),
|
||||
bucket: &str,
|
||||
object_info: &ObjectInfo,
|
||||
err: &Error,
|
||||
compare_part_checksums: bool,
|
||||
capacity: DataMovementOverwriteCapacity,
|
||||
) -> Result<bool> {
|
||||
if !should_check_data_movement_overwrite_resume(err) {
|
||||
return Ok(false);
|
||||
}
|
||||
let (src_pool_idx, target_pool_idx) = pool_indices;
|
||||
|
||||
resolve_data_movement_overwrite_resume_result_for(
|
||||
let equivalent = resolve_data_movement_overwrite_resume_result_for(
|
||||
err,
|
||||
find_data_movement_target_info(store, target_pool_idx, bucket, object_info).await,
|
||||
object_info,
|
||||
src_pool_idx,
|
||||
target_pool_idx,
|
||||
compare_part_checksums,
|
||||
)
|
||||
)?;
|
||||
if equivalent && let Some(owner) = capacity.owner {
|
||||
let expected_data_bytes = capacity
|
||||
.expected_data_bytes
|
||||
.ok_or_else(|| Error::other("equivalent data-movement target cannot reconcile unknown committed data size"))?;
|
||||
store
|
||||
.reconcile_decommission_capacity_after_equivalent_target(owner, target_pool_idx, expected_data_bytes)
|
||||
.await?;
|
||||
}
|
||||
Ok(equivalent)
|
||||
}
|
||||
|
||||
fn data_movement_part_stage_error(
|
||||
@@ -1395,6 +1515,7 @@ pub(crate) async fn migrate_decommission_object(
|
||||
rd: GetObjectReader,
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
) -> Result<()> {
|
||||
let source = rd.object_info.clone();
|
||||
let _mutation_fence = store
|
||||
@@ -1415,6 +1536,7 @@ pub(crate) async fn migrate_decommission_object(
|
||||
source_bucket_incarnation_id,
|
||||
op_label,
|
||||
None,
|
||||
capacity_owner,
|
||||
Some(&_mutation_fence),
|
||||
)
|
||||
.await
|
||||
@@ -1451,6 +1573,7 @@ pub(crate) async fn migrate_object_with_lock_lost_signal(
|
||||
op_label,
|
||||
lock_lost_signal,
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -1464,22 +1587,102 @@ async fn migrate_object_inner(
|
||||
source_bucket_incarnation_id: Option<uuid::Uuid>,
|
||||
op_label: &str,
|
||||
lock_lost_signal: Option<Arc<rustfs_lock::distributed_lock::LockLostSignal>>,
|
||||
capacity_owner: Option<DecommissionCapacityOwner>,
|
||||
mutation_fence: Option<&ObjectLockDiagGuard>,
|
||||
) -> Result<()> {
|
||||
let object_info = rd.object_info.clone();
|
||||
let capacity_owner = capacity_owner.map(|owner| {
|
||||
let version_id = object_info.version_id.map(|version_id| version_id.to_string());
|
||||
let mutation_id = owner.mutation_id.unwrap_or_else(|| {
|
||||
decommission_capacity_mutation_id(
|
||||
owner,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
version_id.as_deref(),
|
||||
object_info.delete_marker,
|
||||
object_info.mod_time,
|
||||
)
|
||||
});
|
||||
owner.with_mutation_id(mutation_id)
|
||||
});
|
||||
let has_part_checksums = object_info
|
||||
.parts
|
||||
.iter()
|
||||
.any(|part| part.checksums.as_ref().is_some_and(|checksums| !checksums.is_empty()));
|
||||
|
||||
let preserve_part_checksums = data_movement_part_checksum_writer_enabled();
|
||||
let capacity_expected_data_bytes = usize::try_from(object_info.size).ok();
|
||||
|
||||
if should_use_multipart_data_movement(&object_info, has_part_checksums) {
|
||||
// The decommission object fence already covers the source/target
|
||||
// namespace for this migration. Acquiring the synthetic multipart
|
||||
// fence while holding that read lock deadlocks local lock domains;
|
||||
// retain the extra fence only for callers without the outer fence.
|
||||
let multipart_mutation_fence = match (capacity_owner, mutation_fence.is_some()) {
|
||||
(Some(owner), false) => Some(store.acquire_decommission_multipart_mutation_fence(owner).await?),
|
||||
_ => None,
|
||||
};
|
||||
let mut new_multipart_opts = data_movement_new_multipart_opts(&object_info, pool_idx);
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut new_multipart_opts);
|
||||
}
|
||||
new_multipart_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
new_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut new_multipart_opts);
|
||||
}
|
||||
if let Some(owner) = capacity_owner {
|
||||
let existing_target_pool_idx = store
|
||||
.select_data_movement_pool_idx(&bucket, &object_info.name, -1, &new_multipart_opts, false)
|
||||
.await?;
|
||||
if existing_target_pool_idx != pool_idx
|
||||
&& let Some(target) =
|
||||
find_data_movement_target_info(store.as_ref(), existing_target_pool_idx, &bucket, &object_info).await?
|
||||
&& is_equivalent_data_movement_object_identity(&object_info, &target, true, preserve_part_checksums)
|
||||
{
|
||||
let expected_data_bytes = capacity_expected_data_bytes
|
||||
.ok_or_else(|| Error::other("equivalent multipart target cannot reconcile unknown committed data size"))?;
|
||||
store
|
||||
.reconcile_decommission_capacity_after_equivalent_target(owner, existing_target_pool_idx, expected_data_bytes)
|
||||
.await?;
|
||||
info!(
|
||||
"{op_label}: multipart upload restart reconciled equivalent target for {}/{}",
|
||||
bucket.as_str(),
|
||||
object_info.name.as_str()
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
let mut cleanup_opts =
|
||||
data_movement_abort_opts(pool_idx, source_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut cleanup_opts);
|
||||
}
|
||||
for target_pool_idx in store.decommission_capacity_cleanup_target_indices(owner).await? {
|
||||
store
|
||||
.reconcile_multipart_uploads_for_data_movement(
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&data_movement_upload_identity(&object_info),
|
||||
&cleanup_opts,
|
||||
)
|
||||
.await
|
||||
.map_err(|err| {
|
||||
data_movement_stage_error(
|
||||
op_label,
|
||||
"reconcile_multipart_upload",
|
||||
bucket.as_str(),
|
||||
object_info.name.as_str(),
|
||||
err,
|
||||
)
|
||||
})?;
|
||||
}
|
||||
}
|
||||
let (res, target_pool_idx, expected_bucket_incarnation_id) = match store
|
||||
.handle_new_multipart_upload_with_pool_idx(&bucket, &object_info.name, &new_multipart_opts, mutation_fence)
|
||||
.await
|
||||
@@ -1532,9 +1735,15 @@ async fn migrate_object_inner(
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut part_opts);
|
||||
}
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
part_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut part_opts);
|
||||
}
|
||||
let pi = match store
|
||||
.put_object_part_for_data_movement(
|
||||
target_pool_idx,
|
||||
@@ -1578,10 +1787,16 @@ async fn migrate_object_inner(
|
||||
err,
|
||||
)
|
||||
})?;
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut complete_multipart_opts);
|
||||
}
|
||||
complete_multipart_opts.expected_bucket_incarnation_id = expected_bucket_incarnation_id;
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
complete_multipart_opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut complete_multipart_opts);
|
||||
}
|
||||
if let Err(err) = store
|
||||
.clone()
|
||||
.complete_multipart_upload_for_data_movement(
|
||||
@@ -1596,12 +1811,15 @@ async fn migrate_object_inner(
|
||||
{
|
||||
if should_treat_data_movement_overwrite_as_complete(
|
||||
store.as_ref(),
|
||||
pool_idx,
|
||||
target_pool_idx,
|
||||
(pool_idx, target_pool_idx),
|
||||
bucket.as_str(),
|
||||
&object_info,
|
||||
&err,
|
||||
preserve_part_checksums,
|
||||
DataMovementOverwriteCapacity {
|
||||
owner: capacity_owner,
|
||||
expected_data_bytes: capacity_expected_data_bytes,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1629,31 +1847,37 @@ async fn migrate_object_inner(
|
||||
.await;
|
||||
|
||||
if multipart_result.is_ok() && should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
let mut abort_opts =
|
||||
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
let abort_result = store
|
||||
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
|
||||
let mut opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: pool_idx,
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
opts
|
||||
})
|
||||
.abort_multipart_upload_for_data_movement(
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&res.upload_id,
|
||||
&abort_opts,
|
||||
)
|
||||
.await;
|
||||
match abort_result {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(abort_err) if is_err_invalid_upload_id(&abort_err) => {
|
||||
if should_treat_data_movement_overwrite_as_complete(
|
||||
store.as_ref(),
|
||||
pool_idx,
|
||||
target_pool_idx,
|
||||
(pool_idx, target_pool_idx),
|
||||
bucket.as_str(),
|
||||
&object_info,
|
||||
&abort_err,
|
||||
preserve_part_checksums,
|
||||
DataMovementOverwriteCapacity {
|
||||
owner: capacity_owner,
|
||||
expected_data_bytes: capacity_expected_data_bytes,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
@@ -1683,6 +1907,7 @@ async fn migrate_object_inner(
|
||||
bucket.clone(),
|
||||
object_info.name.clone(),
|
||||
res.upload_id.clone(),
|
||||
abort_opts,
|
||||
op_label,
|
||||
);
|
||||
return Err(data_movement_stage_error(
|
||||
@@ -1698,19 +1923,24 @@ async fn migrate_object_inner(
|
||||
|
||||
if let Err(primary_err) = multipart_result {
|
||||
if should_abort_multipart_upload(&abort_multipart_flag) {
|
||||
#[cfg(test)]
|
||||
pause_data_movement_multipart_before_abort(&bucket, &object_info.name).await;
|
||||
let mut abort_opts =
|
||||
data_movement_abort_opts(pool_idx, expected_bucket_incarnation_id, lock_lost_signal.as_ref(), capacity_owner);
|
||||
if let Some(fence) = mutation_fence {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
if let Some(fence) = multipart_mutation_fence.as_ref() {
|
||||
fence.add_namespace_lock_fence(&mut abort_opts);
|
||||
}
|
||||
return match store
|
||||
.abort_multipart_upload_for_data_movement(target_pool_idx, &bucket, &object_info.name, &res.upload_id, &{
|
||||
let mut opts = ObjectOptions {
|
||||
data_movement: true,
|
||||
src_pool_idx: pool_idx,
|
||||
expected_bucket_incarnation_id,
|
||||
..Default::default()
|
||||
};
|
||||
if let Some(signal) = lock_lost_signal.as_ref() {
|
||||
opts.add_namespace_lock_lost_signal(Arc::clone(signal));
|
||||
}
|
||||
opts
|
||||
})
|
||||
.abort_multipart_upload_for_data_movement(
|
||||
target_pool_idx,
|
||||
&bucket,
|
||||
&object_info.name,
|
||||
&res.upload_id,
|
||||
&abort_opts,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(()) => Err(primary_err),
|
||||
@@ -1722,6 +1952,7 @@ async fn migrate_object_inner(
|
||||
bucket.clone(),
|
||||
object_info.name.clone(),
|
||||
res.upload_id.clone(),
|
||||
abort_opts,
|
||||
op_label,
|
||||
);
|
||||
Err(resolve_data_movement_abort_result(
|
||||
@@ -1744,6 +1975,9 @@ async fn migrate_object_inner(
|
||||
let mut data = data_movement_put_object_reader(bucket.as_str(), &object_info, rd, op_label)?;
|
||||
|
||||
let mut put_opts = data_movement_put_object_opts(&object_info, pool_idx);
|
||||
if let Some(capacity_owner) = capacity_owner {
|
||||
capacity_owner.apply_to(&mut put_opts);
|
||||
}
|
||||
put_opts.expected_bucket_incarnation_id = source_bucket_incarnation_id;
|
||||
if let Some(signal) = lock_lost_signal {
|
||||
put_opts.add_namespace_lock_lost_signal(signal);
|
||||
@@ -1755,12 +1989,15 @@ async fn migrate_object_inner(
|
||||
if let Err(err) = put_result {
|
||||
if should_treat_data_movement_overwrite_as_complete(
|
||||
store.as_ref(),
|
||||
pool_idx,
|
||||
target_pool_idx,
|
||||
(pool_idx, target_pool_idx),
|
||||
bucket.as_str(),
|
||||
&object_info,
|
||||
&err,
|
||||
preserve_part_checksums,
|
||||
DataMovementOverwriteCapacity {
|
||||
owner: capacity_owner,
|
||||
expected_data_bytes: capacity_expected_data_bytes,
|
||||
},
|
||||
)
|
||||
.await?
|
||||
{
|
||||
|
||||
@@ -109,7 +109,10 @@ static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
/// strictly tighter than beta.11 (usage treated as 0) and strictly more
|
||||
/// available than a blanket 503. The fallback applies to any window without
|
||||
/// authoritative usage, not only pre-v2 upgrades; the values always come from
|
||||
/// the last persisted scanner output. Loads go through the TTL-bounded
|
||||
/// the last persisted scanner output — pre-discard sizes of the
|
||||
/// authoritative snapshot first, backfilled per bucket from the observed
|
||||
/// (nonconverged) snapshot for buckets no authoritative cycle has covered
|
||||
/// yet (issue #6852). Loads go through the TTL-bounded
|
||||
/// snapshot cache, so the quota path adds at most one backend read per
|
||||
/// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent
|
||||
/// from every persisted snapshot — those still fail closed.
|
||||
@@ -168,7 +171,7 @@ fn fresh_cached_data_usage_snapshot(
|
||||
|
||||
fn cache_data_usage_snapshot_result(
|
||||
cache: &mut Option<CachedDataUsageSnapshot>,
|
||||
result: Result<(DataUsageInfo, HashMap<String, u64>), Error>,
|
||||
result: Result<LoadedUsageBaseline, Error>,
|
||||
loaded_at: tokio::time::Instant,
|
||||
refresh_generation: u64,
|
||||
current_generation: u64,
|
||||
@@ -178,7 +181,19 @@ fn cache_data_usage_snapshot_result(
|
||||
}
|
||||
|
||||
Some(match result {
|
||||
Ok((info, degraded_baseline)) => {
|
||||
Ok(LoadedUsageBaseline {
|
||||
info,
|
||||
mut degraded_baseline,
|
||||
observed_unavailable,
|
||||
}) => {
|
||||
// A flaky observed read must not shrink quota coverage for a TTL
|
||||
// window: carry the previous refresh's baseline entries forward,
|
||||
// letting the fresh (authoritative) values win where they exist.
|
||||
if observed_unavailable && let Some(previous) = cache.as_ref() {
|
||||
for (bucket, size) in &previous.degraded_baseline {
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(*size);
|
||||
}
|
||||
}
|
||||
*cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(info.clone()),
|
||||
loaded_at,
|
||||
@@ -1113,24 +1128,78 @@ async fn load_data_usage_snapshot(store: Arc<ECStore>) -> Result<(DataUsageInfo,
|
||||
/// Load data usage info from backend storage
|
||||
#[instrument(skip(store))]
|
||||
pub async fn load_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
Ok(load_data_usage_from_backend_with_baseline(store).await?.0)
|
||||
Ok(load_data_usage_from_backend_with_baseline(store).await?.info)
|
||||
}
|
||||
|
||||
/// One refresh of the persisted usage snapshot plus the quota-admission
|
||||
/// baseline derived from it.
|
||||
struct LoadedUsageBaseline {
|
||||
info: DataUsageInfo,
|
||||
degraded_baseline: HashMap<String, u64>,
|
||||
/// True when the observed snapshot could not be read (a transport error,
|
||||
/// not absence): the cached loader then carries the previous refresh's
|
||||
/// baseline entries forward instead of shrinking quota coverage for a
|
||||
/// whole TTL window over one flaky read.
|
||||
observed_unavailable: bool,
|
||||
}
|
||||
|
||||
/// Like [`load_data_usage_from_backend`], but also returns the pre-discard
|
||||
/// per-bucket sizes so the cached loader can retain them as the degraded
|
||||
/// quota-admission baseline (issue #5716).
|
||||
async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<(DataUsageInfo, HashMap<String, u64>), Error> {
|
||||
let (data_usage_info, source) = load_data_usage_snapshot(store).await?;
|
||||
Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await)
|
||||
async fn load_data_usage_from_backend_with_baseline(store: Arc<ECStore>) -> Result<LoadedUsageBaseline, Error> {
|
||||
let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
// The observed-newness gate below compares against the snapshot as
|
||||
// persisted, before normalization demotes or discards anything.
|
||||
let authoritative_as_persisted = loaded_snapshot.clone();
|
||||
let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await;
|
||||
|
||||
// A bucket without a converged scanner cycle behind it — a freshly joined
|
||||
// replica whose every cycle is superseded by the sustained replication
|
||||
// write stream, or a bucket created after the last converged cycle on a
|
||||
// busy site (#6852) — has no authoritative size, and quota admission
|
||||
// fails its writes closed indefinitely. The observed (nonconverged)
|
||||
// snapshot those superseded cycles still publish is the only grounded
|
||||
// usage in that window, so it backfills buckets the loaded baseline does
|
||||
// not cover; a value already in the baseline always wins. The newness
|
||||
// gate ties the observation to this exact authoritative snapshot, so a
|
||||
// stale observed object left behind by an earlier incarnation (e.g. a
|
||||
// deleted and recreated bucket) cannot inject ghost usage. Loads sit
|
||||
// behind the same TTL cache as the snapshot itself, so this adds at most
|
||||
// one backend read per TTL window.
|
||||
let mut observed_unavailable = false;
|
||||
match load_observed_data_usage_snapshot(store).await {
|
||||
Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => {
|
||||
backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed);
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(_) => observed_unavailable = true,
|
||||
}
|
||||
|
||||
Ok(LoadedUsageBaseline {
|
||||
info,
|
||||
degraded_baseline,
|
||||
observed_unavailable,
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUsageInfo> {
|
||||
/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without
|
||||
/// overriding any bucket the authoritative baseline already covers.
|
||||
fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap<String, u64>, observed: &DataUsageInfo) {
|
||||
for (bucket, usage) in &observed.buckets_usage {
|
||||
degraded_baseline.entry(bucket.clone()).or_insert(usage.size);
|
||||
}
|
||||
}
|
||||
|
||||
/// `Ok(None)` means the observed snapshot is absent or invalid (a settled
|
||||
/// answer); `Err` means it could not be read at all, so the caller may keep
|
||||
/// using what it learned from a previous read.
|
||||
async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Result<Option<DataUsageInfo>, Error> {
|
||||
let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await {
|
||||
Ok(data) => data,
|
||||
Err(Error::ConfigNotFound) => return None,
|
||||
Err(Error::ConfigNotFound) => return Ok(None),
|
||||
Err(err) => {
|
||||
record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
return None;
|
||||
return Err(err);
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1139,7 +1208,7 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
|
||||
if info.usage_snapshot_converged == Some(false)
|
||||
&& (info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()) =>
|
||||
{
|
||||
Some(info)
|
||||
Ok(Some(info))
|
||||
}
|
||||
Ok(_) => {
|
||||
error!(
|
||||
@@ -1150,11 +1219,11 @@ async fn load_observed_data_usage_snapshot(store: Arc<ECStore>) -> Option<DataUs
|
||||
object = %DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(),
|
||||
"observed data usage snapshot was not a structurally complete nonconverged view"
|
||||
);
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
Err(err) => {
|
||||
record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err);
|
||||
None
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1212,7 +1281,9 @@ fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observe
|
||||
|
||||
async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
|
||||
let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?;
|
||||
let observed = load_observed_data_usage_snapshot(store).await;
|
||||
// For the one-shot admin view a failed observed read degrades to "no
|
||||
// observation", same as before the read was fallible.
|
||||
let observed = load_observed_data_usage_snapshot(store).await.ok().flatten();
|
||||
let (selected, selected_is_current_format) =
|
||||
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
|
||||
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
|
||||
@@ -1375,7 +1446,11 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
let refresh_generation = admin_data_usage_snapshot_generation();
|
||||
let result = load_admin_data_usage_from_backend(store.clone())
|
||||
.await
|
||||
.map(|info| (info, HashMap::new()));
|
||||
.map(|info| LoadedUsageBaseline {
|
||||
info,
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
});
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let mut cache = admin_data_usage_snapshot_cache().write().await;
|
||||
if let Some(result) = cache_data_usage_snapshot_result(
|
||||
@@ -2526,6 +2601,37 @@ mod tests {
|
||||
use std::sync::Arc;
|
||||
use tokio::{io::AsyncReadExt, sync::Mutex};
|
||||
|
||||
#[test]
|
||||
fn observed_snapshot_only_backfills_baseline_gaps() {
|
||||
let mut baseline = HashMap::from([("covered".to_string(), 111_u64)]);
|
||||
let observed = DataUsageInfo {
|
||||
buckets_usage: HashMap::from([
|
||||
(
|
||||
"covered".to_string(),
|
||||
BucketUsageInfo {
|
||||
size: 999,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
(
|
||||
"replica-only".to_string(),
|
||||
BucketUsageInfo {
|
||||
size: 42,
|
||||
..Default::default()
|
||||
},
|
||||
),
|
||||
]),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
backfill_degraded_baseline_from_observed(&mut baseline, &observed);
|
||||
|
||||
// The authoritative value must win; only the uncovered bucket (#6852:
|
||||
// a replica that never landed a converged cycle) is filled in.
|
||||
assert_eq!(baseline.get("covered"), Some(&111));
|
||||
assert_eq!(baseline.get("replica-only"), Some(&42));
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct UsageCasState {
|
||||
object: Option<(Vec<u8>, u64)>,
|
||||
@@ -2858,6 +2964,7 @@ mod tests {
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: TokioMutex::new(()),
|
||||
pool_meta_save_gate: TokioMutex::default(),
|
||||
decommission_capacity_entry_gate: TokioMutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
@@ -3479,7 +3586,11 @@ mod tests {
|
||||
|
||||
let first = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok((expected, HashMap::new())),
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: expected,
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
@@ -3494,6 +3605,38 @@ mod tests {
|
||||
assert_snapshot_bucket(&cached, "bucket");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn unavailable_observed_read_keeps_previous_baseline_coverage() {
|
||||
let loaded_at = tokio::time::Instant::now();
|
||||
let refresh_generation = data_usage_snapshot_generation();
|
||||
let mut cache = Some(CachedDataUsageSnapshot {
|
||||
info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)),
|
||||
loaded_at,
|
||||
degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]),
|
||||
});
|
||||
|
||||
cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH),
|
||||
degraded_baseline: HashMap::from([("covered".to_string(), 2_u64)]),
|
||||
observed_unavailable: true,
|
||||
}),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
)
|
||||
.expect("an uninterrupted refresh should populate the cache")
|
||||
.expect("successful load must be returned");
|
||||
|
||||
let baseline = &cache.as_ref().expect("cache must be populated").degraded_baseline;
|
||||
// The bucket only the (now unreadable) observed snapshot covered must
|
||||
// survive the refresh; the freshly loaded value wins where it exists.
|
||||
assert_eq!(baseline.get("observed-only"), Some(&7));
|
||||
assert_eq!(baseline.get("covered"), Some(&2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() {
|
||||
@@ -3508,7 +3651,11 @@ mod tests {
|
||||
|
||||
let stale_result = cache_data_usage_snapshot_result(
|
||||
&mut cache,
|
||||
Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())),
|
||||
Ok(LoadedUsageBaseline {
|
||||
info: data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH),
|
||||
degraded_baseline: HashMap::new(),
|
||||
observed_unavailable: false,
|
||||
}),
|
||||
loaded_at,
|
||||
refresh_generation,
|
||||
data_usage_snapshot_generation(),
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_io_metrics::internode_metrics::INTERNODE_OPERATION_PUT_FILE_STREAM;
|
||||
use rustfs_rio::{InternodeHttpError, InternodeHttpErrorKind};
|
||||
use std::error::Error as StdError;
|
||||
use std::hash::{Hash, Hasher};
|
||||
@@ -229,6 +230,19 @@ fn classify_internode_missing_error(error: &InternodeHttpError) -> Option<DiskEr
|
||||
None
|
||||
}
|
||||
|
||||
fn internode_write_error_is_retryable(error: &InternodeHttpError) -> bool {
|
||||
error.kind().is_retryable()
|
||||
|| (matches!(error.kind(), InternodeHttpErrorKind::HttpStatus(status) if status.as_u16() == 409)
|
||||
&& error.context().operation() == Some(INTERNODE_OPERATION_PUT_FILE_STREAM))
|
||||
}
|
||||
|
||||
fn io_error_contains_retryable_internode_write(error: &io::Error) -> bool {
|
||||
error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.is_some_and(internode_write_error_is_retryable)
|
||||
}
|
||||
|
||||
/// Wrap a terminal shard-read failure without changing its typed
|
||||
/// classification. Timeout-like disk errors retain `TimedOut`; other errors
|
||||
/// retain their inner I/O kind or use `Other` when no more specific kind exists.
|
||||
@@ -336,10 +350,7 @@ impl DiskError {
|
||||
|
||||
pub fn is_retryable_internode_write_failure(&self) -> bool {
|
||||
match self {
|
||||
DiskError::Io(io_error) => io_error
|
||||
.get_ref()
|
||||
.and_then(|source| source.downcast_ref::<InternodeHttpError>())
|
||||
.is_some_and(|err| err.kind().is_retryable()),
|
||||
DiskError::Io(io_error) => io_error_contains_retryable_internode_write(io_error),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
@@ -1240,6 +1251,68 @@ mod tests {
|
||||
assert!(!DiskError::FileNotFound.is_internode_http_status(429));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_file_server_epoch_conflict_is_retryable_write_failure() {
|
||||
let conflict = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::CONFLICT),
|
||||
));
|
||||
let bad_request = DiskError::from(rustfs_rio::new_test_internode_http_io_error(
|
||||
rustfs_rio::InternodeHttpErrorKind::HttpStatus(http::StatusCode::BAD_REQUEST),
|
||||
));
|
||||
|
||||
assert!(conflict.is_retryable_internode_write_failure());
|
||||
assert!(!bad_request.is_retryable_internode_write_failure());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn read_stream_conflict_is_not_a_retryable_put_file_failure() {
|
||||
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
||||
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
||||
.await
|
||||
.expect("bind isolated HTTP fixture");
|
||||
let address = listener.local_addr().expect("fixture address");
|
||||
let server = tokio::spawn(async move {
|
||||
let (mut stream, _) = listener.accept().await.expect("accept read request");
|
||||
let mut request = [0_u8; 4096];
|
||||
let mut read = 0;
|
||||
loop {
|
||||
let count = stream.read(&mut request[read..]).await.expect("read HTTP request");
|
||||
assert!(count > 0, "request ended before its complete headers");
|
||||
read += count;
|
||||
if request[..read].windows(4).any(|bytes| bytes == b"\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
assert!(read < request.len(), "fixture request headers exceed their budget");
|
||||
}
|
||||
stream
|
||||
.write_all(b"HTTP/1.1 409 Conflict\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
|
||||
.await
|
||||
.expect("send typed conflict response");
|
||||
});
|
||||
let error = match rustfs_rio::HttpReader::new(
|
||||
format!("http://{address}/rustfs/rpc/read_file_stream"),
|
||||
http::Method::GET,
|
||||
http::HeaderMap::new(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(_) => panic!("HTTP 409 must fail the read"),
|
||||
Err(error) => DiskError::from(error),
|
||||
};
|
||||
server.await.expect("fixture task should complete");
|
||||
assert!(error.is_internode_http_status(409));
|
||||
assert!(
|
||||
!error.is_retryable_internode_write_failure(),
|
||||
"read-operation 409 must not trigger put-file retry"
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("isolated read-conflict test must finish within its budget");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_internode_missing_errors_preserve_disk_error_types() {
|
||||
let file_missing = DiskError::from(rustfs_rio::new_test_remote_file_not_found_http_io_error());
|
||||
|
||||
@@ -16,10 +16,143 @@ use rustfs_filemeta::{MetacacheReader, MetacacheWriter};
|
||||
use std::io::Cursor;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use tokio::fs;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
/// Test-only lock client whose refresh path can be rejected independently of
|
||||
/// every other lock operation. The observed event is awaitable so lock-loss
|
||||
/// tests do not depend on sleeps or scheduler timing.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct RefreshLossLockClient {
|
||||
inner: rustfs_lock::LocalClient,
|
||||
reject_refresh: AtomicBool,
|
||||
rejected_refresh: AtomicBool,
|
||||
rejected_refresh_notify: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
impl RefreshLossLockClient {
|
||||
pub(crate) fn with_manager(manager: Arc<rustfs_lock::GlobalLockManager>) -> Self {
|
||||
Self {
|
||||
inner: rustfs_lock::LocalClient::with_manager(manager),
|
||||
reject_refresh: AtomicBool::new(false),
|
||||
rejected_refresh: AtomicBool::new(false),
|
||||
rejected_refresh_notify: tokio::sync::Notify::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn reject_refreshes(&self) {
|
||||
self.reject_refresh.store(true, Ordering::Release);
|
||||
}
|
||||
|
||||
pub(crate) fn refreshes_rejected(&self) -> bool {
|
||||
self.rejected_refresh.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
pub(crate) async fn wait_for_rejected_refresh(
|
||||
&self,
|
||||
timeout: std::time::Duration,
|
||||
) -> std::result::Result<(), tokio::time::error::Elapsed> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
loop {
|
||||
let notified = self.rejected_refresh_notify.notified();
|
||||
if self.refreshes_rejected() {
|
||||
return;
|
||||
}
|
||||
notified.await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl rustfs_lock::LockClient for RefreshLossLockClient {
|
||||
async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result<rustfs_lock::LockResponse> {
|
||||
rustfs_lock::LockClient::acquire_lock(&self.inner, request).await
|
||||
}
|
||||
|
||||
async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
rustfs_lock::LockClient::release(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
if self.reject_refresh.load(Ordering::Acquire) {
|
||||
self.rejected_refresh.store(true, Ordering::Release);
|
||||
self.rejected_refresh_notify.notify_waiters();
|
||||
return Ok(false);
|
||||
}
|
||||
rustfs_lock::LockClient::refresh(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<bool> {
|
||||
rustfs_lock::LockClient::force_release(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result<Option<rustfs_lock::LockInfo>> {
|
||||
rustfs_lock::LockClient::check_status(&self.inner, lock_id).await
|
||||
}
|
||||
|
||||
async fn list_lock_leases(&self) -> Vec<rustfs_lock::LockLeaseInfo> {
|
||||
rustfs_lock::LockClient::list_lock_leases(&self.inner).await
|
||||
}
|
||||
|
||||
async fn get_stats(&self) -> rustfs_lock::Result<rustfs_lock::LockStats> {
|
||||
rustfs_lock::LockClient::get_stats(&self.inner).await
|
||||
}
|
||||
|
||||
async fn close(&self) -> rustfs_lock::Result<()> {
|
||||
rustfs_lock::LockClient::close(&self.inner).await
|
||||
}
|
||||
|
||||
async fn is_online(&self) -> bool {
|
||||
rustfs_lock::LockClient::is_online(&self.inner).await
|
||||
}
|
||||
|
||||
async fn is_local(&self) -> bool {
|
||||
rustfs_lock::LockClient::is_local(&self.inner).await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters() {
|
||||
let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new(
|
||||
rustfs_lock::FastObjectLockManager::new(),
|
||||
)));
|
||||
let client = RefreshLossLockClient::with_manager(manager);
|
||||
let resource = rustfs_lock::ObjectKey::new("bucket", "object");
|
||||
let response = rustfs_lock::LockClient::acquire_lock(
|
||||
&client,
|
||||
&rustfs_lock::LockRequest::new(resource, rustfs_lock::LockType::Shared, "refresh-loss-harness"),
|
||||
)
|
||||
.await
|
||||
.expect("acquire should reach the inner local client");
|
||||
let lock_id = response.lock_info.expect("the inner local client should acquire the lock").id;
|
||||
assert_eq!(
|
||||
rustfs_lock::LockClient::list_lock_leases(&client).await.len(),
|
||||
1,
|
||||
"lease diagnostics must remain transparent through the refresh wrapper"
|
||||
);
|
||||
|
||||
client.reject_refreshes();
|
||||
assert!(
|
||||
!rustfs_lock::LockClient::refresh(&client, &lock_id)
|
||||
.await
|
||||
.expect("refresh should return a response")
|
||||
);
|
||||
client
|
||||
.wait_for_rejected_refresh(std::time::Duration::from_millis(50))
|
||||
.await
|
||||
.expect("a waiter registered after rejection must still observe the event");
|
||||
assert!(client.refreshes_rejected());
|
||||
assert!(
|
||||
rustfs_lock::LockClient::release(&client, &lock_id)
|
||||
.await
|
||||
.expect("release should reach the inner local client")
|
||||
);
|
||||
}
|
||||
|
||||
/// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep
|
||||
/// them alive for the test's duration and the directories are removed on drop.
|
||||
pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec<tempfile::TempDir>, Arc<SetDisks>) {
|
||||
|
||||
@@ -46,6 +46,7 @@ type ShardReadFuture<'a> = Pin<Box<dyn Future<Output = (usize, ShardReadCost, Re
|
||||
type OwnedShardReadFuture<'a, R> =
|
||||
Pin<Box<dyn Future<Output = (usize, ShardReadCost, Result<Vec<u8>, Error>, Option<BitrotReader<R>>, bool)> + Send + 'a>>;
|
||||
pub(crate) type DeferredReaderReopener<R> = Arc<dyn Fn(usize) -> Option<BitrotReader<R>> + Send + Sync>;
|
||||
pub(crate) type DecodeOutcome = (usize, Option<std::io::Error>, bool);
|
||||
|
||||
type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>;
|
||||
type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>;
|
||||
@@ -574,6 +575,7 @@ pub(crate) struct ParallelReader<R> {
|
||||
read_timeout: Duration,
|
||||
verify_reconstruction: bool,
|
||||
locality_preference_enabled: bool,
|
||||
demand_bound_lockstep: bool,
|
||||
// Request-scoped shard buffers keyed by shard index. Keeping ownership in
|
||||
// `ParallelReader` avoids dropping unused parity/backup slot buffers between stripes.
|
||||
buffers: ShardBufferPool,
|
||||
@@ -585,10 +587,8 @@ pub(crate) struct ParallelReader<R> {
|
||||
// it to the current stripe when it is engaged mid-object (backlog#923).
|
||||
engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
// Copy-source hedges use a fresh deferred reader so cancelling a hedge
|
||||
// never consumes the unopened reader reserved for a later stripe. The
|
||||
// vector is empty for callers that do not provide a reopen factory (tests
|
||||
// and the ordinary GET path retain the handle-based behavior).
|
||||
// Demand-bound hedges use a fresh deferred reader so cancelling a hedge
|
||||
// never consumes the unopened reader reserved for a later stripe.
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
stripe_index: usize,
|
||||
}
|
||||
@@ -777,9 +777,9 @@ where
|
||||
// reads all live readers on every stripe — the pre-backlog#923
|
||||
// behavior. With the gate on, only data slots start engaged; parity is
|
||||
// engaged on demand, stripe-aligned through its deferred handle.
|
||||
let data_shards_only = get_lockstep_data_shards_only_enabled();
|
||||
let demand_bound_lockstep = get_lockstep_data_shards_only_enabled();
|
||||
let engaged: SmallVec<_> = (0..readers.len())
|
||||
.map(|index| !data_shards_only || index < e.data_shards)
|
||||
.map(|index| !demand_bound_lockstep || index < e.data_shards)
|
||||
.collect();
|
||||
ParallelReader {
|
||||
readers,
|
||||
@@ -793,6 +793,7 @@ where
|
||||
read_timeout,
|
||||
verify_reconstruction,
|
||||
locality_preference_enabled: get_shard_locality_preference_enabled(),
|
||||
demand_bound_lockstep,
|
||||
buffers: ShardBufferPool::new(e.data_shards + e.parity_shards),
|
||||
stripe_state: None,
|
||||
engaged,
|
||||
@@ -1275,7 +1276,7 @@ where
|
||||
/// realigned (no pending deferred handle) is likewise retired instead of
|
||||
/// being read out of position.
|
||||
async fn read_lockstep(&mut self, state: &mut StripeReadState) {
|
||||
if matches!(decode_read_policy(), DecodeReadPolicy::DemandBound) {
|
||||
if self.demand_bound_lockstep {
|
||||
self.read_lockstep_demand_bound(state).await;
|
||||
return;
|
||||
}
|
||||
@@ -1531,17 +1532,18 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Demand-bound lockstep stripe read used by server-side copy sources.
|
||||
/// Demand-bound data-shards-only lockstep stripe read.
|
||||
///
|
||||
/// The ordinary lockstep path can cancel every in-flight reader once it
|
||||
/// has a quorum because all of its parity readers are already engaged.
|
||||
/// Copy sources keep parity unopened until a data reader is missing. A
|
||||
/// hedge therefore has to race the deferred parity reads against the
|
||||
/// original data reads and may retire the latter only after the parity has
|
||||
/// produced an actual decode-plus-verification quorum. The futures own
|
||||
/// their readers so disjoint data/parity slots can be admitted while the
|
||||
/// other group is still pending; dropping an abandoned future retires its
|
||||
/// stream without leaving a borrowed slot behind.
|
||||
/// Copy sources and the data-shards-only rollout gate keep parity unopened
|
||||
/// until a data reader is missing. A hedge therefore has to race the
|
||||
/// deferred parity reads against the original data reads and may retire the
|
||||
/// latter only after parity has produced an actual decode-plus-verification
|
||||
/// quorum. The futures own their readers so disjoint data/parity slots can
|
||||
/// be admitted while the other group is still pending; dropping an
|
||||
/// abandoned future retires its stream without leaving a borrowed slot
|
||||
/// behind.
|
||||
async fn read_lockstep_demand_bound(&mut self, state: &mut StripeReadState) {
|
||||
let num_readers = self.readers.len();
|
||||
state.reset(num_readers, self.data_shards);
|
||||
@@ -1576,14 +1578,14 @@ where
|
||||
let mut completed = 0usize;
|
||||
let mut failed = 0usize;
|
||||
let mut first_shard_recorded = false;
|
||||
let mut active = vec![false; num_readers];
|
||||
let mut temporary_parity = vec![false; num_readers];
|
||||
let mut active: ActiveReaders = smallvec![false; num_readers];
|
||||
let mut temporary_parity: ActiveReaders = smallvec![false; num_readers];
|
||||
// A deferred parity slot is attempted at most once per stripe. A
|
||||
// failed disposable hedge keeps its unopened reserve for the next
|
||||
// stripe, but must not be relaunched in a tight same-stripe retry
|
||||
// loop (which would defeat the bounded fan-out and amplify a remote
|
||||
// outage).
|
||||
let mut attempted_parity = vec![false; num_readers];
|
||||
let mut attempted_parity: ActiveReaders = smallvec![false; num_readers];
|
||||
// Once a data reader has returned an error (or was already missing at
|
||||
// setup), the loss is permanent for lockstep alignment. Use the
|
||||
// deferred handle and keep parity engaged across subsequent stripes;
|
||||
@@ -2189,8 +2191,10 @@ impl Erasure {
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
self.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
|
||||
.await
|
||||
let (written, error, _) = self
|
||||
.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new())
|
||||
.await;
|
||||
(written, error)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")]
|
||||
@@ -2207,8 +2211,10 @@ impl Erasure {
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
self.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
|
||||
.await
|
||||
let (written, error, _) = self
|
||||
.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new())
|
||||
.await;
|
||||
(written, error)
|
||||
}
|
||||
|
||||
/// GET decode entry point that also carries the deferred-parity stripe
|
||||
@@ -2261,6 +2267,37 @@ impl Erasure {
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
) -> (usize, Option<std::io::Error>)
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
let (written, error, _) = self
|
||||
.decode_inner(
|
||||
writer,
|
||||
readers,
|
||||
offset,
|
||||
length,
|
||||
total_length,
|
||||
read_costs,
|
||||
deferred_handles,
|
||||
deferred_reopeners,
|
||||
)
|
||||
.await;
|
||||
(written, error)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(crate) async fn decode_with_stripe_handles_and_reopeners_with_diagnostics<W, R>(
|
||||
&self,
|
||||
writer: &mut W,
|
||||
readers: Vec<Option<BitrotReader<R>>>,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
total_length: usize,
|
||||
read_costs: Option<Vec<ShardReadCost>>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
) -> DecodeOutcome
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
@@ -2298,6 +2335,7 @@ impl Erasure {
|
||||
written: &mut usize,
|
||||
ret_err: &mut Option<std::io::Error>,
|
||||
stage_metrics_enabled: bool,
|
||||
require_surplus_source: bool,
|
||||
) -> StripeFlow
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
@@ -2335,7 +2373,12 @@ impl Erasure {
|
||||
// missing data shard and an extra source shard was available, verify
|
||||
// the reconstructed data against that source before streaming bytes.
|
||||
let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
if let Err(e) = self.decode_data_with_reconstruction_verification(shards) {
|
||||
let decode_result = if require_surplus_source {
|
||||
self.decode_data_with_reconstruction_verification_for_lockstep(shards)
|
||||
} else {
|
||||
self.decode_data_with_reconstruction_verification(shards)
|
||||
};
|
||||
if let Err(e) = decode_result {
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start);
|
||||
let reason = GetObjectFailureReason::DecodeError;
|
||||
error!(
|
||||
@@ -2404,36 +2447,48 @@ impl Erasure {
|
||||
read_costs: Option<Vec<ShardReadCost>>,
|
||||
deferred_handles: Vec<Option<DeferredReaderStripeHandle>>,
|
||||
deferred_reopeners: Vec<Option<DeferredReaderReopener<R>>>,
|
||||
) -> (usize, Option<std::io::Error>)
|
||||
) -> DecodeOutcome
|
||||
where
|
||||
W: AsyncWrite + Send + Sync + Unpin,
|
||||
R: crate::erasure::coding::ShardSource,
|
||||
{
|
||||
if readers.len() != self.data_shards + self.parity_shards {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")));
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")), false);
|
||||
}
|
||||
|
||||
// block_size/data_shards come from on-disk metadata; a corrupt FileInfo with a
|
||||
// zero here must surface as an error, not a divide-by-zero panic on every GET.
|
||||
if self.block_size == 0 || self.data_shards == 0 {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")));
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")),
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
let Some(end_offset) = offset.checked_add(length) else {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
false,
|
||||
);
|
||||
};
|
||||
if end_offset > total_length {
|
||||
record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid);
|
||||
return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")));
|
||||
return (
|
||||
0,
|
||||
Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")),
|
||||
false,
|
||||
);
|
||||
}
|
||||
|
||||
let mut ret_err = None;
|
||||
|
||||
if length == 0 {
|
||||
return (0, ret_err);
|
||||
return (0, ret_err, false);
|
||||
}
|
||||
|
||||
let mut written = 0;
|
||||
@@ -2473,6 +2528,7 @@ impl Erasure {
|
||||
}
|
||||
};
|
||||
|
||||
let mut exact_quorum = false;
|
||||
if legacy_stripe_prefetch_enabled() {
|
||||
// Depth-1 stripe prefetch (backlog#930 HP-9 step 2): while the current
|
||||
// stripe is reconstructed and emitted, the next stripe's shard reads
|
||||
@@ -2515,6 +2571,7 @@ impl Erasure {
|
||||
let Some((mut shards, errs)) = current.take() else {
|
||||
break;
|
||||
};
|
||||
exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
|
||||
|
||||
if idx + 1 < blocks.len() {
|
||||
// Overlap: read stripe idx+1 while reconstructing/emitting idx.
|
||||
@@ -2546,6 +2603,7 @@ impl Erasure {
|
||||
// `shards` are borrowed again below. In the `Stop` case that
|
||||
// drop is what cancels the still-in-flight prefetch read.
|
||||
let (flow, next): (Option<StripeFlow>, Option<StripeReadOutput>) = {
|
||||
let require_surplus_source = reader.demand_bound_lockstep;
|
||||
let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled);
|
||||
let emit_fut = self.emit_decoded_stripe(
|
||||
writer,
|
||||
@@ -2556,6 +2614,7 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
require_surplus_source,
|
||||
);
|
||||
tokio::pin!(read_fut);
|
||||
tokio::pin!(emit_fut);
|
||||
@@ -2603,6 +2662,7 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
reader.demand_bound_lockstep,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2626,6 +2686,7 @@ impl Erasure {
|
||||
let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled();
|
||||
let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled);
|
||||
let (mut shards, errs) = reader.read().await;
|
||||
exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards;
|
||||
record_get_stage_duration_if_enabled(
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_STAGE_STRIPE_READ,
|
||||
@@ -2642,6 +2703,7 @@ impl Erasure {
|
||||
&mut written,
|
||||
&mut ret_err,
|
||||
stage_metrics_enabled,
|
||||
reader.demand_bound_lockstep,
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2654,14 +2716,14 @@ impl Erasure {
|
||||
}
|
||||
|
||||
if ret_err.is_some() {
|
||||
return (written, ret_err);
|
||||
return (written, ret_err, exact_quorum);
|
||||
}
|
||||
|
||||
if written < length {
|
||||
ret_err = Some(Error::LessData.into());
|
||||
}
|
||||
|
||||
(written, ret_err)
|
||||
(written, ret_err, exact_quorum)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2866,6 +2928,7 @@ mod tests {
|
||||
cursor: Cursor<Vec<u8>>,
|
||||
stall: Duration,
|
||||
sleep: Option<Pin<Box<Sleep>>>,
|
||||
stall_polls: Arc<AtomicUsize>,
|
||||
},
|
||||
}
|
||||
|
||||
@@ -2904,7 +2967,12 @@ mod tests {
|
||||
TestShardReader::TerminalFileNotFound => {
|
||||
Poll::Ready(Err(crate::disk::error::terminal_read_error_to_io(Error::FileNotFound)))
|
||||
}
|
||||
TestShardReader::PrefixThenSlow { cursor, stall, sleep } => {
|
||||
TestShardReader::PrefixThenSlow {
|
||||
cursor,
|
||||
stall,
|
||||
sleep,
|
||||
stall_polls,
|
||||
} => {
|
||||
let before = buf.filled().len();
|
||||
match Pin::new(cursor).poll_read(cx, buf) {
|
||||
// Cursor still has bytes for the current stripe: serve them.
|
||||
@@ -2914,6 +2982,7 @@ mod tests {
|
||||
// the task cleanly (no busy `wake_by_ref` spin), letting the
|
||||
// `#[tokio::test(start_paused = true)]` clock auto-advance.
|
||||
Poll::Ready(Ok(())) => {
|
||||
stall_polls.fetch_add(1, Ordering::SeqCst);
|
||||
let stall = *stall;
|
||||
let sleeper = sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep(stall)));
|
||||
let _ = sleeper.as_mut().poll(cx);
|
||||
@@ -2942,6 +3011,29 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct YieldOnceThenFailWriter {
|
||||
yielded: bool,
|
||||
}
|
||||
|
||||
impl AsyncWrite for YieldOnceThenFailWriter {
|
||||
fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
if !self.yielded {
|
||||
self.yielded = true;
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "injected emit failure after prefetch poll")))
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
struct DownstreamClosedWriter;
|
||||
|
||||
impl AsyncWrite for DownstreamClosedWriter {
|
||||
@@ -3878,6 +3970,7 @@ mod tests {
|
||||
(rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some(READ_TIMEOUT_SECS)),
|
||||
];
|
||||
temp_env::async_with_vars(vars, async {
|
||||
let stall_polls = Arc::new(AtomicUsize::new(0));
|
||||
let readers: Vec<Option<BitrotReader<TestShardReader>>> = shard_bufs
|
||||
.iter()
|
||||
.map(|buf| {
|
||||
@@ -3887,12 +3980,13 @@ mod tests {
|
||||
cursor: Cursor::new(prefix),
|
||||
stall: STALL,
|
||||
sleep: None,
|
||||
stall_polls: Arc::clone(&stall_polls),
|
||||
};
|
||||
Some(BitrotReader::new(reader, shard_size, hash_algo.clone(), false))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let mut writer = FailingEmitWriter;
|
||||
let mut writer = YieldOnceThenFailWriter { yielded: false };
|
||||
let start = TokioInstant::now();
|
||||
let (written, err) = erasure.decode(&mut writer, readers, 0, total_len, total_len).await;
|
||||
let elapsed = start.elapsed();
|
||||
@@ -3900,6 +3994,10 @@ mod tests {
|
||||
// Emit failed on stripe 0, so the GET fails with no bytes emitted.
|
||||
assert!(err.is_some(), "emit failure must surface as an error");
|
||||
assert_eq!(written, 0, "the failing writer accepts no bytes");
|
||||
assert!(
|
||||
stall_polls.load(Ordering::SeqCst) > 0,
|
||||
"the speculative next-stripe read must be in flight before emit fails"
|
||||
);
|
||||
// The decisive assertion: the prefetch read was cancelled rather than
|
||||
// awaited. Without cancel-safety this would take READ_TIMEOUT_SECS.
|
||||
assert!(
|
||||
@@ -4911,6 +5009,24 @@ mod tests {
|
||||
/// read timeout even though both parity readers were available to engage.
|
||||
#[tokio::test]
|
||||
async fn test_demand_bound_lockstep_hedges_to_deferred_parity_quorum() {
|
||||
with_decode_read_policy(DecodeReadPolicy::DemandBound, assert_deferred_parity_hedges_slow_data()).await;
|
||||
}
|
||||
|
||||
/// The ordinary GET rollout gate must use the same bounded parity race as
|
||||
/// CopySource. Leaving it on the legacy lockstep loop deadlocks the hedge:
|
||||
/// that loop waits for a parity success before cancelling the slow data
|
||||
/// read, but does not admit deferred parity until after the data read ends.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_data_shards_only_gate_hedges_to_deferred_parity_quorum() {
|
||||
temp_env::async_with_vars(
|
||||
[(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
|
||||
assert_deferred_parity_hedges_slow_data(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn assert_deferred_parity_hedges_slow_data() {
|
||||
const NUM_SHARDS: usize = 1;
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 2;
|
||||
@@ -4951,33 +5067,27 @@ mod tests {
|
||||
];
|
||||
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let (bufs, errs, engaged, readers_remaining) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
erasure,
|
||||
0,
|
||||
NUM_SHARDS * BLOCK_SIZE,
|
||||
None,
|
||||
vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS],
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
);
|
||||
let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read())
|
||||
.await
|
||||
.expect("deferred parity must cover a hedged data shard without waiting for read_timeout");
|
||||
(
|
||||
bufs,
|
||||
errs,
|
||||
parallel_reader.engaged.clone(),
|
||||
parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
erasure,
|
||||
0,
|
||||
NUM_SHARDS * BLOCK_SIZE,
|
||||
None,
|
||||
vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS],
|
||||
Duration::from_secs(60),
|
||||
true,
|
||||
);
|
||||
let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read())
|
||||
.await
|
||||
.expect("deferred parity must cover a hedged data shard without waiting for read_timeout");
|
||||
|
||||
assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut));
|
||||
assert_eq!(bufs.iter().filter(|buf| buf.is_some()).count(), DATA_SHARDS + 1);
|
||||
assert_eq!(engaged.as_slice(), &[true, true, true, true]);
|
||||
assert_eq!(readers_remaining, vec![false, true, true, true]);
|
||||
assert_eq!(parallel_reader.engaged.as_slice(), &[true, true, true, true]);
|
||||
assert_eq!(
|
||||
parallel_reader.readers.iter().map(Option::is_some).collect::<Vec<_>>(),
|
||||
vec![false, true, true, true]
|
||||
);
|
||||
}
|
||||
|
||||
/// A fast data failure must admit deferred parity immediately. There is
|
||||
@@ -5046,6 +5156,24 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_demand_bound_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
|
||||
with_decode_read_policy(
|
||||
DecodeReadPolicy::DemandBound,
|
||||
assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_data_shards_only_gate_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
|
||||
temp_env::async_with_vars(
|
||||
[(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))],
|
||||
assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn assert_canceled_hedge_preserves_deferred_parity_for_next_stripe() {
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
@@ -5094,7 +5222,7 @@ mod tests {
|
||||
Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo, false)),
|
||||
];
|
||||
|
||||
let (first_parity_reserved, second_result) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async {
|
||||
let (first_parity_reserved, second_result) = {
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let mut parallel_reader = ParallelReader::new_with_metrics_path_read_timeout_and_reconstruction_verification(
|
||||
readers,
|
||||
@@ -5155,8 +5283,7 @@ mod tests {
|
||||
parallel_reader.readers[2].is_some() && parallel_reader.readers[3].is_some(),
|
||||
(third_buffers, third_errors),
|
||||
)
|
||||
})
|
||||
.await;
|
||||
};
|
||||
|
||||
assert!(first_parity_reserved);
|
||||
assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS * 2);
|
||||
@@ -5240,6 +5367,58 @@ mod tests {
|
||||
assert!(error.is_none(), "a failed disposable hedge must not fail a recovered stripe: {error:?}");
|
||||
}
|
||||
|
||||
/// Rollout guard for backlog#1308: when a data shard and the first parity
|
||||
/// hedge both fail, the gate-on path must not settle at decode quorum and
|
||||
/// emit an unverified body. The second parity can restore decode quorum but
|
||||
/// cannot provide the extra source required for reconstruction verification,
|
||||
/// so the stripe must fail before exposing bytes.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn test_data_shards_only_gate_data_and_parity_failure_fails_before_output() {
|
||||
const BLOCK_SIZE: usize = 64;
|
||||
const DATA_SHARDS: usize = 2;
|
||||
const PARITY_SHARDS: usize = 2;
|
||||
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], async {
|
||||
let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE);
|
||||
let payload = (0..BLOCK_SIZE).map(|value| value as u8).collect::<Vec<_>>();
|
||||
let shards = erasure.encode_data(&payload).expect("test payload should encode");
|
||||
let shard_size = erasure.shard_size();
|
||||
|
||||
let readers = vec![
|
||||
Some(BitrotReader::new(TestShardReader::TimedOut, shard_size, HashAlgorithm::None, false)),
|
||||
Some(BitrotReader::new(
|
||||
TestShardReader::Ready(Cursor::new(shards[1].to_vec())),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
)),
|
||||
Some(BitrotReader::new(
|
||||
TestShardReader::TerminalFileNotFound,
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
)),
|
||||
Some(BitrotReader::new(
|
||||
TestShardReader::Ready(Cursor::new(shards[3].to_vec())),
|
||||
shard_size,
|
||||
HashAlgorithm::None,
|
||||
false,
|
||||
)),
|
||||
];
|
||||
|
||||
let mut output = Vec::new();
|
||||
let (written, error) = erasure.decode(&mut output, readers, 0, payload.len(), payload.len()).await;
|
||||
|
||||
assert_eq!(written, 0, "an unverified stripe must not report body bytes");
|
||||
assert!(output.is_empty(), "an unverified stripe must not expose a clean short body");
|
||||
let error = error.expect("data plus parity loss must fail closed");
|
||||
assert_eq!(error.kind(), ErrorKind::InvalidData);
|
||||
assert!(error.to_string().contains("insufficient source shards"));
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
/// Lockstep verification-quorum regression (backlog#1156). When a data shard is
|
||||
/// missing, the hedge must settle only at `data_shards + 1` (decode quorum plus
|
||||
/// a reconstruction-verification source), never at exactly `data_shards` — that
|
||||
|
||||
@@ -321,6 +321,13 @@ impl<'a> MultiWriter<'a> {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn take_retryable_internode_write_failure(&mut self) -> Option<Error> {
|
||||
self.errs
|
||||
.iter_mut()
|
||||
.find(|error| error.as_ref().is_some_and(Error::is_retryable_internode_write_failure))
|
||||
.and_then(Option::take)
|
||||
}
|
||||
|
||||
/// Effective budget for one shard operation: the smaller of the per-shard
|
||||
/// stall timeout and the time remaining until the object's absolute cap.
|
||||
/// Returns `None` when neither deadline is configured (wait indefinitely).
|
||||
|
||||
@@ -933,8 +933,29 @@ impl Erasure {
|
||||
}
|
||||
|
||||
pub(crate) fn decode_data_with_reconstruction_verification(&self, shards: &mut [Option<Vec<u8>>]) -> io::Result<()> {
|
||||
self.decode_data_with_reconstruction_verification_policy(shards, false)
|
||||
}
|
||||
|
||||
pub(crate) fn decode_data_with_reconstruction_verification_for_lockstep(
|
||||
&self,
|
||||
shards: &mut [Option<Vec<u8>>],
|
||||
) -> io::Result<()> {
|
||||
self.decode_data_with_reconstruction_verification_policy(shards, true)
|
||||
}
|
||||
|
||||
fn decode_data_with_reconstruction_verification_policy(
|
||||
&self,
|
||||
shards: &mut [Option<Vec<u8>>],
|
||||
require_surplus_source: bool,
|
||||
) -> io::Result<()> {
|
||||
let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none());
|
||||
let available_shards = shards.iter().filter(|shard| shard.is_some()).count();
|
||||
if require_surplus_source && missing_data_source && available_shards == self.data_shards {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"insufficient source shards to verify reconstructed data",
|
||||
));
|
||||
}
|
||||
let source_parity = if missing_data_source && available_shards > self.data_shards {
|
||||
shards
|
||||
.iter()
|
||||
@@ -1868,6 +1889,31 @@ mod tests {
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn decode_data_with_verification_scopes_exact_quorum_to_lockstep() {
|
||||
for uses_legacy in [false, true] {
|
||||
let erasure = Erasure::new_with_options(3, 2, 128, uses_legacy);
|
||||
let data = b"verified reads must not accept reconstruction without a surplus source";
|
||||
let encoded = erasure.encode_data(data).expect("encode should succeed");
|
||||
let mut exact_quorum = optional_shards(&encoded);
|
||||
exact_quorum[0] = None;
|
||||
exact_quorum[erasure.total_shard_count() - 1] = None;
|
||||
|
||||
let mut default_shards = exact_quorum.clone();
|
||||
erasure
|
||||
.decode_data_with_reconstruction_verification(&mut default_shards)
|
||||
.expect("default decode must preserve exact-quorum reconstruction");
|
||||
assert_eq!(default_shards[0].as_deref(), Some(encoded[0].as_ref()));
|
||||
|
||||
let err = erasure
|
||||
.decode_data_with_reconstruction_verification_for_lockstep(&mut exact_quorum)
|
||||
.expect_err("data-shards-only lockstep must reject an exact decode quorum");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("insufficient source shards"));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verify_data_and_parity_rejects_missing_and_mismatched_shards() {
|
||||
let erasure = Erasure::new(4, 2, 128);
|
||||
|
||||
@@ -108,6 +108,13 @@ where
|
||||
(shards, errs)
|
||||
}
|
||||
|
||||
fn heal_writer_failure(writers: &mut MultiWriter<'_>, error: io::Error) -> Error {
|
||||
writers
|
||||
.take_retryable_internode_write_failure()
|
||||
.map(|error| Error::RemoteClientUnavailable(error.to_string()))
|
||||
.unwrap_or_else(|| error.into())
|
||||
}
|
||||
|
||||
impl super::Erasure {
|
||||
pub async fn heal<R>(
|
||||
&self,
|
||||
@@ -202,10 +209,14 @@ impl super::Erasure {
|
||||
.map(|s| Bytes::from(s.unwrap_or_default()))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
writers.write(shards).await?;
|
||||
if let Err(error) = writers.write(shards).await {
|
||||
return Err(heal_writer_failure(&mut writers, error));
|
||||
}
|
||||
}
|
||||
|
||||
writers.shutdown().await?;
|
||||
if let Err(error) = writers.shutdown().await {
|
||||
return Err(heal_writer_failure(&mut writers, error));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -246,6 +257,35 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
struct InternodeFailureWriter {
|
||||
fail_on_write: bool,
|
||||
status: http::StatusCode,
|
||||
}
|
||||
|
||||
impl InternodeFailureWriter {
|
||||
fn error(&self) -> io::Error {
|
||||
rustfs_rio::new_test_internode_http_io_error(rustfs_rio::InternodeHttpErrorKind::HttpStatus(self.status))
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncWrite for InternodeFailureWriter {
|
||||
fn poll_write(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &[u8]) -> Poll<io::Result<usize>> {
|
||||
Poll::Ready(if self.fail_on_write {
|
||||
Err(self.error())
|
||||
} else {
|
||||
Ok(buf.len())
|
||||
})
|
||||
}
|
||||
|
||||
fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
|
||||
fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll<io::Result<()>> {
|
||||
Poll::Ready(Err(self.error()))
|
||||
}
|
||||
}
|
||||
|
||||
struct PendingReader;
|
||||
|
||||
impl AsyncRead for PendingReader {
|
||||
@@ -331,6 +371,94 @@ mod tests {
|
||||
assert!(writers.iter().all(Option::is_some));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_maps_put_file_epoch_conflict_to_retryable_remote_unavailable() {
|
||||
for status in [http::StatusCode::CONFLICT, http::StatusCode::BAD_REQUEST] {
|
||||
for (fail_on_write, data) in [
|
||||
(false, b"".as_slice()),
|
||||
(false, b"payload".as_slice()),
|
||||
(true, b"payload".as_slice()),
|
||||
] {
|
||||
let erasure = Erasure::new(2, 1, 64);
|
||||
let encoded = erasure.encode_data(data).expect("source shards should encode");
|
||||
let readers = encoded
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, shard)| {
|
||||
(index < erasure.data_shards).then(|| {
|
||||
BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut writers = (0..erasure.total_shard_count())
|
||||
.map(|index| {
|
||||
(index == erasure.data_shards).then(|| {
|
||||
BitrotWriterWrapper::new(
|
||||
CustomWriter::new_tokio_writer(InternodeFailureWriter { fail_on_write, status }),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::None,
|
||||
)
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let error = erasure
|
||||
.heal(&mut writers, readers, data.len(), &[])
|
||||
.await
|
||||
.expect_err("failed sole target must not satisfy heal write quorum");
|
||||
assert_eq!(
|
||||
matches!(error, Error::RemoteClientUnavailable(_)),
|
||||
status == http::StatusCode::CONFLICT,
|
||||
"status={status}, fail_on_write={fail_on_write}, len={}, error={error:?}",
|
||||
data.len()
|
||||
);
|
||||
assert!(writers.iter().all(Option::is_none), "failed target must not be committed");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_epoch_conflict_does_not_abort_healthy_target() {
|
||||
for fail_on_write in [false, true] {
|
||||
let erasure = Erasure::new(2, 2, 64);
|
||||
let data = b"healthy target must retain exact reconstructed bytes";
|
||||
let encoded = erasure.encode_data(data).expect("source shards should encode");
|
||||
let readers = encoded
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, shard)| {
|
||||
(index < erasure.data_shards)
|
||||
.then(|| BitrotReader::new(Cursor::new(shard.to_vec()), erasure.shard_size(), HashAlgorithm::None, false))
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let mut writers = vec![
|
||||
None,
|
||||
None,
|
||||
Some(BitrotWriterWrapper::new(
|
||||
CustomWriter::new_tokio_writer(InternodeFailureWriter {
|
||||
fail_on_write,
|
||||
status: http::StatusCode::CONFLICT,
|
||||
}),
|
||||
erasure.shard_size(),
|
||||
HashAlgorithm::None,
|
||||
)),
|
||||
Some(inline_writer(erasure.shard_size())),
|
||||
];
|
||||
erasure
|
||||
.heal(&mut writers, readers, data.len(), &[])
|
||||
.await
|
||||
.expect("one healthy target must still satisfy the existing heal quorum");
|
||||
assert!(writers[2].is_none(), "conflicting target must be dropped");
|
||||
assert_eq!(
|
||||
writers[3]
|
||||
.take()
|
||||
.expect("healthy target remains")
|
||||
.into_inline_data()
|
||||
.expect("inline target data"),
|
||||
encoded[3].to_vec()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_reconstructs_missing_parity_shard() {
|
||||
let erasure = Erasure::new(2, 2, 64);
|
||||
|
||||
@@ -183,8 +183,18 @@ pub enum StorageError {
|
||||
DecommissionNotStarted,
|
||||
#[error("Decommission already running")]
|
||||
DecommissionAlreadyRunning,
|
||||
#[error("Decommission capacity error: {0}")]
|
||||
DecommissionCapacity(String),
|
||||
#[error("decommission_capacity_blocked: Storage reached its minimum free drive threshold.: {message}")]
|
||||
DecommissionCapacityBlocked { message: String },
|
||||
#[error("Rebalance already running")]
|
||||
RebalanceAlreadyRunning,
|
||||
#[error("{operation}: stale pool metadata update rejected for pool {pool_index}; {reason}")]
|
||||
StalePoolMetadataUpdate {
|
||||
operation: String,
|
||||
pool_index: usize,
|
||||
reason: &'static str,
|
||||
},
|
||||
#[error("Operation canceled")]
|
||||
OperationCanceled,
|
||||
#[error("No heal required")]
|
||||
@@ -563,7 +573,20 @@ impl Clone for StorageError {
|
||||
StorageError::EntityTooLarge(a, b) => StorageError::EntityTooLarge(*a, *b),
|
||||
StorageError::DoneForNow => StorageError::DoneForNow,
|
||||
StorageError::DecommissionAlreadyRunning => StorageError::DecommissionAlreadyRunning,
|
||||
StorageError::DecommissionCapacity(message) => StorageError::DecommissionCapacity(message.clone()),
|
||||
StorageError::DecommissionCapacityBlocked { message } => StorageError::DecommissionCapacityBlocked {
|
||||
message: message.clone(),
|
||||
},
|
||||
StorageError::RebalanceAlreadyRunning => StorageError::RebalanceAlreadyRunning,
|
||||
StorageError::StalePoolMetadataUpdate {
|
||||
operation,
|
||||
pool_index,
|
||||
reason,
|
||||
} => StorageError::StalePoolMetadataUpdate {
|
||||
operation: operation.clone(),
|
||||
pool_index: *pool_index,
|
||||
reason,
|
||||
},
|
||||
StorageError::OperationCanceled => StorageError::OperationCanceled,
|
||||
StorageError::ErasureReadQuorum => StorageError::ErasureReadQuorum,
|
||||
StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum,
|
||||
@@ -666,7 +689,10 @@ impl StorageError {
|
||||
StorageError::InvalidPart(_, _, _) => StorageErrorCode::InvalidPart,
|
||||
StorageError::DoneForNow => StorageErrorCode::DoneForNow,
|
||||
StorageError::DecommissionAlreadyRunning => StorageErrorCode::DecommissionAlreadyRunning,
|
||||
StorageError::DecommissionCapacity(_) => StorageErrorCode::InvalidArgument,
|
||||
StorageError::DecommissionCapacityBlocked { .. } => StorageErrorCode::StorageFull,
|
||||
StorageError::RebalanceAlreadyRunning => StorageErrorCode::RebalanceAlreadyRunning,
|
||||
StorageError::StalePoolMetadataUpdate { .. } => StorageErrorCode::InvalidArgument,
|
||||
StorageError::OperationCanceled => StorageErrorCode::OperationCanceled,
|
||||
StorageError::ErasureReadQuorum => StorageErrorCode::ErasureReadQuorum,
|
||||
StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum,
|
||||
@@ -948,10 +974,6 @@ pub fn is_err_data_movement_overwrite(err: &Error) -> bool {
|
||||
matches!(err, &StorageError::DataMovementOverwriteErr(_, _, _))
|
||||
}
|
||||
|
||||
pub fn is_err_decommission_running(err: &Error) -> bool {
|
||||
matches!(err, &StorageError::DecommissionAlreadyRunning)
|
||||
}
|
||||
|
||||
#[allow(dead_code, reason = "predicate asserted by this file's tests (backlog#1823)")]
|
||||
pub fn is_err_rebalance_running(err: &Error) -> bool {
|
||||
matches!(err, &StorageError::RebalanceAlreadyRunning)
|
||||
@@ -1347,9 +1369,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_error_running_state_helpers() {
|
||||
assert!(is_err_decommission_running(&StorageError::DecommissionAlreadyRunning));
|
||||
assert!(!is_err_decommission_running(&StorageError::RebalanceAlreadyRunning));
|
||||
|
||||
assert!(is_err_rebalance_running(&StorageError::RebalanceAlreadyRunning));
|
||||
assert!(!is_err_rebalance_running(&StorageError::DecommissionAlreadyRunning));
|
||||
assert!(is_err_operation_canceled(&StorageError::OperationCanceled));
|
||||
|
||||
@@ -681,6 +681,16 @@ impl Drop for ScannerPublicationCommitScopeInner {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
#[doc(hidden)]
|
||||
pub struct DecommissionCapacityOptions {
|
||||
pub(crate) expected_data_bytes: Option<usize>,
|
||||
pub(crate) operation_id: Option<Uuid>,
|
||||
pub(crate) generation: Option<u64>,
|
||||
pub(crate) owner_nonce: Option<Uuid>,
|
||||
pub(crate) mutation_id: Option<Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
pub struct ObjectOptions {
|
||||
// Use the maximum parity (N/2), used when saving server configuration files
|
||||
@@ -725,6 +735,12 @@ pub struct ObjectOptions {
|
||||
|
||||
pub data_movement: bool,
|
||||
pub raw_data_movement_read: bool,
|
||||
/// Durable reservation identity carried only by decommission writes. Other
|
||||
/// data-movement users, including rebalance, leave it unset. Keep this
|
||||
/// context boxed because `ObjectOptions` is passed by value through deep
|
||||
/// storage futures.
|
||||
#[doc(hidden)]
|
||||
pub decommission_capacity: Option<Box<DecommissionCapacityOptions>>,
|
||||
/// Materialize the data-movement per-part checksum sidecar for APIs that
|
||||
/// return part checksums. Ordinary object reads leave it encoded.
|
||||
pub include_part_checksums: bool,
|
||||
@@ -788,6 +804,36 @@ pub struct ObjectOptions {
|
||||
/// Storage-owned journal writer used by the atomic delete path. This is
|
||||
/// populated only by the `ECStore` wrapper that holds the namespace locks.
|
||||
pub tier_delete_journal_api: Option<Arc<crate::store::ECStore>>,
|
||||
/// Internal staged-mutation admission supplied by `ECStore`; each local
|
||||
/// publish is fenced namespace-first and then by decommission capacity.
|
||||
#[doc(hidden)]
|
||||
pub decommission_capacity_admission: Option<Arc<crate::store::ECStore>>,
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
pub(crate) fn with_capacity_expected_data_bytes(expected_data_bytes: Option<usize>) -> Self {
|
||||
Self {
|
||||
decommission_capacity: expected_data_bytes.map(|expected_data_bytes| {
|
||||
Box::new(DecommissionCapacityOptions {
|
||||
expected_data_bytes: Some(expected_data_bytes),
|
||||
..Default::default()
|
||||
})
|
||||
}),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn capacity_expected_data_bytes(&self) -> Option<usize> {
|
||||
self.decommission_capacity
|
||||
.as_deref()
|
||||
.and_then(|capacity| capacity.expected_data_bytes)
|
||||
}
|
||||
|
||||
pub(crate) fn has_decommission_capacity_reservation(&self) -> bool {
|
||||
self.decommission_capacity
|
||||
.as_deref()
|
||||
.is_some_and(|capacity| capacity.operation_id.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ObjectOptions {
|
||||
|
||||
@@ -368,6 +368,19 @@ impl InstanceContext {
|
||||
Arc::clone(&self.data_movement_generation_notify)
|
||||
}
|
||||
|
||||
pub(crate) fn observe_durable_data_movement_generation(&self, generation: u64) {
|
||||
if generation == 0 || self.data_movement_generation_exhausted.load(Ordering::Acquire) {
|
||||
return;
|
||||
}
|
||||
let previous = self.data_movement_generation.fetch_max(generation, Ordering::AcqRel);
|
||||
if generation == u64::MAX {
|
||||
self.data_movement_generation_exhausted.store(true, Ordering::Release);
|
||||
}
|
||||
if generation > previous {
|
||||
self.data_movement_generation_notify.notify_waiters();
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_publication_state_allowed(&self) -> bool {
|
||||
!self.data_movement_operation_epoch_exhausted()
|
||||
&& !self.data_movement_generation_exhausted()
|
||||
@@ -386,6 +399,20 @@ impl InstanceContext {
|
||||
}
|
||||
|
||||
pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 {
|
||||
let (previous, result) = self.advance_data_movement_operation_epoch_only();
|
||||
if result != previous {
|
||||
let _ = self.advance_data_movement_generation();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
pub(crate) fn advance_data_movement_operation_epoch_to_durable_generation(&self, generation: u64) -> u64 {
|
||||
let (_, result) = self.advance_data_movement_operation_epoch_only();
|
||||
self.observe_durable_data_movement_generation(generation);
|
||||
result
|
||||
}
|
||||
|
||||
fn advance_data_movement_operation_epoch_only(&self) -> (u64, u64) {
|
||||
self.scanner_publication_state
|
||||
.store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release);
|
||||
let previous = self.data_movement_operation_epoch.load(Ordering::Acquire);
|
||||
@@ -396,10 +423,7 @@ impl InstanceContext {
|
||||
if result == u64::MAX {
|
||||
self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release);
|
||||
}
|
||||
if result != previous {
|
||||
let _ = self.advance_data_movement_generation();
|
||||
}
|
||||
result
|
||||
(previous, result)
|
||||
}
|
||||
|
||||
/// Advance the movement generation after a durable movement transition.
|
||||
|
||||
@@ -845,7 +845,7 @@ impl ECStore {
|
||||
|
||||
let mut pool_stats = Vec::with_capacity(self.pools.len());
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
|
||||
|
||||
for disk_stat in disk_stats.iter() {
|
||||
let mut pool_stat = RebalanceStats {
|
||||
@@ -868,8 +868,10 @@ impl ECStore {
|
||||
pool_stats.push(pool_stat);
|
||||
}
|
||||
|
||||
let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating);
|
||||
let meta = RebalanceMeta {
|
||||
id: Uuid::new_v4().to_string(),
|
||||
stopped_at: (!has_participating_pool).then_some(now),
|
||||
percent_free_goal,
|
||||
pool_stats,
|
||||
..Default::default()
|
||||
@@ -963,6 +965,18 @@ impl ECStore {
|
||||
)));
|
||||
}
|
||||
if meta.stopped_at.is_some() {
|
||||
if !is_rebalance_conflicting_with_decommission(meta) {
|
||||
debug!(
|
||||
event = EVENT_REBALANCE_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REBALANCE,
|
||||
state = "start_skipped",
|
||||
reason = "not_started_terminal",
|
||||
rebalance_id = %expected_id,
|
||||
"Skipped rebalance start because metadata is already terminal"
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
return Err(Error::other(format!("rebalance {expected_id} was stopped before start")));
|
||||
}
|
||||
}
|
||||
@@ -1214,11 +1228,11 @@ impl ECStore {
|
||||
};
|
||||
let movement_gate = self.ctx.data_movement_operation_gate();
|
||||
let _movement_guard = movement_gate.write().await;
|
||||
let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
|
||||
let (previous_meta, meta_to_save) = {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
let previous_meta = rebalance_meta.clone();
|
||||
let meta_to_save =
|
||||
stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)?;
|
||||
let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?;
|
||||
(previous_meta, meta_to_save)
|
||||
};
|
||||
|
||||
@@ -1250,14 +1264,10 @@ impl ECStore {
|
||||
.await?;
|
||||
let movement_gate = self.ctx.data_movement_operation_gate();
|
||||
let _movement_guard = movement_gate.write().await;
|
||||
let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
|
||||
let meta_to_save = {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
rollback_rebalance_start_meta_snapshot_for_id(
|
||||
rebalance_meta.as_mut(),
|
||||
OffsetDateTime::now_utc(),
|
||||
expected_id,
|
||||
start_error,
|
||||
)
|
||||
rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error)
|
||||
};
|
||||
|
||||
if let Some(meta_to_save) = meta_to_save {
|
||||
@@ -1326,7 +1336,11 @@ mod tests {
|
||||
use crate::set_disk::{PutObjectCommitBarrier, PutObjectCommitPause, hermetic_set_disks_isolated};
|
||||
|
||||
async fn persist_initialized_identity_then_remove_pool_meta(store: &Arc<ECStore>) {
|
||||
let mut write_state = PoolMetaWriteState::for_startup(store.id, false);
|
||||
let deployment_id = store
|
||||
.ctx
|
||||
.deployment_id()
|
||||
.expect("test store should have a deployment identity");
|
||||
let mut write_state = PoolMetaWriteState::for_startup(deployment_id, false);
|
||||
persist_pool_meta_identity_for_startup(store.pools.clone(), &mut write_state, true)
|
||||
.await
|
||||
.expect("initialized pool metadata identity should persist");
|
||||
@@ -1402,6 +1416,62 @@ mod tests {
|
||||
assert!(cancel.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() {
|
||||
let (_temp_dirs, store, restarted) =
|
||||
crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid");
|
||||
*store.rebalance_meta.write().await = Some(RebalanceMeta {
|
||||
id: "previous-terminal-rebalance".to_string(),
|
||||
stopped_at: Some(movement_floor),
|
||||
..Default::default()
|
||||
});
|
||||
set_rebalance_disk_stats_override_for_test(
|
||||
store.id,
|
||||
vec![
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 50,
|
||||
},
|
||||
DiskStat {
|
||||
total_space: 100,
|
||||
available_space: 50,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
let rebalance_id = store
|
||||
.init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()])
|
||||
.await
|
||||
.expect("equal free ratio admin rebalance should succeed as a terminal no-op");
|
||||
let stopped_at = {
|
||||
let local = store.rebalance_meta.read().await;
|
||||
let local = local.as_ref().expect("no-op rebalance metadata should remain available");
|
||||
assert_eq!(local.id, rebalance_id);
|
||||
assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating));
|
||||
let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp");
|
||||
assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1));
|
||||
stopped_at
|
||||
};
|
||||
|
||||
let stopped_generation =
|
||||
u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation");
|
||||
let live_status = store.scanner_data_movement_pause_status().await;
|
||||
assert!(!live_status.paused);
|
||||
assert_eq!(live_status.movement_generation, stopped_generation);
|
||||
|
||||
restarted
|
||||
.load_rebalance_meta()
|
||||
.await
|
||||
.expect("restarted store should load the persisted no-op rebalance metadata");
|
||||
let status = restarted.scanner_data_movement_pause_status().await;
|
||||
|
||||
assert!(!status.paused);
|
||||
assert_eq!(status.movement_generation, stopped_generation);
|
||||
assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() {
|
||||
|
||||
@@ -83,6 +83,7 @@ pub async fn test_store_with_persisted_rebalance_meta(
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
decommission_capacity_entry_gate: tokio::sync::Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
});
|
||||
@@ -97,7 +98,7 @@ pub(crate) async fn test_two_pool_stores(
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
) {
|
||||
test_two_pool_stores_with_contexts(rebalance_meta, false).await
|
||||
test_pool_stores_with_contexts(rebalance_meta, false, 2, 2).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -108,7 +109,7 @@ pub(crate) async fn test_two_pool_stores_with_isolated_node_contexts(
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
) {
|
||||
test_two_pool_stores_with_contexts(rebalance_meta, true).await
|
||||
test_pool_stores_with_contexts(rebalance_meta, true, 2, 2).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -123,26 +124,58 @@ pub(crate) async fn promote_test_pool_meta_to_v2(store: &std::sync::Arc<crate::s
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn test_two_pool_stores_with_contexts(
|
||||
pub(crate) async fn test_three_pool_stores_with_isolated_node_contexts(
|
||||
rebalance_meta: Option<RebalanceMeta>,
|
||||
) -> (
|
||||
Vec<tempfile::TempDir>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
) {
|
||||
test_pool_stores_with_contexts(rebalance_meta, true, 3, 2).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn test_three_pool_stores_with_three_disk_sets_with_isolated_node_contexts(
|
||||
rebalance_meta: Option<RebalanceMeta>,
|
||||
) -> (
|
||||
Vec<tempfile::TempDir>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
) {
|
||||
test_pool_stores_with_contexts(rebalance_meta, true, 3, 3).await
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn test_pool_stores_with_contexts(
|
||||
rebalance_meta: Option<RebalanceMeta>,
|
||||
isolate_node_contexts: bool,
|
||||
pool_count: usize,
|
||||
set_drive_count: usize,
|
||||
) -> (
|
||||
Vec<tempfile::TempDir>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
std::sync::Arc<crate::store::ECStore>,
|
||||
) {
|
||||
crate::services::notification_sys::install_cross_pool_fence_fleet_proof_for_test();
|
||||
use crate::core::pools::PoolMeta;
|
||||
use crate::core::pools::{POOL_META_VERSION, PoolMeta, PoolMetaWriteState, persist_pool_meta_identity_for_startup};
|
||||
use crate::layout::endpoints::{EndpointServerPools, SetupType};
|
||||
|
||||
let ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let (mut temp_dirs, first_pool) =
|
||||
crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(std::sync::Arc::clone(&ctx), 0).await;
|
||||
let (second_temp_dirs, second_pool) =
|
||||
crate::core::sets::make_local_two_set_sets_for_pool_with_ctx(std::sync::Arc::clone(&ctx), 1).await;
|
||||
temp_dirs.extend(second_temp_dirs);
|
||||
let pools = vec![first_pool, second_pool];
|
||||
let deployment_id = uuid::Uuid::new_v4();
|
||||
ctx.set_deployment_id(deployment_id);
|
||||
let mut temp_dirs = Vec::new();
|
||||
let mut pools = Vec::with_capacity(pool_count);
|
||||
for pool_index in 0..pool_count {
|
||||
let (pool_temp_dirs, pool) = crate::core::sets::make_local_two_set_sets_for_pool_with_drive_count_and_ctx(
|
||||
std::sync::Arc::clone(&ctx),
|
||||
pool_index,
|
||||
set_drive_count,
|
||||
)
|
||||
.await;
|
||||
temp_dirs.extend(pool_temp_dirs);
|
||||
pools.push(pool);
|
||||
}
|
||||
{
|
||||
let local_disk_map = ctx.local_disk_map();
|
||||
let mut local_disk_map = local_disk_map.write().await;
|
||||
@@ -154,11 +187,24 @@ async fn test_two_pool_stores_with_contexts(
|
||||
}
|
||||
}
|
||||
}
|
||||
let pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
|
||||
let mut pool_meta = PoolMeta::new(&pools, &PoolMeta::default());
|
||||
pool_meta.version = POOL_META_VERSION;
|
||||
pool_meta
|
||||
.save_for_startup(pools.clone())
|
||||
.await
|
||||
.expect("baseline pool metadata should be persisted");
|
||||
let mut pool_meta_write_state = PoolMetaWriteState::for_startup(deployment_id, true);
|
||||
persist_pool_meta_identity_for_startup(pools.clone(), &mut pool_meta_write_state, false)
|
||||
.await
|
||||
.expect("pending pool metadata identity should be persisted");
|
||||
let replica_state = pool_meta
|
||||
.load_no_lock_from_replicas_observing(pools.clone(), &mut pool_meta_write_state)
|
||||
.await
|
||||
.expect("baseline pool metadata should remain readable");
|
||||
pool_meta_write_state.observe_replicas(replica_state);
|
||||
persist_pool_meta_identity_for_startup(pools.clone(), &mut pool_meta_write_state, true)
|
||||
.await
|
||||
.expect("initialized pool metadata identity should be persisted");
|
||||
if let Some(meta) = rebalance_meta.as_ref() {
|
||||
meta.save(pools[0].clone())
|
||||
.await
|
||||
@@ -169,6 +215,7 @@ async fn test_two_pool_stores_with_contexts(
|
||||
let other_ctx = if isolate_node_contexts {
|
||||
let other_ctx = std::sync::Arc::new(crate::runtime::instance::InstanceContext::new());
|
||||
other_ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
other_ctx.set_deployment_id(deployment_id);
|
||||
*other_ctx.local_disk_map().write().await = ctx.local_disk_map().read().await.clone();
|
||||
other_ctx.set_endpoints(endpoint_pools.clone());
|
||||
other_ctx
|
||||
@@ -183,9 +230,10 @@ async fn test_two_pool_stores_with_contexts(
|
||||
peer_sys: crate::cluster::rpc::S3PeerSys::new_with_instance_ctx(&endpoint_pools, std::sync::Arc::clone(&store_ctx)),
|
||||
pool_meta: tokio::sync::RwLock::new(pool_meta.clone()),
|
||||
rebalance_meta: tokio::sync::RwLock::new(rebalance_meta.clone()),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None, None]),
|
||||
decommission_cancelers: tokio::sync::RwLock::new(vec![None; pool_count]),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::new(pool_meta_write_state.independent_clone_for_test()),
|
||||
decommission_capacity_entry_gate: tokio::sync::Mutex::default(),
|
||||
ctx: store_ctx,
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
})
|
||||
@@ -195,6 +243,8 @@ async fn test_two_pool_stores_with_contexts(
|
||||
if isolate_node_contexts {
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&store), Vec::new()).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&other_store), Vec::new()).await;
|
||||
} else {
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(std::sync::Arc::clone(&store), Vec::new()).await;
|
||||
}
|
||||
(temp_dirs, store, other_store)
|
||||
}
|
||||
|
||||
@@ -3009,6 +3009,7 @@ fn test_store_with_rebalance_meta(meta: RebalanceMeta) -> Arc<crate::store::ECSt
|
||||
decommission_cancelers: tokio::sync::RwLock::new(Vec::new()),
|
||||
start_gate: tokio::sync::Mutex::new(()),
|
||||
pool_meta_save_gate: tokio::sync::Mutex::default(),
|
||||
decommission_capacity_entry_gate: tokio::sync::Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
})
|
||||
|
||||
@@ -161,6 +161,7 @@ impl ECStore {
|
||||
|
||||
let cancel_tx = CancellationToken::new();
|
||||
let rx = cancel_tx.clone();
|
||||
let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
|
||||
let activation_outcome;
|
||||
let candidate;
|
||||
let expected_cancel;
|
||||
@@ -185,12 +186,8 @@ impl ECStore {
|
||||
return Ok(false);
|
||||
}
|
||||
expected_cancel = meta.cancel.clone();
|
||||
(candidate, activation_outcome, must_persist) = stage_local_rebalance_worker_activation(
|
||||
meta,
|
||||
expected_id.as_ref(),
|
||||
cancel_tx.clone(),
|
||||
OffsetDateTime::now_utc(),
|
||||
)?;
|
||||
(candidate, activation_outcome, must_persist) =
|
||||
stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?;
|
||||
if let Err(err) = activation_fence.ensure_held() {
|
||||
cancel_tx.cancel();
|
||||
return Err(err);
|
||||
@@ -384,11 +381,11 @@ impl ECStore {
|
||||
tokio::select! {
|
||||
result = done_rx.recv() => {
|
||||
quit = true;
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let terminal_event = classify_rebalance_terminal_event(result, now);
|
||||
msg = terminal_event.message().to_string();
|
||||
let movement_gate = store.ctx.data_movement_operation_gate();
|
||||
let movement_guard = movement_gate.write().await;
|
||||
let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await;
|
||||
let terminal_event = classify_rebalance_terminal_event(result, terminal_at);
|
||||
msg = terminal_event.message().to_string();
|
||||
let previous_meta = store.rebalance_meta.read().await.clone();
|
||||
let terminal_state_present = {
|
||||
let mut rebalance_meta = store.rebalance_meta.write().await;
|
||||
@@ -405,7 +402,7 @@ impl ECStore {
|
||||
{
|
||||
pool_stat.info.stopping = false;
|
||||
pool_stat.info.status = RebalStatus::Failed;
|
||||
pool_stat.info.end_time = Some(now);
|
||||
pool_stat.info.end_time = Some(terminal_at);
|
||||
pool_stat.info.last_error = Some(
|
||||
pool_stat
|
||||
.cleanup_warnings
|
||||
@@ -433,7 +430,7 @@ impl ECStore {
|
||||
&mut pool_stat.info.end_time,
|
||||
&mut pool_stat.info.last_error,
|
||||
terminal_event,
|
||||
now,
|
||||
terminal_at,
|
||||
);
|
||||
}
|
||||
true
|
||||
@@ -835,6 +832,10 @@ impl ECStore {
|
||||
opt: RebalSaveOpt,
|
||||
expected_id: Option<&str>,
|
||||
) -> Result<()> {
|
||||
let now = match opt {
|
||||
RebalSaveOpt::Stats => OffsetDateTime::now_utc(),
|
||||
RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await,
|
||||
};
|
||||
let meta_to_save = {
|
||||
let mut rebalance_meta = self.rebalance_meta.write().await;
|
||||
if let Some(expected_id) = expected_id {
|
||||
@@ -844,7 +845,6 @@ impl ECStore {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
let now = OffsetDateTime::now_utc();
|
||||
apply_rebalance_save_option(meta, pool_idx, opt, now);
|
||||
meta.clone()
|
||||
};
|
||||
|
||||
@@ -37,6 +37,8 @@ use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS;
|
||||
#[cfg(test)]
|
||||
use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX;
|
||||
#[cfg(test)]
|
||||
use super::super::ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE;
|
||||
#[cfg(test)]
|
||||
use super::super::get_metadata_slowtail_fault_delay;
|
||||
use super::super::{
|
||||
Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED,
|
||||
@@ -46,13 +48,15 @@ use super::super::{
|
||||
ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RawFileInfo, ReadMultipleReq,
|
||||
ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, StorageError,
|
||||
UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct,
|
||||
capacity_scope_from_disks, coding, collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug,
|
||||
disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset,
|
||||
inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled,
|
||||
is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled, is_object_dangling,
|
||||
capacity_scope_from_disks, codec_streaming_rollout_applies, coding, collect_inline_data_shard_fileinfos_by_index_or_reason,
|
||||
current_dirty_generation, debug, disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info,
|
||||
inline_erasure_shard_file_offset, inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found,
|
||||
is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled,
|
||||
is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling,
|
||||
is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure,
|
||||
merge_file_meta_versions, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, reduce_write_quorum_errs,
|
||||
send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn,
|
||||
merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs,
|
||||
reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err,
|
||||
try_read_inline_data_shards_direct, warn,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
@@ -449,13 +453,29 @@ use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tokio::sync::{Mutex, RwLock, oneshot};
|
||||
use tokio::task::JoinSet;
|
||||
|
||||
struct AbortOnDropJoinHandle<T>(tokio::task::JoinHandle<T>);
|
||||
|
||||
impl<T> Future for AbortOnDropJoinHandle<T> {
|
||||
type Output = std::result::Result<T, tokio::task::JoinError>;
|
||||
|
||||
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
|
||||
Pin::new(&mut self.0).poll(cx)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T> Drop for AbortOnDropJoinHandle<T> {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read";
|
||||
pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE";
|
||||
const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS";
|
||||
const DEFAULT_GET_METADATA_READ_VERSION_COALESCE_DELAY_MICROS: u64 = 200;
|
||||
const METRIC_GET_METADATA_READ_VERSION_COALESCER_TOTAL: &str = "rustfs_get_metadata_read_version_coalescer_total";
|
||||
pub(in crate::set_disk) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
|
||||
pub(crate) const ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE: &str = "RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE";
|
||||
/// Default reader-setup strategy for the GET read path (rustfs/backlog#1215,
|
||||
/// #1159, #923).
|
||||
///
|
||||
@@ -687,6 +707,10 @@ pub(in crate::set_disk) struct MetadataQuorumAccumulator {
|
||||
pub(in crate::set_disk) hard_errors: usize,
|
||||
pub(in crate::set_disk) candidate: Option<FileInfo>,
|
||||
pub(in crate::set_disk) candidate_votes: usize,
|
||||
// Bitset of shard indexes whose metadata matches the candidate. Erasure
|
||||
// layouts are capped at 16 shards, so this stays allocation-free on the
|
||||
// GET metadata hot path.
|
||||
candidate_shard_mask: u16,
|
||||
pub(in crate::set_disk) conflicting_metadata: bool,
|
||||
pub(in crate::set_disk) delete_marker_seen: bool,
|
||||
pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>,
|
||||
@@ -708,6 +732,7 @@ impl MetadataQuorumAccumulator {
|
||||
hard_errors: 0,
|
||||
candidate: None,
|
||||
candidate_votes: 0,
|
||||
candidate_shard_mask: 0,
|
||||
conflicting_metadata: false,
|
||||
delete_marker_seen: false,
|
||||
delete_marker_candidates: Vec::new(),
|
||||
@@ -723,6 +748,14 @@ impl MetadataQuorumAccumulator {
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(None, file_info);
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) {
|
||||
self.observe_file_info_with_index(Some(disk_index), file_info);
|
||||
}
|
||||
|
||||
fn observe_file_info_with_index(&mut self, disk_index: Option<usize>, file_info: &FileInfo) {
|
||||
if !file_info_is_valid_for_metadata(file_info) {
|
||||
self.hard_errors = self.hard_errors.saturating_add(1);
|
||||
return;
|
||||
@@ -762,6 +795,11 @@ impl MetadataQuorumAccumulator {
|
||||
match &self.candidate {
|
||||
Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => {
|
||||
self.candidate_votes = self.candidate_votes.saturating_add(1);
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
Some(_) => {
|
||||
self.conflicting_metadata = true;
|
||||
@@ -769,10 +807,38 @@ impl MetadataQuorumAccumulator {
|
||||
None => {
|
||||
self.candidate = Some(file_info.clone());
|
||||
self.candidate_votes = 1;
|
||||
if let Some(disk_index) = disk_index
|
||||
&& let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index)
|
||||
{
|
||||
self.candidate_shard_mask |= bit;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option<u16> {
|
||||
let &erasure_index = candidate.erasure.distribution.get(disk_index)?;
|
||||
if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index {
|
||||
return None;
|
||||
}
|
||||
Some(1u16 << (erasure_index - 1))
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool {
|
||||
self.candidate_read_reserve_target()
|
||||
.is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option<usize> {
|
||||
let candidate = self.candidate.as_ref()?;
|
||||
Some(
|
||||
candidate
|
||||
.erasure
|
||||
.data_blocks
|
||||
.saturating_add(usize::from(candidate.erasure.parity_blocks > 0)),
|
||||
)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) {
|
||||
match err {
|
||||
DiskError::FileNotFound | DiskError::VolumeNotFound => {
|
||||
@@ -1083,6 +1149,33 @@ fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Op
|
||||
None
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn non_inline_data_read_candidate_is_safe(candidate: &FileInfo) -> bool {
|
||||
if candidate.inline_data()
|
||||
|| candidate.is_compressed()
|
||||
|| candidate.is_remote()
|
||||
|| candidate
|
||||
.metadata
|
||||
.keys()
|
||||
.any(|key| rustfs_utils::http::is_object_encryption_marker(key))
|
||||
|| candidate.parts.len() != 1
|
||||
{
|
||||
return false;
|
||||
}
|
||||
candidate.has_valid_erasure_geometry()
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn late_materialization_candidate_is_safe(candidate: &FileInfo) -> bool {
|
||||
non_inline_data_read_candidate_is_safe(candidate)
|
||||
&& candidate.size > 512 * 1024
|
||||
&& object_fits_single_block(candidate.size, candidate.erasure.block_size)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn non_inline_data_read_early_stop_allowed(read_data: bool, bucket: &str, object: &str) -> bool {
|
||||
read_data && is_get_metadata_non_inline_data_read_early_stop_enabled() && !codec_streaming_rollout_applies(bucket, object)
|
||||
}
|
||||
|
||||
const NON_INLINE_SINGLE_PENDING_HEDGE_DELAY: Duration = Duration::from_millis(100);
|
||||
|
||||
fn data_read_inline_missing_shards_are_pending(
|
||||
candidate: &FileInfo,
|
||||
parts_metadata: &[FileInfo],
|
||||
@@ -1929,14 +2022,10 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
|
||||
return;
|
||||
}
|
||||
|
||||
// Only CopySource uses disposable, stripe-aligned reopeners. Ordinary GET
|
||||
// readers use the existing deferred handle and should not retain one
|
||||
// heap-allocated closure (plus cloned path/disk state) for every parity
|
||||
// slot.
|
||||
let copy_source_demand_bound = matches!(
|
||||
crate::set_disk::get_object_read_policy(),
|
||||
crate::set_disk::GetObjectReadPolicy::CopySource
|
||||
);
|
||||
// Every demand-bound lockstep reader needs a disposable, stripe-aligned
|
||||
// reopener. Otherwise a recovered slow data read can cancel and consume
|
||||
// the only parity reserve needed by a later degraded stripe.
|
||||
let demand_bound_lockstep = crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled();
|
||||
|
||||
for idx in 0..disks.len() {
|
||||
if setup.attempted[idx] {
|
||||
@@ -1951,7 +2040,7 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
|
||||
let disk = disks[idx].clone();
|
||||
let data_dir = files[idx].data_dir.unwrap_or_default();
|
||||
let path = format!("{object}/{data_dir}/part.{part_number}");
|
||||
let reopener = copy_source_demand_bound.then(|| {
|
||||
let reopener = demand_bound_lockstep.then(|| {
|
||||
deferred_reader_reopener(
|
||||
inline_data.clone(),
|
||||
disk.clone(),
|
||||
@@ -1992,7 +2081,7 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
|
||||
// ready/error bookkeeping that quorum decisions rely on is left untouched.
|
||||
// Gate off (default): keep the eagerly opened parity readers exactly as
|
||||
// before — the lockstep path reads them on every stripe.
|
||||
if !crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled() {
|
||||
if !demand_bound_lockstep {
|
||||
return;
|
||||
}
|
||||
for idx in data_shards..disks.len() {
|
||||
@@ -2004,7 +2093,7 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers(
|
||||
let disk = disks[idx].clone();
|
||||
let data_dir = files[idx].data_dir.unwrap_or_default();
|
||||
let path = format!("{object}/{data_dir}/part.{part_number}");
|
||||
let reopener = copy_source_demand_bound.then(|| {
|
||||
let reopener = demand_bound_lockstep.then(|| {
|
||||
deferred_reader_reopener(
|
||||
inline_data.clone(),
|
||||
disk.clone(),
|
||||
@@ -2869,6 +2958,7 @@ impl SetDisks {
|
||||
read_data,
|
||||
healing,
|
||||
incl_free_versions,
|
||||
non_inline_data_read_early_stop_allowed(read_data, bucket, object),
|
||||
default_parity_count,
|
||||
allow_coalescing,
|
||||
)
|
||||
@@ -2934,7 +3024,7 @@ impl SetDisks {
|
||||
let object = object.clone();
|
||||
let version_id = version_id.clone();
|
||||
let slowtail_fault = slowtail_fault.clone();
|
||||
tokio::spawn(async move {
|
||||
AbortOnDropJoinHandle(tokio::spawn(async move {
|
||||
let response_start = observe.then(Instant::now);
|
||||
let result = if let Some(disk) = disk {
|
||||
Self::record_read_version_call(&object, disk_index);
|
||||
@@ -2949,7 +3039,7 @@ impl SetDisks {
|
||||
};
|
||||
let elapsed = response_start.map(|start| start.elapsed());
|
||||
(result, elapsed)
|
||||
})
|
||||
}))
|
||||
});
|
||||
|
||||
// Wait for all futures to complete
|
||||
@@ -3008,6 +3098,7 @@ impl SetDisks {
|
||||
read_data: bool,
|
||||
healing: bool,
|
||||
incl_free_versions: bool,
|
||||
allow_non_inline_data_read_early_stop: bool,
|
||||
default_parity_count: usize,
|
||||
allow_coalescing: bool,
|
||||
) -> disk::error::Result<(Vec<FileInfo>, Vec<Option<DiskError>>, MetadataFanoutDiagnostics)> {
|
||||
@@ -3038,6 +3129,8 @@ impl SetDisks {
|
||||
let mut scheduled_count = 0usize;
|
||||
let mut force_full_wait = false;
|
||||
let mut final_miss_reason_override = None;
|
||||
let mut non_inline_candidate_eligible = None;
|
||||
let mut single_pending_hedge_deadline = None;
|
||||
let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data);
|
||||
let spawn_read_version =
|
||||
|join_set: &mut JoinSet<(usize, disk::error::Result<FileInfo>, Duration)>, index: usize, disk: Option<DiskStore>| {
|
||||
@@ -3085,17 +3178,55 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
while let Some(result) = join_set.join_next().await {
|
||||
loop {
|
||||
let mut defer_pending_inline_data_shard = false;
|
||||
let result = if let Some(deadline) = single_pending_hedge_deadline.take() {
|
||||
tokio::select! {
|
||||
result = join_set.join_next() => result,
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
if bounded_fanout
|
||||
&& !force_full_wait
|
||||
&& join_set.len() == 1
|
||||
&& non_inline_candidate_eligible == Some(true)
|
||||
&& !accumulator.candidate_has_read_reserve()
|
||||
&& next_fanout_index < disks.len()
|
||||
{
|
||||
while next_fanout_index < disks.len() {
|
||||
let disk_index = fanout_order[next_fanout_index];
|
||||
next_fanout_index = next_fanout_index.saturating_add(1);
|
||||
if let Some(disk) = disks.get(disk_index).cloned() {
|
||||
spawn_read_version(&mut join_set, disk_index, disk);
|
||||
scheduled_count = scheduled_count.saturating_add(1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
join_set.join_next().await
|
||||
};
|
||||
let Some(result) = result else { break };
|
||||
match result {
|
||||
Ok((index, res, elapsed)) => match res {
|
||||
Ok(file_info) => {
|
||||
observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed));
|
||||
accumulator.observe_file_info(&file_info);
|
||||
if allow_non_inline_data_read_early_stop {
|
||||
accumulator.observe_file_info_at(index, &file_info);
|
||||
} else {
|
||||
accumulator.observe_file_info(&file_info);
|
||||
}
|
||||
if allow_non_inline_data_read_early_stop && non_inline_candidate_eligible.is_none() {
|
||||
non_inline_candidate_eligible =
|
||||
accumulator.candidate.as_ref().map(non_inline_data_read_candidate_is_safe);
|
||||
}
|
||||
if bounded_fanout
|
||||
&& read_data
|
||||
&& !force_full_wait
|
||||
&& let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info)
|
||||
&& !(non_inline_candidate_eligible == Some(true)
|
||||
&& reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE)
|
||||
{
|
||||
force_full_wait = true;
|
||||
final_miss_reason_override.get_or_insert(reason);
|
||||
@@ -3126,6 +3257,9 @@ impl SetDisks {
|
||||
{
|
||||
let should_return_early = if read_data {
|
||||
match accumulator.candidate.as_ref() {
|
||||
Some(_candidate) if non_inline_candidate_eligible == Some(true) => {
|
||||
accumulator.candidate_has_read_reserve()
|
||||
}
|
||||
Some(candidate) => match data_read_early_stop_inline_body_miss_reason(
|
||||
bucket.as_ref(),
|
||||
object.as_ref(),
|
||||
@@ -3196,12 +3330,37 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let pending_responses = join_set.len();
|
||||
let should_hedge_single_pending_data_read = read_data
|
||||
// Inline verification can still depend on a missing data shard;
|
||||
// issue one immediate spare when only that shard remains. The
|
||||
// non-inline path keeps its delayed hedge below to avoid healthy
|
||||
// reads paying speculative I/O before the candidate is classified.
|
||||
let should_hedge_single_pending_inline_read = read_data
|
||||
&& !force_full_wait
|
||||
&& !defer_pending_inline_data_shard
|
||||
&& pending_responses == 1
|
||||
&& non_inline_candidate_eligible != Some(true)
|
||||
&& accumulator.can_still_reach_early_stop_with_pending(pending_responses);
|
||||
if bounded_fanout && force_full_wait {
|
||||
// A non-inline plan must retain one extra matching shard as a
|
||||
// reconstruction reserve. Schedule that reserve only after the
|
||||
// candidate is known to be eligible, so inline GETs do not pay an
|
||||
// extra fanout and the healthy path remains allocation-free.
|
||||
let needs_non_inline_read_reserve = non_inline_candidate_eligible == Some(true)
|
||||
&& !accumulator.candidate_has_read_reserve()
|
||||
&& accumulator
|
||||
.candidate_read_reserve_target()
|
||||
.is_some_and(|reserve_target| scheduled_count < reserve_target || pending_responses == 0);
|
||||
if bounded_fanout
|
||||
&& !force_full_wait
|
||||
&& (needs_non_inline_read_reserve || should_hedge_single_pending_inline_read)
|
||||
&& next_fanout_index < disks.len()
|
||||
{
|
||||
let disk_index = fanout_order[next_fanout_index];
|
||||
if let Some(disk) = disks.get(disk_index).cloned() {
|
||||
spawn_read_version(&mut join_set, disk_index, disk);
|
||||
scheduled_count = scheduled_count.saturating_add(1);
|
||||
}
|
||||
next_fanout_index = next_fanout_index.saturating_add(1);
|
||||
} else if bounded_fanout && force_full_wait {
|
||||
while next_fanout_index < disks.len() {
|
||||
let disk_index = fanout_order[next_fanout_index];
|
||||
if let Some(disk) = disks.get(disk_index).cloned() {
|
||||
@@ -3213,8 +3372,7 @@ impl SetDisks {
|
||||
} else if bounded_fanout
|
||||
&& !defer_pending_inline_data_shard
|
||||
&& next_fanout_index < disks.len()
|
||||
&& (!accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
||||
|| should_hedge_single_pending_data_read)
|
||||
&& !accumulator.can_still_reach_early_stop_with_pending(pending_responses)
|
||||
{
|
||||
let disk_index = fanout_order[next_fanout_index];
|
||||
if let Some(disk) = disks.get(disk_index).cloned() {
|
||||
@@ -3223,6 +3381,17 @@ impl SetDisks {
|
||||
}
|
||||
next_fanout_index = next_fanout_index.saturating_add(1);
|
||||
}
|
||||
if bounded_fanout
|
||||
&& !force_full_wait
|
||||
&& !defer_pending_inline_data_shard
|
||||
&& join_set.len() == 1
|
||||
&& non_inline_candidate_eligible == Some(true)
|
||||
&& !accumulator.candidate_has_read_reserve()
|
||||
&& accumulator.can_still_reach_early_stop_with_pending(join_set.len())
|
||||
&& next_fanout_index < disks.len()
|
||||
{
|
||||
single_pending_hedge_deadline = Some(tokio::time::Instant::now() + NON_INLINE_SINGLE_PENDING_HEDGE_DELAY);
|
||||
}
|
||||
}
|
||||
|
||||
let accumulator_miss_reason = accumulator.final_miss_reason();
|
||||
@@ -6680,7 +6849,7 @@ pub(in crate::set_disk) mod rename_fanout_barrier_phase {
|
||||
/// Cross-process/black-box fault injection (toxiproxy, blackhole peers, 2-pool)
|
||||
/// is a later cluster-harness block, not this one.
|
||||
#[cfg(test)]
|
||||
pub(in crate::set_disk) mod rename_fanout_barrier {
|
||||
pub(crate) mod rename_fanout_barrier {
|
||||
use std::collections::HashMap;
|
||||
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex, OnceLock};
|
||||
@@ -6880,6 +7049,27 @@ mod tests {
|
||||
use tempfile::TempDir;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(codec_streaming_env)]
|
||||
fn non_inline_early_stop_is_mutually_exclusive_with_codec_rollout() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT", Some("on")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", Some("true")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", Some("true")),
|
||||
],
|
||||
|| assert!(!non_inline_data_read_early_stop_allowed(true, "bucket", "object")),
|
||||
);
|
||||
temp_env::with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
|
||||
("RUSTFS_GET_CODEC_STREAMING_ROLLOUT", Some("off")),
|
||||
],
|
||||
|| assert!(non_inline_data_read_early_stop_allowed(true, "bucket", "object")),
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_delete_owner_survives_waiter_cancellation() {
|
||||
let movement_gate = Arc::new(tokio::sync::RwLock::new(()));
|
||||
@@ -7307,6 +7497,90 @@ mod tests {
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_slowtail_fault_gate_stops_before_unneeded_tail() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "metadata-slowtail-gated-bucket";
|
||||
let object = "objects/metadata-slowtail-gated-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_mapped_metadata_fanout_fileinfo(&disks, bucket, object).await;
|
||||
let order = bounded_metadata_fanout_order(bucket, object, DISKS, 2);
|
||||
let slow_disk = *order.get(3).expect("four-disk fanout should have a deferred tail disk");
|
||||
let slow_disk_env = slow_disk.to_string();
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some(slow_disk_env.as_str())),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let read_with_data =
|
||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2);
|
||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_millis(500), read_with_data)
|
||||
.await
|
||||
.expect("gated metadata read should stop before the deferred slow tail")
|
||||
.expect("gated metadata fanout should resolve");
|
||||
assert!(parts_metadata.iter().filter(|fi| fi.name == object).count() >= 3);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert!(diagnostics.total_responses() < DISKS);
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 0);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn metadata_slowtail_fault_gate_hedges_an_initial_slow_data_shard() {
|
||||
const DISKS: usize = 4;
|
||||
let bucket = "metadata-slowtail-gated-initial-bucket";
|
||||
let object = "objects/metadata-slowtail-gated-initial-object";
|
||||
let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await;
|
||||
install_mapped_metadata_fanout_fileinfo(&disks, bucket, object).await;
|
||||
let order = bounded_metadata_fanout_order(bucket, object, DISKS, 2);
|
||||
let slow_disk = *order.get(1).expect("four-disk fanout should have an initial data disk");
|
||||
let spare_disk = *order.get(3).expect("four-disk fanout should have a spare disk");
|
||||
let slow_disk_env = slow_disk.to_string();
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("500")),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some(slow_disk_env.as_str())),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)),
|
||||
(ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let read_with_data =
|
||||
SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2);
|
||||
let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_millis(300), read_with_data)
|
||||
.await
|
||||
.expect("gated metadata read should hedge the initial slow shard")
|
||||
.expect("gated metadata fanout should resolve");
|
||||
assert!(parts_metadata.iter().filter(|fi| fi.name == object).count() >= 3);
|
||||
assert!(errs.iter().all(Option::is_none));
|
||||
assert!(diagnostics.total_responses() < DISKS);
|
||||
assert_eq!(calls.for_disk(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT, slow_disk), 1);
|
||||
assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, spare_disk), 1);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
|
||||
drop(dirs);
|
||||
}
|
||||
|
||||
/// Demo / regression guard for the backlog#1325 per-disk call counters.
|
||||
///
|
||||
/// The metadata fan-out issues each `read_version` inside its own
|
||||
@@ -7720,6 +7994,32 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn install_mapped_metadata_fanout_fileinfo(disks: &[Option<DiskStore>], bucket: &str, object: &str) {
|
||||
let version_id = Uuid::new_v4();
|
||||
let data_dir = Uuid::new_v4();
|
||||
let mod_time = OffsetDateTime::now_utc();
|
||||
let distribution = FileInfo::new(&metadata_distribution_key(bucket, object), 2, 2)
|
||||
.erasure
|
||||
.distribution;
|
||||
for (index, disk) in disks
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(index, disk)| disk.as_ref().map(|disk| (index, disk)))
|
||||
{
|
||||
disk.write_all(bucket, &format!("{object}/{data_dir}/part.1"), Bytes::from_static(b"x"))
|
||||
.await
|
||||
.expect("part data should be installed on every disk");
|
||||
let mut file_info = valid_metadata_fanout_fileinfo(bucket, object, version_id, data_dir, mod_time);
|
||||
file_info.erasure.distribution = distribution.clone();
|
||||
file_info.erasure.index = *distribution
|
||||
.get(index)
|
||||
.expect("mapped metadata distribution should cover every disk");
|
||||
disk.write_metadata(bucket, bucket, object, file_info)
|
||||
.await
|
||||
.expect("mapped metadata should be installed on every disk");
|
||||
}
|
||||
}
|
||||
|
||||
async fn inline_metadata_fanout_fileinfos_with_mode(
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
@@ -10342,6 +10642,41 @@ mod tests {
|
||||
assert_eq!(accumulator.candidate_latest_quorum(&impossible_parity), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_quorum_accumulator_tracks_mapped_shards_and_requires_a_reserve() {
|
||||
let version_id = Uuid::new_v4();
|
||||
let data_dir = Uuid::new_v4();
|
||||
let base = valid_metadata_fanout_fileinfo("bucket", "object", version_id, data_dir, OffsetDateTime::now_utc());
|
||||
let distribution = base.erasure.distribution.clone();
|
||||
let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true);
|
||||
|
||||
for (disk_index, &erasure_index) in distribution.iter().take(2).enumerate() {
|
||||
let mut file_info = base.clone();
|
||||
file_info.erasure.index = erasure_index;
|
||||
accumulator.observe_file_info_at(disk_index, &file_info);
|
||||
}
|
||||
assert!(
|
||||
!accumulator.candidate_has_read_reserve(),
|
||||
"data quorum without parity reserve must not early-stop"
|
||||
);
|
||||
|
||||
let mut mismatched = base.clone();
|
||||
mismatched.erasure.index = distribution[3];
|
||||
accumulator.observe_file_info_at(2, &mismatched);
|
||||
assert!(
|
||||
!accumulator.candidate_has_read_reserve(),
|
||||
"mapped index mismatch must not count as a reserve"
|
||||
);
|
||||
|
||||
let mut reserve = base;
|
||||
reserve.erasure.index = distribution[2];
|
||||
accumulator.observe_file_info_at(2, &reserve);
|
||||
assert!(
|
||||
accumulator.candidate_has_read_reserve(),
|
||||
"one matching reserve shard should complete the read reserve"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_quorum_accumulator_treats_invalid_default_parity_as_full_fanout() {
|
||||
let accumulator = MetadataQuorumAccumulator::new(2, 2, true);
|
||||
|
||||
@@ -42,7 +42,8 @@ use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::bucket::metadata_sys::ObjectLockConfigState;
|
||||
use crate::bucket::object_lock::objectlock_sys::{
|
||||
check_object_lock_for_deletion_with_state, check_retention_for_modification, replication_write_may_pass_worm_gate,
|
||||
check_object_lock_for_deletion_with_state, check_retention_for_modification, replication_delete_may_bypass_governance,
|
||||
replication_write_may_pass_worm_gate,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::bucket::replication::ReplicationState;
|
||||
@@ -773,6 +774,14 @@ const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false;
|
||||
const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE";
|
||||
const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true;
|
||||
|
||||
// Opt-in non-inline data-read quorum early-stop rollout (backlog#1309). The
|
||||
// existing metadata fanout still reads data-bearing metadata; this gate only
|
||||
// permits a safe plain single-part candidate to stop before the full fanout.
|
||||
// Keep it opt-in until the Linux multi-node slow-tail and small-inline cost
|
||||
// gates are complete. The environment name is retained for compatibility.
|
||||
const ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE: &str = "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE";
|
||||
const DEFAULT_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE: bool = false;
|
||||
|
||||
const ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: &str = "RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT";
|
||||
const DEFAULT_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT: bool = true;
|
||||
|
||||
@@ -849,7 +858,7 @@ static OBJECT_LOCK_DIAG_ENABLED: OnceLock<bool> = OnceLock::new();
|
||||
|
||||
mod core;
|
||||
#[cfg(test)]
|
||||
pub(crate) use core::io_primitives::disk_call_counters;
|
||||
pub(crate) use core::io_primitives::{ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, disk_call_counters, rename_fanout_barrier};
|
||||
mod ctx;
|
||||
mod metadata;
|
||||
mod ops;
|
||||
@@ -886,8 +895,11 @@ struct OwnedGetObjectFileInfo {
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
late_metadata_fanout_disks: Option<Vec<Option<DiskStore>>>,
|
||||
}
|
||||
|
||||
type OwnedGetObjectFileInfoParts = (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>, Option<Vec<Option<DiskStore>>>);
|
||||
|
||||
impl GetObjectFileInfo {
|
||||
fn owned(fi: FileInfo, parts_metadata: Vec<FileInfo>, online_disks: Vec<Option<DiskStore>>) -> Self {
|
||||
Self {
|
||||
@@ -895,6 +907,24 @@ impl GetObjectFileInfo {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
late_metadata_fanout_disks: None,
|
||||
}),
|
||||
shared: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn owned_with_late_metadata_fanout(
|
||||
fi: FileInfo,
|
||||
parts_metadata: Vec<FileInfo>,
|
||||
online_disks: Vec<Option<DiskStore>>,
|
||||
late_metadata_fanout_disks: Vec<Option<DiskStore>>,
|
||||
) -> Self {
|
||||
Self {
|
||||
owned: Some(OwnedGetObjectFileInfo {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
late_metadata_fanout_disks: Some(late_metadata_fanout_disks),
|
||||
}),
|
||||
shared: None,
|
||||
}
|
||||
@@ -931,19 +961,28 @@ impl GetObjectFileInfo {
|
||||
}
|
||||
}
|
||||
|
||||
fn has_late_metadata_fanout(&self) -> bool {
|
||||
self.owned
|
||||
.as_ref()
|
||||
.is_some_and(|snapshot| snapshot.late_metadata_fanout_disks.is_some())
|
||||
}
|
||||
|
||||
fn into_owned(self) -> (FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>) {
|
||||
let (fi, parts_metadata, online_disks, _) = self.into_owned_with_late_metadata_fanout();
|
||||
(fi, parts_metadata, online_disks)
|
||||
}
|
||||
|
||||
fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts {
|
||||
match (self.owned, self.shared) {
|
||||
(Some(snapshot), None) => {
|
||||
let OwnedGetObjectFileInfo {
|
||||
fi,
|
||||
parts_metadata,
|
||||
online_disks,
|
||||
} = snapshot;
|
||||
(fi, parts_metadata, online_disks)
|
||||
}
|
||||
(Some(snapshot), None) => (
|
||||
snapshot.fi,
|
||||
snapshot.parts_metadata,
|
||||
snapshot.online_disks,
|
||||
snapshot.late_metadata_fanout_disks,
|
||||
),
|
||||
(None, Some(entry)) => match Arc::try_unwrap(entry) {
|
||||
Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks),
|
||||
Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone()),
|
||||
Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, None),
|
||||
Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone(), None),
|
||||
},
|
||||
_ => unreachable!("GET metadata snapshot representation must be exclusive"),
|
||||
}
|
||||
@@ -1030,14 +1069,19 @@ mod prepared_get_object_metadata_tests {
|
||||
const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10);
|
||||
|
||||
fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String {
|
||||
object_with_initial_data_shards_for_geometry(bucket, prefix, 4, 2)
|
||||
}
|
||||
|
||||
fn object_with_initial_data_shards_for_geometry(bucket: &str, prefix: &str, total_disks: usize, parity: usize) -> String {
|
||||
(0..1000)
|
||||
.map(|index| format!("{prefix}-{index}.bin"))
|
||||
.find(|name| {
|
||||
let order = bounded_metadata_fanout_order(bucket, name, 4, 2);
|
||||
let distribution = FileInfo::new(&[bucket, name].join("/"), 2, 2).erasure.distribution;
|
||||
let mut seen = [false; 2];
|
||||
for disk_index in order.into_iter().take(3) {
|
||||
if let Some(block_index @ 1..=2) = distribution.get(disk_index).copied() {
|
||||
let order = bounded_metadata_fanout_order(bucket, name, total_disks, parity);
|
||||
let data = total_disks.saturating_sub(parity);
|
||||
let distribution = FileInfo::new(&[bucket, name].join("/"), data, parity).erasure.distribution;
|
||||
let mut seen = vec![false; data];
|
||||
for disk_index in order.into_iter().take(total_disks.saturating_sub(parity).saturating_add(1)) {
|
||||
if let Some(block_index) = distribution.get(disk_index).copied().filter(|index| *index <= data) {
|
||||
seen[block_index - 1] = true;
|
||||
}
|
||||
}
|
||||
@@ -1168,6 +1212,495 @@ mod prepared_get_object_metadata_tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn non_inline_data_read_early_stop_uses_quorum_plan() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "non-inline-read-plan";
|
||||
let object = object_with_initial_data_shards(bucket, "non-inline-object");
|
||||
let payload = vec![0x5a; 2 * 1024 * 1024];
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
reset_test_get_object_reader_path();
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("quorum GET reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("quorum GET body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
assert!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION) < 4,
|
||||
"non-inline quorum GET should retain a reconstruction reserve"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn non_inline_two_phase_read_fetches_late_parity_after_two_selected_shards_fail() {
|
||||
let (dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "non-inline-read-late-parity";
|
||||
let object = object_with_initial_data_shards(bucket, "late-parity-object");
|
||||
let payload = vec![0x5a; 1024 * 1024];
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let order = bounded_metadata_fanout_order(bucket, &object, 4, 2);
|
||||
let distribution = FileInfo::new(&[bucket, object.as_str()].join("/"), 2, 2).erasure.distribution;
|
||||
assert!(
|
||||
order.iter().take(2).all(|disk_index| distribution[*disk_index] <= 2),
|
||||
"the two failed selected shards must be data shards"
|
||||
);
|
||||
assert!(
|
||||
distribution[order[3]] > 2,
|
||||
"the metadata shard omitted by the plan must be healthy parity"
|
||||
);
|
||||
for disk_index in order.iter().take(2) {
|
||||
let object_dir = dirs[*disk_index].path().join(bucket).join(&object);
|
||||
let data_dir = std::fs::read_dir(&object_dir)
|
||||
.expect("object directory should be readable")
|
||||
.find_map(|entry| {
|
||||
let entry = entry.expect("object directory entry should be readable");
|
||||
entry
|
||||
.file_type()
|
||||
.expect("object directory entry type should be readable")
|
||||
.is_dir()
|
||||
.then(|| entry.path())
|
||||
})
|
||||
.expect("object data directory should exist");
|
||||
let part_path = data_dir.join("part.1");
|
||||
let mut shard = std::fs::read(&part_path).expect("selected data shard should be readable before corruption");
|
||||
shard[0] ^= 0xff;
|
||||
std::fs::write(part_path, shard).expect("selected data shard should be corrupted after metadata was written");
|
||||
}
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("two-phase GET should recover using late parity");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("late parity should restore the exact GET body");
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 7);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn non_inline_two_phase_read_fetches_late_parity_when_selected_parts_are_missing() {
|
||||
let (dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "non-inline-read-late-parity-missing";
|
||||
let object = object_with_initial_data_shards(bucket, "late-parity-missing-object");
|
||||
let payload = vec![0x3c; 1024 * 1024];
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let order = bounded_metadata_fanout_order(bucket, &object, 4, 2);
|
||||
for disk_index in order.iter().take(2) {
|
||||
let object_dir = dirs[*disk_index].path().join(bucket).join(&object);
|
||||
let data_dir = std::fs::read_dir(&object_dir)
|
||||
.expect("object directory should be readable")
|
||||
.find_map(|entry| {
|
||||
let entry = entry.expect("object directory entry should be readable");
|
||||
entry
|
||||
.file_type()
|
||||
.expect("entry type should be readable")
|
||||
.is_dir()
|
||||
.then(|| entry.path())
|
||||
})
|
||||
.expect("object data directory should exist");
|
||||
std::fs::remove_file(data_dir.join("part.1")).expect("selected data shard should be removed");
|
||||
}
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("two-phase GET should recover using late parity");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("late parity should restore the exact GET body");
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 7);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn four_data_two_parity_two_phase_read_recovers_one_failed_data_shard() {
|
||||
let (dirs, set_disks) = make_local_set_disks(6, 2).await;
|
||||
let bucket = "four-data-two-parity-late-read";
|
||||
let object = object_with_initial_data_shards_for_geometry(bucket, "one-failed-data", 4, 2);
|
||||
let payload = vec![0x7a; 1024 * 1024];
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let order = bounded_metadata_fanout_order(bucket, &object, 6, 2);
|
||||
let distribution = FileInfo::new(&[bucket, object.as_str()].join("/"), 4, 2).erasure.distribution;
|
||||
let failed_disk = *order
|
||||
.iter()
|
||||
.take(4)
|
||||
.find(|disk_index| distribution[**disk_index] <= 4)
|
||||
.expect("initial fanout should include a data shard");
|
||||
assert!(
|
||||
order.iter().take(4).all(|disk_index| distribution[*disk_index] <= 4),
|
||||
"initial fanout should cover all four data shards"
|
||||
);
|
||||
assert!(distribution[order[5]] > 4, "the final deferred metadata shard should be parity");
|
||||
|
||||
let object_dir = dirs[failed_disk].path().join(bucket).join(&object);
|
||||
let data_dir = std::fs::read_dir(&object_dir)
|
||||
.expect("object directory should be readable")
|
||||
.find_map(|entry| {
|
||||
let entry = entry.expect("object directory entry should be readable");
|
||||
entry
|
||||
.file_type()
|
||||
.expect("object directory entry type should be readable")
|
||||
.is_dir()
|
||||
.then(|| entry.path())
|
||||
})
|
||||
.expect("object data directory should exist");
|
||||
let part_path = data_dir.join("part.1");
|
||||
let mut shard = std::fs::read(&part_path).expect("selected data shard should be readable before corruption");
|
||||
shard[0] ^= 0xff;
|
||||
std::fs::write(part_path, shard).expect("selected data shard should be corrupted after metadata was written");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("two-phase GET should recover with one failed data shard");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("late parity should restore the exact GET body");
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 11);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn four_data_two_parity_two_phase_read_rejects_below_read_quorum() {
|
||||
let (dirs, set_disks) = make_local_set_disks(6, 2).await;
|
||||
let bucket = "four-data-two-parity-quorum-minus-one";
|
||||
let object = object_with_initial_data_shards_for_geometry(bucket, "quorum-minus-one", 4, 2);
|
||||
let payload = vec![0x4b; 1024 * 1024];
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload);
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
let order = bounded_metadata_fanout_order(bucket, &object, 6, 2);
|
||||
for disk_index in order.iter().take(3) {
|
||||
let object_dir = dirs[*disk_index].path().join(bucket).join(&object);
|
||||
let data_dir = std::fs::read_dir(&object_dir)
|
||||
.expect("object directory should be readable")
|
||||
.find_map(|entry| {
|
||||
let entry = entry.expect("object directory entry should be readable");
|
||||
entry
|
||||
.file_type()
|
||||
.expect("entry type should be readable")
|
||||
.is_dir()
|
||||
.then(|| entry.path())
|
||||
})
|
||||
.expect("object data directory should exist");
|
||||
std::fs::remove_file(data_dir.join("part.1")).expect("selected shard should be removed");
|
||||
}
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let result = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await;
|
||||
assert!(result.is_err(), "quorum-minus-one read must fail closed without exposing a body");
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn non_inline_data_read_early_stop_keeps_reserve_on_unequal_layout() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(6, 2).await;
|
||||
let bucket = "non-inline-read-reserve";
|
||||
let object = object_with_initial_data_shards_for_geometry(bucket, "reserve-object", 6, 2);
|
||||
let payload = vec![0x5a; 2 * 1024 * 1024];
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("object should be written");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("quorum GET reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("quorum GET body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
5,
|
||||
"the unequal layout should schedule exactly one reserve beyond its data quorum"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn non_inline_data_read_early_stop_preserves_inline_path() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "non-inline-read-plan-inline";
|
||||
let object = object_with_initial_data_shards(bucket, "inline-object");
|
||||
let payload = b"quorum inline payload".repeat(256);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("inline object should be written");
|
||||
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("inline GET reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("inline GET body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(
|
||||
test_get_object_reader_path_id(),
|
||||
3,
|
||||
"inline GET should retain the direct inline reader path"
|
||||
);
|
||||
assert!(calls.total(disk_call_counters::KIND_READ_VERSION) <= 4);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
async fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() {
|
||||
let (_dirs, set_disks) = make_local_set_disks(6, 2).await;
|
||||
let bucket = "inline-read-plan-unequal";
|
||||
let object = object_with_initial_data_shards_for_geometry(bucket, "inline-object", 6, 2);
|
||||
let payload = b"inline quorum payload".repeat(256);
|
||||
let opts = ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let mut put_reader = PutObjReader::from_vec(payload.clone());
|
||||
set_disks
|
||||
.put_object(bucket, &object, &mut put_reader, &opts)
|
||||
.await
|
||||
.expect("inline object should be written");
|
||||
|
||||
let read_once = |enabled: bool| {
|
||||
let set_disks = Arc::clone(&set_disks);
|
||||
let bucket = bucket.to_string();
|
||||
let object = object.clone();
|
||||
let payload = payload.clone();
|
||||
let opts = opts.clone();
|
||||
async move {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(
|
||||
"RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE",
|
||||
Some(if enabled { "true" } else { "false" }),
|
||||
),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")),
|
||||
("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")),
|
||||
],
|
||||
async {
|
||||
let calls = disk_call_counters::observe(&object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(&bucket, &object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("inline GET reader should open");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("inline GET body should stream");
|
||||
assert_eq!(restored, payload);
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION)
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
};
|
||||
|
||||
let gate_off_calls = read_once(false).await;
|
||||
let gate_on_calls = read_once(true).await;
|
||||
assert_eq!(gate_on_calls, gate_off_calls, "inline gate must not add reserve fanout");
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial(body_cache_hook)]
|
||||
fn inline_data_read_early_stop_defaults_return_exact_body() {
|
||||
@@ -1914,6 +2447,26 @@ fn is_get_metadata_data_read_early_stop_enabled() -> bool {
|
||||
}
|
||||
}
|
||||
|
||||
fn is_get_metadata_non_inline_data_read_early_stop_enabled() -> bool {
|
||||
#[cfg(test)]
|
||||
{
|
||||
rustfs_utils::get_env_bool(
|
||||
ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE,
|
||||
DEFAULT_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE,
|
||||
)
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
static CACHED: OnceLock<bool> = OnceLock::new();
|
||||
*CACHED.get_or_init(|| {
|
||||
rustfs_utils::get_env_bool(
|
||||
ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE,
|
||||
DEFAULT_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE,
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn is_get_metadata_early_stop_bounded_fanout_enabled() -> bool {
|
||||
#[cfg(test)]
|
||||
{
|
||||
@@ -2090,6 +2643,15 @@ fn should_use_codec_streaming(config: GetCodecStreamingConfig, bucket: &str, obj
|
||||
is_optimization_enabled_for_request(config.enabled, config.rollout_pct, bucket, object)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn codec_streaming_rollout_applies(bucket: &str, object: &str) -> bool {
|
||||
let config = get_codec_streaming_config();
|
||||
config.enabled
|
||||
&& config.body_compat_confirmed
|
||||
&& config.header_compat_confirmed
|
||||
&& config.rollout.is_opted_in()
|
||||
&& should_use_codec_streaming(config, bucket, object)
|
||||
}
|
||||
|
||||
/// Should this specific request use metadata early-stop?
|
||||
#[allow(
|
||||
dead_code,
|
||||
@@ -5017,10 +5579,15 @@ async fn check_object_lock_delete(
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// An authorized replicated version purge already passed this gate on the
|
||||
// source with the bypass it carried there, so it clears GOVERNANCE
|
||||
// retention here without the header; COMPLIANCE and legal hold still
|
||||
// block below (see `replication_delete_may_bypass_governance`, #6850).
|
||||
let bypass_governance = opts
|
||||
.object_lock_delete
|
||||
.as_ref()
|
||||
.is_some_and(|delete_opts| delete_opts.bypass_governance);
|
||||
.is_some_and(|delete_opts| delete_opts.bypass_governance)
|
||||
|| replication_delete_may_bypass_governance(opts);
|
||||
let blocked = match opts.object_lock_config_snapshot.as_deref() {
|
||||
Some(snapshot) => check_object_lock_for_deletion_with_state(snapshot.state(), obj_info, bypass_governance)?.is_some(),
|
||||
None => {
|
||||
@@ -6241,6 +6808,7 @@ mod tests {
|
||||
use crate::object_api::BLOCK_SIZE_V2;
|
||||
use crate::object_api::ObjectInfo;
|
||||
use crate::set_disk::core::io_primitives::rename_fanout_barrier;
|
||||
use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause};
|
||||
use crate::storage_api_contracts::{
|
||||
heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart,
|
||||
object::ObjectOperations as _,
|
||||
@@ -11020,6 +11588,100 @@ mod tests {
|
||||
.expect("versioned delete marker creation should not delete the locked version");
|
||||
}
|
||||
|
||||
fn governance_retained_obj_info() -> ObjectInfo {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::GOVERNANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
);
|
||||
ObjectInfo {
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn explicit_version_delete_opts(replication_request: bool) -> ObjectOptions {
|
||||
ObjectOptions {
|
||||
version_id: Some(Uuid::new_v4().to_string()),
|
||||
versioned: true,
|
||||
replication_request,
|
||||
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
// Issue #6850: a replicated version purge carries no bypass header, so the
|
||||
// GOVERNANCE gate must honor the source's already-judged bypass instead of
|
||||
// keeping the sites permanently diverged.
|
||||
#[tokio::test]
|
||||
async fn test_check_object_lock_delete_allows_replicated_governance_version_purge() {
|
||||
let obj_info = governance_retained_obj_info();
|
||||
let opts = explicit_version_delete_opts(true);
|
||||
|
||||
check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts)
|
||||
.await
|
||||
.expect("an authorized replicated version purge must pass GOVERNANCE retention (#6850)");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_check_object_lock_delete_blocks_plain_governance_version_delete_without_bypass() {
|
||||
let obj_info = governance_retained_obj_info();
|
||||
let opts = explicit_version_delete_opts(false);
|
||||
|
||||
let err = check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts)
|
||||
.await
|
||||
.expect_err("a plain client delete without the bypass header must stay blocked by GOVERNANCE retention");
|
||||
|
||||
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_check_object_lock_delete_blocks_replicated_compliance_version_purge() {
|
||||
let retain_until = OffsetDateTime::now_utc() + Duration::from_secs(60 * 60 * 24 * 60);
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(
|
||||
X_AMZ_OBJECT_LOCK_MODE.as_str().to_string(),
|
||||
s3s::dto::ObjectLockRetentionMode::COMPLIANCE.to_string(),
|
||||
);
|
||||
user_defined.insert(
|
||||
X_AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE.as_str().to_string(),
|
||||
retain_until.format(&time::format_description::well_known::Rfc3339).unwrap(),
|
||||
);
|
||||
let obj_info = ObjectInfo {
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = explicit_version_delete_opts(true);
|
||||
|
||||
let err = check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts)
|
||||
.await
|
||||
.expect_err("the source gate can never purge through COMPLIANCE, so a replicated purge fails closed");
|
||||
|
||||
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_check_object_lock_delete_blocks_replicated_legal_hold_version_purge() {
|
||||
let mut user_defined = HashMap::new();
|
||||
user_defined.insert(X_AMZ_OBJECT_LOCK_LEGAL_HOLD.as_str().to_string(), "ON".to_string());
|
||||
let obj_info = ObjectInfo {
|
||||
user_defined: Arc::new(user_defined),
|
||||
..Default::default()
|
||||
};
|
||||
let opts = explicit_version_delete_opts(true);
|
||||
|
||||
let err = check_object_lock_delete(&bootstrap_ctx(), "bucket", "object", &obj_info, &opts)
|
||||
.await
|
||||
.expect_err("the source gate can never purge through a legal hold, so a replicated purge fails closed");
|
||||
|
||||
assert!(matches!(err, StorageError::PrefixAccessDenied(_, _)));
|
||||
}
|
||||
|
||||
// backlog#929 (HP-8): the delete_objects per-object stat is gated on the
|
||||
// bucket object-lock configuration. Lock-enabled buckets (either legacy
|
||||
// lock_enabled flag or an enabled ObjectLockConfiguration) and unknown
|
||||
@@ -11999,6 +12661,7 @@ mod tests {
|
||||
0,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
metrics_size_bucket,
|
||||
@@ -12111,6 +12774,7 @@ mod tests {
|
||||
0,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
metrics_size_bucket,
|
||||
@@ -12681,6 +13345,111 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn multipart_streaming_get_blocks_overwrite_across_part_boundary() {
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true")),
|
||||
(ENV_RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH, Some("false")),
|
||||
],
|
||||
async {
|
||||
let set_disks = make_local_bucket_test_set_disks().await;
|
||||
let bucket = "snapshot-multipart-overwrite";
|
||||
let object = "object";
|
||||
let part_size = usize::try_from(GLOBAL_MIN_PART_SIZE.as_u64()).expect("minimum part size should fit usize");
|
||||
let first_part = vec![0x41; part_size];
|
||||
let second_part = vec![0x42; part_size];
|
||||
let replacement = vec![0x43; first_part.len() + second_part.len()];
|
||||
let opts = ObjectOptions::default();
|
||||
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bucket should be created");
|
||||
let upload = set_disks
|
||||
.new_multipart_upload(bucket, object, &opts)
|
||||
.await
|
||||
.expect("multipart upload should be created");
|
||||
let mut completed_parts = Vec::with_capacity(2);
|
||||
for (part_num, body) in [(1, &first_part), (2, &second_part)] {
|
||||
let mut reader = PutObjReader::from_vec(body.clone());
|
||||
let part = set_disks
|
||||
.put_object_part(bucket, object, &upload.upload_id, part_num, &mut reader, &opts)
|
||||
.await
|
||||
.expect("multipart part should be written");
|
||||
completed_parts.push(CompletePart {
|
||||
part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
let completed = Arc::clone(&set_disks)
|
||||
.complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &opts)
|
||||
.await
|
||||
.expect("multipart upload should complete");
|
||||
assert!(completed.is_multipart());
|
||||
|
||||
let mut snapshot = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("multipart snapshot reader should open");
|
||||
let overwrite_set = Arc::clone(&set_disks);
|
||||
let overwrite_opts = opts.clone();
|
||||
let overwrite_body = replacement.clone();
|
||||
let commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
|
||||
let overwrite = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(overwrite_body);
|
||||
overwrite_set.put_object(bucket, object, &mut reader, &overwrite_opts).await
|
||||
});
|
||||
commit_barrier.wait_until_paused().await;
|
||||
commit_barrier.release_and_wait_until_namespace_pending().await;
|
||||
assert!(
|
||||
!commit_barrier.namespace_acquired(),
|
||||
"overwrite must wait for the multipart response's read lock"
|
||||
);
|
||||
|
||||
let mut restored_first = vec![0; first_part.len()];
|
||||
snapshot
|
||||
.stream
|
||||
.read_exact(&mut restored_first)
|
||||
.await
|
||||
.expect("the first multipart part should stream");
|
||||
assert_eq!(restored_first, first_part);
|
||||
assert!(
|
||||
!commit_barrier.namespace_acquired() && !overwrite.is_finished(),
|
||||
"overwrite must remain blocked at the first/second part boundary"
|
||||
);
|
||||
|
||||
let mut restored_second = Vec::new();
|
||||
snapshot
|
||||
.stream
|
||||
.read_to_end(&mut restored_second)
|
||||
.await
|
||||
.expect("the second multipart part should stream");
|
||||
assert_eq!(restored_second, second_part);
|
||||
tokio::time::timeout(Duration::from_secs(5), overwrite)
|
||||
.await
|
||||
.expect("overwrite should proceed after multipart EOF")
|
||||
.expect("overwrite task should join")
|
||||
.expect("overwrite should succeed");
|
||||
|
||||
let mut latest = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &opts)
|
||||
.await
|
||||
.expect("replacement reader should open");
|
||||
let mut latest_body = Vec::new();
|
||||
latest
|
||||
.stream
|
||||
.read_to_end(&mut latest_body)
|
||||
.await
|
||||
.expect("replacement should stream");
|
||||
assert_eq!(latest_body, replacement);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
#[serial]
|
||||
async fn streaming_get_blocks_concurrent_delete_until_eof() {
|
||||
|
||||
@@ -36,6 +36,20 @@ const EVENT_HEAL_OBJECT_RENAME: &str = "heal_object_rename";
|
||||
const HEAL_RENAME_INCOMPLETE: &str = "heal rename incomplete";
|
||||
const READ_REPAIR_DATA_PHASE_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(60 * 60);
|
||||
|
||||
fn heal_drive_state_for_error(error: &DiskError) -> DriveState {
|
||||
match error {
|
||||
DiskError::DiskNotFound | DiskError::RemoteClientUnavailable(_) => DriveState::Offline,
|
||||
DiskError::FaultyDisk | DiskError::FaultyRemoteDisk => DriveState::Faulty,
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::VolumeNotFound
|
||||
| DiskError::PartMissingOrCorrupt
|
||||
| DiskError::OutdatedXLMeta => DriveState::Missing,
|
||||
DiskError::FileCorrupt => DriveState::Corrupt,
|
||||
_ => DriveState::Unknown(error.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static HEAL_RENAME_FAILURES: std::sync::Mutex<Vec<(String, String, usize)>> = std::sync::Mutex::new(Vec::new());
|
||||
|
||||
@@ -892,16 +906,7 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let drive_state = match reason {
|
||||
Some(err) => match err {
|
||||
DiskError::DiskNotFound => DriveState::Offline.to_string(),
|
||||
DiskError::FileNotFound
|
||||
| DiskError::FileVersionNotFound
|
||||
| DiskError::VolumeNotFound
|
||||
| DiskError::PartMissingOrCorrupt
|
||||
| DiskError::OutdatedXLMeta => DriveState::Missing.to_string(),
|
||||
DiskError::FileCorrupt => DriveState::Corrupt.to_string(),
|
||||
_ => DriveState::Unknown(err.to_string()).to_string(),
|
||||
},
|
||||
Some(err) => heal_drive_state_for_error(&err).to_string(),
|
||||
None => DriveState::Ok.to_string(),
|
||||
};
|
||||
result.before.drives.push(HealDriveInfo {
|
||||
@@ -2673,6 +2678,17 @@ mod heal_result_report_tests {
|
||||
assert!(!super::metadata_less_part_file("xl.meta"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_heal_errors_use_stable_drive_states() {
|
||||
for error in [DiskError::FaultyDisk, DiskError::FaultyRemoteDisk] {
|
||||
assert_eq!(super::heal_drive_state_for_error(&error).to_string(), DriveState::Faulty.to_string());
|
||||
}
|
||||
assert_eq!(
|
||||
super::heal_drive_state_for_error(&DiskError::RemoteClientUnavailable("peer restarting".to_string())).to_string(),
|
||||
DriveState::Offline.to_string()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_repair_commit_fingerprint_tracks_commit_identity_only() {
|
||||
let data_dir = Uuid::parse_str("11111111-1111-1111-1111-111111111111").expect("data dir should parse");
|
||||
@@ -2751,6 +2767,26 @@ mod heal_result_report_tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn remove_current_object_part(temp_dir: &TempDir, bucket: &str, object: &str) -> std::io::Result<()> {
|
||||
let object_dir = temp_dir.path().join(bucket).join(object);
|
||||
let mut entries = tokio::fs::read_dir(&object_dir).await?;
|
||||
while let Some(entry) = entries.next_entry().await? {
|
||||
if !entry.file_type().await?.is_dir() {
|
||||
continue;
|
||||
}
|
||||
let part = entry.path().join("part.1");
|
||||
match tokio::fs::remove_file(&part).await {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
Err(std::io::Error::new(
|
||||
std::io::ErrorKind::NotFound,
|
||||
format!("no current part.1 found under {}", object_dir.display()),
|
||||
))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn heal_writer_error_summary_redacts_io_message() {
|
||||
let error = DiskError::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "/sensitive/storage/path"));
|
||||
@@ -2783,21 +2819,13 @@ mod heal_result_report_tests {
|
||||
.read_version("", &bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("source metadata should be readable");
|
||||
let data_dir = source.data_dir.expect("non-inline source should have a data directory");
|
||||
let mut target_slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1];
|
||||
target_slots.sort_unstable();
|
||||
|
||||
for index in [0, 1] {
|
||||
tokio::fs::remove_file(
|
||||
temp_dirs[index]
|
||||
.path()
|
||||
.join(&bucket)
|
||||
.join(object)
|
||||
.join(data_dir.to_string())
|
||||
.join("part.1"),
|
||||
)
|
||||
.await
|
||||
.expect("target shard should be removed before heal");
|
||||
remove_current_object_part(&temp_dirs[index], &bucket, object)
|
||||
.await
|
||||
.expect("target shard should be removed before heal");
|
||||
}
|
||||
|
||||
let failed_slots = &target_slots[..failed_target_count];
|
||||
@@ -3053,9 +3081,20 @@ mod heal_result_report_tests {
|
||||
|
||||
let payload = vec![0x5a; 1024 * 1024];
|
||||
let mut reader = PutObjReader::from_vec(payload);
|
||||
set.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
// This fixture removes physical shards immediately after PUT. A
|
||||
// lock-owning PUT may quorum-ack before its rename tail drains, so
|
||||
// keep the isolated setup on the full-fanout commit path.
|
||||
set.put_object(
|
||||
&bucket,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("source object should be written");
|
||||
let source = disks[2]
|
||||
.read_version("", &bucket, object, "", &ReadOptions::default())
|
||||
.await
|
||||
|
||||
@@ -27,18 +27,18 @@ use super::super::MetadataCacheInvalidationProbe;
|
||||
#[cfg(test)]
|
||||
use super::super::capacity_scope_from_disks;
|
||||
use super::super::{
|
||||
AMZ_STORAGE_CLASS, Arc, Bytes, CompletePart, Cursor, DiskError, DiskStore, EVENT_SET_DISK_MULTIPART, Error, FileInfo,
|
||||
GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet, HealChannelPriority, Instant, LOG_COMPONENT_ECSTORE,
|
||||
LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo, MAX_PARTS_COUNT, MULTIPART_WRITE_QUORUM_RENAME_PART,
|
||||
MULTIPART_WRITE_QUORUM_UPLOAD_METADATA, MULTIPART_WRITE_QUORUM_WRITER_SETUP, MultipartInfo, MultipartUploadResult,
|
||||
MultipartWriteQuorumContext, NamespaceLockFence, OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectLockDiagGuard, ObjectOptions,
|
||||
ObjectPartInfo, OffsetDateTime, PartInfo, PutObjReader, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY, Result, SLASH_SEPARATOR, SUFFIX_ACTUAL_OBJECT_SIZE_CAP,
|
||||
SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID, SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC,
|
||||
SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError, Uuid, WriteLayout,
|
||||
check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding, complete_multipart_part_error,
|
||||
complete_multipart_part_error_result, complete_part_checksum, completed_multipart_object_part, contains_key_str,
|
||||
create_bitrot_writer, debug, disk, error, get_complete_multipart_md5, get_header_map, get_str, insert_str,
|
||||
AMZ_STORAGE_CLASS, Arc, Bytes, CompletePart, Cursor, DATA_MOVEMENT_MULTIPART_PREFIX, DiskError, DiskStore,
|
||||
EVENT_SET_DISK_MULTIPART, Error, FileInfo, GLOBAL_MIN_PART_SIZE, HashAlgorithm, HashMap, HashReader, HashSet,
|
||||
HealChannelPriority, Instant, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_SET_DISK, ListMultipartsInfo, ListPartsInfo,
|
||||
MAX_PARTS_COUNT, MULTIPART_WRITE_QUORUM_RENAME_PART, MULTIPART_WRITE_QUORUM_UPLOAD_METADATA,
|
||||
MULTIPART_WRITE_QUORUM_WRITER_SETUP, MultipartInfo, MultipartUploadResult, MultipartWriteQuorumContext, NamespaceLockFence,
|
||||
OBJECT_OP_IGNORED_ERRS, ObjectInfo, ObjectLockDiagGuard, ObjectOptions, ObjectPartInfo, OffsetDateTime, PartInfo,
|
||||
PutObjReader, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET, RUSTFS_MULTIPART_BUCKET_KEY, RUSTFS_MULTIPART_OBJECT_KEY,
|
||||
Result, SLASH_SEPARATOR, SUFFIX_ACTUAL_OBJECT_SIZE_CAP, SUFFIX_ACTUAL_SIZE, SUFFIX_BUCKET_INCARNATION_ID,
|
||||
SUFFIX_COMPRESSION_SIZE, SUFFIX_REPLICATION_SSEC_CRC, SUFFIX_RESTORE_OPERATION_ID, SetDisks, SmallWritePath, StorageError,
|
||||
Uuid, WriteLayout, check_object_lock_for_deletion_with_state, classify_multipart_part_write_path, coding,
|
||||
complete_multipart_part_error, complete_multipart_part_error_result, complete_part_checksum, completed_multipart_object_part,
|
||||
contains_key_str, create_bitrot_writer, debug, disk, error, get_complete_multipart_md5, get_header_map, get_str, insert_str,
|
||||
is_err_object_not_found, is_err_version_not_found, is_min_allowed_part_size, log_multipart_write_quorum_failure,
|
||||
parts_after_marker, path_join_buf, record_compression_total_memory, reduce_read_quorum_errs, reduce_write_quorum_errs,
|
||||
remove_header_map, resolve_write_layout, restore_commit_operation_id_from_metadata, should_persist_encryption_original_size,
|
||||
@@ -183,6 +183,45 @@ pub(crate) struct StaleMultipartCleanupGuard {
|
||||
lock_guard: ObjectLockDiagGuard,
|
||||
}
|
||||
|
||||
pub(crate) struct DataMovementMultipartAbortGuard {
|
||||
upload_path: String,
|
||||
write_quorum: Option<usize>,
|
||||
lock_guard: ObjectLockDiagGuard,
|
||||
}
|
||||
|
||||
impl DataMovementMultipartAbortGuard {
|
||||
pub(crate) fn add_namespace_lock_fence(&self, opts: &mut ObjectOptions) {
|
||||
opts.add_namespace_lock_guard(&self.lock_guard.guard);
|
||||
}
|
||||
|
||||
pub(crate) async fn delete(&self, set: &SetDisks, bucket: &str, object: &str, opts: &ObjectOptions) -> Result<()> {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::AbortBeforeDelete).await;
|
||||
fence_commit_on_lock_loss(Some(&self.lock_guard), "abort_multipart_upload_commit", &self.upload_path)?;
|
||||
if opts
|
||||
.namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "abort_multipart_upload_outer_lock",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
if let Some(write_quorum) = self.write_quorum {
|
||||
set.delete_all_with_quorum(RUSTFS_META_MULTIPART_BUCKET, &self.upload_path, write_quorum)
|
||||
.await?;
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::AbortAfterDelete).await;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl StaleMultipartCleanupGuard {
|
||||
pub(crate) fn file_info(&self) -> &FileInfo {
|
||||
&self.file_info
|
||||
@@ -210,7 +249,10 @@ pub enum MultipartCommitPause {
|
||||
NewUploadBeforeLockLost,
|
||||
PutPartBeforeLockAcquire,
|
||||
PutPartBeforeLockLost,
|
||||
PutPartAfterCapacityAdmission,
|
||||
PutPartAfterRename,
|
||||
AbortBeforeDelete,
|
||||
AbortAfterDelete,
|
||||
BeforeLockLost,
|
||||
BeforeQuotaRename,
|
||||
BeforeTransactionEpochVerify,
|
||||
@@ -673,12 +715,28 @@ fn is_corrupt_upload_metadata_error(err: &DiskError) -> bool {
|
||||
)
|
||||
}
|
||||
|
||||
async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str) -> disk::error::Result<Vec<String>> {
|
||||
async fn multipart_upload_paths_on_disk(disk: DiskStore, bucket: &str, root_prefix: &str) -> disk::error::Result<Vec<String>> {
|
||||
if !disk.is_online().await {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
|
||||
let sha_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, "", -1).await {
|
||||
if !root_prefix.is_empty() {
|
||||
let upload_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
return Ok(upload_dirs
|
||||
.into_iter()
|
||||
.filter_map(|upload_dir| {
|
||||
let upload_dir = upload_dir.trim_end_matches('/');
|
||||
(!upload_dir.is_empty() && upload_dir != "." && upload_dir != ".." && !upload_dir.contains(['/', '\\']))
|
||||
.then(|| format!("{root_prefix}/{upload_dir}"))
|
||||
})
|
||||
.collect());
|
||||
}
|
||||
|
||||
let sha_dirs = match disk.list_dir(bucket, RUSTFS_META_MULTIPART_BUCKET, root_prefix, -1).await {
|
||||
Ok(entries) => entries,
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => return Ok(Vec::new()),
|
||||
Err(err) => return Err(err),
|
||||
@@ -778,6 +836,7 @@ impl SetDisks {
|
||||
&self,
|
||||
orig_bucket: &str,
|
||||
error_path: &str,
|
||||
root_prefix: &str,
|
||||
) -> Result<(Vec<Option<DiskStore>>, Vec<String>, usize)> {
|
||||
let disks = self.disks.read().await.clone();
|
||||
if disks.is_empty() {
|
||||
@@ -794,9 +853,10 @@ impl SetDisks {
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
let disk = disk.clone();
|
||||
let orig_bucket = orig_bucket.to_string();
|
||||
let root_prefix = root_prefix.to_string();
|
||||
discovery_tasks.spawn(async move {
|
||||
let result = match disk {
|
||||
Some(disk) => multipart_upload_paths_on_disk(disk, &orig_bucket).await,
|
||||
Some(disk) => multipart_upload_paths_on_disk(disk, &orig_bucket, &root_prefix).await,
|
||||
None => Err(DiskError::DiskNotFound),
|
||||
};
|
||||
(index, result)
|
||||
@@ -832,11 +892,64 @@ impl SetDisks {
|
||||
|
||||
pub(crate) async fn first_multipart_upload_path_for_decommission(&self, bucket: &str) -> Result<Option<String>> {
|
||||
let (_, paths, _) = self
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET)
|
||||
.discover_multipart_upload_paths(bucket, RUSTFS_META_MULTIPART_BUCKET, "")
|
||||
.await?;
|
||||
Ok(paths.into_iter().next())
|
||||
}
|
||||
|
||||
pub(crate) async fn data_movement_multipart_upload_ids(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
upload_identity: &str,
|
||||
) -> Result<Vec<String>> {
|
||||
let expected_parent = format!("{DATA_MOVEMENT_MULTIPART_PREFIX}/{}", Self::get_multipart_sha_dir(bucket, object));
|
||||
let (_, candidate_paths, _) = self.discover_multipart_upload_paths(bucket, object, &expected_parent).await?;
|
||||
let mut upload_ids = Vec::new();
|
||||
for upload_path in candidate_paths {
|
||||
let Some((parent, raw_upload_id)) = upload_path.rsplit_once('/') else {
|
||||
continue;
|
||||
};
|
||||
if parent != expected_parent || raw_upload_id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let upload_id = runtime_sources::deployment_upload_id(raw_upload_id);
|
||||
let file_info = match self
|
||||
.check_multipart_upload_path_exists(bucket, object, &upload_id, &upload_path, false)
|
||||
.await
|
||||
{
|
||||
Ok((file_info, _)) => file_info,
|
||||
Err(err) if crate::error::is_err_invalid_upload_id(&err) || crate::error::is_err_object_not_found(&err) => {
|
||||
continue;
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
if file_info.metadata.get(RUSTFS_MULTIPART_BUCKET_KEY).map(String::as_str) != Some(bucket)
|
||||
|| file_info.metadata.get(RUSTFS_MULTIPART_OBJECT_KEY).map(String::as_str) != Some(object)
|
||||
{
|
||||
return Err(Error::other("data movement multipart upload target metadata is inconsistent"));
|
||||
}
|
||||
if expected_incarnation_id
|
||||
.is_some_and(|expected| !multipart_bucket_incarnation_matches(&file_info.metadata, expected))
|
||||
{
|
||||
return Err(Error::other("data movement multipart upload bucket incarnation is inconsistent"));
|
||||
}
|
||||
let Some(actual_upload_identity) =
|
||||
rustfs_utils::http::get_consistent_str(&file_info.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD)
|
||||
else {
|
||||
return Err(Error::other("data movement multipart upload identity is inconsistent"));
|
||||
};
|
||||
if actual_upload_identity != upload_identity {
|
||||
continue;
|
||||
}
|
||||
upload_ids.push(upload_id);
|
||||
}
|
||||
upload_ids.sort_unstable();
|
||||
upload_ids.dedup();
|
||||
Ok(upload_ids)
|
||||
}
|
||||
|
||||
async fn acquire_multipart_upload_read_lock(
|
||||
&self,
|
||||
op: &'static str,
|
||||
@@ -873,6 +986,55 @@ impl SetDisks {
|
||||
.map(Some)
|
||||
}
|
||||
|
||||
pub(crate) async fn lock_data_movement_multipart_abort(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
expected_upload_identity: Option<&str>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<Option<DataMovementMultipartAbortGuard>> {
|
||||
let upload_path = Self::get_multipart_upload_dir(bucket, object, upload_id, true);
|
||||
let lock_guard = self
|
||||
.acquire_write_lock_diag("abort_data_movement_multipart", RUSTFS_META_MULTIPART_BUCKET, &upload_path)
|
||||
.await?;
|
||||
let file_info = match self
|
||||
.check_multipart_upload_path_exists(bucket, object, upload_id, &upload_path, true)
|
||||
.await
|
||||
{
|
||||
Ok((file_info, _)) => file_info,
|
||||
Err(err) if crate::error::is_err_invalid_upload_id(&err) || crate::error::is_err_object_not_found(&err) => {
|
||||
return Ok(Some(DataMovementMultipartAbortGuard {
|
||||
upload_path,
|
||||
write_quorum: None,
|
||||
lock_guard,
|
||||
}));
|
||||
}
|
||||
Err(err) => return Err(err),
|
||||
};
|
||||
ensure_data_movement_upload_access(&file_info, bucket, object, upload_id, opts)?;
|
||||
let upload_identity =
|
||||
rustfs_utils::http::get_consistent_str(&file_info.metadata, rustfs_utils::http::SUFFIX_DATA_MOVEMENT_UPLOAD);
|
||||
if upload_identity.is_none() || expected_upload_identity.is_some_and(|expected| upload_identity != Some(expected)) {
|
||||
return Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()));
|
||||
}
|
||||
ensure_multipart_bucket_incarnation(
|
||||
&self.ctx,
|
||||
&file_info,
|
||||
bucket,
|
||||
object,
|
||||
upload_id,
|
||||
opts.expected_bucket_incarnation_id,
|
||||
)
|
||||
.await?;
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
Ok(Some(DataMovementMultipartAbortGuard {
|
||||
upload_path,
|
||||
write_quorum: Some(file_info.write_quorum(self.default_write_quorum())),
|
||||
lock_guard,
|
||||
}))
|
||||
}
|
||||
|
||||
pub(super) async fn list_parts(
|
||||
disks: &[Option<DiskStore>],
|
||||
part_path: &str,
|
||||
@@ -1028,7 +1190,7 @@ impl SetDisks {
|
||||
max_uploads: usize,
|
||||
expected_incarnation_id: Option<Uuid>,
|
||||
) -> Result<ListMultipartsInfo> {
|
||||
let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix).await?;
|
||||
let (disks, candidate_paths, discovery_quorum) = self.discover_multipart_upload_paths(bucket, prefix, "").await?;
|
||||
let listed_uploads = stream::iter(candidate_paths)
|
||||
.map(|upload_path| {
|
||||
let disks = &disks;
|
||||
@@ -1624,7 +1786,27 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
admitted_multipart_size(current_size, candidate_size, limit)?;
|
||||
}
|
||||
|
||||
let _ = self
|
||||
let decommission_capacity_guard = if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation")
|
||||
.await?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::PutPartAfterCapacityAdmission).await;
|
||||
if decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_part_decommission_capacity",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
let rename_result = self
|
||||
.rename_part(
|
||||
&shuffle_disks,
|
||||
RUSTFS_META_TMP_BUCKET,
|
||||
@@ -1641,7 +1823,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
part_number: Some(part_id),
|
||||
}),
|
||||
)
|
||||
.await?;
|
||||
.await;
|
||||
drop(decommission_capacity_guard);
|
||||
let _ = rename_result?;
|
||||
#[cfg(test)]
|
||||
observe_multipart_commit(bucket, object, MultipartCommitPause::PutPartBeforeLockLost);
|
||||
|
||||
@@ -2112,6 +2296,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let upload_id_path = Self::get_multipart_upload_dir(bucket, object, upload_id, opts.data_movement);
|
||||
let range_seek_rollout_enabled = crate::object_api::legacy_encrypted_range_seek_enabled() && !opts.no_lock;
|
||||
let mut object_lock_guard = None;
|
||||
let mut decommission_object_lock_guard = None;
|
||||
let mut decommission_target_lock_covered = false;
|
||||
let mut decommission_capacity_guard = None;
|
||||
|
||||
if opts.http_preconditions.is_some() {
|
||||
if !opts.no_lock {
|
||||
@@ -2126,7 +2313,25 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if !opts.no_lock && object_lock_guard.is_none() {
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
crate::core::pools::notify_decommission_external_object_commit_phase_started(store.id);
|
||||
crate::core::pools::wait_for_decommission_external_object_commit_phase_release(store.id).await;
|
||||
}
|
||||
let (object_guard, target_lock_covered, capacity_guard) = store
|
||||
.acquire_external_decommission_commit_guards(
|
||||
self.pool_index,
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock || object_lock_guard.is_some(),
|
||||
)
|
||||
.await?;
|
||||
decommission_object_lock_guard = object_guard;
|
||||
decommission_target_lock_covered = target_lock_covered;
|
||||
decommission_capacity_guard = capacity_guard;
|
||||
}
|
||||
if !opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered {
|
||||
object_lock_guard = Some(
|
||||
self.acquire_write_lock_diag("complete_multipart_upload_commit", bucket, object)
|
||||
.await?,
|
||||
@@ -2728,7 +2933,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
// so a lost lock leaves the upload intact and retryable.
|
||||
#[cfg(test)]
|
||||
pause_multipart_commit(bucket, object, MultipartCommitPause::BeforeLockLost).await;
|
||||
if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "complete_multipart_upload_commit",
|
||||
bucket: bucket.to_string(),
|
||||
@@ -2805,7 +3014,11 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
if object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "complete_multipart_upload_commit",
|
||||
bucket: bucket.to_string(),
|
||||
@@ -2838,6 +3051,19 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
}
|
||||
ensure_multipart_bucket_lifecycle_lock_held(bucket, object, opts)?;
|
||||
|
||||
// Complete has already acquired the object and upload-id namespaces.
|
||||
// Recheck decommission capacity only after those locks, and retain the
|
||||
// guard through the durable rename below.
|
||||
if decommission_capacity_guard.is_none()
|
||||
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
|
||||
{
|
||||
decommission_capacity_guard = Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation")
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
let transaction_fencing_proof = object_transaction_fencing_fleet_proof();
|
||||
if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() {
|
||||
return Err(Error::other("object transaction fencing requires a live fleet capability proof"));
|
||||
@@ -2896,11 +3122,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
let commit_is_versioned = opts.versioned || opts.version_suspended;
|
||||
let commit_capacity_scope_token = opts.capacity_scope_token;
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let commit_allows_early_ack = commit_object_lock_guard.is_some();
|
||||
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some());
|
||||
let detach_commit_owner = commit_allows_early_ack || upload_guard.is_some() || quota_mutation_fence;
|
||||
let commit = async move {
|
||||
let mut _object_lock_guard = commit_object_lock_guard;
|
||||
let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
|
||||
let mut _upload_guard = upload_guard;
|
||||
let mut _decommission_capacity_guard = commit_decommission_capacity_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let complete_tail_stage_start = rustfs_io_metrics::put_stage_metrics_enabled().then(Instant::now);
|
||||
|
||||
@@ -2919,6 +3150,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
@@ -2926,6 +3160,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|| commit_bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| _decommission_capacity_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
@@ -2967,6 +3204,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _upload_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
@@ -2974,6 +3214,9 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
|| commit_bucket_lifecycle_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| _decommission_capacity_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
@@ -3037,6 +3280,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.map(|version_id| version_id.to_string());
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let upload_guard = _upload_guard.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
let decommission_capacity_guard = _decommission_capacity_guard.take();
|
||||
let cleanup_bucket = commit_bucket.clone();
|
||||
let cleanup_object = commit_object.clone();
|
||||
let heal_set = commit_set.clone();
|
||||
@@ -3054,7 +3299,12 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
tokio::spawn(finish_rename_tail_heal(
|
||||
rename_tail_drain,
|
||||
guard_release_rx,
|
||||
(object_lock_guard, upload_guard),
|
||||
(
|
||||
object_lock_guard,
|
||||
upload_guard,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
request,
|
||||
move || async move {
|
||||
if quota_mutation_fence {
|
||||
@@ -3068,7 +3318,8 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
.await;
|
||||
}
|
||||
},
|
||||
move |(object_lock_guard, upload_guard), targets| async move {
|
||||
move |(object_lock_guard, upload_guard, decommission_object_lock_guard, decommission_capacity_guard),
|
||||
targets| async move {
|
||||
drop(object_lock_guard);
|
||||
cleanup_set.cleanup_multipart_path(&cleanup_parts).await;
|
||||
cleanup_set
|
||||
@@ -3093,11 +3344,16 @@ impl crate::storage_api_contracts::multipart::MultipartOperations for SetDisks {
|
||||
);
|
||||
}
|
||||
drop(upload_guard);
|
||||
drop(decommission_object_lock_guard);
|
||||
drop(decommission_capacity_guard);
|
||||
},
|
||||
|request| async move { heal_set.submit_rename_tail_heal(request).await },
|
||||
));
|
||||
}
|
||||
}
|
||||
if !tail_owns_staging_cleanup {
|
||||
drop(_decommission_capacity_guard.take());
|
||||
}
|
||||
if quota_mutation_fence && !tail_owns_staging_cleanup {
|
||||
let _ = SetDisks::release_quota_mutation_fences(
|
||||
&commit_disks,
|
||||
|
||||
@@ -1379,6 +1379,14 @@ fn data_read_metadata_early_stop_request_shape_allowed(range: &Option<HTTPRangeS
|
||||
&& !crate::object_api::restore_request_active(opts)
|
||||
}
|
||||
|
||||
fn prepare_late_materialized_retry(initial_result: &Result<()>, output: &mut Vec<u8>, expected_size: usize) -> bool {
|
||||
if initial_result.is_ok() && output.len() == expected_size {
|
||||
return false;
|
||||
}
|
||||
output.clear();
|
||||
true
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod data_read_metadata_early_stop_request_shape_tests {
|
||||
use super::*;
|
||||
@@ -1434,6 +1442,14 @@ mod data_read_metadata_early_stop_request_shape_tests {
|
||||
restore_opts.transition.restore_request.days = Some(1);
|
||||
assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &restore_opts));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn late_materialized_retry_clears_partial_buffer_after_error() {
|
||||
let mut output = b"partial-prefix".to_vec();
|
||||
let result = Err(Error::FileCorrupt);
|
||||
assert!(prepare_late_materialized_retry(&result, &mut output, 1024));
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
/// Length of the full plaintext body when — and only when — this read's output
|
||||
@@ -1675,7 +1691,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
None
|
||||
};
|
||||
|
||||
let metadata_stage_start = Instant::now();
|
||||
let metadata_stage_start = stage_metrics_enabled.then(Instant::now);
|
||||
let (snapshot, prepared_object_info) = if let Some(prepared) = take_prepared_get_object_metadata() {
|
||||
(prepared.snapshot, prepared.object_info)
|
||||
} else {
|
||||
@@ -1691,7 +1707,11 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
{
|
||||
Ok(snapshot) => (snapshot, None),
|
||||
Err(err) => {
|
||||
rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_stage_start.elapsed().as_secs_f64());
|
||||
if let Some(metadata_stage_start) = metadata_stage_start {
|
||||
rustfs_io_metrics::record_get_object_metadata_phase_duration(
|
||||
metadata_stage_start.elapsed().as_secs_f64(),
|
||||
);
|
||||
}
|
||||
let failure_path = if is_meta_bucketname(bucket) {
|
||||
GET_OBJECT_PATH_INTERNAL_META
|
||||
} else {
|
||||
@@ -1716,15 +1736,17 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
};
|
||||
let size_bucket = rustfs_io_metrics::get_object_size_bucket(metrics_size);
|
||||
record_get_stage_duration_if_enabled(GET_OBJECT_PATH_SET_DISK, GET_STAGE_OBJECT_INFO, object_info_stage_start);
|
||||
let metadata_elapsed = metadata_stage_start.elapsed().as_secs_f64();
|
||||
rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_elapsed);
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_STAGE_METADATA,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
metadata_elapsed,
|
||||
);
|
||||
if let Some(metadata_stage_start) = metadata_stage_start {
|
||||
let metadata_elapsed = metadata_stage_start.elapsed().as_secs_f64();
|
||||
rustfs_io_metrics::record_get_object_metadata_phase_duration(metadata_elapsed);
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
GET_STAGE_METADATA,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
metadata_elapsed,
|
||||
);
|
||||
}
|
||||
|
||||
if object_info.delete_marker {
|
||||
if opts.version_id.is_none() {
|
||||
@@ -2006,6 +2028,88 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if snapshot.has_late_metadata_fanout() {
|
||||
// Keep refresh plus the second decode off the default GET poll stack.
|
||||
// The allocation is limited to the opt-in late-materialization path.
|
||||
return Box::pin(async move {
|
||||
let object_size = usize::try_from(object_info.size)
|
||||
.map_err(|_| to_object_err(Error::other("two-phase GET object size is invalid"), vec![bucket, object]))?;
|
||||
let mut output = Vec::with_capacity(object_size);
|
||||
let (fi, files, disks, late_metadata_fanout_disks) = snapshot.into_owned_with_late_metadata_fanout();
|
||||
let expected_identity = super::super::read::LateMetadataIdentity::from_file_info(&fi);
|
||||
let late_metadata_fanout_disks = late_metadata_fanout_disks.ok_or_else(|| {
|
||||
to_object_err(Error::other("two-phase GET fallback context is missing"), vec![bucket, object])
|
||||
})?;
|
||||
let initial_result = Self::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::clone(&self.erasure_cache),
|
||||
0,
|
||||
object_info.size,
|
||||
&mut output,
|
||||
fi,
|
||||
files,
|
||||
&disks,
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
true,
|
||||
true,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
)
|
||||
.await;
|
||||
if prepare_late_materialized_retry(&initial_result, &mut output, object_size) {
|
||||
let (full_fi, full_parts_metadata, full_online_disks) = Self::refresh_late_metadata_fanout(
|
||||
&late_metadata_fanout_disks,
|
||||
bucket,
|
||||
object,
|
||||
&expected_identity,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
)
|
||||
.await?;
|
||||
Self::get_object_with_fileinfo(
|
||||
bucket,
|
||||
object,
|
||||
Arc::clone(&self.erasure_cache),
|
||||
0,
|
||||
object_info.size,
|
||||
&mut output,
|
||||
full_fi,
|
||||
full_parts_metadata,
|
||||
&full_online_disks,
|
||||
self.set_index,
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
true,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
if output.len() != object_size {
|
||||
return Err(to_object_err(Error::other("two-phase GET decoded length mismatch"), vec![bucket, object]));
|
||||
}
|
||||
|
||||
record_get_object_reader_path_observation(GET_OBJECT_PATH_LEGACY_DUPLEX, object_class, size_bucket);
|
||||
let body = Bytes::from(output);
|
||||
let reader = GetObjectReader {
|
||||
stream: Box::new(Cursor::new(body.clone())),
|
||||
object_info,
|
||||
buffered_body: Some(body),
|
||||
body_source,
|
||||
};
|
||||
if lock_optimization_enabled {
|
||||
release_materialized_read_lock(bucket, object, read_lock_guard.take());
|
||||
}
|
||||
Ok(reader)
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
let direct_memory_decision = get_small_object_direct_memory_decision_with_threshold_and_plan(
|
||||
&range,
|
||||
&object_info,
|
||||
@@ -2067,6 +2171,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
self.pool_index,
|
||||
opts.skip_verify_bitrot,
|
||||
true,
|
||||
false,
|
||||
GET_OBJECT_PATH_DIRECT_MEMORY,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
@@ -2266,6 +2371,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
|
||||
pool_index,
|
||||
skip_verify,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
object_class.as_str(),
|
||||
size_bucket,
|
||||
@@ -2654,6 +2760,9 @@ impl SetDisks {
|
||||
let disks = self.get_disks_internal().await;
|
||||
|
||||
let mut object_lock_guard = None;
|
||||
let mut decommission_object_lock_guard = None;
|
||||
let mut decommission_target_lock_covered = false;
|
||||
let mut decommission_capacity_guard = None;
|
||||
let mut bucket_lifecycle_guard = None;
|
||||
|
||||
// This pre-body check is advisory fast-fail only: the authoritative
|
||||
@@ -3106,7 +3215,25 @@ impl SetDisks {
|
||||
.await?;
|
||||
}
|
||||
|
||||
if !opts.no_lock && object_lock_guard.is_none() {
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
#[cfg(test)]
|
||||
{
|
||||
crate::core::pools::notify_decommission_external_object_commit_phase_started(store.id);
|
||||
crate::core::pools::wait_for_decommission_external_object_commit_phase_release(store.id).await;
|
||||
}
|
||||
let (object_guard, target_lock_covered, capacity_guard) = store
|
||||
.acquire_external_decommission_commit_guards(
|
||||
self.pool_index,
|
||||
bucket,
|
||||
object,
|
||||
opts.no_lock || object_lock_guard.is_some(),
|
||||
)
|
||||
.await?;
|
||||
decommission_object_lock_guard = object_guard;
|
||||
decommission_target_lock_covered = target_lock_covered;
|
||||
decommission_capacity_guard = capacity_guard;
|
||||
}
|
||||
if !opts.no_lock && object_lock_guard.is_none() && !decommission_target_lock_covered {
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
pause_put_object_commit(bucket, object, PutObjectCommitPause::BeforeNamespace).await;
|
||||
if let Some(expected_incarnation_id) = opts.expected_bucket_incarnation_id
|
||||
@@ -3283,6 +3410,33 @@ impl SetDisks {
|
||||
});
|
||||
}
|
||||
|
||||
if decommission_capacity_guard.is_none()
|
||||
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
|
||||
{
|
||||
decommission_capacity_guard = Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation")
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
|
||||
// The object namespace is acquired above, after the input stream
|
||||
// has been fully staged. Only then admit the local publication
|
||||
// against the decommission capacity ledger; holding this guard
|
||||
// through rename_data keeps the namespace -> capacity order.
|
||||
if decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "put_object_external_namespace",
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
required: 1,
|
||||
achieved: 0,
|
||||
});
|
||||
}
|
||||
|
||||
let transaction_fencing_proof = object_transaction_fencing_fleet_proof();
|
||||
if object_transaction_fencing_requested() && transaction_fencing_proof.is_none() {
|
||||
return Err(Error::other("object transaction fencing requires a live fleet capability proof"));
|
||||
@@ -3417,13 +3571,17 @@ impl SetDisks {
|
||||
let commit_object = object.to_owned();
|
||||
let commit_tmp_dir = tmp_dir.clone();
|
||||
let commit_object_lock_guard = object_lock_guard.take();
|
||||
let commit_decommission_object_lock_guard = decommission_object_lock_guard.take();
|
||||
let commit_bucket_lifecycle_guard = bucket_lifecycle_guard.take();
|
||||
let commit_decommission_capacity_guard = decommission_capacity_guard.take();
|
||||
let commit_scanner_publication_scope = opts.scanner_publication_commit_scope.clone();
|
||||
// A scanner publication scope owns the movement permit until the
|
||||
// complete rename fan-out drains. Keep this path synchronous so
|
||||
// its terminal state is known before the coordinator releases
|
||||
// remote leases.
|
||||
let commit_allows_early_ack = commit_object_lock_guard.is_some() && commit_scanner_publication_scope.is_none();
|
||||
let commit_allows_early_ack = !(opts.data_movement && opts.has_decommission_capacity_reservation())
|
||||
&& (commit_object_lock_guard.is_some() || commit_decommission_object_lock_guard.is_some())
|
||||
&& commit_scanner_publication_scope.is_none();
|
||||
let detach_commit_owner = commit_scanner_publication_scope.is_some()
|
||||
|| commit_allows_early_ack
|
||||
|| commit_bucket_lifecycle_guard.is_some()
|
||||
@@ -3443,7 +3601,9 @@ impl SetDisks {
|
||||
|
||||
let commit = move |cancellation: Option<CancellationToken>| async move {
|
||||
let mut _object_lock_guard = commit_object_lock_guard;
|
||||
let mut _decommission_object_lock_guard = commit_decommission_object_lock_guard;
|
||||
let mut _bucket_lifecycle_guard = commit_bucket_lifecycle_guard;
|
||||
let mut _decommission_capacity_guard = commit_decommission_capacity_guard;
|
||||
let mut quota_reservation = quota_reservation;
|
||||
let rename_stage_start = Instant::now();
|
||||
let pre_rename = async {
|
||||
@@ -3455,6 +3615,9 @@ impl SetDisks {
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
@@ -3462,6 +3625,9 @@ impl SetDisks {
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_capacity_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
@@ -3505,6 +3671,9 @@ impl SetDisks {
|
||||
if quota_reservation.is_lock_lost()
|
||||
|| !quota_reservation.capability_proof_matches()
|
||||
|| _object_lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
|| commit_namespace_lock_fence
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
@@ -3512,6 +3681,9 @@ impl SetDisks {
|
||||
.as_ref()
|
||||
.is_some_and(NamespaceLockFence::is_lock_lost)
|
||||
|| _bucket_lifecycle_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| _decommission_capacity_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(StorageError::NamespaceLockQuorumUnavailable {
|
||||
mode: "quota_reservation",
|
||||
@@ -3621,6 +3793,8 @@ impl SetDisks {
|
||||
.map(|version_id| version_id.to_string());
|
||||
let object_lock_guard = _object_lock_guard.take();
|
||||
let bucket_lifecycle_guard = _bucket_lifecycle_guard.take();
|
||||
let decommission_object_lock_guard = _decommission_object_lock_guard.take();
|
||||
let decommission_capacity_guard = _decommission_capacity_guard.take();
|
||||
let cleanup_bucket = commit_bucket.clone();
|
||||
let cleanup_object = commit_object.clone();
|
||||
let heal_set = commit_set.clone();
|
||||
@@ -3635,7 +3809,12 @@ impl SetDisks {
|
||||
tokio::spawn(finish_rename_tail_heal(
|
||||
rename_tail_drain,
|
||||
guard_release_rx,
|
||||
(object_lock_guard, bucket_lifecycle_guard),
|
||||
(
|
||||
object_lock_guard,
|
||||
bucket_lifecycle_guard,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
request,
|
||||
move || async move {
|
||||
if quota_mutation_fence {
|
||||
@@ -3649,7 +3828,13 @@ impl SetDisks {
|
||||
.await;
|
||||
}
|
||||
},
|
||||
move |(object_lock_guard, bucket_lifecycle_guard), targets| async move {
|
||||
move |(
|
||||
object_lock_guard,
|
||||
bucket_lifecycle_guard,
|
||||
decommission_object_lock_guard,
|
||||
decommission_capacity_guard,
|
||||
),
|
||||
targets| async move {
|
||||
drop(object_lock_guard);
|
||||
drop(bucket_lifecycle_guard);
|
||||
cleanup_set
|
||||
@@ -3670,11 +3855,16 @@ impl SetDisks {
|
||||
"issue3031_put_object_tmp_cleanup_done"
|
||||
);
|
||||
}
|
||||
drop(decommission_object_lock_guard);
|
||||
drop(decommission_capacity_guard);
|
||||
},
|
||||
|request| async move { heal_set.submit_rename_tail_heal(request).await },
|
||||
));
|
||||
}
|
||||
}
|
||||
if !tail_owns_tmp_cleanup {
|
||||
drop(_decommission_capacity_guard.take());
|
||||
}
|
||||
#[cfg(any(test, feature = "test-util"))]
|
||||
if rename_result.is_ok() {
|
||||
pause_put_object_commit(&commit_bucket, &commit_object, PutObjectCommitPause::AfterRenameHandoff).await;
|
||||
@@ -5581,6 +5771,7 @@ fn notify_put_object_commit_namespace_acquired(bucket: &str, object: &str) {
|
||||
struct DeleteObjectCommitBarrierState {
|
||||
bucket: String,
|
||||
object: String,
|
||||
pause_after_publish: bool,
|
||||
arrived: tokio::sync::Notify,
|
||||
release: tokio::sync::Notify,
|
||||
}
|
||||
@@ -5597,9 +5788,18 @@ static DELETE_OBJECT_COMMIT_BARRIER: std::sync::OnceLock<std::sync::Mutex<Option
|
||||
#[cfg(test)]
|
||||
impl DeleteObjectCommitBarrier {
|
||||
pub(crate) fn install(bucket: &str, object: &str) -> Self {
|
||||
Self::install_with_mode(bucket, object, false)
|
||||
}
|
||||
|
||||
pub(crate) fn install_after_publish(bucket: &str, object: &str) -> Self {
|
||||
Self::install_with_mode(bucket, object, true)
|
||||
}
|
||||
|
||||
fn install_with_mode(bucket: &str, object: &str, pause_after_publish: bool) -> Self {
|
||||
let state = Arc::new(DeleteObjectCommitBarrierState {
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
pause_after_publish,
|
||||
arrived: tokio::sync::Notify::new(),
|
||||
release: tokio::sync::Notify::new(),
|
||||
});
|
||||
@@ -5644,7 +5844,22 @@ async fn pause_delete_object_commit(bucket: &str, object: &str) {
|
||||
.lock()
|
||||
.expect("delete object commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object)
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && !barrier.pause_after_publish)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
barrier.release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_delete_object_commit_after_publish(bucket: &str, object: &str) {
|
||||
let barrier = DELETE_OBJECT_COMMIT_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("delete object commit barrier mutex should not poison")
|
||||
.as_ref()
|
||||
.filter(|barrier| barrier.bucket == bucket && barrier.object == object && barrier.pause_after_publish)
|
||||
.cloned();
|
||||
if let Some(barrier) = barrier {
|
||||
barrier.arrived.notify_one();
|
||||
@@ -7421,6 +7636,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
self.delete_object_version(bucket, object, &fi, should_force_delete_marker_for_missing_version(&opts))
|
||||
.await
|
||||
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
|
||||
#[cfg(test)]
|
||||
pause_delete_object_commit_after_publish(bucket, object).await;
|
||||
|
||||
let disks = self.disk_inventory().await;
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
@@ -7461,6 +7678,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
self.delete_object_version(bucket, object, &dfi, opts.delete_marker)
|
||||
.await
|
||||
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
|
||||
#[cfg(test)]
|
||||
pause_delete_object_commit_after_publish(bucket, object).await;
|
||||
|
||||
let disks = self.disk_inventory().await;
|
||||
self.record_capacity_scope_if_needed(opts.capacity_scope_token, &disks);
|
||||
@@ -7815,6 +8034,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
pool_index,
|
||||
skip_verify,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_LEGACY_DUPLEX,
|
||||
GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART,
|
||||
metrics_size_bucket,
|
||||
@@ -8212,19 +8432,15 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
part_checksums.to_string(),
|
||||
);
|
||||
}
|
||||
// The restore copy-back re-writes this same object via put_object /
|
||||
// new_multipart_upload / complete_multipart_upload, each of which takes
|
||||
// the object write lock in its commit phase. The caller
|
||||
// (handle_restore_transitioned_object, #4877) already holds that write
|
||||
// lock for the whole restore and forwards no_lock=true, so the inner
|
||||
// writes must inherit it or they self-deadlock on the lock we already
|
||||
// hold and time out. put_restore_opts builds fresh options that default
|
||||
// no_lock=false, so propagate it explicitly here.
|
||||
// Keep the public ECStore capacity admission attached to each local
|
||||
// commit. The tier reads below must remain outside the object write
|
||||
// lock so HEAD/GET do not wait for a slow remote copy-back.
|
||||
ropts.no_lock = opts.no_lock;
|
||||
ropts.expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
|
||||
ropts.bucket_lifecycle_lock_fence = opts.bucket_lifecycle_lock_fence.clone();
|
||||
ropts.namespace_lock_fence = opts.namespace_lock_fence.clone();
|
||||
ropts.object_lock_config_snapshot = opts.object_lock_config_snapshot.clone();
|
||||
ropts.decommission_capacity_admission = opts.decommission_capacity_admission.clone();
|
||||
if oi.parts.len() == 1 {
|
||||
let mut opts = opts.clone();
|
||||
opts.part_number = Some(1);
|
||||
@@ -8355,25 +8571,19 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
|
||||
.expect("multipart restore must contain at least one uploaded part")
|
||||
.etag = Some("injected-invalid-complete-etag".to_string());
|
||||
}
|
||||
let complete_opts = ObjectOptions {
|
||||
mod_time: oi.mod_time,
|
||||
version_id: oi.version_id.map(|version| version.to_string()),
|
||||
expected_bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(),
|
||||
user_defined: restore_commit_metadata,
|
||||
no_lock: opts.no_lock,
|
||||
decommission_capacity_admission: opts.decommission_capacity_admission.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
self_
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
bucket,
|
||||
object,
|
||||
&res.upload_id,
|
||||
uploaded_parts,
|
||||
&ObjectOptions {
|
||||
mod_time: oi.mod_time,
|
||||
version_id: oi.version_id.map(|version| version.to_string()),
|
||||
expected_bucket_incarnation_id: opts.expected_bucket_incarnation_id,
|
||||
bucket_lifecycle_lock_fence: opts.bucket_lifecycle_lock_fence.clone(),
|
||||
user_defined: restore_commit_metadata,
|
||||
// Inherit the restore write lock (see ropts.no_lock above):
|
||||
// the commit phase re-acquires this object's write lock.
|
||||
no_lock: opts.no_lock,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &complete_opts)
|
||||
.await
|
||||
}
|
||||
.await;
|
||||
@@ -9785,6 +9995,9 @@ mod inline_put_commit_path_tests {
|
||||
use super::*;
|
||||
use crate::config::storageclass::{INLINE_BLOCK_ENV, lookup_config_for_pools, lookup_config_for_pools_without_env};
|
||||
use crate::disk::ReadOptions;
|
||||
use crate::ecstore_validation_blackbox::make_local_set_disks;
|
||||
use crate::set_disk::disk_call_counters;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
use rustfs_config::server_config::KVS;
|
||||
use serial_test::serial;
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -9957,6 +10170,69 @@ mod inline_put_commit_path_tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn get_object_reader_codec_rollout_excludes_late_metadata_refresh() {
|
||||
let (_temp_dirs, set_disks) = make_local_set_disks(4, 2).await;
|
||||
let bucket = "one-mib-codec-reader";
|
||||
let object = "object.bin";
|
||||
let payload = vec![0x6b; 1024 * 1024];
|
||||
set_disks
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("codec bucket should be created");
|
||||
let storage_class = temp_env::with_var(INLINE_BLOCK_ENV, Some("1KiB"), || lookup_config_for_pools(&KVS::new(), &[4]))
|
||||
.expect("test storage class should resolve");
|
||||
set_disks.set_test_storage_class_config(storage_class);
|
||||
|
||||
let mut writer = PutObjReader::from_vec(payload.clone());
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")),
|
||||
(ENV_RUSTFS_GET_MID_SIZE_STREAMING_ENABLE, Some("false")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some("legacy")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("true")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, Some("false")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")),
|
||||
(crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")),
|
||||
(rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true")),
|
||||
],
|
||||
async {
|
||||
set_disks
|
||||
.put_object(bucket, object, &mut writer, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("codec fixture should commit");
|
||||
crate::set_disk::reset_test_get_object_reader_path();
|
||||
let calls = disk_call_counters::observe(object);
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("codec GET should succeed");
|
||||
let mut restored = Vec::new();
|
||||
reader
|
||||
.stream
|
||||
.read_to_end(&mut restored)
|
||||
.await
|
||||
.expect("codec GET should stream");
|
||||
assert_eq!(restored, payload);
|
||||
assert_eq!(
|
||||
crate::set_disk::test_get_object_reader_path_id(),
|
||||
5,
|
||||
"codec path must win over late refresh"
|
||||
);
|
||||
assert_eq!(
|
||||
calls.total(disk_call_counters::KIND_READ_VERSION),
|
||||
4,
|
||||
"codec path must use full metadata fanout and must not trigger a second late refresh"
|
||||
);
|
||||
},
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_gets_reuse_the_set_erasure_shell() {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
|
||||
@@ -89,6 +89,8 @@ use super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT;
|
||||
#[cfg(test)]
|
||||
use super::ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE;
|
||||
#[cfg(test)]
|
||||
use super::ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE;
|
||||
#[cfg(test)]
|
||||
use super::ENV_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE;
|
||||
#[cfg(test)]
|
||||
use super::ENV_RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH;
|
||||
@@ -126,6 +128,8 @@ use super::is_get_metadata_early_stop_bounded_fanout_enabled;
|
||||
#[cfg(test)]
|
||||
use super::is_get_metadata_early_stop_enabled;
|
||||
#[cfg(test)]
|
||||
use super::is_get_metadata_non_inline_data_read_early_stop_enabled;
|
||||
#[cfg(test)]
|
||||
use super::is_version_early_stop_enabled;
|
||||
#[cfg(test)]
|
||||
use super::load_get_codec_streaming_config;
|
||||
@@ -605,7 +609,20 @@ impl SetDisks {
|
||||
|
||||
// let online_disks: Vec<Option<DiskStore>> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect();
|
||||
|
||||
Ok(GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks))
|
||||
if !metadata_fanout_complete
|
||||
&& allow_early_stop
|
||||
&& non_inline_data_read_early_stop_allowed(read_data, bucket, object)
|
||||
&& late_materialization_candidate_is_safe(&fi)
|
||||
{
|
||||
Ok(GetObjectFileInfo::owned_with_late_metadata_fanout(
|
||||
fi,
|
||||
parts_metadata,
|
||||
op_online_disks,
|
||||
disks,
|
||||
))
|
||||
} else {
|
||||
Ok(GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks))
|
||||
}
|
||||
}
|
||||
|
||||
#[hotpath::measure(impl_type = "SetDisks")]
|
||||
@@ -815,6 +832,7 @@ impl SetDisks {
|
||||
pool_index: usize,
|
||||
skip_verify_bitrot: bool,
|
||||
prefer_data_blocks_first_reader_setup: bool,
|
||||
require_reconstruction_surplus: bool,
|
||||
metrics_path: &'static str,
|
||||
metrics_object_class: &'static str,
|
||||
metrics_size_bucket: &'static str,
|
||||
@@ -1079,6 +1097,9 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let nil_count = reader_setup.available_shards();
|
||||
if require_reconstruction_surplus && nil_count <= erasure.data_shards {
|
||||
return Err(Error::other("insufficient reconstruction surplus for two-phase read"));
|
||||
}
|
||||
if nil_count < erasure.data_shards {
|
||||
if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards)
|
||||
{
|
||||
@@ -1186,18 +1207,34 @@ impl SetDisks {
|
||||
let readers = reader_setup.readers;
|
||||
let deferred_stripe_handles = reader_setup.deferred_stripe_handles;
|
||||
let deferred_reopeners = reader_setup.deferred_reopeners;
|
||||
let (written, err) = erasure
|
||||
.decode_with_stripe_handles_and_reopeners(
|
||||
writer,
|
||||
readers,
|
||||
part_offset,
|
||||
part_length,
|
||||
part_size,
|
||||
read_costs,
|
||||
deferred_stripe_handles,
|
||||
deferred_reopeners,
|
||||
)
|
||||
.await;
|
||||
let (written, err, exact_quorum) = if require_reconstruction_surplus {
|
||||
erasure
|
||||
.decode_with_stripe_handles_and_reopeners_with_diagnostics(
|
||||
writer,
|
||||
readers,
|
||||
part_offset,
|
||||
part_length,
|
||||
part_size,
|
||||
read_costs,
|
||||
deferred_stripe_handles,
|
||||
deferred_reopeners,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
let (written, err) = erasure
|
||||
.decode_with_stripe_handles_and_reopeners(
|
||||
writer,
|
||||
readers,
|
||||
part_offset,
|
||||
part_length,
|
||||
part_size,
|
||||
read_costs,
|
||||
deferred_stripe_handles,
|
||||
deferred_reopeners,
|
||||
)
|
||||
.await;
|
||||
(written, err, false)
|
||||
};
|
||||
let decode_elapsed = decode_stage_start.elapsed();
|
||||
rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64());
|
||||
rustfs_io_metrics::record_get_object_stage_duration_by_size(
|
||||
@@ -1207,6 +1244,9 @@ impl SetDisks {
|
||||
metrics_size_bucket,
|
||||
decode_elapsed.as_secs_f64(),
|
||||
);
|
||||
if exact_quorum && err.is_none() {
|
||||
return Err(Error::other("two-phase read completed with exact reconstruction quorum"));
|
||||
}
|
||||
if decode_elapsed >= SLOW_OBJECT_READ_LOG_THRESHOLD && err.is_none() {
|
||||
warn!(
|
||||
event = EVENT_SET_DISK_READ,
|
||||
@@ -1754,6 +1794,102 @@ fn multipart_reader_setup_prefetch_enabled(policy: GetObjectReadPolicy) -> bool
|
||||
policy.allows_multipart_setup_prefetch() && is_multipart_reader_setup_prefetch_enabled()
|
||||
}
|
||||
|
||||
pub(super) struct LateMetadataIdentity {
|
||||
volume: String,
|
||||
name: String,
|
||||
algorithm: String,
|
||||
block_size: usize,
|
||||
uses_legacy_checksum: bool,
|
||||
quorum_hash: [u8; 32],
|
||||
distribution: Vec<usize>,
|
||||
parity_blocks: usize,
|
||||
}
|
||||
|
||||
impl LateMetadataIdentity {
|
||||
pub(super) fn from_file_info(file_info: &FileInfo) -> Self {
|
||||
Self {
|
||||
volume: file_info.volume.clone(),
|
||||
name: file_info.name.clone(),
|
||||
algorithm: file_info.erasure.algorithm.clone(),
|
||||
block_size: file_info.erasure.block_size,
|
||||
uses_legacy_checksum: file_info.uses_legacy_checksum,
|
||||
quorum_hash: SetDisks::file_info_quorum_hash(file_info),
|
||||
distribution: file_info.erasure.distribution.clone(),
|
||||
parity_blocks: file_info.erasure.parity_blocks,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn late_metadata_read_identity_matches(expected: &LateMetadataIdentity, actual: &FileInfo) -> bool {
|
||||
expected.volume == actual.volume
|
||||
&& expected.name == actual.name
|
||||
&& expected.algorithm == actual.erasure.algorithm
|
||||
&& expected.block_size == actual.erasure.block_size
|
||||
&& expected.uses_legacy_checksum == actual.uses_legacy_checksum
|
||||
&& expected.quorum_hash == SetDisks::file_info_quorum_hash(actual)
|
||||
}
|
||||
|
||||
fn late_metadata_shard_matches(expected: &LateMetadataIdentity, actual: &FileInfo, disk_index: usize) -> bool {
|
||||
expected
|
||||
.distribution
|
||||
.get(disk_index)
|
||||
.is_some_and(|mapped_index| *mapped_index == actual.erasure.index)
|
||||
&& late_metadata_read_identity_matches(expected, actual)
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
pub(super) async fn refresh_late_metadata_fanout(
|
||||
fallback_disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
expected: &LateMetadataIdentity,
|
||||
metrics_path: &'static str,
|
||||
) -> Result<(FileInfo, Vec<FileInfo>, Vec<Option<DiskStore>>)> {
|
||||
let (mut parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed(
|
||||
fallback_disks,
|
||||
"",
|
||||
bucket,
|
||||
object,
|
||||
"",
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
expected.parity_blocks,
|
||||
)
|
||||
.await?;
|
||||
diagnostics.record(metrics_path);
|
||||
|
||||
let (read_quorum, write_quorum) = SetDisks::object_quorum_from_meta(&parts_metadata, &errs, expected.parity_blocks)
|
||||
.map_err(|err| to_object_err(err.into(), vec![bucket, object]))?;
|
||||
let read_quorum =
|
||||
usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?;
|
||||
let write_quorum = usize::try_from(write_quorum)
|
||||
.map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?;
|
||||
if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) {
|
||||
return Err(to_object_err(err.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
let (mut online_disks, full_fi, _) =
|
||||
SetDisks::select_valid_fileinfo(fallback_disks, &parts_metadata, &errs, "", read_quorum, write_quorum)?;
|
||||
if !late_metadata_read_identity_matches(expected, &full_fi) {
|
||||
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
for (disk_index, (metadata, disk)) in parts_metadata.iter_mut().zip(online_disks.iter_mut()).enumerate() {
|
||||
if !late_metadata_shard_matches(expected, metadata, disk_index) {
|
||||
*metadata = FileInfo::default();
|
||||
*disk = None;
|
||||
}
|
||||
}
|
||||
if online_disks.iter().filter(|disk| disk.is_some()).count() < read_quorum {
|
||||
return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]));
|
||||
}
|
||||
|
||||
Ok((full_fi, parts_metadata, online_disks))
|
||||
}
|
||||
}
|
||||
|
||||
/// Run one part's bitrot reader setup and measure its wall-clock duration.
|
||||
///
|
||||
/// Shared by the synchronous path and the prefetch task in
|
||||
@@ -2345,6 +2481,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
@@ -2376,6 +2513,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
@@ -2400,6 +2538,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
@@ -2422,6 +2561,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
@@ -2446,6 +2586,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
@@ -2484,6 +2625,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"empty",
|
||||
@@ -2517,6 +2659,7 @@ mod metadata_cache_tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"plain",
|
||||
"small",
|
||||
@@ -4267,6 +4410,20 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial(body_cache_hook)]
|
||||
fn non_inline_data_read_early_stop_gate_defaults_off_and_honors_override() {
|
||||
temp_env::with_var(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, None::<&str>, || {
|
||||
assert!(!is_get_metadata_non_inline_data_read_early_stop_enabled());
|
||||
});
|
||||
temp_env::with_var(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true"), || {
|
||||
assert!(is_get_metadata_non_inline_data_read_early_stop_enabled());
|
||||
});
|
||||
temp_env::with_var(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("false"), || {
|
||||
assert!(!is_get_metadata_non_inline_data_read_early_stop_enabled());
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn metadata_early_stop_rejects_healing_and_free_version_requests() {
|
||||
temp_env::with_vars(
|
||||
@@ -4864,6 +5021,7 @@ mod tests {
|
||||
0,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
GET_OBJECT_PATH_SET_DISK,
|
||||
"test-object-class",
|
||||
"test-size-bucket",
|
||||
@@ -5547,9 +5705,10 @@ mod tests {
|
||||
|
||||
/// backlog#923: with the data-shards-only lockstep gate on, every retained
|
||||
/// parity reader must be an unopened deferred reader carrying a stripe
|
||||
/// handle, so the decode path can realign it to a mid-object stripe. With
|
||||
/// the gate off (default), eagerly opened parity readers are kept exactly
|
||||
/// as before and carry no handles.
|
||||
/// handle and disposable reopener, so the decode path can realign it to a
|
||||
/// mid-object stripe without consuming the later-stripe reserve. With the
|
||||
/// gate off (default), eagerly opened parity readers are kept exactly as
|
||||
/// before and carry neither.
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bitrot_reader_setup_gates_parity_stripe_handle_conversion() {
|
||||
@@ -5578,6 +5737,11 @@ mod tests {
|
||||
enabled.is_some(),
|
||||
"parity slot {idx} stripe handle must match the gate (enabled={enabled:?})"
|
||||
);
|
||||
assert_eq!(
|
||||
setup.deferred_reopeners[idx].is_some(),
|
||||
enabled.is_some(),
|
||||
"parity slot {idx} reopener must match the gate (enabled={enabled:?})"
|
||||
);
|
||||
}
|
||||
|
||||
if enabled.is_some() {
|
||||
@@ -5600,13 +5764,17 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn bitrot_reader_setup_data_blocks_first_keeps_deferred_fallback_readers() {
|
||||
let mut setup = setup_inline_bitrot_readers_with_env(
|
||||
vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")],
|
||||
2,
|
||||
2,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
true,
|
||||
let mut setup = temp_env::async_with_vars(
|
||||
[("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true"))],
|
||||
setup_inline_bitrot_readers_with_env(
|
||||
vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")],
|
||||
2,
|
||||
2,
|
||||
BitrotReaderSetupMode::ReadQuorum,
|
||||
true,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
|
||||
@@ -5614,6 +5782,8 @@ mod tests {
|
||||
assert_eq!(setup.available_shards(), 2);
|
||||
assert_eq!(setup.scheduled_shards(), 2);
|
||||
assert_eq!(setup.readers.iter().filter(|reader| reader.is_some()).count(), 4);
|
||||
assert!(setup.deferred_reopeners[2].is_some());
|
||||
assert!(setup.deferred_reopeners[3].is_some());
|
||||
|
||||
let fallback_index = setup
|
||||
.attempted
|
||||
|
||||
@@ -157,7 +157,15 @@ impl SetDisks {
|
||||
.clone()
|
||||
.unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
|
||||
let version_id = expected.version_id.map(|v| v.to_string());
|
||||
let lock_guard = if !opts.no_lock {
|
||||
let (decommission_object_lock_guard, decommission_target_lock_covered, mut decommission_capacity_guard) =
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
store
|
||||
.acquire_external_decommission_commit_guards(self.pool_index, bucket, object, opts.no_lock)
|
||||
.await?
|
||||
} else {
|
||||
(None, false, None)
|
||||
};
|
||||
let lock_guard = if !opts.no_lock && !decommission_target_lock_covered {
|
||||
Some(
|
||||
self.acquire_write_lock_diag("restore_finalize_metadata", bucket, object)
|
||||
.await?,
|
||||
@@ -165,6 +173,15 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if decommission_capacity_guard.is_none()
|
||||
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
|
||||
{
|
||||
decommission_capacity_guard = Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation")
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
let read_opts = ObjectOptions {
|
||||
version_id,
|
||||
versioned: opts.versioned,
|
||||
@@ -198,7 +215,12 @@ impl SetDisks {
|
||||
);
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
ensure_restore_metadata_lock_held(bucket, object, opts, "restore_finalize_metadata")?;
|
||||
if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
|| decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other("restore finalization lock lost before metadata update"));
|
||||
}
|
||||
self.update_object_meta_with_opts(
|
||||
@@ -233,7 +255,15 @@ impl SetDisks {
|
||||
.clone()
|
||||
.unwrap_or_else(|| get_raw_etag(obj_info.user_defined.as_ref()));
|
||||
let version_id = expected.version_id.map(|v| v.to_string());
|
||||
let lock_guard = if !opts.no_lock {
|
||||
let (decommission_object_lock_guard, decommission_target_lock_covered, mut decommission_capacity_guard) =
|
||||
if let Some(store) = opts.decommission_capacity_admission.as_ref() {
|
||||
store
|
||||
.acquire_external_decommission_commit_guards(self.pool_index, bucket, object, opts.no_lock)
|
||||
.await?
|
||||
} else {
|
||||
(None, false, None)
|
||||
};
|
||||
let lock_guard = if !opts.no_lock && !decommission_target_lock_covered {
|
||||
Some(
|
||||
self.acquire_write_lock_diag("restore_cleanup_metadata", bucket, object)
|
||||
.await?,
|
||||
@@ -241,6 +271,15 @@ impl SetDisks {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if decommission_capacity_guard.is_none()
|
||||
&& let Some(store) = opts.decommission_capacity_admission.as_ref()
|
||||
{
|
||||
decommission_capacity_guard = Some(
|
||||
store
|
||||
.acquire_external_decommission_capacity_fence(&[self.pool_index], "mutation")
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
let read_opts = ObjectOptions {
|
||||
version_id,
|
||||
versioned: opts.versioned,
|
||||
@@ -269,7 +308,12 @@ impl SetDisks {
|
||||
&mut fi.metadata,
|
||||
rustfs_utils::http::metadata_compat::SUFFIX_RESTORE_OPERATION_ID,
|
||||
);
|
||||
if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost()) {
|
||||
if lock_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
|| decommission_object_lock_guard
|
||||
.as_ref()
|
||||
.is_some_and(|guard| guard.is_lock_lost())
|
||||
|| decommission_capacity_guard.as_ref().is_some_and(|guard| guard.is_lock_lost())
|
||||
{
|
||||
return Err(Error::other("restore cleanup lock lost before metadata update".to_string()));
|
||||
}
|
||||
self.invalidate_get_object_metadata_cache(bucket, object).await;
|
||||
|
||||
@@ -470,6 +470,27 @@ impl ECStore {
|
||||
let object = encode_dir_object(object);
|
||||
|
||||
let pools = self.get_pools_for_heal_object(opts)?;
|
||||
if let Some(set_idx) = opts.set {
|
||||
for pool in &pools {
|
||||
if set_idx >= pool.disk_set.len() {
|
||||
let err = StorageError::InvalidArgument(
|
||||
"heal".to_string(),
|
||||
"set".to_string(),
|
||||
format!(
|
||||
"invalid heal set index {set_idx} for pool {} with {} sets",
|
||||
pool.pool_idx,
|
||||
pool.disk_set.len()
|
||||
),
|
||||
);
|
||||
if opts.pool.is_some() {
|
||||
return Err(err);
|
||||
}
|
||||
return Ok((HealResultItem::default(), Some(err)));
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
let store_id = self.id;
|
||||
|
||||
let mut futures = Vec::with_capacity(pools.len());
|
||||
for pool in pools.iter() {
|
||||
@@ -499,7 +520,17 @@ impl ECStore {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
futures.push(pool.heal_object(bucket, &object, version_id, opts));
|
||||
let pool_idx = pool.pool_idx;
|
||||
let pool = Arc::clone(pool);
|
||||
let pool_object = object.clone();
|
||||
let opts = *opts;
|
||||
futures.push(
|
||||
self.run_external_decommission_capacity_heal(pool_idx, bucket, &object, opts, move |opts| async move {
|
||||
#[cfg(test)]
|
||||
crate::core::pools::notify_decommission_external_heal_operation_started(store_id);
|
||||
pool.heal_object(bucket, &pool_object, version_id, &opts).await
|
||||
}),
|
||||
);
|
||||
}
|
||||
let results = join_all(futures).await;
|
||||
|
||||
@@ -594,14 +625,18 @@ mod tests {
|
||||
use crate::cluster::rpc::PeerS3Client;
|
||||
use crate::config::com::{delete_config, read_config_no_lock_preserve_empty_with_metadata, save_config};
|
||||
use crate::core::pools::{
|
||||
POOL_META_IDENTITY_NAME, PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus, initialized_pool_meta_identity_for_test,
|
||||
DecommissionCapacityLockOrderBarrier, DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME,
|
||||
PoolDecommissionInfo, PoolMetaReplicaState, PoolStatus, initialized_pool_meta_identity_for_test,
|
||||
set_decommission_capacity_info_overrides_for_test,
|
||||
};
|
||||
use crate::core::sets::HealFormatAfterSaveBarrier;
|
||||
use crate::disk::error::Result as DiskResult;
|
||||
use crate::disk::{DeleteOptions, DiskOption, FORMAT_CONFIG_FILE, format::FormatV3, new_disk};
|
||||
use crate::layout::endpoints::{EndpointServerPools, Endpoints, PoolEndpoints};
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::services::rebalance::{RebalanceInfo, RebalanceStats};
|
||||
use crate::services::rebalance::{
|
||||
RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores,
|
||||
};
|
||||
use crate::storage_api_contracts::bucket::{
|
||||
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
|
||||
};
|
||||
@@ -743,6 +778,7 @@ mod tests {
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
decommission_capacity_entry_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
@@ -1207,6 +1243,292 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn targeted_heal_is_blocked_by_exact_fit_decommission_reservation() {
|
||||
let (_temp_dirs, store, _other_store) = test_two_pool_stores(None).await;
|
||||
let bucket = format!("heal-capacity-{}", Uuid::new_v4().simple());
|
||||
let object = "targeted-heal.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("heal capacity bucket should be created");
|
||||
|
||||
let target_set = store.pools[1].get_disks(0);
|
||||
let mut reader = PutObjReader::from_vec(b"targeted heal body".to_vec());
|
||||
target_set
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("targeted heal fixture should be written");
|
||||
let missing_disk = target_set.disks.read().await[0]
|
||||
.clone()
|
||||
.expect("targeted heal fixture disk should be online");
|
||||
missing_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("targeted heal fixture shard should be removed");
|
||||
assert!(
|
||||
missing_disk.read_xl(&bucket, object, false).await.is_err(),
|
||||
"targeted heal fixture must start with one missing metadata copy"
|
||||
);
|
||||
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
set_decommission_capacity_info_overrides_for_test(
|
||||
store.id,
|
||||
vec![vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
|
||||
]],
|
||||
);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("the exact-fit decommission reservation should activate");
|
||||
{
|
||||
let pool_meta = store.pool_meta.read().await;
|
||||
let reservation = pool_meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("the active decommission reservation should be durable");
|
||||
assert_eq!(reservation.targets.len(), 1);
|
||||
assert_eq!(reservation.targets[0].pool_index, 1);
|
||||
assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
|
||||
}
|
||||
|
||||
let (_, err) = store
|
||||
.handle_heal_object(
|
||||
&bucket,
|
||||
object,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("targeted heal should return a mapped capacity result");
|
||||
assert!(matches!(err, Some(Error::SlowDown)), "targeted heal must be capacity-blocked: {err:?}");
|
||||
assert!(
|
||||
missing_disk.read_xl(&bucket, object, false).await.is_err(),
|
||||
"capacity-blocked targeted heal must not rewrite the missing shard"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn targeted_heal_keeps_target_lock_for_different_lock_domain() {
|
||||
let (_temp_dirs, store, other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let bucket = format!("heal-lock-domain-{}", Uuid::new_v4().simple());
|
||||
let object = "different-domain-heal.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("heal lock-domain bucket should be created");
|
||||
|
||||
let target_set = other_store.pools[1].get_disks(0);
|
||||
let mut reader = PutObjReader::from_vec(b"different lock domain body".to_vec());
|
||||
target_set
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("heal lock-domain fixture should be written");
|
||||
let missing_disk = target_set.disks.read().await[0]
|
||||
.clone()
|
||||
.expect("heal lock-domain fixture disk should be online");
|
||||
missing_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("heal lock-domain fixture shard should be removed");
|
||||
assert!(
|
||||
missing_disk.read_xl(&bucket, object, false).await.is_err(),
|
||||
"heal lock-domain fixture must start with one missing metadata copy"
|
||||
);
|
||||
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
set_decommission_capacity_info_overrides_for_test(
|
||||
store.id,
|
||||
vec![vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 30, 30),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 0, 100, 100),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, 60, 60, 0),
|
||||
]],
|
||||
);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[0], Vec::new())
|
||||
.await
|
||||
.expect("heal lock-domain reservation should activate");
|
||||
{
|
||||
let pool_meta = store.pool_meta.read().await;
|
||||
let reservation = pool_meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("heal lock-domain reservation should be durable");
|
||||
assert_eq!(reservation.targets.len(), 1);
|
||||
assert_eq!(reservation.targets[0].pool_index, 2);
|
||||
}
|
||||
let pool_meta = store.pool_meta.read().await.clone();
|
||||
*other_store.pool_meta.write().await = pool_meta;
|
||||
|
||||
let fixed_set = other_store.pools[0].disk_set[0].clone();
|
||||
assert!(
|
||||
!fixed_set.shares_namespace_lock_domain(&target_set).await,
|
||||
"heal fixture must use different fixed and target lock domains"
|
||||
);
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, other_store.id);
|
||||
let heal_store = Arc::clone(&other_store);
|
||||
let heal_bucket = bucket.clone();
|
||||
let heal_object = object.to_string();
|
||||
let mut heal = tokio::spawn(async move {
|
||||
heal_store
|
||||
.handle_heal_object(
|
||||
&heal_bucket,
|
||||
&heal_object,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(1),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
barrier.wait_until_external_capacity_released().await;
|
||||
barrier.wait_until_external_heal_target_lock_attempted().await;
|
||||
|
||||
let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(30), &mut heal)
|
||||
.await
|
||||
.expect("targeted heal should finish after target lock release")
|
||||
.expect("targeted heal task should not panic")
|
||||
.expect("targeted heal should complete");
|
||||
assert!(err.is_none(), "targeted heal should repair after target lock release: {err:?}");
|
||||
assert!(
|
||||
missing_disk.read_xl(&bucket, object, false).await.is_ok(),
|
||||
"targeted heal should rewrite the missing shard after the target lock is released"
|
||||
);
|
||||
drop(barrier);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn targeted_heal_reuses_shared_target_lock_without_reentrant_lock() {
|
||||
let (_temp_dirs, store, _other_store) = test_three_pool_stores_with_isolated_node_contexts(None).await;
|
||||
let bucket = format!("heal-shared-lock-domain-{}", Uuid::new_v4().simple());
|
||||
let object = "shared-domain-heal.bin";
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("shared heal lock-domain bucket should be created");
|
||||
|
||||
let target_set = store.pools[0].get_disks(0);
|
||||
let mut reader = PutObjReader::from_vec(b"shared lock domain body".to_vec());
|
||||
target_set
|
||||
.put_object(&bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("shared heal lock-domain fixture should be written");
|
||||
let missing_disk = target_set.disks.read().await[0]
|
||||
.clone()
|
||||
.expect("shared heal lock-domain fixture disk should be online");
|
||||
missing_disk
|
||||
.delete(
|
||||
&bucket,
|
||||
object,
|
||||
DeleteOptions {
|
||||
recursive: true,
|
||||
immediate: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("shared heal lock-domain fixture shard should be removed");
|
||||
assert!(
|
||||
missing_disk.read_xl(&bucket, object, false).await.is_err(),
|
||||
"shared heal lock-domain fixture must start with one missing metadata copy"
|
||||
);
|
||||
|
||||
let layout = DecommissionErasureLayout { data: 1, parity: 0 };
|
||||
set_decommission_capacity_info_overrides_for_test(
|
||||
store.id,
|
||||
vec![vec![
|
||||
DecommissionPoolCapacityInfo::for_test(0, layout, 0, 100, 100),
|
||||
DecommissionPoolCapacityInfo::for_test(1, layout, 60, 60, 0),
|
||||
DecommissionPoolCapacityInfo::for_test(2, layout, 0, 30, 30),
|
||||
]],
|
||||
);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[2], Vec::new())
|
||||
.await
|
||||
.expect("shared heal lock-domain reservation should activate");
|
||||
{
|
||||
let pool_meta = store.pool_meta.read().await;
|
||||
let reservation = pool_meta.pools[2]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.and_then(|info| info.capacity_reservation.as_ref())
|
||||
.expect("shared heal lock-domain reservation should be durable");
|
||||
assert_eq!(reservation.targets.len(), 1);
|
||||
assert_eq!(reservation.targets[0].pool_index, 1);
|
||||
assert_eq!(reservation.targets[0].reserved_physical_bytes, 60);
|
||||
}
|
||||
|
||||
let fixed_set = store.pools[0].disk_set[0].clone();
|
||||
assert!(
|
||||
fixed_set.shares_namespace_lock_domain(&target_set).await,
|
||||
"shared heal fixture must use one fixed and target lock domain"
|
||||
);
|
||||
|
||||
let barrier = DecommissionCapacityLockOrderBarrier::install(store.id, store.id);
|
||||
let heal_store = Arc::clone(&store);
|
||||
let heal_bucket = bucket.clone();
|
||||
let heal_object = object.to_string();
|
||||
let mut heal = tokio::spawn(async move {
|
||||
heal_store
|
||||
.handle_heal_object(
|
||||
&heal_bucket,
|
||||
&heal_object,
|
||||
"",
|
||||
&HealOpts {
|
||||
pool: Some(0),
|
||||
set: Some(0),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
|
||||
barrier.wait_until_external_capacity_released().await;
|
||||
barrier.wait_until_external_heal_operation_started().await;
|
||||
let (_, err) = tokio::time::timeout(std::time::Duration::from_secs(5), &mut heal)
|
||||
.await
|
||||
.expect("shared-domain heal must not reenter the fixed namespace lock")
|
||||
.expect("shared-domain heal task should not panic")
|
||||
.expect("shared-domain heal should complete");
|
||||
assert!(err.is_none(), "shared-domain heal should repair after admission: {err:?}");
|
||||
assert!(
|
||||
missing_disk.read_xl(&bucket, object, false).await.is_ok(),
|
||||
"shared-domain heal should rewrite the missing shard"
|
||||
);
|
||||
drop(barrier);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_heal_object_defers_when_requested_pool_is_suspended() {
|
||||
let mut store = minimal_heal_store().await;
|
||||
@@ -1697,8 +2019,10 @@ mod tests {
|
||||
.expect("quorum boundary heal should return a mapped result");
|
||||
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
|
||||
assert!(
|
||||
matches!(quorum_err, Some(Error::ErasureReadQuorum)),
|
||||
"quorum-boundary heal must preserve quorum error, got {quorum_err:?}"
|
||||
quorum_err.as_ref().is_some_and(|err| err
|
||||
.to_string()
|
||||
.contains("pool metadata writes remain blocked after a recovery-required replica state")),
|
||||
"heal must fail closed when capacity admission cannot verify pool metadata, got {quorum_err:?}"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
@@ -1808,6 +2132,7 @@ mod tests {
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
decommission_capacity_entry_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
};
|
||||
|
||||
+273
-231
@@ -17,18 +17,17 @@ use crate::core::pools::{
|
||||
PoolMetaReplicaState, PoolMetaWriteState, load_pool_meta_identity_observing, local_decommission_queue_prefix,
|
||||
persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission,
|
||||
};
|
||||
use crate::error::is_err_decommission_running;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::object::EcstoreObjectIO;
|
||||
use rustfs_config::server_config::KVS;
|
||||
use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token};
|
||||
use std::future::Future;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_STORE_INIT: &str = "store_init";
|
||||
const EVENT_DECOMMISSION_RESUME_RETRY: &str = "decommission_resume_retry";
|
||||
const EVENT_DECOMMISSION_RESUME_FAILED: &str = "decommission_resume_failed";
|
||||
const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry";
|
||||
const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status";
|
||||
const EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED: &str = "store_rpc_secret_preflight_failed";
|
||||
@@ -96,16 +95,13 @@ fn preflight_startup_rpc_secret_with(
|
||||
}
|
||||
}
|
||||
|
||||
const LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES: usize = 6;
|
||||
const LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(60 * 3);
|
||||
const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30);
|
||||
const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5);
|
||||
const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10);
|
||||
const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10);
|
||||
|
||||
fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool {
|
||||
matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES
|
||||
}
|
||||
|
||||
fn should_retry_format_load(err: &Error) -> bool {
|
||||
!matches!(err, Error::CorruptedFormat)
|
||||
}
|
||||
@@ -118,14 +114,6 @@ fn should_defer_rebalance_auto_start(distributed: bool, fleet_proof_available: b
|
||||
distributed && !fleet_proof_available
|
||||
}
|
||||
|
||||
fn should_schedule_local_decommission_resume(
|
||||
pool_indices: &[usize],
|
||||
pool_meta_replica_state: PoolMetaReplicaState,
|
||||
pool_meta_write_safe: bool,
|
||||
) -> bool {
|
||||
!pool_indices.is_empty() && pool_meta_replica_state.repair_write_safe && pool_meta_write_safe
|
||||
}
|
||||
|
||||
async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = rx.cancelled() => false,
|
||||
@@ -133,6 +121,13 @@ async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay:
|
||||
}
|
||||
}
|
||||
|
||||
fn local_decommission_watchdog_retry_delay(consecutive_failures: u32) -> Duration {
|
||||
let exponent = consecutive_failures.saturating_sub(1).min(4);
|
||||
LOCAL_DECOMMISSION_RESUME_RETRY_DELAY
|
||||
.saturating_mul(1_u32 << exponent)
|
||||
.min(LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY)
|
||||
}
|
||||
|
||||
async fn wait_for_rebalance_resume_delay(rx: &CancellationToken, delay: Duration) -> bool {
|
||||
tokio::select! {
|
||||
_ = rx.cancelled() => false,
|
||||
@@ -235,71 +230,63 @@ where
|
||||
Ok(committed)
|
||||
}
|
||||
|
||||
async fn resume_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken, pool_indices: Vec<usize>) {
|
||||
for attempt in 0..=LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES {
|
||||
async fn run_local_decommission_watchdog<F, Fut>(rx: CancellationToken, mut reconcile: F)
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
Fut: Future<Output = Result<()>>,
|
||||
{
|
||||
let mut consecutive_failures = 0_u32;
|
||||
|
||||
loop {
|
||||
if rx.is_cancelled() {
|
||||
return;
|
||||
}
|
||||
|
||||
let result = if pool_indices.len() > 1 {
|
||||
store
|
||||
.spawn_decommission_routines(store.clone(), rx.clone(), pool_indices.clone())
|
||||
.await
|
||||
} else {
|
||||
store.decommission(rx.clone(), pool_indices.clone()).await
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => return,
|
||||
Err(err) if is_err_decommission_running(&err) => {
|
||||
if let Err(spawn_err) = store
|
||||
.spawn_decommission_routines(store.clone(), rx.clone(), pool_indices.clone())
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_RESUME_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_STORE_INIT,
|
||||
pool_indices = ?pool_indices,
|
||||
error = %spawn_err,
|
||||
reason = "spawn_workers_failed",
|
||||
"Failed to resume decommission workers"
|
||||
);
|
||||
}
|
||||
return;
|
||||
let delay = match reconcile().await {
|
||||
Ok(()) => {
|
||||
consecutive_failures = 0;
|
||||
LOCAL_DECOMMISSION_WATCHDOG_INTERVAL
|
||||
}
|
||||
Err(err) if should_retry_local_decommission_resume(&err, attempt) => {
|
||||
Err(err) => {
|
||||
consecutive_failures = consecutive_failures.saturating_add(1);
|
||||
let retry_delay = local_decommission_watchdog_retry_delay(consecutive_failures);
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_RESUME_RETRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_STORE_INIT,
|
||||
pool_indices = ?pool_indices,
|
||||
retry_count = attempt + 1,
|
||||
retry_limit = LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES + 1,
|
||||
consecutive_failures,
|
||||
retry_delay_secs = retry_delay.as_secs(),
|
||||
error = %err,
|
||||
"Retrying decommission resume after missing config"
|
||||
"Retrying decommission worker recovery"
|
||||
);
|
||||
tokio::select! {
|
||||
_ = rx.cancelled() => return,
|
||||
_ = tokio::time::sleep(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY) => {}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_RESUME_FAILED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_STORE_INIT,
|
||||
pool_indices = ?pool_indices,
|
||||
error = %err,
|
||||
reason = "resume_failed",
|
||||
"Failed to resume decommission"
|
||||
);
|
||||
return;
|
||||
retry_delay
|
||||
}
|
||||
};
|
||||
|
||||
if !wait_for_local_decommission_resume_delay(&rx, delay).await {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn supervise_local_decommission_after_init(store: Arc<ECStore>, rx: CancellationToken) {
|
||||
run_local_decommission_watchdog(rx.clone(), || {
|
||||
let store = store.clone();
|
||||
let worker_rx = rx.clone();
|
||||
async move {
|
||||
store
|
||||
.ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery")
|
||||
.await?;
|
||||
if store.has_active_local_decommission_worker().await {
|
||||
return Ok(());
|
||||
}
|
||||
store.refresh_pool_status_meta().await?;
|
||||
store.spawn_missing_local_decommission_routines_with_token(worker_rx).await
|
||||
}
|
||||
})
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn resume_rebalance_after_init(store: Arc<ECStore>, rx: CancellationToken) {
|
||||
if !wait_for_rebalance_resume_delay(&rx, REBALANCE_INITIAL_RESUME_DELAY).await {
|
||||
return;
|
||||
@@ -575,6 +562,7 @@ impl ECStore {
|
||||
decommission_cancelers,
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::new(PoolMetaWriteState::for_startup(deployment_id, fresh_bootstrap_proven)),
|
||||
decommission_capacity_entry_gate: Mutex::default(),
|
||||
// Adopt the caller's context (the process bootstrap one on the
|
||||
// legacy path) so startup writes (erasure type recorded before
|
||||
// this point) and later reads share one cell.
|
||||
@@ -729,31 +717,32 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?;
|
||||
let has_local_decommission_leadership = endpoints.as_ref().iter().any(pool_first_endpoint_is_local);
|
||||
let pool_meta_write_safe = self
|
||||
.ensure_pool_meta_side_effects_safe("decommission resume blocked while pool metadata requires recovery")
|
||||
.await
|
||||
.is_ok();
|
||||
if should_schedule_local_decommission_resume(&local_pool_indices, pool_meta_replica_state, pool_meta_write_safe) {
|
||||
let store = self.clone();
|
||||
let decommission_rx = rx.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await {
|
||||
return;
|
||||
}
|
||||
resume_local_decommission_after_init(store, decommission_rx, local_pool_indices).await;
|
||||
});
|
||||
} else if !local_pool_indices.is_empty() {
|
||||
error!(
|
||||
event = EVENT_DECOMMISSION_RESUME_FAILED,
|
||||
if !pool_meta_replica_state.repair_write_safe || !pool_meta_write_safe {
|
||||
warn!(
|
||||
event = EVENT_DECOMMISSION_RESUME_RETRY,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_STORE_INIT,
|
||||
state = "blocked",
|
||||
pool_indices = ?local_pool_indices,
|
||||
reason = "pool_meta_write_blocked",
|
||||
"Decommission resume blocked until pool metadata replicas are readable and consistent"
|
||||
"Decommission watchdog waiting for pool metadata recovery"
|
||||
);
|
||||
}
|
||||
if has_local_decommission_leadership {
|
||||
let store = self.clone();
|
||||
let decommission_rx = rx.clone();
|
||||
tokio::spawn(async move {
|
||||
if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await {
|
||||
return;
|
||||
}
|
||||
supervise_local_decommission_after_init(store, decommission_rx).await;
|
||||
});
|
||||
}
|
||||
|
||||
runtime_sources::init_bucket_monitor_for_current_endpoints();
|
||||
crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat();
|
||||
@@ -786,12 +775,12 @@ impl ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{
|
||||
LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, PoolMetaWriteState, establish_pool_meta_bootstrap_identity_if_proven,
|
||||
load_pool_meta_for_startup, persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local,
|
||||
pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with,
|
||||
resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
|
||||
should_defer_rebalance_auto_start, should_retry_format_load, should_retry_local_decommission_resume,
|
||||
wait_for_local_decommission_resume_delay,
|
||||
LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState,
|
||||
establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay,
|
||||
persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission,
|
||||
preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result,
|
||||
run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init,
|
||||
should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay,
|
||||
};
|
||||
#[cfg(feature = "test-util")]
|
||||
use crate::disk::DiskAPI;
|
||||
@@ -849,8 +838,9 @@ mod tests {
|
||||
use crate::{
|
||||
bucket::replication::{ReplicationState, ReplicationStatusType, replication_statuses_map},
|
||||
core::pools::{
|
||||
POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION, PoolDecommissionInfo, PoolMeta, PoolStatus,
|
||||
pool_meta_identity_initialized_for_test, pool_meta_v3_commit_state_for_test,
|
||||
DecommissionErasureLayout, DecommissionPoolCapacityInfo, POOL_META_IDENTITY_NAME, POOL_META_NAME, POOL_META_VERSION,
|
||||
PoolDecommissionInfo, PoolMeta, PoolStatus, pool_meta_identity_initialized_for_test,
|
||||
pool_meta_v3_commit_state_for_test, set_decommission_capacity_info_overrides_for_test,
|
||||
},
|
||||
disk::endpoint::Endpoint,
|
||||
error::{Error, Result, StorageError},
|
||||
@@ -1481,15 +1471,6 @@ mod tests {
|
||||
assert!(err.to_string().contains("cannot overwrite an unreadable replica"));
|
||||
assert!(!valid.wrote_without_lock.load(Ordering::SeqCst));
|
||||
assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst));
|
||||
assert!(!super::should_schedule_local_decommission_resume(&[0], replica_state, true));
|
||||
assert!(!super::should_schedule_local_decommission_resume(
|
||||
&[0],
|
||||
crate::core::pools::PoolMetaReplicaState {
|
||||
needs_repair: false,
|
||||
repair_write_safe: true,
|
||||
},
|
||||
false,
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1517,7 +1498,6 @@ mod tests {
|
||||
.contains("restart after all replicas are readable and consistent")
|
||||
);
|
||||
assert!(!repaired.wrote_without_lock.load(Ordering::SeqCst));
|
||||
assert!(!super::should_schedule_local_decommission_resume(&[0], replica_state, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1551,21 +1531,66 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_local_decommission_resume_accepts_config_not_found_before_retry_limit() {
|
||||
assert!(should_retry_local_decommission_resume(&StorageError::ConfigNotFound, 0));
|
||||
fn test_local_decommission_watchdog_retry_delay_is_bounded() {
|
||||
assert_eq!(local_decommission_watchdog_retry_delay(1), LOCAL_DECOMMISSION_RESUME_RETRY_DELAY);
|
||||
assert_eq!(
|
||||
local_decommission_watchdog_retry_delay(u32::MAX),
|
||||
LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_local_decommission_resume_rejects_config_not_found_at_retry_limit() {
|
||||
assert!(!should_retry_local_decommission_resume(
|
||||
&StorageError::ConfigNotFound,
|
||||
LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES
|
||||
));
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() {
|
||||
let rx = CancellationToken::new();
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
|
||||
let attempts = attempts.clone();
|
||||
let rx = rx.clone();
|
||||
move || {
|
||||
let attempts = attempts.clone();
|
||||
let rx = rx.clone();
|
||||
async move {
|
||||
if attempts.fetch_add(1, Ordering::SeqCst) == 0 {
|
||||
Err(StorageError::SlowDown)
|
||||
} else {
|
||||
rx.cancel();
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await;
|
||||
task.await.expect("watchdog task should exit after cancellation");
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_local_decommission_resume_rejects_non_config_errors() {
|
||||
assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0));
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_local_decommission_watchdog_rescans_after_success() {
|
||||
let rx = CancellationToken::new();
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), {
|
||||
let attempts = attempts.clone();
|
||||
let rx = rx.clone();
|
||||
move || {
|
||||
let attempts = attempts.clone();
|
||||
let rx = rx.clone();
|
||||
async move {
|
||||
if attempts.fetch_add(1, Ordering::SeqCst) == 1 {
|
||||
rx.cancel();
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
tokio::task::yield_now().await;
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
tokio::time::advance(super::LOCAL_DECOMMISSION_WATCHDOG_INTERVAL).await;
|
||||
task.await.expect("watchdog task should exit after cancellation");
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2010,6 +2035,18 @@ mod tests {
|
||||
.await
|
||||
.expect("store should build around the fresh context");
|
||||
|
||||
// Capacity admission in these local fixtures must not depend on the
|
||||
// host volume's statvfs values. Keep enough identical snapshots for
|
||||
// startup, recovery, and the mutation probes exercised by each test.
|
||||
let layout = DecommissionErasureLayout { data: 2, parity: 2 };
|
||||
let snapshot: Vec<DecommissionPoolCapacityInfo> = store
|
||||
.pools
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(pool_index, _)| DecommissionPoolCapacityInfo::for_test(pool_index, layout, 1 << 40, 1 << 40, 1 << 30))
|
||||
.collect();
|
||||
set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| snapshot.clone()).collect());
|
||||
|
||||
(instance_ctx, store, shutdown)
|
||||
}
|
||||
|
||||
@@ -2119,8 +2156,16 @@ mod tests {
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
crate::data_movement::migrate_decommission_object(migration_store, 0, migration_bucket, source_reader, None, op_label)
|
||||
.await
|
||||
crate::data_movement::migrate_decommission_object(
|
||||
migration_store,
|
||||
0,
|
||||
migration_bucket,
|
||||
source_reader,
|
||||
None,
|
||||
op_label,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
});
|
||||
barrier.wait_until_paused().await;
|
||||
barrier.release();
|
||||
@@ -2132,12 +2177,31 @@ mod tests {
|
||||
(source_version, expected_source_versions)
|
||||
}
|
||||
|
||||
fn set_test_decommission_capacity_override(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
|
||||
let layout = DecommissionErasureLayout { data: 2, parity: 2 };
|
||||
let source_physical_bytes = 1024 * 1024 * 1024;
|
||||
let target_physical_bytes = source_physical_bytes * 8;
|
||||
let capacity: Vec<DecommissionPoolCapacityInfo> = store
|
||||
.pools
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, _)| {
|
||||
if index == pool_idx {
|
||||
DecommissionPoolCapacityInfo::for_test(index, layout, 0, source_physical_bytes, source_physical_bytes)
|
||||
} else {
|
||||
DecommissionPoolCapacityInfo::for_test(index, layout, target_physical_bytes, target_physical_bytes, 0)
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
set_decommission_capacity_info_overrides_for_test(store.id, (0..128).map(|_| capacity.clone()).collect());
|
||||
}
|
||||
|
||||
async fn mark_test_pool_decommissioning(store: &Arc<crate::store::ECStore>, pool_idx: usize) {
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
pool_meta.pools[pool_idx].decommission = Some(PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
});
|
||||
set_test_decommission_capacity_override(store, pool_idx);
|
||||
store
|
||||
.save_current_pool_meta_for_decommission_start(&[pool_idx], Vec::new())
|
||||
.await
|
||||
.expect("test decommission capacity reservation should activate");
|
||||
}
|
||||
|
||||
const DECOMMISSION_TEST_FAULT_STAGE_DELETE_MARKER: &str = "delete_marker_copy";
|
||||
@@ -2376,45 +2440,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
async fn assert_suspended_decommission_converged(store: &Arc<crate::store::ECStore>, bucket: &str, object: &str) {
|
||||
let source_versions = store.pools[0]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("source versions should remain readable after suspended convergence");
|
||||
assert!(
|
||||
source_versions.is_none_or(|versions| versions.versions.is_empty()),
|
||||
"worker convergence must remove only the decommissioned source null version"
|
||||
);
|
||||
|
||||
let target_versions = store.pools[1]
|
||||
.get_disks_by_key(object)
|
||||
.load_file_info_versions_exact(bucket, object)
|
||||
.await
|
||||
.expect("active target versions should be readable")
|
||||
.expect("active target must retain the suspended DELETE marker");
|
||||
assert!(
|
||||
matches!(target_versions.versions.as_slice(), [marker] if marker.deleted && marker.version_id.is_none_or(|version_id| version_id.is_nil())),
|
||||
"active target must contain only its null delete marker: {target_versions:?}"
|
||||
);
|
||||
|
||||
let err = store
|
||||
.get_object_info(
|
||||
bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("the active null delete marker must hide the migrated source generation");
|
||||
assert!(
|
||||
matches!(err, StorageError::ObjectNotFound(_, _)),
|
||||
"unexpected suspended latest-object result: {err:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(storage_class_env)]
|
||||
async fn tag_updates_skip_active_rebalance_source_pool() {
|
||||
@@ -2609,7 +2634,6 @@ mod tests {
|
||||
drop(lifecycle_guard);
|
||||
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
|
||||
let err = store
|
||||
.ensure_decommission_multipart_uploads_drained_for_test(0)
|
||||
.await
|
||||
@@ -4165,13 +4189,7 @@ mod tests {
|
||||
.put_object(&bucket, &object, &mut source, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write source object to the pool being decommissioned");
|
||||
{
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
pool_meta.pools[0].decommission = Some(PoolDecommissionInfo {
|
||||
start_time: Some(OffsetDateTime::now_utc()),
|
||||
..Default::default()
|
||||
});
|
||||
}
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
assert!(store.is_suspended(0).await, "pool 0 must be a suspended decommission source");
|
||||
|
||||
let barrier = crate::set_disk::PutObjectCommitBarrier::install(
|
||||
@@ -4907,9 +4925,10 @@ mod tests {
|
||||
.then(|| crate::set_disk::NewMultipartUploadCommitObservation::install(&bucket, object));
|
||||
let barrier = crate::set_disk::MultipartCommitBarrier::install(&bucket, object, pause);
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let recovery_source_set = Arc::clone(&source_set);
|
||||
let worker_store = Arc::clone(&store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
let mut worker = tokio::spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -4923,7 +4942,10 @@ mod tests {
|
||||
.await
|
||||
});
|
||||
|
||||
barrier.wait_until_paused().await;
|
||||
tokio::select! {
|
||||
() = barrier.wait_until_paused() => {}
|
||||
result = &mut worker => panic!("decommission multipart worker exited before the commit barrier: {result:?}"),
|
||||
}
|
||||
loss_hook.mark_lost();
|
||||
barrier.release();
|
||||
drop(barrier);
|
||||
@@ -4945,6 +4967,19 @@ mod tests {
|
||||
.await
|
||||
.expect("list target multipart uploads after fenced migration");
|
||||
assert!(uploads.uploads.is_empty(), "fenced multipart migration must not retain target staging");
|
||||
drop(loss_hook);
|
||||
store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
MetaCacheEntry {
|
||||
name: object.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
bucket.clone(),
|
||||
recovery_source_set,
|
||||
)
|
||||
.await
|
||||
.expect("same-mutation retry should recover the durable capacity intent");
|
||||
}
|
||||
|
||||
shutdown.cancel();
|
||||
@@ -5153,6 +5188,12 @@ mod tests {
|
||||
.expect("self-copy should keep using the committed active target");
|
||||
assert_eq!(active_copy_result.data_dir, active_copy_data_dir);
|
||||
|
||||
{
|
||||
let mut pool_meta = store.pool_meta.write().await;
|
||||
pool_meta.pools[1].decommission = None;
|
||||
}
|
||||
mark_test_pool_decommissioning(&store, 1).await;
|
||||
|
||||
let cleanup_barrier = crate::data_movement::SourceCleanupDeleteBarrier::install(&bucket, object);
|
||||
let commit_barrier = crate::set_disk::PutObjectCommitBarrier::install(
|
||||
&bucket,
|
||||
@@ -6166,28 +6207,27 @@ mod tests {
|
||||
write_suspended_decommission_source(&store, &bucket, object).await;
|
||||
mark_test_pool_decommissioning(&store, 0).await;
|
||||
|
||||
let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
|
||||
let delete_store = Arc::clone(&store);
|
||||
let delete_bucket = bucket.clone();
|
||||
let delete = tokio::spawn(async move {
|
||||
delete_store
|
||||
.delete_object(
|
||||
&delete_bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
delete_barrier.wait_until_paused().await;
|
||||
let delete_err = store
|
||||
.delete_object(
|
||||
&bucket,
|
||||
object,
|
||||
ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect_err("capacity-reserved target must reject a concurrent suspended DELETE");
|
||||
assert!(
|
||||
matches!(delete_err, Error::SlowDown),
|
||||
"unexpected suspended DELETE result: {delete_err:?}"
|
||||
);
|
||||
assert_suspended_null_source_present(&store, &bucket, object).await;
|
||||
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let worker_store = Arc::clone(&store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
tokio::spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -6199,25 +6239,25 @@ mod tests {
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
});
|
||||
})
|
||||
.await
|
||||
.expect("suspended decommission worker should join")
|
||||
.expect("worker must migrate the fenced suspended source");
|
||||
|
||||
delete_barrier.release();
|
||||
let marker = delete
|
||||
.await
|
||||
.expect("suspended DELETE task should join")
|
||||
.expect("suspended DELETE should commit its active-pool marker");
|
||||
drop(delete_barrier);
|
||||
assert!(marker.delete_marker, "suspended DELETE must create a marker");
|
||||
assert!(
|
||||
marker.version_id.is_none_or(|version_id| version_id.is_nil()),
|
||||
"suspended DELETE marker must keep the null version identity"
|
||||
assert_decommission_source_absent(
|
||||
&store,
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
read_decommission_target_body(&store, &bucket, object, &ObjectOptions::default()).await,
|
||||
b"suspended source generation"
|
||||
);
|
||||
worker
|
||||
.await
|
||||
.expect("suspended decommission worker should join")
|
||||
.expect("worker must treat the newer active null marker as a completed migration");
|
||||
|
||||
assert_suspended_decommission_converged(&store, &bucket, object).await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
@@ -6250,31 +6290,29 @@ mod tests {
|
||||
},
|
||||
None,
|
||||
));
|
||||
let delete_barrier = crate::store::object::VersionedDeleteMarkerCommitBarrier::install(&bucket, object);
|
||||
let delete_store = Arc::clone(&store);
|
||||
let delete_bucket = bucket.clone();
|
||||
let delete = tokio::spawn(async move {
|
||||
delete_store
|
||||
.delete_objects(
|
||||
&delete_bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(delete_config_snapshot),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
});
|
||||
delete_barrier.wait_until_paused().await;
|
||||
let (_deleted, errors) = store
|
||||
.delete_objects(
|
||||
&bucket,
|
||||
vec![ObjectToDelete {
|
||||
object_name: object.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
ObjectOptions {
|
||||
delete_replication_config_snapshot: Some(delete_config_snapshot),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
matches!(errors.as_slice(), [Some(Error::SlowDown)]),
|
||||
"unexpected suspended batch DELETE result: {errors:?}"
|
||||
);
|
||||
assert_suspended_null_source_present(&store, &bucket, object).await;
|
||||
|
||||
let source_set = store.pools[0].get_disks_by_key(object);
|
||||
let worker_store = Arc::clone(&store);
|
||||
let worker_bucket = bucket.clone();
|
||||
let worker = tokio::spawn(async move {
|
||||
tokio::spawn(async move {
|
||||
worker_store
|
||||
.decommission_entry_for_test(
|
||||
0,
|
||||
@@ -6286,22 +6324,25 @@ mod tests {
|
||||
source_set,
|
||||
)
|
||||
.await
|
||||
});
|
||||
})
|
||||
.await
|
||||
.expect("suspended batch decommission worker should join")
|
||||
.expect("worker must migrate the batch-fenced suspended source");
|
||||
|
||||
delete_barrier.release();
|
||||
let (deleted, errors) = delete.await.expect("suspended batch DELETE task should join");
|
||||
drop(delete_barrier);
|
||||
assert!(errors.iter().all(Option::is_none), "suspended batch DELETE should succeed: {errors:?}");
|
||||
assert!(
|
||||
matches!(deleted.as_slice(), [marker] if marker.delete_marker && marker.delete_marker_version_id.is_none_or(|version_id| version_id.is_nil())),
|
||||
"suspended batch DELETE must create one null marker: {deleted:?}"
|
||||
assert_decommission_source_absent(
|
||||
&store,
|
||||
&bucket,
|
||||
object,
|
||||
&ObjectOptions {
|
||||
version_suspended: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
read_decommission_target_body(&store, &bucket, object, &ObjectOptions::default()).await,
|
||||
b"suspended source generation"
|
||||
);
|
||||
worker
|
||||
.await
|
||||
.expect("suspended batch decommission worker should join")
|
||||
.expect("worker must treat the newer batch null marker as a completed migration");
|
||||
|
||||
assert_suspended_decommission_converged(&store, &bucket, object).await;
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
@@ -7294,6 +7335,7 @@ mod tests {
|
||||
.await
|
||||
.expect("legacy decommission queue should reload after restart");
|
||||
*store.pool_meta.write().await = restarted_pool_meta;
|
||||
set_test_decommission_capacity_override(&store, 0);
|
||||
store
|
||||
.promote_queued_decommission_for_test(0)
|
||||
.await
|
||||
|
||||
+860
-15
@@ -44,7 +44,7 @@ use crate::error::{
|
||||
use crate::runtime::global::DISK_RESERVE_FRACTION;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission};
|
||||
use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission};
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions},
|
||||
list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions},
|
||||
@@ -260,6 +260,12 @@ pub struct ECStore {
|
||||
/// Lock order: acquire `pool_meta_save_gate`, then the distributed
|
||||
/// `pool.bin` fence, then clone `pool_meta` under a short read lock.
|
||||
pub(crate) pool_meta_save_gate: Mutex<PoolMetaWriteState>,
|
||||
/// Serializes decommission entries while the durable capacity ledger has
|
||||
/// one target mutation intent slot.
|
||||
///
|
||||
/// Lock order: acquire this gate before object namespaces or
|
||||
/// `pool_meta_save_gate`.
|
||||
pub(crate) decommission_capacity_entry_gate: Mutex<()>,
|
||||
/// Per-instance runtime state (Phase 5, backlog#939).
|
||||
///
|
||||
/// Carries this instance's identity/runtime out of the process globals so
|
||||
@@ -273,6 +279,215 @@ pub struct ECStore {
|
||||
pub(crate) bucket_fence_registry: Arc<bucket_fence::BucketFenceRegistry>,
|
||||
}
|
||||
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds";
|
||||
const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items";
|
||||
const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause";
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ScannerDataMovementPauseReason {
|
||||
OperationEpochExhausted,
|
||||
MovementGenerationExhausted,
|
||||
DecommissionActive,
|
||||
DecommissionFailed,
|
||||
DecommissionCanceled,
|
||||
RebalanceActive,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)]
|
||||
pub struct ScannerDataMovementPauseStatus {
|
||||
pub paused: bool,
|
||||
pub policy: &'static str,
|
||||
pub reasons: Vec<ScannerDataMovementPauseReason>,
|
||||
pub started_at_unix_secs: u64,
|
||||
pub duration_seconds: u64,
|
||||
pub operation_epoch: u64,
|
||||
pub movement_generation: u64,
|
||||
pub movement_backlog_work_items: u64,
|
||||
pub movement_backlog_estimated: bool,
|
||||
}
|
||||
|
||||
impl Default for ScannerDataMovementPauseStatus {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
paused: false,
|
||||
policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
|
||||
reasons: Vec::new(),
|
||||
started_at_unix_secs: 0,
|
||||
duration_seconds: 0,
|
||||
operation_epoch: 0,
|
||||
movement_generation: 0,
|
||||
movement_backlog_work_items: 0,
|
||||
movement_backlog_estimated: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn offset_unix_seconds(value: OffsetDateTime) -> u64 {
|
||||
u64::try_from(value.unix_timestamp()).unwrap_or(0)
|
||||
}
|
||||
|
||||
fn earliest_timestamp(current: Option<OffsetDateTime>, candidate: Option<OffsetDateTime>) -> Option<OffsetDateTime> {
|
||||
match (current, candidate) {
|
||||
(Some(current), Some(candidate)) => Some(current.min(candidate)),
|
||||
(Some(current), None) => Some(current),
|
||||
(None, candidate) => candidate,
|
||||
}
|
||||
}
|
||||
|
||||
fn usize_to_u64(value: usize) -> u64 {
|
||||
u64::try_from(value).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
fn metric_u64(value: u64) -> f64 {
|
||||
f64::from(u32::try_from(value).unwrap_or(u32::MAX))
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 {
|
||||
let timestamp = value.unix_timestamp_nanos();
|
||||
if timestamp <= 0 {
|
||||
0
|
||||
} else {
|
||||
u64::try_from(timestamp).unwrap_or(u64::MAX)
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option<u64> {
|
||||
let generation = scanner_data_movement_timestamp_generation(value);
|
||||
(generation != 0 && generation != u64::MAX).then_some(generation)
|
||||
}
|
||||
|
||||
fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 {
|
||||
let mut generation = 0;
|
||||
for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) {
|
||||
let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else {
|
||||
return u64::MAX;
|
||||
};
|
||||
generation = generation.max(pool_generation);
|
||||
}
|
||||
|
||||
for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| {
|
||||
meta.stopped_at.into_iter().chain(
|
||||
meta.pool_stats
|
||||
.iter()
|
||||
.flat_map(|pool| [pool.info.start_time, pool.info.end_time])
|
||||
.flatten(),
|
||||
)
|
||||
}) {
|
||||
let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else {
|
||||
return u64::MAX;
|
||||
};
|
||||
generation = generation.max(rebalance_generation);
|
||||
}
|
||||
|
||||
if generation == 0
|
||||
&& rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some())
|
||||
{
|
||||
u64::MAX
|
||||
} else {
|
||||
generation
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
struct ScannerDataMovementSequenceState {
|
||||
operation_epoch: u64,
|
||||
operation_epoch_exhausted: bool,
|
||||
movement_generation: u64,
|
||||
movement_generation_exhausted: bool,
|
||||
}
|
||||
|
||||
fn resolve_scanner_data_movement_pause_status(
|
||||
pool_meta: &PoolMeta,
|
||||
rebalance_meta: Option<&RebalanceMeta>,
|
||||
decommission_worker_active: bool,
|
||||
sequence: ScannerDataMovementSequenceState,
|
||||
now: OffsetDateTime,
|
||||
) -> ScannerDataMovementPauseStatus {
|
||||
let mut decommission_active = decommission_worker_active;
|
||||
let mut decommission_failed = false;
|
||||
let mut decommission_canceled = false;
|
||||
let mut rebalance_active = false;
|
||||
let mut started_at = None;
|
||||
let mut movement_backlog_work_items = 0_u64;
|
||||
|
||||
for pool in &pool_meta.pools {
|
||||
let Some(info) = pool.decommission.as_ref() else {
|
||||
continue;
|
||||
};
|
||||
let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled;
|
||||
let failed = !info.queued && info.failed;
|
||||
let canceled = !info.queued && info.canceled;
|
||||
if !(active || failed || canceled) {
|
||||
continue;
|
||||
}
|
||||
|
||||
decommission_active |= active;
|
||||
decommission_failed |= failed;
|
||||
decommission_canceled |= canceled;
|
||||
started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update)));
|
||||
let queued = usize_to_u64(info.queued_buckets.len());
|
||||
let current_bucket = if info.bucket.is_empty() { 0 } else { 1 };
|
||||
movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket));
|
||||
}
|
||||
|
||||
if let Some(rebalance_meta) = rebalance_meta {
|
||||
for pool in &rebalance_meta.pool_stats {
|
||||
let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping;
|
||||
if !active {
|
||||
continue;
|
||||
}
|
||||
rebalance_active = true;
|
||||
started_at = earliest_timestamp(started_at, pool.info.start_time);
|
||||
movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len()));
|
||||
}
|
||||
}
|
||||
|
||||
let mut reasons = Vec::with_capacity(6);
|
||||
if sequence.operation_epoch_exhausted {
|
||||
reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted);
|
||||
}
|
||||
if sequence.movement_generation_exhausted {
|
||||
reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted);
|
||||
}
|
||||
if decommission_active {
|
||||
reasons.push(ScannerDataMovementPauseReason::DecommissionActive);
|
||||
}
|
||||
if decommission_failed {
|
||||
reasons.push(ScannerDataMovementPauseReason::DecommissionFailed);
|
||||
}
|
||||
if decommission_canceled {
|
||||
reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled);
|
||||
}
|
||||
if rebalance_active {
|
||||
reasons.push(ScannerDataMovementPauseReason::RebalanceActive);
|
||||
}
|
||||
let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0);
|
||||
let duration_seconds = started_at
|
||||
.and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok())
|
||||
.unwrap_or(0);
|
||||
let paused = !reasons.is_empty();
|
||||
|
||||
ScannerDataMovementPauseStatus {
|
||||
paused,
|
||||
policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY,
|
||||
reasons,
|
||||
started_at_unix_secs,
|
||||
duration_seconds,
|
||||
operation_epoch: sequence.operation_epoch,
|
||||
movement_generation: sequence.movement_generation,
|
||||
movement_backlog_work_items,
|
||||
movement_backlog_estimated: paused,
|
||||
}
|
||||
}
|
||||
|
||||
fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) {
|
||||
metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 });
|
||||
metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds));
|
||||
metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items));
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ECStore {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum();
|
||||
@@ -300,6 +515,28 @@ impl ECStore {
|
||||
self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect()
|
||||
}
|
||||
|
||||
/// Erasure sets that may receive scanner pause-backlog replicas.
|
||||
///
|
||||
/// An actively decommissioning or already decommissioned source pool is
|
||||
/// excluded so an operational record acknowledged during movement always
|
||||
/// has a copy on storage that remains in the cluster. The record is kept
|
||||
/// separate from pool and rebalance metadata.
|
||||
pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec<Arc<crate::set_disk::SetDisks>> {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
self.pools
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(pool_index, _)| {
|
||||
!pool_meta.pools.get(*pool_index).is_some_and(|pool| {
|
||||
pool.decommission
|
||||
.as_ref()
|
||||
.is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled)
|
||||
})
|
||||
})
|
||||
.flat_map(|(_, pool)| pool.disk_set.iter().cloned())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Get server configuration (delegates to global)
|
||||
pub fn get_server_config(&self) -> Option<Config> {
|
||||
runtime_sources::server_config()
|
||||
@@ -454,14 +691,14 @@ impl ECStore {
|
||||
self.scanner_data_usage_publication_snapshot_blocked().await
|
||||
}
|
||||
|
||||
pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus {
|
||||
let operation_gate = self.ctx.data_movement_operation_gate();
|
||||
let _operation_guard = operation_gate.read_owned().await;
|
||||
self.scanner_data_movement_pause_snapshot().await
|
||||
}
|
||||
|
||||
async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool {
|
||||
if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() {
|
||||
self.ctx.set_scanner_publication_state(true);
|
||||
return true;
|
||||
}
|
||||
let (_, blocked) = self.scanner_data_movement_snapshot_locked().await;
|
||||
self.ctx.set_scanner_publication_state(blocked);
|
||||
blocked
|
||||
self.scanner_data_movement_pause_snapshot().await.paused
|
||||
}
|
||||
|
||||
async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) {
|
||||
@@ -481,19 +718,56 @@ impl ECStore {
|
||||
.as_ref()
|
||||
.is_some_and(|info| !info.queued && (info.failed || info.canceled))
|
||||
});
|
||||
drop(pool_meta);
|
||||
|
||||
let rebalance_active = self
|
||||
.rebalance_meta
|
||||
.read()
|
||||
.await
|
||||
let rebalance_meta = self.rebalance_meta.read().await;
|
||||
let rebalance_active = rebalance_meta
|
||||
.as_ref()
|
||||
.is_some_and(is_rebalance_conflicting_with_decommission);
|
||||
self.ctx
|
||||
.observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
|
||||
&pool_meta,
|
||||
rebalance_meta.as_ref(),
|
||||
));
|
||||
|
||||
let blocked = decommission_active || decommission_terminal || rebalance_active;
|
||||
(decommission_active || rebalance_active, blocked)
|
||||
}
|
||||
|
||||
async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus {
|
||||
let decommission_active = {
|
||||
let decommission_cancelers = self.decommission_cancelers.read().await;
|
||||
decommission_cancelers
|
||||
.iter()
|
||||
.any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active))
|
||||
};
|
||||
let pool_meta = self.pool_meta.read().await.clone();
|
||||
let rebalance_meta = self.rebalance_meta.read().await.clone();
|
||||
self.ctx
|
||||
.observe_durable_data_movement_generation(durable_scanner_data_movement_generation(
|
||||
&pool_meta,
|
||||
rebalance_meta.as_ref(),
|
||||
));
|
||||
let status = resolve_scanner_data_movement_pause_status(
|
||||
&pool_meta,
|
||||
rebalance_meta.as_ref(),
|
||||
decommission_active,
|
||||
ScannerDataMovementSequenceState {
|
||||
operation_epoch: self.ctx.data_movement_operation_epoch(),
|
||||
operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(),
|
||||
movement_generation: self.ctx.data_movement_generation(),
|
||||
movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(),
|
||||
},
|
||||
OffsetDateTime::now_utc(),
|
||||
);
|
||||
self.ctx.set_scanner_publication_state(status.paused);
|
||||
record_scanner_data_movement_pause_status(&status);
|
||||
status
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus {
|
||||
self.scanner_data_movement_pause_snapshot().await
|
||||
}
|
||||
|
||||
/// Admit one short data-usage publication commit under the same
|
||||
/// per-instance gate used by decommission side effects and transitions.
|
||||
/// The epoch is sampled while the read guard is held, so a transition
|
||||
@@ -1196,7 +1470,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::core::pools::{PoolDecommissionInfo, PoolStatus};
|
||||
use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus};
|
||||
use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::runtime::global::reset_local_disk_test_state;
|
||||
@@ -1205,6 +1479,23 @@ mod tests {
|
||||
use serial_test::serial;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn g_d2_008_default_versioning_config_keeps_persisted_bytes() {
|
||||
let bytes = crate::bucket::utils::serialize::<VersioningConfiguration>(&ENABLED_VERSIONING_CONFIG)
|
||||
.expect("the default Versioning configuration must serialize");
|
||||
assert_eq!(bytes, b"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn g_d2_009_default_object_lock_config_keeps_persisted_bytes() {
|
||||
let bytes = crate::bucket::utils::serialize::<ObjectLockConfiguration>(&ENABLED_OBJECT_LOCK_CONFIG)
|
||||
.expect("the default Object Lock configuration must serialize");
|
||||
assert_eq!(
|
||||
bytes,
|
||||
b"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_get_disk_infos() {
|
||||
let disks = vec![None, None]; // Empty disks for testing
|
||||
@@ -1229,6 +1520,7 @@ mod tests {
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
decommission_capacity_entry_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
};
|
||||
@@ -1304,11 +1596,564 @@ mod tests {
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
decommission_capacity_entry_gate: Mutex::default(),
|
||||
ctx,
|
||||
bucket_fence_registry: Arc::default(),
|
||||
})
|
||||
}
|
||||
|
||||
fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState {
|
||||
ScannerDataMovementSequenceState {
|
||||
operation_epoch,
|
||||
operation_epoch_exhausted: false,
|
||||
movement_generation,
|
||||
movement_generation_exhausted: false,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_pause_status_derives_restart_stable_decommission_fields() {
|
||||
let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid");
|
||||
let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid");
|
||||
let pool_meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: started_at,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
start_time: Some(started_at),
|
||||
queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
|
||||
bucket: "bucket-a".to_string(),
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now);
|
||||
|
||||
assert!(status.paused);
|
||||
assert_eq!(status.policy, "global_pause");
|
||||
assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]);
|
||||
assert_eq!(status.started_at_unix_secs, 1_000);
|
||||
assert_eq!(status.duration_seconds, 90);
|
||||
assert_eq!(status.operation_epoch, 7);
|
||||
assert_eq!(status.movement_generation, 11);
|
||||
assert_eq!(status.movement_backlog_work_items, 2);
|
||||
assert!(status.movement_backlog_estimated);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_decommission_restores_durable_movement_generation() {
|
||||
let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
|
||||
let pool_meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: completed_at,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
complete: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
|
||||
let ctx = InstanceContext::new();
|
||||
|
||||
ctx.observe_durable_data_movement_generation(durable_generation);
|
||||
|
||||
assert_eq!(durable_generation, 1_100_000_000_000);
|
||||
assert_eq!(ctx.data_movement_generation(), durable_generation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cleared_decommission_restores_durable_movement_generation_after_restart() {
|
||||
let mut pool_meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: OffsetDateTime::UNIX_EPOCH,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear"));
|
||||
assert!(
|
||||
pool_meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.is_some_and(|info| !info.has_decommission_state())
|
||||
);
|
||||
let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None);
|
||||
let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
*restarted.pool_meta.write().await = pool_meta;
|
||||
|
||||
let status = restarted.scanner_data_movement_pause_status().await;
|
||||
|
||||
assert_ne!(durable_generation, 0);
|
||||
assert!(!status.paused);
|
||||
assert_eq!(status.movement_generation, durable_generation);
|
||||
assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() {
|
||||
let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid");
|
||||
let mut pool_meta = PoolMeta {
|
||||
pools: vec![
|
||||
PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: same_tick,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
PoolStatus {
|
||||
id: 1,
|
||||
cmd_line: "pool-1".to_string(),
|
||||
last_update: same_tick,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
canceled: true,
|
||||
..Default::default()
|
||||
}),
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
pool_meta
|
||||
.clear_decommission_at_for_test(0, same_tick, None)
|
||||
.expect("first terminal decommission should clear")
|
||||
);
|
||||
let first_generation = durable_scanner_data_movement_generation(&pool_meta, None);
|
||||
assert_eq!(
|
||||
first_generation,
|
||||
scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1))
|
||||
);
|
||||
|
||||
assert!(
|
||||
pool_meta
|
||||
.clear_decommission_at_for_test(1, same_tick, None)
|
||||
.expect("second terminal decommission should clear")
|
||||
);
|
||||
let second_generation = durable_scanner_data_movement_generation(&pool_meta, None);
|
||||
assert_eq!(
|
||||
second_generation,
|
||||
scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2))
|
||||
);
|
||||
assert!(second_generation > first_generation);
|
||||
|
||||
let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
*restarted.pool_meta.write().await = pool_meta;
|
||||
let status = restarted.scanner_data_movement_pause_status().await;
|
||||
|
||||
assert!(!status.paused);
|
||||
assert_eq!(status.movement_generation, second_generation);
|
||||
assert_eq!(restarted.scanner_data_movement_generation(), second_generation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() {
|
||||
let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid");
|
||||
let earlier_tick = same_tick - time::Duration::nanoseconds(10);
|
||||
let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
|
||||
let rebalance = RebalanceMeta {
|
||||
stopped_at: Some(rebalance_floor),
|
||||
id: "completed-rebalance".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let active_decommission = |id| PoolStatus {
|
||||
id,
|
||||
cmd_line: format!("pool-{id}"),
|
||||
last_update: same_tick,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
start_time: Some(same_tick),
|
||||
..Default::default()
|
||||
}),
|
||||
};
|
||||
let mut pool_meta = PoolMeta {
|
||||
pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance)));
|
||||
assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1));
|
||||
|
||||
assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance)));
|
||||
assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2));
|
||||
|
||||
assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance)));
|
||||
assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3));
|
||||
let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
|
||||
assert_eq!(
|
||||
durable_generation,
|
||||
scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3))
|
||||
);
|
||||
|
||||
let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
*restarted.pool_meta.write().await = pool_meta;
|
||||
*restarted.rebalance_meta.write().await = Some(rebalance);
|
||||
let status = restarted.scanner_data_movement_pause_status().await;
|
||||
|
||||
assert_eq!(status.movement_generation, durable_generation);
|
||||
assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
|
||||
assert_eq!(
|
||||
status.reasons,
|
||||
vec![
|
||||
ScannerDataMovementPauseReason::DecommissionFailed,
|
||||
ScannerDataMovementPauseReason::DecommissionCanceled
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() {
|
||||
let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid");
|
||||
let earlier_tick = same_tick - time::Duration::nanoseconds(10);
|
||||
let rebalance_floor = same_tick + time::Duration::nanoseconds(5);
|
||||
let rebalance = RebalanceMeta {
|
||||
stopped_at: Some(rebalance_floor),
|
||||
id: "completed-rebalance".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut pool_meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: same_tick,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
pool_meta
|
||||
.clear_decommission_at_for_test(0, same_tick, Some(&rebalance))
|
||||
.expect("failed decommission should clear")
|
||||
);
|
||||
let cleared_at = rebalance_floor + time::Duration::nanoseconds(1);
|
||||
assert_eq!(pool_meta.pools[0].last_update, cleared_at);
|
||||
|
||||
pool_meta
|
||||
.decommission_at_for_test(
|
||||
0,
|
||||
PoolSpaceInfo {
|
||||
total: 200,
|
||||
free: 50,
|
||||
used: 150,
|
||||
},
|
||||
earlier_tick,
|
||||
Some(&rebalance),
|
||||
)
|
||||
.expect("decommission restart after clear should be allowed");
|
||||
let started_at = cleared_at + time::Duration::nanoseconds(1);
|
||||
assert_eq!(pool_meta.pools[0].last_update, started_at);
|
||||
assert_eq!(
|
||||
pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time),
|
||||
Some(started_at)
|
||||
);
|
||||
|
||||
assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance)));
|
||||
let completed_at = started_at + time::Duration::nanoseconds(1);
|
||||
assert_eq!(pool_meta.pools[0].last_update, completed_at);
|
||||
let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
|
||||
assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at));
|
||||
|
||||
let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
*restarted.pool_meta.write().await = pool_meta;
|
||||
*restarted.rebalance_meta.write().await = Some(rebalance);
|
||||
let status = restarted.scanner_data_movement_pause_status().await;
|
||||
|
||||
assert!(!status.paused);
|
||||
assert_eq!(status.movement_generation, durable_generation);
|
||||
assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() {
|
||||
let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid");
|
||||
let earlier_tick = terminal_at - time::Duration::nanoseconds(10);
|
||||
let rebalance_floor = terminal_at + time::Duration::nanoseconds(5);
|
||||
let rebalance = RebalanceMeta {
|
||||
stopped_at: Some(rebalance_floor),
|
||||
id: "completed-rebalance".to_string(),
|
||||
..Default::default()
|
||||
};
|
||||
let mut pool_meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: terminal_at,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
start_time: Some(terminal_at),
|
||||
complete: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert!(
|
||||
pool_meta
|
||||
.record_decommission_terminal_reload_failure_at_for_test(
|
||||
0,
|
||||
"complete_decommission",
|
||||
"peer reload failed".to_string(),
|
||||
earlier_tick,
|
||||
Some(&rebalance),
|
||||
)
|
||||
.expect("reload failure should be recorded")
|
||||
);
|
||||
let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1);
|
||||
assert_eq!(pool_meta.pools[0].last_update, reload_failure_at);
|
||||
let info = pool_meta.pools[0]
|
||||
.decommission
|
||||
.as_ref()
|
||||
.expect("decommission metadata should exist");
|
||||
assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at));
|
||||
assert_eq!(
|
||||
info.terminal_reload_failures,
|
||||
vec!["complete_decommission: peer reload failed".to_string()]
|
||||
);
|
||||
let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance));
|
||||
assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at));
|
||||
|
||||
let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
*restarted.pool_meta.write().await = pool_meta;
|
||||
*restarted.rebalance_meta.write().await = Some(rebalance);
|
||||
let status = restarted.scanner_data_movement_pause_status().await;
|
||||
|
||||
assert!(!status.paused);
|
||||
assert_eq!(status.movement_generation, durable_generation);
|
||||
assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() {
|
||||
let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid");
|
||||
let earlier_tick = same_tick - time::Duration::nanoseconds(10);
|
||||
let decommission_floor = same_tick + time::Duration::nanoseconds(5);
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
*store.pool_meta.write().await = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: decommission_floor,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
complete: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let started_at = store.next_scanner_data_movement_update(same_tick).await;
|
||||
assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1));
|
||||
*store.rebalance_meta.write().await = Some(RebalanceMeta {
|
||||
id: "rebalance-generation".to_string(),
|
||||
pool_stats: vec![crate::services::rebalance::RebalanceStats {
|
||||
participating: true,
|
||||
info: crate::services::rebalance::RebalanceInfo {
|
||||
start_time: Some(started_at),
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
let completed_at = store.next_scanner_data_movement_update(same_tick).await;
|
||||
assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2));
|
||||
{
|
||||
let mut rebalance_meta = store.rebalance_meta.write().await;
|
||||
let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
|
||||
meta.pool_stats[0].info.status = RebalStatus::Completed;
|
||||
meta.pool_stats[0].info.end_time = Some(completed_at);
|
||||
}
|
||||
|
||||
let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await;
|
||||
assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3));
|
||||
{
|
||||
let mut rebalance_meta = store.rebalance_meta.write().await;
|
||||
let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present");
|
||||
meta.stopped_at = Some(stopped_at);
|
||||
}
|
||||
let pool_meta = store.pool_meta.read().await.clone();
|
||||
let rebalance_meta = store.rebalance_meta.read().await.clone();
|
||||
let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref());
|
||||
assert_eq!(
|
||||
durable_generation,
|
||||
scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3))
|
||||
);
|
||||
|
||||
let restarted = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
*restarted.pool_meta.write().await = pool_meta;
|
||||
*restarted.rebalance_meta.write().await = rebalance_meta;
|
||||
let status = restarted.scanner_data_movement_pause_status().await;
|
||||
|
||||
assert!(!status.paused);
|
||||
assert_eq!(status.movement_generation, durable_generation);
|
||||
assert_eq!(restarted.scanner_data_movement_generation(), durable_generation);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() {
|
||||
let pool_meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: OffsetDateTime::UNIX_EPOCH,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
complete: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX);
|
||||
let exhausted_generation =
|
||||
OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid");
|
||||
assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() {
|
||||
let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid");
|
||||
let pool_meta = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update: OffsetDateTime::UNIX_EPOCH,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
complete: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let rebalance_meta = RebalanceMeta {
|
||||
id: "completed-rebalance".to_string(),
|
||||
stopped_at: Some(valid_rebalance_at),
|
||||
pool_stats: vec![crate::services::rebalance::RebalanceStats {
|
||||
participating: true,
|
||||
info: crate::services::rebalance::RebalanceInfo {
|
||||
start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)),
|
||||
end_time: Some(valid_rebalance_at),
|
||||
status: RebalStatus::Completed,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn durable_movement_generation_without_records_is_zero() {
|
||||
assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0);
|
||||
assert_eq!(
|
||||
durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())),
|
||||
0
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() {
|
||||
let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid");
|
||||
let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid");
|
||||
let failed = PoolMeta {
|
||||
pools: vec![PoolStatus {
|
||||
id: 0,
|
||||
cmd_line: "pool-0".to_string(),
|
||||
last_update,
|
||||
decommission: Some(PoolDecommissionInfo {
|
||||
failed: true,
|
||||
..Default::default()
|
||||
}),
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now);
|
||||
assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]);
|
||||
assert_eq!(failed_status.started_at_unix_secs, 2_000);
|
||||
assert_eq!(failed_status.duration_seconds, 30);
|
||||
|
||||
let rebalance = RebalanceMeta {
|
||||
pool_stats: vec![crate::services::rebalance::RebalanceStats {
|
||||
buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()],
|
||||
participating: true,
|
||||
info: crate::services::rebalance::RebalanceInfo {
|
||||
start_time: Some(last_update),
|
||||
status: RebalStatus::Started,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
}],
|
||||
..Default::default()
|
||||
};
|
||||
let rebalance_status = resolve_scanner_data_movement_pause_status(
|
||||
&PoolMeta::default(),
|
||||
Some(&rebalance),
|
||||
false,
|
||||
scanner_sequence_state(4, 13),
|
||||
now,
|
||||
);
|
||||
assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]);
|
||||
assert_eq!(rebalance_status.movement_backlog_work_items, 2);
|
||||
|
||||
let exhausted = resolve_scanner_data_movement_pause_status(
|
||||
&PoolMeta::default(),
|
||||
None,
|
||||
false,
|
||||
ScannerDataMovementSequenceState {
|
||||
operation_epoch: u64::MAX,
|
||||
operation_epoch_exhausted: true,
|
||||
movement_generation: 14,
|
||||
movement_generation_exhausted: false,
|
||||
},
|
||||
now,
|
||||
);
|
||||
assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]);
|
||||
assert_eq!(exhausted.started_at_unix_secs, 0);
|
||||
|
||||
let generation_exhausted = resolve_scanner_data_movement_pause_status(
|
||||
&PoolMeta::default(),
|
||||
None,
|
||||
false,
|
||||
ScannerDataMovementSequenceState {
|
||||
operation_epoch: 5,
|
||||
operation_epoch_exhausted: false,
|
||||
movement_generation: u64::MAX,
|
||||
movement_generation_exhausted: true,
|
||||
},
|
||||
now,
|
||||
);
|
||||
assert_eq!(
|
||||
generation_exhausted.reasons,
|
||||
vec![ScannerDataMovementPauseReason::MovementGenerationExhausted]
|
||||
);
|
||||
|
||||
let idle =
|
||||
resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now);
|
||||
assert!(!idle.paused);
|
||||
assert!(idle.reasons.is_empty());
|
||||
assert!(!idle.movement_backlog_estimated);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() {
|
||||
let store = build_store_with_ctx(Arc::new(InstanceContext::new()));
|
||||
|
||||
@@ -13,6 +13,7 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::core::pools::{DecommissionCapacityOwner, ensure_decommission_capacity_mutation_id};
|
||||
use crate::multipart_listing::paginate_multipart_listing;
|
||||
use crate::set_disk::get_lock_acquire_timeout;
|
||||
use crate::storage_api_contracts::multipart::MultipartOperations as _;
|
||||
@@ -196,6 +197,21 @@ async fn list_pool_multipart_uploads_for_incarnation(
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
pub(crate) async fn acquire_decommission_multipart_mutation_fence(
|
||||
&self,
|
||||
owner: DecommissionCapacityOwner,
|
||||
) -> Result<ObjectLockDiagGuard> {
|
||||
let mutation_id = owner
|
||||
.mutation_id
|
||||
.ok_or_else(|| Error::other("decommission multipart mutation identity is missing"))?;
|
||||
let object = format!(
|
||||
"decommission-multipart/{}/{}/{}/{}",
|
||||
owner.source_pool_index, owner.operation_id, owner.generation, mutation_id
|
||||
);
|
||||
self.acquire_object_write_lock("decommission_multipart_mutation", crate::disk::RUSTFS_META_MULTIPART_BUCKET, &object)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn existing_multipart_pool_order(&self) -> Vec<usize> {
|
||||
// A draining source must not hide a valid UploadID in an active target,
|
||||
// while physical order within each phase preserves fail-closed errors.
|
||||
@@ -215,6 +231,24 @@ impl ECStore {
|
||||
active
|
||||
}
|
||||
|
||||
async fn multipart_upload_pool_idx(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_id: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<usize> {
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
match self.pools[pool_idx].get_multipart_info(bucket, object, upload_id, opts).await {
|
||||
Ok(_) => return Ok(pool_idx),
|
||||
Err(err) if is_err_invalid_upload_id(&err) => continue,
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub async fn list_multipart_uploads_for_bucket_incarnation(
|
||||
&self,
|
||||
@@ -433,8 +467,10 @@ impl ECStore {
|
||||
if self.single_pool() {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
return self.pools[0]
|
||||
.new_multipart_upload(bucket, object, &opts)
|
||||
return self
|
||||
.run_decommission_capacity_admitted_mutation(0, None, None, || async {
|
||||
self.pools[0].new_multipart_upload(bucket, object, &opts).await
|
||||
})
|
||||
.await
|
||||
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
@@ -450,7 +486,14 @@ impl ECStore {
|
||||
}
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
let res = self
|
||||
.run_decommission_capacity_temporary_mutation(
|
||||
idx,
|
||||
DecommissionCapacityOwner::from_options(&opts),
|
||||
None,
|
||||
|| async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
|
||||
)
|
||||
.await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
@@ -475,8 +518,19 @@ impl ECStore {
|
||||
if !res.uploads.is_empty() {
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
return Ok((res, idx, opts.expected_bucket_incarnation_id));
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
|
||||
let lock_object = encode_dir_object(object);
|
||||
let res = self
|
||||
.run_external_decommission_capacity_object_mutation(
|
||||
idx,
|
||||
bucket,
|
||||
&lock_object,
|
||||
object,
|
||||
opts,
|
||||
|opts| async move { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
|
||||
)
|
||||
.await?;
|
||||
return Ok((res, idx, expected_bucket_incarnation_id));
|
||||
}
|
||||
}
|
||||
let idx = self.get_pool_idx(bucket, object, -1).await?;
|
||||
@@ -490,8 +544,23 @@ impl ECStore {
|
||||
|
||||
self.apply_decommission_target_mutation_fence(idx, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
let res = self.pools[idx].new_multipart_upload(bucket, object, &opts).await?;
|
||||
Ok((res, idx, opts.expected_bucket_incarnation_id))
|
||||
let expected_bucket_incarnation_id = opts.expected_bucket_incarnation_id;
|
||||
let res = if opts.data_movement {
|
||||
self.run_decommission_capacity_temporary_mutation(
|
||||
idx,
|
||||
DecommissionCapacityOwner::from_options(&opts),
|
||||
None,
|
||||
|| async { self.pools[idx].new_multipart_upload(bucket, object, &opts).await },
|
||||
)
|
||||
.await?
|
||||
} else {
|
||||
let lock_object = encode_dir_object(object);
|
||||
self.run_external_decommission_capacity_object_mutation(idx, bucket, &lock_object, object, opts, |opts| async move {
|
||||
self.pools[idx].new_multipart_upload(bucket, object, &opts).await
|
||||
})
|
||||
.await?
|
||||
};
|
||||
Ok((res, idx, expected_bucket_incarnation_id))
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -529,7 +598,8 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
let opts = &opts;
|
||||
|
||||
if self.single_pool() {
|
||||
@@ -538,26 +608,10 @@ impl ECStore {
|
||||
.await;
|
||||
}
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
let pool = &self.pools[pool_idx];
|
||||
let err = match pool.put_object_part(bucket, object, upload_id, part_id, data, opts).await {
|
||||
Ok(res) => return Ok(res),
|
||||
Err(err) => {
|
||||
if is_err_invalid_upload_id(&err) {
|
||||
None
|
||||
} else {
|
||||
Some(err)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(err) = err {
|
||||
error!("put_object_part err: {:?}", err);
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||
let pool_idx = self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await?;
|
||||
self.pools[pool_idx]
|
||||
.put_object_part(bucket, object, upload_id, part_id, data, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn put_object_part_for_data_movement(
|
||||
@@ -576,12 +630,24 @@ impl ECStore {
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart upload requires data_movement options"));
|
||||
}
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let pool = self
|
||||
.pools
|
||||
.get(target_pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
|
||||
pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
let pool = self.pools.get(target_pool_idx).ok_or_else(|| {
|
||||
Error::InvalidArgument("data-movement".to_string(), "target-pool".to_string(), target_pool_idx.to_string())
|
||||
})?;
|
||||
let expected_data_bytes = usize::try_from(data.size()).ok();
|
||||
self.run_decommission_capacity_temporary_mutation_with_capacity_lease(
|
||||
target_pool_idx,
|
||||
DecommissionCapacityOwner::from_options(&opts),
|
||||
expected_data_bytes,
|
||||
|capacity_lease| async move {
|
||||
if let Some(capacity_lease) = capacity_lease {
|
||||
opts.add_namespace_lock_lost_signal(capacity_lease);
|
||||
}
|
||||
pool.put_object_part(bucket, object, upload_id, part_id, data, &opts).await
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -662,16 +728,95 @@ impl ECStore {
|
||||
upload_id: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart abort requires data_movement options"));
|
||||
}
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &[upload_id.to_owned()], None, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn reconcile_multipart_uploads_for_data_movement(
|
||||
&self,
|
||||
target_pool_idx: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_identity: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
let pool = self
|
||||
.pools
|
||||
.get(target_pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
|
||||
pool.abort_multipart_upload(bucket, object, upload_id, &opts).await
|
||||
let owner = DecommissionCapacityOwner::from_options(opts);
|
||||
let has_capacity_state = match owner {
|
||||
Some(owner) => {
|
||||
self.has_decommission_capacity_temporary_mutation_state(target_pool_idx, owner)
|
||||
.await
|
||||
}
|
||||
None => false,
|
||||
};
|
||||
if !has_capacity_state {
|
||||
return Ok(());
|
||||
}
|
||||
let set = pool.get_disks_by_key(object);
|
||||
let upload_ids = set
|
||||
.data_movement_multipart_upload_ids(bucket, object, opts.expected_bucket_incarnation_id, upload_identity)
|
||||
.await?;
|
||||
self.abort_multipart_uploads_for_data_movement(target_pool_idx, bucket, object, &upload_ids, Some(upload_identity), opts)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn abort_multipart_uploads_for_data_movement(
|
||||
&self,
|
||||
target_pool_idx: usize,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
upload_ids: &[String],
|
||||
expected_upload_identity: Option<&str>,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<()> {
|
||||
check_new_multipart_args(bucket, object)?;
|
||||
for upload_id in upload_ids {
|
||||
check_abort_multipart_args(bucket, object, upload_id)?;
|
||||
}
|
||||
if !opts.data_movement {
|
||||
return Err(Error::other("targeted multipart abort requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
let pool = self
|
||||
.pools
|
||||
.get(target_pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?;
|
||||
let set = pool.get_disks_by_key(object);
|
||||
let mut guards = Vec::with_capacity(upload_ids.len());
|
||||
for upload_id in upload_ids {
|
||||
if let Some(guard) = set
|
||||
.lock_data_movement_multipart_abort(bucket, object, upload_id, expected_upload_identity, &opts)
|
||||
.await?
|
||||
{
|
||||
guard.add_namespace_lock_fence(&mut opts);
|
||||
guards.push(guard);
|
||||
}
|
||||
}
|
||||
opts.no_lock = true;
|
||||
let capacity_owner = DecommissionCapacityOwner::from_options(&opts);
|
||||
// Keep every upload namespace guard alive through the final capacity progress save.
|
||||
let result = self
|
||||
.run_decommission_capacity_temporary_release_with_capacity_lease(target_pool_idx, capacity_owner, |capacity_lease| {
|
||||
let mut delete_opts = opts.clone();
|
||||
let guards = &guards;
|
||||
let set = &set;
|
||||
async move {
|
||||
if let Some(capacity_lease) = capacity_lease {
|
||||
delete_opts.add_namespace_lock_lost_signal(capacity_lease);
|
||||
}
|
||||
for guard in guards {
|
||||
guard.delete(set, bucket, object, &delete_opts).await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
})
|
||||
.await;
|
||||
drop(guards);
|
||||
result
|
||||
}
|
||||
|
||||
#[instrument(skip(self))]
|
||||
@@ -684,7 +829,8 @@ impl ECStore {
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let (opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
let opts = &opts;
|
||||
|
||||
if self.single_pool() {
|
||||
@@ -694,27 +840,10 @@ impl ECStore {
|
||||
.await;
|
||||
}
|
||||
|
||||
for pool_idx in self.existing_multipart_pool_order().await {
|
||||
let pool = &self.pools[pool_idx];
|
||||
|
||||
let pool = pool.clone();
|
||||
let err = match pool
|
||||
.complete_multipart_upload(bucket, object, upload_id, uploaded_parts.clone(), opts)
|
||||
.await
|
||||
{
|
||||
Ok(res) => return Ok(res),
|
||||
Err(err) => {
|
||||
//
|
||||
if is_err_invalid_upload_id(&err) { None } else { Some(err) }
|
||||
}
|
||||
};
|
||||
|
||||
if let Some(er) = err {
|
||||
return Err(er);
|
||||
}
|
||||
}
|
||||
|
||||
Err(StorageError::InvalidUploadID(bucket.to_owned(), object.to_owned(), upload_id.to_owned()))
|
||||
let pool_idx = self.multipart_upload_pool_idx(bucket, object, upload_id, opts).await?;
|
||||
let pool = self.pools[pool_idx].clone();
|
||||
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, opts)
|
||||
.await
|
||||
}
|
||||
|
||||
pub(crate) async fn complete_multipart_upload_for_data_movement(
|
||||
@@ -732,6 +861,7 @@ impl ECStore {
|
||||
return Err(Error::other("targeted multipart completion requires data_movement options"));
|
||||
}
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
ensure_decommission_capacity_mutation_id(bucket, object, &mut opts);
|
||||
if opts.overwrites_existing_version() && !is_meta_bucketname(bucket) {
|
||||
let expected_incarnation_id = opts
|
||||
.expected_bucket_incarnation_id
|
||||
@@ -764,12 +894,24 @@ impl ECStore {
|
||||
.get(target_pool_idx)
|
||||
.ok_or_else(|| Error::other(format!("data movement target pool {target_pool_idx} is out of range")))?
|
||||
.clone();
|
||||
let result = enqueue_transition_after_write(
|
||||
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts)
|
||||
.await,
|
||||
LcEventSrc::S3CompleteMultipartUpload,
|
||||
)
|
||||
.await;
|
||||
// Data movement already owns the pool-meta write lease. Forward its
|
||||
// loss signal into SetDisks so commit fencing observes the same lease
|
||||
// without trying to reacquire the namespace.
|
||||
let result = self
|
||||
.run_decommission_capacity_admitted_mutation_with_capacity_lease(
|
||||
target_pool_idx,
|
||||
DecommissionCapacityOwner::from_options(&opts),
|
||||
opts.capacity_expected_data_bytes(),
|
||||
|capacity_lease| async move {
|
||||
if let Some(capacity_lease) = capacity_lease {
|
||||
opts.add_namespace_lock_lost_signal(capacity_lease);
|
||||
}
|
||||
pool.complete_multipart_upload(bucket, object, upload_id, uploaded_parts, &opts)
|
||||
.await
|
||||
},
|
||||
)
|
||||
.await;
|
||||
let result = enqueue_transition_after_write(result, LcEventSrc::S3CompleteMultipartUpload).await;
|
||||
if result.is_ok() {
|
||||
list_objects::observe_list_objects_mutation(self.as_ref(), bucket).await;
|
||||
}
|
||||
@@ -961,6 +1103,7 @@ mod tests {
|
||||
decommission_cancelers: RwLock::new(Vec::new()),
|
||||
start_gate: Mutex::new(()),
|
||||
pool_meta_save_gate: Mutex::default(),
|
||||
decommission_capacity_entry_gate: Mutex::default(),
|
||||
ctx: crate::runtime::instance::bootstrap_ctx(),
|
||||
bucket_fence_registry: std::sync::Arc::default(),
|
||||
}
|
||||
|
||||
+936
-154
File diff suppressed because it is too large
Load Diff
@@ -18,18 +18,117 @@ use crate::core::pools::merge_pool_status_refresh;
|
||||
use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space};
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _};
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
pub(in crate::store) mod support;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_POOLS: &str = "pools";
|
||||
const EVENT_POOL_META_RELOAD: &str = "pool_meta_reload";
|
||||
|
||||
#[cfg(test)]
|
||||
struct PreparedPoolReadFallbackBarrierState {
|
||||
object: String,
|
||||
pause_before_refetch: bool,
|
||||
fanout_arrived: tokio::sync::Notify,
|
||||
fanout_release: tokio::sync::Notify,
|
||||
refetch_arrived: tokio::sync::Notify,
|
||||
refetch_release: tokio::sync::Notify,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(in crate::store) struct PreparedPoolReadFallbackBarrier {
|
||||
state: Arc<PreparedPoolReadFallbackBarrierState>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
static PREPARED_POOL_READ_FALLBACK_BARRIER: std::sync::OnceLock<
|
||||
std::sync::Mutex<Option<Arc<PreparedPoolReadFallbackBarrierState>>>,
|
||||
> = std::sync::OnceLock::new();
|
||||
|
||||
#[cfg(test)]
|
||||
impl PreparedPoolReadFallbackBarrier {
|
||||
pub(in crate::store) fn install(object: &str, pause_before_refetch: bool) -> Self {
|
||||
let state = Arc::new(PreparedPoolReadFallbackBarrierState {
|
||||
object: object.to_string(),
|
||||
pause_before_refetch,
|
||||
fanout_arrived: tokio::sync::Notify::new(),
|
||||
fanout_release: tokio::sync::Notify::new(),
|
||||
refetch_arrived: tokio::sync::Notify::new(),
|
||||
refetch_release: tokio::sync::Notify::new(),
|
||||
});
|
||||
*PREPARED_POOL_READ_FALLBACK_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("prepared pool read fallback barrier must not be poisoned") = Some(Arc::clone(&state));
|
||||
Self { state }
|
||||
}
|
||||
|
||||
pub(in crate::store) async fn wait_after_fanout(&self) {
|
||||
self.state.fanout_arrived.notified().await;
|
||||
}
|
||||
|
||||
pub(in crate::store) fn release_after_fanout(&self) {
|
||||
self.state.fanout_release.notify_one();
|
||||
}
|
||||
|
||||
pub(in crate::store) async fn wait_before_refetch(&self) {
|
||||
self.state.refetch_arrived.notified().await;
|
||||
}
|
||||
|
||||
pub(in crate::store) fn release_before_refetch(&self) {
|
||||
self.state.refetch_release.notify_one();
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl Drop for PreparedPoolReadFallbackBarrier {
|
||||
fn drop(&mut self) {
|
||||
self.state.fanout_release.notify_waiters();
|
||||
self.state.refetch_release.notify_waiters();
|
||||
if let Some(barrier) = PREPARED_POOL_READ_FALLBACK_BARRIER.get() {
|
||||
*barrier
|
||||
.lock()
|
||||
.expect("prepared pool read fallback barrier must not be poisoned") = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_prepared_pool_read_after_fanout(object: &str) {
|
||||
let state = PREPARED_POOL_READ_FALLBACK_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("prepared pool read fallback barrier must not be poisoned")
|
||||
.as_ref()
|
||||
.filter(|state| state.object == object)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.fanout_arrived.notify_one();
|
||||
state.fanout_release.notified().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
async fn pause_prepared_pool_read_before_refetch(object: &str) {
|
||||
let state = PREPARED_POOL_READ_FALLBACK_BARRIER
|
||||
.get_or_init(|| std::sync::Mutex::new(None))
|
||||
.lock()
|
||||
.expect("prepared pool read fallback barrier must not be poisoned")
|
||||
.as_ref()
|
||||
.filter(|state| state.object == object && state.pause_before_refetch)
|
||||
.cloned();
|
||||
if let Some(state) = state {
|
||||
state.refetch_arrived.notify_one();
|
||||
state.refetch_release.notified().await;
|
||||
}
|
||||
}
|
||||
#[cfg(test)]
|
||||
use support::resolve_latest_object_info_candidates;
|
||||
use support::{
|
||||
LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error,
|
||||
rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates_with_pool_state,
|
||||
resolve_rebalance_delete_from_all_pools_result, resolve_rebalance_delete_from_all_pools_results,
|
||||
resolve_store_rebalance_pool_meta_reload_result,
|
||||
resolve_store_rebalance_pool_meta_reload_result, validate_prepared_pool_refetch_identity,
|
||||
};
|
||||
|
||||
#[derive(Debug, Default, Eq, PartialEq)]
|
||||
@@ -675,6 +774,134 @@ impl ECStore {
|
||||
resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)
|
||||
}
|
||||
|
||||
pub(super) async fn prepare_latest_object_metadata_with_idx(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
opts: &ObjectOptions,
|
||||
) -> Result<(crate::set_disk::PreparedGetObjectMetadata, usize)> {
|
||||
let suspended_pools = if opts.skip_decommissioned {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
Some(
|
||||
(0..self.pools.len())
|
||||
.map(|idx| pool_meta.is_suspended(idx))
|
||||
.collect::<Vec<_>>(),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let mut futures = FuturesUnordered::new();
|
||||
for (idx, pool) in self.pools.iter().enumerate() {
|
||||
if suspended_pools.as_ref().is_some_and(|pools| pools[idx]) {
|
||||
continue;
|
||||
}
|
||||
|
||||
if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await {
|
||||
continue;
|
||||
}
|
||||
|
||||
futures.push(async move {
|
||||
let result = pool
|
||||
.prepare_get_object_reader_metadata(bucket, object, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket, object]));
|
||||
(idx, result)
|
||||
});
|
||||
}
|
||||
|
||||
let mut candidates = (0..self.pools.len()).map(|_| None).collect::<Vec<_>>();
|
||||
// Retain one provisional winner. Other pools only need their lightweight
|
||||
// identity for final conflict checks; if pool state changes while the
|
||||
// fanout runs, the final winner is refetched and revalidated below.
|
||||
let mut latest_prepared = None;
|
||||
let mut latest_mod_time = None;
|
||||
let mut provisional_dynamic_pool_state = None;
|
||||
while let Some((idx, result)) = futures.next().await {
|
||||
match result {
|
||||
Ok(metadata) => {
|
||||
let mod_time = metadata.object_info().mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
let info = metadata.object_info().clone();
|
||||
let retain = match (latest_mod_time, latest_prepared.as_ref()) {
|
||||
(None, _) => true,
|
||||
(Some(current), _) if mod_time > current => true,
|
||||
(Some(current), _) if mod_time < current => false,
|
||||
(Some(_), Some((current_idx, _))) => {
|
||||
if suspended_pools.is_none() && provisional_dynamic_pool_state.is_none() {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
provisional_dynamic_pool_state = Some(
|
||||
(0..self.pools.len())
|
||||
.map(|pool_idx| pool_meta.is_suspended(pool_idx))
|
||||
.collect::<Vec<_>>(),
|
||||
);
|
||||
}
|
||||
let provisional_pool_state = suspended_pools
|
||||
.as_ref()
|
||||
.or(provisional_dynamic_pool_state.as_ref())
|
||||
.ok_or_else(|| Error::other("GET pool state snapshot is unavailable"))?;
|
||||
let new_key = (provisional_pool_state.get(idx).copied().unwrap_or(false), std::cmp::Reverse(idx));
|
||||
let current_key = (
|
||||
provisional_pool_state.get(*current_idx).copied().unwrap_or(false),
|
||||
std::cmp::Reverse(*current_idx),
|
||||
);
|
||||
new_key < current_key
|
||||
}
|
||||
(Some(_), None) => true,
|
||||
};
|
||||
if retain {
|
||||
if latest_mod_time.is_none_or(|current| mod_time > current) {
|
||||
latest_mod_time = Some(mod_time);
|
||||
}
|
||||
latest_prepared = Some((idx, metadata));
|
||||
}
|
||||
candidates[idx] = Some(LatestObjectInfoCandidate {
|
||||
info: Some(info),
|
||||
idx,
|
||||
err: None,
|
||||
});
|
||||
}
|
||||
Err(err) => {
|
||||
candidates[idx] = Some(LatestObjectInfoCandidate {
|
||||
info: None,
|
||||
idx,
|
||||
err: Some(err),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pause_prepared_pool_read_after_fanout(object).await;
|
||||
|
||||
let suspended_pools = match suspended_pools {
|
||||
Some(pools) => pools,
|
||||
None => {
|
||||
let pool_meta = self.pool_meta.read().await;
|
||||
(0..self.pools.len())
|
||||
.map(|idx| pool_meta.is_suspended(idx))
|
||||
.collect::<Vec<_>>()
|
||||
}
|
||||
};
|
||||
|
||||
let candidates = candidates.into_iter().flatten().collect();
|
||||
let (winner_info, winner_idx) =
|
||||
resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)?;
|
||||
if let Some((prepared_idx, metadata)) = latest_prepared
|
||||
&& prepared_idx == winner_idx
|
||||
{
|
||||
return Ok((metadata, winner_idx));
|
||||
}
|
||||
|
||||
let pool = self.pools.get(winner_idx).ok_or(Error::ErasureReadQuorum)?;
|
||||
#[cfg(test)]
|
||||
pause_prepared_pool_read_before_refetch(object).await;
|
||||
let metadata = pool
|
||||
.prepare_get_object_reader_metadata(bucket, object, opts)
|
||||
.await
|
||||
.map_err(|err| to_object_err(err, vec![bucket, object]))?;
|
||||
validate_prepared_pool_refetch_identity(&winner_info, metadata.object_info())?;
|
||||
Ok((metadata, winner_idx))
|
||||
}
|
||||
|
||||
pub(super) async fn delete_object_from_all_pools(
|
||||
&self,
|
||||
bucket: &str,
|
||||
@@ -698,9 +925,19 @@ impl ECStore {
|
||||
}
|
||||
|
||||
if let Some(idx) = pe.index {
|
||||
let pool = self.pools[idx].clone();
|
||||
results.push(RebalanceDeletePoolResult {
|
||||
pool_idx: idx,
|
||||
result: self.pools[idx].delete_object(bucket, object, opts.clone()).await,
|
||||
result: self
|
||||
.run_external_decommission_capacity_object_delete(
|
||||
idx,
|
||||
bucket,
|
||||
object,
|
||||
object,
|
||||
opts.clone(),
|
||||
|opts| async move { pool.delete_object(bucket, object, opts).await },
|
||||
)
|
||||
.await,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -218,7 +218,7 @@ fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
/// excluded. The selected winner still carries the chosen pool's layout, while
|
||||
/// the remaining read-visible fields must agree before the pool index can
|
||||
/// provide a deterministic tie-break.
|
||||
fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
pub(super) fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool {
|
||||
let same_read_surface = left.bucket == right.bucket
|
||||
&& left.name == right.name
|
||||
&& left.is_dir == right.is_dir
|
||||
@@ -277,6 +277,14 @@ fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bo
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn validate_prepared_pool_refetch_identity(expected: &ObjectInfo, refetched: &ObjectInfo) -> Result<()> {
|
||||
if same_latest_object_info_identity(expected, refetched) {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(Error::ErasureReadQuorum)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn resolve_latest_object_info_candidates(
|
||||
candidates: Vec<LatestObjectInfoCandidate>,
|
||||
@@ -328,7 +336,11 @@ pub(super) fn resolve_latest_object_info_candidates_with_pool_state(
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
return Ok((winner_info.clone(), winner.idx));
|
||||
let winner = latest_candidates.swap_remove(0);
|
||||
let Some(winner_info) = winner.info else {
|
||||
return Err(Error::ErasureReadQuorum);
|
||||
};
|
||||
return Ok((winner_info, winner.idx));
|
||||
}
|
||||
|
||||
for candidate in candidates {
|
||||
@@ -347,6 +359,23 @@ pub(super) fn resolve_latest_object_info_candidates_with_pool_state(
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn prepared_pool_refetch_identity_fails_closed_on_generation_change() {
|
||||
let expected = ObjectInfo {
|
||||
mod_time: Some(OffsetDateTime::from_unix_timestamp(10).expect("test timestamp should be valid")),
|
||||
version_id: Some(uuid::Uuid::from_u128(1)),
|
||||
etag: Some("etag-a".to_string()),
|
||||
..Default::default()
|
||||
};
|
||||
let mut refetched = expected.clone();
|
||||
refetched.etag = Some("etag-b".to_string());
|
||||
|
||||
let error = validate_prepared_pool_refetch_identity(&expected, &refetched)
|
||||
.expect_err("refetched metadata from a changed generation must fail closed");
|
||||
|
||||
assert_eq!(error, Error::ErasureReadQuorum);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rebalance_delete_result_preserves_precondition_failed() {
|
||||
let err = resolve_rebalance_delete_from_all_pools_result(Err(Error::PreconditionFailed), "bucket", "object")
|
||||
|
||||
@@ -24,7 +24,7 @@ use crate::heal::{
|
||||
use crate::{Error, Result};
|
||||
use metrics::{counter, histogram};
|
||||
use rustfs_common::trace_bus::{TraceEvent, TraceFunc, TraceKind, trace_emit};
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_heal_contracts::heal_channel::{DriveState, HealOpts, HealRequestSource, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::path::SLASH_SEPARATOR;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
@@ -16,6 +16,22 @@ use super::*;
|
||||
use crate::heal::progress::{add_bytes, increment_counter, stable_generation};
|
||||
use crate::heal::utils::format_set_disk_id;
|
||||
|
||||
fn unavailable_recreate_error(result: &HealResultItem, opts: &HealOpts) -> Option<Error> {
|
||||
if opts.dry_run || !opts.recreate {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut offline = false;
|
||||
for drive in &result.after.drives {
|
||||
if drive.state == DriveState::Faulty.to_str() {
|
||||
return Some(Error::Disk(DiskError::FaultyDisk));
|
||||
}
|
||||
offline |= drive.state == DriveState::Offline.to_str();
|
||||
}
|
||||
|
||||
offline.then_some(Error::Disk(DiskError::DiskNotFound))
|
||||
}
|
||||
|
||||
impl HealTask {
|
||||
pub(super) async fn heal_bucket(&self, bucket: &str) -> Result<()> {
|
||||
debug!(
|
||||
@@ -335,13 +351,16 @@ impl HealTask {
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok((result, None)) => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |=
|
||||
!add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
|
||||
self.record_result_item(result).await;
|
||||
None
|
||||
}
|
||||
Ok((result, None)) => match unavailable_recreate_error(&result, &heal_opts) {
|
||||
Some(error) => Some(error),
|
||||
None => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
telemetry_unknown |=
|
||||
!add_bytes(&mut bytes, u64::try_from(result.object_size).unwrap_or(u64::MAX));
|
||||
self.record_result_item(result).await;
|
||||
None
|
||||
}
|
||||
},
|
||||
Ok((_, Some(err))) if is_missing_object_dir_heal_result(object, &err) => {
|
||||
telemetry_unknown |= !increment_counter(&mut healed);
|
||||
debug!(
|
||||
|
||||
@@ -706,11 +706,28 @@ enum MockHealObjectOutcome {
|
||||
OkWithOtherError(&'static str),
|
||||
ErrOther(&'static str),
|
||||
DanglingGraceDeferred,
|
||||
UnavailableDrive(DriveState),
|
||||
RetryableReadQuorum,
|
||||
RetryableSlowDown,
|
||||
PermanentOther(&'static str),
|
||||
}
|
||||
|
||||
fn unavailable_drive_heal_result(state: DriveState) -> (HealResultItem, Option<Error>) {
|
||||
(
|
||||
HealResultItem {
|
||||
after: Infos {
|
||||
drives: vec![HealDriveInfo {
|
||||
endpoint: "remote-target".to_string(),
|
||||
state: state.to_string(),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
..Default::default()
|
||||
},
|
||||
None,
|
||||
)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum MockObjectExists {
|
||||
Exists(bool),
|
||||
@@ -813,6 +830,7 @@ impl HealStorageAPI for MockStorage {
|
||||
"dangling object deletion deferred by heal grace window; retry_after_secs=3599; grace_secs=3600",
|
||||
))),
|
||||
)),
|
||||
MockHealObjectOutcome::UnavailableDrive(state) => Ok(unavailable_drive_heal_result(state)),
|
||||
MockHealObjectOutcome::RetryableReadQuorum => Err(Error::Storage(EcstoreError::InsufficientReadQuorum(
|
||||
bucket.to_string(),
|
||||
object.to_string(),
|
||||
@@ -833,6 +851,7 @@ impl HealStorageAPI for MockStorage {
|
||||
"dangling object deletion deferred by heal grace window; retry_after_secs=3599; grace_secs=3600",
|
||||
))),
|
||||
)),
|
||||
MockHealObjectOutcome::UnavailableDrive(state) => Ok(unavailable_drive_heal_result(state)),
|
||||
MockHealObjectOutcome::OkWithOtherError(message) => Ok((HealResultItem::default(), Some(Error::other(message)))),
|
||||
MockHealObjectOutcome::ErrOther(message) | MockHealObjectOutcome::PermanentOther(message) => {
|
||||
Err(Error::other(message))
|
||||
@@ -1449,6 +1468,46 @@ async fn test_recursive_bucket_heal_retries_only_retryable_objects() {
|
||||
assert_eq!(progress.objects_failed, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn recursive_bucket_heal_retries_when_recreate_target_is_unavailable() {
|
||||
for state in [DriveState::Offline, DriveState::Faulty] {
|
||||
let state_name = state.to_string();
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::UnavailableDrive(state)]));
|
||||
let request = HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
recreate_missing: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
);
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
task.heal_bucket("bucket-a")
|
||||
.await
|
||||
.expect("an unavailable recreate target should be retried after it returns");
|
||||
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().unwrap().as_slice(),
|
||||
["object-a".to_string(), "object-b".to_string(), "object-a".to_string()],
|
||||
"unexpected calls for unavailable state {state_name}"
|
||||
);
|
||||
let progress = task.get_progress().await;
|
||||
assert_eq!(progress.objects_scanned, 2, "unexpected scanned count for state {state_name}");
|
||||
assert_eq!(progress.objects_healed, 2, "unexpected healed count for state {state_name}");
|
||||
assert_eq!(progress.objects_failed, 0, "unexpected failed count for state {state_name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn recursive_bucket_heal_skips_dangling_delete_grace_without_batch_failure() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
@@ -1486,6 +1545,90 @@ async fn recursive_bucket_heal_skips_dangling_delete_grace_without_batch_failure
|
||||
assert_eq!(progress.skipped_objects, 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn recursive_bucket_heal_preserves_non_recreate_and_non_availability_results() {
|
||||
for (dry_run, recreate_missing, state) in [
|
||||
(true, true, DriveState::Offline),
|
||||
(true, true, DriveState::Faulty),
|
||||
(false, false, DriveState::Offline),
|
||||
(false, false, DriveState::Faulty),
|
||||
(false, true, DriveState::Ok),
|
||||
(false, true, DriveState::Missing),
|
||||
(false, true, DriveState::Corrupt),
|
||||
(false, true, DriveState::PermissionDenied),
|
||||
(false, true, DriveState::Unknown("other failure".to_string())),
|
||||
] {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage
|
||||
.heal_object_outcomes
|
||||
.lock()
|
||||
.expect("test outcome lock")
|
||||
.insert("object-a".to_string(), VecDeque::from([MockHealObjectOutcome::UnavailableDrive(state)]));
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
dry_run,
|
||||
recreate_missing,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
task.heal_bucket("bucket-a")
|
||||
.await
|
||||
.expect("unchanged best-effort result should not schedule an availability retry");
|
||||
assert_eq!(
|
||||
storage.heal_object_calls.lock().expect("test call lock").as_slice(),
|
||||
["object-a".to_string(), "object-b".to_string()]
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn recursive_bucket_heal_exhausts_unavailable_target_without_rescanning_healthy_objects() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
storage.heal_object_outcomes.lock().expect("test outcome lock").insert(
|
||||
"object-a".to_string(),
|
||||
(0..4)
|
||||
.map(|_| MockHealObjectOutcome::UnavailableDrive(DriveState::Faulty))
|
||||
.collect(),
|
||||
);
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::new(
|
||||
HealType::Bucket {
|
||||
bucket: "bucket-a".to_string(),
|
||||
},
|
||||
HealOptions {
|
||||
recursive: true,
|
||||
recreate_missing: true,
|
||||
timeout: None,
|
||||
..Default::default()
|
||||
},
|
||||
HealPriority::Normal,
|
||||
),
|
||||
storage.clone(),
|
||||
);
|
||||
let error = task
|
||||
.heal_bucket("bucket-a")
|
||||
.await
|
||||
.expect_err("persistent unavailable target must not report success");
|
||||
assert!(matches!(error, Error::TaskExecutionFailed { .. }));
|
||||
let failure = task
|
||||
.take_batch_failure()
|
||||
.await
|
||||
.expect("exhausted availability failure should be retained");
|
||||
assert_eq!((failure.failed, failure.retryable, failure.permanent), (1, 1, 0));
|
||||
let calls = storage.heal_object_calls.lock().expect("test call lock");
|
||||
assert_eq!(calls.iter().filter(|object| object.as_str() == "object-a").count(), 4);
|
||||
assert_eq!(calls.iter().filter(|object| object.as_str() == "object-b").count(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn test_recursive_bucket_heal_reports_typed_exhausted_and_permanent_failures() {
|
||||
let storage = Arc::new(MockStorage::default());
|
||||
|
||||
@@ -1537,7 +1537,7 @@ where
|
||||
Ok(deleted_at)
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<()> {
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
if access_key.is_empty() || secret_key.is_empty() {
|
||||
return Err(Error::InvalidArgument);
|
||||
}
|
||||
@@ -1552,7 +1552,16 @@ where
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
// Status is captured from the same credential snapshot the new secret
|
||||
// is persisted with, so a caller replicating the rotation broadcasts
|
||||
// exactly what was written rather than re-reading racily.
|
||||
let status = if cred.is_valid() {
|
||||
AccountStatus::Enabled
|
||||
} else {
|
||||
AccountStatus::Disabled
|
||||
};
|
||||
let u = UserIdentity::from(cred);
|
||||
let updated_at = u.update_at.unwrap_or_else(OffsetDateTime::now_utc);
|
||||
drop(cache);
|
||||
drop(users);
|
||||
|
||||
@@ -1560,7 +1569,8 @@ where
|
||||
.save_user_identity(access_key, UserType::Reg, u.clone(), None)
|
||||
.await?;
|
||||
|
||||
self.update_user_with_claims(access_key, u)
|
||||
self.update_user_with_claims(access_key, u)?;
|
||||
Ok((updated_at, status))
|
||||
}
|
||||
|
||||
/// Add SSH public key for a user (for SFTP authentication)
|
||||
|
||||
@@ -960,7 +960,11 @@ impl<T: Store> IamSys<T> {
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
pub async fn set_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<()> {
|
||||
pub async fn set_user_secret_key(
|
||||
&self,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus)> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
}
|
||||
@@ -969,7 +973,9 @@ impl<T: Store> IamSys<T> {
|
||||
return Err(IamError::InvalidSecretKeyLength);
|
||||
}
|
||||
|
||||
self.store.update_user_secret_key(access_key, secret_key).await
|
||||
let (updated_at, status) = self.store.update_user_secret_key(access_key, secret_key).await?;
|
||||
self.notify_for_user(access_key, false).await;
|
||||
Ok((updated_at, status))
|
||||
}
|
||||
|
||||
/// Add SSH public key for a user (for SFTP authentication)
|
||||
|
||||
@@ -75,13 +75,13 @@ pub(super) fn rules() -> Vec<Rule> {
|
||||
},
|
||||
Rule {
|
||||
evidence_fields: strings(["required", "target_free"]),
|
||||
anchors: strings(["insufficient target pool capacity"]),
|
||||
anchors: strings(["insufficient reserved physical target capacity"]),
|
||||
..base(
|
||||
"decom-capacity-insufficient",
|
||||
P1Unavailable,
|
||||
"capacity",
|
||||
"pool 下线目标容量不足",
|
||||
contains("insufficient target pool capacity"),
|
||||
contains("insufficient reserved physical target capacity"),
|
||||
"pool 下线迁移目标容量不足,decommission 无法开始。",
|
||||
"先扩容目标 pool。",
|
||||
)
|
||||
|
||||
@@ -262,7 +262,9 @@ fn every_rule_has_a_positive_sample() {
|
||||
),
|
||||
(
|
||||
"decom-capacity-insufficient",
|
||||
msg("failed to start decommission: insufficient target pool capacity: required 100 bytes available 50 bytes"),
|
||||
msg(
|
||||
"failed to start decommission: insufficient reserved physical target capacity: required 100 bytes available 50 bytes",
|
||||
),
|
||||
),
|
||||
// ops
|
||||
("decom-object-failed", msg("Decommission object migration failed")),
|
||||
|
||||
@@ -282,10 +282,10 @@ The configuration constants are defined in `crates/config/src/constants/capacity
|
||||
|
||||
| Environment Variable | Default | Description |
|
||||
| --- | --- | --- |
|
||||
| `RUSTFS_CAPACITY_SCHEDULED_INTERVAL` | `120s` | Scheduled refresh interval |
|
||||
| `RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY` | `5s` | Debounce delay after writes |
|
||||
| `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` | `5` | Recent 60-second write-frequency threshold |
|
||||
| `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` | `30s` | Cache age required before fast refresh is considered |
|
||||
| `RUSTFS_CAPACITY_SCHEDULED_INTERVAL` | `600s` | Scheduled refresh interval |
|
||||
| `RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY` | `30s` | Debounce delay after writes |
|
||||
| `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` | `20` | Recent 60-second write-frequency threshold |
|
||||
| `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` | `120s` | Cache age required before fast refresh is considered |
|
||||
| `RUSTFS_CAPACITY_MAX_FILES_THRESHOLD` | `200000` | Exact-count file threshold |
|
||||
| `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3s` (`60s` with `RUSTFS_DRIVE_TIMEOUT_PROFILE=high_latency`) | Base scan timeout |
|
||||
| `RUSTFS_CAPACITY_SAMPLE_RATE` | `200` | Overflow-file sampling interval |
|
||||
|
||||
@@ -282,10 +282,10 @@ get_capacity_manager()
|
||||
|
||||
| 环境变量 | 默认值 | 说明 |
|
||||
| --- | --- | --- |
|
||||
| `RUSTFS_CAPACITY_SCHEDULED_INTERVAL` | `120s` | 定时刷新间隔 |
|
||||
| `RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY` | `5s` | 写后防抖延迟 |
|
||||
| `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` | `5` | 最近 60 秒写频率阈值 |
|
||||
| `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` | `30s` | 缓存超过该年龄后才考虑快速刷新 |
|
||||
| `RUSTFS_CAPACITY_SCHEDULED_INTERVAL` | `600s` | 定时刷新间隔 |
|
||||
| `RUSTFS_CAPACITY_WRITE_TRIGGER_DELAY` | `30s` | 写后防抖延迟 |
|
||||
| `RUSTFS_CAPACITY_WRITE_FREQUENCY_THRESHOLD` | `20` | 最近 60 秒写频率阈值 |
|
||||
| `RUSTFS_CAPACITY_FAST_UPDATE_THRESHOLD` | `120s` | 缓存超过该年龄后才考虑快速刷新 |
|
||||
| `RUSTFS_CAPACITY_MAX_FILES_THRESHOLD` | `200000` | 精确统计文件数阈值 |
|
||||
| `RUSTFS_CAPACITY_STAT_TIMEOUT` | `3s` | 基础扫描超时 |
|
||||
| `RUSTFS_CAPACITY_SAMPLE_RATE` | `200` | overflow 文件采样间隔 |
|
||||
|
||||
@@ -1705,22 +1705,22 @@ mod tests {
|
||||
(
|
||||
ENV_CAPACITY_SCHEDULED_INTERVAL,
|
||||
|| get_scheduled_update_interval().as_secs(),
|
||||
120,
|
||||
600,
|
||||
"600",
|
||||
600,
|
||||
),
|
||||
(ENV_CAPACITY_WRITE_TRIGGER_DELAY, || get_write_trigger_delay().as_secs(), 5, "20", 20),
|
||||
(ENV_CAPACITY_WRITE_TRIGGER_DELAY, || get_write_trigger_delay().as_secs(), 30, "20", 20),
|
||||
(
|
||||
ENV_CAPACITY_WRITE_FREQUENCY_THRESHOLD,
|
||||
|| get_write_frequency_threshold() as u64,
|
||||
5,
|
||||
20,
|
||||
"20",
|
||||
20,
|
||||
),
|
||||
(
|
||||
ENV_CAPACITY_FAST_UPDATE_THRESHOLD,
|
||||
|| get_fast_update_threshold().as_secs(),
|
||||
30,
|
||||
120,
|
||||
"120",
|
||||
120,
|
||||
),
|
||||
@@ -2773,10 +2773,10 @@ mod tests {
|
||||
let config = HybridStrategyConfig::from_env();
|
||||
|
||||
// Check default values
|
||||
assert_eq!(config.scheduled_update_interval, Duration::from_secs(120));
|
||||
assert_eq!(config.write_trigger_delay, Duration::from_secs(5));
|
||||
assert_eq!(config.write_frequency_threshold, 5);
|
||||
assert_eq!(config.fast_update_threshold, Duration::from_secs(30));
|
||||
assert_eq!(config.scheduled_update_interval, Duration::from_secs(600));
|
||||
assert_eq!(config.write_trigger_delay, Duration::from_secs(30));
|
||||
assert_eq!(config.write_frequency_threshold, 20);
|
||||
assert_eq!(config.fast_update_threshold, Duration::from_secs(120));
|
||||
assert!(config.enable_smart_update);
|
||||
assert!(config.enable_write_trigger);
|
||||
}
|
||||
|
||||
@@ -198,6 +198,9 @@ pub enum S3KeyName {
|
||||
#[strum(serialize = "s3:object-lock-retain-until-date")]
|
||||
S3ObjectLockRetainUntilDate,
|
||||
|
||||
#[strum(serialize = "s3:object-lock-legal-hold")]
|
||||
S3ObjectLockLegalHold,
|
||||
|
||||
#[strum(serialize = "s3:object-lock-mode")]
|
||||
S3ObjectLockMode,
|
||||
|
||||
@@ -389,6 +392,7 @@ mod tests {
|
||||
#[test_case("s3:VersionId", KeyName::S3(S3KeyName::S3VersionId) ; "aws_version_id")]
|
||||
#[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")]
|
||||
#[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))]
|
||||
#[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))]
|
||||
#[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))]
|
||||
#[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))]
|
||||
#[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))]
|
||||
@@ -412,6 +416,7 @@ mod tests {
|
||||
#[test_case("s3:VersionId", KeyName::S3(S3KeyName::S3VersionId) ; "aws_version_id")]
|
||||
#[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId) ; "minio_version_id")]
|
||||
#[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))]
|
||||
#[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))]
|
||||
#[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))]
|
||||
#[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))]
|
||||
#[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))]
|
||||
@@ -431,6 +436,7 @@ mod tests {
|
||||
#[test_case("s3:x-amz-copy-source", KeyName::S3(S3KeyName::S3XAmzCopySource))]
|
||||
#[test_case("s3:versionid", KeyName::S3(S3KeyName::S3VersionId))]
|
||||
#[test_case("s3:object-lock-mode", KeyName::S3(S3KeyName::S3ObjectLockMode))]
|
||||
#[test_case("s3:object-lock-legal-hold", KeyName::S3(S3KeyName::S3ObjectLockLegalHold))]
|
||||
#[test_case("aws:SecureTransport", KeyName::Aws(AwsKeyName::AWSSecureTransport))]
|
||||
#[test_case("jwt:sub", KeyName::Jwt(JwtKeyName::JWTSub))]
|
||||
#[test_case("ldap:user", KeyName::Ldap(LdapKeyName::User))]
|
||||
|
||||
@@ -287,7 +287,7 @@ mod tests {
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::policy::function::key_name::S3KeyName::{S3LocationConstraint, S3ObjectLockMode};
|
||||
use crate::policy::function::key_name::S3KeyName::{S3LocationConstraint, S3ObjectLockLegalHold, S3ObjectLockMode};
|
||||
use test_case::test_case;
|
||||
|
||||
fn new_func(name: KeyName, variable: Option<String>, values: Vec<&str>) -> StringFunc {
|
||||
@@ -309,6 +309,7 @@ mod tests {
|
||||
#[test_case(r#"{"aws:username/value": ["johndoe", "aaa"]}"#, new_func(Aws(AWSUsername), Some("value".into()), vec!["johndoe", "aaa"]
|
||||
))]
|
||||
#[test_case(r#"{"s3:object-lock-mode": "COMPLIANCE"}"#, new_func(S3(S3ObjectLockMode), None, vec!["COMPLIANCE"]))]
|
||||
#[test_case(r#"{"s3:object-lock-legal-hold": "ON"}"#, new_func(S3(S3ObjectLockLegalHold), None, vec!["ON"]))]
|
||||
fn test_deser(input: &str, expect: StringFunc) -> Result<(), serde_json::Error> {
|
||||
let v: StringFunc = serde_json::from_str(input)?;
|
||||
assert_eq!(v, expect);
|
||||
|
||||
@@ -1671,6 +1671,94 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn replication_writable_fields_bind_to_typed_dto_fields() {
|
||||
let mut rule = replication_rule("id-marker", "arn:bucket-marker");
|
||||
rule.priority = Some(37);
|
||||
rule.filter = Some(s3s::dto::ReplicationRuleFilter {
|
||||
prefix: Some("prefix-marker/".to_string()),
|
||||
tag: Some(s3s::dto::Tag {
|
||||
key: Some("tag-key-marker".to_string()),
|
||||
value: Some("tag-value-marker".to_string()),
|
||||
}),
|
||||
and: Some(s3s::dto::ReplicationRuleAndOperator {
|
||||
prefix: Some("and-prefix-marker/".to_string()),
|
||||
tags: Some(vec![s3s::dto::Tag {
|
||||
key: Some("and-tag-key-marker".to_string()),
|
||||
value: Some("and-tag-value-marker".to_string()),
|
||||
}]),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
rule.delete_marker_replication = Some(DeleteMarkerReplication {
|
||||
status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)),
|
||||
});
|
||||
rule.delete_replication = Some(DeleteReplication {
|
||||
status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED),
|
||||
});
|
||||
rule.source_selection_criteria = Some(SourceSelectionCriteria {
|
||||
replica_modifications: Some(ReplicaModifications {
|
||||
status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED),
|
||||
}),
|
||||
sse_kms_encrypted_objects: None,
|
||||
});
|
||||
let config = ReplicationConfiguration {
|
||||
role: "role-marker".to_string(),
|
||||
rules: vec![rule],
|
||||
};
|
||||
|
||||
let rule = config.rules.first().expect("fixture should contain one rule");
|
||||
let filter = rule.filter.as_ref().expect("fixture should contain a rule filter");
|
||||
let field_hits = [
|
||||
("Role", config.role == "role-marker"),
|
||||
("Rule.ID", rule.id.as_deref() == Some("id-marker")),
|
||||
("Rule.Status", rule.status.as_str() == ReplicationRuleStatus::ENABLED),
|
||||
("Rule.Priority", rule.priority == Some(37)),
|
||||
("Rule.Filter.Prefix", filter.prefix.as_deref() == Some("prefix-marker/")),
|
||||
(
|
||||
"Rule.Filter.Tag",
|
||||
filter.tag.as_ref().and_then(|tag| tag.key.as_deref()) == Some("tag-key-marker"),
|
||||
),
|
||||
(
|
||||
"Rule.Filter.And",
|
||||
filter.and.as_ref().and_then(|and| and.prefix.as_deref()) == Some("and-prefix-marker/"),
|
||||
),
|
||||
("Rule.Destination.Bucket", rule.destination.bucket == "arn:bucket-marker"),
|
||||
(
|
||||
"Rule.ExistingObjectReplication.Status",
|
||||
rule.existing_object_replication
|
||||
.as_ref()
|
||||
.is_some_and(|existing| existing.status.as_str() == ExistingObjectReplicationStatus::ENABLED),
|
||||
),
|
||||
(
|
||||
"Rule.DeleteMarkerReplication.Status",
|
||||
rule.delete_marker_replication
|
||||
.as_ref()
|
||||
.and_then(|delete_marker| delete_marker.status.as_ref())
|
||||
.is_some_and(|status| status.as_str() == DeleteMarkerReplicationStatus::ENABLED),
|
||||
),
|
||||
(
|
||||
"Rule.DeleteReplication.Status",
|
||||
rule.delete_replication
|
||||
.as_ref()
|
||||
.is_some_and(|delete| delete.status.as_str() == DeleteReplicationStatus::ENABLED),
|
||||
),
|
||||
(
|
||||
"Rule.SourceSelectionCriteria.ReplicaModifications.Status",
|
||||
rule.source_selection_criteria
|
||||
.as_ref()
|
||||
.and_then(|criteria| criteria.replica_modifications.as_ref())
|
||||
.is_some_and(|modifications| modifications.status.as_str() == ReplicaModificationsStatus::ENABLED),
|
||||
),
|
||||
];
|
||||
let bound_paths = field_hits.iter().map(|(path, _)| *path).collect::<Vec<_>>();
|
||||
assert_eq!(bound_paths, REPLICATION_WRITABLE_FIELDS);
|
||||
|
||||
for (path, hit) in field_hits {
|
||||
assert!(hit, "typed field probe did not reach {path}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_replication_status_fields_are_reported_before_persistence() {
|
||||
let arn = "arn:rustfs:replication:us-east-1:target:bucket";
|
||||
|
||||
@@ -126,6 +126,27 @@ pub fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool {
|
||||
dobj.delete_marker_version_id.is_some()
|
||||
}
|
||||
|
||||
/// True when the target denied a replicated delete because object-lock
|
||||
/// retention or a legal hold protects that version on the replica (its
|
||||
/// deletion gate answers `AccessDenied` with the lock reason, and a
|
||||
/// replication request carries no governance bypass, rustfs#6850). Retrying
|
||||
/// cannot succeed until the lock itself lapses, so callers treat this as a
|
||||
/// policy denial rather than a transient fault.
|
||||
///
|
||||
/// The reason text is the RustFS deletion-gate wording; a MinIO/AWS peer
|
||||
/// phrases its WORM denial differently and simply stays unclassified — the
|
||||
/// caller then falls back to plain retry behavior, never a wrong state.
|
||||
pub fn is_object_lock_denied_delete(code: Option<&str>, message: Option<&str>) -> bool {
|
||||
if !matches!(code, Some("AccessDenied")) {
|
||||
return false;
|
||||
}
|
||||
let Some(message) = message else {
|
||||
return false;
|
||||
};
|
||||
let message = message.to_ascii_lowercase();
|
||||
message.contains("retention") || message.contains("legal hold")
|
||||
}
|
||||
|
||||
fn admitted_target_arns_from_replication_state(state: &ReplicationState) -> Vec<String> {
|
||||
let mut target_arns = state.targets.keys().cloned().collect::<Vec<_>>();
|
||||
target_arns.extend(state.purge_targets.keys().cloned());
|
||||
@@ -237,9 +258,9 @@ mod tests {
|
||||
|
||||
use super::{
|
||||
DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
delete_replication_creates_marker, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, resync_existing_delete_replication_info, should_retry_delete_marker_purge,
|
||||
target_delete_version_id,
|
||||
delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error,
|
||||
is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info,
|
||||
should_retry_delete_marker_purge, target_delete_version_id,
|
||||
};
|
||||
use crate::storage_api::DeletedObject;
|
||||
use crate::{
|
||||
@@ -615,4 +636,24 @@ mod tests {
|
||||
corrupt.target_delete_marker_version_ids_corrupt = true;
|
||||
assert_eq!(delete_marker_purge_version_id(Some(&corrupt), arn, source), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn object_lock_denied_delete_is_recognized_by_code_and_reason() {
|
||||
// The peer's deletion gate answers AccessDenied with the lock reason.
|
||||
assert!(is_object_lock_denied_delete(
|
||||
Some("AccessDenied"),
|
||||
Some("Object is under GOVERNANCE retention and cannot be deleted until 2026-09-01T00:00:00Z")
|
||||
));
|
||||
assert!(is_object_lock_denied_delete(
|
||||
Some("AccessDenied"),
|
||||
Some("Object has a legal hold and cannot be deleted. Remove the legal hold first.")
|
||||
));
|
||||
|
||||
// A plain policy denial (misconfigured replicator) is not a lock denial.
|
||||
assert!(!is_object_lock_denied_delete(Some("AccessDenied"), Some("Access Denied.")));
|
||||
assert!(!is_object_lock_denied_delete(Some("AccessDenied"), None));
|
||||
// Other errors mentioning retention must not match.
|
||||
assert!(!is_object_lock_denied_delete(Some("InternalError"), Some("retention lookup failed")));
|
||||
assert!(!is_object_lock_denied_delete(None, Some("legal hold")));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -41,9 +41,9 @@ pub use config::{
|
||||
};
|
||||
pub use delete::{
|
||||
DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id,
|
||||
delete_replication_creates_marker, is_retryable_delete_replication_head_error, is_version_delete_replication,
|
||||
replicate_delete_outcome, resync_existing_delete_replication_info, should_retry_delete_marker_purge,
|
||||
target_delete_version_id,
|
||||
delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error,
|
||||
is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info,
|
||||
should_retry_delete_marker_purge, target_delete_version_id,
|
||||
};
|
||||
pub use filemeta::{
|
||||
NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING,
|
||||
@@ -65,7 +65,8 @@ pub use multipart::{
|
||||
pub use object::{
|
||||
ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag,
|
||||
is_replication_target_offline_error, replication_action_for_target, replication_etags_match,
|
||||
ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
};
|
||||
pub use operation::{
|
||||
MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource,
|
||||
|
||||
@@ -71,6 +71,32 @@ pub fn replication_etags_match(source: Option<&str>, target: Option<&str>) -> bo
|
||||
source_etag.is_some() && source_etag == target_etag
|
||||
}
|
||||
|
||||
fn is_plain_single_part_md5(etag: &str) -> bool {
|
||||
etag.len() == 32 && etag.bytes().all(|b| b.is_ascii_hexdigit())
|
||||
}
|
||||
|
||||
/// Whether the ETag the target returned for a single-part replica proves the
|
||||
/// stored bytes differ from what the source sent — e.g. a target that does not
|
||||
/// decode `aws-chunked` framing stores the frames verbatim and returns their
|
||||
/// ETag. Only a plain single-part MD5 ETag on both sides is decidable; a
|
||||
/// multipart or opaque (encrypted) ETag, or a withheld replica ETag, returns
|
||||
/// `false` because no corruption can be concluded from it.
|
||||
pub fn single_part_replica_etag_mismatch(source_etag: Option<&str>, replica_etag: Option<&str>) -> bool {
|
||||
let Some(source) = source_etag.map(trim_etag) else {
|
||||
return false;
|
||||
};
|
||||
if !is_plain_single_part_md5(&source) {
|
||||
return false;
|
||||
}
|
||||
let Some(replica) = replica_etag.map(trim_etag) else {
|
||||
return false;
|
||||
};
|
||||
if !is_plain_single_part_md5(&replica) {
|
||||
return false;
|
||||
}
|
||||
!source.eq_ignore_ascii_case(&replica)
|
||||
}
|
||||
|
||||
pub fn target_is_newer_than_source_null_version(
|
||||
source: &ReplicationSourceObject<'_>,
|
||||
target: &ReplicationTargetObject<'_>,
|
||||
@@ -276,11 +302,41 @@ pub fn ssec_passthrough_evidence_present(sse_customer_algorithm: Option<&str>) -
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
const SOURCE_MD5: &str = "9a0364b9e99bb480dd25e1f0284c8555";
|
||||
const FRAMED_MD5: &str = "0f343b0931126a20f133d67c2b018a3b";
|
||||
|
||||
#[test]
|
||||
fn single_part_replica_mismatch_is_only_decided_on_plain_md5_pairs() {
|
||||
// The #6853 shape: the target stored aws-chunked frames verbatim and
|
||||
// returned the framed bytes' ETag.
|
||||
assert!(single_part_replica_etag_mismatch(Some(SOURCE_MD5), Some(FRAMED_MD5)));
|
||||
assert!(single_part_replica_etag_mismatch(
|
||||
Some(&format!("\"{SOURCE_MD5}\"")),
|
||||
Some(&format!("\"{FRAMED_MD5}\""))
|
||||
));
|
||||
|
||||
// A faithful replica, quoted or not, passes; hex case must not matter
|
||||
// (a target may return the same MD5 uppercased).
|
||||
assert!(!single_part_replica_etag_mismatch(Some(SOURCE_MD5), Some(SOURCE_MD5)));
|
||||
assert!(!single_part_replica_etag_mismatch(Some(&format!("\"{SOURCE_MD5}\"")), Some(SOURCE_MD5)));
|
||||
assert!(!single_part_replica_etag_mismatch(
|
||||
Some(SOURCE_MD5),
|
||||
Some(&SOURCE_MD5.to_ascii_uppercase())
|
||||
));
|
||||
|
||||
// Not decidable: multipart source, opaque replica ETag, or either side
|
||||
// missing must never be reported as corruption.
|
||||
assert!(!single_part_replica_etag_mismatch(Some(&format!("{SOURCE_MD5}-3")), Some(FRAMED_MD5)));
|
||||
assert!(!single_part_replica_etag_mismatch(Some(SOURCE_MD5), Some(&format!("{FRAMED_MD5}-3"))));
|
||||
assert!(!single_part_replica_etag_mismatch(Some(SOURCE_MD5), None));
|
||||
assert!(!single_part_replica_etag_mismatch(None, Some(FRAMED_MD5)));
|
||||
}
|
||||
|
||||
use super::{
|
||||
ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate,
|
||||
content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target, replication_etags_match,
|
||||
ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version,
|
||||
version_identity_drifted,
|
||||
single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
target_is_newer_than_source_null_version, version_identity_drifted,
|
||||
};
|
||||
use crate::filemeta::{ReplicationAction, ReplicationType};
|
||||
use crate::http::AMZ_OBJECT_LOCK_MODE;
|
||||
|
||||
@@ -12,10 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use minlz::{Encoder as MinlzEncoder, crc::crc, decode};
|
||||
use minlz::{Encoder as MinlzEncoder, crc::crc};
|
||||
use pin_project_lite::pin_project;
|
||||
use rand::RngExt;
|
||||
use rustfs_rio::{EtagResolvable, HashReaderDetector, HashReaderMut, Index, TryGetIndex};
|
||||
use rustfs_rio::{
|
||||
EtagResolvable, HashReaderDetector, HashReaderMut, Index, MAX_S2_DECOMPRESSED_BLOCK_SIZE, S2Decoder, TryGetIndex,
|
||||
};
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use std::cmp::min;
|
||||
use std::fmt;
|
||||
@@ -25,18 +27,16 @@ use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
const MAGIC_CHUNK: &[u8] = b"\xff\x06\x00\x00S2sTwO";
|
||||
const MAGIC_CHUNK_SNAPPY: &[u8] = b"\xff\x06\x00\x00sNaPpY";
|
||||
const CHUNK_TYPE_COMPRESSED_DATA: u8 = 0x00;
|
||||
const CHUNK_TYPE_UNCOMPRESSED_DATA: u8 = 0x01;
|
||||
const CHUNK_TYPE_INDEX: u8 = 0x99;
|
||||
const CHUNK_TYPE_PADDING: u8 = 0xfe;
|
||||
const CHUNK_TYPE_STREAM_IDENTIFIER: u8 = 0xff;
|
||||
const DEFAULT_BLOCK_SIZE: usize = 1 << 20;
|
||||
const MAX_CHUNK_SIZE: usize = (1 << 24) - 1;
|
||||
const CHECKSUM_SIZE: usize = 4;
|
||||
const CHUNK_HEADER_LEN: usize = 4;
|
||||
const ENCRYPTED_PADDING_MULTIPLE: usize = 256;
|
||||
const MIN_INDEX_SIZE: usize = 8 << 20;
|
||||
const MAX_READY_READS_PER_POLL: usize = 64;
|
||||
|
||||
pin_project! {
|
||||
#[derive(Debug)]
|
||||
@@ -88,7 +88,17 @@ where
|
||||
Self::with_block_size(inner, DEFAULT_BLOCK_SIZE, CompressionAlgorithm::default())
|
||||
}
|
||||
|
||||
/// Create an encoder with a caller-selected S2 block size.
|
||||
///
|
||||
/// Zero selects the default. Larger values are capped at the maximum block
|
||||
/// accepted by the paired decoder, preserving this infallible API without
|
||||
/// allowing it to emit a stream that RustFS cannot read back.
|
||||
pub fn with_block_size(inner: R, block_size: usize, _compression_algorithm: CompressionAlgorithm) -> Self {
|
||||
let block_size = if block_size == 0 {
|
||||
DEFAULT_BLOCK_SIZE
|
||||
} else {
|
||||
block_size.min(MAX_S2_DECOMPRESSED_BLOCK_SIZE)
|
||||
};
|
||||
Self {
|
||||
inner,
|
||||
buffer: Vec::new(),
|
||||
@@ -125,6 +135,7 @@ where
|
||||
{
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let mut this = self.project();
|
||||
let mut ready_reads = 0usize;
|
||||
|
||||
if *this.pos < this.buffer.len() {
|
||||
let to_copy = min(buf.remaining(), this.buffer.len() - *this.pos);
|
||||
@@ -142,6 +153,11 @@ where
|
||||
}
|
||||
|
||||
while this.temp_buffer.len() < *this.block_size {
|
||||
if ready_reads >= MAX_READY_READS_PER_POLL {
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
|
||||
let remaining = *this.block_size - this.temp_buffer.len();
|
||||
let mut read_buf = ReadBuf::new(&mut this.read_buffer[..remaining]);
|
||||
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
|
||||
@@ -149,6 +165,7 @@ where
|
||||
return Poll::Pending;
|
||||
}
|
||||
Poll::Ready(Ok(())) => {
|
||||
ready_reads += 1;
|
||||
let n = read_buf.filled().len();
|
||||
if n == 0 {
|
||||
break;
|
||||
@@ -243,18 +260,7 @@ pin_project! {
|
||||
#[derive(Debug)]
|
||||
pub struct DecompressReader<R> {
|
||||
#[pin]
|
||||
inner: R,
|
||||
buffer: Vec<u8>,
|
||||
buffer_pos: usize,
|
||||
finished: bool,
|
||||
header_buf: [u8; CHUNK_HEADER_LEN],
|
||||
header_read: usize,
|
||||
chunk_type: u8,
|
||||
chunk_buf: Vec<u8>,
|
||||
chunk_len: usize,
|
||||
chunk_read: usize,
|
||||
reading_chunk: bool,
|
||||
stream_initialized: bool,
|
||||
inner: S2Decoder<R>,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,18 +270,7 @@ where
|
||||
{
|
||||
pub fn new(inner: R, _compression_algorithm: CompressionAlgorithm) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
buffer: Vec::new(),
|
||||
buffer_pos: 0,
|
||||
finished: false,
|
||||
header_buf: [0u8; CHUNK_HEADER_LEN],
|
||||
header_read: 0,
|
||||
chunk_type: 0,
|
||||
chunk_buf: Vec::new(),
|
||||
chunk_len: 0,
|
||||
chunk_read: 0,
|
||||
reading_chunk: false,
|
||||
stream_initialized: false,
|
||||
inner: S2Decoder::new_at_legacy_chunk_boundary(inner),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -285,125 +280,7 @@ where
|
||||
R: AsyncRead + Unpin + Send + Sync,
|
||||
{
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let mut this = self.project();
|
||||
|
||||
if *this.buffer_pos < this.buffer.len() {
|
||||
let to_copy = min(buf.remaining(), this.buffer.len() - *this.buffer_pos);
|
||||
buf.put_slice(&this.buffer[*this.buffer_pos..*this.buffer_pos + to_copy]);
|
||||
*this.buffer_pos += to_copy;
|
||||
if *this.buffer_pos == this.buffer.len() {
|
||||
this.buffer.clear();
|
||||
*this.buffer_pos = 0;
|
||||
}
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
loop {
|
||||
if *this.finished {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
if !*this.reading_chunk {
|
||||
while *this.header_read < CHUNK_HEADER_LEN {
|
||||
let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..]);
|
||||
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(Ok(())) => {
|
||||
let n = read_buf.filled().len();
|
||||
if n == 0 {
|
||||
if *this.header_read == 0 {
|
||||
*this.finished = true;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"unexpected EOF while reading S2 chunk header",
|
||||
)));
|
||||
}
|
||||
*this.header_read += n;
|
||||
}
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
}
|
||||
}
|
||||
|
||||
*this.chunk_type = this.header_buf[0];
|
||||
*this.chunk_len =
|
||||
(this.header_buf[1] as usize) | ((this.header_buf[2] as usize) << 8) | ((this.header_buf[3] as usize) << 16);
|
||||
*this.header_read = 0;
|
||||
|
||||
if this.chunk_buf.len() < *this.chunk_len {
|
||||
this.chunk_buf.resize(*this.chunk_len, 0);
|
||||
}
|
||||
*this.chunk_read = 0;
|
||||
*this.reading_chunk = true;
|
||||
}
|
||||
|
||||
while *this.chunk_read < *this.chunk_len {
|
||||
let mut read_buf = ReadBuf::new(&mut this.chunk_buf[*this.chunk_read..*this.chunk_len]);
|
||||
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(Ok(())) => {
|
||||
let n = read_buf.filled().len();
|
||||
if n == 0 {
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"unexpected EOF while reading S2 chunk body",
|
||||
)));
|
||||
}
|
||||
*this.chunk_read += n;
|
||||
}
|
||||
Poll::Ready(Err(err)) => return Poll::Ready(Err(err)),
|
||||
}
|
||||
}
|
||||
|
||||
let chunk = &this.chunk_buf[..*this.chunk_len];
|
||||
*this.reading_chunk = false;
|
||||
match *this.chunk_type {
|
||||
CHUNK_TYPE_STREAM_IDENTIFIER => {
|
||||
if chunk != &MAGIC_CHUNK[CHUNK_HEADER_LEN..] && chunk != &MAGIC_CHUNK_SNAPPY[CHUNK_HEADER_LEN..] {
|
||||
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid S2 stream identifier")));
|
||||
}
|
||||
*this.stream_initialized = true;
|
||||
continue;
|
||||
}
|
||||
CHUNK_TYPE_COMPRESSED_DATA => {
|
||||
*this.stream_initialized = true;
|
||||
let decompressed = decode_chunk(chunk, true)?;
|
||||
*this.buffer = decompressed;
|
||||
}
|
||||
CHUNK_TYPE_UNCOMPRESSED_DATA => {
|
||||
*this.stream_initialized = true;
|
||||
let decompressed = decode_chunk(chunk, false)?;
|
||||
*this.buffer = decompressed;
|
||||
}
|
||||
CHUNK_TYPE_INDEX | CHUNK_TYPE_PADDING | 0x80..=0xfd => {
|
||||
*this.stream_initialized = true;
|
||||
continue;
|
||||
}
|
||||
_ => {
|
||||
if !*this.stream_initialized && *this.chunk_type != CHUNK_TYPE_COMPRESSED_DATA {
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type),
|
||||
)));
|
||||
}
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type),
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
*this.buffer_pos = 0;
|
||||
let to_copy = min(buf.remaining(), this.buffer.len());
|
||||
buf.put_slice(&this.buffer[..to_copy]);
|
||||
*this.buffer_pos += to_copy;
|
||||
if *this.buffer_pos == this.buffer.len() {
|
||||
this.buffer.clear();
|
||||
*this.buffer_pos = 0;
|
||||
}
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
self.project().inner.poll_read(cx, buf)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -412,7 +289,7 @@ where
|
||||
R: EtagResolvable,
|
||||
{
|
||||
fn try_resolve_etag(&mut self) -> Option<String> {
|
||||
self.inner.try_resolve_etag()
|
||||
self.inner.get_mut().try_resolve_etag()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -421,11 +298,11 @@ where
|
||||
R: HashReaderDetector,
|
||||
{
|
||||
fn is_hash_reader(&self) -> bool {
|
||||
self.inner.is_hash_reader()
|
||||
self.inner.get_ref().is_hash_reader()
|
||||
}
|
||||
|
||||
fn as_hash_reader_mut(&mut self) -> Option<&mut dyn HashReaderMut> {
|
||||
self.inner.as_hash_reader_mut()
|
||||
self.inner.get_mut().as_hash_reader_mut()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -483,42 +360,57 @@ fn build_padding_chunk(current_size: usize, padding_multiple: usize) -> io::Resu
|
||||
Ok(Some(out))
|
||||
}
|
||||
|
||||
fn decode_chunk(chunk: &[u8], compressed: bool) -> io::Result<Vec<u8>> {
|
||||
if chunk.len() < CHECKSUM_SIZE {
|
||||
return Err(io::Error::new(io::ErrorKind::InvalidData, "S2 chunk smaller than checksum header"));
|
||||
}
|
||||
|
||||
let expected_crc = u32::from_le_bytes(chunk[..CHECKSUM_SIZE].try_into().expect("checksum header"));
|
||||
let payload = &chunk[CHECKSUM_SIZE..];
|
||||
let decompressed = if compressed {
|
||||
decode(payload).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("S2 decode error: {err}")))?
|
||||
} else {
|
||||
payload.to_vec()
|
||||
};
|
||||
|
||||
let actual_crc = crc(&decompressed);
|
||||
if actual_crc != expected_crc {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!(
|
||||
"S2 CRC mismatch: expected={expected_crc:08x} actual={actual_crc:08x} compressed={compressed} payload_len={} decompressed_len={}",
|
||||
payload.len(),
|
||||
decompressed.len()
|
||||
),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(decompressed)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::task::{Context, Poll, Wake, Waker};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[derive(Default)]
|
||||
struct WakeCounter(AtomicUsize);
|
||||
|
||||
impl Wake for WakeCounter {
|
||||
fn wake(self: Arc<Self>) {
|
||||
self.0.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
struct AlwaysReadyOneByte {
|
||||
bytes: Vec<u8>,
|
||||
position: usize,
|
||||
read_calls: Arc<AtomicUsize>,
|
||||
}
|
||||
|
||||
impl AlwaysReadyOneByte {
|
||||
fn new(bytes: Vec<u8>, read_calls: Arc<AtomicUsize>) -> Self {
|
||||
Self {
|
||||
bytes,
|
||||
position: 0,
|
||||
read_calls,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for AlwaysReadyOneByte {
|
||||
fn poll_read(mut self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
self.read_calls.fetch_add(1, Ordering::Relaxed);
|
||||
if self.position == self.bytes.len() || buf.remaining() == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
let byte = self.bytes[self.position];
|
||||
self.position += 1;
|
||||
buf.put_slice(&[byte]);
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
|
||||
struct PendingAfterBytes<R> {
|
||||
inner: R,
|
||||
max_chunk: usize,
|
||||
@@ -583,7 +475,7 @@ mod tests {
|
||||
let plaintext = b"compressible-rio-v2-block-".repeat(4096);
|
||||
let mut encoder = S2BlockEncoder::new();
|
||||
let compressed = encode_block(&plaintext, &mut encoder);
|
||||
let decoded = decode(&compressed).expect("decode payload");
|
||||
let decoded = minlz::decode(&compressed).expect("decode payload");
|
||||
|
||||
assert_eq!(decoded, plaintext);
|
||||
}
|
||||
@@ -604,6 +496,97 @@ mod tests {
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s2_compress_reader_yields_after_ready_read_budget() {
|
||||
let read_calls = Arc::new(AtomicUsize::new(0));
|
||||
let source = AlwaysReadyOneByte::new(vec![b'x'; MAX_READY_READS_PER_POLL + 1], read_calls.clone());
|
||||
let mut reader = CompressReader::new(source, CompressionAlgorithm::default());
|
||||
let wake_counter = Arc::new(WakeCounter::default());
|
||||
let waker = Waker::from(wake_counter.clone());
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
let mut output = [0u8; 1];
|
||||
let mut read_buf = ReadBuf::new(&mut output);
|
||||
|
||||
assert!(Pin::new(&mut reader).poll_read(&mut cx, &mut read_buf).is_pending());
|
||||
assert!(read_buf.filled().is_empty());
|
||||
assert_eq!(read_calls.load(Ordering::Relaxed), MAX_READY_READS_PER_POLL);
|
||||
assert_eq!(reader.temp_buffer.len(), MAX_READY_READS_PER_POLL);
|
||||
assert_eq!(wake_counter.0.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s2_compress_reader_normalizes_non_decodable_block_sizes() {
|
||||
let zero = CompressReader::with_block_size(Cursor::new(Vec::<u8>::new()), 0, CompressionAlgorithm::default());
|
||||
assert_eq!(zero.block_size, DEFAULT_BLOCK_SIZE);
|
||||
|
||||
let oversized = CompressReader::with_block_size(
|
||||
Cursor::new(Vec::<u8>::new()),
|
||||
MAX_S2_DECOMPRESSED_BLOCK_SIZE + 1,
|
||||
CompressionAlgorithm::default(),
|
||||
);
|
||||
assert_eq!(oversized.block_size, MAX_S2_DECOMPRESSED_BLOCK_SIZE);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s2_compress_reader_max_block_roundtrips_with_paired_decoder() {
|
||||
let plaintext = pseudo_random_bytes(MAX_S2_DECOMPRESSED_BLOCK_SIZE);
|
||||
let mut reader = CompressReader::with_block_size(
|
||||
Cursor::new(plaintext.clone()),
|
||||
MAX_S2_DECOMPRESSED_BLOCK_SIZE,
|
||||
CompressionAlgorithm::default(),
|
||||
);
|
||||
let mut compressed = Vec::new();
|
||||
reader.read_to_end(&mut compressed).await.expect("read maximum S2 block");
|
||||
|
||||
let mut decompressor = DecompressReader::new(Cursor::new(compressed), CompressionAlgorithm::default());
|
||||
let mut actual = Vec::new();
|
||||
decompressor
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("paired decoder should accept maximum S2 block");
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s2_decompress_reader_accepts_legacy_block_above_16_mib() {
|
||||
const PRE_CAP_LEGACY_BLOCK_SIZE: usize = (16 << 20) + 1;
|
||||
let plaintext = vec![b'x'; PRE_CAP_LEGACY_BLOCK_SIZE];
|
||||
let mut encoder = S2BlockEncoder::new();
|
||||
let mut fixture = MAGIC_CHUNK.to_vec();
|
||||
fixture.extend_from_slice(
|
||||
&build_s2_chunk(&plaintext, &mut encoder).expect("the pre-cap writer format should encode a block above 16 MiB"),
|
||||
);
|
||||
assert_eq!(fixture[MAGIC_CHUNK.len()], CHUNK_TYPE_COMPRESSED_DATA);
|
||||
|
||||
let mut decompressor = DecompressReader::new(Cursor::new(fixture), CompressionAlgorithm::default());
|
||||
let mut actual = Vec::new();
|
||||
decompressor
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("legacy rio-v2 blocks above the current writer limit should remain readable");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s2_decompress_reader_accepts_an_indexed_headerless_tail() {
|
||||
let plaintext = b"indexed-rio-v2-s2-tail-".repeat(32_768);
|
||||
let mut reader = CompressReader::new(Cursor::new(plaintext.clone()), CompressionAlgorithm::default());
|
||||
let mut compressed = Vec::new();
|
||||
reader.read_to_end(&mut compressed).await.expect("read compressed data");
|
||||
assert!(compressed.starts_with(MAGIC_CHUNK));
|
||||
|
||||
let mut decompressor =
|
||||
DecompressReader::new(Cursor::new(compressed[MAGIC_CHUNK.len()..].to_vec()), CompressionAlgorithm::default());
|
||||
let mut actual = Vec::new();
|
||||
decompressor
|
||||
.read_to_end(&mut actual)
|
||||
.await
|
||||
.expect("indexed tail should decode without the stream header");
|
||||
|
||||
assert_eq!(actual, plaintext);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn s2_compress_reader_roundtrip_near_erasure_boundary() {
|
||||
let size = 4 * 1024 * 1024 - 97;
|
||||
|
||||
@@ -79,6 +79,7 @@ rustfs-tls-runtime.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
md-5 = { workspace = true }
|
||||
minlz.workspace = true
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
base64-simd.workspace = true
|
||||
|
||||
@@ -100,6 +100,9 @@ mod limit_reader;
|
||||
|
||||
pub use limit_reader::LimitReader;
|
||||
|
||||
mod s2_decoder;
|
||||
pub use s2_decoder::{MAX_S2_DECOMPRESSED_BLOCK_SIZE, S2Decoder};
|
||||
|
||||
mod etag_reader;
|
||||
pub use etag_reader::EtagReader;
|
||||
|
||||
|
||||
@@ -0,0 +1,858 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Bounded asynchronous decoder for the S2/Snappy framed stream format.
|
||||
|
||||
use minlz::{MAX_DECODE_DST_SIZE, crc::crc, decode, decode_into, decode_len};
|
||||
use pin_project_lite::pin_project;
|
||||
use std::cmp::min;
|
||||
use std::io;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
|
||||
const S2_MAGIC_BODY: &[u8] = b"S2sTwO";
|
||||
const SNAPPY_MAGIC_BODY: &[u8] = b"sNaPpY";
|
||||
const CHUNK_TYPE_COMPRESSED_DATA: u8 = 0x00;
|
||||
const CHUNK_TYPE_UNCOMPRESSED_DATA: u8 = 0x01;
|
||||
const CHUNK_TYPE_PADDING: u8 = 0xfe;
|
||||
const CHUNK_TYPE_STREAM_IDENTIFIER: u8 = 0xff;
|
||||
const CHECKSUM_SIZE: usize = 4;
|
||||
const CHUNK_HEADER_LEN: usize = 4;
|
||||
const MAX_READY_READS_PER_POLL: usize = 64;
|
||||
const MAX_CHUNKS_PER_POLL: usize = 64;
|
||||
const MAX_INPUT_BYTES_PER_POLL: usize = 256 * 1024;
|
||||
const MAX_SNAPPY_DECOMPRESSED_BLOCK_SIZE: usize = 64 << 10;
|
||||
const MAX_FRAMED_CHUNK_SIZE: usize = (1 << 24) - 1;
|
||||
const MAX_LEGACY_S2_DECOMPRESSED_BLOCK_SIZE: usize = MAX_DECODE_DST_SIZE;
|
||||
|
||||
// This is checksum size + klauspost/s2 MaxEncodedLen(4 MiB).
|
||||
// MaxEncodedLen adds a four-byte varint and four-byte literal header. The Go
|
||||
// reader keeps this encoded-input cap even for Snappy frames, then applies the
|
||||
// tighter 64 KiB limit to the decoded size.
|
||||
const MAX_S2_COMPRESSED_CHUNK_SIZE: usize = CHECKSUM_SIZE + MAX_S2_DECOMPRESSED_BLOCK_SIZE + 4 + 4;
|
||||
|
||||
/// S2 writers, including minio-go's Snowball writer, cap a decoded block at
|
||||
/// 4 MiB. Enforcing the framing limit before allocation prevents a tiny block
|
||||
/// length varint from requesting the block codec's much larger generic limit.
|
||||
pub const MAX_S2_DECOMPRESSED_BLOCK_SIZE: usize = 4 << 20;
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum FrameMode {
|
||||
Uninitialized,
|
||||
S2,
|
||||
Snappy,
|
||||
MidstreamS2,
|
||||
}
|
||||
|
||||
impl FrameMode {
|
||||
fn max_compressed_chunk_size(self, s2_limit: usize) -> usize {
|
||||
match self {
|
||||
Self::Snappy => MAX_S2_COMPRESSED_CHUNK_SIZE,
|
||||
Self::Uninitialized | Self::S2 | Self::MidstreamS2 => s2_limit,
|
||||
}
|
||||
}
|
||||
|
||||
fn max_decompressed_block_size(self, s2_limit: usize) -> usize {
|
||||
match self {
|
||||
Self::Snappy => MAX_SNAPPY_DECOMPRESSED_BLOCK_SIZE,
|
||||
Self::Uninitialized | Self::S2 | Self::MidstreamS2 => s2_limit,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pin_project! {
|
||||
/// Decode an S2 or Snappy framed stream without blocking the async reader.
|
||||
#[derive(Debug)]
|
||||
pub struct S2Decoder<R> {
|
||||
#[pin]
|
||||
inner: R,
|
||||
output: Vec<u8>,
|
||||
output_pos: usize,
|
||||
finished: bool,
|
||||
poisoned: bool,
|
||||
header_buf: [u8; CHUNK_HEADER_LEN],
|
||||
header_read: usize,
|
||||
chunk_type: u8,
|
||||
chunk_buf: Vec<u8>,
|
||||
chunk_len: usize,
|
||||
chunk_read: usize,
|
||||
reading_chunk: bool,
|
||||
skipping_chunk: bool,
|
||||
frame_mode: FrameMode,
|
||||
max_s2_compressed_chunk_size: usize,
|
||||
max_s2_decompressed_block_size: usize,
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> S2Decoder<R> {
|
||||
pub fn new(inner: R) -> Self {
|
||||
Self::with_limits(
|
||||
inner,
|
||||
FrameMode::Uninitialized,
|
||||
MAX_S2_COMPRESSED_CHUNK_SIZE,
|
||||
MAX_S2_DECOMPRESSED_BLOCK_SIZE,
|
||||
)
|
||||
}
|
||||
|
||||
/// Create a decoder positioned at a trusted S2 data-chunk boundary.
|
||||
///
|
||||
/// Indexed range reads start after the stream identifier. General stream
|
||||
/// consumers should use [`S2Decoder::new`] so a missing identifier remains
|
||||
/// an error.
|
||||
pub fn new_at_chunk_boundary(inner: R) -> Self {
|
||||
Self::with_limits(
|
||||
inner,
|
||||
FrameMode::MidstreamS2,
|
||||
MAX_S2_COMPRESSED_CHUNK_SIZE,
|
||||
MAX_S2_DECOMPRESSED_BLOCK_SIZE,
|
||||
)
|
||||
}
|
||||
|
||||
/// Create a bounded decoder for rio-v2 data written before its block-size
|
||||
/// API was capped at 4 MiB.
|
||||
///
|
||||
/// This compatibility mode accepts decoded S2 blocks up to the block
|
||||
/// decoder's 256 MiB safety cap and an encoded chunk up to the format's
|
||||
/// 24-bit framing limit. That preserves streams written through rio-v2's
|
||||
/// former uncapped block-size API without restoring unbounded allocation.
|
||||
/// New streams and general S2 consumers should use the stricter
|
||||
/// constructors above.
|
||||
pub fn new_at_legacy_chunk_boundary(inner: R) -> Self {
|
||||
Self::with_limits(
|
||||
inner,
|
||||
FrameMode::MidstreamS2,
|
||||
MAX_FRAMED_CHUNK_SIZE,
|
||||
MAX_LEGACY_S2_DECOMPRESSED_BLOCK_SIZE,
|
||||
)
|
||||
}
|
||||
|
||||
fn with_limits(
|
||||
inner: R,
|
||||
frame_mode: FrameMode,
|
||||
max_s2_compressed_chunk_size: usize,
|
||||
max_s2_decompressed_block_size: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
output: Vec::new(),
|
||||
output_pos: 0,
|
||||
finished: false,
|
||||
poisoned: false,
|
||||
header_buf: [0u8; CHUNK_HEADER_LEN],
|
||||
header_read: 0,
|
||||
chunk_type: 0,
|
||||
chunk_buf: Vec::new(),
|
||||
chunk_len: 0,
|
||||
chunk_read: 0,
|
||||
reading_chunk: false,
|
||||
skipping_chunk: false,
|
||||
frame_mode,
|
||||
max_s2_compressed_chunk_size,
|
||||
max_s2_decompressed_block_size,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_ref(&self) -> &R {
|
||||
&self.inner
|
||||
}
|
||||
|
||||
pub fn get_mut(&mut self) -> &mut R {
|
||||
&mut self.inner
|
||||
}
|
||||
|
||||
pub fn into_inner(self) -> R {
|
||||
self.inner
|
||||
}
|
||||
}
|
||||
|
||||
impl<R> AsyncRead for S2Decoder<R>
|
||||
where
|
||||
R: AsyncRead,
|
||||
{
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<io::Result<()>> {
|
||||
let mut this = self.project();
|
||||
let mut ready_reads = 0;
|
||||
let mut completed_chunks = 0;
|
||||
let mut input_bytes = 0;
|
||||
|
||||
if buf.remaining() == 0 {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
if *this.poisoned {
|
||||
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "S2 decoder previously failed")));
|
||||
}
|
||||
|
||||
if *this.output_pos < this.output.len() {
|
||||
let to_copy = min(buf.remaining(), this.output.len() - *this.output_pos);
|
||||
buf.put_slice(&this.output[*this.output_pos..*this.output_pos + to_copy]);
|
||||
*this.output_pos += to_copy;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
if *this.finished {
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
|
||||
loop {
|
||||
if ready_reads >= MAX_READY_READS_PER_POLL
|
||||
|| completed_chunks >= MAX_CHUNKS_PER_POLL
|
||||
|| input_bytes >= MAX_INPUT_BYTES_PER_POLL
|
||||
{
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
|
||||
if !*this.reading_chunk {
|
||||
while *this.header_read < CHUNK_HEADER_LEN {
|
||||
let remaining_poll_bytes = MAX_INPUT_BYTES_PER_POLL - input_bytes;
|
||||
if ready_reads >= MAX_READY_READS_PER_POLL || remaining_poll_bytes == 0 {
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
let read_end = (*this.header_read + remaining_poll_bytes).min(CHUNK_HEADER_LEN);
|
||||
let mut read_buf = ReadBuf::new(&mut this.header_buf[*this.header_read..read_end]);
|
||||
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(Ok(())) => {
|
||||
ready_reads += 1;
|
||||
let read = read_buf.filled().len();
|
||||
if read == 0 {
|
||||
if *this.header_read == 0 {
|
||||
*this.finished = true;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"unexpected EOF while reading S2 chunk header",
|
||||
)));
|
||||
}
|
||||
input_bytes += read;
|
||||
*this.header_read += read;
|
||||
}
|
||||
Poll::Ready(Err(err)) => {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
if ready_reads >= MAX_READY_READS_PER_POLL || input_bytes >= MAX_INPUT_BYTES_PER_POLL {
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
|
||||
*this.chunk_type = this.header_buf[0];
|
||||
*this.chunk_len = usize::from(this.header_buf[1])
|
||||
| (usize::from(this.header_buf[2]) << 8)
|
||||
| (usize::from(this.header_buf[3]) << 16);
|
||||
*this.header_read = 0;
|
||||
|
||||
if *this.frame_mode == FrameMode::Uninitialized && *this.chunk_type != CHUNK_TYPE_STREAM_IDENTIFIER {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"S2 stream identifier must be the first chunk",
|
||||
)));
|
||||
}
|
||||
|
||||
let skippable = matches!(*this.chunk_type, CHUNK_TYPE_PADDING | 0x80..=0xfd);
|
||||
let invalid_length = match *this.chunk_type {
|
||||
CHUNK_TYPE_STREAM_IDENTIFIER => *this.chunk_len != S2_MAGIC_BODY.len(),
|
||||
CHUNK_TYPE_COMPRESSED_DATA => {
|
||||
*this.chunk_len < CHECKSUM_SIZE
|
||||
|| *this.chunk_len > this.frame_mode.max_compressed_chunk_size(*this.max_s2_compressed_chunk_size)
|
||||
}
|
||||
CHUNK_TYPE_UNCOMPRESSED_DATA => {
|
||||
*this.chunk_len < CHECKSUM_SIZE
|
||||
|| *this.chunk_len - CHECKSUM_SIZE
|
||||
> this
|
||||
.frame_mode
|
||||
.max_decompressed_block_size(*this.max_s2_decompressed_block_size)
|
||||
}
|
||||
_ if skippable => false,
|
||||
_ => {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("unknown S2 chunk type: 0x{:02x}", *this.chunk_type),
|
||||
)));
|
||||
}
|
||||
};
|
||||
if invalid_length {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid S2 chunk length")));
|
||||
}
|
||||
|
||||
if !skippable && this.chunk_buf.len() < *this.chunk_len {
|
||||
this.chunk_buf.resize(*this.chunk_len, 0);
|
||||
}
|
||||
*this.chunk_read = 0;
|
||||
*this.reading_chunk = true;
|
||||
*this.skipping_chunk = skippable;
|
||||
}
|
||||
|
||||
while *this.chunk_read < *this.chunk_len {
|
||||
let remaining_poll_bytes = MAX_INPUT_BYTES_PER_POLL - input_bytes;
|
||||
if ready_reads >= MAX_READY_READS_PER_POLL || remaining_poll_bytes == 0 {
|
||||
cx.waker().wake_by_ref();
|
||||
return Poll::Pending;
|
||||
}
|
||||
let mut discard = [0u8; 8192];
|
||||
let mut read_buf = if *this.skipping_chunk {
|
||||
let remaining = *this.chunk_len - *this.chunk_read;
|
||||
let discard_len = remaining.min(discard.len()).min(remaining_poll_bytes);
|
||||
ReadBuf::new(&mut discard[..discard_len])
|
||||
} else {
|
||||
let read_end = (*this.chunk_read + remaining_poll_bytes).min(*this.chunk_len);
|
||||
ReadBuf::new(&mut this.chunk_buf[*this.chunk_read..read_end])
|
||||
};
|
||||
match this.inner.as_mut().poll_read(cx, &mut read_buf) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(Ok(())) => {
|
||||
ready_reads += 1;
|
||||
let read = read_buf.filled().len();
|
||||
if read == 0 {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::UnexpectedEof,
|
||||
"unexpected EOF while reading S2 chunk body",
|
||||
)));
|
||||
}
|
||||
input_bytes += read;
|
||||
*this.chunk_read += read;
|
||||
}
|
||||
Poll::Ready(Err(err)) => {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(err));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
completed_chunks += 1;
|
||||
*this.reading_chunk = false;
|
||||
if *this.skipping_chunk {
|
||||
*this.skipping_chunk = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
let chunk = &this.chunk_buf[..*this.chunk_len];
|
||||
match *this.chunk_type {
|
||||
CHUNK_TYPE_STREAM_IDENTIFIER => {
|
||||
*this.frame_mode = if chunk == S2_MAGIC_BODY {
|
||||
FrameMode::S2
|
||||
} else if chunk == SNAPPY_MAGIC_BODY {
|
||||
FrameMode::Snappy
|
||||
} else {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, "invalid S2 stream identifier")));
|
||||
};
|
||||
continue;
|
||||
}
|
||||
CHUNK_TYPE_COMPRESSED_DATA | CHUNK_TYPE_UNCOMPRESSED_DATA => {
|
||||
if *this.frame_mode == FrameMode::Uninitialized {
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"S2 data chunk before stream identifier",
|
||||
)));
|
||||
}
|
||||
if let Err(err) = decode_chunk_into(
|
||||
this.output,
|
||||
chunk,
|
||||
*this.chunk_type == CHUNK_TYPE_COMPRESSED_DATA,
|
||||
this.frame_mode
|
||||
.max_decompressed_block_size(*this.max_s2_decompressed_block_size),
|
||||
) {
|
||||
this.output.clear();
|
||||
*this.output_pos = 0;
|
||||
*this.poisoned = true;
|
||||
return Poll::Ready(Err(err));
|
||||
}
|
||||
}
|
||||
_ => unreachable!("chunk type validated before reading its payload"),
|
||||
}
|
||||
|
||||
if this.output.is_empty() {
|
||||
*this.output_pos = 0;
|
||||
continue;
|
||||
}
|
||||
|
||||
*this.output_pos = 0;
|
||||
let to_copy = min(buf.remaining(), this.output.len());
|
||||
buf.put_slice(&this.output[..to_copy]);
|
||||
*this.output_pos += to_copy;
|
||||
return Poll::Ready(Ok(()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
delegate_reader_capabilities_generic_no_index!(S2Decoder<R>, inner);
|
||||
|
||||
fn decode_chunk_into(output: &mut Vec<u8>, chunk: &[u8], compressed: bool, max_decompressed_block_size: usize) -> io::Result<()> {
|
||||
let expected_crc = u32::from_le_bytes(
|
||||
chunk[..CHECKSUM_SIZE]
|
||||
.try_into()
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::InvalidData, "S2 chunk smaller than checksum header"))?,
|
||||
);
|
||||
let payload = &chunk[CHECKSUM_SIZE..];
|
||||
if compressed {
|
||||
let (decoded_len, _) = decode_len(payload)
|
||||
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("S2 length decode error: {err}")))?;
|
||||
if decoded_len > max_decompressed_block_size {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("S2 decompressed block size exceeds limit: size={decoded_len}, limit={max_decompressed_block_size}"),
|
||||
));
|
||||
}
|
||||
if output.len() >= decoded_len {
|
||||
let written = decode_into(&mut output[..decoded_len], payload)
|
||||
.map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("S2 decode error: {err}")))?;
|
||||
output.truncate(written);
|
||||
} else {
|
||||
*output =
|
||||
decode(payload).map_err(|err| io::Error::new(io::ErrorKind::InvalidData, format!("S2 decode error: {err}")))?;
|
||||
}
|
||||
} else {
|
||||
output.clear();
|
||||
output.extend_from_slice(payload);
|
||||
}
|
||||
|
||||
let actual_crc = crc(output);
|
||||
if actual_crc != expected_crc {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
format!("S2 CRC mismatch: expected={expected_crc:08x} actual={actual_crc:08x}"),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Cursor;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
};
|
||||
use std::task::{Wake, Waker};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[derive(Default)]
|
||||
struct WakeCounter(AtomicUsize);
|
||||
|
||||
impl Wake for WakeCounter {
|
||||
fn wake(self: Arc<Self>) {
|
||||
self.0.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
fn append_chunk_header(stream: &mut Vec<u8>, chunk_type: u8, payload_len: usize) {
|
||||
assert!(payload_len <= 0x00ff_ffff, "test chunk payload must fit the framing length");
|
||||
stream.push(chunk_type);
|
||||
stream.push(u8::try_from(payload_len & 0xff).expect("low length byte must fit u8"));
|
||||
stream.push(u8::try_from((payload_len >> 8) & 0xff).expect("middle length byte must fit u8"));
|
||||
stream.push(u8::try_from((payload_len >> 16) & 0xff).expect("high length byte must fit u8"));
|
||||
}
|
||||
|
||||
fn append_chunk(stream: &mut Vec<u8>, chunk_type: u8, payload: &[u8]) {
|
||||
append_chunk_header(stream, chunk_type, payload.len());
|
||||
stream.extend_from_slice(payload);
|
||||
}
|
||||
|
||||
fn append_uncompressed_chunk(stream: &mut Vec<u8>, payload: &[u8]) {
|
||||
let mut chunk = crc(payload).to_le_bytes().to_vec();
|
||||
chunk.extend_from_slice(payload);
|
||||
append_chunk(stream, CHUNK_TYPE_UNCOMPRESSED_DATA, &chunk);
|
||||
}
|
||||
|
||||
fn append_compressed_chunk(stream: &mut Vec<u8>, payload: &[u8]) {
|
||||
let mut chunk = crc(payload).to_le_bytes().to_vec();
|
||||
chunk.extend_from_slice(&minlz::encode(payload));
|
||||
append_chunk(stream, CHUNK_TYPE_COMPRESSED_DATA, &chunk);
|
||||
}
|
||||
|
||||
fn append_uvarint(output: &mut Vec<u8>, mut value: usize) {
|
||||
loop {
|
||||
let mut byte = u8::try_from(value & 0x7f).expect("varint byte must fit u8");
|
||||
value >>= 7;
|
||||
if value != 0 {
|
||||
byte |= 0x80;
|
||||
}
|
||||
output.push(byte);
|
||||
if value == 0 {
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const GO_S2_GOLDEN_PLAINTEXT: &[u8] = b"This is a test file with some repeated content to compress.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nThe quick brown fox jumps over the lazy dog.\nLorem ipsum dolor sit amet, consectetur adipiscing elit.\nLorem ipsum dolor sit amet, consectetur adipiscing elit.\nLorem ipsum dolor sit amet, consectetur adipiscing elit.\nBinary compatibility testing with S2 compression format.\nBinary compatibility testing with S2 compression format.\nBinary compatibility testing with S2 compression format.\nPerformance benchmarking and optimization verification.\nPerformance benchmarking and optimization verification.\nPerformance benchmarking and optimization verification.\n";
|
||||
|
||||
// Fixed interoperability fixture generated with the same dependency and
|
||||
// option as minio-go PutObjectsSnowball:
|
||||
// minio-go commit 0e78d3f18efe14e352e20d3a262b99df97b516b8
|
||||
// github.com/klauspost/compress v1.19.2
|
||||
// s2.NewWriter(dst, s2.WriterBetterCompression())
|
||||
// Generator body: create that writer on os.Stdout, io.Copy from os.Stdin,
|
||||
// then Close; invoke as `go run generator.go < input | xxd -p -c 100000`.
|
||||
// The exact input is GO_S2_GOLDEN_PLAINTEXT, so this test never creates its
|
||||
// expected stream through minlz.
|
||||
const GO_S2_GOLDEN_HEX: &str = "ff06000053327354774f00120100f8de94fbc105f06654686973206973206120746573742066696c65207769746820736f6d6520726570656174656420636f6e74656e7420746f20636f6d70726573732e0a54686520717569636b2062726f776e20666f78206a756d7073206f76657220746865206c617a7920646f67112d15004c684c6f72656d20697073756d20646f6c6f722073697420616d65742c01b85c73656374657475722061646970697363696e6720656c697411391500641442696e61727925432061746962696c697479257901952577045332356120696f6e20666f726d61113915006508506572057d306e63652062656e63686d61726b01a730616e64206f7074696d697a617401a41876657269666963050d1138150062";
|
||||
|
||||
#[tokio::test]
|
||||
async fn decodes_fixed_go_s2_fixture() {
|
||||
let fixture = hex_simd::decode_to_vec(GO_S2_GOLDEN_HEX).expect("golden fixture must be valid hex");
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
decoder.read_to_end(&mut output).await.expect("Go S2 fixture must decode");
|
||||
|
||||
assert_eq!(output, GO_S2_GOLDEN_PLAINTEXT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn indexed_chunk_boundary_mode_decodes_a_headerless_tail() {
|
||||
let fixture = hex_simd::decode_to_vec(GO_S2_GOLDEN_HEX).expect("golden fixture must be valid hex");
|
||||
let header_len = CHUNK_HEADER_LEN + S2_MAGIC_BODY.len();
|
||||
let mut decoder = S2Decoder::new_at_chunk_boundary(Cursor::new(fixture[header_len..].to_vec()));
|
||||
let mut output = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect("trusted indexed tail should decode without a stream identifier");
|
||||
|
||||
assert_eq!(output, GO_S2_GOLDEN_PLAINTEXT);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn strict_mode_rejects_a_headerless_data_chunk() {
|
||||
let fixture = hex_simd::decode_to_vec(GO_S2_GOLDEN_HEX).expect("golden fixture must be valid hex");
|
||||
let header_len = CHUNK_HEADER_LEN + S2_MAGIC_BODY.len();
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture[header_len..].to_vec()));
|
||||
let mut output = Vec::new();
|
||||
let err = decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect_err("untrusted stream must include a stream identifier");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("must be the first chunk"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn strict_mode_rejects_skippable_chunk_before_identifier() {
|
||||
let mut fixture = Vec::new();
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_PADDING, &[]);
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_STREAM_IDENTIFIER, S2_MAGIC_BODY);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
let err = decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect_err("strict streams must begin with an identifier even when the first chunk is skippable");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("must be the first chunk"));
|
||||
assert_eq!(
|
||||
decoder.get_ref().position(),
|
||||
u64::try_from(CHUNK_HEADER_LEN).expect("chunk header length must fit u64")
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snappy_identifier_enforces_the_64_kib_decoded_limit() {
|
||||
let payload = vec![0u8; MAX_SNAPPY_DECOMPRESSED_BLOCK_SIZE + 1];
|
||||
let mut fixture = Vec::new();
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_STREAM_IDENTIFIER, SNAPPY_MAGIC_BODY);
|
||||
append_compressed_chunk(&mut fixture, &payload);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
let err = decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect_err("Snappy frames must reject decoded blocks larger than 64 KiB");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("limit=65536"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn snappy_frame_accepts_large_noncanonical_encoding_with_small_output() {
|
||||
let payload = vec![b'x'; 40_000];
|
||||
let mut encoded = Vec::with_capacity(payload.len() * 2 + 3);
|
||||
append_uvarint(&mut encoded, payload.len());
|
||||
for byte in &payload {
|
||||
encoded.push(0); // One-byte literal tag.
|
||||
encoded.push(*byte);
|
||||
}
|
||||
assert!(encoded.len() > MAX_SNAPPY_DECOMPRESSED_BLOCK_SIZE);
|
||||
|
||||
let mut chunk = crc(&payload).to_le_bytes().to_vec();
|
||||
chunk.extend_from_slice(&encoded);
|
||||
let mut fixture = Vec::new();
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_STREAM_IDENTIFIER, SNAPPY_MAGIC_BODY);
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_COMPRESSED_DATA, &chunk);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect("Snappy encoded input uses the S2 reader cap while decoded output stays below 64 KiB");
|
||||
|
||||
assert_eq!(output, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn repeated_identifier_switches_the_frame_limit() {
|
||||
let payload = vec![b'x'; MAX_SNAPPY_DECOMPRESSED_BLOCK_SIZE + 1];
|
||||
let mut fixture = Vec::new();
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_STREAM_IDENTIFIER, SNAPPY_MAGIC_BODY);
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_STREAM_IDENTIFIER, S2_MAGIC_BODY);
|
||||
append_uncompressed_chunk(&mut fixture, &payload);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect("a later S2 identifier must restore the S2 block limit");
|
||||
|
||||
assert_eq!(output, payload);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_oversized_compressed_chunk_before_allocation() {
|
||||
let declared_len = MAX_S2_COMPRESSED_CHUNK_SIZE + 1;
|
||||
let mut fixture = Vec::new();
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_STREAM_IDENTIFIER, S2_MAGIC_BODY);
|
||||
append_chunk_header(&mut fixture, CHUNK_TYPE_COMPRESSED_DATA, declared_len);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
let err = decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect_err("oversized compressed chunks must fail from their header");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.to_string(), "invalid S2 chunk length");
|
||||
assert_eq!(
|
||||
decoder.get_ref().position(),
|
||||
u64::try_from(CHUNK_HEADER_LEN * 2 + S2_MAGIC_BODY.len()).expect("fixture prefix length must fit u64")
|
||||
);
|
||||
assert_eq!(decoder.chunk_buf.len(), S2_MAGIC_BODY.len());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn accepts_empty_uncompressed_and_compressed_chunks() {
|
||||
let mut fixture = Vec::new();
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_STREAM_IDENTIFIER, S2_MAGIC_BODY);
|
||||
append_uncompressed_chunk(&mut fixture, &[]);
|
||||
append_compressed_chunk(&mut fixture, &[]);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect("empty data chunks are valid framed-stream no-ops");
|
||||
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_blocks_reuse_the_decoded_output_allocation() {
|
||||
let first_payload = vec![b'a'; 32 * 1024];
|
||||
let second_payload = vec![b'b'; first_payload.len()];
|
||||
let mut first_chunk = crc(&first_payload).to_le_bytes().to_vec();
|
||||
first_chunk.extend_from_slice(&minlz::encode(&first_payload));
|
||||
let mut second_chunk = crc(&second_payload).to_le_bytes().to_vec();
|
||||
second_chunk.extend_from_slice(&minlz::encode(&second_payload));
|
||||
|
||||
let mut output = Vec::new();
|
||||
decode_chunk_into(&mut output, &first_chunk, true, MAX_S2_DECOMPRESSED_BLOCK_SIZE)
|
||||
.expect("first compressed block should decode");
|
||||
let allocation = output.as_ptr();
|
||||
decode_chunk_into(&mut output, &second_chunk, true, MAX_S2_DECOMPRESSED_BLOCK_SIZE)
|
||||
.expect("same-sized compressed block should decode into the existing allocation");
|
||||
|
||||
assert_eq!(output, second_payload);
|
||||
assert_eq!(output.as_ptr(), allocation);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_decompressed_block_length_above_framing_limit() {
|
||||
let mut fixture = b"\xff\x06\x00\x00S2sTwO".to_vec();
|
||||
fixture.extend_from_slice(&[CHUNK_TYPE_COMPRESSED_DATA, 8, 0, 0]);
|
||||
fixture.extend_from_slice(&[0, 0, 0, 0]);
|
||||
fixture.extend_from_slice(&[0x81, 0x80, 0x80, 0x02]);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
let err = decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect_err("oversized decoded block declaration must fail before allocation");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("exceeds limit"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_chunk_boundary_mode_rejects_blocks_above_decoder_safety_limit() {
|
||||
let mut chunk = vec![0u8; CHECKSUM_SIZE];
|
||||
append_uvarint(&mut chunk, MAX_LEGACY_S2_DECOMPRESSED_BLOCK_SIZE + 1);
|
||||
let mut fixture = Vec::new();
|
||||
append_chunk(&mut fixture, CHUNK_TYPE_COMPRESSED_DATA, &chunk);
|
||||
|
||||
let mut decoder = S2Decoder::new_at_legacy_chunk_boundary(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
let err = decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect_err("legacy compatibility must remain bounded before allocation");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains(&format!("limit={MAX_DECODE_DST_SIZE}")));
|
||||
assert!(output.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_crc_mismatch() {
|
||||
let mut fixture = hex_simd::decode_to_vec(GO_S2_GOLDEN_HEX).expect("golden fixture must be valid hex");
|
||||
fixture[14] ^= 0xff;
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = [0u8; 128];
|
||||
let err = decoder.read(&mut output).await.expect_err("CRC mismatch must fail");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert!(err.to_string().contains("CRC mismatch"));
|
||||
assert!(decoder.output.is_empty(), "unverified decoded bytes must be discarded");
|
||||
|
||||
let err = decoder
|
||||
.read(&mut output)
|
||||
.await
|
||||
.expect_err("a poisoned decoder must remain fail-closed on later reads");
|
||||
assert_eq!(err.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(err.to_string(), "S2 decoder previously failed");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn rejects_truncated_chunk() {
|
||||
let mut fixture = hex_simd::decode_to_vec(GO_S2_GOLDEN_HEX).expect("golden fixture must be valid hex");
|
||||
fixture.truncate(fixture.len() - 1);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
let err = decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect_err("truncated S2 chunk must fail");
|
||||
|
||||
assert_eq!(err.kind(), io::ErrorKind::UnexpectedEof);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn skips_large_extension_chunks_without_buffering_them() {
|
||||
const EXTENSION_SIZE: usize = 64 * 1024;
|
||||
|
||||
let mut fixture = b"\xff\x06\x00\x00S2sTwO".to_vec();
|
||||
append_chunk_header(&mut fixture, CHUNK_TYPE_PADDING, EXTENSION_SIZE);
|
||||
fixture.resize(fixture.len() + EXTENSION_SIZE, 0);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let mut output = Vec::new();
|
||||
decoder
|
||||
.read_to_end(&mut output)
|
||||
.await
|
||||
.expect("skippable extension must be consumed");
|
||||
|
||||
assert!(output.is_empty());
|
||||
assert_eq!(decoder.chunk_buf.len(), S2_MAGIC_BODY.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn yields_after_bounded_number_of_non_data_chunks() {
|
||||
let mut fixture = b"\xff\x06\x00\x00S2sTwO".to_vec();
|
||||
for _ in 0..MAX_CHUNKS_PER_POLL {
|
||||
fixture.extend_from_slice(&[CHUNK_TYPE_PADDING, 0, 0, 0]);
|
||||
}
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let wake_counter = Arc::new(WakeCounter::default());
|
||||
let waker = Waker::from(wake_counter.clone());
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
let mut output = [0u8; 1];
|
||||
let mut read_buf = ReadBuf::new(&mut output);
|
||||
|
||||
assert!(Pin::new(&mut decoder).poll_read(&mut cx, &mut read_buf).is_pending());
|
||||
assert!(read_buf.filled().is_empty());
|
||||
assert_eq!(wake_counter.0.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn yields_while_streaming_a_large_skippable_chunk() {
|
||||
const EXTENSION_SIZE: usize = MAX_INPUT_BYTES_PER_POLL * 2;
|
||||
|
||||
let mut fixture = b"\xff\x06\x00\x00S2sTwO".to_vec();
|
||||
append_chunk_header(&mut fixture, CHUNK_TYPE_PADDING, EXTENSION_SIZE);
|
||||
fixture.resize(fixture.len() + EXTENSION_SIZE, 0);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let wake_counter = Arc::new(WakeCounter::default());
|
||||
let waker = Waker::from(wake_counter.clone());
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
let mut output = [0u8; 1];
|
||||
let mut read_buf = ReadBuf::new(&mut output);
|
||||
|
||||
assert!(Pin::new(&mut decoder).poll_read(&mut cx, &mut read_buf).is_pending());
|
||||
assert!(read_buf.filled().is_empty());
|
||||
assert_eq!(
|
||||
decoder.get_ref().position(),
|
||||
u64::try_from(MAX_INPUT_BYTES_PER_POLL).expect("poll byte budget must fit u64")
|
||||
);
|
||||
assert_eq!(wake_counter.0.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn header_reads_respect_the_remaining_poll_byte_budget() {
|
||||
const FIRST_EXTENSION_SIZE: usize = MAX_INPUT_BYTES_PER_POLL - 15;
|
||||
|
||||
let mut fixture = b"\xff\x06\x00\x00S2sTwO".to_vec();
|
||||
append_chunk_header(&mut fixture, CHUNK_TYPE_PADDING, FIRST_EXTENSION_SIZE);
|
||||
fixture.resize(fixture.len() + FIRST_EXTENSION_SIZE, 0);
|
||||
fixture.extend_from_slice(&[CHUNK_TYPE_PADDING, 0, 0, 0]);
|
||||
|
||||
let mut decoder = S2Decoder::new(Cursor::new(fixture));
|
||||
let wake_counter = Arc::new(WakeCounter::default());
|
||||
let waker = Waker::from(wake_counter.clone());
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
let mut output = [0u8; 1];
|
||||
let mut read_buf = ReadBuf::new(&mut output);
|
||||
|
||||
assert!(Pin::new(&mut decoder).poll_read(&mut cx, &mut read_buf).is_pending());
|
||||
assert!(read_buf.filled().is_empty());
|
||||
assert_eq!(
|
||||
decoder.get_ref().position(),
|
||||
u64::try_from(MAX_INPUT_BYTES_PER_POLL).expect("poll byte budget must fit u64")
|
||||
);
|
||||
assert_eq!(wake_counter.0.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
}
|
||||
@@ -60,21 +60,26 @@ hotpath-cpu = [
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
metrics = { workspace = true }
|
||||
async-compression = { workspace = true, features = ["tokio", "gzip", "bzip2"] }
|
||||
async-trait.workspace = true
|
||||
arc-swap.workspace = true
|
||||
bytes = { workspace = true, features = ["serde"] }
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
crc-fast.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] }
|
||||
rustfs-ecstore.workspace = true
|
||||
rustfs-storage-api.workspace = true
|
||||
futures = { workspace = true }
|
||||
futures-core = { workspace = true }
|
||||
flate2.workspace = true
|
||||
http.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
serde_json = { workspace = true, features = ["raw_value"] }
|
||||
thiserror = { workspace = true }
|
||||
parking_lot.workspace = true
|
||||
tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
|
||||
tokio-stream.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
tracing.workspace = true
|
||||
uuid.workspace = true
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -23,10 +23,13 @@ use datafusion::{
|
||||
use std::{error::Error as StdError, fmt::Display};
|
||||
use thiserror::Error;
|
||||
|
||||
mod input_stream;
|
||||
mod metrics;
|
||||
pub mod object_store;
|
||||
pub mod query;
|
||||
pub mod server;
|
||||
mod storage_api;
|
||||
pub use metrics::{SelectInputMetrics, SelectInputMetricsSnapshot};
|
||||
pub use storage_api::SelectObjectSnapshot;
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -77,6 +80,9 @@ pub enum SelectError {
|
||||
#[error("The file is not in a supported compression format. Only GZIP and BZIP2 are supported.")]
|
||||
InvalidCompressionFormat,
|
||||
|
||||
#[error("{compression} is not applicable to the queried object. Please correct the request and try again.")]
|
||||
InvalidCompressionFormatForObject { compression: &'static str },
|
||||
|
||||
#[error("The data source type is not valid. Only CSV, JSON, and Parquet are supported.")]
|
||||
InvalidDataSource,
|
||||
|
||||
@@ -85,6 +91,9 @@ pub enum SelectError {
|
||||
)]
|
||||
TruncatedInput,
|
||||
|
||||
#[error("Scan range queries are not supported on this type of object.")]
|
||||
UnsupportedScanRangeInput,
|
||||
|
||||
#[error("An error occurred while parsing the CSV file. Check the file and try again.")]
|
||||
CsvParsingError,
|
||||
|
||||
@@ -94,6 +103,9 @@ pub enum SelectError {
|
||||
#[error("An error occurred while parsing the Parquet file. Check the file and try again.")]
|
||||
ParquetParsingError,
|
||||
|
||||
#[error("The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB.")]
|
||||
OverMaxRecordSize,
|
||||
|
||||
#[error("{message}")]
|
||||
ParseSelectFailure { message: String },
|
||||
|
||||
|
||||
@@ -0,0 +1,161 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use arc_swap::ArcSwap;
|
||||
use std::sync::{
|
||||
Arc,
|
||||
atomic::{AtomicU64, Ordering},
|
||||
};
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)]
|
||||
pub struct SelectInputMetricsSnapshot {
|
||||
pub bytes_scanned: u64,
|
||||
pub bytes_processed: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct SelectInputMetrics {
|
||||
active: ArcSwap<SelectInputMetricBank>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct SelectInputMetricBank {
|
||||
uncompressed_bytes: AtomicU64,
|
||||
compressed_bytes_scanned: AtomicU64,
|
||||
compressed_bytes_processed: AtomicU64,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct SelectInputMetricsRecorder {
|
||||
bank: Arc<SelectInputMetricBank>,
|
||||
}
|
||||
|
||||
impl Default for SelectInputMetrics {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
active: ArcSwap::from_pointee(SelectInputMetricBank::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SelectInputMetrics {
|
||||
pub fn snapshot(&self) -> SelectInputMetricsSnapshot {
|
||||
let bank = self.active.load();
|
||||
let uncompressed_bytes = bank.uncompressed_bytes.load(Ordering::Relaxed);
|
||||
SelectInputMetricsSnapshot {
|
||||
bytes_scanned: uncompressed_bytes.saturating_add(bank.compressed_bytes_scanned.load(Ordering::Relaxed)),
|
||||
bytes_processed: uncompressed_bytes.saturating_add(bank.compressed_bytes_processed.load(Ordering::Relaxed)),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn recorder(&self) -> SelectInputMetricsRecorder {
|
||||
SelectInputMetricsRecorder {
|
||||
bank: self.active.load_full(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Publishes a fresh bank so late planner writes remain isolated.
|
||||
pub fn reset(&self) {
|
||||
self.active.store(Arc::new(SelectInputMetricBank::default()));
|
||||
}
|
||||
}
|
||||
|
||||
impl SelectInputMetricsRecorder {
|
||||
pub(crate) fn record_uncompressed(&self, bytes: usize) {
|
||||
saturating_add(&self.bank.uncompressed_bytes, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_scanned(&self, bytes: usize) {
|
||||
saturating_add(&self.bank.compressed_bytes_scanned, bytes);
|
||||
}
|
||||
|
||||
pub(crate) fn record_processed(&self, bytes: usize) {
|
||||
saturating_add(&self.bank.compressed_bytes_processed, bytes);
|
||||
}
|
||||
}
|
||||
|
||||
fn saturating_add(counter: &AtomicU64, bytes: usize) {
|
||||
let increment = u64::try_from(bytes).unwrap_or(u64::MAX);
|
||||
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(increment)));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn records_uncompressed_input_at_both_boundaries() {
|
||||
let metrics = SelectInputMetrics::default();
|
||||
metrics.recorder().record_uncompressed(7);
|
||||
|
||||
assert_eq!(
|
||||
metrics.snapshot(),
|
||||
SelectInputMetricsSnapshot {
|
||||
bytes_scanned: 7,
|
||||
bytes_processed: 7,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn counters_saturate_instead_of_wrapping() {
|
||||
let metrics = SelectInputMetrics::default();
|
||||
metrics
|
||||
.active
|
||||
.load()
|
||||
.uncompressed_bytes
|
||||
.store(u64::MAX - 1, Ordering::Relaxed);
|
||||
|
||||
metrics.recorder().record_uncompressed(2);
|
||||
|
||||
assert_eq!(metrics.snapshot().bytes_scanned, u64::MAX);
|
||||
assert_eq!(metrics.snapshot().bytes_processed, u64::MAX);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_boundaries_are_counted_independently() {
|
||||
let metrics = SelectInputMetrics::default();
|
||||
let recorder = metrics.recorder();
|
||||
recorder.record_scanned(39);
|
||||
recorder.record_processed(19);
|
||||
|
||||
assert_eq!(
|
||||
metrics.snapshot(),
|
||||
SelectInputMetricsSnapshot {
|
||||
bytes_scanned: 39,
|
||||
bytes_processed: 19,
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reset_clears_schema_inference_bytes() {
|
||||
let metrics = SelectInputMetrics::default();
|
||||
let planning = metrics.recorder();
|
||||
planning.record_uncompressed(9);
|
||||
|
||||
metrics.reset();
|
||||
planning.record_uncompressed(5);
|
||||
let execution = metrics.recorder();
|
||||
execution.record_uncompressed(3);
|
||||
|
||||
assert_eq!(
|
||||
metrics.snapshot(),
|
||||
SelectInputMetricsSnapshot {
|
||||
bytes_scanned: 3,
|
||||
bytes_processed: 3,
|
||||
}
|
||||
);
|
||||
}
|
||||
}
|
||||
+2025
-272
File diff suppressed because it is too large
Load Diff
@@ -13,6 +13,43 @@
|
||||
// limitations under the License.
|
||||
|
||||
use datafusion::sql::sqlparser::ast::Statement;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum JsonPathSegment {
|
||||
Key { name: String, quoted: bool },
|
||||
Index(usize),
|
||||
ArrayWildcard,
|
||||
ObjectWildcard,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct JsonSource {
|
||||
path: Arc<[JsonPathSegment]>,
|
||||
scalar_column: Option<String>,
|
||||
}
|
||||
|
||||
impl JsonSource {
|
||||
pub fn new(path: Vec<JsonPathSegment>, scalar_column: Option<String>) -> Self {
|
||||
Self {
|
||||
path: path.into(),
|
||||
scalar_column,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn from_path(path: Vec<JsonPathSegment>) -> Self {
|
||||
Self::new(path, None)
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &[JsonPathSegment] {
|
||||
&self.path
|
||||
}
|
||||
|
||||
pub fn scalar_column(&self) -> Option<&str> {
|
||||
self.scalar_column.as_deref()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ExtStatement {
|
||||
|
||||
@@ -25,6 +25,8 @@ use super::{
|
||||
session::QueryAdmission,
|
||||
};
|
||||
|
||||
pub type DispatchedQuery = (Query, Output);
|
||||
|
||||
#[async_trait]
|
||||
pub trait QueryDispatcher: Send + Sync {
|
||||
// fn create_query_id(&self) -> QueryId;
|
||||
@@ -41,6 +43,18 @@ pub trait QueryDispatcher: Send + Sync {
|
||||
self.execute_query(query).await
|
||||
}
|
||||
|
||||
async fn dispatch_query(&self, query: &Query) -> QueryResult<DispatchedQuery> {
|
||||
let execution_query = query.for_execution();
|
||||
let output = self.execute_query(&execution_query).await?;
|
||||
Ok((execution_query, output))
|
||||
}
|
||||
|
||||
async fn dispatch_query_admitted(&self, query: &Query, admission: QueryAdmission) -> QueryResult<DispatchedQuery> {
|
||||
let execution_query = query.for_execution();
|
||||
let output = self.execute_query_admitted(&execution_query, admission).await?;
|
||||
Ok((execution_query, output))
|
||||
}
|
||||
|
||||
async fn build_logical_plan(&self, query_state_machine: Arc<QueryStateMachine>) -> QueryResult<Option<Plan>>;
|
||||
|
||||
async fn execute_logical_plan(&self, logical_plan: Plan, query_state_machine: Arc<QueryStateMachine>) -> QueryResult<Output>;
|
||||
@@ -53,3 +67,155 @@ pub trait QueryDispatcher: Send + Sync {
|
||||
|
||||
// fn cancel_query(&self, id: &QueryId);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::query::test_query;
|
||||
use parking_lot::Mutex;
|
||||
|
||||
#[derive(Default)]
|
||||
struct DefaultDispatchDispatcher {
|
||||
executed_metrics: Mutex<Vec<Arc<crate::SelectInputMetrics>>>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl QueryDispatcher for DefaultDispatchDispatcher {
|
||||
async fn execute_query(&self, query: &Query) -> QueryResult<Output> {
|
||||
self.executed_metrics.lock().push(Arc::clone(query.input_metrics()));
|
||||
Ok(Output::Nil(()))
|
||||
}
|
||||
|
||||
async fn build_logical_plan(&self, _query_state_machine: Arc<QueryStateMachine>) -> QueryResult<Option<Plan>> {
|
||||
unreachable!("default dispatch test does not plan queries")
|
||||
}
|
||||
|
||||
async fn execute_logical_plan(
|
||||
&self,
|
||||
_logical_plan: Plan,
|
||||
_query_state_machine: Arc<QueryStateMachine>,
|
||||
) -> QueryResult<Output> {
|
||||
unreachable!("default dispatch test does not execute plans")
|
||||
}
|
||||
|
||||
async fn build_query_state_machine(&self, _query: Query) -> QueryResult<Arc<QueryStateMachine>> {
|
||||
unreachable!("default dispatch test does not build state machines")
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct DistinctAdmittedDispatcher {
|
||||
plain_metrics: Mutex<Vec<Arc<crate::SelectInputMetrics>>>,
|
||||
admitted_metrics: Mutex<Vec<Arc<crate::SelectInputMetrics>>>,
|
||||
fail_plain: bool,
|
||||
fail_admitted: bool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl QueryDispatcher for DistinctAdmittedDispatcher {
|
||||
async fn execute_query(&self, query: &Query) -> QueryResult<Output> {
|
||||
self.plain_metrics.lock().push(Arc::clone(query.input_metrics()));
|
||||
if self.fail_plain {
|
||||
Err(crate::QueryError::Cancel)
|
||||
} else {
|
||||
Ok(Output::Nil(()))
|
||||
}
|
||||
}
|
||||
|
||||
async fn execute_query_admitted(&self, query: &Query, _admission: QueryAdmission) -> QueryResult<Output> {
|
||||
self.admitted_metrics.lock().push(Arc::clone(query.input_metrics()));
|
||||
if self.fail_admitted {
|
||||
Err(crate::QueryError::Cancel)
|
||||
} else {
|
||||
Ok(Output::Nil(()))
|
||||
}
|
||||
}
|
||||
|
||||
async fn build_logical_plan(&self, _query_state_machine: Arc<QueryStateMachine>) -> QueryResult<Option<Plan>> {
|
||||
unreachable!("dispatch routing test does not plan queries")
|
||||
}
|
||||
|
||||
async fn execute_logical_plan(
|
||||
&self,
|
||||
_logical_plan: Plan,
|
||||
_query_state_machine: Arc<QueryStateMachine>,
|
||||
) -> QueryResult<Output> {
|
||||
unreachable!("dispatch routing test does not execute plans")
|
||||
}
|
||||
|
||||
async fn build_query_state_machine(&self, _query: Query) -> QueryResult<Arc<QueryStateMachine>> {
|
||||
unreachable!("dispatch routing test does not build state machines")
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn plain_dispatch_propagates_override_errors() {
|
||||
let dispatcher = DistinctAdmittedDispatcher {
|
||||
fail_plain: true,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let error = match dispatcher.dispatch_query(&test_query()).await {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("plain override error should propagate"),
|
||||
};
|
||||
assert!(matches!(error, crate::QueryError::Cancel));
|
||||
assert_eq!(dispatcher.plain_metrics.lock().len(), 1);
|
||||
assert!(dispatcher.admitted_metrics.lock().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn default_dispatch_methods_use_distinct_execution_metrics() {
|
||||
let dispatcher = DefaultDispatchDispatcher::default();
|
||||
let query = test_query();
|
||||
|
||||
let (first, _) = dispatcher
|
||||
.dispatch_query(&query)
|
||||
.await
|
||||
.expect("first dispatch should execute");
|
||||
let (second, _) = dispatcher
|
||||
.dispatch_query(&query)
|
||||
.await
|
||||
.expect("second dispatch should execute");
|
||||
let (admitted, _) = dispatcher
|
||||
.dispatch_query_admitted(&query, QueryAdmission::unmanaged())
|
||||
.await
|
||||
.expect("admitted dispatch should execute");
|
||||
let executed_metrics = dispatcher.executed_metrics.lock();
|
||||
|
||||
assert!(!Arc::ptr_eq(first.input_metrics(), second.input_metrics()));
|
||||
assert!(!Arc::ptr_eq(first.input_metrics(), admitted.input_metrics()));
|
||||
assert!(Arc::ptr_eq(first.input_metrics(), &executed_metrics[0]));
|
||||
assert!(Arc::ptr_eq(second.input_metrics(), &executed_metrics[1]));
|
||||
assert!(Arc::ptr_eq(admitted.input_metrics(), &executed_metrics[2]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn admitted_dispatch_uses_the_admitted_override_and_propagates_errors() {
|
||||
let dispatcher = DistinctAdmittedDispatcher::default();
|
||||
let query = test_query();
|
||||
|
||||
let (dispatched, _) = dispatcher
|
||||
.dispatch_query_admitted(&query, QueryAdmission::unmanaged())
|
||||
.await
|
||||
.expect("admitted dispatch should execute through its override");
|
||||
assert!(dispatcher.plain_metrics.lock().is_empty());
|
||||
{
|
||||
let admitted_metrics = dispatcher.admitted_metrics.lock();
|
||||
assert_eq!(admitted_metrics.len(), 1);
|
||||
assert!(Arc::ptr_eq(dispatched.input_metrics(), &admitted_metrics[0]));
|
||||
}
|
||||
|
||||
let failing = DistinctAdmittedDispatcher {
|
||||
fail_admitted: true,
|
||||
..Default::default()
|
||||
};
|
||||
let error = match failing.dispatch_query_admitted(&query, QueryAdmission::unmanaged()).await {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("admitted override error should propagate"),
|
||||
};
|
||||
assert!(matches!(error, crate::QueryError::Cancel));
|
||||
assert!(failing.plain_metrics.lock().is_empty());
|
||||
assert_eq!(failing.admitted_metrics.lock().len(), 1);
|
||||
}
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user