diff --git a/.agents/skills/security-advisory-lessons/references/advisory-patterns.md b/.agents/skills/security-advisory-lessons/references/advisory-patterns.md index 954e0d0ef..7380499ca 100644 --- a/.agents/skills/security-advisory-lessons/references/advisory-patterns.md +++ b/.agents/skills/security-advisory-lessons/references/advisory-patterns.md @@ -48,6 +48,7 @@ Update this file only when an advisory adds or changes a reusable lesson, affect ### S3 object actions, copy, multipart, and upload policy validation +- `GHSA-g8w9-qw9q-fghr`: a valid presigned `PutObject` accepted extra `x-amz-tagging`, website redirect, and storage-class headers omitted from `SignedHeaders`. Lesson: a presigned URL is a bounded capability; reject `x-amz-*` headers that are not cryptographically bound by the signature so unsigned metadata cannot change authorization, lifecycle, redirect, cost, or durability semantics. - `GHSA-3ppv-fx5m-m749`: explicit `versionId` reads and copy sources authorized `s3:GetObject` instead of `s3:GetObjectVersion`. Lesson: version-specific object access must select version-specific actions for direct reads, `CopyObject`, and `UploadPartCopy`, with tests proving the backend is not reached on denial. - `GHSA-x298-9x87-fvjq`: anonymous `ListObjectVersions` fell back to `ListBucket` and returned before public-access-block gates. Lesson: compatibility fallbacks must converge on the same post-authorization checks as direct grants, especially `RestrictPublicBuckets` and anonymous data-plane denies. - `GHSA-mx42-j6wv-px98`: `UploadPartCopy` missed source authorization and allowed cross-bucket object exfiltration. Lesson: multipart copy must enforce the same source and destination contract as `CopyObject`. @@ -119,7 +120,7 @@ Use these targeted searches when a diff touches security-sensitive code: ```bash rg -n "validate_admin_request|check_permissions|AdminAction::|deny_only|is_allowed" rustfs crates rg -n "authorize_operation|FtpsDriver|SftpDriver|RETR|MKD|SIZE|MDTM|CreateBucket|GetObject|HeadObject" crates/protocols rustfs -rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|content-length-range|starts-with" rustfs crates +rg -n "UploadPartCopy|upload_part_copy|CompleteMultipart|PostObject|presign|SignedHeaders|content-length-range|starts-with" rustfs crates rg -n "ListBucketVersions|GetObjectVersion|versionId|VersionId|ExistingObjectTag|ForAllValues|ForAnyValue|POLICY_PLUGIN|opa" rustfs crates rg -n "normalize_extract_entry_key|Snowball|auto-extract|PathBuf::join|canonicalize|\\.\\.|x-forwarded-for|x-real-ip|SourceIp" rustfs crates rg -n "DEFAULT_SECRET|DEFAULT_ACCESS|TEST_PRIVATE_KEY|rustfs rpc|RUSTFS_RPC_SECRET" rustfs crates @@ -136,6 +137,7 @@ rg -n "deny_unknown_fields|serde.default|as u32|as usize|as i32" rustfs crates - Protocol frontend authz fixes: include denied `RETR`, `SIZE`/`MDTM`, `MKD`, bucket probe, and sibling allowed-operation cases, and assert denied paths do not reach the storage backend. - IAM fixes: include import/update/list service-account cases with attacker-controlled parent, claims, access key, secret key, and policy. - Copy/upload fixes: include cross-bucket, cross-user, source-denied, destination-denied, copy-source-condition, and multipart completion cases. +- Presigned upload fixes: include a valid presign with extra unsigned tagging, redirect, and storage-class headers; require rejection before storage access, and verify explicitly signed equivalents still work. - Version-action fixes: include historical UUID, explicit current version, `null`, range, partNumber, presigned, STS/session, service-account, anonymous bucket-policy, copy source, and multipart-copy source cases. - Policy-condition fixes: include reserved-key header collisions, missing keys, partially overlapping multi-value sets, plugin mode, and built-in policy mode. - Path fixes: include encoded traversal, absolute path, nested traversal, archive entries with `..`, valid object keys that resemble traversal text but should be rejected, and canonical bucket/prefix boundary checks. diff --git a/.config/e2e-nightly-selection.txt b/.config/e2e-nightly-selection.txt index 1c8b22d22..a3a2c14ad 100644 --- a/.config/e2e-nightly-selection.txt +++ b/.config/e2e-nightly-selection.txt @@ -1 +1 @@ -sha256=071be531eef021e9b772837d47bb32b0aa2c146c68baf055ce6e7f2cc3fce4c1 +sha256=9c2b958035a038ffd5ab98cac5f59a1b8e6a16e141f109ec7fb956afc0f11105 diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index d890335e5..3e366481b 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -70,11 +70,24 @@ jobs: warp --version || true df -h /data | tail -1 - - name: Reset test environment (before) + - name: Cleanup environment (before) if: ${{ inputs.cleanup_before != 'false' }} run: | - chmod +x auto-testing/rustfs_heal_test.sh - ./auto-testing/rustfs_heal_test.sh --reset -y + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done - name: Install RustFS package & start cluster run: | @@ -115,10 +128,24 @@ jobs: /tmp/rustfs-warp.*.log if-no-files-found: warn - - name: Reset test environment (after) + - name: Cleanup environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./auto-testing/rustfs_heal_test.sh --reset -y + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done - name: Notify on failure if: failure() diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 07f80d520..0cc3dd16a 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -12,7 +12,7 @@ on: required: false type: string workflow_run: - # Strict shared-environment order: run after S3 compatibility test succeeds. + # Strict shared-environment order: run after S3 compatibility test completes. workflows: ["RustFS S3 Compatibility Test"] types: [completed] @@ -38,8 +38,9 @@ env: jobs: kms-test: runs-on: smoke-testing + continue-on-error: true timeout-minutes: 420 - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }} steps: - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -86,6 +87,7 @@ jobs: - name: Run KMS suite id: test + continue-on-error: true env: LOG_FILE: /tmp/rustfs-kms.log run: | @@ -119,12 +121,65 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi + CASE_TABLE="/tmp/rustfs-kms-cases.md" + python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' + import re + import sys + + log_file, out_file = sys.argv[1], sys.argv[2] + ansi = re.compile(r'\x1b\[[0-9;]*m') + start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$') + done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b') + + rows = [] + index = {} + try: + with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: + for raw in fh: + line = ansi.sub('', raw).strip() + m = start_re.match(line) + if m: + case_id, name = m.group(1), m.group(2) + if case_id not in index: + index[case_id] = len(rows) + rows.append([case_id, name, 'RUNNING']) + continue + m = done_re.match(line) + if m: + status, case_id = m.group(1), m.group(2) + if case_id in index: + rows[index[case_id]][2] = status + else: + rows.append([case_id, case_id, status]) + index[case_id] = len(rows) - 1 + except FileNotFoundError: + rows = [] + + counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} + for _, _, status in rows: + counts[status] = counts.get(status, 0) + 1 + + with open(out_file, 'w', encoding='utf-8') as out: + out.write('## Case Summary\n\n') + out.write(f"- Total: {len(rows)}\\n") + out.write(f"- PASS: {counts.get('PASS', 0)}\\n") + out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") + out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") + out.write('\\n') + out.write('| Case | Name | Status |\\n') + out.write('| --- | --- | --- |\\n') + for case_id, name, status in rows: + out.write(f'| {case_id} | {name} | {status} |\\n') + PY { echo "# RustFS KMS test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" + echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "" + cat "${CASE_TABLE}" || true echo "" echo "## Log tail" echo '```text' @@ -133,6 +188,129 @@ jobs: } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + - name: Upload functional report to dashboard + if: always() + continue-on-error: true + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + REPORT_FILE: /tmp/rustfs-kms-report.md + SUITE: kms + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" + exit 0 + fi + DATE="$(date -u +%Y-%m-%d)" + REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" + CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${SHA}" ]; then + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + else + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + fi + + cat > /tmp/rustfs-functional-index.html <<'EOF' + + + + + + RustFS Functional Test Reports + + + +
+
+

RustFS Functional Test Reports

+

S3, KMS, Tier report tabs. Each tab lists reports by date.

+
+ +
+
+ + + + EOF + + INDEX_PATH="functional/index.html" + INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')" + INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${INDEX_SHA}" ]; then + jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + else + jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + fi + - name: Upload report and logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index 7dedb7787..31db6579a 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -46,7 +46,7 @@ on: type: boolean default: true workflow_run: - # Strict shared-environment order: run after tier test succeeds. + # Strict shared-environment order: run after tier test completes. workflows: ["RustFS Tier Test"] types: [completed] @@ -75,11 +75,124 @@ env: RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} jobs: + heal-test: + name: Heal test + runs-on: smoke-testing + timeout-minutes: 480 + if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }} + steps: + - name: Checkout auto-testing scripts + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + repository: rustfs/auto-testing + ref: main + path: auto-testing + persist-credentials: false + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + + - name: Show environment + run: | + uname -a + jq --version + openssl version + df -h /data | tail -1 + + - name: Cleanup environment (before) + if: ${{ inputs.cleanup_before != 'false' }} + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done + + - name: Install RustFS package & start cluster + run: | + ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}") + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + + - name: Preflight checks + run: | + ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}") + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + + - name: Run heal test (write -> outage -> heal -> verify) + run: | + ARGS=(--steps "3,4,5,6,7" -y \ + --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ + --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \ + --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \ + --log-file /tmp/rustfs-heal-test.log) + if [ -n "${{ inputs.package_url }}" ]; then + ARGS+=(--package-url "${{ inputs.package_url }}") + else + ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") + fi + ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: rustfs-heal-test-${{ github.run_id }} + path: | + /tmp/rustfs-heal-test.log + /tmp/rustfs-warp.*.log + if-no-files-found: warn + + - name: Cleanup environment (after) + if: ${{ always() && inputs.cleanup_after != 'false' }} + run: | + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done + + - name: Notify on failure + if: failure() + run: | + echo "RustFS heal test failed" + echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}" + echo "See the uploaded log artifact for details." + + # Pool expansion runs after heal regardless of heal outcome. pool-expansion-test: name: Pool expansion / decommission test runs-on: smoke-testing timeout-minutes: 360 - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + needs: heal-test + if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run') }} steps: - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -98,13 +211,26 @@ jobs: warp --version || true df -h /data | tail -1 - - name: Reset test environment (before) + - name: Cleanup environment (before) if: ${{ inputs.cleanup_before != 'false' }} run: | - chmod +x auto-testing/rustfs_pool_expand.sh - ./auto-testing/rustfs_pool_expand.sh --reset -y + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done - - name: Install RustFS package & start first pool + - name: Install RustFS package & start cluster run: | ARGS=(--steps "1,2,3" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}") if [ -n "${{ inputs.package_url }}" ]; then @@ -163,10 +289,24 @@ jobs: /tmp/rustfs-warp.*.log if-no-files-found: warn - - name: Reset test environment (after) + - name: Cleanup environment (after) if: ${{ always() && inputs.cleanup_after != 'false' }} run: | - ./auto-testing/rustfs_pool_expand.sh --reset -y + set -euo pipefail + read -r -a NODES <<< "${RUSTFS_NODES:-vm000 vm001 vm002}" + SSH_USER="${RUSTFS_SSH_USER:-azureuser}" + for node in "${NODES[@]}"; do + ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "${SSH_USER}@${node}" ' + set -euo pipefail + SUDO=""; [ "$(id -u)" -ne 0 ] && SUDO="sudo -n" + ${SUDO} systemctl stop rustfs 2>/dev/null || true + if ${SUDO} dpkg -l rustfs 2>/dev/null | grep -q "^ii"; then + ${SUDO} dpkg -P rustfs + fi + for i in 1 2 3 4; do ${SUDO} rm -rf /data/rustfs${i}/mnmd; done + ${SUDO} rm -rf /var/log/rustfs /var/lib/rustfs/kms /var/lib/rustfs/kms-backup + ' + done - name: Notify on failure if: failure() @@ -174,82 +314,3 @@ jobs: echo "RustFS pool expansion test failed" echo "Package source: ${{ inputs.package_url || inputs.rustfs_version || 'nightly (R2 latest)' }}" echo "See the uploaded log artifact for details." - - # Heal regression runs after the pool test regardless of its outcome: a pool - # failure must be reported (it makes the run red) but must not block heal. - heal-test: - name: Heal test (after pool test) - runs-on: smoke-testing - timeout-minutes: 480 - needs: pool-expansion-test - if: ${{ always() && (github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success') }} - steps: - - name: Checkout auto-testing scripts - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 - with: - repository: rustfs/auto-testing - ref: main - path: auto-testing - persist-credentials: false - token: ${{ secrets.PF_TESTING_GH_TOKEN }} - - - name: Reset test environment (before) - if: ${{ inputs.cleanup_before != 'false' }} - run: | - chmod +x auto-testing/rustfs_heal_test.sh - ./auto-testing/rustfs_heal_test.sh --reset -y - - - name: Install RustFS package & start cluster - run: | - ARGS=(--steps "1,2" -y --endpoint "${{ env.RUSTFS_API_ENDPOINT }}") - if [ -n "${{ inputs.package_url }}" ]; then - ARGS+=(--package-url "${{ inputs.package_url }}") - else - ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") - fi - ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - - - name: Preflight checks - run: | - ARGS=(--preflight --endpoint "${{ env.RUSTFS_API_ENDPOINT }}") - if [ -n "${{ inputs.package_url }}" ]; then - ARGS+=(--package-url "${{ inputs.package_url }}") - else - ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") - fi - ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - - - name: Run heal test (write -> outage -> heal -> verify) - run: | - ARGS=(--steps "3,4,5,6,7" -y \ - --endpoint "${{ env.RUSTFS_API_ENDPOINT }}" \ - --stop-node-gb "${{ inputs.stop_node_gb || '15' }}" \ - --warp-stop-gb "${{ inputs.warp_stop_gb || '40' }}" \ - --log-file /tmp/rustfs-heal-test.log) - if [ -n "${{ inputs.package_url }}" ]; then - ARGS+=(--package-url "${{ inputs.package_url }}") - else - ARGS+=(--package-url "${{ env.RUSTFS_NIGHTLY_PACKAGE_URL }}") - fi - ./auto-testing/rustfs_heal_test.sh "${ARGS[@]}" - - - name: Upload test logs - if: always() - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 - with: - name: rustfs-heal-test-${{ github.run_id }} - path: | - /tmp/rustfs-heal-test.log - /tmp/rustfs-warp.*.log - if-no-files-found: warn - - - name: Reset test environment (after) - if: ${{ always() && inputs.cleanup_after != 'false' }} - run: | - ./auto-testing/rustfs_heal_test.sh --reset -y - - - name: Notify on failure - if: failure() - run: | - echo "RustFS heal test failed" - echo "See the uploaded log artifact for details." diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index 2efe2986b..fffd0621f 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -33,10 +33,12 @@ env: RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }} RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }} RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }} + PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }} jobs: s3-compat-test: runs-on: smoke-testing + continue-on-error: true timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} steps: @@ -76,6 +78,7 @@ jobs: - name: Run S3 compatibility suite id: test + continue-on-error: true env: LOG_FILE: /tmp/rustfs-s3-compat.log run: | @@ -109,12 +112,68 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi + CASE_TABLE="/tmp/rustfs-s3-compat-cases.md" + python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' + import re + import sys + + log_file, out_file = sys.argv[1], sys.argv[2] + ansi = re.compile(r'\x1b\[[0-9;]*m') + start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$') + done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b') + + rows = [] + index = {} + current = None + try: + with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: + for raw in fh: + line = ansi.sub('', raw).strip() + m = start_re.match(line) + if m: + case_id, name = m.group(1), m.group(2) + current = case_id + if case_id not in index: + index[case_id] = len(rows) + rows.append([case_id, name, 'RUNNING']) + continue + m = done_re.match(line) + if m: + status, case_id = m.group(1), m.group(2) + if case_id in index: + rows[index[case_id]][2] = status + else: + rows.append([case_id, case_id, status]) + index[case_id] = len(rows) - 1 + current = None + except FileNotFoundError: + rows = [] + + counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} + for _, _, status in rows: + counts[status] = counts.get(status, 0) + 1 + + with open(out_file, 'w', encoding='utf-8') as out: + out.write('## Case Summary\n\n') + out.write(f"- Total: {len(rows)}\\n") + out.write(f"- PASS: {counts.get('PASS', 0)}\\n") + out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") + out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") + out.write('\\n') + out.write('| Case | Name | Status |\\n') + out.write('| --- | --- | --- |\\n') + for case_id, name, status in rows: + out.write(f'| {case_id} | {name} | {status} |\\n') + PY { echo "# RustFS S3 compatibility test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" + echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "" + cat "${CASE_TABLE}" || true echo "" echo "## Log tail" echo '```text' @@ -123,6 +182,129 @@ jobs: } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + - name: Upload functional report to dashboard + if: always() + continue-on-error: true + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + REPORT_FILE: /tmp/rustfs-s3-compat-report.md + SUITE: s3 + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" + exit 0 + fi + DATE="$(date -u +%Y-%m-%d)" + REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" + CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${SHA}" ]; then + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + else + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + fi + + cat > /tmp/rustfs-functional-index.html <<'EOF' + + + + + + RustFS Functional Test Reports + + + +
+
+

RustFS Functional Test Reports

+

S3, KMS, Tier report tabs. Each tab lists reports by date.

+
+ +
+
+ + + + EOF + + INDEX_PATH="functional/index.html" + INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')" + INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${INDEX_SHA}" ]; then + jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + else + jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + fi + - name: Upload report and logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 2fda50b70..765e0d867 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -12,7 +12,7 @@ on: required: false type: string workflow_run: - # Strict shared-environment order: run after KMS test succeeds. + # Strict shared-environment order: run after KMS test completes. workflows: ["RustFS KMS Test"] types: [completed] @@ -38,8 +38,9 @@ env: jobs: tier-test: runs-on: smoke-testing + continue-on-error: true timeout-minutes: 420 - if: ${{ github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' }} + if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_run' }} steps: - name: Checkout auto-testing scripts uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 @@ -107,6 +108,7 @@ jobs: - name: Run tier suite id: test + continue-on-error: true env: LOG_FILE: /tmp/rustfs-tier.log run: | @@ -140,12 +142,65 @@ jobs: else PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}" fi + CASE_TABLE="/tmp/rustfs-tier-cases.md" + python3 - "${LOG_FILE}" "${CASE_TABLE}" <<'PY' + import re + import sys + + log_file, out_file = sys.argv[1], sys.argv[2] + ansi = re.compile(r'\x1b\[[0-9;]*m') + start_re = re.compile(r'^---\s+([A-Z]+-[0-9]+)\s+(.+?)\s+---$') + done_re = re.compile(r'^\[(PASS|FAIL|UNSUPPORTED)\]\s+([A-Z]+-[0-9]+)\b') + + rows = [] + index = {} + try: + with open(log_file, 'r', encoding='utf-8', errors='replace') as fh: + for raw in fh: + line = ansi.sub('', raw).strip() + m = start_re.match(line) + if m: + case_id, name = m.group(1), m.group(2) + if case_id not in index: + index[case_id] = len(rows) + rows.append([case_id, name, 'RUNNING']) + continue + m = done_re.match(line) + if m: + status, case_id = m.group(1), m.group(2) + if case_id in index: + rows[index[case_id]][2] = status + else: + rows.append([case_id, case_id, status]) + index[case_id] = len(rows) - 1 + except FileNotFoundError: + rows = [] + + counts = {'PASS': 0, 'FAIL': 0, 'UNSUPPORTED': 0, 'RUNNING': 0} + for _, _, status in rows: + counts[status] = counts.get(status, 0) + 1 + + with open(out_file, 'w', encoding='utf-8') as out: + out.write('## Case Summary\n\n') + out.write(f"- Total: {len(rows)}\\n") + out.write(f"- PASS: {counts.get('PASS', 0)}\\n") + out.write(f"- FAIL: {counts.get('FAIL', 0)}\\n") + out.write(f"- UNSUPPORTED: {counts.get('UNSUPPORTED', 0)}\\n") + out.write('\\n') + out.write('| Case | Name | Status |\\n') + out.write('| --- | --- | --- |\\n') + for case_id, name, status in rows: + out.write(f'| {case_id} | {name} | {status} |\\n') + PY { echo "# RustFS tier test report" echo "" echo "- Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" echo "- Trigger: ${{ github.event_name }}" echo "- Package: ${PACKAGE_SOURCE}" + echo "- Test Step Outcome: ${{ steps.test.outcome }}" + echo "" + cat "${CASE_TABLE}" || true echo "" echo "## Log tail" echo '```text' @@ -154,6 +209,129 @@ jobs: } | tee "${REPORT_FILE}" cat "${REPORT_FILE}" >> "${GITHUB_STEP_SUMMARY}" + - name: Upload functional report to dashboard + if: always() + continue-on-error: true + env: + GH_TOKEN: ${{ env.PF_TESTING_GH_TOKEN }} + REPORT_FILE: /tmp/rustfs-tier-report.md + SUITE: tier + run: | + set -euo pipefail + if [ -z "${GH_TOKEN:-}" ]; then + echo "PF_TESTING_GH_TOKEN is not configured; skipping dashboard upload" + exit 0 + fi + DATE="$(date -u +%Y-%m-%d)" + REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" + CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${SHA}" ]; then + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + else + jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null + fi + + cat > /tmp/rustfs-functional-index.html <<'EOF' + + + + + + RustFS Functional Test Reports + + + +
+
+

RustFS Functional Test Reports

+

S3, KMS, Tier report tabs. Each tab lists reports by date.

+
+ +
+
+ + + + EOF + + INDEX_PATH="functional/index.html" + INDEX_CONTENT="$(python3 -c 'import base64;print(base64.b64encode(open("/tmp/rustfs-functional-index.html","rb").read()).decode())')" + INDEX_SHA="$(gh api "repos/rustfs/dashboard/contents/${INDEX_PATH}" -q '.sha' 2>/dev/null || true)" + if [ -n "${INDEX_SHA}" ]; then + jq -n --arg msg "functional ui update" --arg content "${INDEX_CONTENT}" --arg sha "${INDEX_SHA}" \ + '{message:$msg, content:$content, sha:$sha}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + else + jq -n --arg msg "functional ui init" --arg content "${INDEX_CONTENT}" \ + '{message:$msg, content:$content}' \ + | gh api --method PUT "repos/rustfs/dashboard/contents/${INDEX_PATH}" --input - >/dev/null + fi + - name: Upload report and logs if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/Cargo.lock b/Cargo.lock index 1f6087f27..1375bf3b3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3926,6 +3926,7 @@ dependencies = [ "aws-sdk-s3", "aws-sdk-sts", "aws-smithy-http-client", + "aws-smithy-types", "base64-simd", "bytes", "chrono", @@ -10557,10 +10558,14 @@ dependencies = [ name = "rustfs-s3select-api" version = "1.0.0-rc.4" dependencies = [ + "arc-swap", + "async-compression", "async-trait", "bytes", "chrono", + "crc-fast", "datafusion", + "flate2", "futures", "futures-core", "hotpath", @@ -10576,6 +10581,7 @@ dependencies = [ "serial_test", "thiserror 2.0.20", "tokio", + "tokio-stream", "tokio-util", "tracing", "transform-stream", diff --git a/crates/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml index 75a539561..0a0a70872 100644 --- a/crates/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -100,6 +100,7 @@ aws-sdk-s3 = { workspace = true, default-features = false, features = ["sigv4a", aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] } aws-config = { workspace = true } aws-smithy-http-client = { workspace = true, default-features = false, features = ["rustls-aws-lc"] } +aws-smithy-types.workspace = true async-compression = { workspace = true, features = ["tokio", "bzip2", "xz"] } async-trait = { workspace = true } flate2.workspace = true diff --git a/crates/e2e_test/src/cluster_multidrive_pool_test.rs b/crates/e2e_test/src/cluster_multidrive_pool_test.rs index 3f88e5af4..411336754 100644 --- a/crates/e2e_test/src/cluster_multidrive_pool_test.rs +++ b/crates/e2e_test/src/cluster_multidrive_pool_test.rs @@ -27,8 +27,10 @@ //! Readiness is established by the harness's `start()` handshake (TCP reachability //! plus an S3 `ListBuckets` poll) — there are no fixed sleeps. //! -//! Out of scope for this block (tracked separately): network fault injection -//! (toxiproxy / socket proxy) and 5GiB large-object budgets. +//! The volume-proxy smoke below also proves that the socket-level fault proxy +//! can be installed before startup without changing the client-facing node URL. +//! A full lock-plane partition matrix and 5GiB large-object budget remain +//! tracked separately. use crate::common::{ClusterTopology, RustFSTestClusterEnvironment}; @@ -76,6 +78,28 @@ async fn cluster_multidrive_single_pool_smoke() -> TestResult { Ok(()) } +/// 4 nodes x 4 drives, single pool: exercise the maximum local erasure layout +/// supported by the cluster harness. This remains in the nightly lane because +/// it starts four real server processes and sixteen data directories. +#[tokio::test] +async fn cluster_four_node_four_drive_single_pool_smoke() -> TestResult { + crate::common::init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(4, 4)).await?; + + let volumes = cluster.rustfs_volumes_arg(); + assert_eq!(volumes.split(' ').count(), 16, "expected 16 explicit endpoints, got: {volumes}"); + assert!(!volumes.contains('{'), "single-pool layout must not use ellipses: {volumes}"); + assert!(cluster.nodes.iter().all(|node| node.data_dirs.len() == 4)); + + cluster.start().await?; + cluster.create_test_bucket(BUCKET).await?; + + let payload = vec![0x3Cu8; 1024 * 1024]; + put_get_roundtrip(&cluster, "multidrive-4/object", &payload).await?; + Ok(()) +} + /// Two single-node pools, 2 drives each: the multi-pool layout boots and /// round-trips. Every pool is a distinct erasure pool (`pool_idx` 0 and 1). #[tokio::test] @@ -103,3 +127,27 @@ async fn cluster_two_pool_smoke() -> TestResult { put_get_roundtrip(&cluster, "twopool/object", &payload).await?; Ok(()) } + +/// A real cluster smoke for the volume FaultProxy wiring. The proxy target is +/// not listening yet when it is created; cluster startup must still converge +/// once the target node starts, and peer disk/RPC traffic must traverse it. +#[tokio::test] +async fn cluster_volume_fault_proxy_pass_smoke() -> TestResult { + crate::common::init_logging(); + + let mut cluster = RustFSTestClusterEnvironment::with_topology(ClusterTopology::single_pool_multidrive(2, 2)).await?; + let proxy = cluster.start_volume_proxy_for_node(0).await?; + let proxied = proxy.local_addr().to_string(); + assert!(cluster.rustfs_volumes_arg().contains(&proxied)); + + let result: TestResult = async { + cluster.start().await?; + cluster.create_test_bucket(BUCKET).await?; + let payload = vec![0x6Du8; 256 * 1024]; + put_get_roundtrip(&cluster, "volume-proxy/object", &payload).await + } + .await; + + proxy.shutdown().await; + result +} diff --git a/crates/e2e_test/src/common.rs b/crates/e2e_test/src/common.rs index 79a6f6864..cb942830a 100644 --- a/crates/e2e_test/src/common.rs +++ b/crates/e2e_test/src/common.rs @@ -34,6 +34,7 @@ use serde_json; use std::ffi::OsStr; use std::fs as stdfs; use std::io::ErrorKind; +use std::net::SocketAddr; use std::path::{Path, PathBuf}; use std::process::{Child, Command, Stdio}; use std::sync::Once; @@ -1214,6 +1215,9 @@ pub struct RustFSTestClusterEnvironment { pub node_extra_env: Vec>, pub node_capture_log_paths: Vec>, pub topology: ClusterTopology, + /// Optional socket proxies used for the corresponding node's volume + /// endpoints. Proxies must be installed before [`Self::start`]. + volume_proxy_addresses: Vec>, } impl RustFSTestClusterEnvironment { @@ -1305,6 +1309,7 @@ impl RustFSTestClusterEnvironment { extra_env.push(("RUSTFS_UNSAFE_BYPASS_DISK_CHECK".to_string(), "true".to_string())); } + let node_count = topology.node_count; Ok(Self { nodes, temp_dir, @@ -1314,6 +1319,7 @@ impl RustFSTestClusterEnvironment { node_extra_env: vec![Vec::new(); topology.node_count], node_capture_log_paths: vec![None; topology.node_count], topology, + volume_proxy_addresses: vec![None; node_count], }) } @@ -1381,6 +1387,34 @@ impl RustFSTestClusterEnvironment { self.build_volumes_arg() } + /// Start a socket proxy for one node's volume endpoints and route all + /// subsequent `RUSTFS_VOLUMES` references for that node through it. + /// + /// Call this before [`Self::start`], then use the returned proxy's + /// [`crate::fault_proxy::FaultProxy::set_mode`] to inject latency, + /// blackhole, or one-way partition faults. The node's own listen address + /// remains direct, so S3 clients can still reach it while peer disk/RPC + /// traffic is steered through the proxy. + pub async fn start_volume_proxy_for_node( + &mut self, + node_idx: usize, + ) -> Result> { + self.ensure_node_index(node_idx)?; + if self.volume_proxy_addresses[node_idx].is_some() { + return Err(format!("a volume proxy is already configured for node {node_idx}").into()); + } + let target = self.nodes[node_idx].address.parse::()?; + let proxy = crate::fault_proxy::FaultProxy::start(target).await?; + self.volume_proxy_addresses[node_idx] = Some(proxy.local_addr()); + Ok(proxy) + } + + fn volume_address(&self, node_idx: usize) -> String { + self.volume_proxy_addresses[node_idx] + .map(|address| address.to_string()) + .unwrap_or_else(|| self.nodes[node_idx].address.clone()) + } + fn build_volumes_arg(&self) -> String { let pools = self.topology.normalized_pools(); @@ -1389,7 +1423,11 @@ impl RustFSTestClusterEnvironment { return self .nodes .iter() - .flat_map(|n| n.data_dirs.iter().map(move |dir| format!("http://{}{}", n.address, dir))) + .enumerate() + .flat_map(|(node_idx, n)| { + let address = self.volume_address(node_idx); + n.data_dirs.iter().map(move |dir| format!("http://{}{}", address, dir)) + }) .collect::>() .join(" "); } @@ -1400,13 +1438,19 @@ impl RustFSTestClusterEnvironment { pools .iter() .map(|nodes| { - let node = &self.nodes[nodes[0]]; + let node_idx = nodes[0]; + let node = &self.nodes[node_idx]; let base = node .data_dirs .first() .and_then(|d| d.rsplit_once('/').map(|(parent, _)| parent)) .unwrap_or(&node.data_dir); - format!("http://{}{}/drive{{0...{}}}", node.address, base, self.topology.drives_per_node - 1) + format!( + "http://{}{}/drive{{0...{}}}", + self.volume_address(node_idx), + base, + self.topology.drives_per_node - 1 + ) }) .collect::>() .join(" ") @@ -2000,7 +2044,7 @@ mod tests { } let multidrive = topology.drives_per_node > 1; - let nodes = (0..topology.node_count) + let nodes: Vec = (0..topology.node_count) .map(|i| { let address = format!("127.0.0.1:{}", 9000 + i); let data_dirs: Vec = if multidrive { @@ -2021,6 +2065,7 @@ mod tests { }) .collect(); + let node_count = nodes.len(); RustFSTestClusterEnvironment { nodes, temp_dir, @@ -2030,6 +2075,7 @@ mod tests { node_extra_env: vec![Vec::new(); topology.node_count], node_capture_log_paths: vec![None; topology.node_count], topology, + volume_proxy_addresses: vec![None; node_count], } } @@ -2114,6 +2160,25 @@ mod tests { assert!(ClusterTopology::single_pool_multidrive(1, 1).validate().is_ok()); } + #[tokio::test] + async fn volume_proxy_rewrites_cluster_volume_endpoint() { + let mut env = RustFSTestClusterEnvironment::new(1) + .await + .expect("cluster environment should allocate a node"); + let direct = env.nodes[0].address.clone(); + let proxy = env + .start_volume_proxy_for_node(0) + .await + .expect("volume proxy should bind before the target server starts"); + let proxied = proxy.local_addr().to_string(); + let volumes = env.rustfs_volumes_arg(); + + assert!(volumes.contains(&proxied), "volumes must use the proxy address: {volumes}"); + assert!(!volumes.contains(&direct), "volumes must not retain the direct address: {volumes}"); + + proxy.shutdown().await; + } + #[test] fn cluster_node_env_supports_per_node_overrides() { let mut env = fake_cluster(ClusterTopology::single_pool(4)); diff --git a/crates/e2e_test/src/reliant/mod.rs b/crates/e2e_test/src/reliant/mod.rs index ddc42a855..80a18ac11 100644 --- a/crates/e2e_test/src/reliant/mod.rs +++ b/crates/e2e_test/src/reliant/mod.rs @@ -21,5 +21,6 @@ mod head_tls_bodyless_test; mod lifecycle; mod lock; mod node_interact_test; +mod s3_select_compression; mod sql; mod tiering; diff --git a/crates/e2e_test/src/reliant/s3_select_compression.rs b/crates/e2e_test/src/reliant/s3_select_compression.rs new file mode 100644 index 000000000..379352fe2 --- /dev/null +++ b/crates/e2e_test/src/reliant/s3_select_compression.rs @@ -0,0 +1,351 @@ +#![cfg(test)] +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::common::{RustFSTestEnvironment, init_logging}; +use async_compression::tokio::write::BzEncoder; +use aws_sdk_s3::{ + Client, + error::ProvideErrorMetadata, + operation::select_object_content::{SelectObjectContentOutput, builders::SelectObjectContentFluentBuilder}, + types::{ + CompressionType, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, JsonOutput, + JsonType, OutputSerialization, SelectObjectContentEventStream, + }, +}; +use aws_smithy_types::event_stream::RawMessage; +use bytes::Bytes; +use flate2::{Compression, write::GzEncoder}; +use std::{error::Error, io::Cursor, time::Duration}; +use tokio::io::AsyncWriteExt; + +const BUCKET: &str = "s3-select-compression"; +const SELECT_RESPONSE_TIMEOUT: Duration = Duration::from_secs(30); + +type TestResult = Result>; + +async fn create_test_environment(extra_env: &[(&str, &str)]) -> TestResult<(RustFSTestEnvironment, Client)> { + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server_with_env(vec![], extra_env).await?; + let client = env.create_s3_client(); + client.create_bucket().bucket(BUCKET).send().await?; + Ok((env, client)) +} + +async fn put_object(client: &Client, key: &str, body: &[u8]) -> TestResult<()> { + client + .put_object() + .bucket(BUCKET) + .key(key) + .body(Bytes::copy_from_slice(body).into()) + .send() + .await?; + Ok(()) +} + +fn gzip(input: &[u8]) -> TestResult> { + let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); + std::io::Write::write_all(&mut encoder, input)?; + Ok(encoder.finish()?) +} + +async fn bzip2(input: &[u8]) -> TestResult> { + let mut encoder = BzEncoder::new(Cursor::new(Vec::new())); + encoder.write_all(input).await?; + encoder.shutdown().await?; + Ok(encoder.into_inner().into_inner()) +} + +fn csv_select_request( + client: &Client, + key: &str, + compression: CompressionType, + expression: &str, +) -> SelectObjectContentFluentBuilder { + client + .select_object_content() + .bucket(BUCKET) + .key(key) + .expression(expression) + .expression_type(ExpressionType::Sql) + .input_serialization( + InputSerialization::builder() + .compression_type(compression) + .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build()) + .build(), + ) + .output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build()) +} + +fn json_select_request( + client: &Client, + key: &str, + compression: CompressionType, + json_type: JsonType, +) -> SelectObjectContentFluentBuilder { + client + .select_object_content() + .bucket(BUCKET) + .key(key) + .expression("SELECT name FROM S3Object") + .expression_type(ExpressionType::Sql) + .input_serialization( + InputSerialization::builder() + .compression_type(compression) + .json(JsonInput::builder().set_type(Some(json_type)).build()) + .build(), + ) + .output_serialization(OutputSerialization::builder().json(JsonOutput::builder().build()).build()) +} + +async fn collect_success( + mut response: SelectObjectContentOutput, + compressed_bytes: usize, + processed_bytes: usize, +) -> TestResult> { + tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move { + let mut records = Vec::new(); + let mut stats = None; + let mut saw_end = false; + + while let Some(event) = response.payload.recv().await? { + assert!(!saw_end, "Select emitted an event after End"); + match event { + SelectObjectContentEventStream::Records(event) => { + assert!(stats.is_none(), "Select emitted Records after Stats"); + if let Some(payload) = event.payload { + records.extend_from_slice(payload.as_ref()); + } + } + SelectObjectContentEventStream::Stats(event) => { + assert!(stats.is_none(), "Select emitted more than one Stats event"); + stats = event.details; + } + SelectObjectContentEventStream::End(_) => { + assert!(stats.is_some(), "Select emitted End before Stats"); + saw_end = true; + } + _ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"), + } + } + + let stats = stats.ok_or("Select response ended without a Stats event")?; + assert_eq!(stats.bytes_scanned(), Some(i64::try_from(compressed_bytes)?)); + assert_eq!(stats.bytes_processed(), Some(i64::try_from(processed_bytes)?)); + assert_eq!(stats.bytes_returned(), Some(i64::try_from(records.len())?)); + assert!(saw_end, "Select response ended without an End event"); + Ok::<_, Box>(records) + }) + .await + .map_err(|_| -> Box { "Select response timed out".into() })? +} + +async fn assert_truncated_stream_failure(mut response: SelectObjectContentOutput) -> TestResult<()> { + tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async move { + loop { + match response.payload.recv().await { + Err(error) => { + // S3 Select request-level errors use `error` frames, which this SDK version exposes as raw response errors. + if let Some(code) = error.code() { + assert_eq!(code, "TruncatedInput", "unexpected modeled event-stream error: {error:?}"); + } else if let aws_sdk_s3::error::SdkError::ResponseError(context) = &error + && let RawMessage::Decoded(message) = context.raw() + { + let header = |name: &str| { + message + .headers() + .iter() + .find(|header| header.name().as_str() == name) + .and_then(|header| header.value().as_string().ok()) + .map(|value| value.as_str()) + }; + assert_eq!(header(":message-type"), Some("error")); + assert_eq!(header(":error-code"), Some("TruncatedInput")); + } else { + panic!("unexpected event-stream error: {error:?}"); + } + return Ok(()); + } + Ok(Some(SelectObjectContentEventStream::Stats(_))) | Ok(Some(SelectObjectContentEventStream::End(_))) => { + return Err("truncated compressed input reached a success terminal event".into()); + } + Ok(Some(_)) => {} + Ok(None) => return Err("truncated compressed input ended without an error event".into()), + } + } + }) + .await + .map_err(|_| -> Box { "truncated Select response timed out".into() })? +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_select_object_content_compressed_csv_and_json() -> TestResult<()> { + const CSV: &[u8] = b"name,age\nAlice,30\nBob,25\n"; + const JSON_LINES: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n"; + const JSON_DOCUMENT: &[u8] = br#"[{"name":"Alice"},{"name":"Bob"}]"#; + + let (_env, client) = create_test_environment(&[]).await?; + + let gzip_csv = gzip(CSV)?; + put_object(&client, "records.csv.gz", &gzip_csv).await?; + let gzip_csv_records = collect_success( + csv_select_request(&client, "records.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object") + .send() + .await?, + gzip_csv.len(), + CSV.len(), + ) + .await?; + assert_eq!(gzip_csv_records, b"Alice,30\nBob,25\n"); + + let bzip_csv = bzip2(CSV).await?; + put_object(&client, "records.csv.bz2", &bzip_csv).await?; + let bzip_csv_records = collect_success( + csv_select_request(&client, "records.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object") + .send() + .await?, + bzip_csv.len(), + CSV.len(), + ) + .await?; + assert_eq!(bzip_csv_records, gzip_csv_records); + + let gzip_json_lines = gzip(JSON_LINES)?; + put_object(&client, "json-lines", &gzip_json_lines).await?; + let gzip_json_records = collect_success( + json_select_request(&client, "json-lines", CompressionType::Gzip, JsonType::Lines) + .send() + .await?, + gzip_json_lines.len(), + JSON_LINES.len(), + ) + .await?; + assert_eq!(gzip_json_records, JSON_LINES); + + let bzip_json_lines = bzip2(JSON_LINES).await?; + put_object(&client, "records.jsonl.bz2", &bzip_json_lines).await?; + let bzip_json_records = collect_success( + json_select_request(&client, "records.jsonl.bz2", CompressionType::Bzip2, JsonType::Lines) + .send() + .await?, + bzip_json_lines.len(), + JSON_LINES.len(), + ) + .await?; + assert_eq!(bzip_json_records, gzip_json_records); + + let gzip_json_document = gzip(JSON_DOCUMENT)?; + put_object(&client, "document.json.gz", &gzip_json_document).await?; + let document_records = collect_success( + json_select_request(&client, "document.json.gz", CompressionType::Gzip, JsonType::Document) + .send() + .await?, + gzip_json_document.len(), + JSON_DOCUMENT.len(), + ) + .await?; + assert_eq!(document_records, JSON_LINES); + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_select_object_content_invalid_compressed_stream_fails() -> TestResult<()> { + const CSV: &[u8] = b"name\nAlice\n"; + + let (_env, client) = create_test_environment(&[]).await?; + + put_object(&client, "invalid.csv.gz", CSV).await?; + let invalid = csv_select_request(&client, "invalid.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object") + .send() + .await + .expect_err("invalid GZIP header must fail before streaming"); + assert_eq!( + invalid.as_service_error().and_then(ProvideErrorMetadata::code), + Some("InvalidCompressionFormat") + ); + + put_object(&client, "empty.csv.gz", b"").await?; + let empty = csv_select_request(&client, "empty.csv.gz", CompressionType::Gzip, "SELECT * FROM S3Object") + .send() + .await + .expect_err("empty GZIP input must fail as truncated"); + assert_eq!(empty.as_service_error().and_then(ProvideErrorMetadata::code), Some("TruncatedInput")); + + let mut truncated = bzip2(CSV).await?; + truncated.pop(); + put_object(&client, "truncated.csv.bz2", &truncated).await?; + let truncated = csv_select_request(&client, "truncated.csv.bz2", CompressionType::Bzip2, "SELECT * FROM S3Object") + .send() + .await?; + assert_truncated_stream_failure(truncated).await?; + + Ok(()) +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_select_object_content_compressed_disconnect_releases_query() -> TestResult<()> { + const OBJECT: &str = "disconnect.csv.gz"; + const ROWS: usize = 16 * 1024; + const RELEASE_ATTEMPTS: usize = 20; + const RELEASE_BACKOFF: Duration = Duration::from_millis(25); + + let (_env, client) = create_test_environment(&[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")]).await?; + let row = format!("{}\n", "x".repeat(1023)); + let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len()); + body.extend_from_slice(b"value\n"); + for _ in 0..ROWS { + body.extend_from_slice(row.as_bytes()); + } + let compressed = gzip(&body)?; + put_object(&client, OBJECT, &compressed).await?; + + let first = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object") + .send() + .await?; + let saturated = csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object") + .send() + .await + .expect_err("the unread compressed response should retain the only query permit"); + assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown")); + + drop(first); + let second = tokio::time::timeout(Duration::from_secs(5), async { + for attempt in 0..RELEASE_ATTEMPTS { + match csv_select_request(&client, OBJECT, CompressionType::Gzip, "SELECT * FROM S3Object") + .send() + .await + { + Ok(response) => return Ok::<_, Box>(response), + Err(error) + if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") + && attempt + 1 < RELEASE_ATTEMPTS => + { + tokio::time::sleep(RELEASE_BACKOFF).await; + } + Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => { + return Err("disconnected compressed Select retained its query permit".into()); + } + Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()), + } + } + Err("query permit release retry loop ended unexpectedly".into()) + }) + .await + .map_err(|_| -> Box { "compressed Select did not release its query permit".into() })??; + drop(second); + + Ok(()) +} diff --git a/crates/e2e_test/src/reliant/sql.rs b/crates/e2e_test/src/reliant/sql.rs index ab2d385ab..d779363e7 100644 --- a/crates/e2e_test/src/reliant/sql.rs +++ b/crates/e2e_test/src/reliant/sql.rs @@ -122,6 +122,24 @@ async fn select_json_document(client: &Client, key: &str, expression: &str) -> T process_select_response(response).await } +fn csv_select_request( + client: &Client, + key: &str, +) -> aws_sdk_s3::operation::select_object_content::builders::SelectObjectContentFluentBuilder { + client + .select_object_content() + .bucket(BUCKET) + .key(key) + .expression("SELECT * FROM S3Object") + .expression_type(ExpressionType::Sql) + .input_serialization( + InputSerialization::builder() + .csv(CsvInput::builder().file_header_info(FileHeaderInfo::Use).build()) + .build(), + ) + .output_serialization(OutputSerialization::builder().csv(CsvOutput::builder().build()).build()) +} + async fn process_select_response( mut event_stream: aws_sdk_s3::operation::select_object_content::SelectObjectContentOutput, ) -> TestResult { @@ -188,30 +206,42 @@ async fn assert_input_byte_stats( let mut last_progress: Option = None; let mut stats = None; let mut saw_end = false; - while let Some(event) = payload.recv().await? { - match event { - aws_sdk_s3::types::SelectObjectContentEventStream::Records(records) => { - if let Some(bytes) = records.payload { - records_len = records_len.saturating_add(u64::try_from(bytes.as_ref().len())?); + tokio::time::timeout(SELECT_RESPONSE_TIMEOUT, async { + // The AWS SDK validates both event-stream CRCs before yielding an event. + while let Some(event) = payload.recv().await? { + assert!(!saw_end, "Select emitted an event after End"); + match event { + aws_sdk_s3::types::SelectObjectContentEventStream::Records(records) => { + assert!(stats.is_none(), "Select emitted Records after Stats"); + if let Some(bytes) = records.payload { + records_len = records_len.saturating_add(u64::try_from(bytes.as_ref().len())?); + } } - } - aws_sdk_s3::types::SelectObjectContentEventStream::Progress(event) => { - let details = event.details.ok_or("Progress event did not contain details")?; - if let Some(previous) = last_progress.as_ref() { - assert!(details.bytes_scanned() >= previous.bytes_scanned()); - assert!(details.bytes_processed() >= previous.bytes_processed()); - assert!(details.bytes_returned() >= previous.bytes_returned()); + aws_sdk_s3::types::SelectObjectContentEventStream::Progress(event) => { + assert!(stats.is_none(), "Select emitted Progress after Stats"); + let details = event.details.ok_or("Progress event did not contain details")?; + if let Some(previous) = last_progress.as_ref() { + assert!(details.bytes_scanned() >= previous.bytes_scanned()); + assert!(details.bytes_processed() >= previous.bytes_processed()); + assert!(details.bytes_returned() >= previous.bytes_returned()); + } + last_progress = Some(details); } - last_progress = Some(details); + aws_sdk_s3::types::SelectObjectContentEventStream::Stats(event) => { + assert!(stats.is_none(), "Select emitted more than one Stats event"); + stats = event.details; + } + aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => { + assert!(stats.is_some(), "Select emitted End before Stats"); + saw_end = true; + } + _ => assert!(stats.is_none(), "Select emitted a non-terminal event after Stats"), } - aws_sdk_s3::types::SelectObjectContentEventStream::Stats(event) => stats = event.details, - aws_sdk_s3::types::SelectObjectContentEventStream::End(_) => { - saw_end = true; - break; - } - _ => {} } - } + Ok::<(), Box>(()) + }) + .await + .map_err(|_| -> Box { "Select response timed out".into() })??; let stats = stats.ok_or("Select response ended without a Stats event")?; let input_len = i64::try_from(body.len())?; @@ -219,10 +249,11 @@ async fn assert_input_byte_stats( assert_eq!(stats.bytes_processed(), Some(input_len)); assert_eq!(stats.bytes_returned(), Some(i64::try_from(records_len)?)); if progress_enabled { - let progress = last_progress.ok_or("Select response ended without a Progress event")?; - assert_eq!(progress.bytes_scanned(), stats.bytes_scanned()); - assert_eq!(progress.bytes_processed(), stats.bytes_processed()); - assert_eq!(progress.bytes_returned(), stats.bytes_returned()); + if let Some(progress) = last_progress { + assert!(stats.bytes_scanned() >= progress.bytes_scanned()); + assert!(stats.bytes_processed() >= progress.bytes_processed()); + assert!(stats.bytes_returned() >= progress.bytes_returned()); + } } else { assert!(last_progress.is_none(), "disabled request progress emitted a Progress event"); } @@ -231,7 +262,7 @@ async fn assert_input_byte_stats( } #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn test_select_object_content_reports_input_byte_stats() -> TestResult<()> { +async fn test_select_object_content_http_event_order_crc_and_input_byte_stats() -> TestResult<()> { const CSV_BODY: &[u8] = b"name,age\nAlice,30\nBob,25\n"; const JSON_LINES_BODY: &[u8] = b"{\"name\":\"Alice\"}\n{\"name\":\"Bob\"}\n"; const JSON_DOCUMENT_BODY: &[u8] = b"[{\"name\":\"Alice\"},{\"name\":\"Bob\"}]"; @@ -289,6 +320,60 @@ async fn test_select_object_content_reports_input_byte_stats() -> TestResult<()> Ok(()) } +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn test_select_object_content_http_disconnect_releases_query() -> TestResult<()> { + const OBJECT: &str = "disconnect.csv"; + const ROWS: usize = 16 * 1024; + const RELEASE_BACKOFF: Duration = Duration::from_millis(25); + + init_logging(); + let mut env = RustFSTestEnvironment::new().await?; + env.start_rustfs_server_with_env(vec![], &[("RUSTFS_S3SELECT_MAX_CONCURRENT_QUERIES", "1")]) + .await?; + let client = env.create_s3_client(); + setup_test_bucket(&client).await?; + + let row = format!("{}\n", "x".repeat(1023)); + let mut body = Vec::with_capacity("value\n".len() + ROWS * row.len()); + body.extend_from_slice(b"value\n"); + for _ in 0..ROWS { + body.extend_from_slice(row.as_bytes()); + } + client + .put_object() + .bucket(BUCKET) + .key(OBJECT) + .body(Bytes::from(body).into()) + .send() + .await?; + + // Leaving this response body unread fills the bounded HTTP/event channels before the query can finish. + let first = csv_select_request(&client, OBJECT).send().await?; + let saturated = csv_select_request(&client, OBJECT) + .send() + .await + .expect_err("the first HTTP stream should retain the only query permit"); + assert_eq!(saturated.as_service_error().and_then(ProvideErrorMetadata::code), Some("SlowDown")); + + drop(first); + let second = tokio::time::timeout(Duration::from_secs(5), async { + loop { + match csv_select_request(&client, OBJECT).send().await { + Ok(response) => return Ok::<_, Box>(response), + Err(error) if error.as_service_error().and_then(ProvideErrorMetadata::code) == Some("SlowDown") => { + tokio::time::sleep(RELEASE_BACKOFF).await; + } + Err(error) => return Err(format!("unexpected Select error after disconnect: {error}").into()), + } + } + }) + .await + .map_err(|_| -> Box { "disconnected Select did not release its query permit".into() })??; + drop(second); + + Ok(()) +} + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn test_select_object_content_csv_basic() -> TestResult<()> { let (_env, client) = create_test_environment().await?; diff --git a/crates/ecstore/src/api/mod.rs b/crates/ecstore/src/api/mod.rs index 791c0c56d..6ea3f15d1 100644 --- a/crates/ecstore/src/api/mod.rs +++ b/crates/ecstore/src/api/mod.rs @@ -461,8 +461,8 @@ pub mod rpc { tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, - verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, - verify_tonic_rpc_signature_with_bootstrap, + verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof, + verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap, }; } @@ -489,9 +489,9 @@ pub mod storage { pub use crate::core::pools::HealLifecycleExpiryContext; pub use crate::store::HealWalkVersion; pub use crate::store::{ - ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, init_local_disks, - init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map, - prewarm_local_disk_id_map_with_instance_ctx, + ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path, + find_local_disk_by_ref, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, + prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx, }; } diff --git a/crates/ecstore/src/bucket/bucket_target_sys.rs b/crates/ecstore/src/bucket/bucket_target_sys.rs index 2966647c2..e30e3dd0a 100644 --- a/crates/ecstore/src/bucket/bucket_target_sys.rs +++ b/crates/ecstore/src/bucket/bucket_target_sys.rs @@ -24,6 +24,7 @@ use crate::runtime::sources as runtime_sources; use aws_credential_types::Credentials as SdkCredentials; use aws_credential_types::provider::{ProvideCredentials, error::CredentialsError, future}; use aws_sdk_s3::config::Region as SdkRegion; +use aws_sdk_s3::config::RequestChecksumCalculation; use aws_sdk_s3::config::SharedHttpClient; use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::error::SdkError; @@ -39,6 +40,7 @@ use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::Tagging as SdkTagging; use aws_sdk_s3::types::{ ChecksumMode, CompletedMultipartUpload, CompletedPart, ObjectLockLegalHoldStatus, ObjectLockRetentionMode, + ServerSideEncryption, }; use aws_sdk_s3::{Client as S3Client, Config as S3Config, operation::head_object::HeadObjectOutput}; use aws_sdk_s3::{config::SharedCredentialsProvider, types::BucketVersioningStatus}; @@ -1071,7 +1073,8 @@ impl BucketTargetSys { .endpoint_url(endpoint.clone()) .credentials_provider(SharedCredentialsProvider::new(RemoteTargetCredentialsProvider { credentials: creds })) .region(SdkRegion::new(target.region.clone())) - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()); + .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) + .request_checksum_calculation(replication_request_checksum_calculation()); if should_force_path_style(target) { config_builder = config_builder.force_path_style(true); @@ -1367,6 +1370,25 @@ fn loopback_replication_targets_allowed() -> bool { .unwrap_or(false) } +const REPLICATION_STREAMING_CHECKSUMS_ENV: &str = "RUSTFS_REPLICATION_STREAMING_CHECKSUMS"; + +/// Streaming trailer checksums make the SDK frame request bodies as +/// `aws-chunked`; a target that does not decode that framing stores the frames +/// verbatim, silently corrupting every replica while the transfer itself +/// succeeds (#6853). Plain signed payloads are the compatible default; the env +/// knob restores trailer checksums for fleets whose targets are all known to +/// decode them. +fn replication_request_checksum_calculation() -> RequestChecksumCalculation { + if std::env::var(REPLICATION_STREAMING_CHECKSUMS_ENV) + .map(|v| v.eq_ignore_ascii_case("true") || v == "1") + .unwrap_or(false) + { + RequestChecksumCalculation::WhenSupported + } else { + RequestChecksumCalculation::WhenRequired + } +} + fn validate_replication_target_endpoint(url: &Url) -> Result<(), OutboundUrlError> { validate_replication_target_endpoint_inner(url, loopback_replication_targets_allowed()) } @@ -1746,6 +1768,17 @@ impl Default for AdvancedPutOptions { } } +/// The subset of the target's PutObject response replication audits. +#[derive(Debug, Clone)] +pub struct RemotePutObjectResponse { + /// Version id the target assigned (`x-amz-version-id`). + pub version_id: Option, + /// ETag of what the target stored; `None` when the target withheld it or + /// when its encryption mode (SSE-KMS / SSE-C) makes it incomparable to + /// the source ETag. `None` is therefore "not decidable", never evidence. + pub etag: Option, +} + #[derive(Clone)] pub struct PutObjectOptions { pub user_metadata: HashMap, @@ -2291,7 +2324,9 @@ impl TargetClient { /// On success returns the version id the target assigned (from /// `x-amz-version-id`), letting callers audit the version-identity - /// contract — a target that adopts the source version echoes it back. + /// contract — a target that adopts the source version echoes it back — + /// together with the ETag of what the target actually stored, so callers + /// can detect a target that persisted transformed bytes (#6853). pub async fn put_object( &self, bucket: &str, @@ -2299,7 +2334,7 @@ impl TargetClient { size: i64, body: ByteStream, opts: &PutObjectOptions, - ) -> Result, S3ClientError> { + ) -> Result { let mut headers = opts.header(); let builder = self.client.put_object(); @@ -2334,7 +2369,25 @@ impl TargetClient { .send() .await { - Ok(output) => Ok(output.version_id().map(ToOwned::to_owned)), + Ok(output) => { + // Under SSE-KMS/DSSE or SSE-C the target's ETag is not the MD5 + // of the stored plaintext, so it cannot be compared against the + // source ETag; withhold it rather than let a caller conclude + // corruption from an opaque value. + let etag_comparable = output.sse_customer_algorithm().is_none() + && !matches!( + output.server_side_encryption(), + Some(ServerSideEncryption::AwsKms) | Some(ServerSideEncryption::AwsKmsDsse) + ); + Ok(RemotePutObjectResponse { + version_id: output.version_id().map(ToOwned::to_owned), + etag: if etag_comparable { + output.e_tag().map(ToOwned::to_owned) + } else { + None + }, + }) + } Err(e) => match e { SdkError::ServiceError(service_err) => { let err = service_err.into_err(); @@ -2673,6 +2726,145 @@ mod tests { } } + type RecordedHeaders = Arc>>>; + + /// Records full request headers and answers with canned response headers, + /// for asserting wire framing and response parsing. + #[derive(Clone, Debug)] + struct RecordingHeaderConnector { + request_headers: RecordedHeaders, + response_headers: Vec<(String, String)>, + } + + impl SmithyHttpConnector for RecordingHeaderConnector { + fn call(&self, request: HttpRequest) -> HttpConnectorFuture { + self.request_headers + .lock() + .expect("recorded header lock should not be poisoned") + .push( + request + .headers() + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + ); + let mut response = HttpResponse::new( + aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"), + SdkBody::empty(), + ); + for (name, value) in &self.response_headers { + response.headers_mut().insert(name.clone(), value.clone()); + } + HttpConnectorFuture::ready(Ok(response)) + } + } + + fn header_recording_target_client(response_headers: Vec<(String, String)>) -> (TargetClient, RecordedHeaders) { + let request_headers: RecordedHeaders = Arc::new(std::sync::Mutex::new(Vec::new())); + let connector = SharedHttpConnector::new(RecordingHeaderConnector { + request_headers: Arc::clone(&request_headers), + response_headers, + }); + let http_client = http_client_fn(move |_settings, _components| connector.clone()); + let client = s3_client_for_test(443, Some(http_client)); + ( + TargetClient { + endpoint: "https://localhost:443".to_string(), + credentials: None, + bucket: "target-bucket".to_string(), + storage_class: String::new(), + disable_proxy: false, + arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(), + reset_id: String::new(), + secure: true, + health_check_duration: Duration::from_secs(5), + replicate_sync: false, + client: Arc::new(client), + }, + request_headers, + ) + } + + fn streaming_test_body(payload: &'static [u8]) -> ByteStream { + let stream = tokio_util::io::ReaderStream::new(std::io::Cursor::new(payload)); + let body = http_body_util::StreamBody::new(futures::StreamExt::map(stream, |r| r.map(http_body::Frame::data))); + ByteStream::new(SdkBody::from_body_1_x(body)) + } + + #[test] + fn replication_checksums_default_to_plain_payloads() { + assert!(matches!( + replication_request_checksum_calculation(), + RequestChecksumCalculation::WhenRequired + )); + } + + #[tokio::test] + async fn replication_put_object_sends_plain_signed_payloads_by_default() { + let (client, recorded) = header_recording_target_client(Vec::new()); + client + .put_object("target-bucket", "object", 4, streaming_test_body(b"data"), &PutObjectOptions::default()) + .await + .expect("recorded put_object should succeed"); + + let recorded = recorded.lock().expect("recorded header lock should not be poisoned"); + let headers = &recorded[0]; + let header = |name: &str| { + headers + .iter() + .find(|(k, _)| k.eq_ignore_ascii_case(name)) + .map(|(_, v)| v.as_str()) + }; + // The #6853 regression shape: trailer checksums force aws-chunked + // framing, which a non-decoding target stores verbatim as the object. + assert_eq!(header("x-amz-trailer"), None, "streaming uploads must not carry a trailer checksum"); + assert!( + header("content-encoding").is_none_or(|v| !v.contains("aws-chunked")), + "streaming uploads must not be aws-chunked framed" + ); + assert_eq!(header("x-amz-decoded-content-length"), None); + assert_eq!(header("content-length"), Some("4")); + } + + #[tokio::test] + async fn put_object_returns_the_etag_the_target_stored() { + let (client, _) = + header_recording_target_client(vec![("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string())]); + let response = client + .put_object( + "target-bucket", + "object", + 4, + ByteStream::from_static(b"data"), + &PutObjectOptions::default(), + ) + .await + .expect("recorded put_object should succeed"); + assert_eq!(response.etag.as_deref(), Some("\"9a0364b9e99bb480dd25e1f0284c8555\"")); + } + + #[tokio::test] + async fn put_object_withholds_the_etag_under_target_side_kms() { + let (client, _) = header_recording_target_client(vec![ + ("etag".to_string(), "\"9a0364b9e99bb480dd25e1f0284c8555\"".to_string()), + ("x-amz-server-side-encryption".to_string(), "aws:kms".to_string()), + ]); + let response = client + .put_object( + "target-bucket", + "object", + 4, + ByteStream::from_static(b"data"), + &PutObjectOptions::default(), + ) + .await + .expect("recorded put_object should succeed"); + assert!( + response.etag.is_none(), + "a KMS-encrypted replica's etag is not the content MD5 and must be withheld" + ); + } + #[derive(Clone, Debug)] struct RecordingAuthConnector { signed_requests: Arc>>, @@ -2969,7 +3161,10 @@ mod tests { .credentials_provider(SharedCredentialsProvider::new(credentials)) .region(SdkRegion::new("us-east-1")) .force_path_style(true) - .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()); + .behavior_version(aws_sdk_s3::config::BehaviorVersion::latest()) + // Mirror the production remote-target builder so recorded requests + // exercise the same checksum/framing behavior (#6853). + .request_checksum_calculation(replication_request_checksum_calculation()); if let Some(http_client) = http_client { config = config.http_client(http_client); } diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index aa67a9bd6..c1e51a14c 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -12195,7 +12195,7 @@ mod tests { #[tokio::test] #[serial] async fn tier_free_version_recovery_continues_after_deleted_marker_bucket() { - let (_paths, ecstore) = setup_test_env().await; + let (disk_paths, ecstore) = setup_test_env().await; let suffix = Uuid::new_v4().simple(); let earlier_bucket = format!("zzzz-recovery-{suffix}-a"); let deleted_marker = format!("zzzz-recovery-{suffix}-m"); @@ -12203,11 +12203,7 @@ mod tests { let later_object = "a-before-stale-marker"; create_test_bucket(&ecstore, &earlier_bucket).await; create_test_bucket(&ecstore, &later_bucket).await; - let mut reader = PutObjReader::from_vec(b"cursor reset probe".to_vec()); - ecstore - .put_object(&later_bucket, later_object, &mut reader, &ObjectOptions::default()) - .await - .expect("successor bucket object should be created"); + seed_recoverable_free_version(&disk_paths, &later_bucket, later_object, None, None).await; let page = list_tier_free_versions( Arc::clone(&ecstore), @@ -12220,14 +12216,10 @@ mod tests { .expect("recovery should resume at the first bucket after a deleted marker bucket"); assert_eq!(page.buckets_scanned, 1, "the later bucket must not be skipped"); - assert_eq!( - page.scanned_entries, 1, - "the deleted bucket's object marker must not skip objects in the successor bucket" - ); - ecstore - .delete_object(&later_bucket, later_object, ObjectOptions::default()) - .await - .expect("successor bucket object should be removed"); + assert_eq!(page.items.len(), 1, "the successor bucket's recoverable object must be returned"); + assert_eq!(page.items[0].bucket, later_bucket); + assert_eq!(page.items[0].name, later_object); + remove_seeded_free_version(&disk_paths, &later_bucket, later_object).await; for bucket in [&earlier_bucket, &later_bucket] { ecstore .delete_bucket(bucket, &DeleteBucketOptions::default()) diff --git a/crates/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs index 3e0e6ded6..52edc4350 100644 --- a/crates/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -2512,11 +2512,169 @@ pub(crate) mod test_support { mod tests { use super::test_support::isolated_store_over_temp_disks; use super::*; + use crate::bucket::metadata::{ + BUCKET_ACCELERATE_CONFIG, BUCKET_CORS_CONFIG, BUCKET_LIFECYCLE_CONFIG, BUCKET_LOGGING_CONFIG, BUCKET_NOTIFICATION_CONFIG, + BUCKET_POLICY_CONFIG, BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, BUCKET_REPLICATION_CONFIG, BUCKET_REQUEST_PAYMENT_CONFIG, + BUCKET_SSECONFIG, BUCKET_TAGGING_CONFIG, BUCKET_VERSIONING_CONFIG, BUCKET_WEBSITE_CONFIG, OBJECT_LOCK_CONFIG, + }; use crate::bucket::target::{BucketTarget, BucketTargetType, Credentials}; + use crate::config::com::read_config; use crate::storage_api_contracts::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions}; + use byteorder::{ByteOrder as _, LittleEndian}; use serial_test::serial; use tokio::time::timeout; + const NEW_WRITER_REPLICATION_XML: &[u8] = br#"arn:aws:iam::111122223333:role/replication-rolerollback1documents/Enabledarn:aws:s3:::replica-bucketDisabled"#; + + const NEW_WRITER_CONFIGS: [(&str, &[u8]); 14] = [ + (BUCKET_POLICY_CONFIG, br#"{"Version":"2012-10-17","Statement":[]}"#), + (BUCKET_NOTIFICATION_CONFIG, br#""#), + ( + BUCKET_LIFECYCLE_CONFIG, + br#"expireEnabledlogs/30"#, + ), + ( + OBJECT_LOCK_CONFIG, + br#"EnabledGOVERNANCE7"#, + ), + ( + BUCKET_VERSIONING_CONFIG, + br#"Enabled"#, + ), + ( + BUCKET_SSECONFIG, + br#"AES256"#, + ), + ( + BUCKET_TAGGING_CONFIG, + r#"environment测试-🦀"#.as_bytes(), + ), + (BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML), + ( + BUCKET_CORS_CONFIG, + br#"GEThttps://example.test"#, + ), + (BUCKET_LOGGING_CONFIG, br#""#), + ( + BUCKET_WEBSITE_CONFIG, + br#"index.html"#, + ), + ( + BUCKET_ACCELERATE_CONFIG, + br#"Enabled"#, + ), + ( + BUCKET_REQUEST_PAYMENT_CONFIG, + br#"Requester"#, + ), + ( + BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, + br#"truetruetruefalse"#, + ), + ]; + + #[tokio::test] + async fn g_d3_003_new_writer_replication_loads_without_fail_closed_state() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "rollback-new-replication"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created"); + } + + let writer = BucketMetadataSys::new(store.clone()); + let mut metadata = BucketMetadata::new(bucket); + metadata + .update_config(BUCKET_REPLICATION_CONFIG, NEW_WRITER_REPLICATION_XML.to_vec()) + .expect("new-writer replication XML should be accepted before persistence"); + writer + .persist_new_and_set(metadata) + .await + .expect("new-writer replication metadata should persist"); + + let old_reader = BucketMetadataSys::new(store); + let (loaded, _) = old_reader + .get_replication_config(bucket) + .await + .expect("old metadata_sys must not classify new-writer replication XML as invalid"); + assert_eq!(loaded.role, "arn:aws:iam::111122223333:role/replication-role"); + assert_eq!(loaded.rules.len(), 1); + assert_eq!(loaded.rules[0].id.as_deref(), Some("rollback")); + } + + #[tokio::test] + async fn g_d3_004_new_writer_metadata_blob_keeps_legacy_header_and_configs() { + let (dirs, store) = isolated_store_over_temp_disks().await; + let bucket = "rollback-new-metadata"; + for dir in &dirs { + std::fs::create_dir_all(dir.path().join(bucket)).expect("rollback fixture bucket should be created"); + } + + let writer = BucketMetadataSys::new(store.clone()); + let mut metadata = BucketMetadata::new(bucket); + for (config_file, bytes) in NEW_WRITER_CONFIGS { + metadata + .update_config(config_file, bytes.to_vec()) + .unwrap_or_else(|err| panic!("new-writer {config_file} fixture must be valid: {err}")); + } + writer + .persist_new_and_set(metadata) + .await + .expect("new-writer metadata should persist"); + + let path = BucketMetadata::new(bucket).save_file_path(); + let blob = read_config(store.clone(), &path) + .await + .expect("persisted .metadata.bin should be readable"); + assert_eq!( + LittleEndian::read_u16(&blob[0..2]), + 1, + "bucket metadata format must stay rollback-readable" + ); + assert_eq!( + LittleEndian::read_u16(&blob[2..4]), + 1, + "bucket metadata version must stay rollback-readable" + ); + + let loaded = load_bucket_metadata(store, bucket) + .await + .expect("old read_bucket_metadata path must load the new-writer blob"); + let loaded_configs: [(&str, &[u8]); 14] = [ + (BUCKET_POLICY_CONFIG, &loaded.policy_config_json), + (BUCKET_NOTIFICATION_CONFIG, &loaded.notification_config_xml), + (BUCKET_LIFECYCLE_CONFIG, &loaded.lifecycle_config_xml), + (OBJECT_LOCK_CONFIG, &loaded.object_lock_config_xml), + (BUCKET_VERSIONING_CONFIG, &loaded.versioning_config_xml), + (BUCKET_SSECONFIG, &loaded.encryption_config_xml), + (BUCKET_TAGGING_CONFIG, &loaded.tagging_config_xml), + (BUCKET_REPLICATION_CONFIG, &loaded.replication_config_xml), + (BUCKET_CORS_CONFIG, &loaded.cors_config_xml), + (BUCKET_LOGGING_CONFIG, &loaded.logging_config_xml), + (BUCKET_WEBSITE_CONFIG, &loaded.website_config_xml), + (BUCKET_ACCELERATE_CONFIG, &loaded.accelerate_config_xml), + (BUCKET_REQUEST_PAYMENT_CONFIG, &loaded.request_payment_config_xml), + (BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG, &loaded.public_access_block_config_xml), + ]; + for ((expected_name, expected), (loaded_name, actual)) in NEW_WRITER_CONFIGS.into_iter().zip(loaded_configs) { + assert_eq!(loaded_name, expected_name); + assert_eq!(actual, expected, "old read_bucket_metadata changed {expected_name} bytes"); + } + assert!(loaded.policy_config.is_some()); + assert!(loaded.notification_config.is_some()); + assert!(loaded.lifecycle_config.is_some()); + assert!(loaded.object_lock_config.is_some()); + assert!(loaded.versioning_config.is_some()); + assert!(loaded.sse_config.is_some()); + assert!(loaded.tagging_config.is_some()); + assert!(loaded.replication_config.is_some()); + assert!(loaded.cors_config.is_some()); + assert!(loaded.logging_config.is_some()); + assert!(loaded.website_config.is_some()); + assert!(loaded.accelerate_config.is_some()); + assert!(loaded.request_payment_config.is_some()); + assert!(loaded.public_access_block_config.is_some()); + } + #[tokio::test] async fn malformed_delete_configs_are_not_treated_as_absent() { let (_dirs, ecstore) = isolated_store_over_temp_disks().await; diff --git a/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs b/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs index c84099a30..da6b7e121 100644 --- a/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_object_decision_boundary.rs @@ -20,8 +20,8 @@ pub use rustfs_replication::{ pub(crate) use rustfs_replication::{ ReplicationDeleteSource, ReplicationMultipartPartInput, ReplicationResyncTargetObject, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, delete_replication_creates_marker, delete_replication_missing_source_decision, - delete_replication_object_opts, heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, - is_version_delete_replication, replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size, - replication_multipart_part_plan, resync_existing_delete_replication_info, resync_target_for_object, - should_retry_delete_marker_purge, target_delete_version_id, + delete_replication_object_opts, heal_uses_delete_replication_path, is_object_lock_denied_delete, + is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match, + replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info, + resync_target_for_object, should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id, }; diff --git a/crates/ecstore/src/bucket/replication/replication_pool.rs b/crates/ecstore/src/bucket/replication/replication_pool.rs index a6d5cf857..9b227c7c4 100644 --- a/crates/ecstore/src/bucket/replication/replication_pool.rs +++ b/crates/ecstore/src/bucket/replication/replication_pool.rs @@ -3177,6 +3177,19 @@ pub(crate) async fn queue_replication_heal_internal( } } ReplicationHealQueueAction::QueueDelete(dv) => { + // A purge the peer denied under object lock cannot succeed until + // the lock lapses (#6850); requeuing it every heal cycle only + // burns bandwidth and failure counters. The backoff expires on + // its own, so the purge is probed again — and converges — once + // the retention window has a chance of being over. + if super::replication_object_decision_boundary::is_version_delete_replication(&dv.delete_object) + && super::replication_resyncer::object_lock_denied_purge_backoff_active(&dv) + { + return ReplicationHealQueueResult { + object_info: roi, + admission: ReplicationQueueAdmission::Skipped, + }; + } let admission = if let Some(pool) = runtime_sources::replication_pool() { pool.queue_replica_delete_task(dv).await } else { diff --git a/crates/ecstore/src/bucket/replication/replication_resyncer.rs b/crates/ecstore/src/bucket/replication/replication_resyncer.rs index e62a6b04f..3aa28ddfa 100644 --- a/crates/ecstore/src/bucket/replication/replication_resyncer.rs +++ b/crates/ecstore/src/bucket/replication/replication_resyncer.rs @@ -30,10 +30,10 @@ use super::replication_msgp_boundary::ReplicationMsgpCodec; use super::replication_object_config::{ReplicationConfig, get_replication_config, must_replicate}; use super::replication_object_decision_boundary::{ MustReplicateOptions, ReplicationMultipartPartInput, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, - delete_replication_creates_marker, heal_uses_delete_replication_path, is_retryable_delete_replication_head_error, - is_version_delete_replication, replicate_delete_outcome, replication_etags_match, replication_multipart_complete_actual_size, - replication_multipart_part_plan, resync_existing_delete_replication_info, should_retry_delete_marker_purge, - target_delete_version_id, + delete_replication_creates_marker, heal_uses_delete_replication_path, is_object_lock_denied_delete, + is_retryable_delete_replication_head_error, is_version_delete_replication, replicate_delete_outcome, replication_etags_match, + replication_multipart_complete_actual_size, replication_multipart_part_plan, resync_existing_delete_replication_info, + should_retry_delete_marker_purge, single_part_replica_etag_mismatch, target_delete_version_id, }; use super::replication_queue_boundary::{DeletedObjectReplicationInfo, ReplicationQueueAdmission}; use super::replication_resync_boundary::ResyncStatusType; @@ -54,7 +54,7 @@ use super::replication_storage_boundary::{ }; use super::replication_target_boundary::{ ERR_REPLICATION_SSEC_PASSTHROUGH_UNSUPPORTED, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, - ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, + RemotePutObjectResponse, ReplicationTargetStore, S3ClientError, SsecPassthroughCapability, SsecPassthroughGate, TargetClient, is_replication_target_offline_error, replication_action_for_target_head, replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options, replication_force_delete_remove_options, replication_object_is_ssec_encrypted, replication_put_object_header_size, replication_put_object_options, @@ -96,7 +96,7 @@ use tokio::task::{JoinHandle, JoinSet}; use tokio::time::Duration as TokioDuration; use tokio_util::io::ReaderStream; use tokio_util::sync::CancellationToken; -use tracing::{debug, error, instrument, trace, warn}; +use tracing::{debug, error, info, instrument, trace, warn}; const BACKGROUND_WALKDIR_TIMEOUT: TokioDuration = TokioDuration::from_secs(60); const ENV_REPL_RESYNC_MAX_JOBS: &str = "RUSTFS_REPL_RESYNC_MAX_JOBS"; @@ -112,11 +112,13 @@ const EVENT_REPLICATION_DELETE_SKIPPED: &str = "replication_delete_skipped"; const EVENT_REPLICATION_FORCE_DELETE_SKIPPED: &str = "replication_force_delete_skipped"; const EVENT_RESYNC_TASK_FAILED: &str = "replication_resync_task_failed"; const EVENT_RESYNC_TARGET_OPERATION_FAILED: &str = "replication_resync_target_operation_failed"; +const EVENT_REPLICATION_ABORT_RETRY_RESOLVED: &str = "replication_abort_retry_resolved"; const EVENT_RESYNC_RUNTIME_CHANNEL_FAILED: &str = "replication_resync_runtime_channel_failed"; const EVENT_DELETE_MARKER_PURGE_FAILED: &str = "replication_delete_marker_purge_failed"; const EVENT_DELETE_MARKER_PURGE_MRF: &str = "replication_delete_marker_purge_mrf"; const METRIC_DELETE_MARKER_PURGE_TOTAL: &str = "rustfs_replication_delete_marker_purge_total"; const EVENT_REPLICATION_VERSION_IDENTITY_DRIFT: &str = "replication_version_identity_drift"; +const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied"; #[allow( dead_code, @@ -194,6 +196,123 @@ const METRIC_VERSION_IDENTITY_DRIFT_TOTAL: &str = "rustfs_replication_version_id /// after a restart is acceptable. static VERSION_IDENTITY_WARNED_ARNS: LazyLock>> = LazyLock::new(|| StdMutex::new(HashSet::new())); +/// Version purges the peer denied under object lock (#6850). Replication +/// carries no governance bypass, so such a purge cannot succeed until the +/// lock on the replica lapses — retrying every heal cycle only burns +/// bandwidth and failure counters. Entries suppress heal requeues for the +/// backoff window; after it expires one probe runs again, so the purge still +/// converges on its own once retention ends. In-process only: a restart +/// costs at most one extra probe per entry. +const OBJECT_LOCK_DENIED_PURGE_BACKOFF: std::time::Duration = std::time::Duration::from_secs(60 * 60); +const OBJECT_LOCK_DENIED_PURGE_CACHE_MAX: usize = 4096; +type ObjectLockDeniedPurgeKey = (String, String, String); + +struct ObjectLockDeniedPurge { + denied_at: std::time::Instant, + denied_arns: HashSet, +} + +static OBJECT_LOCK_DENIED_PURGES: LazyLock>> = + LazyLock::new(|| StdMutex::new(HashMap::new())); + +fn object_lock_denied_purge_key(dobj: &DeletedObjectReplicationInfo) -> ObjectLockDeniedPurgeKey { + let version_id = dobj + .delete_object + .delete_marker_version_id + .or(dobj.delete_object.version_id) + .unwrap_or_default(); + (dobj.bucket.clone(), dobj.delete_object.object_name.clone(), version_id.to_string()) +} + +fn record_object_lock_denied_purge(dobj: &DeletedObjectReplicationInfo, arn: &str) { + let mut denied = OBJECT_LOCK_DENIED_PURGES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if denied.len() >= OBJECT_LOCK_DENIED_PURGE_CACHE_MAX { + denied.retain(|_, entry| entry.denied_at.elapsed() < OBJECT_LOCK_DENIED_PURGE_BACKOFF); + } + let key = object_lock_denied_purge_key(dobj); + if denied.len() < OBJECT_LOCK_DENIED_PURGE_CACHE_MAX || denied.contains_key(&key) { + let entry = denied.entry(key).or_insert_with(|| ObjectLockDeniedPurge { + denied_at: std::time::Instant::now(), + denied_arns: HashSet::new(), + }); + entry.denied_at = std::time::Instant::now(); + entry.denied_arns.insert(arn.to_string()); + } + // Still full after dropping expired entries: skip recording — the purge + // then simply keeps retrying, which is the pre-#6850 behavior. +} + +/// Whether a heal requeue of this delete can only reach targets that denied +/// it under object lock within the backoff window. A target the entry does +/// not cover (another peer, or one whose denial expired) keeps the requeue +/// flowing — suppressing it would delay a purge that could succeed there. +pub(crate) fn object_lock_denied_purge_backoff_active(dobj: &DeletedObjectReplicationInfo) -> bool { + let key = object_lock_denied_purge_key(dobj); + let mut denied = OBJECT_LOCK_DENIED_PURGES + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + match denied.get(&key) { + Some(entry) if entry.denied_at.elapsed() < OBJECT_LOCK_DENIED_PURGE_BACKOFF => { + let admitted = dobj.admitted_target_arns(); + !admitted.is_empty() && admitted.iter().all(|arn| entry.denied_arns.contains(arn)) + } + Some(_) => { + denied.remove(&key); + false + } + None => false, + } +} + +const REPLICA_ETAG_VERIFY_ENV: &str = "RUSTFS_REPLICATION_REPLICA_ETAG_VERIFY"; + +/// Escape hatch for a target whose 32-hex ETags are legitimately not the +/// content MD5 (e.g. a gateway hashing its own ciphertext without announcing +/// SSE in the response) — such a target would otherwise fail every object. +fn replica_etag_verification_enabled() -> bool { + std::env::var(REPLICA_ETAG_VERIFY_ENV) + .map(|v| !(v.eq_ignore_ascii_case("false") || v == "0")) + .unwrap_or(true) +} + +/// A 200 from the target is not proof the replica holds the source bytes: a +/// target that stores a transformed payload (e.g. undecoded `aws-chunked` +/// frames, #6853) returns the ETag of what it actually wrote. Reporting +/// COMPLETED over such a replica is silent corruption, so a decidable +/// mismatch fails the replication instead. An SSE-C ciphertext passthrough +/// transfer is exempt: the wire bytes are ciphertext while the source ETag is +/// the plaintext MD5, and that path has its own HEAD-back audit. +fn verify_single_part_replica( + object_info: &ObjectInfo, + response: &RemotePutObjectResponse, + ciphertext_passthrough: bool, +) -> std::result::Result<(), std::io::Error> { + if ciphertext_passthrough || !replica_etag_verification_enabled() { + return Ok(()); + } + if single_part_replica_etag_mismatch(object_info.etag.as_deref(), response.etag.as_deref()) { + // The differing ETags go into the structured log; the error message + // stays constant so same-cause failures bucket together downstream. + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = %object_info.bucket, + object = %object_info.name, + source_etag = ?object_info.etag, + replica_etag = ?response.etag, + operation = "verify_replica_etag", + "Replication target operation failed" + ); + return Err(std::io::Error::other(REPLICA_ETAG_MISMATCH_ERROR)); + } + Ok(()) +} + +const REPLICA_ETAG_MISMATCH_ERROR: &str = "replica etag mismatch: the target persisted different bytes than were sent"; + fn audit_target_version_identity(tgt_client: &TargetClient, source_version_id: &str, assigned_version_id: Option<&str>) { if !version_identity_drifted(source_version_id, assigned_version_id) { return; @@ -2708,19 +2827,42 @@ async fn replicate_delete_to_target(dobj: &DeletedObjectReplicationInfo, tgt_cli } } Err(e) => { - warn!( - event = EVENT_RESYNC_TARGET_OPERATION_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, - bucket = tgt_client.bucket, - object = dobj.delete_object.object_name, - version_id = ?version_id, - delete_marker = dobj.delete_object.delete_marker, - is_version_purge, - error = %e, - operation = "replicate_delete_to_target", - "Replication target operation failed" - ); + let object_lock_denied = is_version_purge && is_object_lock_denied_delete(e.code.as_deref(), e.message.as_deref()); + if object_lock_denied { + // Terminal for as long as the lock holds: the peer retains + // this version and replication carries no governance bypass + // (#6850), so the sites stay diverged until the retention or + // legal hold on the replica lapses. Surface it loudly instead + // of letting a silent failed counter and a hot heal-retry + // loop stand in for the divergence. + record_object_lock_denied_purge(dobj, &tgt_client.arn); + error!( + event = EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = tgt_client.bucket, + object = dobj.delete_object.object_name, + version_id = ?version_id, + arn = %tgt_client.arn, + error = %e, + operation = "replicate_delete_to_target", + "Replicated version purge denied by object lock on the target; the sites stay diverged until the lock lapses" + ); + } else { + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + bucket = tgt_client.bucket, + object = dobj.delete_object.object_name, + version_id = ?version_id, + delete_marker = dobj.delete_object.delete_marker, + is_version_purge, + error = %e, + operation = "replicate_delete_to_target", + "Replication target operation failed" + ); + } rinfo.error = Some(e.to_string()); if !is_version_purge { rinfo.replication_status = ReplicationStatusType::Failed; @@ -3274,14 +3416,15 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo { let result = tgt_client .put_object(&tgt_client.bucket, &object, transfer_size, byte_stream, &put_opts) .await - .map(|assigned_version_id| { + .map_err(|e| std::io::Error::other(e.to_string())) + .and_then(|response| { audit_target_version_identity( &tgt_client, &put_opts.internal.source_version_id, - assigned_version_id.as_deref(), - ) - }) - .map_err(|e| std::io::Error::other(e.to_string())); + response.version_id.as_deref(), + ); + verify_single_part_replica(&object_info, &response, obj_opts.raw_data_movement_read) + }); result.err() } { rinfo.replication_status = ReplicationStatusType::Failed; @@ -3942,14 +4085,15 @@ async fn replicate_all_payload_to_target( .tgt_client .put_object(&ctx.tgt_client.bucket, ctx.object, ctx.transfer_size, byte_stream, &ctx.put_opts) .await - .map(|assigned_version_id| { + .map_err(|e| std::io::Error::other(e.to_string())) + .and_then(|response| { audit_target_version_identity( ctx.tgt_client, &ctx.put_opts.internal.source_version_id, - assigned_version_id.as_deref(), - ) - }) - .map_err(|e| std::io::Error::other(e.to_string())); + response.version_id.as_deref(), + ); + verify_single_part_replica(ctx.object_info, &response, ctx.obj_opts.raw_data_movement_read) + }); result.err() } } @@ -4036,28 +4180,132 @@ async fn replicate_object_with_multipart(ctx: MultipartR let arn = ctx.arn; let result = replicate_multipart_parts_and_complete(ctx, &upload_id).await; - abort_multipart_on_failure(result, dst_bucket, object, &upload_id, arn, || async { - cli.abort_multipart_upload(dst_bucket, object, &upload_id).await - }) + abort_multipart_on_failure( + result, + dst_bucket, + object, + &upload_id, + arn, + || async { cli.abort_multipart_upload(dst_bucket, object, &upload_id).await }, + || { + schedule_replication_abort_retry( + cli.clone(), + dst_bucket.to_string(), + object.to_string(), + upload_id.clone(), + arn.to_string(), + ) + }, + ) .await } +const REPLICATION_ABORT_RETRY_ATTEMPTS: u32 = 5; +const REPLICATION_ABORT_RETRY_INITIAL_DELAY_SECS: u64 = 30; + +/// The immediate abort usually fails for the same reason the transfer did — +/// the target is unreachable — and MRF only retries the *object*: every replay +/// mints a fresh upload id, so a failed abort would leak its upload on the +/// target forever (#6854). Retry the abort on a detached, bounded backoff +/// (~30s..8m) so it lands once the target comes back; an upload the target no +/// longer knows counts as cleaned up. +fn schedule_replication_abort_retry(cli: Arc, dst_bucket: String, object: String, upload_id: String, arn: String) { + tokio::spawn(async move { + let mut delay_secs = REPLICATION_ABORT_RETRY_INITIAL_DELAY_SECS; + for attempt in 1..=REPLICATION_ABORT_RETRY_ATTEMPTS { + tokio::time::sleep(tokio::time::Duration::from_secs(delay_secs)).await; + delay_secs = delay_secs.saturating_mul(2); + + match cli.abort_multipart_upload(&dst_bucket, &object, &upload_id).await { + Ok(()) => { + info!( + event = EVENT_REPLICATION_ABORT_RETRY_RESOLVED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + target_bucket = %dst_bucket, + object = %object, + arn = %arn, + upload_id = %upload_id, + operation = "abort_multipart_upload_retry", + attempt, + "Replication abort retry cleaned up the orphaned upload" + ); + return; + } + Err(err) if target_upload_already_removed(&err) => { + info!( + event = EVENT_REPLICATION_ABORT_RETRY_RESOLVED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + target_bucket = %dst_bucket, + object = %object, + arn = %arn, + upload_id = %upload_id, + operation = "abort_multipart_upload_retry", + attempt, + "Replication abort retry found the upload already removed" + ); + return; + } + Err(err) => { + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + target_bucket = %dst_bucket, + object = %object, + arn = %arn, + upload_id = %upload_id, + operation = "abort_multipart_upload_retry", + attempt, + error = %err, + "Replication target operation failed" + ); + } + } + } + + // Terminal: the upload id stays in the log so an operator can reap it + // with list-multipart-uploads/abort by hand (the #6840 contract). + warn!( + event = EVENT_RESYNC_TARGET_OPERATION_FAILED, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC, + target_bucket = %dst_bucket, + object = %object, + arn = %arn, + upload_id = %upload_id, + operation = "abort_multipart_upload_retry", + result = "gave_up", + "Replication abort retries exhausted; the incomplete upload remains on the target" + ); + }); +} + +/// AWS answers an abort for an unknown upload with `NoSuchUpload`; that means +/// the orphan is gone (aborted elsewhere or expired), which is the goal state. +fn target_upload_already_removed(err: &S3ClientError) -> bool { + err.code.as_deref() == Some("NoSuchUpload") +} + /// Best-effort abort of the target-side multipart upload once the transfer has /// failed past CreateMultipartUpload; without it every failed attempt leaves an /// invisible incomplete upload on the target that keeps billing for its parts. /// The abort outcome never replaces the transfer error: an abort failure is /// only logged and `result` is returned as-is. -async fn abort_multipart_on_failure( +async fn abort_multipart_on_failure( result: std::io::Result<()>, dst_bucket: &str, object: &str, upload_id: &str, arn: &str, abort: F, + schedule_abort_retry: R, ) -> std::io::Result<()> where F: FnOnce() -> Fut, Fut: std::future::Future>, + R: FnOnce(), { if result.is_ok() { return result; @@ -4075,6 +4323,9 @@ where error = %abort_err, "Replication target operation failed" ); + if !target_upload_already_removed(&abort_err) { + schedule_abort_retry(); + } } result } @@ -5354,27 +5605,72 @@ mod tests { assert!(!resync_state_accepts_update(¤t, &stale)); } + #[test] + fn object_lock_denied_purge_backoff_tracks_version_and_target() { + let denied = DeletedObjectReplicationInfo { + bucket: "worm-backoff-test-bucket".to_string(), + target_arn: "arn:rustfs:replication::worm-test:t1".to_string(), + delete_object: ReplicationDeletedObject { + object_name: "locked-object".to_string(), + version_id: Some(uuid::Uuid::new_v4()), + ..Default::default() + }, + ..Default::default() + }; + assert!(!object_lock_denied_purge_backoff_active(&denied)); + + record_object_lock_denied_purge(&denied, "arn:rustfs:replication::worm-test:t1"); + assert!(object_lock_denied_purge_backoff_active(&denied)); + + // A requeue that can also reach a target this denial does not cover + // must keep flowing: the purge may succeed there. + let mut other_target = denied.clone(); + other_target.target_arn = "arn:rustfs:replication::worm-test:t2".to_string(); + assert!(!object_lock_denied_purge_backoff_active(&other_target)); + + // A different version of the same object must not be suppressed. + let mut other_version = denied; + other_version.delete_object.version_id = Some(uuid::Uuid::new_v4()); + assert!(!object_lock_denied_purge_backoff_active(&other_version)); + } + #[tokio::test] async fn abort_multipart_on_failure_skips_abort_when_transfer_succeeded() { let aborted = Arc::new(AtomicBool::new(false)); let flag = aborted.clone(); + let retry_scheduled = Arc::new(AtomicBool::new(false)); + let retry_flag = retry_scheduled.clone(); - let result = abort_multipart_on_failure(Ok(()), "dst-bucket", "obj", "upload-1", "arn:dest", move || async move { - flag.store(true, Ordering::SeqCst); - Ok(()) - }) + let result = abort_multipart_on_failure( + Ok(()), + "dst-bucket", + "obj", + "upload-1", + "arn:dest", + move || async move { + flag.store(true, Ordering::SeqCst); + Ok(()) + }, + move || retry_flag.store(true, Ordering::SeqCst), + ) .await; assert!(result.is_ok()); assert!(!aborted.load(Ordering::SeqCst)); + assert!(!retry_scheduled.load(Ordering::SeqCst)); } #[tokio::test] async fn abort_multipart_on_failure_aborts_and_keeps_transfer_error() { let aborted = Arc::new(AtomicBool::new(false)); let flag = aborted.clone(); + let retry_scheduled = Arc::new(AtomicBool::new(false)); + let retry_flag = retry_scheduled.clone(); - // The abort itself failing must not mask the transfer error. + // The abort itself failing must not mask the transfer error, and a + // failed abort must hand the upload id to the retry schedule (#6854): + // the object itself is re-replicated under a fresh upload id, so + // nothing else will ever abort this one. let result = abort_multipart_on_failure( Err(std::io::Error::other("transfer failed")), "dst-bucket", @@ -5385,10 +5681,34 @@ mod tests { flag.store(true, Ordering::SeqCst); Err(S3ClientError::new("abort failed")) }, + move || retry_flag.store(true, Ordering::SeqCst), ) .await; assert!(aborted.load(Ordering::SeqCst)); + assert!(retry_scheduled.load(Ordering::SeqCst)); + assert_eq!(result.unwrap_err().to_string(), "transfer failed"); + } + + #[tokio::test] + async fn abort_multipart_on_failure_does_not_retry_a_gone_upload() { + let retry_scheduled = Arc::new(AtomicBool::new(false)); + let retry_flag = retry_scheduled.clone(); + + let result = abort_multipart_on_failure( + Err(std::io::Error::other("transfer failed")), + "dst-bucket", + "obj", + "upload-1", + "arn:dest", + || async { Err(S3ClientError::with_metadata("gone", None, Some("NoSuchUpload".to_string()), None)) }, + move || retry_flag.store(true, Ordering::SeqCst), + ) + .await; + + // NoSuchUpload means the orphan no longer exists; retrying would only + // produce noise. + assert!(!retry_scheduled.load(Ordering::SeqCst)); assert_eq!(result.unwrap_err().to_string(), "transfer failed"); } } diff --git a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs index ff94159cd..194f9d85e 100644 --- a/crates/ecstore/src/bucket/replication/replication_target_boundary.rs +++ b/crates/ecstore/src/bucket/replication/replication_target_boundary.rs @@ -36,8 +36,8 @@ use time::OffsetDateTime; use time::format_description::well_known::Rfc3339; pub(crate) use crate::bucket::bucket_target_sys::{ - AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, S3ClientError, - TargetClient, resolve_read_api_version_id, + AdvancedPutOptions, HeadObjectSdkError, PutObjectOptions, PutObjectPartOptions, RemotePutObjectResponse, RemoveObjectOptions, + S3ClientError, TargetClient, resolve_read_api_version_id, }; #[cfg(test)] pub(crate) use crate::bucket::target::BucketTarget; diff --git a/crates/ecstore/src/bucket/utils.rs b/crates/ecstore/src/bucket/utils.rs index d24ef95b5..f659468bb 100644 --- a/crates/ecstore/src/bucket/utils.rs +++ b/crates/ecstore/src/bucket/utils.rs @@ -73,6 +73,7 @@ pub fn check_valid_bucket_name_strict(bucket_name: &str) -> Result<()> { check_bucket_name_common(bucket_name, true) } +// RUSTFS_COMPAT_TODO(s3gate-metadata-xml): the s3s codec reads persisted XML during migration. Remove after every supported writer uses the gateway codec and every retained metadata object and backup archive is verified or rewritten. pub fn deserialize(input: &[u8]) -> xml::DeResult where T: for<'xml> xml::Deserialize<'xml>, diff --git a/crates/ecstore/src/cluster/rpc/http_auth.rs b/crates/ecstore/src/cluster/rpc/http_auth.rs index 157da2c57..32ab79a68 100644 --- a/crates/ecstore/src/cluster/rpc/http_auth.rs +++ b/crates/ecstore/src/cluster/rpc/http_auth.rs @@ -1307,6 +1307,32 @@ pub fn verify_tonic_mutation_body_digest(request: &tonic::Request, canonic verify_tonic_mutation_body_digest_with_strictness(request, canonical_body, internode_rpc_body_digest_strict()) } +/// Verify a non-disk mutation without accepting a newly-generated unsigned v2 body. +/// +/// The disk mutation lane has a rolling-upgrade exception for `UNSIGNED-PAYLOAD` +/// while peer replay-cache capability is being discovered. Historical v2 peers +/// used the fixed `unsigned` nonce before body-digest rollout; preserve that +/// exact marker for mixed-version compatibility, but reject unsigned v2 +/// requests that omit it or present a different nonce. +pub fn verify_tonic_mutation_body_digest_reject_unsigned( + request: &tonic::Request, + canonical_body: &[u8], +) -> std::io::Result<()> { + let version = request + .metadata() + .get(RPC_AUTH_VERSION_HEADER) + .and_then(|value| value.to_str().ok()); + let digest = request + .metadata() + .get(RPC_CONTENT_SHA256_HEADER) + .and_then(|value| value.to_str().ok()); + let nonce = request.metadata().get(RPC_NONCE_HEADER).and_then(|value| value.to_str().ok()); + if version == Some(RPC_AUTH_VERSION_V2) && digest == Some(UNSIGNED_PAYLOAD) && nonce != Some("unsigned") { + return Err(std::io::Error::other("RPC mutation requires a body-bound v2 signature")); + } + verify_tonic_mutation_body_digest(request, canonical_body) +} + /// [`verify_tonic_mutation_body_digest`] with the strict gate injected as a parameter, so both /// rollout postures are unit-testable without racing on process-global environment variables. fn verify_tonic_mutation_body_digest_with_strictness( diff --git a/crates/ecstore/src/cluster/rpc/mod.rs b/crates/ecstore/src/cluster/rpc/mod.rs index e8c3d5434..36886b36e 100644 --- a/crates/ecstore/src/cluster/rpc/mod.rs +++ b/crates/ecstore/src/cluster/rpc/mod.rs @@ -39,8 +39,8 @@ pub use http_auth::{ sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_tier_registry_generation, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, - verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, - verify_tonic_rpc_signature_with_bootstrap, + verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof, + verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap, }; #[cfg(test)] pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport; diff --git a/crates/ecstore/src/core/pools.rs b/crates/ecstore/src/core/pools.rs index e3c1194d8..c5d3bfe4c 100644 --- a/crates/ecstore/src/core/pools.rs +++ b/crates/ecstore/src/core/pools.rs @@ -952,6 +952,8 @@ fn record_decommission_unresolved_entry( idx: usize, generation: OffsetDateTime, entry: DecommissionUnresolvedEntry, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, ) -> Result { ensure_decommission_generation(meta, idx, generation)?; if entry.pool_index != idx || entry.source_generation != generation { @@ -959,34 +961,37 @@ fn record_decommission_unresolved_entry( } let pool_count = meta.pools.len(); - let Some(pool) = meta.pools.get_mut(idx) else { + let Some(pool) = meta.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; - let Some(info) = pool.decommission.as_mut() else { + let Some(info) = pool.decommission.as_ref() else { return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry")); }; - let existing = info.unresolved_entries.iter_mut().find(|existing| { + let existing_index = info.unresolved_entries.iter().position(|existing| { existing.bucket == entry.bucket && existing.object == entry.object && existing.pool_index == entry.pool_index && existing.set_index == entry.set_index && existing.source_generation == entry.source_generation }); - let changed = match existing { - Some(existing) if existing == &entry => false, - Some(existing) => { - *existing = entry; - true - } - None => { - info.unresolved_entries.push(entry); - true - } - }; - if changed { - pool.last_update = OffsetDateTime::now_utc(); + if existing_index.is_some_and(|index| info.unresolved_entries[index] == entry) { + return Ok(false); } - Ok(changed) + + let last_update = meta.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = meta.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_mut() else { + return Err(decommission_metadata_not_initialized_error("record decommission unresolved entry")); + }; + if let Some(index) = existing_index { + info.unresolved_entries[index] = entry; + } else { + info.unresolved_entries.push(entry); + } + pool.last_update = last_update; + Ok(true) } type DecommissionUnresolvedEntryIdentity = (usize, String, String); @@ -1760,7 +1765,11 @@ fn merge_pool_meta_updates_for_save( if current_pool.id != idx || persisted_pool.id != idx || current_pool.cmd_line != persisted_pool.cmd_line { return Err(Error::other(format!("{operation}: pool metadata layout changed for pool {idx}"))); } - if current_pool.decommission.is_none() + let current_clears_decommission = current_pool + .decommission + .as_ref() + .is_none_or(|info| !info.has_decommission_state()); + if current_clears_decommission && persisted_pool.decommission.as_ref().is_some_and(|info| { info.has_decommission_state() && is_decommission_active(info.complete, info.failed, info.canceled) }) @@ -1770,6 +1779,19 @@ fn merge_pool_meta_updates_for_save( "{operation}: stale pool metadata update rejected for pool {idx}; persisted active or queued decommission cannot be cleared" ))); } + if current_clears_decommission + && persisted_pool + .decommission + .as_ref() + .is_some_and(|info| info.complete || !info.unresolved_entries.is_empty()) + { + record_pool_meta_stale_write_rejection("unsafe_terminal_decommission_clear"); + return Err(Error::StalePoolMetadataUpdate { + operation: operation.to_string(), + pool_index: idx, + reason: "completed or unresolved decommission state cannot be cleared", + }); + } if current_pool.last_update < persisted_pool.last_update { record_pool_meta_stale_write_rejection("older_pool_revision"); return Err(Error::other(format!( @@ -1778,6 +1800,7 @@ fn merge_pool_meta_updates_for_save( } if let (Some(persisted_info), Some(current_info)) = (persisted_pool.decommission.as_ref(), current_pool.decommission.as_ref()) + && current_info.has_decommission_state() { let persisted_terminal = (persisted_info.complete, persisted_info.failed, persisted_info.canceled); let current_terminal = (current_info.complete, current_info.failed, current_info.canceled); @@ -4594,6 +4617,45 @@ impl PoolMetaSaveOutcome { } impl PoolMeta { + fn current_decommission_movement_update(&self) -> Option { + self.pools + .iter() + .filter(|pool| pool.decommission.is_some()) + .map(|pool| pool.last_update) + .max() + } + + fn current_rebalance_movement_update(rebalance_meta: Option<&RebalanceMeta>) -> Option { + rebalance_meta + .into_iter() + .flat_map(|meta| { + meta.stopped_at.into_iter().chain( + meta.pool_stats + .iter() + .flat_map(|pool| [pool.info.start_time, pool.info.end_time]) + .flatten(), + ) + }) + .max() + } + + pub(crate) fn next_scanner_data_movement_update( + &self, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> OffsetDateTime { + let max_seen = self + .current_decommission_movement_update() + .max(Self::current_rebalance_movement_update(rebalance_meta)); + match max_seen { + Some(max_seen) => max_seen + .checked_add(Duration::nanoseconds(1)) + .map(|next| now.max(next)) + .unwrap_or(max_seen), + None => now, + } + } + fn decode_pool_meta_payload(version: u16, payload: &[u8]) -> Result { match version { POOL_META_VERSION => rmp_serde::from_slice::(payload) @@ -4673,6 +4735,7 @@ impl PoolMeta { idx: usize, duration: Duration, now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, ) -> Result> { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; @@ -4698,7 +4761,7 @@ impl PoolMeta { start_time: info.start_time, queued: info.queued, counted_items: info.counted_items(), - checkpoint_at: now, + checkpoint_at: self.next_scanner_data_movement_update(now, rebalance_meta), })) } @@ -5245,63 +5308,101 @@ impl PoolMeta { } pub fn decommission_cancel(&mut self, idx: usize) -> bool { - if let Some(stats) = self.pools.get_mut(idx) { - if let Some(d) = &stats.decommission { - if !d.canceled { - stats.last_update = OffsetDateTime::now_utc(); + self.decommission_cancel_at(idx, OffsetDateTime::now_utc(), None) + } - let mut pd = d.clone(); - pd.canceled = true; - pd.failed = false; - pd.complete = false; - pd.start_time = None; - pd.terminal_reload_attempt_at = None; - pd.terminal_reload_failures.clear(); + #[cfg(test)] + pub(crate) fn decommission_cancel_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.decommission_cancel_at(idx, now, rebalance_meta) + } - stats.decommission = Some(pd); - true - } else { - false - } - } else { - false - } - } else { - false + fn decommission_cancel_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool { + let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else { + return false; + }; + if d.canceled { + return false; } + + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let mut pd = d.clone(); + pd.canceled = true; + pd.failed = false; + pd.complete = false; + pd.start_time = None; + pd.terminal_reload_attempt_at = None; + pd.terminal_reload_failures.clear(); + + let Some(stats) = self.pools.get_mut(idx) else { + return false; + }; + stats.last_update = last_update; + stats.decommission = Some(pd); + true } pub fn decommission_failed(&mut self, idx: usize) -> bool { - if let Some(stats) = self.pools.get_mut(idx) { - if let Some(d) = &stats.decommission { - if is_decommission_active(d.complete, d.failed, d.canceled) { - stats.last_update = OffsetDateTime::now_utc(); + self.decommission_failed_at(idx, OffsetDateTime::now_utc(), None) + } - let mut pd = d.clone(); - pd.canceled = false; - pd.failed = true; - pd.complete = false; - pd.start_time = None; - pd.terminal_reload_attempt_at = None; - pd.terminal_reload_failures.clear(); + #[cfg(test)] + pub(crate) fn decommission_failed_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.decommission_failed_at(idx, now, rebalance_meta) + } - stats.decommission = Some(pd); - true - } else { - false - } - } else { - false - } - } else { - false + fn decommission_failed_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool { + let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else { + return false; + }; + if !is_decommission_active(d.complete, d.failed, d.canceled) { + return false; } + + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let mut pd = d.clone(); + pd.canceled = false; + pd.failed = true; + pd.complete = false; + pd.start_time = None; + pd.terminal_reload_attempt_at = None; + pd.terminal_reload_failures.clear(); + + let Some(stats) = self.pools.get_mut(idx) else { + return false; + }; + stats.last_update = last_update; + stats.decommission = Some(pd); + true } pub fn clear_decommission(&mut self, idx: usize) -> Result { + self.clear_decommission_at(idx, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn clear_decommission_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result { + self.clear_decommission_at(idx, now, rebalance_meta) + } + + fn clear_decommission_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> Result { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; - let Some(pool) = self.pools.get_mut(idx) else { + let Some(pool) = self.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; @@ -5321,54 +5422,97 @@ impl PoolMeta { ensure_decommission_clear_allowed(true, decommission_present, complete, failed, canceled, unresolved_entries)?; - pool.last_update = OffsetDateTime::now_utc(); - pool.decommission = None; + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + pool.last_update = last_update; + // Preserve a state-empty tombstone so scanner catch-up can recover the + // durable movement generation after a clear followed by a restart. + pool.decommission = Some(PoolDecommissionInfo::default()); Ok(true) } pub fn decommission_complete(&mut self, idx: usize) -> bool { - if let Some(stats) = self.pools.get_mut(idx) { - if let Some(d) = &stats.decommission { - if is_decommission_active(d.complete, d.failed, d.canceled) { - stats.last_update = OffsetDateTime::now_utc(); - - let mut pd = d.clone(); - pd.canceled = false; - pd.failed = false; - pd.complete = true; - pd.terminal_reload_attempt_at = None; - pd.terminal_reload_failures.clear(); - - stats.decommission = Some(pd); - true - } else { - false - } - } else { - false - } - } else { - false - } + self.decommission_complete_at(idx, OffsetDateTime::now_utc(), None) } - fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> { + + #[cfg(test)] + pub(crate) fn decommission_complete_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.decommission_complete_at(idx, now, rebalance_meta) + } + + fn decommission_complete_at(&mut self, idx: usize, now: OffsetDateTime, rebalance_meta: Option<&RebalanceMeta>) -> bool { + let Some(d) = self.pools.get(idx).and_then(|stats| stats.decommission.as_ref()) else { + return false; + }; + if !is_decommission_active(d.complete, d.failed, d.canceled) { + return false; + } + + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let mut pd = d.clone(); + pd.canceled = false; + pd.failed = false; + pd.complete = true; + pd.terminal_reload_attempt_at = None; + pd.terminal_reload_failures.clear(); + + let Some(stats) = self.pools.get_mut(idx) else { + return false; + }; + stats.last_update = last_update; + stats.decommission = Some(pd); + true + } + fn set_decommission_state_at( + &mut self, + idx: usize, + pi: PoolSpaceInfo, + queued: bool, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result<()> { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; - let Some(pool) = self.pools.get_mut(idx) else { + let Some(pool) = self.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; ensure_decommission_start_allowed(decommission_start_pool_state(Some(pool)))?; + let generation = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; let previous = pool.decommission.as_ref(); - let now = OffsetDateTime::now_utc(); - pool.last_update = now; - pool.decommission = Some(build_decommission_start_state(pi, queued, now, previous)); + pool.last_update = generation; + pool.decommission = Some(build_decommission_start_state(pi, queued, generation, previous)); Ok(()) } + fn set_decommission_state(&mut self, idx: usize, pi: PoolSpaceInfo, queued: bool) -> Result<()> { + self.set_decommission_state_at(idx, pi, queued, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn decommission_at_for_test( + &mut self, + idx: usize, + pi: PoolSpaceInfo, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result<()> { + self.set_decommission_state_at(idx, pi, false, now, rebalance_meta) + } + pub fn decommission(&mut self, idx: usize, pi: PoolSpaceInfo) -> Result<()> { self.set_decommission_state(idx, pi, false) } @@ -5378,13 +5522,36 @@ impl PoolMeta { } pub fn record_decommission_terminal_reload_failure(&mut self, idx: usize, stage: &str, message: String) -> Result { + self.record_decommission_terminal_reload_failure_at(idx, stage, message, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn record_decommission_terminal_reload_failure_at_for_test( + &mut self, + idx: usize, + stage: &str, + message: String, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result { + self.record_decommission_terminal_reload_failure_at(idx, stage, message, now, rebalance_meta) + } + + fn record_decommission_terminal_reload_failure_at( + &mut self, + idx: usize, + stage: &str, + message: String, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> Result { let pool_count = self.pools.len(); ensure_valid_decommission_pool_index(pool_count, idx)?; - let Some(pool) = self.pools.get_mut(idx) else { + let Some(pool) = self.pools.get(idx) else { return Err(invalid_decommission_pool_index_error(pool_count, idx)); }; - let Some(info) = pool.decommission.as_mut() else { + let Some(info) = pool.decommission.as_ref() else { return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure")); }; @@ -5393,29 +5560,60 @@ impl PoolMeta { return Ok(false); } - pool.last_update = OffsetDateTime::now_utc(); - info.terminal_reload_attempt_at = Some(pool.last_update); + let last_update = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return Err(invalid_decommission_pool_index_error(pool_count, idx)); + }; + let Some(info) = pool.decommission.as_mut() else { + return Err(decommission_metadata_not_initialized_error("record decommission terminal reload failure")); + }; + pool.last_update = last_update; + info.terminal_reload_attempt_at = Some(last_update); info.terminal_reload_failures.push(failure); Ok(true) } pub fn promote_queued_decommission(&mut self, idx: usize) -> bool { - if let Some(pool) = self.pools.get_mut(idx) - && let Some(info) = pool.decommission.as_mut() - && info.queued - && is_decommission_active(info.complete, info.failed, info.canceled) - { - let now = OffsetDateTime::now_utc(); - pool.last_update = now; - info.queued = false; - let generation = *info.start_time.get_or_insert(now); - for entry in &mut info.unresolved_entries { - entry.source_generation = generation; - } - return true; + self.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), None) + } + + #[cfg(test)] + pub(crate) fn promote_queued_decommission_at_for_test( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + self.promote_queued_decommission_at(idx, now, rebalance_meta) + } + + fn promote_queued_decommission_at( + &mut self, + idx: usize, + now: OffsetDateTime, + rebalance_meta: Option<&RebalanceMeta>, + ) -> bool { + let Some(info) = self.pools.get(idx).and_then(|pool| pool.decommission.as_ref()) else { + return false; + }; + if !info.queued || !is_decommission_active(info.complete, info.failed, info.canceled) { + return false; } - false + let generation = self.next_scanner_data_movement_update(now, rebalance_meta); + let Some(pool) = self.pools.get_mut(idx) else { + return false; + }; + let Some(info) = pool.decommission.as_mut() else { + return false; + }; + pool.last_update = generation; + info.queued = false; + info.start_time = Some(generation); + for entry in &mut info.unresolved_entries { + entry.source_generation = generation; + } + true } pub fn queue_buckets(&mut self, idx: usize, bks: Vec) { if let Some(pool) = self.pools.get_mut(idx) @@ -5494,7 +5692,7 @@ impl PoolMeta { pub fn update_after(&mut self, idx: usize, duration: Duration) -> Result { Ok(self - .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc())? + .decommission_progress_checkpoint(idx, duration, OffsetDateTime::now_utc(), None)? .is_some()) } @@ -6241,9 +6439,17 @@ impl ECStore { entry: DecommissionUnresolvedEntry, ) -> Result<()> { { + let rebalance_meta = self.rebalance_meta.read().await.clone(); let mut pool_meta = self.pool_meta.write().await; ensure_decommission_ledger_persistence_supported(&pool_meta)?; - record_decommission_unresolved_entry(&mut pool_meta, idx, generation, entry)?; + record_decommission_unresolved_entry( + &mut pool_meta, + idx, + generation, + entry, + OffsetDateTime::now_utc(), + rebalance_meta.as_ref(), + )?; } self.save_current_pool_meta(&[idx]) .await @@ -6251,19 +6457,32 @@ impl ECStore { } async fn save_decommission_progress_checkpoint(&self, idx: usize, generation: OffsetDateTime) -> Result { - // Lock order: save gate, then the short pool metadata read/write sections. Peer - // reloads are intentionally performed by the caller after both locks are released. + self.save_decommission_progress_checkpoint_at(idx, generation, OffsetDateTime::now_utc()) + .await + } + + async fn save_decommission_progress_checkpoint_at( + &self, + idx: usize, + generation: OffsetDateTime, + now: OffsetDateTime, + ) -> Result { + // Lock order: save gate, rebalance metadata, then the short pool + // metadata read/write sections. Peer reloads are intentionally + // performed by the caller after both locks are released. let mut save_guard = self.pool_meta_save_gate.lock().await; let (pool_meta_guard, mut snapshot) = self .acquire_pool_meta_write_guard(&mut save_guard, "decommission progress save failed") .await?; let (snapshot, checkpoint) = { + let rebalance_meta = self.rebalance_meta.read().await.clone(); let pool_meta = self.pool_meta.read().await; ensure_decommission_generation(&pool_meta, idx, generation)?; let Some(checkpoint) = pool_meta.decommission_progress_checkpoint( idx, DECOMMISSION_PROGRESS_SAVE_INTERVAL, - OffsetDateTime::now_utc(), + now, + rebalance_meta.as_ref(), )? else { return Ok(false); @@ -6398,11 +6617,13 @@ impl ECStore { let previous_pool_meta = latest_pool_meta.clone(); let first_idx = indices.first().copied(); for (idx, pi) in space_infos { - if Some(idx) == first_idx { - latest_pool_meta.decommission(idx, pi)?; - } else { - latest_pool_meta.queue_decommission(idx, pi)?; - } + latest_pool_meta.set_decommission_state_at( + idx, + pi, + Some(idx) != first_idx, + OffsetDateTime::now_utc(), + Some(&rebalance_meta), + )?; latest_pool_meta.queue_buckets(idx, decom_buckets.clone()); } @@ -6517,6 +6738,12 @@ impl ECStore { Ok(space_infos) } + pub(crate) async fn next_scanner_data_movement_update(&self, now: OffsetDateTime) -> OffsetDateTime { + let pool_meta = self.pool_meta.read().await; + let rebalance_meta = self.rebalance_meta.read().await; + pool_meta.next_scanner_data_movement_update(now, rebalance_meta.as_ref()) + } + #[tracing::instrument(skip(self))] pub async fn decommission_cancel(self: &Arc, idx: usize) -> Result<()> { self.decommission_cancel_with_owner(idx, None).await @@ -6575,12 +6802,14 @@ impl ECStore { .and_then(rustfs_lock::NamespaceLockGuard::lock_lost_signal); // Lock order: start gate, save gate, distributed pool metadata fence, - // decommission_cancelers, then pool_meta. The state guards stay held - // across persistence so the active generation cannot change before - // the cancel is published. + // rebalance_meta, decommission_cancelers, then pool_meta. The state + // guards stay held across persistence so the active generation cannot + // change before the cancel is published. + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let mut cancelers = self.decommission_cancelers.write().await; let mut pool_meta = self.pool_meta.write().await; - let (pending, should_reload_pool_meta, already_canceled, terminal_canceler) = { + let (pending, should_reload_pool_meta, already_canceled, terminal_canceler, durable_movement_generation) = { let mut already_canceled = false; let (pool_present, decommission_present, terminal) = if let Some(pool) = pool_meta.pools.get(idx) { if let Some(info) = pool.decommission.as_ref() { @@ -6609,7 +6838,7 @@ impl ECStore { .ok_or_else(|| decommission_metadata_not_initialized_error("cancel decommission"))?; let mut snapshot = pool_meta.clone(); let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut snapshot, idx, owner, |pool_meta| { - pool_meta.decommission_cancel(idx) + pool_meta.decommission_cancel_at(idx, terminal_at, rebalance_meta.as_ref()) }) else { return Ok(()); }; @@ -6640,11 +6869,16 @@ impl ECStore { } else { cancelers.get(idx).and_then(Option::as_ref).cloned() }; + let durable_movement_generation = pending + .as_ref() + .map(|(_, commit)| crate::store::scanner_data_movement_timestamp_generation(commit.canceled_pool.last_update)) + .unwrap_or_default(); ( pending, should_retry_decommission_cancel_reload(changed, already_canceled), already_canceled, terminal_canceler, + durable_movement_generation, ) }; let active_worker = terminal_canceler.as_ref().is_some_and(DecommissionCanceler::is_active); @@ -6698,7 +6932,8 @@ impl ECStore { // the shared gate without deadlocking the terminal transition. let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; - self.ctx.advance_data_movement_operation_epoch(); + self.ctx + .advance_data_movement_operation_epoch_to_durable_generation(durable_movement_generation); } if should_reload_pool_meta && let Some(notification_sys) = runtime_sources::notification_sys() { @@ -6897,9 +7132,11 @@ impl ECStore { let _movement_guard = movement_gate.write().await; let (should_reload_pool_meta, previous_pool_meta) = { + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); - let changed = pool_meta.clear_decommission(idx)?; + let changed = pool_meta.clear_decommission_at(idx, terminal_at, rebalance_meta.as_ref())?; (changed, changed.then_some(previous_pool_meta)) }; @@ -6946,12 +7183,13 @@ impl ECStore { let (pool_meta_guard, mut snapshot) = self .acquire_pool_meta_write_guard(&mut save_guard, "decommission promotion failed") .await?; + let rebalance_meta = self.rebalance_meta.read().await.clone(); let mut pool_meta = self.pool_meta.write().await; if pool_meta.pools.get(idx).is_none() { return Err(Error::other("failed to start decommission: target pool was not found")); } let reconciled = reconcile_decommission_meta_buckets(&mut pool_meta, idx); - let promoted = pool_meta.promote_queued_decommission(idx); + let promoted = pool_meta.promote_queued_decommission_at(idx, OffsetDateTime::now_utc(), rebalance_meta.as_ref()); let changed = reconciled || promoted; if changed { merge_pool_meta_updates_for_save(&mut snapshot, &pool_meta, &[idx], "decommission promotion failed")?; @@ -7019,8 +7257,15 @@ impl ECStore { let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; let changed = { + let rebalance_meta = self.rebalance_meta.read().await.clone(); let mut pool_meta = self.pool_meta.write().await; - pool_meta.record_decommission_terminal_reload_failure(idx, stage, err.to_string())? + pool_meta.record_decommission_terminal_reload_failure_at( + idx, + stage, + err.to_string(), + OffsetDateTime::now_utc(), + rebalance_meta.as_ref(), + )? }; if changed { @@ -7135,27 +7380,20 @@ impl ECStore { self.reserve_decommission_routines(rx, indices.as_slice()).await } - pub(crate) async fn spawn_decommission_routines( - &self, - store: Arc, - rx: CancellationToken, - indices: Vec, - ) -> Result<()> { - let index_cancelers = self.reserve_decommission_routines(&rx, indices.as_slice()).await?; - if !index_cancelers.is_empty() { - drop(spawn_decommission_index_cancelers( - store, - rx, - index_cancelers, - Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), - )); - } - - Ok(()) + pub async fn spawn_missing_local_decommission_routines(self: &Arc) -> Result<()> { + self.spawn_missing_local_decommission_routines_with_token(CancellationToken::new()) + .await } - pub async fn spawn_missing_local_decommission_routines(self: &Arc) -> Result<()> { - let rx = CancellationToken::new(); + pub(crate) async fn has_active_local_decommission_worker(&self) -> bool { + let cancelers = self.decommission_cancelers.read().await; + has_active_decommission_canceler(cancelers.as_slice()) + } + + pub(crate) async fn spawn_missing_local_decommission_routines_with_token( + self: &Arc, + rx: CancellationToken, + ) -> Result<()> { let endpoints = self.endpoints(); let index_cancelers = self.reserve_missing_local_decommission_routines(&rx, &endpoints).await?; if index_cancelers.is_empty() { @@ -9025,16 +9263,18 @@ impl ECStore { let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; - // Lock order: movement gate, then decommission_cancelers, then pool_meta. - // Holding both state locks makes owner validation and the terminal - // transition one atomic operation. + // Lock order: movement gate, rebalance_meta, decommission_cancelers, + // then pool_meta. Holding both state locks makes owner validation and + // the terminal transition one atomic operation. + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let (should_reload_pool_meta, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; let previous_pool_meta = pool_meta.clone(); let Some(changed) = update_decommission_for_operation(cancelers.as_slice(), &mut pool_meta, idx, owner, |pool_meta| { - pool_meta.decommission_failed(idx) + pool_meta.decommission_failed_at(idx, terminal_at, rebalance_meta.as_ref()) }) else { return Ok(()); @@ -9142,9 +9382,11 @@ impl ECStore { let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; - // Lock order: movement gate, then decommission_cancelers, then pool_meta. - // Holding both state locks makes owner validation and the terminal - // transition one atomic operation. + // Lock order: movement gate, rebalance_meta, decommission_cancelers, + // then pool_meta. Holding both state locks makes owner validation and + // the terminal transition one atomic operation. + let rebalance_meta = self.rebalance_meta.read().await.clone(); + let terminal_at = OffsetDateTime::now_utc(); let (should_reload_pool_meta, completed, previous_pool_meta, terminal_canceler) = { let cancelers = self.decommission_cancelers.read().await; let mut pool_meta = self.pool_meta.write().await; @@ -9157,7 +9399,7 @@ impl ECStore { verified_generation, verified_unresolved_entries.as_deref(), )?; - Ok::(pool_meta.decommission_complete(idx)) + Ok::(pool_meta.decommission_complete_at(idx, terminal_at, rebalance_meta.as_ref())) }) else { return Ok(()); @@ -11132,10 +11374,16 @@ mod tests { assert!(replica_state.repair_write_safe); let worker_cancel = CancellationToken::new(); - store - .spawn_decommission_routines(Arc::clone(&store), worker_cancel.clone(), vec![0]) + let index_cancelers = store + .reserve_decommission_routines(&worker_cancel, &[0]) .await .expect("the committed activation should admit its decommission worker"); + drop(spawn_decommission_index_cancelers( + Arc::clone(&store), + worker_cancel.clone(), + index_cancelers, + Arc::new(Semaphore::new(decommission_entry_concurrency_limit())), + )); let admitted_cancel = store.decommission_cancelers.read().await[0] .clone() .expect("the admitted decommission worker should have a cancellation token"); @@ -13662,8 +13910,10 @@ mod pools_tests { ); let pool_meta = store.pool_meta.read().await; assert!( - pool_meta.pools[0].decommission.is_none(), - "the detached transaction should publish the persisted clear" + pool_meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) ); } @@ -14101,7 +14351,12 @@ mod pools_tests { merge_pool_meta_updates_for_save(&mut persisted, ¤t, &[0], "clear decommission") .expect("the target pool update should merge into the latest snapshot"); - assert!(persisted.pools[0].decommission.is_none()); + assert!( + persisted.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); assert_eq!(persisted.pools[1].last_update, newer); assert!(persisted.pools[1].decommission.as_ref().is_some_and(|info| info.failed)); } @@ -14142,6 +14397,59 @@ mod pools_tests { assert!(persisted.pools[0].decommission.as_ref().is_some_and(|info| info.canceled)); } + #[test] + fn test_pool_meta_save_merge_rejects_clear_over_completed_or_unresolved_state() { + let timestamp = OffsetDateTime::from_unix_timestamp(2_000).expect("test timestamp should be valid"); + for persisted_info in [ + PoolDecommissionInfo { + complete: true, + ..Default::default() + }, + PoolDecommissionInfo { + failed: true, + unresolved_entries: vec![DecommissionUnresolvedEntry { + bucket: "bucket-a".to_string(), + object: "object-a".to_string(), + pool_index: 0, + set_index: 0, + source_generation: timestamp, + candidate_count: 1, + disk_error_count: 1, + observed_at: timestamp, + reason: "test unresolved entry".to_string(), + }], + ..Default::default() + }, + ] { + let mut persisted = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: timestamp, + decommission: Some(persisted_info), + }], + ..Default::default() + }; + let cleared = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::now_utc(), + decommission: Some(PoolDecommissionInfo::default()), + }], + ..Default::default() + }; + + let err = merge_pool_meta_updates_for_save(&mut persisted, &cleared, &[0], "clear decommission") + .expect_err("a stale clear must not erase completed or unresolved state"); + + assert!( + err.to_string() + .contains("completed or unresolved decommission state cannot be cleared") + ); + } + } + #[test] fn test_pool_meta_publish_preserves_untouched_runtime_progress() { let mut current = PoolMeta { @@ -15533,8 +15841,15 @@ mod pools_tests { }; assert!( - record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry.clone()) - .expect("active generation should accept unresolved entry") + record_decommission_unresolved_entry( + &mut pool_meta, + 0, + generation, + unresolved_entry.clone(), + generation + Duration::nanoseconds(1), + None, + ) + .expect("active generation should accept unresolved entry") ); let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, None, None) .expect_err("unverified completion must retain unresolved entries"); @@ -15558,8 +15873,15 @@ mod pools_tests { replacement.disk_error_count = 2; replacement.observed_at = generation + Duration::seconds(1); assert!( - record_decommission_unresolved_entry(&mut pool_meta, 0, generation, replacement.clone()) - .expect("a newer observation should replace the ledger entry") + record_decommission_unresolved_entry( + &mut pool_meta, + 0, + generation, + replacement.clone(), + generation + Duration::nanoseconds(2), + None, + ) + .expect("a newer observation should replace the ledger entry") ); let err = reconcile_decommission_unresolved_entries_for_completion(&mut pool_meta, 0, Some(generation), Some(&stale_verified)) @@ -15655,6 +15977,100 @@ mod pools_tests { ); } + #[tokio::test] + async fn decommission_metadata_saves_stay_monotonic_across_clock_rollback_before_terminal_restart() { + let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await; + let persisted_floor = store.pool_meta.read().await.pools[0].last_update; + let generation = persisted_floor + .checked_add(Duration::seconds(1)) + .expect("test generation should advance the initialized pool metadata"); + let earlier_tick = generation - Duration::nanoseconds(10); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.dont_save = false; + pool_meta.pools[0].last_update = generation; + pool_meta.pools[0].decommission = Some(PoolDecommissionInfo { + start_time: Some(generation), + ..Default::default() + }); + } + store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("active decommission metadata should persist before rollback checkpoints"); + + let unresolved_entry = DecommissionUnresolvedEntry { + bucket: "bucket-a".to_string(), + object: "object-a".to_string(), + pool_index: 0, + set_index: 0, + source_generation: generation, + candidate_count: 1, + disk_error_count: 1, + observed_at: earlier_tick, + reason: "metadata_resolution_failed".to_string(), + }; + { + let mut pool_meta = store.pool_meta.write().await; + assert!( + record_decommission_unresolved_entry(&mut pool_meta, 0, generation, unresolved_entry, earlier_tick, None) + .expect("unresolved entry should record under active generation") + ); + assert_eq!(pool_meta.pools[0].last_update, generation + Duration::nanoseconds(1)); + } + store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("rollback unresolved-entry save should not stale-reject"); + store.pool_meta.write().await.pools[0] + .decommission + .as_mut() + .expect("decommission metadata should exist") + .items_decommissioned = DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD; + + assert!( + store + .save_decommission_progress_checkpoint_at(0, generation, earlier_tick) + .await + .expect("rollback progress checkpoint should use a monotonic durable identity") + ); + let progress_at = generation + Duration::nanoseconds(2); + assert_eq!(store.pool_meta.read().await.pools[0].last_update, progress_at); + + let terminal_at = generation + Duration::nanoseconds(3); + { + let mut pool_meta = store.pool_meta.write().await; + assert!(pool_meta.decommission_failed_at_for_test(0, earlier_tick, None)); + assert_eq!(pool_meta.pools[0].last_update, terminal_at); + } + store + .save_current_pool_meta_for_test(&[0]) + .await + .expect("terminal failure after rollback checkpoints should persist"); + + let mut restored = PoolMeta::default(); + restored + .load_no_lock_from_replicas(vec![store.pools[0].clone()]) + .await + .expect("terminal metadata should reload after restart"); + assert_eq!(restored.pools[0].last_update, terminal_at); + assert!( + restored.pools[0] + .decommission + .as_ref() + .is_some_and(|info| info.failed && !info.complete && !info.canceled) + ); + + let restarted = decommission_worker_test_store(restored, Vec::new()); + let status = restarted.scanner_data_movement_pause_status().await; + let expected_generation = + u64::try_from(terminal_at.unix_timestamp_nanos()).expect("fixed positive timestamp should fit generation"); + + assert_eq!(status.movement_generation, expected_generation); + assert_eq!(status.reasons, vec![crate::store::ScannerDataMovementPauseReason::DecommissionFailed]); + assert_eq!(restarted.scanner_data_movement_generation(), expected_generation); + } + #[tokio::test] async fn unresolved_entry_probe_verifies_absence_on_every_source_disk() { let (_dirs, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await; @@ -16127,7 +16543,7 @@ mod pools_tests { }; let checkpoint = meta - .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("valid decommission state should produce a checkpoint") .expect("item threshold should produce a checkpoint"); meta.count_item(0, 1, false); @@ -16159,13 +16575,13 @@ mod pools_tests { }; let checkpoint = meta - .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("valid decommission state should produce a checkpoint") .expect("item threshold should produce a checkpoint"); meta.defer_decommission_progress_checkpoint(0, checkpoint, retry_after); assert!( - meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + meta.decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("retry backoff check should succeed") .is_none() ); @@ -16201,7 +16617,7 @@ mod pools_tests { for _ in 0..(DECOMMISSION_PROGRESS_SAVE_ITEM_THRESHOLD * 10) { meta.count_item(0, 1, false); if let Some(checkpoint) = meta - .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at) + .decommission_progress_checkpoint(0, DECOMMISSION_PROGRESS_SAVE_INTERVAL, checkpoint_at, None) .expect("valid decommission state should produce a checkpoint") { checkpoint_count += 1; @@ -17082,7 +17498,12 @@ mod pools_tests { assert!(meta.is_suspended(0)); assert!(meta.clear_decommission(0).expect("terminal decommission should clear")); - assert!(meta.pools[0].decommission.is_none()); + assert!( + meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); assert!(!meta.is_suspended(0)); } } @@ -17300,6 +17721,40 @@ mod pools_tests { assert_eq!(promoted.unresolved_entries[0].source_generation, generation); } + #[test] + fn test_queued_decommission_promotion_advances_generation_after_clock_rollback() { + let queued_at = OffsetDateTime::from_unix_timestamp(1_260).expect("fixed timestamp should be valid"); + let earlier_tick = queued_at - Duration::nanoseconds(10); + let rebalance_floor = queued_at + Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let mut meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: queued_at, + decommission: Some(PoolDecommissionInfo { + queued: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!(meta.promote_queued_decommission_at_for_test(0, earlier_tick, Some(&rebalance))); + let expected_generation = rebalance_floor + Duration::nanoseconds(1); + let promoted = meta.pools[0] + .decommission + .as_ref() + .expect("promoted decommission metadata should exist"); + assert_eq!(meta.pools[0].last_update, expected_generation); + assert_eq!(promoted.start_time, Some(expected_generation)); + assert!(!promoted.queued); + } + #[test] fn test_pool_meta_queued_decommission_is_not_suspended_until_promoted() { let mut meta = PoolMeta { @@ -17410,7 +17865,12 @@ mod pools_tests { assert_eq!(blocked.bytes_done, 1024); assert!(meta.clear_decommission(0).expect("failed decommission should clear")); - assert!(meta.pools[0].decommission.is_none()); + assert!( + meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); meta.decommission( 0, @@ -17497,7 +17957,12 @@ mod pools_tests { assert_eq!(blocked.bytes_done, 512); assert!(meta.clear_decommission(0).expect("canceled decommission should clear")); - assert!(meta.pools[0].decommission.is_none()); + assert!( + meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); meta.queue_decommission( 0, @@ -18515,6 +18980,7 @@ mod pools_tests { let save_started = Arc::new(tokio::sync::Notify::new()); let save_release = Arc::new(tokio::sync::Notify::new()); let save_entered = Arc::new(AtomicBool::new(false)); + let persisted = Arc::new(std::sync::Mutex::new(None)); let mut cancel = tokio::spawn({ let store = store.clone(); @@ -18522,9 +18988,12 @@ mod pools_tests { let save_started = save_started.clone(); let save_release = save_release.clone(); let save_entered = save_entered.clone(); + let persisted = persisted.clone(); async move { store - .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |_, _| async move { + .decommission_cancel_with_owner_and_save(0, Some(&canceler), move |snapshot, _| async move { + *persisted.lock().expect("persisted cancel lock should not be poisoned") = + Some(snapshot.encode_config_data()?); save_entered.store(true, Ordering::SeqCst); save_started.notify_one(); save_release.notified().await; @@ -18561,7 +19030,7 @@ mod pools_tests { .await .expect("the durable cancel must signal the active worker"); assert!(!cancel.is_finished(), "cancel must wait for in-flight movement after the durable signal"); - { + let canceled_at = { let pool_meta = store.pool_meta.read().await; let info = pool_meta.pools[0] .decommission @@ -18571,7 +19040,11 @@ mod pools_tests { assert!(!info.complete); assert!(!info.failed); assert!(info.start_time.is_none()); - } + pool_meta.pools[0].last_update + }; + let expected_generation = crate::store::scanner_data_movement_timestamp_generation(canceled_at); + let scanner_status = store.scanner_data_movement_pause_snapshot_for_test().await; + assert_eq!(scanner_status.movement_generation, expected_generation); drop(side_effect); tokio::time::timeout(StdDuration::from_secs(1), &mut cancel) @@ -18598,6 +19071,27 @@ mod pools_tests { assert!(canceler.is_cancelled()); assert!(!canceler.is_active()); assert!(store.decommission_cancelers.read().await[0].is_none()); + assert_eq!(store.scanner_data_movement_generation(), expected_generation); + + let persisted = persisted + .lock() + .expect("persisted cancel lock should not be poisoned") + .take() + .expect("cancel should persist a durable snapshot"); + let mut durable = PoolMeta::default(); + durable + .load_from_config_data(persisted) + .expect("durable cancel snapshot should decode after restart"); + let restarted = decommission_worker_test_store(durable.clone(), Vec::new()); + let restarted_status = restarted.scanner_data_movement_pause_status().await; + assert_eq!(restarted_status.movement_generation, expected_generation); + + assert!( + durable + .clear_decommission_at_for_test(0, canceled_at, None) + .expect("same-tick clear should succeed") + ); + assert!(durable.pools[0].last_update > canceled_at); } #[test] diff --git a/crates/ecstore/src/data_usage/mod.rs b/crates/ecstore/src/data_usage/mod.rs index 734e1a982..b829208ea 100644 --- a/crates/ecstore/src/data_usage/mod.rs +++ b/crates/ecstore/src/data_usage/mod.rs @@ -109,7 +109,10 @@ static USAGE_MEMORY_GENERATION: AtomicU64 = AtomicU64::new(0); /// strictly tighter than beta.11 (usage treated as 0) and strictly more /// available than a blanket 503. The fallback applies to any window without /// authoritative usage, not only pre-v2 upgrades; the values always come from -/// the last persisted scanner output. Loads go through the TTL-bounded +/// the last persisted scanner output — pre-discard sizes of the +/// authoritative snapshot first, backfilled per bucket from the observed +/// (nonconverged) snapshot for buckets no authoritative cycle has covered +/// yet (issue #6852). Loads go through the TTL-bounded /// snapshot cache, so the quota path adds at most one backend read per /// [`DATA_USAGE_CACHE_TTL_SECS`] window. Returns `None` for buckets absent /// from every persisted snapshot — those still fail closed. @@ -168,7 +171,7 @@ fn fresh_cached_data_usage_snapshot( fn cache_data_usage_snapshot_result( cache: &mut Option, - result: Result<(DataUsageInfo, HashMap), Error>, + result: Result, loaded_at: tokio::time::Instant, refresh_generation: u64, current_generation: u64, @@ -178,7 +181,19 @@ fn cache_data_usage_snapshot_result( } Some(match result { - Ok((info, degraded_baseline)) => { + Ok(LoadedUsageBaseline { + info, + mut degraded_baseline, + observed_unavailable, + }) => { + // A flaky observed read must not shrink quota coverage for a TTL + // window: carry the previous refresh's baseline entries forward, + // letting the fresh (authoritative) values win where they exist. + if observed_unavailable && let Some(previous) = cache.as_ref() { + for (bucket, size) in &previous.degraded_baseline { + degraded_baseline.entry(bucket.clone()).or_insert(*size); + } + } *cache = Some(CachedDataUsageSnapshot { info: Some(info.clone()), loaded_at, @@ -1113,24 +1128,78 @@ async fn load_data_usage_snapshot(store: Arc) -> Result<(DataUsageInfo, /// Load data usage info from backend storage #[instrument(skip(store))] pub async fn load_data_usage_from_backend(store: Arc) -> Result { - Ok(load_data_usage_from_backend_with_baseline(store).await?.0) + Ok(load_data_usage_from_backend_with_baseline(store).await?.info) +} + +/// One refresh of the persisted usage snapshot plus the quota-admission +/// baseline derived from it. +struct LoadedUsageBaseline { + info: DataUsageInfo, + degraded_baseline: HashMap, + /// True when the observed snapshot could not be read (a transport error, + /// not absence): the cached loader then carries the previous refresh's + /// baseline entries forward instead of shrinking quota coverage for a + /// whole TTL window over one flaky read. + observed_unavailable: bool, } /// Like [`load_data_usage_from_backend`], but also returns the pre-discard /// per-bucket sizes so the cached loader can retain them as the degraded /// quota-admission baseline (issue #5716). -async fn load_data_usage_from_backend_with_baseline(store: Arc) -> Result<(DataUsageInfo, HashMap), Error> { - let (data_usage_info, source) = load_data_usage_snapshot(store).await?; - Ok(normalize_loaded_data_usage(data_usage_info, source.is_authoritative()).await) +async fn load_data_usage_from_backend_with_baseline(store: Arc) -> Result { + let (loaded_snapshot, source) = load_data_usage_snapshot(store.clone()).await?; + // The observed-newness gate below compares against the snapshot as + // persisted, before normalization demotes or discards anything. + let authoritative_as_persisted = loaded_snapshot.clone(); + let (info, mut degraded_baseline) = normalize_loaded_data_usage(loaded_snapshot, source.is_authoritative()).await; + + // A bucket without a converged scanner cycle behind it — a freshly joined + // replica whose every cycle is superseded by the sustained replication + // write stream, or a bucket created after the last converged cycle on a + // busy site (#6852) — has no authoritative size, and quota admission + // fails its writes closed indefinitely. The observed (nonconverged) + // snapshot those superseded cycles still publish is the only grounded + // usage in that window, so it backfills buckets the loaded baseline does + // not cover; a value already in the baseline always wins. The newness + // gate ties the observation to this exact authoritative snapshot, so a + // stale observed object left behind by an earlier incarnation (e.g. a + // deleted and recreated bucket) cannot inject ghost usage. Loads sit + // behind the same TTL cache as the snapshot itself, so this adds at most + // one backend read per TTL window. + let mut observed_unavailable = false; + match load_observed_data_usage_snapshot(store).await { + Ok(Some(observed)) if observed_data_usage_is_newer(&observed, &authoritative_as_persisted) => { + backfill_degraded_baseline_from_observed(&mut degraded_baseline, &observed); + } + Ok(_) => {} + Err(_) => observed_unavailable = true, + } + + Ok(LoadedUsageBaseline { + info, + degraded_baseline, + observed_unavailable, + }) } -async fn load_observed_data_usage_snapshot(store: Arc) -> Option { +/// Fill quota-baseline gaps from an observed (nonconverged) snapshot without +/// overriding any bucket the authoritative baseline already covers. +fn backfill_degraded_baseline_from_observed(degraded_baseline: &mut HashMap, observed: &DataUsageInfo) { + for (bucket, usage) in &observed.buckets_usage { + degraded_baseline.entry(bucket.clone()).or_insert(usage.size); + } +} + +/// `Ok(None)` means the observed snapshot is absent or invalid (a settled +/// answer); `Err` means it could not be read at all, so the caller may keep +/// using what it learned from a previous read. +async fn load_observed_data_usage_snapshot(store: Arc) -> Result, Error> { let data = match read_config_preserve_empty(store, &DATA_USAGE_OBSERVED_OBJ_NAME_PATH).await { Ok(data) => data, - Err(Error::ConfigNotFound) => return None, + Err(Error::ConfigNotFound) => return Ok(None), Err(err) => { record_usage_snapshot_failure("read_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err); - return None; + return Err(err); } }; @@ -1139,7 +1208,7 @@ async fn load_observed_data_usage_snapshot(store: Arc) -> Option { - Some(info) + Ok(Some(info)) } Ok(_) => { error!( @@ -1150,11 +1219,11 @@ async fn load_observed_data_usage_snapshot(store: Arc) -> Option { record_usage_snapshot_decode_failure("parse_observed", DATA_USAGE_OBSERVED_OBJ_NAME_PATH.as_str(), &err); - None + Ok(None) } } } @@ -1212,7 +1281,9 @@ fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observe async fn load_admin_data_usage_from_backend(store: Arc) -> Result { let (authoritative, source) = load_data_usage_snapshot(store.clone()).await?; - let observed = load_observed_data_usage_snapshot(store).await; + // For the one-shot admin view a failed observed read degrades to "no + // observation", same as before the read was fallible. + let observed = load_observed_data_usage_snapshot(store).await.ok().flatten(); let (selected, selected_is_current_format) = select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed); Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0) @@ -1375,7 +1446,11 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc) -> R let refresh_generation = admin_data_usage_snapshot_generation(); let result = load_admin_data_usage_from_backend(store.clone()) .await - .map(|info| (info, HashMap::new())); + .map(|info| LoadedUsageBaseline { + info, + degraded_baseline: HashMap::new(), + observed_unavailable: false, + }); let loaded_at = tokio::time::Instant::now(); let mut cache = admin_data_usage_snapshot_cache().write().await; if let Some(result) = cache_data_usage_snapshot_result( @@ -2526,6 +2601,37 @@ mod tests { use std::sync::Arc; use tokio::{io::AsyncReadExt, sync::Mutex}; + #[test] + fn observed_snapshot_only_backfills_baseline_gaps() { + let mut baseline = HashMap::from([("covered".to_string(), 111_u64)]); + let observed = DataUsageInfo { + buckets_usage: HashMap::from([ + ( + "covered".to_string(), + BucketUsageInfo { + size: 999, + ..Default::default() + }, + ), + ( + "replica-only".to_string(), + BucketUsageInfo { + size: 42, + ..Default::default() + }, + ), + ]), + ..Default::default() + }; + + backfill_degraded_baseline_from_observed(&mut baseline, &observed); + + // The authoritative value must win; only the uncovered bucket (#6852: + // a replica that never landed a converged cycle) is filled in. + assert_eq!(baseline.get("covered"), Some(&111)); + assert_eq!(baseline.get("replica-only"), Some(&42)); + } + #[derive(Debug, Default)] struct UsageCasState { object: Option<(Vec, u64)>, @@ -3479,7 +3585,11 @@ mod tests { let first = cache_data_usage_snapshot_result( &mut cache, - Ok((expected, HashMap::new())), + Ok(LoadedUsageBaseline { + info: expected, + degraded_baseline: HashMap::new(), + observed_unavailable: false, + }), loaded_at, refresh_generation, data_usage_snapshot_generation(), @@ -3494,6 +3604,38 @@ mod tests { assert_snapshot_bucket(&cached, "bucket"); } + #[test] + #[serial] + fn unavailable_observed_read_keeps_previous_baseline_coverage() { + let loaded_at = tokio::time::Instant::now(); + let refresh_generation = data_usage_snapshot_generation(); + let mut cache = Some(CachedDataUsageSnapshot { + info: Some(data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH)), + loaded_at, + degraded_baseline: HashMap::from([("observed-only".to_string(), 7_u64), ("covered".to_string(), 1)]), + }); + + cache_data_usage_snapshot_result( + &mut cache, + Ok(LoadedUsageBaseline { + info: data_usage_info_for_test("bucket", 1, 42, SystemTime::UNIX_EPOCH), + degraded_baseline: HashMap::from([("covered".to_string(), 2_u64)]), + observed_unavailable: true, + }), + loaded_at, + refresh_generation, + data_usage_snapshot_generation(), + ) + .expect("an uninterrupted refresh should populate the cache") + .expect("successful load must be returned"); + + let baseline = &cache.as_ref().expect("cache must be populated").degraded_baseline; + // The bucket only the (now unreadable) observed snapshot covered must + // survive the refresh; the freshly loaded value wins where it exists. + assert_eq!(baseline.get("observed-only"), Some(&7)); + assert_eq!(baseline.get("covered"), Some(&2)); + } + #[test] #[serial] fn cache_invalidation_during_refresh_prevents_stale_snapshot_resurrection() { @@ -3508,7 +3650,11 @@ mod tests { let stale_result = cache_data_usage_snapshot_result( &mut cache, - Ok((data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), HashMap::new())), + Ok(LoadedUsageBaseline { + info: data_usage_info_for_test("stale", 1, 42, SystemTime::UNIX_EPOCH), + degraded_baseline: HashMap::new(), + observed_unavailable: false, + }), loaded_at, refresh_generation, data_usage_snapshot_generation(), diff --git a/crates/ecstore/src/ecstore_validation_blackbox.rs b/crates/ecstore/src/ecstore_validation_blackbox.rs index 4db7e5104..b89caafe3 100644 --- a/crates/ecstore/src/ecstore_validation_blackbox.rs +++ b/crates/ecstore/src/ecstore_validation_blackbox.rs @@ -16,10 +16,143 @@ use rustfs_filemeta::{MetacacheReader, MetacacheWriter}; use std::io::Cursor; use std::path::PathBuf; use std::sync::Arc; +use std::sync::atomic::{AtomicBool, Ordering}; use tokio::fs; use tokio::io::AsyncReadExt; use tokio::sync::RwLock; +/// Test-only lock client whose refresh path can be rejected independently of +/// every other lock operation. The observed event is awaitable so lock-loss +/// tests do not depend on sleeps or scheduler timing. +#[derive(Debug)] +pub(crate) struct RefreshLossLockClient { + inner: rustfs_lock::LocalClient, + reject_refresh: AtomicBool, + rejected_refresh: AtomicBool, + rejected_refresh_notify: tokio::sync::Notify, +} + +impl RefreshLossLockClient { + pub(crate) fn with_manager(manager: Arc) -> Self { + Self { + inner: rustfs_lock::LocalClient::with_manager(manager), + reject_refresh: AtomicBool::new(false), + rejected_refresh: AtomicBool::new(false), + rejected_refresh_notify: tokio::sync::Notify::new(), + } + } + + pub(crate) fn reject_refreshes(&self) { + self.reject_refresh.store(true, Ordering::Release); + } + + pub(crate) fn refreshes_rejected(&self) -> bool { + self.rejected_refresh.load(Ordering::Acquire) + } + + pub(crate) async fn wait_for_rejected_refresh( + &self, + timeout: std::time::Duration, + ) -> std::result::Result<(), tokio::time::error::Elapsed> { + tokio::time::timeout(timeout, async { + loop { + let notified = self.rejected_refresh_notify.notified(); + if self.refreshes_rejected() { + return; + } + notified.await; + } + }) + .await + } +} + +#[async_trait::async_trait] +impl rustfs_lock::LockClient for RefreshLossLockClient { + async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result { + rustfs_lock::LockClient::acquire_lock(&self.inner, request).await + } + + async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + rustfs_lock::LockClient::release(&self.inner, lock_id).await + } + + async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + if self.reject_refresh.load(Ordering::Acquire) { + self.rejected_refresh.store(true, Ordering::Release); + self.rejected_refresh_notify.notify_waiters(); + return Ok(false); + } + rustfs_lock::LockClient::refresh(&self.inner, lock_id).await + } + + async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { + rustfs_lock::LockClient::force_release(&self.inner, lock_id).await + } + + async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result> { + rustfs_lock::LockClient::check_status(&self.inner, lock_id).await + } + + async fn list_lock_leases(&self) -> Vec { + rustfs_lock::LockClient::list_lock_leases(&self.inner).await + } + + async fn get_stats(&self) -> rustfs_lock::Result { + rustfs_lock::LockClient::get_stats(&self.inner).await + } + + async fn close(&self) -> rustfs_lock::Result<()> { + rustfs_lock::LockClient::close(&self.inner).await + } + + async fn is_online(&self) -> bool { + rustfs_lock::LockClient::is_online(&self.inner).await + } + + async fn is_local(&self) -> bool { + rustfs_lock::LockClient::is_local(&self.inner).await + } +} + +#[tokio::test] +async fn refresh_loss_lock_client_keeps_rejection_observable_for_late_waiters() { + let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new( + rustfs_lock::FastObjectLockManager::new(), + ))); + let client = RefreshLossLockClient::with_manager(manager); + let resource = rustfs_lock::ObjectKey::new("bucket", "object"); + let response = rustfs_lock::LockClient::acquire_lock( + &client, + &rustfs_lock::LockRequest::new(resource, rustfs_lock::LockType::Shared, "refresh-loss-harness"), + ) + .await + .expect("acquire should reach the inner local client"); + let lock_id = response.lock_info.expect("the inner local client should acquire the lock").id; + assert_eq!( + rustfs_lock::LockClient::list_lock_leases(&client).await.len(), + 1, + "lease diagnostics must remain transparent through the refresh wrapper" + ); + + client.reject_refreshes(); + assert!( + !rustfs_lock::LockClient::refresh(&client, &lock_id) + .await + .expect("refresh should return a response") + ); + client + .wait_for_rejected_refresh(std::time::Duration::from_millis(50)) + .await + .expect("a waiter registered after rejection must still observe the event"); + assert!(client.refreshes_rejected()); + assert!( + rustfs_lock::LockClient::release(&client, &lock_id) + .await + .expect("release should reach the inner local client") + ); +} + /// Returns the backing [`tempfile::TempDir`]s alongside the set so callers keep /// them alive for the test's duration and the directories are removed on drop. pub(crate) async fn make_local_set_disks(drive_count: usize, parity_count: usize) -> (Vec, Arc) { diff --git a/crates/ecstore/src/erasure/coding/decode.rs b/crates/ecstore/src/erasure/coding/decode.rs index 7c8b38b54..b0c67c62e 100644 --- a/crates/ecstore/src/erasure/coding/decode.rs +++ b/crates/ecstore/src/erasure/coding/decode.rs @@ -46,6 +46,7 @@ type ShardReadFuture<'a> = Pin = Pin, Error>, Option>, bool)> + Send + 'a>>; pub(crate) type DeferredReaderReopener = Arc Option> + Send + Sync>; +pub(crate) type DecodeOutcome = (usize, Option, bool); type ShardIndexes = SmallVec<[usize; INLINE_SHARD_SLOTS]>; type ActiveReaders = SmallVec<[bool; INLINE_SHARD_SLOTS]>; @@ -574,6 +575,7 @@ pub(crate) struct ParallelReader { read_timeout: Duration, verify_reconstruction: bool, locality_preference_enabled: bool, + demand_bound_lockstep: bool, // Request-scoped shard buffers keyed by shard index. Keeping ownership in // `ParallelReader` avoids dropping unused parity/backup slot buffers between stripes. buffers: ShardBufferPool, @@ -585,10 +587,8 @@ pub(crate) struct ParallelReader { // it to the current stripe when it is engaged mid-object (backlog#923). engaged: SmallVec<[bool; INLINE_SHARD_SLOTS]>, deferred_handles: Vec>, - // Copy-source hedges use a fresh deferred reader so cancelling a hedge - // never consumes the unopened reader reserved for a later stripe. The - // vector is empty for callers that do not provide a reopen factory (tests - // and the ordinary GET path retain the handle-based behavior). + // Demand-bound hedges use a fresh deferred reader so cancelling a hedge + // never consumes the unopened reader reserved for a later stripe. deferred_reopeners: Vec>>, stripe_index: usize, } @@ -777,9 +777,9 @@ where // reads all live readers on every stripe — the pre-backlog#923 // behavior. With the gate on, only data slots start engaged; parity is // engaged on demand, stripe-aligned through its deferred handle. - let data_shards_only = get_lockstep_data_shards_only_enabled(); + let demand_bound_lockstep = get_lockstep_data_shards_only_enabled(); let engaged: SmallVec<_> = (0..readers.len()) - .map(|index| !data_shards_only || index < e.data_shards) + .map(|index| !demand_bound_lockstep || index < e.data_shards) .collect(); ParallelReader { readers, @@ -793,6 +793,7 @@ where read_timeout, verify_reconstruction, locality_preference_enabled: get_shard_locality_preference_enabled(), + demand_bound_lockstep, buffers: ShardBufferPool::new(e.data_shards + e.parity_shards), stripe_state: None, engaged, @@ -1275,7 +1276,7 @@ where /// realigned (no pending deferred handle) is likewise retired instead of /// being read out of position. async fn read_lockstep(&mut self, state: &mut StripeReadState) { - if matches!(decode_read_policy(), DecodeReadPolicy::DemandBound) { + if self.demand_bound_lockstep { self.read_lockstep_demand_bound(state).await; return; } @@ -1531,17 +1532,18 @@ where } } - /// Demand-bound lockstep stripe read used by server-side copy sources. + /// Demand-bound data-shards-only lockstep stripe read. /// /// The ordinary lockstep path can cancel every in-flight reader once it /// has a quorum because all of its parity readers are already engaged. - /// Copy sources keep parity unopened until a data reader is missing. A - /// hedge therefore has to race the deferred parity reads against the - /// original data reads and may retire the latter only after the parity has - /// produced an actual decode-plus-verification quorum. The futures own - /// their readers so disjoint data/parity slots can be admitted while the - /// other group is still pending; dropping an abandoned future retires its - /// stream without leaving a borrowed slot behind. + /// Copy sources and the data-shards-only rollout gate keep parity unopened + /// until a data reader is missing. A hedge therefore has to race the + /// deferred parity reads against the original data reads and may retire the + /// latter only after parity has produced an actual decode-plus-verification + /// quorum. The futures own their readers so disjoint data/parity slots can + /// be admitted while the other group is still pending; dropping an + /// abandoned future retires its stream without leaving a borrowed slot + /// behind. async fn read_lockstep_demand_bound(&mut self, state: &mut StripeReadState) { let num_readers = self.readers.len(); state.reset(num_readers, self.data_shards); @@ -1576,14 +1578,14 @@ where let mut completed = 0usize; let mut failed = 0usize; let mut first_shard_recorded = false; - let mut active = vec![false; num_readers]; - let mut temporary_parity = vec![false; num_readers]; + let mut active: ActiveReaders = smallvec![false; num_readers]; + let mut temporary_parity: ActiveReaders = smallvec![false; num_readers]; // A deferred parity slot is attempted at most once per stripe. A // failed disposable hedge keeps its unopened reserve for the next // stripe, but must not be relaunched in a tight same-stripe retry // loop (which would defeat the bounded fan-out and amplify a remote // outage). - let mut attempted_parity = vec![false; num_readers]; + let mut attempted_parity: ActiveReaders = smallvec![false; num_readers]; // Once a data reader has returned an error (or was already missing at // setup), the loss is permanent for lockstep alignment. Use the // deferred handle and keep parity engaged across subsequent stripes; @@ -2189,8 +2191,10 @@ impl Erasure { W: AsyncWrite + Send + Sync + Unpin, R: crate::erasure::coding::ShardSource, { - self.decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new()) - .await + let (written, error, _) = self + .decode_inner(writer, readers, offset, length, total_length, None, Vec::new(), Vec::new()) + .await; + (written, error) } #[allow(dead_code, reason = "read-cost decode path asserted by this file's tests (backlog#1823)")] @@ -2207,8 +2211,10 @@ impl Erasure { W: AsyncWrite + Send + Sync + Unpin, R: crate::erasure::coding::ShardSource, { - self.decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new()) - .await + let (written, error, _) = self + .decode_inner(writer, readers, offset, length, total_length, Some(read_costs), Vec::new(), Vec::new()) + .await; + (written, error) } /// GET decode entry point that also carries the deferred-parity stripe @@ -2261,6 +2267,37 @@ impl Erasure { deferred_handles: Vec>, deferred_reopeners: Vec>>, ) -> (usize, Option) + where + W: AsyncWrite + Send + Sync + Unpin, + R: crate::erasure::coding::ShardSource, + { + let (written, error, _) = self + .decode_inner( + writer, + readers, + offset, + length, + total_length, + read_costs, + deferred_handles, + deferred_reopeners, + ) + .await; + (written, error) + } + + #[allow(clippy::too_many_arguments)] + pub(crate) async fn decode_with_stripe_handles_and_reopeners_with_diagnostics( + &self, + writer: &mut W, + readers: Vec>>, + offset: usize, + length: usize, + total_length: usize, + read_costs: Option>, + deferred_handles: Vec>, + deferred_reopeners: Vec>>, + ) -> DecodeOutcome where W: AsyncWrite + Send + Sync + Unpin, R: crate::erasure::coding::ShardSource, @@ -2298,6 +2335,7 @@ impl Erasure { written: &mut usize, ret_err: &mut Option, stage_metrics_enabled: bool, + require_surplus_source: bool, ) -> StripeFlow where W: AsyncWrite + Send + Sync + Unpin, @@ -2335,7 +2373,12 @@ impl Erasure { // missing data shard and an extra source shard was available, verify // the reconstructed data against that source before streaming bytes. let reconstruct_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); - if let Err(e) = self.decode_data_with_reconstruction_verification(shards) { + let decode_result = if require_surplus_source { + self.decode_data_with_reconstruction_verification_for_lockstep(shards) + } else { + self.decode_data_with_reconstruction_verification(shards) + }; + if let Err(e) = decode_result { record_get_stage_duration_if_enabled(GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_RECONSTRUCT, reconstruct_stage_start); let reason = GetObjectFailureReason::DecodeError; error!( @@ -2404,36 +2447,48 @@ impl Erasure { read_costs: Option>, deferred_handles: Vec>, deferred_reopeners: Vec>>, - ) -> (usize, Option) + ) -> DecodeOutcome where W: AsyncWrite + Send + Sync + Unpin, R: crate::erasure::coding::ShardSource, { if readers.len() != self.data_shards + self.parity_shards { record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid); - return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers"))); + return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid number of readers")), false); } // block_size/data_shards come from on-disk metadata; a corrupt FileInfo with a // zero here must surface as an error, not a divide-by-zero panic on every GET. if self.block_size == 0 || self.data_shards == 0 { record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid); - return (0, Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters"))); + return ( + 0, + Some(io::Error::new(ErrorKind::InvalidInput, "Invalid erasure coding parameters")), + false, + ); } let Some(end_offset) = offset.checked_add(length) else { record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid); - return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))); + return ( + 0, + Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")), + false, + ); }; if end_offset > total_length { record_get_object_pipeline_failure(GET_STAGE_RANGE, GetObjectFailureReason::RangeOrLengthInvalid); - return (0, Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length"))); + return ( + 0, + Some(io::Error::new(ErrorKind::InvalidInput, "offset + length exceeds total length")), + false, + ); } let mut ret_err = None; if length == 0 { - return (0, ret_err); + return (0, ret_err, false); } let mut written = 0; @@ -2473,6 +2528,7 @@ impl Erasure { } }; + let mut exact_quorum = false; if legacy_stripe_prefetch_enabled() { // Depth-1 stripe prefetch (backlog#930 HP-9 step 2): while the current // stripe is reconstructed and emitted, the next stripe's shard reads @@ -2515,6 +2571,7 @@ impl Erasure { let Some((mut shards, errs)) = current.take() else { break; }; + exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards; if idx + 1 < blocks.len() { // Overlap: read stripe idx+1 while reconstructing/emitting idx. @@ -2546,6 +2603,7 @@ impl Erasure { // `shards` are borrowed again below. In the `Stop` case that // drop is what cancels the still-in-flight prefetch read. let (flow, next): (Option, Option) = { + let require_surplus_source = reader.demand_bound_lockstep; let read_fut = read_stripe_timed(&mut reader, stage_metrics_enabled); let emit_fut = self.emit_decoded_stripe( writer, @@ -2556,6 +2614,7 @@ impl Erasure { &mut written, &mut ret_err, stage_metrics_enabled, + require_surplus_source, ); tokio::pin!(read_fut); tokio::pin!(emit_fut); @@ -2603,6 +2662,7 @@ impl Erasure { &mut written, &mut ret_err, stage_metrics_enabled, + reader.demand_bound_lockstep, ) .await { @@ -2626,6 +2686,7 @@ impl Erasure { let stage_metrics_enabled = rustfs_io_metrics::get_stage_metrics_enabled(); let stripe_read_stage_start = get_stage_timer_if_enabled(stage_metrics_enabled); let (mut shards, errs) = reader.read().await; + exact_quorum |= shards.iter().filter(|shard| shard.is_some()).count() == self.data_shards; record_get_stage_duration_if_enabled( GET_OBJECT_PATH_LEGACY_DUPLEX, GET_STAGE_STRIPE_READ, @@ -2642,6 +2703,7 @@ impl Erasure { &mut written, &mut ret_err, stage_metrics_enabled, + reader.demand_bound_lockstep, ) .await { @@ -2654,14 +2716,14 @@ impl Erasure { } if ret_err.is_some() { - return (written, ret_err); + return (written, ret_err, exact_quorum); } if written < length { ret_err = Some(Error::LessData.into()); } - (written, ret_err) + (written, ret_err, exact_quorum) } } @@ -2866,6 +2928,7 @@ mod tests { cursor: Cursor>, stall: Duration, sleep: Option>>, + stall_polls: Arc, }, } @@ -2904,7 +2967,12 @@ mod tests { TestShardReader::TerminalFileNotFound => { Poll::Ready(Err(crate::disk::error::terminal_read_error_to_io(Error::FileNotFound))) } - TestShardReader::PrefixThenSlow { cursor, stall, sleep } => { + TestShardReader::PrefixThenSlow { + cursor, + stall, + sleep, + stall_polls, + } => { let before = buf.filled().len(); match Pin::new(cursor).poll_read(cx, buf) { // Cursor still has bytes for the current stripe: serve them. @@ -2914,6 +2982,7 @@ mod tests { // the task cleanly (no busy `wake_by_ref` spin), letting the // `#[tokio::test(start_paused = true)]` clock auto-advance. Poll::Ready(Ok(())) => { + stall_polls.fetch_add(1, Ordering::SeqCst); let stall = *stall; let sleeper = sleep.get_or_insert_with(|| Box::pin(tokio::time::sleep(stall))); let _ = sleeper.as_mut().poll(cx); @@ -2942,6 +3011,29 @@ mod tests { } } + struct YieldOnceThenFailWriter { + yielded: bool, + } + + impl AsyncWrite for YieldOnceThenFailWriter { + fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, _buf: &[u8]) -> Poll> { + if !self.yielded { + self.yielded = true; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + Poll::Ready(Err(io::Error::new(ErrorKind::BrokenPipe, "injected emit failure after prefetch poll"))) + } + + fn poll_flush(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn poll_shutdown(self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + } + struct DownstreamClosedWriter; impl AsyncWrite for DownstreamClosedWriter { @@ -3878,6 +3970,7 @@ mod tests { (rustfs_config::ENV_OBJECT_DISK_READ_TIMEOUT, Some(READ_TIMEOUT_SECS)), ]; temp_env::async_with_vars(vars, async { + let stall_polls = Arc::new(AtomicUsize::new(0)); let readers: Vec>> = shard_bufs .iter() .map(|buf| { @@ -3887,12 +3980,13 @@ mod tests { cursor: Cursor::new(prefix), stall: STALL, sleep: None, + stall_polls: Arc::clone(&stall_polls), }; Some(BitrotReader::new(reader, shard_size, hash_algo.clone(), false)) }) .collect(); - let mut writer = FailingEmitWriter; + let mut writer = YieldOnceThenFailWriter { yielded: false }; let start = TokioInstant::now(); let (written, err) = erasure.decode(&mut writer, readers, 0, total_len, total_len).await; let elapsed = start.elapsed(); @@ -3900,6 +3994,10 @@ mod tests { // Emit failed on stripe 0, so the GET fails with no bytes emitted. assert!(err.is_some(), "emit failure must surface as an error"); assert_eq!(written, 0, "the failing writer accepts no bytes"); + assert!( + stall_polls.load(Ordering::SeqCst) > 0, + "the speculative next-stripe read must be in flight before emit fails" + ); // The decisive assertion: the prefetch read was cancelled rather than // awaited. Without cancel-safety this would take READ_TIMEOUT_SECS. assert!( @@ -4911,6 +5009,24 @@ mod tests { /// read timeout even though both parity readers were available to engage. #[tokio::test] async fn test_demand_bound_lockstep_hedges_to_deferred_parity_quorum() { + with_decode_read_policy(DecodeReadPolicy::DemandBound, assert_deferred_parity_hedges_slow_data()).await; + } + + /// The ordinary GET rollout gate must use the same bounded parity race as + /// CopySource. Leaving it on the legacy lockstep loop deadlocks the hedge: + /// that loop waits for a parity success before cancelling the slow data + /// read, but does not admit deferred parity until after the data read ends. + #[tokio::test] + #[serial_test::serial] + async fn test_data_shards_only_gate_hedges_to_deferred_parity_quorum() { + temp_env::async_with_vars( + [(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], + assert_deferred_parity_hedges_slow_data(), + ) + .await; + } + + async fn assert_deferred_parity_hedges_slow_data() { const NUM_SHARDS: usize = 1; const BLOCK_SIZE: usize = 64; const DATA_SHARDS: usize = 2; @@ -4951,33 +5067,27 @@ mod tests { ]; let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); - let (bufs, errs, engaged, readers_remaining) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async { - let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification( - readers, - erasure, - 0, - NUM_SHARDS * BLOCK_SIZE, - None, - vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS], - Duration::from_secs(60), - true, - ); - let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read()) - .await - .expect("deferred parity must cover a hedged data shard without waiting for read_timeout"); - ( - bufs, - errs, - parallel_reader.engaged.clone(), - parallel_reader.readers.iter().map(Option::is_some).collect::>(), - ) - }) - .await; + let mut parallel_reader = ParallelReader::new_with_metrics_path_read_costs_timeout_and_reconstruction_verification( + readers, + erasure, + 0, + NUM_SHARDS * BLOCK_SIZE, + None, + vec![ShardReadCost::Unknown; DATA_SHARDS + PARITY_SHARDS], + Duration::from_secs(60), + true, + ); + let (bufs, errs) = tokio::time::timeout(Duration::from_secs(2), parallel_reader.read()) + .await + .expect("deferred parity must cover a hedged data shard without waiting for read_timeout"); assert!(matches!(&errs[0], Some(DiskError::Io(err)) if err.kind() == ErrorKind::TimedOut)); assert_eq!(bufs.iter().filter(|buf| buf.is_some()).count(), DATA_SHARDS + 1); - assert_eq!(engaged.as_slice(), &[true, true, true, true]); - assert_eq!(readers_remaining, vec![false, true, true, true]); + assert_eq!(parallel_reader.engaged.as_slice(), &[true, true, true, true]); + assert_eq!( + parallel_reader.readers.iter().map(Option::is_some).collect::>(), + vec![false, true, true, true] + ); } /// A fast data failure must admit deferred parity immediately. There is @@ -5046,6 +5156,24 @@ mod tests { #[tokio::test] async fn test_demand_bound_canceled_hedge_preserves_deferred_parity_for_next_stripe() { + with_decode_read_policy( + DecodeReadPolicy::DemandBound, + assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(), + ) + .await; + } + + #[tokio::test] + #[serial_test::serial] + async fn test_data_shards_only_gate_canceled_hedge_preserves_deferred_parity_for_next_stripe() { + temp_env::async_with_vars( + [(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], + assert_canceled_hedge_preserves_deferred_parity_for_next_stripe(), + ) + .await; + } + + async fn assert_canceled_hedge_preserves_deferred_parity_for_next_stripe() { const BLOCK_SIZE: usize = 64; const DATA_SHARDS: usize = 2; const PARITY_SHARDS: usize = 2; @@ -5094,7 +5222,7 @@ mod tests { Some(BitrotReader::new(TestShardReader::Pending, SHARD_SIZE, hash_algo, false)), ]; - let (first_parity_reserved, second_result) = with_decode_read_policy(DecodeReadPolicy::DemandBound, async { + let (first_parity_reserved, second_result) = { let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); let mut parallel_reader = ParallelReader::new_with_metrics_path_read_timeout_and_reconstruction_verification( readers, @@ -5155,8 +5283,7 @@ mod tests { parallel_reader.readers[2].is_some() && parallel_reader.readers[3].is_some(), (third_buffers, third_errors), ) - }) - .await; + }; assert!(first_parity_reserved); assert_eq!(parity_calls.load(Ordering::SeqCst), PARITY_SHARDS * 2); @@ -5240,6 +5367,58 @@ mod tests { assert!(error.is_none(), "a failed disposable hedge must not fail a recovered stripe: {error:?}"); } + /// Rollout guard for backlog#1308: when a data shard and the first parity + /// hedge both fail, the gate-on path must not settle at decode quorum and + /// emit an unverified body. The second parity can restore decode quorum but + /// cannot provide the extra source required for reconstruction verification, + /// so the stripe must fail before exposing bytes. + #[tokio::test] + #[serial_test::serial] + async fn test_data_shards_only_gate_data_and_parity_failure_fails_before_output() { + const BLOCK_SIZE: usize = 64; + const DATA_SHARDS: usize = 2; + const PARITY_SHARDS: usize = 2; + + temp_env::async_with_vars([(ENV_RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE, Some("true"))], async { + let erasure = Erasure::new(DATA_SHARDS, PARITY_SHARDS, BLOCK_SIZE); + let payload = (0..BLOCK_SIZE).map(|value| value as u8).collect::>(); + let shards = erasure.encode_data(&payload).expect("test payload should encode"); + let shard_size = erasure.shard_size(); + + let readers = vec![ + Some(BitrotReader::new(TestShardReader::TimedOut, shard_size, HashAlgorithm::None, false)), + Some(BitrotReader::new( + TestShardReader::Ready(Cursor::new(shards[1].to_vec())), + shard_size, + HashAlgorithm::None, + false, + )), + Some(BitrotReader::new( + TestShardReader::TerminalFileNotFound, + shard_size, + HashAlgorithm::None, + false, + )), + Some(BitrotReader::new( + TestShardReader::Ready(Cursor::new(shards[3].to_vec())), + shard_size, + HashAlgorithm::None, + false, + )), + ]; + + let mut output = Vec::new(); + let (written, error) = erasure.decode(&mut output, readers, 0, payload.len(), payload.len()).await; + + assert_eq!(written, 0, "an unverified stripe must not report body bytes"); + assert!(output.is_empty(), "an unverified stripe must not expose a clean short body"); + let error = error.expect("data plus parity loss must fail closed"); + assert_eq!(error.kind(), ErrorKind::InvalidData); + assert!(error.to_string().contains("insufficient source shards")); + }) + .await; + } + /// Lockstep verification-quorum regression (backlog#1156). When a data shard is /// missing, the hedge must settle only at `data_shards + 1` (decode quorum plus /// a reconstruction-verification source), never at exactly `data_shards` — that diff --git a/crates/ecstore/src/erasure/coding/erasure.rs b/crates/ecstore/src/erasure/coding/erasure.rs index a9e4c8bc3..602b24789 100644 --- a/crates/ecstore/src/erasure/coding/erasure.rs +++ b/crates/ecstore/src/erasure/coding/erasure.rs @@ -933,8 +933,29 @@ impl Erasure { } pub(crate) fn decode_data_with_reconstruction_verification(&self, shards: &mut [Option>]) -> io::Result<()> { + self.decode_data_with_reconstruction_verification_policy(shards, false) + } + + pub(crate) fn decode_data_with_reconstruction_verification_for_lockstep( + &self, + shards: &mut [Option>], + ) -> io::Result<()> { + self.decode_data_with_reconstruction_verification_policy(shards, true) + } + + fn decode_data_with_reconstruction_verification_policy( + &self, + shards: &mut [Option>], + require_surplus_source: bool, + ) -> io::Result<()> { let missing_data_source = shards.iter().take(self.data_shards).any(|shard| shard.is_none()); let available_shards = shards.iter().filter(|shard| shard.is_some()).count(); + if require_surplus_source && missing_data_source && available_shards == self.data_shards { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "insufficient source shards to verify reconstructed data", + )); + } let source_parity = if missing_data_source && available_shards > self.data_shards { shards .iter() @@ -1868,6 +1889,31 @@ mod tests { assert_eq!(err.kind(), io::ErrorKind::InvalidData); } + #[test] + fn decode_data_with_verification_scopes_exact_quorum_to_lockstep() { + for uses_legacy in [false, true] { + let erasure = Erasure::new_with_options(3, 2, 128, uses_legacy); + let data = b"verified reads must not accept reconstruction without a surplus source"; + let encoded = erasure.encode_data(data).expect("encode should succeed"); + let mut exact_quorum = optional_shards(&encoded); + exact_quorum[0] = None; + exact_quorum[erasure.total_shard_count() - 1] = None; + + let mut default_shards = exact_quorum.clone(); + erasure + .decode_data_with_reconstruction_verification(&mut default_shards) + .expect("default decode must preserve exact-quorum reconstruction"); + assert_eq!(default_shards[0].as_deref(), Some(encoded[0].as_ref())); + + let err = erasure + .decode_data_with_reconstruction_verification_for_lockstep(&mut exact_quorum) + .expect_err("data-shards-only lockstep must reject an exact decode quorum"); + + assert_eq!(err.kind(), io::ErrorKind::InvalidData); + assert!(err.to_string().contains("insufficient source shards")); + } + } + #[test] fn verify_data_and_parity_rejects_missing_and_mismatched_shards() { let erasure = Erasure::new(4, 2, 128); diff --git a/crates/ecstore/src/error/mod.rs b/crates/ecstore/src/error/mod.rs index 9895018f7..c6002250c 100644 --- a/crates/ecstore/src/error/mod.rs +++ b/crates/ecstore/src/error/mod.rs @@ -185,6 +185,12 @@ pub enum StorageError { DecommissionAlreadyRunning, #[error("Rebalance already running")] RebalanceAlreadyRunning, + #[error("{operation}: stale pool metadata update rejected for pool {pool_index}; {reason}")] + StalePoolMetadataUpdate { + operation: String, + pool_index: usize, + reason: &'static str, + }, #[error("Operation canceled")] OperationCanceled, #[error("No heal required")] @@ -564,6 +570,15 @@ impl Clone for StorageError { StorageError::DoneForNow => StorageError::DoneForNow, StorageError::DecommissionAlreadyRunning => StorageError::DecommissionAlreadyRunning, StorageError::RebalanceAlreadyRunning => StorageError::RebalanceAlreadyRunning, + StorageError::StalePoolMetadataUpdate { + operation, + pool_index, + reason, + } => StorageError::StalePoolMetadataUpdate { + operation: operation.clone(), + pool_index: *pool_index, + reason, + }, StorageError::OperationCanceled => StorageError::OperationCanceled, StorageError::ErasureReadQuorum => StorageError::ErasureReadQuorum, StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum, @@ -667,6 +682,7 @@ impl StorageError { StorageError::DoneForNow => StorageErrorCode::DoneForNow, StorageError::DecommissionAlreadyRunning => StorageErrorCode::DecommissionAlreadyRunning, StorageError::RebalanceAlreadyRunning => StorageErrorCode::RebalanceAlreadyRunning, + StorageError::StalePoolMetadataUpdate { .. } => StorageErrorCode::InvalidArgument, StorageError::OperationCanceled => StorageErrorCode::OperationCanceled, StorageError::ErasureReadQuorum => StorageErrorCode::ErasureReadQuorum, StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum, @@ -948,10 +964,6 @@ pub fn is_err_data_movement_overwrite(err: &Error) -> bool { matches!(err, &StorageError::DataMovementOverwriteErr(_, _, _)) } -pub fn is_err_decommission_running(err: &Error) -> bool { - matches!(err, &StorageError::DecommissionAlreadyRunning) -} - #[allow(dead_code, reason = "predicate asserted by this file's tests (backlog#1823)")] pub fn is_err_rebalance_running(err: &Error) -> bool { matches!(err, &StorageError::RebalanceAlreadyRunning) @@ -1347,9 +1359,6 @@ mod tests { #[test] fn test_error_running_state_helpers() { - assert!(is_err_decommission_running(&StorageError::DecommissionAlreadyRunning)); - assert!(!is_err_decommission_running(&StorageError::RebalanceAlreadyRunning)); - assert!(is_err_rebalance_running(&StorageError::RebalanceAlreadyRunning)); assert!(!is_err_rebalance_running(&StorageError::DecommissionAlreadyRunning)); assert!(is_err_operation_canceled(&StorageError::OperationCanceled)); diff --git a/crates/ecstore/src/runtime/instance.rs b/crates/ecstore/src/runtime/instance.rs index 673de7b92..b7090bcdd 100644 --- a/crates/ecstore/src/runtime/instance.rs +++ b/crates/ecstore/src/runtime/instance.rs @@ -368,6 +368,19 @@ impl InstanceContext { Arc::clone(&self.data_movement_generation_notify) } + pub(crate) fn observe_durable_data_movement_generation(&self, generation: u64) { + if generation == 0 || self.data_movement_generation_exhausted.load(Ordering::Acquire) { + return; + } + let previous = self.data_movement_generation.fetch_max(generation, Ordering::AcqRel); + if generation == u64::MAX { + self.data_movement_generation_exhausted.store(true, Ordering::Release); + } + if generation > previous { + self.data_movement_generation_notify.notify_waiters(); + } + } + pub(crate) fn scanner_publication_state_allowed(&self) -> bool { !self.data_movement_operation_epoch_exhausted() && !self.data_movement_generation_exhausted() @@ -386,6 +399,20 @@ impl InstanceContext { } pub(crate) fn advance_data_movement_operation_epoch(&self) -> u64 { + let (previous, result) = self.advance_data_movement_operation_epoch_only(); + if result != previous { + let _ = self.advance_data_movement_generation(); + } + result + } + + pub(crate) fn advance_data_movement_operation_epoch_to_durable_generation(&self, generation: u64) -> u64 { + let (_, result) = self.advance_data_movement_operation_epoch_only(); + self.observe_durable_data_movement_generation(generation); + result + } + + fn advance_data_movement_operation_epoch_only(&self) -> (u64, u64) { self.scanner_publication_state .store(SCANNER_PUBLICATION_STATE_UNKNOWN, Ordering::Release); let previous = self.data_movement_operation_epoch.load(Ordering::Acquire); @@ -396,10 +423,7 @@ impl InstanceContext { if result == u64::MAX { self.data_movement_operation_epoch_exhausted.store(true, Ordering::Release); } - if result != previous { - let _ = self.advance_data_movement_generation(); - } - result + (previous, result) } /// Advance the movement generation after a durable movement transition. diff --git a/crates/ecstore/src/services/rebalance/control.rs b/crates/ecstore/src/services/rebalance/control.rs index 731a3299b..d9939702d 100644 --- a/crates/ecstore/src/services/rebalance/control.rs +++ b/crates/ecstore/src/services/rebalance/control.rs @@ -845,7 +845,7 @@ impl ECStore { let mut pool_stats = Vec::with_capacity(self.pools.len()); - let now = OffsetDateTime::now_utc(); + let now = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; for disk_stat in disk_stats.iter() { let mut pool_stat = RebalanceStats { @@ -868,8 +868,10 @@ impl ECStore { pool_stats.push(pool_stat); } + let has_participating_pool = pool_stats.iter().any(|pool_stat| pool_stat.participating); let meta = RebalanceMeta { id: Uuid::new_v4().to_string(), + stopped_at: (!has_participating_pool).then_some(now), percent_free_goal, pool_stats, ..Default::default() @@ -963,6 +965,18 @@ impl ECStore { ))); } if meta.stopped_at.is_some() { + if !is_rebalance_conflicting_with_decommission(meta) { + debug!( + event = EVENT_REBALANCE_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_REBALANCE, + state = "start_skipped", + reason = "not_started_terminal", + rebalance_id = %expected_id, + "Skipped rebalance start because metadata is already terminal" + ); + return Ok(()); + } return Err(Error::other(format!("rebalance {expected_id} was stopped before start"))); } } @@ -1214,11 +1228,11 @@ impl ECStore { }; let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; + let stopped_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; let (previous_meta, meta_to_save) = { let mut rebalance_meta = self.rebalance_meta.write().await; let previous_meta = rebalance_meta.clone(); - let meta_to_save = - stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), OffsetDateTime::now_utc(), expected_id)?; + let meta_to_save = stop_rebalance_meta_snapshot_for_id(rebalance_meta.as_mut(), stopped_at, expected_id)?; (previous_meta, meta_to_save) }; @@ -1250,14 +1264,10 @@ impl ECStore { .await?; let movement_gate = self.ctx.data_movement_operation_gate(); let _movement_guard = movement_gate.write().await; + let failed_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; - rollback_rebalance_start_meta_snapshot_for_id( - rebalance_meta.as_mut(), - OffsetDateTime::now_utc(), - expected_id, - start_error, - ) + rollback_rebalance_start_meta_snapshot_for_id(rebalance_meta.as_mut(), failed_at, expected_id, start_error) }; if let Some(meta_to_save) = meta_to_save { @@ -1402,6 +1412,62 @@ mod tests { assert!(cancel.is_cancelled()); } + #[tokio::test] + #[serial_test::serial] + async fn equal_free_ratio_admin_no_participant_rebalance_succeeds_and_persists_terminal_generation_after_restart() { + let (_temp_dirs, store, restarted) = + crate::services::rebalance::test_two_pool_stores_with_isolated_node_contexts(None).await; + let movement_floor = OffsetDateTime::from_unix_timestamp(4_100_000_000).expect("future test timestamp should be valid"); + *store.rebalance_meta.write().await = Some(RebalanceMeta { + id: "previous-terminal-rebalance".to_string(), + stopped_at: Some(movement_floor), + ..Default::default() + }); + set_rebalance_disk_stats_override_for_test( + store.id, + vec![ + DiskStat { + total_space: 100, + available_space: 50, + }, + DiskStat { + total_space: 100, + available_space: 50, + }, + ], + ); + + let rebalance_id = store + .init_and_start_rebalance(vec!["equal-ratio-no-op".to_string()]) + .await + .expect("equal free ratio admin rebalance should succeed as a terminal no-op"); + let stopped_at = { + let local = store.rebalance_meta.read().await; + let local = local.as_ref().expect("no-op rebalance metadata should remain available"); + assert_eq!(local.id, rebalance_id); + assert!(local.pool_stats.iter().all(|pool_stat| !pool_stat.participating)); + let stopped_at = local.stopped_at.expect("no-op rebalance must persist a terminal timestamp"); + assert_eq!(stopped_at, movement_floor + time::Duration::nanoseconds(1)); + stopped_at + }; + + let stopped_generation = + u64::try_from(stopped_at.unix_timestamp_nanos()).expect("terminal timestamp should map to scanner generation"); + let live_status = store.scanner_data_movement_pause_status().await; + assert!(!live_status.paused); + assert_eq!(live_status.movement_generation, stopped_generation); + + restarted + .load_rebalance_meta() + .await + .expect("restarted store should load the persisted no-op rebalance metadata"); + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, stopped_generation); + assert_eq!(restarted.scanner_data_movement_generation(), stopped_generation); + } + #[tokio::test] #[serial_test::serial] async fn rebalance_activation_rejects_initialized_cluster_with_all_pool_meta_missing() { diff --git a/crates/ecstore/src/services/rebalance/runtime.rs b/crates/ecstore/src/services/rebalance/runtime.rs index fe8bb77f9..f85184742 100644 --- a/crates/ecstore/src/services/rebalance/runtime.rs +++ b/crates/ecstore/src/services/rebalance/runtime.rs @@ -161,6 +161,7 @@ impl ECStore { let cancel_tx = CancellationToken::new(); let rx = cancel_tx.clone(); + let activation_at = self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; let activation_outcome; let candidate; let expected_cancel; @@ -185,12 +186,8 @@ impl ECStore { return Ok(false); } expected_cancel = meta.cancel.clone(); - (candidate, activation_outcome, must_persist) = stage_local_rebalance_worker_activation( - meta, - expected_id.as_ref(), - cancel_tx.clone(), - OffsetDateTime::now_utc(), - )?; + (candidate, activation_outcome, must_persist) = + stage_local_rebalance_worker_activation(meta, expected_id.as_ref(), cancel_tx.clone(), activation_at)?; if let Err(err) = activation_fence.ensure_held() { cancel_tx.cancel(); return Err(err); @@ -384,11 +381,11 @@ impl ECStore { tokio::select! { result = done_rx.recv() => { quit = true; - let now = OffsetDateTime::now_utc(); - let terminal_event = classify_rebalance_terminal_event(result, now); - msg = terminal_event.message().to_string(); let movement_gate = store.ctx.data_movement_operation_gate(); let movement_guard = movement_gate.write().await; + let terminal_at = store.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await; + let terminal_event = classify_rebalance_terminal_event(result, terminal_at); + msg = terminal_event.message().to_string(); let previous_meta = store.rebalance_meta.read().await.clone(); let terminal_state_present = { let mut rebalance_meta = store.rebalance_meta.write().await; @@ -405,7 +402,7 @@ impl ECStore { { pool_stat.info.stopping = false; pool_stat.info.status = RebalStatus::Failed; - pool_stat.info.end_time = Some(now); + pool_stat.info.end_time = Some(terminal_at); pool_stat.info.last_error = Some( pool_stat .cleanup_warnings @@ -433,7 +430,7 @@ impl ECStore { &mut pool_stat.info.end_time, &mut pool_stat.info.last_error, terminal_event, - now, + terminal_at, ); } true @@ -835,6 +832,10 @@ impl ECStore { opt: RebalSaveOpt, expected_id: Option<&str>, ) -> Result<()> { + let now = match opt { + RebalSaveOpt::Stats => OffsetDateTime::now_utc(), + RebalSaveOpt::StoppedAt => self.next_scanner_data_movement_update(OffsetDateTime::now_utc()).await, + }; let meta_to_save = { let mut rebalance_meta = self.rebalance_meta.write().await; if let Some(expected_id) = expected_id { @@ -844,7 +845,6 @@ impl ECStore { return Ok(()); }; - let now = OffsetDateTime::now_utc(); apply_rebalance_save_option(meta, pool_idx, opt, now); meta.clone() }; diff --git a/crates/ecstore/src/set_disk/core/io_primitives.rs b/crates/ecstore/src/set_disk/core/io_primitives.rs index 251bbc6eb..c79bd6868 100644 --- a/crates/ecstore/src/set_disk/core/io_primitives.rs +++ b/crates/ecstore/src/set_disk/core/io_primitives.rs @@ -37,6 +37,8 @@ use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS; #[cfg(test)] use super::super::ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX; #[cfg(test)] +use super::super::ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE; +#[cfg(test)] use super::super::get_metadata_slowtail_fault_delay; use super::super::{ Bytes, CHECK_PART_DISK_NOT_FOUND, DeleteOptions, DiskError, DiskStore, EVENT_SET_DISK_RENAME_TAIL_DRAIN_FAILED, @@ -46,13 +48,14 @@ use super::super::{ ObjectPartInfo, OffsetDateTime, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RawFileInfo, ReadMultipleReq, ReadMultipleResp, ReadOptions, Result, SLASH_SEPARATOR, STORAGE_FORMAT_FILE, SetDisks, SnapshotLeaseToken, StorageError, UpdateMetadataOpts, Uuid, build_inline_bitrot_readers_from_refs, can_try_inline_data_shards_direct, - capacity_scope_from_disks, coding, collect_inline_data_shard_fileinfos_by_index_or_reason, current_dirty_generation, debug, - disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, inline_erasure_shard_file_offset, - inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, is_get_metadata_data_read_early_stop_enabled, - is_get_metadata_early_stop_bounded_fanout_enabled, is_get_metadata_early_stop_enabled, - is_get_metadata_two_phase_read_plan_enabled, is_object_dangling, is_version_early_stop_enabled, issue3031_diag_enabled, - join_all, join_errs, log_multipart_write_quorum_failure, merge_file_meta_versions, path_join_buf, record_global_dirty_scope, - reduce_read_quorum_errs, reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err, + capacity_scope_from_disks, codec_streaming_rollout_applies, coding, collect_inline_data_shard_fileinfos_by_index_or_reason, + current_dirty_generation, debug, disk, file_info_is_valid_for_metadata, get_metadata_slowtail_fault_request, info, + inline_erasure_shard_file_offset, inline_erasure_shard_size, is_err_object_not_found, is_err_version_not_found, + is_get_metadata_data_read_early_stop_enabled, is_get_metadata_early_stop_bounded_fanout_enabled, + is_get_metadata_early_stop_enabled, is_get_metadata_non_inline_data_read_early_stop_enabled, is_object_dangling, + is_version_early_stop_enabled, issue3031_diag_enabled, join_all, join_errs, log_multipart_write_quorum_failure, + merge_file_meta_versions, object_fits_single_block, path_join_buf, record_global_dirty_scope, reduce_read_quorum_errs, + reduce_write_quorum_errs, send_heal_request_with_admission, should_prevent_write, to_object_err, try_read_inline_data_shards_direct, warn, }; #[cfg(test)] @@ -450,6 +453,22 @@ use tokio::io::{AsyncRead, ReadBuf}; use tokio::sync::{Mutex, RwLock, oneshot}; use tokio::task::JoinSet; +struct AbortOnDropJoinHandle(tokio::task::JoinHandle); + +impl Future for AbortOnDropJoinHandle { + type Output = std::result::Result; + + fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + Pin::new(&mut self.0).poll(cx) + } +} + +impl Drop for AbortOnDropJoinHandle { + fn drop(&mut self) { + self.0.abort(); + } +} + pub(in crate::set_disk) const EVENT_SET_DISK_READ: &str = "set_disk_read"; pub(in crate::set_disk) const ENV_RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP: &str = "RUSTFS_GET_DATA_BLOCKS_FIRST_READER_SETUP"; const ENV_RUSTFS_GET_METADATA_READ_VERSION_COALESCE: &str = "RUSTFS_GET_METADATA_READ_VERSION_COALESCE"; @@ -688,6 +707,10 @@ pub(in crate::set_disk) struct MetadataQuorumAccumulator { pub(in crate::set_disk) hard_errors: usize, pub(in crate::set_disk) candidate: Option, pub(in crate::set_disk) candidate_votes: usize, + // Bitset of shard indexes whose metadata matches the candidate. Erasure + // layouts are capped at 16 shards, so this stays allocation-free on the + // GET metadata hot path. + candidate_shard_mask: u16, pub(in crate::set_disk) conflicting_metadata: bool, pub(in crate::set_disk) delete_marker_seen: bool, pub(in crate::set_disk) delete_marker_candidates: Vec<(FileInfo, usize)>, @@ -709,6 +732,7 @@ impl MetadataQuorumAccumulator { hard_errors: 0, candidate: None, candidate_votes: 0, + candidate_shard_mask: 0, conflicting_metadata: false, delete_marker_seen: false, delete_marker_candidates: Vec::new(), @@ -724,6 +748,14 @@ impl MetadataQuorumAccumulator { } pub(in crate::set_disk) fn observe_file_info(&mut self, file_info: &FileInfo) { + self.observe_file_info_with_index(None, file_info); + } + + pub(in crate::set_disk) fn observe_file_info_at(&mut self, disk_index: usize, file_info: &FileInfo) { + self.observe_file_info_with_index(Some(disk_index), file_info); + } + + fn observe_file_info_with_index(&mut self, disk_index: Option, file_info: &FileInfo) { if !file_info_is_valid_for_metadata(file_info) { self.hard_errors = self.hard_errors.saturating_add(1); return; @@ -763,6 +795,11 @@ impl MetadataQuorumAccumulator { match &self.candidate { Some(candidate) if metadata_early_stop_candidate_matches(candidate, file_info) => { self.candidate_votes = self.candidate_votes.saturating_add(1); + if let Some(disk_index) = disk_index + && let Some(bit) = Self::candidate_shard_bit(candidate, file_info, disk_index) + { + self.candidate_shard_mask |= bit; + } } Some(_) => { self.conflicting_metadata = true; @@ -770,10 +807,38 @@ impl MetadataQuorumAccumulator { None => { self.candidate = Some(file_info.clone()); self.candidate_votes = 1; + if let Some(disk_index) = disk_index + && let Some(bit) = Self::candidate_shard_bit(file_info, file_info, disk_index) + { + self.candidate_shard_mask |= bit; + } } } } + fn candidate_shard_bit(candidate: &FileInfo, file_info: &FileInfo, disk_index: usize) -> Option { + let &erasure_index = candidate.erasure.distribution.get(disk_index)?; + if erasure_index == 0 || erasure_index > u16::BITS as usize || file_info.erasure.index != erasure_index { + return None; + } + Some(1u16 << (erasure_index - 1)) + } + + pub(in crate::set_disk) fn candidate_has_read_reserve(&self) -> bool { + self.candidate_read_reserve_target() + .is_some_and(|required| self.candidate_shard_mask.count_ones() as usize >= required) + } + + pub(in crate::set_disk) fn candidate_read_reserve_target(&self) -> Option { + let candidate = self.candidate.as_ref()?; + Some( + candidate + .erasure + .data_blocks + .saturating_add(usize::from(candidate.erasure.parity_blocks > 0)), + ) + } + pub(in crate::set_disk) fn observe_error(&mut self, err: &DiskError) { match err { DiskError::FileNotFound | DiskError::VolumeNotFound => { @@ -1084,11 +1149,7 @@ fn data_read_early_stop_inline_candidate_miss_reason(candidate: &FileInfo) -> Op None } -fn non_inline_data_read_candidate_is_safe( - candidate: &FileInfo, - parts_metadata: &[FileInfo], - disks: &[Option], -) -> bool { +pub(in crate::set_disk) fn non_inline_data_read_candidate_is_safe(candidate: &FileInfo) -> bool { if candidate.inline_data() || candidate.is_compressed() || candidate.is_remote() @@ -1100,34 +1161,21 @@ fn non_inline_data_read_candidate_is_safe( { return false; } - let Ok(erasure) = coding::Erasure::try_new_with_options( - candidate.erasure.data_blocks, - candidate.erasure.parity_blocks, - candidate.erasure.block_size, - candidate.uses_legacy_checksum, - ) else { - return false; - }; - // The regular reader setup can reconstruct missing data shards from any - // `data_shards` matching metadata entries. Requiring every data slot here - // would unnecessarily wait for one slow data disk even when parity and - // the remaining data shards already form a read quorum. - let mut available_shards = vec![false; erasure.data_shards + erasure.parity_shards]; - for ((file_info, disk), &erasure_index) in parts_metadata - .iter() - .zip(disks.iter()) - .zip(candidate.erasure.distribution.iter()) - { - if erasure_index == 0 || erasure_index > available_shards.len() || disk.is_none() { - continue; - } - if metadata_early_stop_candidate_matches(file_info, candidate) && file_info.erasure.index == erasure_index { - available_shards[erasure_index - 1] = true; - } - } - available_shards.into_iter().filter(|present| *present).count() >= erasure.data_shards + candidate.has_valid_erasure_geometry() } +pub(in crate::set_disk) fn late_materialization_candidate_is_safe(candidate: &FileInfo) -> bool { + non_inline_data_read_candidate_is_safe(candidate) + && candidate.size > 512 * 1024 + && object_fits_single_block(candidate.size, candidate.erasure.block_size) +} + +pub(in crate::set_disk) fn non_inline_data_read_early_stop_allowed(read_data: bool, bucket: &str, object: &str) -> bool { + read_data && is_get_metadata_non_inline_data_read_early_stop_enabled() && !codec_streaming_rollout_applies(bucket, object) +} + +const NON_INLINE_SINGLE_PENDING_HEDGE_DELAY: Duration = Duration::from_millis(100); + fn data_read_inline_missing_shards_are_pending( candidate: &FileInfo, parts_metadata: &[FileInfo], @@ -1974,14 +2022,10 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers( return; } - // Only CopySource uses disposable, stripe-aligned reopeners. Ordinary GET - // readers use the existing deferred handle and should not retain one - // heap-allocated closure (plus cloned path/disk state) for every parity - // slot. - let copy_source_demand_bound = matches!( - crate::set_disk::get_object_read_policy(), - crate::set_disk::GetObjectReadPolicy::CopySource - ); + // Every demand-bound lockstep reader needs a disposable, stripe-aligned + // reopener. Otherwise a recovered slow data read can cancel and consume + // the only parity reserve needed by a later degraded stripe. + let demand_bound_lockstep = crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled(); for idx in 0..disks.len() { if setup.attempted[idx] { @@ -1996,7 +2040,7 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers( let disk = disks[idx].clone(); let data_dir = files[idx].data_dir.unwrap_or_default(); let path = format!("{object}/{data_dir}/part.{part_number}"); - let reopener = copy_source_demand_bound.then(|| { + let reopener = demand_bound_lockstep.then(|| { deferred_reader_reopener( inline_data.clone(), disk.clone(), @@ -2037,7 +2081,7 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers( // ready/error bookkeeping that quorum decisions rely on is left untouched. // Gate off (default): keep the eagerly opened parity readers exactly as // before — the lockstep path reads them on every stripe. - if !crate::erasure::coding::decode::get_lockstep_data_shards_only_enabled() { + if !demand_bound_lockstep { return; } for idx in data_shards..disks.len() { @@ -2049,7 +2093,7 @@ pub(in crate::set_disk) fn fill_deferred_bitrot_readers( let disk = disks[idx].clone(); let data_dir = files[idx].data_dir.unwrap_or_default(); let path = format!("{object}/{data_dir}/part.{part_number}"); - let reopener = copy_source_demand_bound.then(|| { + let reopener = demand_bound_lockstep.then(|| { deferred_reader_reopener( inline_data.clone(), disk.clone(), @@ -2914,7 +2958,7 @@ impl SetDisks { read_data, healing, incl_free_versions, - read_data && is_get_metadata_two_phase_read_plan_enabled(), + non_inline_data_read_early_stop_allowed(read_data, bucket, object), default_parity_count, allow_coalescing, ) @@ -2980,7 +3024,7 @@ impl SetDisks { let object = object.clone(); let version_id = version_id.clone(); let slowtail_fault = slowtail_fault.clone(); - tokio::spawn(async move { + AbortOnDropJoinHandle(tokio::spawn(async move { let response_start = observe.then(Instant::now); let result = if let Some(disk) = disk { Self::record_read_version_call(&object, disk_index); @@ -2995,7 +3039,7 @@ impl SetDisks { }; let elapsed = response_start.map(|start| start.elapsed()); (result, elapsed) - }) + })) }); // Wait for all futures to complete @@ -3085,6 +3129,8 @@ impl SetDisks { let mut scheduled_count = 0usize; let mut force_full_wait = false; let mut final_miss_reason_override = None; + let mut non_inline_candidate_eligible = None; + let mut single_pending_hedge_deadline = None; let slowtail_fault = get_metadata_slowtail_fault_request(bucket.as_ref(), object.as_ref(), read_data); let spawn_read_version = |join_set: &mut JoinSet<(usize, disk::error::Result, Duration)>, index: usize, disk: Option| { @@ -3132,18 +3178,54 @@ impl SetDisks { } } - while let Some(result) = join_set.join_next().await { + loop { let mut defer_pending_inline_data_shard = false; + let result = if let Some(deadline) = single_pending_hedge_deadline.take() { + tokio::select! { + result = join_set.join_next() => result, + _ = tokio::time::sleep_until(deadline) => { + if bounded_fanout + && !force_full_wait + && join_set.len() == 1 + && non_inline_candidate_eligible == Some(true) + && !accumulator.candidate_has_read_reserve() + && next_fanout_index < disks.len() + { + while next_fanout_index < disks.len() { + let disk_index = fanout_order[next_fanout_index]; + next_fanout_index = next_fanout_index.saturating_add(1); + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + break; + } + } + } + continue; + } + } + } else { + join_set.join_next().await + }; + let Some(result) = result else { break }; match result { Ok((index, res, elapsed)) => match res { Ok(file_info) => { observations.push(MetadataFanoutObservation::from_file_info(&file_info, elapsed)); - accumulator.observe_file_info(&file_info); + if allow_non_inline_data_read_early_stop { + accumulator.observe_file_info_at(index, &file_info); + } else { + accumulator.observe_file_info(&file_info); + } + if allow_non_inline_data_read_early_stop && non_inline_candidate_eligible.is_none() { + non_inline_candidate_eligible = + accumulator.candidate.as_ref().map(non_inline_data_read_candidate_is_safe); + } if bounded_fanout && read_data && !force_full_wait && let Some(reason) = data_read_early_stop_inline_candidate_miss_reason(&file_info) - && !(allow_non_inline_data_read_early_stop + && !(non_inline_candidate_eligible == Some(true) && reason == GET_METADATA_EARLY_STOP_REASON_DATA_READ_INLINE_NOT_INLINE) { force_full_wait = true; @@ -3175,11 +3257,8 @@ impl SetDisks { { let should_return_early = if read_data { match accumulator.candidate.as_ref() { - Some(candidate) - if allow_non_inline_data_read_early_stop - && non_inline_data_read_candidate_is_safe(candidate, &ress, disks) => - { - true + Some(_candidate) if non_inline_candidate_eligible == Some(true) => { + accumulator.candidate_has_read_reserve() } Some(candidate) => match data_read_early_stop_inline_body_miss_reason( bucket.as_ref(), @@ -3251,12 +3330,37 @@ impl SetDisks { } let pending_responses = join_set.len(); - let should_hedge_single_pending_data_read = read_data + // Inline verification can still depend on a missing data shard; + // issue one immediate spare when only that shard remains. The + // non-inline path keeps its delayed hedge below to avoid healthy + // reads paying speculative I/O before the candidate is classified. + let should_hedge_single_pending_inline_read = read_data && !force_full_wait && !defer_pending_inline_data_shard && pending_responses == 1 + && non_inline_candidate_eligible != Some(true) && accumulator.can_still_reach_early_stop_with_pending(pending_responses); - if bounded_fanout && force_full_wait { + // A non-inline plan must retain one extra matching shard as a + // reconstruction reserve. Schedule that reserve only after the + // candidate is known to be eligible, so inline GETs do not pay an + // extra fanout and the healthy path remains allocation-free. + let needs_non_inline_read_reserve = non_inline_candidate_eligible == Some(true) + && !accumulator.candidate_has_read_reserve() + && accumulator + .candidate_read_reserve_target() + .is_some_and(|reserve_target| scheduled_count < reserve_target || pending_responses == 0); + if bounded_fanout + && !force_full_wait + && (needs_non_inline_read_reserve || should_hedge_single_pending_inline_read) + && next_fanout_index < disks.len() + { + let disk_index = fanout_order[next_fanout_index]; + if let Some(disk) = disks.get(disk_index).cloned() { + spawn_read_version(&mut join_set, disk_index, disk); + scheduled_count = scheduled_count.saturating_add(1); + } + next_fanout_index = next_fanout_index.saturating_add(1); + } else if bounded_fanout && force_full_wait { while next_fanout_index < disks.len() { let disk_index = fanout_order[next_fanout_index]; if let Some(disk) = disks.get(disk_index).cloned() { @@ -3268,8 +3372,7 @@ impl SetDisks { } else if bounded_fanout && !defer_pending_inline_data_shard && next_fanout_index < disks.len() - && (!accumulator.can_still_reach_early_stop_with_pending(pending_responses) - || should_hedge_single_pending_data_read) + && !accumulator.can_still_reach_early_stop_with_pending(pending_responses) { let disk_index = fanout_order[next_fanout_index]; if let Some(disk) = disks.get(disk_index).cloned() { @@ -3278,6 +3381,17 @@ impl SetDisks { } next_fanout_index = next_fanout_index.saturating_add(1); } + if bounded_fanout + && !force_full_wait + && !defer_pending_inline_data_shard + && join_set.len() == 1 + && non_inline_candidate_eligible == Some(true) + && !accumulator.candidate_has_read_reserve() + && accumulator.can_still_reach_early_stop_with_pending(join_set.len()) + && next_fanout_index < disks.len() + { + single_pending_hedge_deadline = Some(tokio::time::Instant::now() + NON_INLINE_SINGLE_PENDING_HEDGE_DELAY); + } } let accumulator_miss_reason = accumulator.final_miss_reason(); @@ -6935,6 +7049,27 @@ mod tests { use tempfile::TempDir; use tokio::io::AsyncReadExt; + #[test] + #[serial_test::serial(codec_streaming_env)] + fn non_inline_early_stop_is_mutually_exclusive_with_codec_rollout() { + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")), + ("RUSTFS_GET_CODEC_STREAMING_ROLLOUT", Some("on")), + ("RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED", Some("true")), + ("RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED", Some("true")), + ], + || assert!(!non_inline_data_read_early_stop_allowed(true, "bucket", "object")), + ); + temp_env::with_vars( + [ + (ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")), + ("RUSTFS_GET_CODEC_STREAMING_ROLLOUT", Some("off")), + ], + || assert!(non_inline_data_read_early_stop_allowed(true, "bucket", "object")), + ); + } + #[tokio::test] async fn scanner_delete_owner_survives_waiter_cancellation() { let movement_gate = Arc::new(tokio::sync::RwLock::new(())); @@ -7362,6 +7497,90 @@ mod tests { drop(dirs); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn metadata_slowtail_fault_gate_stops_before_unneeded_tail() { + const DISKS: usize = 4; + let bucket = "metadata-slowtail-gated-bucket"; + let object = "objects/metadata-slowtail-gated-object"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_mapped_metadata_fanout_fileinfo(&disks, bucket, object).await; + let order = bounded_metadata_fanout_order(bucket, object, DISKS, 2); + let slow_disk = *order.get(3).expect("four-disk fanout should have a deferred tail disk"); + let slow_disk_env = slow_disk.to_string(); + + temp_env::async_with_vars( + [ + (ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("150")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some(slow_disk_env.as_str())), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")), + ], + async { + let calls = disk_call_counters::observe(object); + let read_with_data = + SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2); + let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_millis(500), read_with_data) + .await + .expect("gated metadata read should stop before the deferred slow tail") + .expect("gated metadata fanout should resolve"); + assert!(parts_metadata.iter().filter(|fi| fi.name == object).count() >= 3); + assert!(errs.iter().all(Option::is_none)); + assert!(diagnostics.total_responses() < DISKS); + assert_eq!(calls.total(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT), 0); + }, + ) + .await; + + drop(dirs); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn metadata_slowtail_fault_gate_hedges_an_initial_slow_data_shard() { + const DISKS: usize = 4; + let bucket = "metadata-slowtail-gated-initial-bucket"; + let object = "objects/metadata-slowtail-gated-initial-object"; + let (dirs, disks) = call_counter_local_disks(bucket, DISKS).await; + install_mapped_metadata_fanout_fileinfo(&disks, bucket, object).await; + let order = bounded_metadata_fanout_order(bucket, object, DISKS, 2); + let slow_disk = *order.get(1).expect("four-disk fanout should have an initial data disk"); + let spare_disk = *order.get(3).expect("four-disk fanout should have a spare disk"); + let slow_disk_env = slow_disk.to_string(); + + temp_env::async_with_vars( + [ + (ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE, Some("true")), + (ENV_RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT, Some("true")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DELAY_MS, Some("500")), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_DISKS, Some(slow_disk_env.as_str())), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_BUCKET, Some(bucket)), + (ENV_RUSTFS_GET_METADATA_SLOWTAIL_FAULT_OBJECT_PREFIX, Some("objects/")), + ], + async { + let calls = disk_call_counters::observe(object); + let read_with_data = + SetDisks::read_all_fileinfo_observed(&disks, bucket, bucket, object, "", true, false, false, true, 2); + let (parts_metadata, errs, diagnostics) = tokio::time::timeout(Duration::from_millis(300), read_with_data) + .await + .expect("gated metadata read should hedge the initial slow shard") + .expect("gated metadata fanout should resolve"); + assert!(parts_metadata.iter().filter(|fi| fi.name == object).count() >= 3); + assert!(errs.iter().all(Option::is_none)); + assert!(diagnostics.total_responses() < DISKS); + assert_eq!(calls.for_disk(disk_call_counters::KIND_METADATA_SLOWTAIL_FAULT, slow_disk), 1); + assert_eq!(calls.for_disk(disk_call_counters::KIND_READ_VERSION, spare_disk), 1); + }, + ) + .await; + + drop(dirs); + } + /// Demo / regression guard for the backlog#1325 per-disk call counters. /// /// The metadata fan-out issues each `read_version` inside its own @@ -7775,6 +7994,32 @@ mod tests { } } + async fn install_mapped_metadata_fanout_fileinfo(disks: &[Option], bucket: &str, object: &str) { + let version_id = Uuid::new_v4(); + let data_dir = Uuid::new_v4(); + let mod_time = OffsetDateTime::now_utc(); + let distribution = FileInfo::new(&metadata_distribution_key(bucket, object), 2, 2) + .erasure + .distribution; + for (index, disk) in disks + .iter() + .enumerate() + .filter_map(|(index, disk)| disk.as_ref().map(|disk| (index, disk))) + { + disk.write_all(bucket, &format!("{object}/{data_dir}/part.1"), Bytes::from_static(b"x")) + .await + .expect("part data should be installed on every disk"); + let mut file_info = valid_metadata_fanout_fileinfo(bucket, object, version_id, data_dir, mod_time); + file_info.erasure.distribution = distribution.clone(); + file_info.erasure.index = *distribution + .get(index) + .expect("mapped metadata distribution should cover every disk"); + disk.write_metadata(bucket, bucket, object, file_info) + .await + .expect("mapped metadata should be installed on every disk"); + } + } + async fn inline_metadata_fanout_fileinfos_with_mode( bucket: &str, object: &str, @@ -10397,6 +10642,41 @@ mod tests { assert_eq!(accumulator.candidate_latest_quorum(&impossible_parity), None); } + #[test] + fn metadata_quorum_accumulator_tracks_mapped_shards_and_requires_a_reserve() { + let version_id = Uuid::new_v4(); + let data_dir = Uuid::new_v4(); + let base = valid_metadata_fanout_fileinfo("bucket", "object", version_id, data_dir, OffsetDateTime::now_utc()); + let distribution = base.erasure.distribution.clone(); + let mut accumulator = MetadataQuorumAccumulator::new(4, 2, true); + + for (disk_index, &erasure_index) in distribution.iter().take(2).enumerate() { + let mut file_info = base.clone(); + file_info.erasure.index = erasure_index; + accumulator.observe_file_info_at(disk_index, &file_info); + } + assert!( + !accumulator.candidate_has_read_reserve(), + "data quorum without parity reserve must not early-stop" + ); + + let mut mismatched = base.clone(); + mismatched.erasure.index = distribution[3]; + accumulator.observe_file_info_at(2, &mismatched); + assert!( + !accumulator.candidate_has_read_reserve(), + "mapped index mismatch must not count as a reserve" + ); + + let mut reserve = base; + reserve.erasure.index = distribution[2]; + accumulator.observe_file_info_at(2, &reserve); + assert!( + accumulator.candidate_has_read_reserve(), + "one matching reserve shard should complete the read reserve" + ); + } + #[test] fn metadata_quorum_accumulator_treats_invalid_default_parity_as_full_fanout() { let accumulator = MetadataQuorumAccumulator::new(2, 2, true); diff --git a/crates/ecstore/src/set_disk/mod.rs b/crates/ecstore/src/set_disk/mod.rs index 4b67f6c80..4fac5362a 100644 --- a/crates/ecstore/src/set_disk/mod.rs +++ b/crates/ecstore/src/set_disk/mod.rs @@ -773,10 +773,11 @@ const DEFAULT_RUSTFS_GET_METADATA_VERSION_EARLY_STOP_ENABLE: bool = false; const ENV_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: &str = "RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE"; const DEFAULT_RUSTFS_GET_METADATA_DATA_READ_EARLY_STOP_ENABLE: bool = true; -// Two-phase metadata/read-plan rollout (backlog#1309). The first phase reads -// metadata without inline payloads and only fetches inline data from the -// selected data-shard slots. Keep this opt-in until the Linux multi-node -// slow-tail and small-inline cost gates are complete. +// Opt-in non-inline data-read quorum early-stop rollout (backlog#1309). The +// existing metadata fanout still reads data-bearing metadata; this gate only +// permits a safe plain single-part candidate to stop before the full fanout. +// Keep it opt-in until the Linux multi-node slow-tail and small-inline cost +// gates are complete. The environment name is retained for compatibility. const ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE: &str = "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE"; const DEFAULT_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE: bool = false; @@ -893,8 +894,11 @@ struct OwnedGetObjectFileInfo { fi: FileInfo, parts_metadata: Vec, online_disks: Vec>, + late_metadata_fanout_disks: Option>>, } +type OwnedGetObjectFileInfoParts = (FileInfo, Vec, Vec>, Option>>); + impl GetObjectFileInfo { fn owned(fi: FileInfo, parts_metadata: Vec, online_disks: Vec>) -> Self { Self { @@ -902,6 +906,24 @@ impl GetObjectFileInfo { fi, parts_metadata, online_disks, + late_metadata_fanout_disks: None, + }), + shared: None, + } + } + + fn owned_with_late_metadata_fanout( + fi: FileInfo, + parts_metadata: Vec, + online_disks: Vec>, + late_metadata_fanout_disks: Vec>, + ) -> Self { + Self { + owned: Some(OwnedGetObjectFileInfo { + fi, + parts_metadata, + online_disks, + late_metadata_fanout_disks: Some(late_metadata_fanout_disks), }), shared: None, } @@ -938,19 +960,28 @@ impl GetObjectFileInfo { } } + fn has_late_metadata_fanout(&self) -> bool { + self.owned + .as_ref() + .is_some_and(|snapshot| snapshot.late_metadata_fanout_disks.is_some()) + } + fn into_owned(self) -> (FileInfo, Vec, Vec>) { + let (fi, parts_metadata, online_disks, _) = self.into_owned_with_late_metadata_fanout(); + (fi, parts_metadata, online_disks) + } + + fn into_owned_with_late_metadata_fanout(self) -> OwnedGetObjectFileInfoParts { match (self.owned, self.shared) { - (Some(snapshot), None) => { - let OwnedGetObjectFileInfo { - fi, - parts_metadata, - online_disks, - } = snapshot; - (fi, parts_metadata, online_disks) - } + (Some(snapshot), None) => ( + snapshot.fi, + snapshot.parts_metadata, + snapshot.online_disks, + snapshot.late_metadata_fanout_disks, + ), (None, Some(entry)) => match Arc::try_unwrap(entry) { - Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks), - Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone()), + Ok(entry) => (entry.fi, entry.parts_metadata, entry.online_disks, None), + Err(entry) => (entry.fi.clone(), entry.parts_metadata.clone(), entry.online_disks.clone(), None), }, _ => unreachable!("GET metadata snapshot representation must be exclusive"), } @@ -1037,14 +1068,19 @@ mod prepared_get_object_metadata_tests { const READ_VERSION_BARRIER_GUARD: std::time::Duration = std::time::Duration::from_secs(10); fn object_with_initial_data_shards(bucket: &str, prefix: &str) -> String { + object_with_initial_data_shards_for_geometry(bucket, prefix, 4, 2) + } + + fn object_with_initial_data_shards_for_geometry(bucket: &str, prefix: &str, total_disks: usize, parity: usize) -> String { (0..1000) .map(|index| format!("{prefix}-{index}.bin")) .find(|name| { - let order = bounded_metadata_fanout_order(bucket, name, 4, 2); - let distribution = FileInfo::new(&[bucket, name].join("/"), 2, 2).erasure.distribution; - let mut seen = [false; 2]; - for disk_index in order.into_iter().take(3) { - if let Some(block_index @ 1..=2) = distribution.get(disk_index).copied() { + let order = bounded_metadata_fanout_order(bucket, name, total_disks, parity); + let data = total_disks.saturating_sub(parity); + let distribution = FileInfo::new(&[bucket, name].join("/"), data, parity).erasure.distribution; + let mut seen = vec![false; data]; + for disk_index in order.into_iter().take(total_disks.saturating_sub(parity).saturating_add(1)) { + if let Some(block_index) = distribution.get(disk_index).copied().filter(|index| *index <= data) { seen[block_index - 1] = true; } } @@ -1177,9 +1213,9 @@ mod prepared_get_object_metadata_tests { #[tokio::test] #[serial_test::serial(body_cache_hook)] - async fn two_phase_read_plan_uses_metadata_only_for_non_inline_get() { + async fn non_inline_data_read_early_stop_uses_quorum_plan() { let (_dirs, set_disks) = make_local_set_disks(4, 2).await; - let bucket = "two-phase-read-plan"; + let bucket = "non-inline-read-plan"; let object = object_with_initial_data_shards(bucket, "non-inline-object"); let payload = vec![0x5a; 2 * 1024 * 1024]; let opts = ObjectOptions { @@ -1209,17 +1245,17 @@ mod prepared_get_object_metadata_tests { let mut reader = set_disks .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts) .await - .expect("two-phase GET reader should open"); + .expect("quorum GET reader should open"); let mut restored = Vec::new(); reader .stream .read_to_end(&mut restored) .await - .expect("two-phase GET body should stream"); + .expect("quorum GET body should stream"); assert_eq!(restored, payload); assert!( calls.total(disk_call_counters::KIND_READ_VERSION) < 4, - "non-inline two-phase GET should stop metadata fanout at a quorum" + "non-inline quorum GET should retain a reconstruction reserve" ); }, ) @@ -1228,11 +1264,334 @@ mod prepared_get_object_metadata_tests { #[tokio::test] #[serial_test::serial(body_cache_hook)] - async fn two_phase_read_plan_preserves_inline_early_stop_path() { + async fn non_inline_two_phase_read_fetches_late_parity_after_two_selected_shards_fail() { + let (dirs, set_disks) = make_local_set_disks(4, 2).await; + let bucket = "non-inline-read-late-parity"; + let object = object_with_initial_data_shards(bucket, "late-parity-object"); + let payload = vec![0x5a; 1024 * 1024]; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + let order = bounded_metadata_fanout_order(bucket, &object, 4, 2); + let distribution = FileInfo::new(&[bucket, object.as_str()].join("/"), 2, 2).erasure.distribution; + assert!( + order.iter().take(2).all(|disk_index| distribution[*disk_index] <= 2), + "the two failed selected shards must be data shards" + ); + assert!( + distribution[order[3]] > 2, + "the metadata shard omitted by the plan must be healthy parity" + ); + for disk_index in order.iter().take(2) { + let object_dir = dirs[*disk_index].path().join(bucket).join(&object); + let data_dir = std::fs::read_dir(&object_dir) + .expect("object directory should be readable") + .find_map(|entry| { + let entry = entry.expect("object directory entry should be readable"); + entry + .file_type() + .expect("object directory entry type should be readable") + .is_dir() + .then(|| entry.path()) + }) + .expect("object data directory should exist"); + let part_path = data_dir.join("part.1"); + let mut shard = std::fs::read(&part_path).expect("selected data shard should be readable before corruption"); + shard[0] ^= 0xff; + std::fs::write(part_path, shard).expect("selected data shard should be corrupted after metadata was written"); + } + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let mut reader = set_disks + .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts) + .await + .expect("two-phase GET should recover using late parity"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("late parity should restore the exact GET body"); + assert_eq!(restored, payload); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 7); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn non_inline_two_phase_read_fetches_late_parity_when_selected_parts_are_missing() { + let (dirs, set_disks) = make_local_set_disks(4, 2).await; + let bucket = "non-inline-read-late-parity-missing"; + let object = object_with_initial_data_shards(bucket, "late-parity-missing-object"); + let payload = vec![0x3c; 1024 * 1024]; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + let order = bounded_metadata_fanout_order(bucket, &object, 4, 2); + for disk_index in order.iter().take(2) { + let object_dir = dirs[*disk_index].path().join(bucket).join(&object); + let data_dir = std::fs::read_dir(&object_dir) + .expect("object directory should be readable") + .find_map(|entry| { + let entry = entry.expect("object directory entry should be readable"); + entry + .file_type() + .expect("entry type should be readable") + .is_dir() + .then(|| entry.path()) + }) + .expect("object data directory should exist"); + std::fs::remove_file(data_dir.join("part.1")).expect("selected data shard should be removed"); + } + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let mut reader = set_disks + .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts) + .await + .expect("two-phase GET should recover using late parity"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("late parity should restore the exact GET body"); + assert_eq!(restored, payload); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 7); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn four_data_two_parity_two_phase_read_recovers_one_failed_data_shard() { + let (dirs, set_disks) = make_local_set_disks(6, 2).await; + let bucket = "four-data-two-parity-late-read"; + let object = object_with_initial_data_shards_for_geometry(bucket, "one-failed-data", 4, 2); + let payload = vec![0x7a; 1024 * 1024]; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + let order = bounded_metadata_fanout_order(bucket, &object, 6, 2); + let distribution = FileInfo::new(&[bucket, object.as_str()].join("/"), 4, 2).erasure.distribution; + let failed_disk = *order + .iter() + .take(4) + .find(|disk_index| distribution[**disk_index] <= 4) + .expect("initial fanout should include a data shard"); + assert!( + order.iter().take(4).all(|disk_index| distribution[*disk_index] <= 4), + "initial fanout should cover all four data shards" + ); + assert!(distribution[order[5]] > 4, "the final deferred metadata shard should be parity"); + + let object_dir = dirs[failed_disk].path().join(bucket).join(&object); + let data_dir = std::fs::read_dir(&object_dir) + .expect("object directory should be readable") + .find_map(|entry| { + let entry = entry.expect("object directory entry should be readable"); + entry + .file_type() + .expect("object directory entry type should be readable") + .is_dir() + .then(|| entry.path()) + }) + .expect("object data directory should exist"); + let part_path = data_dir.join("part.1"); + let mut shard = std::fs::read(&part_path).expect("selected data shard should be readable before corruption"); + shard[0] ^= 0xff; + std::fs::write(part_path, shard).expect("selected data shard should be corrupted after metadata was written"); + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let mut reader = set_disks + .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts) + .await + .expect("two-phase GET should recover with one failed data shard"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("late parity should restore the exact GET body"); + assert_eq!(restored, payload); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), 11); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn four_data_two_parity_two_phase_read_rejects_below_read_quorum() { + let (dirs, set_disks) = make_local_set_disks(6, 2).await; + let bucket = "four-data-two-parity-quorum-minus-one"; + let object = object_with_initial_data_shards_for_geometry(bucket, "quorum-minus-one", 4, 2); + let payload = vec![0x4b; 1024 * 1024]; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + let order = bounded_metadata_fanout_order(bucket, &object, 6, 2); + for disk_index in order.iter().take(3) { + let object_dir = dirs[*disk_index].path().join(bucket).join(&object); + let data_dir = std::fs::read_dir(&object_dir) + .expect("object directory should be readable") + .find_map(|entry| { + let entry = entry.expect("object directory entry should be readable"); + entry + .file_type() + .expect("entry type should be readable") + .is_dir() + .then(|| entry.path()) + }) + .expect("object data directory should exist"); + std::fs::remove_file(data_dir.join("part.1")).expect("selected shard should be removed"); + } + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let result = set_disks + .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts) + .await; + assert!(result.is_err(), "quorum-minus-one read must fail closed without exposing a body"); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn non_inline_data_read_early_stop_keeps_reserve_on_unequal_layout() { + let (_dirs, set_disks) = make_local_set_disks(6, 2).await; + let bucket = "non-inline-read-reserve"; + let object = object_with_initial_data_shards_for_geometry(bucket, "reserve-object", 6, 2); + let payload = vec![0x5a; 2 * 1024 * 1024]; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("object should be written"); + + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let mut reader = set_disks + .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts) + .await + .expect("quorum GET reader should open"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("quorum GET body should stream"); + assert_eq!(restored, payload); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + 5, + "the unequal layout should schedule exactly one reserve beyond its data quorum" + ); + }, + ) + .await; + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn non_inline_data_read_early_stop_preserves_inline_path() { let (_dirs, set_disks) = make_local_set_disks(4, 2).await; - let bucket = "two-phase-read-plan-inline"; + let bucket = "non-inline-read-plan-inline"; let object = object_with_initial_data_shards(bucket, "inline-object"); - let payload = b"two-phase inline payload".repeat(256); + let payload = b"quorum inline payload".repeat(256); let opts = ObjectOptions { no_lock: true, ..Default::default() @@ -1259,13 +1618,13 @@ mod prepared_get_object_metadata_tests { let mut reader = set_disks .get_object_reader(bucket, &object, None, HeaderMap::new(), &opts) .await - .expect("two-phase inline GET reader should open"); + .expect("inline GET reader should open"); let mut restored = Vec::new(); reader .stream .read_to_end(&mut restored) .await - .expect("two-phase inline GET body should stream"); + .expect("inline GET body should stream"); assert_eq!(restored, payload); assert_eq!( test_get_object_reader_path_id(), @@ -1278,6 +1637,69 @@ mod prepared_get_object_metadata_tests { .await; } + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn non_inline_data_read_early_stop_does_not_add_inline_fanout_on_unequal_layout() { + let (_dirs, set_disks) = make_local_set_disks(6, 2).await; + let bucket = "inline-read-plan-unequal"; + let object = object_with_initial_data_shards_for_geometry(bucket, "inline-object", 6, 2); + let payload = b"inline quorum payload".repeat(256); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, &object, &mut put_reader, &opts) + .await + .expect("inline object should be written"); + + let read_once = |enabled: bool| { + let set_disks = Arc::clone(&set_disks); + let bucket = bucket.to_string(); + let object = object.clone(); + let payload = payload.clone(); + let opts = opts.clone(); + async move { + temp_env::async_with_vars( + [ + ( + "RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", + Some(if enabled { "true" } else { "false" }), + ), + ("RUSTFS_GET_METADATA_EARLY_STOP_ENABLE", Some("true")), + ("RUSTFS_GET_METADATA_EARLY_STOP_BOUNDED_FANOUT", Some("true")), + ], + async { + let calls = disk_call_counters::observe(&object); + let mut reader = set_disks + .get_object_reader(&bucket, &object, None, HeaderMap::new(), &opts) + .await + .expect("inline GET reader should open"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("inline GET body should stream"); + assert_eq!(restored, payload); + calls.total(disk_call_counters::KIND_READ_VERSION) + }, + ) + .await + } + }; + + let gate_off_calls = read_once(false).await; + let gate_on_calls = read_once(true).await; + assert_eq!(gate_on_calls, gate_off_calls, "inline gate must not add reserve fanout"); + } + #[test] #[serial_test::serial(body_cache_hook)] fn inline_data_read_early_stop_defaults_return_exact_body() { @@ -2024,7 +2446,7 @@ fn is_get_metadata_data_read_early_stop_enabled() -> bool { } } -fn is_get_metadata_two_phase_read_plan_enabled() -> bool { +fn is_get_metadata_non_inline_data_read_early_stop_enabled() -> bool { #[cfg(test)] { rustfs_utils::get_env_bool( @@ -2220,6 +2642,15 @@ fn should_use_codec_streaming(config: GetCodecStreamingConfig, bucket: &str, obj is_optimization_enabled_for_request(config.enabled, config.rollout_pct, bucket, object) } +pub(in crate::set_disk) fn codec_streaming_rollout_applies(bucket: &str, object: &str) -> bool { + let config = get_codec_streaming_config(); + config.enabled + && config.body_compat_confirmed + && config.header_compat_confirmed + && config.rollout.is_opted_in() + && should_use_codec_streaming(config, bucket, object) +} + /// Should this specific request use metadata early-stop? #[allow( dead_code, @@ -6371,6 +6802,7 @@ mod tests { use crate::object_api::BLOCK_SIZE_V2; use crate::object_api::ObjectInfo; use crate::set_disk::core::io_primitives::rename_fanout_barrier; + use crate::set_disk::ops::object::{PutObjectCommitBarrier, PutObjectCommitPause}; use crate::storage_api_contracts::{ heal::HealOperations as _, lifecycle::TransitionedObject, list::ListOperations as _, multipart::CompletePart, object::ObjectOperations as _, @@ -12129,6 +12561,7 @@ mod tests { 0, true, false, + false, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, metrics_size_bucket, @@ -12241,6 +12674,7 @@ mod tests { 0, true, false, + false, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, metrics_size_bucket, @@ -12811,6 +13245,111 @@ mod tests { .await; } + #[tokio::test(flavor = "multi_thread")] + #[serial] + async fn multipart_streaming_get_blocks_overwrite_across_part_boundary() { + temp_env::async_with_vars( + [ + (rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true")), + (ENV_RUSTFS_GET_MULTIPART_READER_SETUP_PREFETCH, Some("false")), + ], + async { + let set_disks = make_local_bucket_test_set_disks().await; + let bucket = "snapshot-multipart-overwrite"; + let object = "object"; + let part_size = usize::try_from(GLOBAL_MIN_PART_SIZE.as_u64()).expect("minimum part size should fit usize"); + let first_part = vec![0x41; part_size]; + let second_part = vec![0x42; part_size]; + let replacement = vec![0x43; first_part.len() + second_part.len()]; + let opts = ObjectOptions::default(); + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let upload = set_disks + .new_multipart_upload(bucket, object, &opts) + .await + .expect("multipart upload should be created"); + let mut completed_parts = Vec::with_capacity(2); + for (part_num, body) in [(1, &first_part), (2, &second_part)] { + let mut reader = PutObjReader::from_vec(body.clone()); + let part = set_disks + .put_object_part(bucket, object, &upload.upload_id, part_num, &mut reader, &opts) + .await + .expect("multipart part should be written"); + completed_parts.push(CompletePart { + part_num, + etag: part.etag, + ..Default::default() + }); + } + let completed = Arc::clone(&set_disks) + .complete_multipart_upload(bucket, object, &upload.upload_id, completed_parts, &opts) + .await + .expect("multipart upload should complete"); + assert!(completed.is_multipart()); + + let mut snapshot = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + .await + .expect("multipart snapshot reader should open"); + let overwrite_set = Arc::clone(&set_disks); + let overwrite_opts = opts.clone(); + let overwrite_body = replacement.clone(); + let commit_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace); + let overwrite = tokio::spawn(async move { + let mut reader = PutObjReader::from_vec(overwrite_body); + overwrite_set.put_object(bucket, object, &mut reader, &overwrite_opts).await + }); + commit_barrier.wait_until_paused().await; + commit_barrier.release_and_wait_until_namespace_pending().await; + assert!( + !commit_barrier.namespace_acquired(), + "overwrite must wait for the multipart response's read lock" + ); + + let mut restored_first = vec![0; first_part.len()]; + snapshot + .stream + .read_exact(&mut restored_first) + .await + .expect("the first multipart part should stream"); + assert_eq!(restored_first, first_part); + assert!( + !commit_barrier.namespace_acquired() && !overwrite.is_finished(), + "overwrite must remain blocked at the first/second part boundary" + ); + + let mut restored_second = Vec::new(); + snapshot + .stream + .read_to_end(&mut restored_second) + .await + .expect("the second multipart part should stream"); + assert_eq!(restored_second, second_part); + tokio::time::timeout(Duration::from_secs(5), overwrite) + .await + .expect("overwrite should proceed after multipart EOF") + .expect("overwrite task should join") + .expect("overwrite should succeed"); + + let mut latest = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + .await + .expect("replacement reader should open"); + let mut latest_body = Vec::new(); + latest + .stream + .read_to_end(&mut latest_body) + .await + .expect("replacement should stream"); + assert_eq!(latest_body, replacement); + }, + ) + .await; + } + #[tokio::test(flavor = "multi_thread")] #[serial] async fn streaming_get_blocks_concurrent_delete_until_eof() { diff --git a/crates/ecstore/src/set_disk/ops/heal.rs b/crates/ecstore/src/set_disk/ops/heal.rs index ab4f21f0b..11df66bc2 100644 --- a/crates/ecstore/src/set_disk/ops/heal.rs +++ b/crates/ecstore/src/set_disk/ops/heal.rs @@ -2767,6 +2767,26 @@ mod heal_result_report_tests { } } + async fn remove_current_object_part(temp_dir: &TempDir, bucket: &str, object: &str) -> std::io::Result<()> { + let object_dir = temp_dir.path().join(bucket).join(object); + let mut entries = tokio::fs::read_dir(&object_dir).await?; + while let Some(entry) = entries.next_entry().await? { + if !entry.file_type().await?.is_dir() { + continue; + } + let part = entry.path().join("part.1"); + match tokio::fs::remove_file(&part).await { + Ok(()) => return Ok(()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => continue, + Err(err) => return Err(err), + } + } + Err(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("no current part.1 found under {}", object_dir.display()), + )) + } + #[test] fn heal_writer_error_summary_redacts_io_message() { let error = DiskError::Io(std::io::Error::new(std::io::ErrorKind::PermissionDenied, "/sensitive/storage/path")); @@ -2799,21 +2819,13 @@ mod heal_result_report_tests { .read_version("", &bucket, object, "", &ReadOptions::default()) .await .expect("source metadata should be readable"); - let data_dir = source.data_dir.expect("non-inline source should have a data directory"); let mut target_slots = [source.erasure.distribution[0] - 1, source.erasure.distribution[1] - 1]; target_slots.sort_unstable(); for index in [0, 1] { - tokio::fs::remove_file( - temp_dirs[index] - .path() - .join(&bucket) - .join(object) - .join(data_dir.to_string()) - .join("part.1"), - ) - .await - .expect("target shard should be removed before heal"); + remove_current_object_part(&temp_dirs[index], &bucket, object) + .await + .expect("target shard should be removed before heal"); } let failed_slots = &target_slots[..failed_target_count]; @@ -3069,9 +3081,20 @@ mod heal_result_report_tests { let payload = vec![0x5a; 1024 * 1024]; let mut reader = PutObjReader::from_vec(payload); - set.put_object(&bucket, object, &mut reader, &ObjectOptions::default()) - .await - .expect("source object should be written"); + // This fixture removes physical shards immediately after PUT. A + // lock-owning PUT may quorum-ack before its rename tail drains, so + // keep the isolated setup on the full-fanout commit path. + set.put_object( + &bucket, + object, + &mut reader, + &ObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + .expect("source object should be written"); let source = disks[2] .read_version("", &bucket, object, "", &ReadOptions::default()) .await diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index fa9aebe71..0e2f27178 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -1379,6 +1379,14 @@ fn data_read_metadata_early_stop_request_shape_allowed(range: &Option, output: &mut Vec, expected_size: usize) -> bool { + if initial_result.is_ok() && output.len() == expected_size { + return false; + } + output.clear(); + true +} + #[cfg(test)] mod data_read_metadata_early_stop_request_shape_tests { use super::*; @@ -1434,6 +1442,14 @@ mod data_read_metadata_early_stop_request_shape_tests { restore_opts.transition.restore_request.days = Some(1); assert!(!data_read_metadata_early_stop_request_shape_allowed(&None, &restore_opts)); } + + #[test] + fn late_materialized_retry_clears_partial_buffer_after_error() { + let mut output = b"partial-prefix".to_vec(); + let result = Err(Error::FileCorrupt); + assert!(prepare_late_materialized_retry(&result, &mut output, 1024)); + assert!(output.is_empty()); + } } /// Length of the full plaintext body when — and only when — this read's output @@ -2012,6 +2028,88 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { } } + if snapshot.has_late_metadata_fanout() { + // Keep refresh plus the second decode off the default GET poll stack. + // The allocation is limited to the opt-in late-materialization path. + return Box::pin(async move { + let object_size = usize::try_from(object_info.size) + .map_err(|_| to_object_err(Error::other("two-phase GET object size is invalid"), vec![bucket, object]))?; + let mut output = Vec::with_capacity(object_size); + let (fi, files, disks, late_metadata_fanout_disks) = snapshot.into_owned_with_late_metadata_fanout(); + let expected_identity = super::super::read::LateMetadataIdentity::from_file_info(&fi); + let late_metadata_fanout_disks = late_metadata_fanout_disks.ok_or_else(|| { + to_object_err(Error::other("two-phase GET fallback context is missing"), vec![bucket, object]) + })?; + let initial_result = Self::get_object_with_fileinfo( + bucket, + object, + Arc::clone(&self.erasure_cache), + 0, + object_info.size, + &mut output, + fi, + files, + &disks, + self.set_index, + self.pool_index, + opts.skip_verify_bitrot, + true, + true, + GET_OBJECT_PATH_LEGACY_DUPLEX, + object_class.as_str(), + size_bucket, + ) + .await; + if prepare_late_materialized_retry(&initial_result, &mut output, object_size) { + let (full_fi, full_parts_metadata, full_online_disks) = Self::refresh_late_metadata_fanout( + &late_metadata_fanout_disks, + bucket, + object, + &expected_identity, + GET_OBJECT_PATH_LEGACY_DUPLEX, + ) + .await?; + Self::get_object_with_fileinfo( + bucket, + object, + Arc::clone(&self.erasure_cache), + 0, + object_info.size, + &mut output, + full_fi, + full_parts_metadata, + &full_online_disks, + self.set_index, + self.pool_index, + opts.skip_verify_bitrot, + true, + false, + GET_OBJECT_PATH_LEGACY_DUPLEX, + object_class.as_str(), + size_bucket, + ) + .await?; + } + if output.len() != object_size { + return Err(to_object_err(Error::other("two-phase GET decoded length mismatch"), vec![bucket, object])); + } + + record_get_object_reader_path_observation(GET_OBJECT_PATH_LEGACY_DUPLEX, object_class, size_bucket); + let body = Bytes::from(output); + let reader = GetObjectReader { + stream: Box::new(Cursor::new(body.clone())), + object_info, + buffered_body: Some(body), + body_source, + }; + if lock_optimization_enabled { + release_materialized_read_lock(bucket, object, read_lock_guard.take()); + } + Ok(reader) + }) + .await; + } + let direct_memory_decision = get_small_object_direct_memory_decision_with_threshold_and_plan( &range, &object_info, @@ -2073,6 +2171,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { self.pool_index, opts.skip_verify_bitrot, true, + false, GET_OBJECT_PATH_DIRECT_MEMORY, object_class.as_str(), size_bucket, @@ -2272,6 +2371,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks { pool_index, skip_verify, false, + false, GET_OBJECT_PATH_LEGACY_DUPLEX, object_class.as_str(), size_bucket, @@ -7821,6 +7921,7 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { pool_index, skip_verify, false, + false, GET_OBJECT_PATH_LEGACY_DUPLEX, GET_CODEC_STREAMING_OBJECT_CLASS_PLAIN_SINGLE_PART, metrics_size_bucket, @@ -9791,6 +9892,9 @@ mod inline_put_commit_path_tests { use super::*; use crate::config::storageclass::{INLINE_BLOCK_ENV, lookup_config_for_pools, lookup_config_for_pools_without_env}; use crate::disk::ReadOptions; + use crate::ecstore_validation_blackbox::make_local_set_disks; + use crate::set_disk::disk_call_counters; + use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions}; use rustfs_config::server_config::KVS; use serial_test::serial; use tokio::io::AsyncReadExt; @@ -9963,6 +10067,69 @@ mod inline_put_commit_path_tests { .await; } + #[tokio::test] + #[serial] + async fn get_object_reader_codec_rollout_excludes_late_metadata_refresh() { + let (_temp_dirs, set_disks) = make_local_set_disks(4, 2).await; + let bucket = "one-mib-codec-reader"; + let object = "object.bin"; + let payload = vec![0x6b; 1024 * 1024]; + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("codec bucket should be created"); + let storage_class = temp_env::with_var(INLINE_BLOCK_ENV, Some("1KiB"), || lookup_config_for_pools(&KVS::new(), &[4])) + .expect("test storage class should resolve"); + set_disks.set_test_storage_class_config(storage_class); + + let mut writer = PutObjReader::from_vec(payload.clone()); + temp_env::async_with_vars( + [ + ("RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE", Some("true")), + (ENV_RUSTFS_GET_MID_SIZE_STREAMING_ENABLE, Some("false")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ENABLE, Some("true")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_MIN_SIZE, Some("1")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ENGINE, Some("legacy")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_MULTIPART_ENABLE, Some("true")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_DATA_BLOCKS_FIRST_ENABLE, Some("false")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_BODY_COMPAT_CONFIRMED, Some("true")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_HEADER_COMPAT_CONFIRMED, Some("true")), + (crate::set_disk::ENV_RUSTFS_GET_CODEC_STREAMING_ROLLOUT, Some("on")), + (rustfs_config::ENV_OBJECT_LOCK_OPTIMIZATION_ENABLE, Some("true")), + ], + async { + set_disks + .put_object(bucket, object, &mut writer, &ObjectOptions::default()) + .await + .expect("codec fixture should commit"); + crate::set_disk::reset_test_get_object_reader_path(); + let calls = disk_call_counters::observe(object); + let mut reader = set_disks + .get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("codec GET should succeed"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("codec GET should stream"); + assert_eq!(restored, payload); + assert_eq!( + crate::set_disk::test_get_object_reader_path_id(), + 5, + "codec path must win over late refresh" + ); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + 4, + "codec path must use full metadata fanout and must not trigger a second late refresh" + ); + }, + ) + .await; + } + #[tokio::test] async fn repeated_gets_reuse_the_set_erasure_shell() { let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await; diff --git a/crates/ecstore/src/set_disk/read.rs b/crates/ecstore/src/set_disk/read.rs index 1c093abc3..2e8f58ead 100644 --- a/crates/ecstore/src/set_disk/read.rs +++ b/crates/ecstore/src/set_disk/read.rs @@ -128,7 +128,7 @@ use super::is_get_metadata_early_stop_bounded_fanout_enabled; #[cfg(test)] use super::is_get_metadata_early_stop_enabled; #[cfg(test)] -use super::is_get_metadata_two_phase_read_plan_enabled; +use super::is_get_metadata_non_inline_data_read_early_stop_enabled; #[cfg(test)] use super::is_version_early_stop_enabled; #[cfg(test)] @@ -609,7 +609,20 @@ impl SetDisks { // let online_disks: Vec> = op_online_disks.iter().filter(|v| v.is_some()).cloned().collect(); - Ok(GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks)) + if !metadata_fanout_complete + && allow_early_stop + && non_inline_data_read_early_stop_allowed(read_data, bucket, object) + && late_materialization_candidate_is_safe(&fi) + { + Ok(GetObjectFileInfo::owned_with_late_metadata_fanout( + fi, + parts_metadata, + op_online_disks, + disks, + )) + } else { + Ok(GetObjectFileInfo::owned(fi, parts_metadata, op_online_disks)) + } } #[hotpath::measure(impl_type = "SetDisks")] @@ -819,6 +832,7 @@ impl SetDisks { pool_index: usize, skip_verify_bitrot: bool, prefer_data_blocks_first_reader_setup: bool, + require_reconstruction_surplus: bool, metrics_path: &'static str, metrics_object_class: &'static str, metrics_size_bucket: &'static str, @@ -1083,6 +1097,9 @@ impl SetDisks { } let nil_count = reader_setup.available_shards(); + if require_reconstruction_surplus && nil_count <= erasure.data_shards { + return Err(Error::other("insufficient reconstruction surplus for two-phase read")); + } if nil_count < erasure.data_shards { if let Some(read_err) = reduce_read_quorum_errs(&reader_setup.errors, OBJECT_OP_IGNORED_ERRS, erasure.data_shards) { @@ -1190,18 +1207,34 @@ impl SetDisks { let readers = reader_setup.readers; let deferred_stripe_handles = reader_setup.deferred_stripe_handles; let deferred_reopeners = reader_setup.deferred_reopeners; - let (written, err) = erasure - .decode_with_stripe_handles_and_reopeners( - writer, - readers, - part_offset, - part_length, - part_size, - read_costs, - deferred_stripe_handles, - deferred_reopeners, - ) - .await; + let (written, err, exact_quorum) = if require_reconstruction_surplus { + erasure + .decode_with_stripe_handles_and_reopeners_with_diagnostics( + writer, + readers, + part_offset, + part_length, + part_size, + read_costs, + deferred_stripe_handles, + deferred_reopeners, + ) + .await + } else { + let (written, err) = erasure + .decode_with_stripe_handles_and_reopeners( + writer, + readers, + part_offset, + part_length, + part_size, + read_costs, + deferred_stripe_handles, + deferred_reopeners, + ) + .await; + (written, err, false) + }; let decode_elapsed = decode_stage_start.elapsed(); rustfs_io_metrics::record_get_object_decode_duration(decode_elapsed.as_secs_f64()); rustfs_io_metrics::record_get_object_stage_duration_by_size( @@ -1211,6 +1244,9 @@ impl SetDisks { metrics_size_bucket, decode_elapsed.as_secs_f64(), ); + if exact_quorum && err.is_none() { + return Err(Error::other("two-phase read completed with exact reconstruction quorum")); + } if decode_elapsed >= SLOW_OBJECT_READ_LOG_THRESHOLD && err.is_none() { warn!( event = EVENT_SET_DISK_READ, @@ -1758,6 +1794,102 @@ fn multipart_reader_setup_prefetch_enabled(policy: GetObjectReadPolicy) -> bool policy.allows_multipart_setup_prefetch() && is_multipart_reader_setup_prefetch_enabled() } +pub(super) struct LateMetadataIdentity { + volume: String, + name: String, + algorithm: String, + block_size: usize, + uses_legacy_checksum: bool, + quorum_hash: [u8; 32], + distribution: Vec, + parity_blocks: usize, +} + +impl LateMetadataIdentity { + pub(super) fn from_file_info(file_info: &FileInfo) -> Self { + Self { + volume: file_info.volume.clone(), + name: file_info.name.clone(), + algorithm: file_info.erasure.algorithm.clone(), + block_size: file_info.erasure.block_size, + uses_legacy_checksum: file_info.uses_legacy_checksum, + quorum_hash: SetDisks::file_info_quorum_hash(file_info), + distribution: file_info.erasure.distribution.clone(), + parity_blocks: file_info.erasure.parity_blocks, + } + } +} + +fn late_metadata_read_identity_matches(expected: &LateMetadataIdentity, actual: &FileInfo) -> bool { + expected.volume == actual.volume + && expected.name == actual.name + && expected.algorithm == actual.erasure.algorithm + && expected.block_size == actual.erasure.block_size + && expected.uses_legacy_checksum == actual.uses_legacy_checksum + && expected.quorum_hash == SetDisks::file_info_quorum_hash(actual) +} + +fn late_metadata_shard_matches(expected: &LateMetadataIdentity, actual: &FileInfo, disk_index: usize) -> bool { + expected + .distribution + .get(disk_index) + .is_some_and(|mapped_index| *mapped_index == actual.erasure.index) + && late_metadata_read_identity_matches(expected, actual) +} + +impl SetDisks { + pub(super) async fn refresh_late_metadata_fanout( + fallback_disks: &[Option], + bucket: &str, + object: &str, + expected: &LateMetadataIdentity, + metrics_path: &'static str, + ) -> Result<(FileInfo, Vec, Vec>)> { + let (mut parts_metadata, errs, diagnostics) = SetDisks::read_all_fileinfo_observed( + fallback_disks, + "", + bucket, + object, + "", + true, + false, + false, + false, + expected.parity_blocks, + ) + .await?; + diagnostics.record(metrics_path); + + let (read_quorum, write_quorum) = SetDisks::object_quorum_from_meta(&parts_metadata, &errs, expected.parity_blocks) + .map_err(|err| to_object_err(err.into(), vec![bucket, object]))?; + let read_quorum = + usize::try_from(read_quorum).map_err(|_| to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object]))?; + let write_quorum = usize::try_from(write_quorum) + .map_err(|_| to_object_err(DiskError::ErasureWriteQuorum.into(), vec![bucket, object]))?; + if let Some(err) = reduce_read_quorum_errs(&errs, OBJECT_OP_IGNORED_ERRS, read_quorum) { + return Err(to_object_err(err.into(), vec![bucket, object])); + } + + let (mut online_disks, full_fi, _) = + SetDisks::select_valid_fileinfo(fallback_disks, &parts_metadata, &errs, "", read_quorum, write_quorum)?; + if !late_metadata_read_identity_matches(expected, &full_fi) { + return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object])); + } + + for (disk_index, (metadata, disk)) in parts_metadata.iter_mut().zip(online_disks.iter_mut()).enumerate() { + if !late_metadata_shard_matches(expected, metadata, disk_index) { + *metadata = FileInfo::default(); + *disk = None; + } + } + if online_disks.iter().filter(|disk| disk.is_some()).count() < read_quorum { + return Err(to_object_err(DiskError::ErasureReadQuorum.into(), vec![bucket, object])); + } + + Ok((full_fi, parts_metadata, online_disks)) + } +} + /// Run one part's bitrot reader setup and measure its wall-clock duration. /// /// Shared by the synchronous path and the prefetch task in @@ -2349,6 +2481,7 @@ mod metadata_cache_tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "plain", "small", @@ -2380,6 +2513,7 @@ mod metadata_cache_tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "plain", "small", @@ -2404,6 +2538,7 @@ mod metadata_cache_tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "plain", "small", @@ -2426,6 +2561,7 @@ mod metadata_cache_tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "plain", "small", @@ -2450,6 +2586,7 @@ mod metadata_cache_tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "plain", "small", @@ -2488,6 +2625,7 @@ mod metadata_cache_tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "plain", "empty", @@ -2521,6 +2659,7 @@ mod metadata_cache_tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "plain", "small", @@ -4272,15 +4411,16 @@ mod tests { } #[test] - fn two_phase_read_plan_gate_defaults_off_and_honors_override() { + #[serial(body_cache_hook)] + fn non_inline_data_read_early_stop_gate_defaults_off_and_honors_override() { temp_env::with_var(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, None::<&str>, || { - assert!(!is_get_metadata_two_phase_read_plan_enabled()); + assert!(!is_get_metadata_non_inline_data_read_early_stop_enabled()); }); temp_env::with_var(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("true"), || { - assert!(is_get_metadata_two_phase_read_plan_enabled()); + assert!(is_get_metadata_non_inline_data_read_early_stop_enabled()); }); temp_env::with_var(ENV_RUSTFS_GET_METADATA_TWO_PHASE_READ_PLAN_ENABLE, Some("false"), || { - assert!(!is_get_metadata_two_phase_read_plan_enabled()); + assert!(!is_get_metadata_non_inline_data_read_early_stop_enabled()); }); } @@ -4881,6 +5021,7 @@ mod tests { 0, false, false, + false, GET_OBJECT_PATH_SET_DISK, "test-object-class", "test-size-bucket", @@ -5564,9 +5705,10 @@ mod tests { /// backlog#923: with the data-shards-only lockstep gate on, every retained /// parity reader must be an unopened deferred reader carrying a stripe - /// handle, so the decode path can realign it to a mid-object stripe. With - /// the gate off (default), eagerly opened parity readers are kept exactly - /// as before and carry no handles. + /// handle and disposable reopener, so the decode path can realign it to a + /// mid-object stripe without consuming the later-stripe reserve. With the + /// gate off (default), eagerly opened parity readers are kept exactly as + /// before and carry neither. #[tokio::test] #[serial_test::serial] async fn bitrot_reader_setup_gates_parity_stripe_handle_conversion() { @@ -5595,6 +5737,11 @@ mod tests { enabled.is_some(), "parity slot {idx} stripe handle must match the gate (enabled={enabled:?})" ); + assert_eq!( + setup.deferred_reopeners[idx].is_some(), + enabled.is_some(), + "parity slot {idx} reopener must match the gate (enabled={enabled:?})" + ); } if enabled.is_some() { @@ -5617,13 +5764,17 @@ mod tests { } #[tokio::test] + #[serial_test::serial] async fn bitrot_reader_setup_data_blocks_first_keeps_deferred_fallback_readers() { - let mut setup = setup_inline_bitrot_readers_with_env( - vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], - 2, - 2, - BitrotReaderSetupMode::ReadQuorum, - true, + let mut setup = temp_env::async_with_vars( + [("RUSTFS_GET_LOCKSTEP_DATA_SHARDS_ONLY_ENABLE", Some("true"))], + setup_inline_bitrot_readers_with_env( + vec![Some(b"aaaa"), Some(b"bbbb"), Some(b"cccc"), Some(b"dddd")], + 2, + 2, + BitrotReaderSetupMode::ReadQuorum, + true, + ), ) .await; @@ -5631,6 +5782,8 @@ mod tests { assert_eq!(setup.available_shards(), 2); assert_eq!(setup.scheduled_shards(), 2); assert_eq!(setup.readers.iter().filter(|reader| reader.is_some()).count(), 4); + assert!(setup.deferred_reopeners[2].is_some()); + assert!(setup.deferred_reopeners[3].is_some()); let fallback_index = setup .attempted diff --git a/crates/ecstore/src/store/init.rs b/crates/ecstore/src/store/init.rs index 7e0c24a1c..fdaaf6d88 100644 --- a/crates/ecstore/src/store/init.rs +++ b/crates/ecstore/src/store/init.rs @@ -17,18 +17,17 @@ use crate::core::pools::{ PoolMetaReplicaState, PoolMetaWriteState, load_pool_meta_identity_observing, local_decommission_queue_prefix, persist_pool_meta_identity_for_startup, pool_meta_has_active_decommission, }; -use crate::error::is_err_decommission_running; use crate::runtime::instance::InstanceContext; use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::object::EcstoreObjectIO; use rustfs_config::server_config::KVS; use rustfs_credentials::{RPC_SECRET_REQUIRED_OPERATOR_MESSAGE, try_get_rpc_token}; +use std::future::Future; use tracing::{debug, error, info, warn}; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_STORE_INIT: &str = "store_init"; const EVENT_DECOMMISSION_RESUME_RETRY: &str = "decommission_resume_retry"; -const EVENT_DECOMMISSION_RESUME_FAILED: &str = "decommission_resume_failed"; const EVENT_STORE_FORMAT_RETRY: &str = "store_format_retry"; const EVENT_ECSTORE_INIT_STATUS: &str = "ecstore_init_status"; const EVENT_STORE_RPC_SECRET_PREFLIGHT_FAILED: &str = "store_rpc_secret_preflight_failed"; @@ -96,16 +95,13 @@ fn preflight_startup_rpc_secret_with( } } -const LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES: usize = 6; const LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(60 * 3); const LOCAL_DECOMMISSION_RESUME_RETRY_DELAY: Duration = Duration::from_secs(30); +const LOCAL_DECOMMISSION_WATCHDOG_INTERVAL: Duration = Duration::from_secs(30); +const LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY: Duration = Duration::from_secs(60 * 5); const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10); const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10); -fn should_retry_local_decommission_resume(err: &Error, attempt: usize) -> bool { - matches!(err, Error::ConfigNotFound) && attempt < LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES -} - fn should_retry_format_load(err: &Error) -> bool { !matches!(err, Error::CorruptedFormat) } @@ -118,14 +114,6 @@ fn should_defer_rebalance_auto_start(distributed: bool, fleet_proof_available: b distributed && !fleet_proof_available } -fn should_schedule_local_decommission_resume( - pool_indices: &[usize], - pool_meta_replica_state: PoolMetaReplicaState, - pool_meta_write_safe: bool, -) -> bool { - !pool_indices.is_empty() && pool_meta_replica_state.repair_write_safe && pool_meta_write_safe -} - async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: Duration) -> bool { tokio::select! { _ = rx.cancelled() => false, @@ -133,6 +121,13 @@ async fn wait_for_local_decommission_resume_delay(rx: &CancellationToken, delay: } } +fn local_decommission_watchdog_retry_delay(consecutive_failures: u32) -> Duration { + let exponent = consecutive_failures.saturating_sub(1).min(4); + LOCAL_DECOMMISSION_RESUME_RETRY_DELAY + .saturating_mul(1_u32 << exponent) + .min(LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY) +} + async fn wait_for_rebalance_resume_delay(rx: &CancellationToken, delay: Duration) -> bool { tokio::select! { _ = rx.cancelled() => false, @@ -235,71 +230,63 @@ where Ok(committed) } -async fn resume_local_decommission_after_init(store: Arc, rx: CancellationToken, pool_indices: Vec) { - for attempt in 0..=LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES { +async fn run_local_decommission_watchdog(rx: CancellationToken, mut reconcile: F) +where + F: FnMut() -> Fut, + Fut: Future>, +{ + let mut consecutive_failures = 0_u32; + + loop { if rx.is_cancelled() { return; } - let result = if pool_indices.len() > 1 { - store - .spawn_decommission_routines(store.clone(), rx.clone(), pool_indices.clone()) - .await - } else { - store.decommission(rx.clone(), pool_indices.clone()).await - }; - - match result { - Ok(()) => return, - Err(err) if is_err_decommission_running(&err) => { - if let Err(spawn_err) = store - .spawn_decommission_routines(store.clone(), rx.clone(), pool_indices.clone()) - .await - { - error!( - event = EVENT_DECOMMISSION_RESUME_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_STORE_INIT, - pool_indices = ?pool_indices, - error = %spawn_err, - reason = "spawn_workers_failed", - "Failed to resume decommission workers" - ); - } - return; + let delay = match reconcile().await { + Ok(()) => { + consecutive_failures = 0; + LOCAL_DECOMMISSION_WATCHDOG_INTERVAL } - Err(err) if should_retry_local_decommission_resume(&err, attempt) => { + Err(err) => { + consecutive_failures = consecutive_failures.saturating_add(1); + let retry_delay = local_decommission_watchdog_retry_delay(consecutive_failures); warn!( event = EVENT_DECOMMISSION_RESUME_RETRY, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_STORE_INIT, - pool_indices = ?pool_indices, - retry_count = attempt + 1, - retry_limit = LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES + 1, + consecutive_failures, + retry_delay_secs = retry_delay.as_secs(), error = %err, - "Retrying decommission resume after missing config" + "Retrying decommission worker recovery" ); - tokio::select! { - _ = rx.cancelled() => return, - _ = tokio::time::sleep(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY) => {} - } - } - Err(err) => { - error!( - event = EVENT_DECOMMISSION_RESUME_FAILED, - component = LOG_COMPONENT_ECSTORE, - subsystem = LOG_SUBSYSTEM_STORE_INIT, - pool_indices = ?pool_indices, - error = %err, - reason = "resume_failed", - "Failed to resume decommission" - ); - return; + retry_delay } + }; + + if !wait_for_local_decommission_resume_delay(&rx, delay).await { + return; } } } +async fn supervise_local_decommission_after_init(store: Arc, rx: CancellationToken) { + run_local_decommission_watchdog(rx.clone(), || { + let store = store.clone(); + let worker_rx = rx.clone(); + async move { + store + .ensure_pool_meta_side_effects_safe("decommission worker recovery blocked while pool metadata requires recovery") + .await?; + if store.has_active_local_decommission_worker().await { + return Ok(()); + } + store.refresh_pool_status_meta().await?; + store.spawn_missing_local_decommission_routines_with_token(worker_rx).await + } + }) + .await; +} + async fn resume_rebalance_after_init(store: Arc, rx: CancellationToken) { if !wait_for_rebalance_resume_delay(&rx, REBALANCE_INITIAL_RESUME_DELAY).await { return; @@ -729,31 +716,32 @@ impl ECStore { } let local_pool_indices = local_decommission_queue_prefix(&endpoints, &pool_indices)?; + let has_local_decommission_leadership = endpoints.as_ref().iter().any(pool_first_endpoint_is_local); let pool_meta_write_safe = self .ensure_pool_meta_side_effects_safe("decommission resume blocked while pool metadata requires recovery") .await .is_ok(); - if should_schedule_local_decommission_resume(&local_pool_indices, pool_meta_replica_state, pool_meta_write_safe) { - let store = self.clone(); - let decommission_rx = rx.clone(); - - tokio::spawn(async move { - if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await { - return; - } - resume_local_decommission_after_init(store, decommission_rx, local_pool_indices).await; - }); - } else if !local_pool_indices.is_empty() { - error!( - event = EVENT_DECOMMISSION_RESUME_FAILED, + if !pool_meta_replica_state.repair_write_safe || !pool_meta_write_safe { + warn!( + event = EVENT_DECOMMISSION_RESUME_RETRY, component = LOG_COMPONENT_ECSTORE, subsystem = LOG_SUBSYSTEM_STORE_INIT, state = "blocked", pool_indices = ?local_pool_indices, reason = "pool_meta_write_blocked", - "Decommission resume blocked until pool metadata replicas are readable and consistent" + "Decommission watchdog waiting for pool metadata recovery" ); } + if has_local_decommission_leadership { + let store = self.clone(); + let decommission_rx = rx.clone(); + tokio::spawn(async move { + if !wait_for_local_decommission_resume_delay(&decommission_rx, LOCAL_DECOMMISSION_INITIAL_RESUME_DELAY).await { + return; + } + supervise_local_decommission_after_init(store, decommission_rx).await; + }); + } runtime_sources::init_bucket_monitor_for_current_endpoints(); crate::bucket::bucket_target_sys::BucketTargetSys::get().start_heartbeat(); @@ -786,12 +774,12 @@ impl ECStore { #[cfg(test)] mod tests { use super::{ - LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES, PoolMetaWriteState, establish_pool_meta_bootstrap_identity_if_proven, - load_pool_meta_for_startup, persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, - pool_meta_has_active_decommission, preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, - resolve_store_init_stage_result, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init, - should_defer_rebalance_auto_start, should_retry_format_load, should_retry_local_decommission_resume, - wait_for_local_decommission_resume_delay, + LOCAL_DECOMMISSION_RESUME_RETRY_DELAY, LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY, PoolMetaWriteState, + establish_pool_meta_bootstrap_identity_if_proven, load_pool_meta_for_startup, local_decommission_watchdog_retry_delay, + persist_pool_meta_for_startup_if_safe, pool_first_endpoint_is_local, pool_meta_has_active_decommission, + preflight_startup_rpc_secret_with, resolve_startup_pool_defaults_with, resolve_store_init_stage_result, + run_local_decommission_watchdog, save_validated_pool_meta_for_startup, should_auto_start_rebalance_after_init, + should_defer_rebalance_auto_start, should_retry_format_load, wait_for_local_decommission_resume_delay, }; #[cfg(feature = "test-util")] use crate::disk::DiskAPI; @@ -1481,15 +1469,6 @@ mod tests { assert!(err.to_string().contains("cannot overwrite an unreadable replica")); assert!(!valid.wrote_without_lock.load(Ordering::SeqCst)); assert!(!unreadable.wrote_without_lock.load(Ordering::SeqCst)); - assert!(!super::should_schedule_local_decommission_resume(&[0], replica_state, true)); - assert!(!super::should_schedule_local_decommission_resume( - &[0], - crate::core::pools::PoolMetaReplicaState { - needs_repair: false, - repair_write_safe: true, - }, - false, - )); } #[tokio::test] @@ -1517,7 +1496,6 @@ mod tests { .contains("restart after all replicas are readable and consistent") ); assert!(!repaired.wrote_without_lock.load(Ordering::SeqCst)); - assert!(!super::should_schedule_local_decommission_resume(&[0], replica_state, false)); } #[test] @@ -1551,21 +1529,66 @@ mod tests { } #[test] - fn test_should_retry_local_decommission_resume_accepts_config_not_found_before_retry_limit() { - assert!(should_retry_local_decommission_resume(&StorageError::ConfigNotFound, 0)); + fn test_local_decommission_watchdog_retry_delay_is_bounded() { + assert_eq!(local_decommission_watchdog_retry_delay(1), LOCAL_DECOMMISSION_RESUME_RETRY_DELAY); + assert_eq!( + local_decommission_watchdog_retry_delay(u32::MAX), + LOCAL_DECOMMISSION_WATCHDOG_MAX_RETRY_DELAY + ); } - #[test] - fn test_should_retry_local_decommission_resume_rejects_config_not_found_at_retry_limit() { - assert!(!should_retry_local_decommission_resume( - &StorageError::ConfigNotFound, - LOCAL_DECOMMISSION_RESUME_MAX_CONFIG_RETRIES - )); + #[tokio::test(start_paused = true)] + async fn test_local_decommission_watchdog_retries_general_failures_until_cancelled() { + let rx = CancellationToken::new(); + let attempts = Arc::new(AtomicUsize::new(0)); + let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), { + let attempts = attempts.clone(); + let rx = rx.clone(); + move || { + let attempts = attempts.clone(); + let rx = rx.clone(); + async move { + if attempts.fetch_add(1, Ordering::SeqCst) == 0 { + Err(StorageError::SlowDown) + } else { + rx.cancel(); + Ok(()) + } + } + } + })); + + tokio::task::yield_now().await; + assert_eq!(attempts.load(Ordering::SeqCst), 1); + tokio::time::advance(LOCAL_DECOMMISSION_RESUME_RETRY_DELAY).await; + task.await.expect("watchdog task should exit after cancellation"); + assert_eq!(attempts.load(Ordering::SeqCst), 2); } - #[test] - fn test_should_retry_local_decommission_resume_rejects_non_config_errors() { - assert!(!should_retry_local_decommission_resume(&StorageError::SlowDown, 0)); + #[tokio::test(start_paused = true)] + async fn test_local_decommission_watchdog_rescans_after_success() { + let rx = CancellationToken::new(); + let attempts = Arc::new(AtomicUsize::new(0)); + let task = tokio::spawn(run_local_decommission_watchdog(rx.clone(), { + let attempts = attempts.clone(); + let rx = rx.clone(); + move || { + let attempts = attempts.clone(); + let rx = rx.clone(); + async move { + if attempts.fetch_add(1, Ordering::SeqCst) == 1 { + rx.cancel(); + } + Ok(()) + } + } + })); + + tokio::task::yield_now().await; + assert_eq!(attempts.load(Ordering::SeqCst), 1); + tokio::time::advance(super::LOCAL_DECOMMISSION_WATCHDOG_INTERVAL).await; + task.await.expect("watchdog task should exit after cancellation"); + assert_eq!(attempts.load(Ordering::SeqCst), 2); } #[test] diff --git a/crates/ecstore/src/store/mod.rs b/crates/ecstore/src/store/mod.rs index c07a25fab..c3dd77605 100644 --- a/crates/ecstore/src/store/mod.rs +++ b/crates/ecstore/src/store/mod.rs @@ -44,7 +44,7 @@ use crate::error::{ use crate::runtime::global::DISK_RESERVE_FRACTION; use crate::runtime::instance::InstanceContext; use crate::runtime::sources as runtime_sources; -use crate::services::rebalance::{RebalanceMeta, is_rebalance_conflicting_with_decommission}; +use crate::services::rebalance::{RebalStatus, RebalanceMeta, is_rebalance_conflicting_with_decommission}; use crate::storage_api_contracts::{ bucket::{BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions}, list::{StorageListObjectVersionsInfo, StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions}, @@ -273,6 +273,215 @@ pub struct ECStore { pub(crate) bucket_fence_registry: Arc, } +const METRIC_SCANNER_DATA_MOVEMENT_PAUSED: &str = "rustfs_scanner_data_movement_paused"; +const METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_data_movement_pause_duration_seconds"; +const METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS: &str = "rustfs_scanner_data_movement_backlog_work_items"; +const SCANNER_DATA_MOVEMENT_PAUSE_POLICY: &str = "global_pause"; + +#[derive(Clone, Copy, Debug, PartialEq, Eq, serde::Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScannerDataMovementPauseReason { + OperationEpochExhausted, + MovementGenerationExhausted, + DecommissionActive, + DecommissionFailed, + DecommissionCanceled, + RebalanceActive, +} + +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize)] +pub struct ScannerDataMovementPauseStatus { + pub paused: bool, + pub policy: &'static str, + pub reasons: Vec, + pub started_at_unix_secs: u64, + pub duration_seconds: u64, + pub operation_epoch: u64, + pub movement_generation: u64, + pub movement_backlog_work_items: u64, + pub movement_backlog_estimated: bool, +} + +impl Default for ScannerDataMovementPauseStatus { + fn default() -> Self { + Self { + paused: false, + policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY, + reasons: Vec::new(), + started_at_unix_secs: 0, + duration_seconds: 0, + operation_epoch: 0, + movement_generation: 0, + movement_backlog_work_items: 0, + movement_backlog_estimated: false, + } + } +} + +fn offset_unix_seconds(value: OffsetDateTime) -> u64 { + u64::try_from(value.unix_timestamp()).unwrap_or(0) +} + +fn earliest_timestamp(current: Option, candidate: Option) -> Option { + match (current, candidate) { + (Some(current), Some(candidate)) => Some(current.min(candidate)), + (Some(current), None) => Some(current), + (None, candidate) => candidate, + } +} + +fn usize_to_u64(value: usize) -> u64 { + u64::try_from(value).unwrap_or(u64::MAX) +} + +fn metric_u64(value: u64) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +pub(crate) fn scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> u64 { + let timestamp = value.unix_timestamp_nanos(); + if timestamp <= 0 { + 0 + } else { + u64::try_from(timestamp).unwrap_or(u64::MAX) + } +} + +fn valid_scanner_data_movement_timestamp_generation(value: OffsetDateTime) -> Option { + let generation = scanner_data_movement_timestamp_generation(value); + (generation != 0 && generation != u64::MAX).then_some(generation) +} + +fn durable_scanner_data_movement_generation(pool_meta: &PoolMeta, rebalance_meta: Option<&RebalanceMeta>) -> u64 { + let mut generation = 0; + for pool in pool_meta.pools.iter().filter(|pool| pool.decommission.is_some()) { + let Some(pool_generation) = valid_scanner_data_movement_timestamp_generation(pool.last_update) else { + return u64::MAX; + }; + generation = generation.max(pool_generation); + } + + for movement_timestamp in rebalance_meta.into_iter().flat_map(|meta| { + meta.stopped_at.into_iter().chain( + meta.pool_stats + .iter() + .flat_map(|pool| [pool.info.start_time, pool.info.end_time]) + .flatten(), + ) + }) { + let Some(rebalance_generation) = valid_scanner_data_movement_timestamp_generation(movement_timestamp) else { + return u64::MAX; + }; + generation = generation.max(rebalance_generation); + } + + if generation == 0 + && rebalance_meta.is_some_and(|meta| !meta.id.is_empty() || !meta.pool_stats.is_empty() || meta.stopped_at.is_some()) + { + u64::MAX + } else { + generation + } +} + +#[derive(Clone, Copy)] +struct ScannerDataMovementSequenceState { + operation_epoch: u64, + operation_epoch_exhausted: bool, + movement_generation: u64, + movement_generation_exhausted: bool, +} + +fn resolve_scanner_data_movement_pause_status( + pool_meta: &PoolMeta, + rebalance_meta: Option<&RebalanceMeta>, + decommission_worker_active: bool, + sequence: ScannerDataMovementSequenceState, + now: OffsetDateTime, +) -> ScannerDataMovementPauseStatus { + let mut decommission_active = decommission_worker_active; + let mut decommission_failed = false; + let mut decommission_canceled = false; + let mut rebalance_active = false; + let mut started_at = None; + let mut movement_backlog_work_items = 0_u64; + + for pool in &pool_meta.pools { + let Some(info) = pool.decommission.as_ref() else { + continue; + }; + let active = info.has_decommission_state() && !info.complete && !info.failed && !info.canceled; + let failed = !info.queued && info.failed; + let canceled = !info.queued && info.canceled; + if !(active || failed || canceled) { + continue; + } + + decommission_active |= active; + decommission_failed |= failed; + decommission_canceled |= canceled; + started_at = earliest_timestamp(started_at, info.start_time.or(Some(pool.last_update))); + let queued = usize_to_u64(info.queued_buckets.len()); + let current_bucket = if info.bucket.is_empty() { 0 } else { 1 }; + movement_backlog_work_items = movement_backlog_work_items.saturating_add(queued.max(current_bucket)); + } + + if let Some(rebalance_meta) = rebalance_meta { + for pool in &rebalance_meta.pool_stats { + let active = (pool.participating && pool.info.status == RebalStatus::Started) || pool.info.stopping; + if !active { + continue; + } + rebalance_active = true; + started_at = earliest_timestamp(started_at, pool.info.start_time); + movement_backlog_work_items = movement_backlog_work_items.saturating_add(usize_to_u64(pool.buckets.len())); + } + } + + let mut reasons = Vec::with_capacity(6); + if sequence.operation_epoch_exhausted { + reasons.push(ScannerDataMovementPauseReason::OperationEpochExhausted); + } + if sequence.movement_generation_exhausted { + reasons.push(ScannerDataMovementPauseReason::MovementGenerationExhausted); + } + if decommission_active { + reasons.push(ScannerDataMovementPauseReason::DecommissionActive); + } + if decommission_failed { + reasons.push(ScannerDataMovementPauseReason::DecommissionFailed); + } + if decommission_canceled { + reasons.push(ScannerDataMovementPauseReason::DecommissionCanceled); + } + if rebalance_active { + reasons.push(ScannerDataMovementPauseReason::RebalanceActive); + } + let started_at_unix_secs = started_at.map(offset_unix_seconds).unwrap_or(0); + let duration_seconds = started_at + .and_then(|started_at| u64::try_from((now - started_at).whole_seconds()).ok()) + .unwrap_or(0); + let paused = !reasons.is_empty(); + + ScannerDataMovementPauseStatus { + paused, + policy: SCANNER_DATA_MOVEMENT_PAUSE_POLICY, + reasons, + started_at_unix_secs, + duration_seconds, + operation_epoch: sequence.operation_epoch, + movement_generation: sequence.movement_generation, + movement_backlog_work_items, + movement_backlog_estimated: paused, + } +} + +fn record_scanner_data_movement_pause_status(status: &ScannerDataMovementPauseStatus) { + metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSED).set(if status.paused { 1.0 } else { 0.0 }); + metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_PAUSE_DURATION_SECONDS).set(metric_u64(status.duration_seconds)); + metrics::gauge!(METRIC_SCANNER_DATA_MOVEMENT_BACKLOG_WORK_ITEMS).set(metric_u64(status.movement_backlog_work_items)); +} + impl std::fmt::Debug for ECStore { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let disk_slot_count: usize = self.disk_map.values().map(Vec::len).sum(); @@ -300,6 +509,28 @@ impl ECStore { self.pools.iter().flat_map(|pool| pool.disk_set.iter().cloned()).collect() } + /// Erasure sets that may receive scanner pause-backlog replicas. + /// + /// An actively decommissioning or already decommissioned source pool is + /// excluded so an operational record acknowledged during movement always + /// has a copy on storage that remains in the cluster. The record is kept + /// separate from pool and rebalance metadata. + pub async fn scanner_pause_backlog_writable_set_disks(&self) -> Vec> { + let pool_meta = self.pool_meta.read().await; + self.pools + .iter() + .enumerate() + .filter(|(pool_index, _)| { + !pool_meta.pools.get(*pool_index).is_some_and(|pool| { + pool.decommission + .as_ref() + .is_some_and(|info| info.has_decommission_state() && !info.failed && !info.canceled) + }) + }) + .flat_map(|(_, pool)| pool.disk_set.iter().cloned()) + .collect() + } + /// Get server configuration (delegates to global) pub fn get_server_config(&self) -> Option { runtime_sources::server_config() @@ -454,14 +685,14 @@ impl ECStore { self.scanner_data_usage_publication_snapshot_blocked().await } + pub async fn scanner_data_movement_pause_status(&self) -> ScannerDataMovementPauseStatus { + let operation_gate = self.ctx.data_movement_operation_gate(); + let _operation_guard = operation_gate.read_owned().await; + self.scanner_data_movement_pause_snapshot().await + } + async fn scanner_data_usage_publication_snapshot_blocked(&self) -> bool { - if self.ctx.data_movement_operation_epoch_exhausted() || self.ctx.data_movement_generation_exhausted() { - self.ctx.set_scanner_publication_state(true); - return true; - } - let (_, blocked) = self.scanner_data_movement_snapshot_locked().await; - self.ctx.set_scanner_publication_state(blocked); - blocked + self.scanner_data_movement_pause_snapshot().await.paused } async fn scanner_data_movement_snapshot_locked(&self) -> (bool, bool) { @@ -481,19 +712,56 @@ impl ECStore { .as_ref() .is_some_and(|info| !info.queued && (info.failed || info.canceled)) }); - drop(pool_meta); - - let rebalance_active = self - .rebalance_meta - .read() - .await + let rebalance_meta = self.rebalance_meta.read().await; + let rebalance_active = rebalance_meta .as_ref() .is_some_and(is_rebalance_conflicting_with_decommission); + self.ctx + .observe_durable_data_movement_generation(durable_scanner_data_movement_generation( + &pool_meta, + rebalance_meta.as_ref(), + )); let blocked = decommission_active || decommission_terminal || rebalance_active; (decommission_active || rebalance_active, blocked) } + async fn scanner_data_movement_pause_snapshot(&self) -> ScannerDataMovementPauseStatus { + let decommission_active = { + let decommission_cancelers = self.decommission_cancelers.read().await; + decommission_cancelers + .iter() + .any(|canceler| canceler.as_ref().is_some_and(DecommissionCanceler::is_active)) + }; + let pool_meta = self.pool_meta.read().await.clone(); + let rebalance_meta = self.rebalance_meta.read().await.clone(); + self.ctx + .observe_durable_data_movement_generation(durable_scanner_data_movement_generation( + &pool_meta, + rebalance_meta.as_ref(), + )); + let status = resolve_scanner_data_movement_pause_status( + &pool_meta, + rebalance_meta.as_ref(), + decommission_active, + ScannerDataMovementSequenceState { + operation_epoch: self.ctx.data_movement_operation_epoch(), + operation_epoch_exhausted: self.ctx.data_movement_operation_epoch_exhausted(), + movement_generation: self.ctx.data_movement_generation(), + movement_generation_exhausted: self.ctx.data_movement_generation_exhausted(), + }, + OffsetDateTime::now_utc(), + ); + self.ctx.set_scanner_publication_state(status.paused); + record_scanner_data_movement_pause_status(&status); + status + } + + #[cfg(test)] + pub(crate) async fn scanner_data_movement_pause_snapshot_for_test(&self) -> ScannerDataMovementPauseStatus { + self.scanner_data_movement_pause_snapshot().await + } + /// Admit one short data-usage publication commit under the same /// per-instance gate used by decommission side effects and transitions. /// The epoch is sampled while the read guard is held, so a transition @@ -1196,7 +1464,7 @@ impl crate::storage_api_contracts::admin::StorageAdminApi for ECStore { #[cfg(test)] mod tests { use super::*; - use crate::core::pools::{PoolDecommissionInfo, PoolStatus}; + use crate::core::pools::{PoolDecommissionInfo, PoolSpaceInfo, PoolStatus}; use crate::layout::endpoints::{Endpoints, PoolEndpoints, SetupType}; use crate::object_api::ObjectOptions; use crate::runtime::global::reset_local_disk_test_state; @@ -1205,6 +1473,23 @@ mod tests { use serial_test::serial; use tempfile::TempDir; + #[test] + fn g_d2_008_default_versioning_config_keeps_persisted_bytes() { + let bytes = crate::bucket::utils::serialize::(&ENABLED_VERSIONING_CONFIG) + .expect("the default Versioning configuration must serialize"); + assert_eq!(bytes, b"Enabled"); + } + + #[test] + fn g_d2_009_default_object_lock_config_keeps_persisted_bytes() { + let bytes = crate::bucket::utils::serialize::(&ENABLED_OBJECT_LOCK_CONFIG) + .expect("the default Object Lock configuration must serialize"); + assert_eq!( + bytes, + b"Enabled" + ); + } + #[tokio::test] async fn test_get_disk_infos() { let disks = vec![None, None]; // Empty disks for testing @@ -1309,6 +1594,558 @@ mod tests { }) } + fn scanner_sequence_state(operation_epoch: u64, movement_generation: u64) -> ScannerDataMovementSequenceState { + ScannerDataMovementSequenceState { + operation_epoch, + operation_epoch_exhausted: false, + movement_generation, + movement_generation_exhausted: false, + } + } + + #[test] + fn scanner_pause_status_derives_restart_stable_decommission_fields() { + let started_at = OffsetDateTime::from_unix_timestamp(1_000).expect("fixed timestamp should be valid"); + let now = OffsetDateTime::from_unix_timestamp(1_090).expect("fixed timestamp should be valid"); + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: started_at, + decommission: Some(PoolDecommissionInfo { + start_time: Some(started_at), + queued_buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + bucket: "bucket-a".to_string(), + ..Default::default() + }), + }], + ..Default::default() + }; + + let status = resolve_scanner_data_movement_pause_status(&pool_meta, None, false, scanner_sequence_state(7, 11), now); + + assert!(status.paused); + assert_eq!(status.policy, "global_pause"); + assert_eq!(status.reasons, vec![ScannerDataMovementPauseReason::DecommissionActive]); + assert_eq!(status.started_at_unix_secs, 1_000); + assert_eq!(status.duration_seconds, 90); + assert_eq!(status.operation_epoch, 7); + assert_eq!(status.movement_generation, 11); + assert_eq!(status.movement_backlog_work_items, 2); + assert!(status.movement_backlog_estimated); + } + + #[test] + fn completed_decommission_restores_durable_movement_generation() { + let completed_at = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid"); + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: completed_at, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None); + let ctx = InstanceContext::new(); + + ctx.observe_durable_data_movement_generation(durable_generation); + + assert_eq!(durable_generation, 1_100_000_000_000); + assert_eq!(ctx.data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn cleared_decommission_restores_durable_movement_generation_after_restart() { + let mut pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }], + ..Default::default() + }; + assert!(pool_meta.clear_decommission(0).expect("failed decommission should clear")); + assert!( + pool_meta.pools[0] + .decommission + .as_ref() + .is_some_and(|info| !info.has_decommission_state()) + ); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, None); + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + + let status = restarted.scanner_data_movement_pause_status().await; + + assert_ne!(durable_generation, 0); + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn same_tick_cleared_decommission_tombstones_advance_durable_movement_generation() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_100).expect("fixed timestamp should be valid"); + let mut pool_meta = PoolMeta { + pools: vec![ + PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }, + PoolStatus { + id: 1, + cmd_line: "pool-1".to_string(), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + canceled: true, + ..Default::default() + }), + }, + ], + ..Default::default() + }; + + assert!( + pool_meta + .clear_decommission_at_for_test(0, same_tick, None) + .expect("first terminal decommission should clear") + ); + let first_generation = durable_scanner_data_movement_generation(&pool_meta, None); + assert_eq!( + first_generation, + scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(1)) + ); + + assert!( + pool_meta + .clear_decommission_at_for_test(1, same_tick, None) + .expect("second terminal decommission should clear") + ); + let second_generation = durable_scanner_data_movement_generation(&pool_meta, None); + assert_eq!( + second_generation, + scanner_data_movement_timestamp_generation(same_tick + time::Duration::nanoseconds(2)) + ); + assert!(second_generation > first_generation); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, second_generation); + assert_eq!(restarted.scanner_data_movement_generation(), second_generation); + } + + #[tokio::test] + async fn terminal_decommission_transitions_advance_durable_generation_across_same_or_earlier_clocks() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_200).expect("fixed timestamp should be valid"); + let earlier_tick = same_tick - time::Duration::nanoseconds(10); + let rebalance_floor = same_tick + time::Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let active_decommission = |id| PoolStatus { + id, + cmd_line: format!("pool-{id}"), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + start_time: Some(same_tick), + ..Default::default() + }), + }; + let mut pool_meta = PoolMeta { + pools: vec![active_decommission(0), active_decommission(1), active_decommission(2)], + ..Default::default() + }; + + assert!(pool_meta.decommission_complete_at_for_test(0, same_tick, Some(&rebalance))); + assert_eq!(pool_meta.pools[0].last_update, rebalance_floor + time::Duration::nanoseconds(1)); + + assert!(pool_meta.decommission_cancel_at_for_test(1, same_tick, Some(&rebalance))); + assert_eq!(pool_meta.pools[1].last_update, rebalance_floor + time::Duration::nanoseconds(2)); + + assert!(pool_meta.decommission_failed_at_for_test(2, earlier_tick, Some(&rebalance))); + assert_eq!(pool_meta.pools[2].last_update, rebalance_floor + time::Duration::nanoseconds(3)); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance)); + assert_eq!( + durable_generation, + scanner_data_movement_timestamp_generation(rebalance_floor + time::Duration::nanoseconds(3)) + ); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = Some(rebalance); + let status = restarted.scanner_data_movement_pause_status().await; + + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + assert_eq!( + status.reasons, + vec![ + ScannerDataMovementPauseReason::DecommissionFailed, + ScannerDataMovementPauseReason::DecommissionCanceled + ] + ); + } + + #[tokio::test] + async fn decommission_start_after_clear_advances_durable_generation_across_clock_rollback_after_restart() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_250).expect("fixed timestamp should be valid"); + let earlier_tick = same_tick - time::Duration::nanoseconds(10); + let rebalance_floor = same_tick + time::Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let mut pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: same_tick, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!( + pool_meta + .clear_decommission_at_for_test(0, same_tick, Some(&rebalance)) + .expect("failed decommission should clear") + ); + let cleared_at = rebalance_floor + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, cleared_at); + + pool_meta + .decommission_at_for_test( + 0, + PoolSpaceInfo { + total: 200, + free: 50, + used: 150, + }, + earlier_tick, + Some(&rebalance), + ) + .expect("decommission restart after clear should be allowed"); + let started_at = cleared_at + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, started_at); + assert_eq!( + pool_meta.pools[0].decommission.as_ref().and_then(|info| info.start_time), + Some(started_at) + ); + + assert!(pool_meta.decommission_complete_at_for_test(0, earlier_tick, Some(&rebalance))); + let completed_at = started_at + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, completed_at); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance)); + assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(completed_at)); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = Some(rebalance); + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn decommission_terminal_reload_failure_advances_durable_generation_across_clock_rollback_after_restart() { + let terminal_at = OffsetDateTime::from_unix_timestamp(1_280).expect("fixed timestamp should be valid"); + let earlier_tick = terminal_at - time::Duration::nanoseconds(10); + let rebalance_floor = terminal_at + time::Duration::nanoseconds(5); + let rebalance = RebalanceMeta { + stopped_at: Some(rebalance_floor), + id: "completed-rebalance".to_string(), + ..Default::default() + }; + let mut pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: terminal_at, + decommission: Some(PoolDecommissionInfo { + start_time: Some(terminal_at), + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert!( + pool_meta + .record_decommission_terminal_reload_failure_at_for_test( + 0, + "complete_decommission", + "peer reload failed".to_string(), + earlier_tick, + Some(&rebalance), + ) + .expect("reload failure should be recorded") + ); + let reload_failure_at = rebalance_floor + time::Duration::nanoseconds(1); + assert_eq!(pool_meta.pools[0].last_update, reload_failure_at); + let info = pool_meta.pools[0] + .decommission + .as_ref() + .expect("decommission metadata should exist"); + assert_eq!(info.terminal_reload_attempt_at, Some(reload_failure_at)); + assert_eq!( + info.terminal_reload_failures, + vec!["complete_decommission: peer reload failed".to_string()] + ); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance)); + assert_eq!(durable_generation, scanner_data_movement_timestamp_generation(reload_failure_at)); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = Some(rebalance); + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[tokio::test] + async fn rebalance_transitions_advance_durable_generation_across_same_or_earlier_clocks_after_restart() { + let same_tick = OffsetDateTime::from_unix_timestamp(1_300).expect("fixed timestamp should be valid"); + let earlier_tick = same_tick - time::Duration::nanoseconds(10); + let decommission_floor = same_tick + time::Duration::nanoseconds(5); + let store = build_store_with_ctx(Arc::new(InstanceContext::new())); + *store.pool_meta.write().await = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: decommission_floor, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + let started_at = store.next_scanner_data_movement_update(same_tick).await; + assert_eq!(started_at, decommission_floor + time::Duration::nanoseconds(1)); + *store.rebalance_meta.write().await = Some(RebalanceMeta { + id: "rebalance-generation".to_string(), + pool_stats: vec![crate::services::rebalance::RebalanceStats { + participating: true, + info: crate::services::rebalance::RebalanceInfo { + start_time: Some(started_at), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }); + + let completed_at = store.next_scanner_data_movement_update(same_tick).await; + assert_eq!(completed_at, decommission_floor + time::Duration::nanoseconds(2)); + { + let mut rebalance_meta = store.rebalance_meta.write().await; + let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present"); + meta.pool_stats[0].info.status = RebalStatus::Completed; + meta.pool_stats[0].info.end_time = Some(completed_at); + } + + let stopped_at = store.next_scanner_data_movement_update(earlier_tick).await; + assert_eq!(stopped_at, decommission_floor + time::Duration::nanoseconds(3)); + { + let mut rebalance_meta = store.rebalance_meta.write().await; + let meta = rebalance_meta.as_mut().expect("rebalance metadata should be present"); + meta.stopped_at = Some(stopped_at); + } + let pool_meta = store.pool_meta.read().await.clone(); + let rebalance_meta = store.rebalance_meta.read().await.clone(); + let durable_generation = durable_scanner_data_movement_generation(&pool_meta, rebalance_meta.as_ref()); + assert_eq!( + durable_generation, + scanner_data_movement_timestamp_generation(decommission_floor + time::Duration::nanoseconds(3)) + ); + + let restarted = build_store_with_ctx(Arc::new(InstanceContext::new())); + *restarted.pool_meta.write().await = pool_meta; + *restarted.rebalance_meta.write().await = rebalance_meta; + let status = restarted.scanner_data_movement_pause_status().await; + + assert!(!status.paused); + assert_eq!(status.movement_generation, durable_generation); + assert_eq!(restarted.scanner_data_movement_generation(), durable_generation); + } + + #[test] + fn malformed_durable_movement_timestamp_exhausts_generation_fail_closed() { + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + + assert_eq!(durable_scanner_data_movement_generation(&pool_meta, None), u64::MAX); + let exhausted_generation = + OffsetDateTime::from_unix_timestamp(253_402_300_799).expect("the largest RFC 3339 timestamp should be valid"); + assert_eq!(scanner_data_movement_timestamp_generation(exhausted_generation), u64::MAX); + } + + #[test] + fn malformed_durable_movement_timestamp_is_not_masked_by_valid_rebalance_generation() { + let valid_rebalance_at = OffsetDateTime::from_unix_timestamp(2_400).expect("fixed timestamp should be valid"); + let pool_meta = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update: OffsetDateTime::UNIX_EPOCH, + decommission: Some(PoolDecommissionInfo { + complete: true, + ..Default::default() + }), + }], + ..Default::default() + }; + let rebalance_meta = RebalanceMeta { + id: "completed-rebalance".to_string(), + stopped_at: Some(valid_rebalance_at), + pool_stats: vec![crate::services::rebalance::RebalanceStats { + participating: true, + info: crate::services::rebalance::RebalanceInfo { + start_time: Some(valid_rebalance_at - time::Duration::nanoseconds(1)), + end_time: Some(valid_rebalance_at), + status: RebalStatus::Completed, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + + assert_eq!(durable_scanner_data_movement_generation(&pool_meta, Some(&rebalance_meta)), u64::MAX); + } + + #[test] + fn durable_movement_generation_without_records_is_zero() { + assert_eq!(durable_scanner_data_movement_generation(&PoolMeta::default(), None), 0); + assert_eq!( + durable_scanner_data_movement_generation(&PoolMeta::default(), Some(&RebalanceMeta::default())), + 0 + ); + } + + #[test] + fn scanner_pause_status_distinguishes_terminal_rebalance_epoch_and_idle() { + let last_update = OffsetDateTime::from_unix_timestamp(2_000).expect("fixed timestamp should be valid"); + let now = OffsetDateTime::from_unix_timestamp(2_030).expect("fixed timestamp should be valid"); + let failed = PoolMeta { + pools: vec![PoolStatus { + id: 0, + cmd_line: "pool-0".to_string(), + last_update, + decommission: Some(PoolDecommissionInfo { + failed: true, + ..Default::default() + }), + }], + ..Default::default() + }; + let failed_status = resolve_scanner_data_movement_pause_status(&failed, None, false, scanner_sequence_state(3, 12), now); + assert_eq!(failed_status.reasons, vec![ScannerDataMovementPauseReason::DecommissionFailed]); + assert_eq!(failed_status.started_at_unix_secs, 2_000); + assert_eq!(failed_status.duration_seconds, 30); + + let rebalance = RebalanceMeta { + pool_stats: vec![crate::services::rebalance::RebalanceStats { + buckets: vec!["bucket-a".to_string(), "bucket-b".to_string()], + participating: true, + info: crate::services::rebalance::RebalanceInfo { + start_time: Some(last_update), + status: RebalStatus::Started, + ..Default::default() + }, + ..Default::default() + }], + ..Default::default() + }; + let rebalance_status = resolve_scanner_data_movement_pause_status( + &PoolMeta::default(), + Some(&rebalance), + false, + scanner_sequence_state(4, 13), + now, + ); + assert_eq!(rebalance_status.reasons, vec![ScannerDataMovementPauseReason::RebalanceActive]); + assert_eq!(rebalance_status.movement_backlog_work_items, 2); + + let exhausted = resolve_scanner_data_movement_pause_status( + &PoolMeta::default(), + None, + false, + ScannerDataMovementSequenceState { + operation_epoch: u64::MAX, + operation_epoch_exhausted: true, + movement_generation: 14, + movement_generation_exhausted: false, + }, + now, + ); + assert_eq!(exhausted.reasons, vec![ScannerDataMovementPauseReason::OperationEpochExhausted]); + assert_eq!(exhausted.started_at_unix_secs, 0); + + let generation_exhausted = resolve_scanner_data_movement_pause_status( + &PoolMeta::default(), + None, + false, + ScannerDataMovementSequenceState { + operation_epoch: 5, + operation_epoch_exhausted: false, + movement_generation: u64::MAX, + movement_generation_exhausted: true, + }, + now, + ); + assert_eq!( + generation_exhausted.reasons, + vec![ScannerDataMovementPauseReason::MovementGenerationExhausted] + ); + + let idle = + resolve_scanner_data_movement_pause_status(&PoolMeta::default(), None, false, scanner_sequence_state(5, 15), now); + assert!(!idle.paused); + assert!(idle.reasons.is_empty()); + assert!(!idle.movement_backlog_estimated); + } + #[tokio::test] async fn scanner_data_usage_publication_blocks_active_and_unqueued_terminal_decommission() { let store = build_store_with_ctx(Arc::new(InstanceContext::new())); diff --git a/crates/ecstore/src/store/object.rs b/crates/ecstore/src/store/object.rs index 6e3ce4214..2f86d94f3 100644 --- a/crates/ecstore/src/store/object.rs +++ b/crates/ecstore/src/store/object.rs @@ -884,8 +884,7 @@ impl AsyncRead for SelectObjectSnapshotReader { } let filled_before = buf.filled().len(); let poll = Pin::new(&mut self.inner).poll_read(cx, buf); - let reached_eof = matches!(&poll, Poll::Ready(Ok(()))) && buf.filled().len() == filled_before; - if self.lease.is_lost() || (reached_eof && self.lease.check().is_err()) { + if self.lease.check().is_err() { buf.set_filled(filled_before); return Poll::Ready(Err(std::io::Error::other(SnapshotConsistencyError::LockLost))); } @@ -1757,15 +1756,22 @@ impl ECStore { return Err(SnapshotConsistencyError::LockLost.into()); } - let pool = if self.single_pool() { - Arc::clone(&self.pools[0]) + let (mut metadata, pool) = if self.single_pool() { + let pool = Arc::clone(&self.pools[0]); + let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?; + (metadata, pool) } else { - let (_, pool_idx) = self.get_latest_object_info_with_idx(bucket, &object, &opts).await?; - self.pools.get(pool_idx).cloned().ok_or_else(|| { - StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds")) - })? + // Keep the large multi-pool selection future off the caller stack. + // Debug builds otherwise exceed the common 2 MiB worker stack. + Box::pin(async { + let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?; + let pool = self.pools.get(pool_idx).cloned().ok_or_else(|| { + StorageError::other(format!("resolved SelectObjectContent pool index {pool_idx} is out of bounds")) + })?; + Ok::<_, StorageError>((metadata, pool)) + }) + .await? }; - let mut metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?; if read_lock_guards.iter().any(ObjectLockDiagGuard::is_lock_lost) { return Err(SnapshotConsistencyError::LockLost.into()); } @@ -1817,16 +1823,21 @@ impl ECStore { let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?; (metadata, pool) } else { - let (_, pool_idx) = self - .get_latest_accessible_object_info_with_idx(bucket, &object, &opts) - .await?; - let pool = self - .pools - .get(pool_idx) - .cloned() - .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?; - let metadata = pool.prepare_get_object_reader_metadata(bucket, &object, &opts).await?; - (metadata, pool) + // Keep the large multi-pool selection future off the caller stack. + // Debug builds otherwise exceed the common 2 MiB worker stack. + Box::pin(async { + let (metadata, pool_idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?; + if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) { + return Err(error); + } + let pool = self + .pools + .get(pool_idx) + .cloned() + .ok_or_else(|| Error::other(format!("resolved GET pool index {pool_idx} is out of bounds")))?; + Ok((metadata, pool)) + }) + .await? }; Ok(PreparedGetObjectReader { @@ -2518,12 +2529,18 @@ impl ECStore { .get_object_reader(bucket, object.as_ref(), range, h, &opts) .await? } else { - let (_, idx) = self - .get_latest_accessible_object_info_with_idx(bucket, &object, &opts) - .await?; - self.pools[idx] - .get_object_reader(bucket, object.as_ref(), range, h, &opts) - .await? + // Keep selection plus prepared-open state off the caller stack. + // Debug builds otherwise exceed the common 2 MiB worker stack. + Box::pin(async { + let (metadata, idx) = self.prepare_latest_object_metadata_with_idx(bucket, &object, &opts).await?; + if let Some(error) = latest_object_access_delete_marker_error(bucket, &object, metadata.object_info(), &opts) { + return Err(error); + } + self.pools[idx] + .get_object_reader_with_prepared_metadata(bucket, object.as_ref(), range, h, &opts, metadata) + .await + }) + .await? }; Ok(Self::attach_read_lock_guard(reader, read_lock_guard)) @@ -3914,8 +3931,9 @@ mod tests { ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_state_to_filemeta, replication_statuses_map, version_purge_statuses_map, }; + use crate::core::pools::{PoolDecommissionInfo, PoolStatus}; use crate::core::sets::make_local_two_set_sets_with_ctx; - use crate::ecstore_validation_blackbox::{make_local_set_disks, make_local_set_disks_with_ctx}; + use crate::ecstore_validation_blackbox::{RefreshLossLockClient, make_local_set_disks, make_local_set_disks_with_ctx}; use crate::layout::{ endpoints::{Endpoints, PoolEndpoints, SetupType}, format::FormatV3, @@ -3930,7 +3948,7 @@ mod tests { use bytes::Bytes; use std::io::Cursor; use std::sync::Arc; - use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::io::AsyncReadExt; struct WaitForLockLossReader { @@ -3972,68 +3990,17 @@ mod tests { calls: AtomicUsize, } - #[derive(Debug)] - struct RefreshFailureLockClient { - inner: LocalClient, - fail_refresh: AtomicBool, - } - - #[async_trait::async_trait] - impl rustfs_lock::LockClient for RefreshFailureLockClient { - async fn acquire_lock(&self, request: &rustfs_lock::LockRequest) -> rustfs_lock::Result { - rustfs_lock::LockClient::acquire_lock(&self.inner, request).await - } - - async fn release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { - rustfs_lock::LockClient::release(&self.inner, lock_id).await - } - - async fn refresh(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { - if self.fail_refresh.load(Ordering::Acquire) { - return Ok(false); - } - rustfs_lock::LockClient::refresh(&self.inner, lock_id).await - } - - async fn force_release(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result { - rustfs_lock::LockClient::force_release(&self.inner, lock_id).await - } - - async fn check_status(&self, lock_id: &rustfs_lock::LockId) -> rustfs_lock::Result> { - rustfs_lock::LockClient::check_status(&self.inner, lock_id).await - } - - async fn get_stats(&self) -> rustfs_lock::Result { - rustfs_lock::LockClient::get_stats(&self.inner).await - } - - async fn close(&self) -> rustfs_lock::Result<()> { - rustfs_lock::LockClient::close(&self.inner).await - } - - async fn is_online(&self) -> bool { - rustfs_lock::LockClient::is_online(&self.inner).await - } - - async fn is_local(&self) -> bool { - rustfs_lock::LockClient::is_local(&self.inner).await - } - } - async fn refresh_failure_test_guard( owner: &'static str, ) -> ( ObjectLockDiagGuard, Arc, - Arc, + Arc, ) { let manager = Arc::new(rustfs_lock::GlobalLockManager::Enabled(Arc::new( rustfs_lock::FastObjectLockManager::new(), ))); - let client = Arc::new(RefreshFailureLockClient { - inner: LocalClient::with_manager(manager), - fail_refresh: AtomicBool::new(false), - }); + let client = Arc::new(RefreshLossLockClient::with_manager(manager)); let namespace_lock = rustfs_lock::NamespaceLock::with_clients_and_quorum( owner.to_string(), vec![Arc::clone(&client) as Arc], @@ -4068,7 +4035,7 @@ mod tests { ) -> ( Arc, Arc, - Arc, + Arc, ) { let (guard, signal, client) = refresh_failure_test_guard(owner).await; (Arc::new(SelectObjectSnapshotLease::new(vec![guard])), signal, client) @@ -4201,7 +4168,11 @@ mod tests { let release_signal = Arc::clone(&signal); let release_task = tokio::spawn(async move { poll_started_rx.await.expect("reader poll should start"); - release_client.fail_refresh.store(true, Ordering::Release); + release_client.reject_refreshes(); + release_client + .wait_for_rejected_refresh(Duration::from_secs(5)) + .await + .expect("refresh rejection should be observed"); tokio::time::timeout(Duration::from_secs(5), release_signal.notified()) .await .expect("heartbeat should observe the rejected refresh"); @@ -4237,9 +4208,13 @@ mod tests { } #[tokio::test(flavor = "current_thread")] - async fn select_snapshot_reader_checks_guards_at_eof_before_monitor_runs() { + async fn select_snapshot_reader_checks_guards_before_monitor_runs() { let (guard, signal, client) = refresh_failure_test_guard("select-snapshot-eof-fence").await; - client.fail_refresh.store(true, Ordering::Release); + client.reject_refreshes(); + client + .wait_for_rejected_refresh(Duration::from_secs(5)) + .await + .expect("refresh rejection should be observed"); tokio::time::timeout(Duration::from_secs(5), signal.notified()) .await .expect("heartbeat should observe the rejected refresh"); @@ -4258,7 +4233,7 @@ mod tests { .await .expect_err("EOF fence must reject a lease lost before its monitor is scheduled"); - assert_eq!(output, b"old-generation"); + assert!(output.is_empty(), "bytes from a known-lost snapshot must not escape"); assert_eq!(error.kind(), std::io::ErrorKind::Other); } @@ -4304,7 +4279,11 @@ mod tests { second_started_rx .await .expect("second inner reader should reach Poll::Pending"); - second_client.fail_refresh.store(true, Ordering::Release); + second_client.reject_refreshes(); + second_client + .wait_for_rejected_refresh(Duration::from_secs(5)) + .await + .expect("refresh rejection should be observed"); let (first_result, second_result) = tokio::join!( tokio::time::timeout(Duration::from_secs(5), first_read_task), tokio::time::timeout(Duration::from_secs(5), second_read_task), @@ -4317,7 +4296,7 @@ mod tests { assert_eq!(error.kind(), std::io::ErrorKind::Other); } - assert!(!first_client.fail_refresh.load(Ordering::Acquire)); + assert!(!first_client.refreshes_rejected()); assert!(!first_signal.is_lost()); assert!(second_signal.is_lost()); } @@ -6085,13 +6064,112 @@ mod tests { #[tokio::test] #[serial_test::serial(body_cache_hook)] - async fn prepared_reader_resolves_object_from_second_pool() { + async fn prepared_reader_reuses_metadata_across_three_pools() { + let (_first_dirs, first_set) = make_local_set_disks(4, 2).await; + let (_second_dirs, second_set) = make_local_set_disks(4, 2).await; + let (_third_dirs, third_set) = make_local_set_disks(4, 2).await; + let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await; + let bucket = "prepared-reader-three-pools"; + let object = "object.bin"; + let payload = b"prepared-reader-three-pool-payload-".repeat(40_000); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + for pool in &store.pools { + pool.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created in each pool"); + } + let mut put_reader = PutObjReader::from_vec(payload.clone()); + store.pools[2] + .put_object(bucket, object, &mut put_reader, &opts) + .await + .expect("object should be written only to the third pool"); + + let calls = disk_call_counters::observe(object); + let prepared = store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + .await + .expect("prepared reader should resolve the third-pool object"); + assert_eq!(prepared.object_info().size, payload.len() as i64); + let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION); + assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each"); + let mut reader = prepared.into_reader().await.expect("prepared body reader should open"); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + metadata_calls, + "the selected pool must reuse its prepared metadata" + ); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("prepared body should stream"); + assert_eq!(restored, payload); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn select_snapshot_reuses_metadata_across_three_pools() { + let (_first_dirs, first_set) = make_local_set_disks(4, 2).await; + let (_second_dirs, second_set) = make_local_set_disks(4, 2).await; + let (_third_dirs, third_set) = make_local_set_disks(4, 2).await; + let store = new_prepared_reader_test_store(&[first_set, second_set, third_set]).await; + let bucket = "select-snapshot-three-pools"; + let object = "object.bin"; + let payload = b"select-snapshot-three-pool-payload-".repeat(40_000); + let write_opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + for pool in &store.pools { + pool.make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created in each pool"); + } + let mut put_reader = PutObjReader::from_vec(payload.clone()); + store.pools[2] + .put_object(bucket, object, &mut put_reader, &write_opts) + .await + .expect("object should be written only to the third pool"); + + let calls = disk_call_counters::observe(object); + let snapshot = store + .prepare_select_object_snapshot(bucket, object, &HeaderMap::new(), &ObjectOptions::default()) + .await + .expect("SelectObjectContent snapshot should resolve the third-pool object"); + assert_eq!(snapshot.object_info().size, payload.len() as i64); + let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION); + assert_eq!(metadata_calls, 12, "three 4-disk pools must fan out metadata exactly once each"); + + let mut reader = snapshot.open_reader(None).await.expect("snapshot body reader should open"); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + metadata_calls, + "SelectObjectContent must consume the prepared winner without a second fanout" + ); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("snapshot body should stream"); + assert_eq!(restored, payload); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn legacy_reader_reuses_selected_pool_metadata() { let (_first_dirs, first_set) = make_local_set_disks(4, 2).await; let (_second_dirs, second_set) = make_local_set_disks(4, 2).await; let store = new_prepared_reader_test_store(&[first_set, second_set]).await; - let bucket = "prepared-reader-second-pool"; + let bucket = "legacy-reader-second-pool"; let object = "object.bin"; - let payload = b"prepared-reader-second-pool-payload-".repeat(40_000); + let payload = b"legacy-reader-second-pool-payload-".repeat(40_000); let opts = ObjectOptions { no_lock: true, ..Default::default() @@ -6108,18 +6186,287 @@ mod tests { .await .expect("object should be written only to the second pool"); - let prepared = store - .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &opts) + clear_get_object_body_cache_hook(); + let hook = Arc::new(CountingMissHook { + calls: AtomicUsize::new(0), + }); + register_get_object_body_cache_hook(Arc::clone(&hook) as Arc); + let _hook_guard = BodyCacheHookGuard; + + let calls = disk_call_counters::observe(object); + let mut reader = store + .handle_get_object_reader(bucket, object, None, HeaderMap::new(), &opts) .await - .expect("prepared reader should resolve the second-pool object"); - assert_eq!(prepared.object_info().size, payload.len() as i64); - let mut reader = prepared.into_reader().await.expect("prepared body reader should open"); + .expect("legacy reader should resolve the second-pool object"); + assert_eq!( + hook.calls.load(Ordering::Relaxed), + 1, + "legacy reader must probe the body cache exactly once" + ); + assert_eq!(reader.body_source, GetObjectBodySource::HookMissed); + assert!( + calls.total(disk_call_counters::KIND_READ_VERSION) <= 8, + "legacy reader must fan out each 4-disk pool at most once" + ); let mut restored = Vec::new(); reader .stream .read_to_end(&mut restored) .await - .expect("prepared body should stream"); + .expect("legacy reader body should stream"); + assert_eq!(restored, payload); + } + + fn prepared_pool_test_status(id: usize, suspended: bool) -> PoolStatus { + PoolStatus { + id, + cmd_line: format!("prepared-pool-{id}"), + last_update: OffsetDateTime::now_utc(), + decommission: suspended.then(|| PoolDecommissionInfo { + start_time: Some(OffsetDateTime::now_utc()), + ..Default::default() + }), + } + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_refetches_when_final_pool_state_changes_winner() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await); + let bucket = "prepared-reader-pool-state-fallback"; + let object = "object.bin"; + let payload = b"pool-state fallback payload".repeat(8_000); + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut put_reader = PutObjReader::from_vec(payload.clone()); + set_disks + .put_object(bucket, object, &mut put_reader, &opts) + .await + .expect("shared object should be written"); + + let calls = disk_call_counters::observe(object); + let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, false); + let read_store = Arc::clone(&store); + let read_opts = opts.clone(); + let read = tokio::spawn(async move { + read_store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts) + .await + }); + barrier.wait_after_fanout().await; + *store.pool_meta.write().await = PoolMeta { + pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)], + ..Default::default() + }; + barrier.release_after_fanout(); + + let prepared = read + .await + .expect("prepared read task should not panic") + .expect("final active pool should be refetched"); + assert!(Arc::ptr_eq(&prepared.pool, &store.pools[0])); + assert_eq!( + calls.total(disk_call_counters::KIND_READ_VERSION), + 12, + "two initial 4-disk fanouts plus one fallback refetch are required" + ); + let mut reader = prepared.into_reader().await.expect("fallback body reader should open"); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("fallback body should stream"); + assert_eq!(restored, payload); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_fallback_rejects_generation_change_before_refetch() { + let (_dirs, set_disks) = make_local_set_disks(4, 2).await; + let store = Arc::new(new_prepared_reader_test_store(&[Arc::clone(&set_disks), Arc::clone(&set_disks)]).await); + let bucket = "prepared-reader-pool-state-generation-change"; + let object = "object.bin"; + let opts = ObjectOptions { + no_lock: true, + ..Default::default() + }; + + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + let mut initial_reader = PutObjReader::from_vec(b"initial generation".to_vec()); + set_disks + .put_object(bucket, object, &mut initial_reader, &opts) + .await + .expect("initial object should be written"); + + let barrier = crate::store::rebalance::PreparedPoolReadFallbackBarrier::install(object, true); + let read_store = Arc::clone(&store); + let read_opts = opts.clone(); + let read = tokio::spawn(async move { + read_store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts) + .await + }); + barrier.wait_after_fanout().await; + *store.pool_meta.write().await = PoolMeta { + pools: vec![prepared_pool_test_status(0, false), prepared_pool_test_status(1, true)], + ..Default::default() + }; + barrier.release_after_fanout(); + barrier.wait_before_refetch().await; + + let mut replacement_reader = PutObjReader::from_vec(b"replacement generation".to_vec()); + set_disks + .put_object(bucket, object, &mut replacement_reader, &opts) + .await + .expect("replacement generation should be written before fallback refetch"); + barrier.release_before_refetch(); + + let error = match read.await.expect("prepared read task should not panic") { + Ok(_) => panic!("changed fallback generation must not be accepted"), + Err(error) => error, + }; + assert_eq!(error, Error::ErasureReadQuorum); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_rejects_latest_delete_marker_without_refetching_metadata() { + let ctx = Arc::new(crate::runtime::instance::InstanceContext::new()); + let (_first_dirs, first_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let (_second_dirs, second_set) = make_local_set_disks_with_ctx(4, 2, Arc::clone(&ctx)).await; + let store = new_prepared_reader_test_store_with_ctx(&[Arc::clone(&first_set), Arc::clone(&second_set)], ctx).await; + let bucket = "prepared-reader-latest-delete-marker"; + let object = "versioned-object.bin"; + let versioned_opts = ObjectOptions { + no_lock: true, + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), + ..Default::default() + }; + + for set_disks in [&first_set, &second_set] { + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + } + let mut older_reader = PutObjReader::from_vec(b"older visible generation".to_vec()); + first_set + .put_object(bucket, object, &mut older_reader, &versioned_opts) + .await + .expect("older object should be written"); + let mut hidden_reader = PutObjReader::from_vec(b"hidden generation".to_vec()); + second_set + .put_object(bucket, object, &mut hidden_reader, &versioned_opts) + .await + .expect("newer object should be written"); + let marker = second_set + .delete_object(bucket, object, versioned_opts.clone()) + .await + .expect("delete marker should be committed"); + assert!(marker.delete_marker); + + let calls = disk_call_counters::observe(object); + let error = match store + .prepare_get_object_reader( + bucket, + object, + None, + HeaderMap::new(), + &ObjectOptions { + no_lock: true, + versioned: true, + ..Default::default() + }, + ) + .await + { + Ok(_) => panic!("latest delete marker should hide the older live object"), + Err(error) => error, + }; + + assert!(is_err_object_not_found(&error)); + assert!( + calls.total(disk_call_counters::KIND_READ_VERSION) <= 8, + "delete-marker resolution must fan out each pool at most once" + ); + } + + #[tokio::test] + #[serial_test::serial(body_cache_hook)] + async fn prepared_reader_explicit_version_reuses_the_matching_pool_metadata() { + let (_first_dirs, first_set) = make_local_set_disks(4, 2).await; + let (_second_dirs, second_set) = make_local_set_disks(4, 2).await; + let store = new_prepared_reader_test_store(&[Arc::clone(&first_set), Arc::clone(&second_set)]).await; + let bucket = "prepared-reader-explicit-version"; + let object = "versioned-object.bin"; + let payload = b"explicit version from first pool".repeat(8_000); + let versioned_opts = ObjectOptions { + no_lock: true, + versioned: true, + object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(ObjectLockConfigState::ConfirmedAbsent))), + ..Default::default() + }; + + for set_disks in [&first_set, &second_set] { + set_disks + .make_bucket(bucket, &MakeBucketOptions::default()) + .await + .expect("bucket should be created"); + } + let mut first_reader = PutObjReader::from_vec(payload.clone()); + let first = first_set + .put_object(bucket, object, &mut first_reader, &versioned_opts) + .await + .expect("requested version should be written to the first pool"); + let mut second_reader = PutObjReader::from_vec(b"different pool version".to_vec()); + second_set + .put_object(bucket, object, &mut second_reader, &versioned_opts) + .await + .expect("a different version should be written to the second pool"); + + let requested_version = first + .version_id + .expect("versioned PUT should return a version id") + .to_string(); + let read_opts = ObjectOptions { + no_lock: true, + versioned: true, + version_id: Some(requested_version), + ..Default::default() + }; + let calls = disk_call_counters::observe(object); + let prepared = store + .prepare_get_object_reader(bucket, object, None, HeaderMap::new(), &read_opts) + .await + .expect("explicit version should resolve from the matching pool"); + assert_eq!(prepared.object_info().version_id, first.version_id); + let metadata_calls = calls.total(disk_call_counters::KIND_READ_VERSION); + assert!(metadata_calls <= 8, "explicit-version lookup must fan out each pool at most once"); + + let mut reader = prepared + .into_reader() + .await + .expect("prepared explicit-version body should open"); + assert_eq!(calls.total(disk_call_counters::KIND_READ_VERSION), metadata_calls); + let mut restored = Vec::new(); + reader + .stream + .read_to_end(&mut restored) + .await + .expect("explicit-version body should stream"); assert_eq!(restored, payload); } diff --git a/crates/ecstore/src/store/rebalance.rs b/crates/ecstore/src/store/rebalance.rs index ddcd918ac..6c2fe8642 100644 --- a/crates/ecstore/src/store/rebalance.rs +++ b/crates/ecstore/src/store/rebalance.rs @@ -18,18 +18,117 @@ use crate::core::pools::merge_pool_status_refresh; use crate::layout::pool_space::{ServerPoolsAvailableSpace, build_server_pools_available_space}; use crate::runtime::sources as runtime_sources; use crate::storage_api_contracts::{admin::StorageAdminApi, namespace::NamespaceLocking as _, object::ObjectOperations as _}; +use futures::stream::{FuturesUnordered, StreamExt}; pub(in crate::store) mod support; const LOG_COMPONENT_ECSTORE: &str = "ecstore"; const LOG_SUBSYSTEM_POOLS: &str = "pools"; const EVENT_POOL_META_RELOAD: &str = "pool_meta_reload"; + +#[cfg(test)] +struct PreparedPoolReadFallbackBarrierState { + object: String, + pause_before_refetch: bool, + fanout_arrived: tokio::sync::Notify, + fanout_release: tokio::sync::Notify, + refetch_arrived: tokio::sync::Notify, + refetch_release: tokio::sync::Notify, +} + +#[cfg(test)] +pub(in crate::store) struct PreparedPoolReadFallbackBarrier { + state: Arc, +} + +#[cfg(test)] +static PREPARED_POOL_READ_FALLBACK_BARRIER: std::sync::OnceLock< + std::sync::Mutex>>, +> = std::sync::OnceLock::new(); + +#[cfg(test)] +impl PreparedPoolReadFallbackBarrier { + pub(in crate::store) fn install(object: &str, pause_before_refetch: bool) -> Self { + let state = Arc::new(PreparedPoolReadFallbackBarrierState { + object: object.to_string(), + pause_before_refetch, + fanout_arrived: tokio::sync::Notify::new(), + fanout_release: tokio::sync::Notify::new(), + refetch_arrived: tokio::sync::Notify::new(), + refetch_release: tokio::sync::Notify::new(), + }); + *PREPARED_POOL_READ_FALLBACK_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("prepared pool read fallback barrier must not be poisoned") = Some(Arc::clone(&state)); + Self { state } + } + + pub(in crate::store) async fn wait_after_fanout(&self) { + self.state.fanout_arrived.notified().await; + } + + pub(in crate::store) fn release_after_fanout(&self) { + self.state.fanout_release.notify_one(); + } + + pub(in crate::store) async fn wait_before_refetch(&self) { + self.state.refetch_arrived.notified().await; + } + + pub(in crate::store) fn release_before_refetch(&self) { + self.state.refetch_release.notify_one(); + } +} + +#[cfg(test)] +impl Drop for PreparedPoolReadFallbackBarrier { + fn drop(&mut self) { + self.state.fanout_release.notify_waiters(); + self.state.refetch_release.notify_waiters(); + if let Some(barrier) = PREPARED_POOL_READ_FALLBACK_BARRIER.get() { + *barrier + .lock() + .expect("prepared pool read fallback barrier must not be poisoned") = None; + } + } +} + +#[cfg(test)] +async fn pause_prepared_pool_read_after_fanout(object: &str) { + let state = PREPARED_POOL_READ_FALLBACK_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("prepared pool read fallback barrier must not be poisoned") + .as_ref() + .filter(|state| state.object == object) + .cloned(); + if let Some(state) = state { + state.fanout_arrived.notify_one(); + state.fanout_release.notified().await; + } +} + +#[cfg(test)] +async fn pause_prepared_pool_read_before_refetch(object: &str) { + let state = PREPARED_POOL_READ_FALLBACK_BARRIER + .get_or_init(|| std::sync::Mutex::new(None)) + .lock() + .expect("prepared pool read fallback barrier must not be poisoned") + .as_ref() + .filter(|state| state.object == object && state.pause_before_refetch) + .cloned(); + if let Some(state) = state { + state.refetch_arrived.notify_one(); + state.refetch_release.notified().await; + } +} #[cfg(test)] use support::resolve_latest_object_info_candidates; use support::{ LatestObjectInfoCandidate, PoolErr, PoolObjInfo, RebalanceDeletePoolResult, pool_lookup_not_found_error, rebalance_disk_set_lookup_error, resolve_latest_object_info_candidates_with_pool_state, resolve_rebalance_delete_from_all_pools_result, resolve_rebalance_delete_from_all_pools_results, - resolve_store_rebalance_pool_meta_reload_result, + resolve_store_rebalance_pool_meta_reload_result, validate_prepared_pool_refetch_identity, }; #[derive(Debug, Default, Eq, PartialEq)] @@ -675,6 +774,134 @@ impl ECStore { resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts) } + pub(super) async fn prepare_latest_object_metadata_with_idx( + &self, + bucket: &str, + object: &str, + opts: &ObjectOptions, + ) -> Result<(crate::set_disk::PreparedGetObjectMetadata, usize)> { + let suspended_pools = if opts.skip_decommissioned { + let pool_meta = self.pool_meta.read().await; + Some( + (0..self.pools.len()) + .map(|idx| pool_meta.is_suspended(idx)) + .collect::>(), + ) + } else { + None + }; + let mut futures = FuturesUnordered::new(); + for (idx, pool) in self.pools.iter().enumerate() { + if suspended_pools.as_ref().is_some_and(|pools| pools[idx]) { + continue; + } + + if opts.skip_rebalancing && self.is_pool_rebalancing(idx).await { + continue; + } + + futures.push(async move { + let result = pool + .prepare_get_object_reader_metadata(bucket, object, opts) + .await + .map_err(|err| to_object_err(err, vec![bucket, object])); + (idx, result) + }); + } + + let mut candidates = (0..self.pools.len()).map(|_| None).collect::>(); + // Retain one provisional winner. Other pools only need their lightweight + // identity for final conflict checks; if pool state changes while the + // fanout runs, the final winner is refetched and revalidated below. + let mut latest_prepared = None; + let mut latest_mod_time = None; + let mut provisional_dynamic_pool_state = None; + while let Some((idx, result)) = futures.next().await { + match result { + Ok(metadata) => { + let mod_time = metadata.object_info().mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH); + let info = metadata.object_info().clone(); + let retain = match (latest_mod_time, latest_prepared.as_ref()) { + (None, _) => true, + (Some(current), _) if mod_time > current => true, + (Some(current), _) if mod_time < current => false, + (Some(_), Some((current_idx, _))) => { + if suspended_pools.is_none() && provisional_dynamic_pool_state.is_none() { + let pool_meta = self.pool_meta.read().await; + provisional_dynamic_pool_state = Some( + (0..self.pools.len()) + .map(|pool_idx| pool_meta.is_suspended(pool_idx)) + .collect::>(), + ); + } + let provisional_pool_state = suspended_pools + .as_ref() + .or(provisional_dynamic_pool_state.as_ref()) + .ok_or_else(|| Error::other("GET pool state snapshot is unavailable"))?; + let new_key = (provisional_pool_state.get(idx).copied().unwrap_or(false), std::cmp::Reverse(idx)); + let current_key = ( + provisional_pool_state.get(*current_idx).copied().unwrap_or(false), + std::cmp::Reverse(*current_idx), + ); + new_key < current_key + } + (Some(_), None) => true, + }; + if retain { + if latest_mod_time.is_none_or(|current| mod_time > current) { + latest_mod_time = Some(mod_time); + } + latest_prepared = Some((idx, metadata)); + } + candidates[idx] = Some(LatestObjectInfoCandidate { + info: Some(info), + idx, + err: None, + }); + } + Err(err) => { + candidates[idx] = Some(LatestObjectInfoCandidate { + info: None, + idx, + err: Some(err), + }); + } + } + } + + #[cfg(test)] + pause_prepared_pool_read_after_fanout(object).await; + + let suspended_pools = match suspended_pools { + Some(pools) => pools, + None => { + let pool_meta = self.pool_meta.read().await; + (0..self.pools.len()) + .map(|idx| pool_meta.is_suspended(idx)) + .collect::>() + } + }; + + let candidates = candidates.into_iter().flatten().collect(); + let (winner_info, winner_idx) = + resolve_latest_object_info_candidates_with_pool_state(candidates, &suspended_pools, bucket, object, opts)?; + if let Some((prepared_idx, metadata)) = latest_prepared + && prepared_idx == winner_idx + { + return Ok((metadata, winner_idx)); + } + + let pool = self.pools.get(winner_idx).ok_or(Error::ErasureReadQuorum)?; + #[cfg(test)] + pause_prepared_pool_read_before_refetch(object).await; + let metadata = pool + .prepare_get_object_reader_metadata(bucket, object, opts) + .await + .map_err(|err| to_object_err(err, vec![bucket, object]))?; + validate_prepared_pool_refetch_identity(&winner_info, metadata.object_info())?; + Ok((metadata, winner_idx)) + } + pub(super) async fn delete_object_from_all_pools( &self, bucket: &str, diff --git a/crates/ecstore/src/store/rebalance/support.rs b/crates/ecstore/src/store/rebalance/support.rs index e1a773a54..376ac05b9 100644 --- a/crates/ecstore/src/store/rebalance/support.rs +++ b/crates/ecstore/src/store/rebalance/support.rs @@ -218,7 +218,7 @@ fn same_user_defined_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { /// excluded. The selected winner still carries the chosen pool's layout, while /// the remaining read-visible fields must agree before the pool index can /// provide a deterministic tie-break. -fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { +pub(super) fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bool { let same_read_surface = left.bucket == right.bucket && left.name == right.name && left.is_dir == right.is_dir @@ -277,6 +277,14 @@ fn same_latest_object_info_identity(left: &ObjectInfo, right: &ObjectInfo) -> bo } } +pub(super) fn validate_prepared_pool_refetch_identity(expected: &ObjectInfo, refetched: &ObjectInfo) -> Result<()> { + if same_latest_object_info_identity(expected, refetched) { + Ok(()) + } else { + Err(Error::ErasureReadQuorum) + } +} + #[cfg(test)] pub(super) fn resolve_latest_object_info_candidates( candidates: Vec, @@ -328,7 +336,11 @@ pub(super) fn resolve_latest_object_info_candidates_with_pool_state( return Err(Error::ErasureReadQuorum); } - return Ok((winner_info.clone(), winner.idx)); + let winner = latest_candidates.swap_remove(0); + let Some(winner_info) = winner.info else { + return Err(Error::ErasureReadQuorum); + }; + return Ok((winner_info, winner.idx)); } for candidate in candidates { @@ -347,6 +359,23 @@ pub(super) fn resolve_latest_object_info_candidates_with_pool_state( mod tests { use super::*; + #[test] + fn prepared_pool_refetch_identity_fails_closed_on_generation_change() { + let expected = ObjectInfo { + mod_time: Some(OffsetDateTime::from_unix_timestamp(10).expect("test timestamp should be valid")), + version_id: Some(uuid::Uuid::from_u128(1)), + etag: Some("etag-a".to_string()), + ..Default::default() + }; + let mut refetched = expected.clone(); + refetched.etag = Some("etag-b".to_string()); + + let error = validate_prepared_pool_refetch_identity(&expected, &refetched) + .expect_err("refetched metadata from a changed generation must fail closed"); + + assert_eq!(error, Error::ErasureReadQuorum); + } + #[test] fn rebalance_delete_result_preserves_precondition_failed() { let err = resolve_rebalance_delete_from_all_pools_result(Err(Error::PreconditionFailed), "bucket", "object") diff --git a/crates/iam/src/manager.rs b/crates/iam/src/manager.rs index 30090a240..384ed9a0a 100644 --- a/crates/iam/src/manager.rs +++ b/crates/iam/src/manager.rs @@ -1537,7 +1537,7 @@ where Ok(deleted_at) } - pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<()> { + pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> { if access_key.is_empty() || secret_key.is_empty() { return Err(Error::InvalidArgument); } @@ -1552,7 +1552,16 @@ where let mut cred = u.credentials.clone(); cred.secret_key = secret_key.to_string(); + // Status is captured from the same credential snapshot the new secret + // is persisted with, so a caller replicating the rotation broadcasts + // exactly what was written rather than re-reading racily. + let status = if cred.is_valid() { + AccountStatus::Enabled + } else { + AccountStatus::Disabled + }; let u = UserIdentity::from(cred); + let updated_at = u.update_at.unwrap_or_else(OffsetDateTime::now_utc); drop(cache); drop(users); @@ -1560,7 +1569,8 @@ where .save_user_identity(access_key, UserType::Reg, u.clone(), None) .await?; - self.update_user_with_claims(access_key, u) + self.update_user_with_claims(access_key, u)?; + Ok((updated_at, status)) } /// Add SSH public key for a user (for SFTP authentication) diff --git a/crates/iam/src/sys.rs b/crates/iam/src/sys.rs index fecbf9f5c..cd38d5f4e 100644 --- a/crates/iam/src/sys.rs +++ b/crates/iam/src/sys.rs @@ -960,7 +960,11 @@ impl IamSys { Ok(updated_at) } - pub async fn set_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<()> { + pub async fn set_user_secret_key( + &self, + access_key: &str, + secret_key: &str, + ) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus)> { if !is_access_key_valid(access_key) { return Err(IamError::InvalidAccessKeyLength); } @@ -969,7 +973,9 @@ impl IamSys { return Err(IamError::InvalidSecretKeyLength); } - self.store.update_user_secret_key(access_key, secret_key).await + let (updated_at, status) = self.store.update_user_secret_key(access_key, secret_key).await?; + self.notify_for_user(access_key, false).await; + Ok((updated_at, status)) } /// Add SSH public key for a user (for SFTP authentication) diff --git a/crates/replication/src/config.rs b/crates/replication/src/config.rs index abdb6cab4..e413434cf 100644 --- a/crates/replication/src/config.rs +++ b/crates/replication/src/config.rs @@ -1671,6 +1671,94 @@ mod tests { } } + #[test] + fn replication_writable_fields_bind_to_typed_dto_fields() { + let mut rule = replication_rule("id-marker", "arn:bucket-marker"); + rule.priority = Some(37); + rule.filter = Some(s3s::dto::ReplicationRuleFilter { + prefix: Some("prefix-marker/".to_string()), + tag: Some(s3s::dto::Tag { + key: Some("tag-key-marker".to_string()), + value: Some("tag-value-marker".to_string()), + }), + and: Some(s3s::dto::ReplicationRuleAndOperator { + prefix: Some("and-prefix-marker/".to_string()), + tags: Some(vec![s3s::dto::Tag { + key: Some("and-tag-key-marker".to_string()), + value: Some("and-tag-value-marker".to_string()), + }]), + }), + ..Default::default() + }); + rule.delete_marker_replication = Some(DeleteMarkerReplication { + status: Some(DeleteMarkerReplicationStatus::from_static(DeleteMarkerReplicationStatus::ENABLED)), + }); + rule.delete_replication = Some(DeleteReplication { + status: DeleteReplicationStatus::from_static(DeleteReplicationStatus::ENABLED), + }); + rule.source_selection_criteria = Some(SourceSelectionCriteria { + replica_modifications: Some(ReplicaModifications { + status: ReplicaModificationsStatus::from_static(ReplicaModificationsStatus::ENABLED), + }), + sse_kms_encrypted_objects: None, + }); + let config = ReplicationConfiguration { + role: "role-marker".to_string(), + rules: vec![rule], + }; + + let rule = config.rules.first().expect("fixture should contain one rule"); + let filter = rule.filter.as_ref().expect("fixture should contain a rule filter"); + let field_hits = [ + ("Role", config.role == "role-marker"), + ("Rule.ID", rule.id.as_deref() == Some("id-marker")), + ("Rule.Status", rule.status.as_str() == ReplicationRuleStatus::ENABLED), + ("Rule.Priority", rule.priority == Some(37)), + ("Rule.Filter.Prefix", filter.prefix.as_deref() == Some("prefix-marker/")), + ( + "Rule.Filter.Tag", + filter.tag.as_ref().and_then(|tag| tag.key.as_deref()) == Some("tag-key-marker"), + ), + ( + "Rule.Filter.And", + filter.and.as_ref().and_then(|and| and.prefix.as_deref()) == Some("and-prefix-marker/"), + ), + ("Rule.Destination.Bucket", rule.destination.bucket == "arn:bucket-marker"), + ( + "Rule.ExistingObjectReplication.Status", + rule.existing_object_replication + .as_ref() + .is_some_and(|existing| existing.status.as_str() == ExistingObjectReplicationStatus::ENABLED), + ), + ( + "Rule.DeleteMarkerReplication.Status", + rule.delete_marker_replication + .as_ref() + .and_then(|delete_marker| delete_marker.status.as_ref()) + .is_some_and(|status| status.as_str() == DeleteMarkerReplicationStatus::ENABLED), + ), + ( + "Rule.DeleteReplication.Status", + rule.delete_replication + .as_ref() + .is_some_and(|delete| delete.status.as_str() == DeleteReplicationStatus::ENABLED), + ), + ( + "Rule.SourceSelectionCriteria.ReplicaModifications.Status", + rule.source_selection_criteria + .as_ref() + .and_then(|criteria| criteria.replica_modifications.as_ref()) + .is_some_and(|modifications| modifications.status.as_str() == ReplicaModificationsStatus::ENABLED), + ), + ]; + let bound_paths = field_hits.iter().map(|(path, _)| *path).collect::>(); + assert_eq!(bound_paths, REPLICATION_WRITABLE_FIELDS); + + for (path, hit) in field_hits { + assert!(hit, "typed field probe did not reach {path}"); + } + } + #[test] fn invalid_replication_status_fields_are_reported_before_persistence() { let arn = "arn:rustfs:replication:us-east-1:target:bucket"; diff --git a/crates/replication/src/delete.rs b/crates/replication/src/delete.rs index 49d305b40..ffd014796 100644 --- a/crates/replication/src/delete.rs +++ b/crates/replication/src/delete.rs @@ -126,6 +126,27 @@ pub fn should_retry_delete_marker_purge(dobj: &DeletedObject) -> bool { dobj.delete_marker_version_id.is_some() } +/// True when the target denied a replicated delete because object-lock +/// retention or a legal hold protects that version on the replica (its +/// deletion gate answers `AccessDenied` with the lock reason, and a +/// replication request carries no governance bypass, rustfs#6850). Retrying +/// cannot succeed until the lock itself lapses, so callers treat this as a +/// policy denial rather than a transient fault. +/// +/// The reason text is the RustFS deletion-gate wording; a MinIO/AWS peer +/// phrases its WORM denial differently and simply stays unclassified — the +/// caller then falls back to plain retry behavior, never a wrong state. +pub fn is_object_lock_denied_delete(code: Option<&str>, message: Option<&str>) -> bool { + if !matches!(code, Some("AccessDenied")) { + return false; + } + let Some(message) = message else { + return false; + }; + let message = message.to_ascii_lowercase(); + message.contains("retention") || message.contains("legal hold") +} + fn admitted_target_arns_from_replication_state(state: &ReplicationState) -> Vec { let mut target_arns = state.targets.keys().cloned().collect::>(); target_arns.extend(state.purge_targets.keys().cloned()); @@ -237,9 +258,9 @@ mod tests { use super::{ DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, - delete_replication_creates_marker, is_retryable_delete_replication_head_error, is_version_delete_replication, - replicate_delete_outcome, resync_existing_delete_replication_info, should_retry_delete_marker_purge, - target_delete_version_id, + delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error, + is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info, + should_retry_delete_marker_purge, target_delete_version_id, }; use crate::storage_api::DeletedObject; use crate::{ @@ -615,4 +636,24 @@ mod tests { corrupt.target_delete_marker_version_ids_corrupt = true; assert_eq!(delete_marker_purge_version_id(Some(&corrupt), arn, source), None); } + + #[test] + fn object_lock_denied_delete_is_recognized_by_code_and_reason() { + // The peer's deletion gate answers AccessDenied with the lock reason. + assert!(is_object_lock_denied_delete( + Some("AccessDenied"), + Some("Object is under GOVERNANCE retention and cannot be deleted until 2026-09-01T00:00:00Z") + )); + assert!(is_object_lock_denied_delete( + Some("AccessDenied"), + Some("Object has a legal hold and cannot be deleted. Remove the legal hold first.") + )); + + // A plain policy denial (misconfigured replicator) is not a lock denial. + assert!(!is_object_lock_denied_delete(Some("AccessDenied"), Some("Access Denied."))); + assert!(!is_object_lock_denied_delete(Some("AccessDenied"), None)); + // Other errors mentioning retention must not match. + assert!(!is_object_lock_denied_delete(Some("InternalError"), Some("retention lookup failed"))); + assert!(!is_object_lock_denied_delete(None, Some("legal hold"))); + } } diff --git a/crates/replication/src/lib.rs b/crates/replication/src/lib.rs index e9ab2801d..98464e7de 100644 --- a/crates/replication/src/lib.rs +++ b/crates/replication/src/lib.rs @@ -41,9 +41,9 @@ pub use config::{ }; pub use delete::{ DeletedObjectReplicationInfo, delete_marker_purge_mrf_entry, delete_marker_purge_version_id, - delete_replication_creates_marker, is_retryable_delete_replication_head_error, is_version_delete_replication, - replicate_delete_outcome, resync_existing_delete_replication_info, should_retry_delete_marker_purge, - target_delete_version_id, + delete_replication_creates_marker, is_object_lock_denied_delete, is_retryable_delete_replication_head_error, + is_version_delete_replication, replicate_delete_outcome, resync_existing_delete_replication_info, + should_retry_delete_marker_purge, target_delete_version_id, }; pub use filemeta::{ NULL_VERSION_ID, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATE_HEAL, REPLICATE_HEAL_DELETE, REPLICATE_INCOMING, @@ -65,7 +65,8 @@ pub use multipart::{ pub use object::{ ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target, replication_etags_match, - ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version, version_identity_drifted, + single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate, + target_is_newer_than_source_null_version, version_identity_drifted, }; pub use operation::{ MustReplicateOptions, ReplicationDeleteScheduleInput, ReplicationDeleteSource, ReplicationDeleteStateSource, diff --git a/crates/replication/src/object.rs b/crates/replication/src/object.rs index 7ce7761ca..505907df4 100644 --- a/crates/replication/src/object.rs +++ b/crates/replication/src/object.rs @@ -71,6 +71,32 @@ pub fn replication_etags_match(source: Option<&str>, target: Option<&str>) -> bo source_etag.is_some() && source_etag == target_etag } +fn is_plain_single_part_md5(etag: &str) -> bool { + etag.len() == 32 && etag.bytes().all(|b| b.is_ascii_hexdigit()) +} + +/// Whether the ETag the target returned for a single-part replica proves the +/// stored bytes differ from what the source sent — e.g. a target that does not +/// decode `aws-chunked` framing stores the frames verbatim and returns their +/// ETag. Only a plain single-part MD5 ETag on both sides is decidable; a +/// multipart or opaque (encrypted) ETag, or a withheld replica ETag, returns +/// `false` because no corruption can be concluded from it. +pub fn single_part_replica_etag_mismatch(source_etag: Option<&str>, replica_etag: Option<&str>) -> bool { + let Some(source) = source_etag.map(trim_etag) else { + return false; + }; + if !is_plain_single_part_md5(&source) { + return false; + } + let Some(replica) = replica_etag.map(trim_etag) else { + return false; + }; + if !is_plain_single_part_md5(&replica) { + return false; + } + !source.eq_ignore_ascii_case(&replica) +} + pub fn target_is_newer_than_source_null_version( source: &ReplicationSourceObject<'_>, target: &ReplicationTargetObject<'_>, @@ -276,11 +302,41 @@ pub fn ssec_passthrough_evidence_present(sse_customer_algorithm: Option<&str>) - #[cfg(test)] mod tests { + const SOURCE_MD5: &str = "9a0364b9e99bb480dd25e1f0284c8555"; + const FRAMED_MD5: &str = "0f343b0931126a20f133d67c2b018a3b"; + + #[test] + fn single_part_replica_mismatch_is_only_decided_on_plain_md5_pairs() { + // The #6853 shape: the target stored aws-chunked frames verbatim and + // returned the framed bytes' ETag. + assert!(single_part_replica_etag_mismatch(Some(SOURCE_MD5), Some(FRAMED_MD5))); + assert!(single_part_replica_etag_mismatch( + Some(&format!("\"{SOURCE_MD5}\"")), + Some(&format!("\"{FRAMED_MD5}\"")) + )); + + // A faithful replica, quoted or not, passes; hex case must not matter + // (a target may return the same MD5 uppercased). + assert!(!single_part_replica_etag_mismatch(Some(SOURCE_MD5), Some(SOURCE_MD5))); + assert!(!single_part_replica_etag_mismatch(Some(&format!("\"{SOURCE_MD5}\"")), Some(SOURCE_MD5))); + assert!(!single_part_replica_etag_mismatch( + Some(SOURCE_MD5), + Some(&SOURCE_MD5.to_ascii_uppercase()) + )); + + // Not decidable: multipart source, opaque replica ETag, or either side + // missing must never be reported as corruption. + assert!(!single_part_replica_etag_mismatch(Some(&format!("{SOURCE_MD5}-3")), Some(FRAMED_MD5))); + assert!(!single_part_replica_etag_mismatch(Some(SOURCE_MD5), Some(&format!("{FRAMED_MD5}-3")))); + assert!(!single_part_replica_etag_mismatch(Some(SOURCE_MD5), None)); + assert!(!single_part_replica_etag_mismatch(None, Some(FRAMED_MD5))); + } + use super::{ ReplicationSourceObject, ReplicationTargetObject, SsecPassthroughCapability, SsecPassthroughGate, content_matches_by_etag, is_replication_target_offline_error, replication_action_for_target, replication_etags_match, - ssec_passthrough_evidence_present, ssec_passthrough_gate, target_is_newer_than_source_null_version, - version_identity_drifted, + single_part_replica_etag_mismatch, ssec_passthrough_evidence_present, ssec_passthrough_gate, + target_is_newer_than_source_null_version, version_identity_drifted, }; use crate::filemeta::{ReplicationAction, ReplicationType}; use crate::http::AMZ_OBJECT_LOCK_MODE; diff --git a/crates/s3select-api/Cargo.toml b/crates/s3select-api/Cargo.toml index a68ce1a1d..38c9eabf5 100644 --- a/crates/s3select-api/Cargo.toml +++ b/crates/s3select-api/Cargo.toml @@ -60,21 +60,26 @@ hotpath-cpu = [ [dependencies] hotpath.workspace = true metrics = { workspace = true } +async-compression = { workspace = true, features = ["tokio", "gzip", "bzip2"] } async-trait.workspace = true +arc-swap.workspace = true bytes = { workspace = true, features = ["serde"] } chrono = { workspace = true, features = ["serde"] } +crc-fast.workspace = true rustfs-common.workspace = true datafusion = { workspace = true, default-features = false, features = ["parquet", "recursive_protection", "sql"] } rustfs-ecstore.workspace = true rustfs-storage-api.workspace = true futures = { workspace = true } futures-core = { workspace = true } +flate2.workspace = true http.workspace = true s3s = { workspace = true, features = ["minio"] } serde_json = { workspace = true, features = ["raw_value"] } thiserror = { workspace = true } parking_lot.workspace = true tokio = { workspace = true, features = ["fs", "rt-multi-thread"] } +tokio-stream.workspace = true tokio-util = { workspace = true, features = ["io", "compat"] } tracing.workspace = true uuid.workspace = true diff --git a/crates/s3select-api/src/input_stream.rs b/crates/s3select-api/src/input_stream.rs new file mode 100644 index 000000000..ff2b501dd --- /dev/null +++ b/crates/s3select-api/src/input_stream.rs @@ -0,0 +1,1570 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::{ + MAX_ERROR_SOURCE_DEPTH, SelectError, SelectInputMetrics, metrics::SelectInputMetricsRecorder, + query::session::QueryExecutionGuard, +}; +use async_compression::tokio::bufread::BzDecoder; +use bytes::{Buf as _, Bytes}; +use datafusion::object_store::{Error as ObjectStoreError, Result as ObjectStoreResult}; +use flate2::bufread::GzDecoder; +use futures::{StreamExt, stream}; +use futures_core::stream::BoxStream; +use std::{ + error::Error as StdError, + io::{self, BufRead as _, Read as _}, + pin::Pin, + sync::Arc, + task::{Context, Poll}, +}; +use tokio::{ + io::{AsyncRead, AsyncReadExt, BufReader, ReadBuf}, + sync::{mpsc, oneshot}, +}; +use tokio_stream::wrappers::ReceiverStream; +use tokio_util::io::{ReaderStream, StreamReader}; + +pub(crate) const MAX_SELECT_RECORD_BYTES: usize = 1024 * 1024; +const MAX_SELECT_PROCESSED_BYTES: u64 = 5 * 1024 * 1024 * 1024 * 1024; +pub(crate) const SELECT_DECODE_CHUNK_BYTES: usize = 64 * 1024; +const DECOMPRESSION_CHANNEL_CAPACITY: usize = 2; + +pub(crate) type SelectInputReader = Box; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum CompressionFormat { + Gzip, + Bzip2, +} + +impl CompressionFormat { + fn name(self) -> &'static str { + match self { + Self::Gzip => "GZIP", + Self::Bzip2 => "BZIP2", + } + } + + fn invalid_header_error(self) -> SelectError { + SelectError::InvalidCompressionFormatForObject { + compression: self.name(), + } + } +} + +pub(crate) fn processed_bytes_limit() -> u64 { + // Processed throughput is independent of compression ratio and live + // memory; use the S3 Select object-size ceiling as the absolute bound. + MAX_SELECT_PROCESSED_BYTES +} + +pub(crate) fn compressed_input_reader( + reader: SelectInputReader, + compressed_size: u64, + format: CompressionFormat, + input_metrics: Arc, + max_processed_bytes: u64, + query_guard: Option, +) -> SelectInputReader { + // rustfs-zip does not expose Select's streaming metrics, member validation, + // typed errors, or cancellation contract, so the protocol adapter lives here. + let input_metrics = input_metrics.recorder(); + let reader = ScannedReader::new(reader, compressed_size, input_metrics.clone()); + let reader = CooperativeReader::new(reader); + let decoder = match format { + CompressionFormat::Gzip => blocking_gzip_reader(Box::new(reader), query_guard), + CompressionFormat::Bzip2 => blocking_bzip2_reader(Box::new(Bzip2HeaderValidatingReader::new(reader)), query_guard), + }; + Box::new(CooperativeReader::new(ProcessedReader::new(decoder, input_metrics, max_processed_bytes))) +} + +fn blocking_gzip_reader(reader: SelectInputReader, query_guard: Option) -> SelectInputReader { + // The bounded bridge keeps RFC 1952 decoding off Tokio workers while + // preserving member validation, backpressure, and reader cancellation. + let (compressed_tx, compressed_rx) = mpsc::channel(DECOMPRESSION_CHANNEL_CAPACITY); + let (decoded_tx, decoded_rx) = mpsc::channel(DECOMPRESSION_CHANNEL_CAPACITY); + + let decoded_closed = decoded_tx.clone(); + drop(tokio::spawn(async move { + let mut stream = ReaderStream::with_capacity(reader, SELECT_DECODE_CHUNK_BYTES); + loop { + let item = tokio::select! { + biased; + _ = decoded_closed.closed() => break, + item = stream.next() => item, + }; + let Some(item) = item else { + break; + }; + let sent = tokio::select! { + biased; + _ = decoded_closed.closed() => false, + result = compressed_tx.send(item) => result.is_ok(), + }; + if !sent { + break; + } + } + })); + + let blocking_output = decoded_tx.clone(); + spawn_decoder_thread("s3select-gzip", decoded_tx, query_guard, move || { + decode_gzip(compressed_rx, blocking_output) + }); + + Box::new(StreamReader::new(ReceiverStream::new(decoded_rx))) +} + +fn blocking_bzip2_reader(reader: SelectInputReader, query_guard: Option) -> SelectInputReader { + let (decoded_tx, decoded_rx) = mpsc::channel(DECOMPRESSION_CHANNEL_CAPACITY); + let runtime = tokio::runtime::Handle::current(); + let blocking_output = decoded_tx.clone(); + // A dedicated thread avoids occupying Tokio's blocking pool while the + // decoder waits for asynchronous object reads. Query admission bounds the + // number of concurrent Select decoder threads. + spawn_decoder_thread("s3select-bzip2", decoded_tx, query_guard, move || { + runtime.block_on(decode_bzip2(reader, blocking_output)) + }); + Box::new(StreamReader::new(ReceiverStream::new(decoded_rx))) +} + +fn spawn_decoder_thread( + name: &'static str, + decoded: mpsc::Sender>, + query_guard: Option, + task: impl FnOnce() -> io::Result<()> + Send + 'static, +) { + let (finished_tx, finished_rx) = oneshot::channel(); + let spawn_result = std::thread::Builder::new().name(name.to_string()).spawn(move || { + let _query_guard = query_guard; + let _ = finished_tx.send(task()); + }); + if spawn_result.is_err() { + let _ = decoded.try_send(Err(io::Error::other(SelectError::InternalError))); + return; + } + + drop(tokio::spawn(async move { + let result = finished_rx + .await + .unwrap_or_else(|_| Err(io::Error::other(SelectError::InternalError))); + if let Err(error) = result { + let _ = decoded.send(Err(error)).await; + } + })); +} + +async fn decode_bzip2(reader: SelectInputReader, decoded: mpsc::Sender>) -> io::Result<()> { + let reader = BufReader::with_capacity(SELECT_DECODE_CHUNK_BYTES, reader); + let mut decoder = BzDecoder::new(reader); + decoder.multiple_members(true); + let mut buffer = vec![0; SELECT_DECODE_CHUNK_BYTES]; + loop { + let read = tokio::select! { + biased; + _ = decoded.closed() => return Ok(()), + result = decoder.read(&mut buffer) => result?, + }; + if read == 0 { + return Ok(()); + } + let bytes = Bytes::copy_from_slice(&buffer[..read]); + tokio::select! { + biased; + _ = decoded.closed() => return Ok(()), + result = decoded.send(Ok(bytes)) => { + if result.is_err() { + return Ok(()); + } + } + } + } +} + +fn decode_gzip(compressed: mpsc::Receiver>, decoded: mpsc::Sender>) -> io::Result<()> { + let mut reader = io::BufReader::with_capacity(SELECT_DECODE_CHUNK_BYTES, BlockingChannelReader::new(compressed)); + let mut buffer = vec![0; SELECT_DECODE_CHUNK_BYTES]; + let mut decoded_member = false; + loop { + if decoded.is_closed() { + return Ok(()); + } + + if reader.fill_buf()?.is_empty() { + return if decoded_member { + Ok(()) + } else { + Err(io::Error::new(io::ErrorKind::UnexpectedEof, SelectError::TruncatedInput)) + }; + } + let header = read_gzip_header(&mut reader).map_err(|error| { + if decoded_member + && find_error_source::(&error) + .is_some_and(|source| matches!(source, SelectError::InvalidCompressionFormatForObject { .. })) + { + io::Error::new(error.kind(), SelectError::TruncatedInput) + } else { + error + } + })?; + let mut decoder = GzDecoder::new(std::io::Read::chain(io::Cursor::new(header), reader)); + loop { + let read = decoder.read(&mut buffer)?; + if read == 0 { + break; + } + if decoded.blocking_send(Ok(Bytes::copy_from_slice(&buffer[..read]))).is_err() { + return Ok(()); + } + } + let (_, remaining) = decoder.into_inner().into_inner(); + reader = remaining; + decoded_member = true; + } +} + +fn read_gzip_header(reader: &mut R) -> io::Result<[u8; 10]> { + let mut fixed = [0; 10]; + read_gzip_exact(reader, &mut fixed)?; + if fixed[..3] != [0x1f, 0x8b, 0x08] || fixed[3] & 0xe0 != 0 { + return Err(invalid_gzip_header_error()); + } + let flags = fixed[3]; + let mut header_crc = (flags & GZIP_FLAG_HEADER_CRC != 0).then(|| crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc)); + update_gzip_header_crc(&mut header_crc, &fixed); + + if flags & GZIP_FLAG_EXTRA != 0 { + let mut length = [0; 2]; + read_gzip_exact(reader, &mut length)?; + update_gzip_header_crc(&mut header_crc, &length); + let extra_len = usize::from(u16::from_le_bytes(length)); + read_gzip_header_bytes(reader, extra_len, &mut header_crc)?; + } + if flags & GZIP_FLAG_NAME != 0 { + read_gzip_text_field(reader, &mut header_crc)?; + } + if flags & GZIP_FLAG_COMMENT != 0 { + read_gzip_text_field(reader, &mut header_crc)?; + } + if let Some(digest) = header_crc { + let expected = + u16::try_from(digest.finalize() & u64::from(u16::MAX)).map_err(|_| io::Error::other(SelectError::InternalError))?; + let mut actual = [0; 2]; + read_gzip_exact(reader, &mut actual)?; + if u16::from_le_bytes(actual) != expected { + return Err(invalid_gzip_header_error()); + } + } + + fixed[3] &= GZIP_FLAG_TEXT; + Ok(fixed) +} + +fn read_gzip_exact(reader: &mut R, bytes: &mut [u8]) -> io::Result<()> { + reader.read_exact(bytes).map_err(|error| { + if error.kind() == io::ErrorKind::UnexpectedEof && !error_chain_contains::(&error) { + io::Error::new(io::ErrorKind::UnexpectedEof, SelectError::TruncatedInput) + } else { + error + } + }) +} + +fn read_gzip_header_bytes( + reader: &mut R, + mut remaining: usize, + header_crc: &mut Option, +) -> io::Result<()> { + while remaining > 0 { + let available = reader.fill_buf()?; + if available.is_empty() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, SelectError::TruncatedInput)); + } + let consumed = remaining.min(available.len()); + update_gzip_header_crc(header_crc, &available[..consumed]); + reader.consume(consumed); + remaining -= consumed; + } + Ok(()) +} + +fn read_gzip_text_field(reader: &mut R, header_crc: &mut Option) -> io::Result<()> { + loop { + let available = reader.fill_buf()?; + if available.is_empty() { + return Err(io::Error::new(io::ErrorKind::UnexpectedEof, SelectError::TruncatedInput)); + } + let terminator = available.iter().position(|byte| *byte == 0); + let consumed = terminator.map_or(available.len(), |position| position + 1); + update_gzip_header_crc(header_crc, &available[..consumed]); + reader.consume(consumed); + if terminator.is_some() { + return Ok(()); + } + } +} + +fn update_gzip_header_crc(header_crc: &mut Option, bytes: &[u8]) { + if let Some(digest) = header_crc { + digest.update(bytes); + } +} + +fn invalid_gzip_header_error() -> io::Error { + io::Error::new(io::ErrorKind::InvalidData, CompressionFormat::Gzip.invalid_header_error()) +} + +struct BlockingChannelReader { + receiver: mpsc::Receiver>, + current: Bytes, +} + +impl BlockingChannelReader { + fn new(receiver: mpsc::Receiver>) -> Self { + Self { + receiver, + current: Bytes::new(), + } + } +} + +impl io::Read for BlockingChannelReader { + fn read(&mut self, buffer: &mut [u8]) -> io::Result { + while self.current.is_empty() { + match self.receiver.blocking_recv() { + Some(Ok(bytes)) => self.current = bytes, + Some(Err(error)) => return Err(error), + None => return Ok(0), + } + } + let read = buffer.len().min(self.current.len()); + buffer[..read].copy_from_slice(&self.current[..read]); + self.current.advance(read); + Ok(read) + } +} + +pub(crate) fn compressed_input_stream( + reader: SelectInputReader, + compressed_size: u64, + format: CompressionFormat, + input_metrics: Arc, + record_delimiter: Vec, + max_processed_bytes: u64, + query_guard: Option, +) -> ObjectStoreResult>> { + let record_size = RecordSizeTracker::new(record_delimiter).map_err(select_object_store_error)?; + let reader = compressed_input_reader(reader, compressed_size, format, input_metrics, max_processed_bytes, query_guard); + let stream = ReaderStream::with_capacity(reader, SELECT_DECODE_CHUNK_BYTES); + Ok(stream::try_unfold((stream, record_size), |(mut stream, mut record_size)| async move { + match stream.next().await { + Some(Ok(bytes)) => { + record_size.observe(&bytes).map_err(select_object_store_error)?; + Ok(Some((bytes, (stream, record_size)))) + } + Some(Err(error)) => Err(input_io_error(error)), + None => { + record_size.finish().map_err(select_object_store_error)?; + Ok(None) + } + } + }) + .boxed()) +} + +pub(crate) fn input_io_error(source: io::Error) -> ObjectStoreError { + let source: Box = match find_error_source::(&source) { + Some(error) => Box::new(error.clone()), + None => Box::new(source), + }; + ObjectStoreError::Generic { + store: "EcObjectStore", + source, + } +} + +fn select_object_store_error(source: SelectError) -> ObjectStoreError { + ObjectStoreError::Generic { + store: "EcObjectStore", + source: Box::new(source), + } +} + +#[derive(Debug, thiserror::Error)] +#[error("compressed object source read failed")] +struct CompressedSourceReadError { + #[source] + source: io::Error, +} + +struct ScannedReader { + inner: tokio::io::Take, + input_metrics: SelectInputMetricsRecorder, +} + +impl ScannedReader { + fn new(reader: R, compressed_size: u64, input_metrics: SelectInputMetricsRecorder) -> Self { + Self { + inner: reader.take(compressed_size), + input_metrics, + } + } +} + +impl AsyncRead for ScannedReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let before = buf.filled().len(); + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(source)) => Poll::Ready(Err(source_read_error(source))), + Poll::Ready(Ok(())) => { + let read = buf.filled().len() - before; + if read == 0 && self.inner.limit() > 0 { + let source = io::Error::new( + io::ErrorKind::UnexpectedEof, + format!("compressed object stream ended with {} bytes remaining", self.inner.limit()), + ); + return Poll::Ready(Err(source_read_error(source))); + } + self.input_metrics.record_scanned(read); + Poll::Ready(Ok(())) + } + } + } +} + +struct CooperativeReader { + inner: R, + bytes_since_yield: usize, + yield_pending: bool, +} + +impl CooperativeReader { + fn new(inner: R) -> Self { + Self { + inner, + bytes_since_yield: 0, + yield_pending: false, + } + } +} + +impl AsyncRead for CooperativeReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if self.yield_pending { + self.yield_pending = false; + self.bytes_since_yield = 0; + cx.waker().wake_by_ref(); + return Poll::Pending; + } + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + + let remaining_budget = SELECT_DECODE_CHUNK_BYTES - self.bytes_since_yield; + let read_limit = remaining_budget.min(buf.remaining()); + let read = { + let unfilled = buf.initialize_unfilled_to(read_limit); + let mut limited = ReadBuf::new(unfilled); + match Pin::new(&mut self.inner).poll_read(cx, &mut limited) { + Poll::Pending => return Poll::Pending, + Poll::Ready(Err(error)) => return Poll::Ready(Err(error)), + Poll::Ready(Ok(())) => limited.filled().len(), + } + }; + buf.advance(read); + self.bytes_since_yield += read; + self.yield_pending = read > 0 && self.bytes_since_yield == SELECT_DECODE_CHUNK_BYTES; + Poll::Ready(Ok(())) + } +} + +fn source_read_error(source: io::Error) -> io::Error { + let kind = source.kind(); + io::Error::new(kind, CompressedSourceReadError { source }) +} + +struct Bzip2HeaderValidatingReader { + inner: R, + position: usize, + pending_error: Option, +} + +impl Bzip2HeaderValidatingReader { + fn new(inner: R) -> Self { + Self { + inner, + position: 0, + pending_error: None, + } + } +} + +impl AsyncRead for Bzip2HeaderValidatingReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + if let Some(error) = self.pending_error.take() { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, error))); + } + if buf.remaining() == 0 { + return Poll::Ready(Ok(())); + } + if self.position == 4 { + return Pin::new(&mut self.inner).poll_read(cx, buf); + } + + let before = buf.filled().len(); + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(error)) => Poll::Ready(Err(error)), + Poll::Ready(Ok(())) => { + let after = buf.filled().len(); + if after == before { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::UnexpectedEof, SelectError::TruncatedInput))); + } + if let Err(error) = validate_bzip2_header(&mut self.position, &buf.filled()[before..after]) { + buf.set_filled(before + error.offset); + if error.offset == 0 { + return Poll::Ready(Err(io::Error::new(io::ErrorKind::InvalidData, error.source))); + } + self.pending_error = Some(error.source); + } + Poll::Ready(Ok(())) + } + } + } +} + +struct HeaderValidationError { + offset: usize, + source: SelectError, +} + +fn validate_bzip2_header(position: &mut usize, bytes: &[u8]) -> Result<(), HeaderValidationError> { + for (offset, byte) in bytes.iter().copied().enumerate() { + let valid = match *position { + 0 => byte == b'B', + 1 => byte == b'Z', + 2 => byte == b'h', + 3 => matches!(byte, b'1'..=b'9'), + _ => break, + }; + if !valid { + return Err(HeaderValidationError { + offset, + source: CompressionFormat::Bzip2.invalid_header_error(), + }); + } + *position += 1; + } + Ok(()) +} + +const GZIP_FLAG_HEADER_CRC: u8 = 0x02; +const GZIP_FLAG_EXTRA: u8 = 0x04; +const GZIP_FLAG_NAME: u8 = 0x08; +const GZIP_FLAG_COMMENT: u8 = 0x10; +const GZIP_FLAG_TEXT: u8 = 0x01; + +struct ProcessedReader { + inner: R, + input_metrics: SelectInputMetricsRecorder, + processed_bytes: u64, + max_processed_bytes: u64, +} + +impl ProcessedReader { + fn new(inner: R, input_metrics: SelectInputMetricsRecorder, max_processed_bytes: u64) -> Self { + Self { + inner, + input_metrics, + processed_bytes: 0, + max_processed_bytes, + } + } +} + +impl AsyncRead for ProcessedReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + let before = buf.filled().len(); + match Pin::new(&mut self.inner).poll_read(cx, buf) { + Poll::Pending => Poll::Pending, + Poll::Ready(Err(error)) => Poll::Ready(Err(classify_decoder_error(error))), + Poll::Ready(Ok(())) => { + let read = buf.filled().len() - before; + let read = u64::try_from(read).unwrap_or(u64::MAX); + let Some(processed_bytes) = self.processed_bytes.checked_add(read) else { + return Poll::Ready(Err(processed_bytes_limit_error())); + }; + if processed_bytes > self.max_processed_bytes { + return Poll::Ready(Err(processed_bytes_limit_error())); + } + self.processed_bytes = processed_bytes; + self.input_metrics.record_processed(buf.filled().len() - before); + Poll::Ready(Ok(())) + } + } + } +} + +fn processed_bytes_limit_error() -> io::Error { + io::Error::new(io::ErrorKind::OutOfMemory, SelectError::ResourceExhausted) +} + +fn classify_decoder_error(error: io::Error) -> io::Error { + if error_chain_contains::(&error) || error_chain_contains::(&error) { + return error; + } + + let select_error = if error.kind() == io::ErrorKind::OutOfMemory { + SelectError::ResourceExhausted + } else { + SelectError::TruncatedInput + }; + io::Error::new(error.kind(), select_error) +} + +fn error_chain_contains(error: &(dyn StdError + 'static)) -> bool { + find_error_source::(error).is_some() +} + +fn find_error_source<'a, T: StdError + 'static>(error: &'a (dyn StdError + 'static)) -> Option<&'a T> { + let mut current = Some(error); + for _ in 0..MAX_ERROR_SOURCE_DEPTH { + let Some(error) = current else { + break; + }; + if let Some(error) = error.downcast_ref::() { + return Some(error); + } + current = error + .downcast_ref::() + .and_then(|error| error.get_ref()) + .map(|source| source as &(dyn StdError + 'static)) + .or_else(|| error.source()); + } + None +} + +struct RecordSizeTracker { + delimiter: Vec, + prefix: Vec, + record_bytes: usize, + matched: usize, +} + +impl RecordSizeTracker { + fn new(delimiter: Vec) -> Result { + if delimiter.is_empty() { + return Err(SelectError::InvalidDataSource); + } + + let mut prefix = vec![0; delimiter.len()]; + let mut matched = 0; + for index in 1..delimiter.len() { + while matched > 0 && delimiter[index] != delimiter[matched] { + matched = prefix[matched - 1]; + } + if delimiter[index] == delimiter[matched] { + matched += 1; + } + prefix[index] = matched; + } + + Ok(Self { + delimiter, + prefix, + record_bytes: 0, + matched: 0, + }) + } + + fn observe(&mut self, bytes: &[u8]) -> Result<(), SelectError> { + if self.delimiter.len() == 1 { + return self.observe_single_byte_delimiter(bytes); + } + + for &byte in bytes { + self.record_bytes = self.record_bytes.checked_add(1).ok_or(SelectError::OverMaxRecordSize)?; + while self.matched > 0 && byte != self.delimiter[self.matched] { + self.matched = self.prefix[self.matched - 1]; + } + if byte == self.delimiter[self.matched] { + self.matched += 1; + } + if self.matched == self.delimiter.len() { + let payload_bytes = self.record_bytes - self.delimiter.len(); + if payload_bytes > MAX_SELECT_RECORD_BYTES { + return Err(SelectError::OverMaxRecordSize); + } + self.record_bytes = 0; + self.matched = 0; + } else if self.record_bytes - self.matched > MAX_SELECT_RECORD_BYTES { + return Err(SelectError::OverMaxRecordSize); + } + } + Ok(()) + } + + fn observe_single_byte_delimiter(&mut self, mut bytes: &[u8]) -> Result<(), SelectError> { + let delimiter = self.delimiter[0]; + while let Some(index) = bytes.iter().position(|byte| *byte == delimiter) { + let payload_bytes = self.record_bytes.checked_add(index).ok_or(SelectError::OverMaxRecordSize)?; + if payload_bytes > MAX_SELECT_RECORD_BYTES { + return Err(SelectError::OverMaxRecordSize); + } + self.record_bytes = 0; + bytes = &bytes[index + 1..]; + } + self.record_bytes = self + .record_bytes + .checked_add(bytes.len()) + .ok_or(SelectError::OverMaxRecordSize)?; + if self.record_bytes > MAX_SELECT_RECORD_BYTES { + return Err(SelectError::OverMaxRecordSize); + } + Ok(()) + } + + fn finish(&self) -> Result<(), SelectError> { + if self.record_bytes > MAX_SELECT_RECORD_BYTES { + Err(SelectError::OverMaxRecordSize) + } else { + Ok(()) + } + } +} + +#[cfg(test)] +pub(crate) async fn encode_compressed_fixture(format: CompressionFormat, input: &[u8]) -> Vec { + use tokio::io::AsyncWriteExt as _; + + let cursor = std::io::Cursor::new(Vec::new()); + match format { + CompressionFormat::Gzip => { + let mut encoder = async_compression::tokio::write::GzipEncoder::new(cursor); + encoder.write_all(input).await.expect("gzip fixture should encode"); + encoder.shutdown().await.expect("gzip fixture should finish"); + encoder.into_inner().into_inner() + } + CompressionFormat::Bzip2 => { + let mut encoder = async_compression::tokio::write::BzEncoder::new(cursor); + encoder.write_all(input).await.expect("bzip2 fixture should encode"); + encoder.shutdown().await.expect("bzip2 fixture should finish"); + encoder.into_inner().into_inner() + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::SelectInputMetricsSnapshot; + use futures::TryStreamExt; + use std::{io::Cursor, sync::Mutex as StdMutex, thread::ThreadId}; + use tokio::io::{AsyncWriteExt, DuplexStream}; + + struct ThreadRecordingReader { + inner: Cursor>, + thread_id: Arc>>, + } + + impl AsyncRead for ThreadRecordingReader { + fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buffer: &mut ReadBuf<'_>) -> Poll> { + let this = self.get_mut(); + let mut thread_id = this.thread_id.lock().expect("thread recorder mutex should not be poisoned"); + thread_id.get_or_insert_with(|| std::thread::current().id()); + drop(thread_id); + Pin::new(&mut this.inner).poll_read(cx, buffer) + } + } + + struct ErrorAfterReader { + inner: Cursor>, + end: u64, + failed: bool, + } + + impl AsyncRead for ErrorAfterReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buffer: &mut ReadBuf<'_>) -> Poll> { + if self.inner.position() < self.end { + return Pin::new(&mut self.inner).poll_read(cx, buffer); + } + if !self.failed { + self.failed = true; + return Poll::Ready(Err(io::Error::new(io::ErrorKind::ConnectionReset, "injected source failure"))); + } + Poll::Ready(Ok(())) + } + } + + async fn decode(format: CompressionFormat, compressed: Vec) -> (ObjectStoreResult>, SelectInputMetricsSnapshot) { + let compressed_len = u64::try_from(compressed.len()).expect("fixture length should fit in u64"); + let metrics = Arc::new(SelectInputMetrics::default()); + let stream = compressed_input_stream( + Box::new(Cursor::new(compressed)), + compressed_len, + format, + Arc::clone(&metrics), + b"\n".to_vec(), + u64::MAX, + None, + ) + .expect("record delimiter should be valid"); + let result = stream.try_collect::>().await.map(|chunks| chunks.concat()); + (result, metrics.snapshot()) + } + + fn select_error(error: &ObjectStoreError) -> Option { + find_error_source::(error).cloned() + } + + #[test] + fn protocol_limits_match_the_s3_select_contract() { + assert_eq!(MAX_SELECT_RECORD_BYTES, 1_048_576); + assert_eq!(MAX_SELECT_PROCESSED_BYTES, 5_497_558_138_880); + } + + #[tokio::test] + async fn gzip_and_bzip2_preserve_bytes_and_metric_boundaries() { + const INPUT: &[u8] = b"name,age\nAlice,30\n"; + for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] { + let compressed = encode_compressed_fixture(format, INPUT).await; + let compressed_len = u64::try_from(compressed.len()).expect("fixture length should fit in u64"); + let (decoded, metrics) = decode(format, compressed).await; + + assert_eq!(decoded.expect("valid compressed input should decode"), INPUT); + assert_eq!(metrics.bytes_scanned, compressed_len); + assert_eq!( + metrics.bytes_processed, + u64::try_from(INPUT.len()).expect("fixture length should fit in u64") + ); + } + } + + #[tokio::test] + async fn source_read_errors_are_not_reclassified_as_truncated_input() { + for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] { + let compressed = encode_compressed_fixture(format, b"name\nAlice\n").await; + let compressed_len = u64::try_from(compressed.len()).expect("fixture length should fit in u64"); + let expected_size = u64::try_from(compressed.len() + 1).expect("fixture length should fit in u64"); + let stream = compressed_input_stream( + Box::new(ErrorAfterReader { + inner: Cursor::new(compressed), + end: compressed_len, + failed: false, + }), + expected_size, + format, + Arc::new(SelectInputMetrics::default()), + b"\n".to_vec(), + u64::MAX, + None, + ) + .expect("record delimiter should be valid"); + + let error = stream + .try_collect::>() + .await + .expect_err("a storage read failure must terminate decoding"); + + assert_eq!(select_error(&error), None, "{format:?} must not report TruncatedInput: {error:?}"); + assert!( + find_error_source::(&error).is_some(), + "{format:?} must preserve the source error class" + ); + } + } + + #[tokio::test] + async fn source_read_error_inside_gzip_header_is_not_truncated_input() { + let compressed = encode_compressed_fixture(CompressionFormat::Gzip, b"name\nAlice\n").await; + let partial_header = compressed[..5].to_vec(); + let stream = compressed_input_stream( + Box::new(Cursor::new(partial_header)), + 10, + CompressionFormat::Gzip, + Arc::new(SelectInputMetrics::default()), + b"\n".to_vec(), + u64::MAX, + None, + ) + .expect("record delimiter should be valid"); + + let error = stream + .try_collect::>() + .await + .expect_err("a source failure inside the GZIP header must terminate decoding"); + + assert_eq!(select_error(&error), None, "source failure must not report TruncatedInput: {error:?}"); + assert!( + find_error_source::(&error).is_some(), + "the source error class must survive GZIP header validation" + ); + } + + #[tokio::test] + async fn concatenated_members_decode_in_order() { + for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] { + let mut compressed = encode_compressed_fixture(format, b"a\n").await; + compressed.extend_from_slice(&encode_compressed_fixture(format, b"b\n").await); + let (decoded, _) = decode(format, compressed).await; + assert_eq!(decoded.expect("valid concatenated members should decode"), b"a\nb\n"); + } + + let mut compressed = encode_compressed_fixture(CompressionFormat::Gzip, b"a\n").await; + let (second, _) = gzip_with_optional_header_crc(b"b\n").await; + compressed.extend_from_slice(&second); + let (decoded, _) = decode(CompressionFormat::Gzip, compressed).await; + assert_eq!(decoded.expect("optional headers must work in later GZIP members"), b"a\nb\n"); + } + + #[tokio::test] + async fn invalid_initial_headers_are_typed_as_invalid_compression() { + for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] { + let (result, _) = decode(format, b"not compressed\n".to_vec()).await; + let error = result.expect_err("invalid compression header must fail"); + let compression = format.name(); + assert_eq!(select_error(&error), Some(SelectError::InvalidCompressionFormatForObject { compression })); + assert_eq!( + select_error(&error).expect("typed compression error").to_string(), + format!("{compression} is not applicable to the queried object. Please correct the request and try again.") + ); + } + } + + #[tokio::test] + async fn fixed_header_variants_are_validated_before_decoding() { + let encoded = encode_compressed_fixture(CompressionFormat::Gzip, b"name\nAlice\n").await; + for (position, invalid) in [(2, 0), (3, 0x20)] { + let mut mutated = encoded.clone(); + mutated[position] = invalid; + let (result, _) = decode(CompressionFormat::Gzip, mutated).await; + assert_eq!( + select_error(&result.expect_err("invalid GZIP fixed header must fail")), + Some(CompressionFormat::Gzip.invalid_header_error()) + ); + } + + let (result, _) = decode(CompressionFormat::Bzip2, b"BZh0".to_vec()).await; + assert_eq!( + select_error(&result.expect_err("invalid BZIP2 block size must fail")), + Some(CompressionFormat::Bzip2.invalid_header_error()) + ); + } + + #[tokio::test] + async fn incomplete_valid_headers_are_typed_as_truncated_input() { + for (format, prefixes) in [ + (CompressionFormat::Gzip, vec![b"".as_slice(), b"\x1f", b"\x1f\x8b", b"\x1f\x8b\x08"]), + (CompressionFormat::Bzip2, vec![b"".as_slice(), b"B", b"BZ", b"BZh"]), + ] { + for prefix in prefixes { + let (result, _) = decode(format, prefix.to_vec()).await; + assert_eq!( + select_error(&result.expect_err("incomplete compression header must fail")), + Some(SelectError::TruncatedInput) + ); + } + } + } + + async fn gzip_with_optional_header_crc(input: &[u8]) -> (Vec, usize) { + const EXTRA: &[u8] = b"s3-select"; + const FILE_NAME: &[u8] = b"select.csv"; + const COMMENT: &[u8] = b"fixture"; + + let encoded = encode_compressed_fixture(CompressionFormat::Gzip, input).await; + let mut header = encoded[..10].to_vec(); + header[3] = GZIP_FLAG_EXTRA | GZIP_FLAG_NAME | GZIP_FLAG_COMMENT | GZIP_FLAG_HEADER_CRC; + header.extend_from_slice( + &u16::try_from(EXTRA.len()) + .expect("GZIP extra fixture should fit its length field") + .to_le_bytes(), + ); + header.extend_from_slice(EXTRA); + header.extend_from_slice(FILE_NAME); + header.push(0); + header.extend_from_slice(COMMENT); + header.push(0); + let mut digest = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + digest.update(&header); + let crc = digest.finalize(); + let crc = u16::try_from(crc & u64::from(u16::MAX)).expect("masked header CRC should fit in u16"); + let crc_offset = header.len(); + header.extend_from_slice(&crc.to_le_bytes()); + header.extend_from_slice(&encoded[10..]); + (header, crc_offset) + } + + async fn gzip_with_text_header(input: &[u8], flag: u8, field_bytes: usize) -> Vec { + let encoded = encode_compressed_fixture(CompressionFormat::Gzip, input).await; + let mut member = encoded[..10].to_vec(); + member[3] = flag; + member.extend(std::iter::repeat_n(b'x', field_bytes)); + member.push(0); + member.extend_from_slice(&encoded[10..]); + member + } + + async fn gzip_with_text_header_crc(input: &[u8], flag: u8, field_bytes: usize) -> (Vec, usize) { + let encoded = encode_compressed_fixture(CompressionFormat::Gzip, input).await; + let mut member = encoded[..10].to_vec(); + member[3] = flag | GZIP_FLAG_HEADER_CRC; + member.extend(std::iter::repeat_n(b'x', field_bytes)); + member.push(0); + let mut digest = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + digest.update(&member); + let crc = u16::try_from(digest.finalize() & u64::from(u16::MAX)).expect("masked header CRC should fit in u16"); + let crc_offset = member.len(); + member.extend_from_slice(&crc.to_le_bytes()); + member.extend_from_slice(&encoded[10..]); + (member, crc_offset) + } + + async fn gzip_with_extra_header_crc(input: &[u8], extra_bytes: usize) -> (Vec, usize) { + let encoded = encode_compressed_fixture(CompressionFormat::Gzip, input).await; + let mut member = encoded[..10].to_vec(); + member[3] = GZIP_FLAG_EXTRA | GZIP_FLAG_HEADER_CRC; + member.extend_from_slice( + &u16::try_from(extra_bytes) + .expect("GZIP extra fixture should fit its length field") + .to_le_bytes(), + ); + member.extend(std::iter::repeat_n(b'x', extra_bytes)); + let mut digest = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc); + digest.update(&member); + let crc = u16::try_from(digest.finalize() & u64::from(u16::MAX)).expect("masked header CRC should fit in u16"); + let crc_offset = member.len(); + member.extend_from_slice(&crc.to_le_bytes()); + member.extend_from_slice(&encoded[10..]); + (member, crc_offset) + } + + #[tokio::test] + async fn gzip_optional_header_crc_is_validated_without_error_strings() { + const INPUT: &[u8] = b"name\nAlice\n"; + + let (encoded, crc_offset) = gzip_with_optional_header_crc(INPUT).await; + let (decoded, _) = decode(CompressionFormat::Gzip, encoded.clone()).await; + assert_eq!(decoded.expect("valid optional GZIP header should decode"), INPUT); + + let mut corrupt = encoded; + corrupt[crc_offset] ^= 0xff; + let (result, _) = decode(CompressionFormat::Gzip, corrupt).await; + assert_eq!( + select_error(&result.expect_err("invalid GZIP header CRC must fail")), + Some(SelectError::InvalidCompressionFormatForObject { + compression: CompressionFormat::Gzip.name(), + }) + ); + + let mut truncated = encode_compressed_fixture(CompressionFormat::Gzip, INPUT).await; + truncated.truncate(10); + truncated[3] = GZIP_FLAG_NAME; + truncated.extend_from_slice(b"unterminated-name"); + let (result, _) = decode(CompressionFormat::Gzip, truncated).await; + assert_eq!( + select_error(&result.expect_err("incomplete optional GZIP header must fail")), + Some(SelectError::TruncatedInput) + ); + } + + #[tokio::test] + async fn gzip_optional_header_can_cross_source_chunks() { + const INPUT: &[u8] = b"name\nAlice\n"; + const LARGE_GZIP_TEXT_FIELD_BYTES: usize = SELECT_DECODE_CHUNK_BYTES * 3 + 17; + + let encoded = encode_compressed_fixture(CompressionFormat::Gzip, INPUT).await; + let mut with_name = encoded[..10].to_vec(); + with_name[3] = GZIP_FLAG_NAME; + with_name.extend(std::iter::repeat_n(b'x', LARGE_GZIP_TEXT_FIELD_BYTES)); + with_name.push(0); + with_name.extend_from_slice(&encoded[10..]); + let expected_scanned = u64::try_from(with_name.len()).expect("fixture length should fit in u64"); + let (mut writer, reader) = tokio::io::duplex(257); + let writer = tokio::spawn(async move { + writer + .write_all(&with_name) + .await + .expect("fixture source should accept bytes"); + writer.shutdown().await.expect("fixture source should close"); + }); + let metrics = Arc::new(SelectInputMetrics::default()); + let stream = compressed_input_stream( + Box::new(reader), + expected_scanned, + CompressionFormat::Gzip, + Arc::clone(&metrics), + b"\n".to_vec(), + u64::MAX, + None, + ) + .expect("record delimiter should be valid"); + let decoded = stream + .try_collect::>() + .await + .map(|chunks| chunks.concat()) + .expect("chunked optional GZIP names should decode"); + writer.await.expect("fixture writer should complete"); + + assert_eq!(decoded, INPUT); + assert_eq!(metrics.snapshot().bytes_scanned, expected_scanned); + } + + #[tokio::test] + async fn valid_512_byte_text_fields_decode_in_every_gzip_member() { + for flag in [GZIP_FLAG_NAME, GZIP_FLAG_COMMENT] { + let mut encoded = encode_compressed_fixture(CompressionFormat::Gzip, b"a\n").await; + encoded.extend_from_slice(&gzip_with_text_header(b"b\n", flag, 512).await); + let (decoded, _) = decode(CompressionFormat::Gzip, encoded).await; + assert_eq!(decoded.expect("RFC 1952 does not limit zero-terminated text fields"), b"a\nb\n"); + } + } + + #[tokio::test] + async fn long_text_field_header_crc_accumulates_across_source_chunks() { + const FIELD_BYTES: usize = SELECT_DECODE_CHUNK_BYTES * 3 + 17; + + for flag in [GZIP_FLAG_NAME, GZIP_FLAG_COMMENT] { + let first = encode_compressed_fixture(CompressionFormat::Gzip, b"a\n").await; + let (second, crc_offset) = gzip_with_text_header_crc(b"b\n", flag, FIELD_BYTES).await; + + let mut valid = first.clone(); + valid.extend_from_slice(&second); + let (decoded, _) = decode(CompressionFormat::Gzip, valid).await; + assert_eq!(decoded.expect("multi-chunk GZIP header CRC should validate"), b"a\nb\n"); + + let mut corrupt_second = second; + corrupt_second[crc_offset] ^= 0xff; + let mut corrupt = first; + corrupt.extend_from_slice(&corrupt_second); + let (result, _) = decode(CompressionFormat::Gzip, corrupt).await; + assert_eq!( + select_error(&result.expect_err("corrupt multi-chunk GZIP header CRC must fail")), + Some(SelectError::TruncatedInput) + ); + } + } + + #[tokio::test] + async fn maximum_extra_field_header_crc_accumulates_across_source_chunks() { + const INPUT: &[u8] = b"name\nAlice\n"; + let (valid, crc_offset) = gzip_with_extra_header_crc(INPUT, usize::from(u16::MAX)).await; + + let (decoded, _) = decode(CompressionFormat::Gzip, valid.clone()).await; + assert_eq!(decoded.expect("maximum GZIP extra field should decode"), INPUT); + + let mut corrupt = valid; + corrupt[crc_offset] ^= 0xff; + let (result, _) = decode(CompressionFormat::Gzip, corrupt).await; + assert_eq!( + select_error(&result.expect_err("corrupt multi-chunk GZIP extra-field CRC must fail")), + Some(CompressionFormat::Gzip.invalid_header_error()) + ); + } + + #[test] + fn processed_byte_limit_is_independent_of_live_memory_budget() { + assert_eq!(processed_bytes_limit(), MAX_SELECT_PROCESSED_BYTES); + assert!(processed_bytes_limit() > 80 * 1024 * 1024); + } + + #[test] + fn cooperative_reader_yields_after_bounded_ready_input() { + let source = Cursor::new(vec![0_u8; SELECT_DECODE_CHUNK_BYTES + 1]); + let mut reader = CooperativeReader::new(source); + let mut output = vec![0_u8; SELECT_DECODE_CHUNK_BYTES + 1]; + let mut read_buf = ReadBuf::new(&mut output); + let waker = futures::task::noop_waker_ref(); + let mut context = Context::from_waker(waker); + + assert!(Pin::new(&mut reader).poll_read(&mut context, &mut read_buf).is_ready()); + assert_eq!(read_buf.filled().len(), SELECT_DECODE_CHUNK_BYTES); + assert!(Pin::new(&mut reader).poll_read(&mut context, &mut read_buf).is_pending()); + assert!(Pin::new(&mut reader).poll_read(&mut context, &mut read_buf).is_ready()); + assert_eq!(read_buf.filled().len(), SELECT_DECODE_CHUNK_BYTES + 1); + } + + #[tokio::test] + async fn bzip2_decoded_output_yields_at_the_cooperative_quantum() { + let input = vec![b'x'; SELECT_DECODE_CHUNK_BYTES * 2]; + let compressed = encode_compressed_fixture(CompressionFormat::Bzip2, &input).await; + let compressed_len = u64::try_from(compressed.len()).expect("fixture length should fit in u64"); + let mut reader = compressed_input_reader( + Box::new(Cursor::new(compressed)), + compressed_len, + CompressionFormat::Bzip2, + Arc::new(SelectInputMetrics::default()), + processed_bytes_limit(), + None, + ); + let mut output = vec![0; SELECT_DECODE_CHUNK_BYTES * 2]; + let mut read_buf = ReadBuf::new(&mut output); + while read_buf.filled().len() < SELECT_DECODE_CHUNK_BYTES { + futures::future::poll_fn(|cx| Pin::new(&mut reader).poll_read(cx, &mut read_buf)) + .await + .expect("valid BZIP2 input should decode"); + } + assert_eq!(read_buf.filled().len(), SELECT_DECODE_CHUNK_BYTES); + + let waker = futures::task::noop_waker_ref(); + let mut context = Context::from_waker(waker); + assert!(Pin::new(&mut reader).poll_read(&mut context, &mut read_buf).is_pending()); + } + + #[tokio::test(flavor = "current_thread")] + async fn bzip2_decoder_polls_source_off_runtime_thread() { + let input = b"name\nAlice\n"; + let compressed = encode_compressed_fixture(CompressionFormat::Bzip2, input).await; + let compressed_len = u64::try_from(compressed.len()).expect("fixture length should fit in u64"); + let runtime_thread = std::thread::current().id(); + let source_thread = Arc::new(StdMutex::new(None)); + let source = ThreadRecordingReader { + inner: Cursor::new(compressed), + thread_id: Arc::clone(&source_thread), + }; + let mut reader = compressed_input_reader( + Box::new(source), + compressed_len, + CompressionFormat::Bzip2, + Arc::new(SelectInputMetrics::default()), + processed_bytes_limit(), + None, + ); + let mut decoded = Vec::new(); + + reader + .read_to_end(&mut decoded) + .await + .expect("valid BZIP2 input should decode off the runtime thread"); + + assert_eq!(decoded, input); + assert_ne!( + source_thread + .lock() + .expect("thread recorder mutex should not be poisoned") + .expect("compressed source should be polled"), + runtime_thread, + "BZIP2 decoding must not poll codec work on a Tokio runtime worker" + ); + } + + #[tokio::test] + async fn decoder_thread_holds_query_admission_until_exit() { + let admission = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = Arc::new( + Arc::clone(&admission) + .try_acquire_owned() + .expect("query admission should be available"), + ); + let (started_tx, started_rx) = oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let (decoded_tx, _decoded_rx) = mpsc::channel(1); + + spawn_decoder_thread("s3select-guard-test", decoded_tx, Some(permit), move || { + let _ = started_tx.send(()); + release_rx.recv().map_err(io::Error::other) + }); + + started_rx.await.expect("decoder thread should start"); + assert!(Arc::clone(&admission).try_acquire_owned().is_err()); + release_tx.send(()).expect("decoder thread should be releasable"); + let recovered = tokio::time::timeout(std::time::Duration::from_secs(1), Arc::clone(&admission).acquire_owned()) + .await + .expect("decoder thread should release admission promptly") + .expect("query admission should remain open"); + drop(recovered); + } + + #[test] + fn bzip2_decoder_does_not_wait_for_tokio_blocking_pool() { + let runtime = tokio::runtime::Builder::new_multi_thread() + .worker_threads(1) + .max_blocking_threads(1) + .enable_all() + .build() + .expect("test runtime should build"); + + runtime.block_on(async { + let (blocker_started_tx, blocker_started_rx) = tokio::sync::oneshot::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + let blocker = tokio::task::spawn_blocking(move || { + let _ = blocker_started_tx.send(()); + let _ = release_rx.recv(); + }); + blocker_started_rx.await.expect("blocking worker should be occupied"); + + let input = b"name\nAlice\n"; + let compressed = encode_compressed_fixture(CompressionFormat::Bzip2, input).await; + let decode_result = + tokio::time::timeout(std::time::Duration::from_secs(2), decode(CompressionFormat::Bzip2, compressed)).await; + + let _ = release_tx.send(()); + blocker.await.expect("blocking worker should finish"); + + let (decoded, _) = decode_result.expect("BZIP2 decoder must not queue behind Tokio blocking work"); + assert_eq!(decoded.expect("valid BZIP2 input should decode"), input); + }); + } + + #[tokio::test] + async fn processed_reader_streams_past_the_default_memory_budget() { + let expected = 64 * 1024 * 1024 + 1; + let metrics = Arc::new(SelectInputMetrics::default()); + let source = tokio::io::repeat(b'x').take(expected); + let mut reader = ProcessedReader::new(source, metrics.recorder(), processed_bytes_limit()); + let mut buffer = vec![0; SELECT_DECODE_CHUNK_BYTES]; + let mut total = 0_u64; + + loop { + let read = reader + .read(&mut buffer) + .await + .expect("streaming input within the expansion budget should pass"); + if read == 0 { + break; + } + total += u64::try_from(read).expect("read buffer length fits in u64"); + } + + assert_eq!(total, expected); + assert_eq!(metrics.snapshot().bytes_processed, expected); + } + + #[tokio::test] + async fn truncated_and_corrupt_gzip_are_typed_as_truncated_input() { + let encoded = encode_compressed_fixture(CompressionFormat::Gzip, b"name\nAlice\n").await; + + let mut truncated = encoded.clone(); + truncated.truncate(truncated.len() - 1); + let (result, _) = decode(CompressionFormat::Gzip, truncated).await; + assert_eq!( + select_error(&result.expect_err("truncated gzip trailer must fail")), + Some(SelectError::TruncatedInput) + ); + + let mut corrupt = encoded; + let checksum_index = corrupt.len() - 8; + corrupt[checksum_index] ^= 0xff; + let (result, _) = decode(CompressionFormat::Gzip, corrupt).await; + assert_eq!( + select_error(&result.expect_err("corrupt gzip checksum must fail")), + Some(SelectError::TruncatedInput) + ); + + let mut corrupt_size = encode_compressed_fixture(CompressionFormat::Gzip, b"name\nAlice\n").await; + let size_index = corrupt_size.len() - 1; + corrupt_size[size_index] ^= 0xff; + let (result, _) = decode(CompressionFormat::Gzip, corrupt_size).await; + assert_eq!( + select_error(&result.expect_err("corrupt gzip uncompressed size must fail")), + Some(SelectError::TruncatedInput) + ); + } + + #[tokio::test] + async fn truncated_bzip2_is_typed_as_truncated_input() { + let mut encoded = encode_compressed_fixture(CompressionFormat::Bzip2, b"name\nAlice\n").await; + encoded.truncate(encoded.len() - 1); + let (result, _) = decode(CompressionFormat::Bzip2, encoded).await; + assert_eq!( + select_error(&result.expect_err("truncated bzip2 trailer must fail")), + Some(SelectError::TruncatedInput) + ); + + let mut corrupt = encode_compressed_fixture(CompressionFormat::Bzip2, b"name\nAlice\n").await; + let checksum_index = corrupt.len() - 2; + corrupt[checksum_index] ^= 0xff; + let (result, _) = decode(CompressionFormat::Bzip2, corrupt).await; + assert_eq!( + select_error(&result.expect_err("corrupt bzip2 checksum must fail")), + Some(SelectError::TruncatedInput) + ); + } + + #[tokio::test] + async fn trailing_non_member_bytes_fail_closed() { + for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] { + let mut encoded = encode_compressed_fixture(format, b"name\nAlice\n").await; + encoded.extend_from_slice(b"trailing garbage"); + let (result, _) = decode(format, encoded).await; + assert_eq!( + select_error(&result.expect_err("trailing bytes must not be ignored")), + Some(SelectError::TruncatedInput) + ); + } + } + + #[tokio::test] + async fn oversized_compressed_record_fails_before_unbounded_buffering() { + let mut input = vec![b'x'; MAX_SELECT_RECORD_BYTES + 1]; + input.push(b'\n'); + let compressed = encode_compressed_fixture(CompressionFormat::Gzip, &input).await; + let (result, _) = decode(CompressionFormat::Gzip, compressed).await; + assert_eq!( + select_error(&result.expect_err("oversized compressed record must fail")), + Some(SelectError::OverMaxRecordSize) + ); + } + + #[tokio::test] + async fn one_megabyte_record_is_accepted_with_or_without_delimiter() { + for terminated in [false, true] { + let mut input = vec![b'x'; MAX_SELECT_RECORD_BYTES]; + if terminated { + input.push(b'\n'); + } + let compressed = encode_compressed_fixture(CompressionFormat::Gzip, &input).await; + let (decoded, _) = decode(CompressionFormat::Gzip, compressed).await; + assert_eq!(decoded.expect("record at the protocol limit should decode"), input); + } + } + + #[tokio::test] + async fn processed_byte_limit_rejects_many_small_records() { + let input = b"{}\n".repeat(1024); + let compressed = encode_compressed_fixture(CompressionFormat::Gzip, &input).await; + let compressed_len = u64::try_from(compressed.len()).expect("fixture length should fit in u64"); + let exact_stream = compressed_input_stream( + Box::new(Cursor::new(compressed.clone())), + compressed_len, + CompressionFormat::Gzip, + Arc::new(SelectInputMetrics::default()), + b"\n".to_vec(), + u64::try_from(input.len()).expect("fixture length should fit in u64"), + None, + ) + .expect("record delimiter should be valid"); + assert_eq!( + exact_stream + .try_collect::>() + .await + .expect("decoded bytes at the processed limit should pass") + .concat(), + input + ); + + let stream = compressed_input_stream( + Box::new(Cursor::new(compressed)), + compressed_len, + CompressionFormat::Gzip, + Arc::new(SelectInputMetrics::default()), + b"\n".to_vec(), + u64::try_from(input.len() - 1).expect("fixture length should fit in u64"), + None, + ) + .expect("record delimiter should be valid"); + + let error = stream + .try_collect::>() + .await + .expect_err("decoded bytes over the decompression budget must fail"); + assert_eq!(select_error(&error), Some(SelectError::ResourceExhausted)); + } + + #[tokio::test] + async fn oversized_final_record_with_partial_delimiter_fails_at_eof() { + let mut input = vec![b'x'; MAX_SELECT_RECORD_BYTES + 1]; + input.push(b'\r'); + let compressed = encode_compressed_fixture(CompressionFormat::Gzip, &input).await; + let compressed_len = u64::try_from(compressed.len()).expect("fixture length should fit in u64"); + let metrics = Arc::new(SelectInputMetrics::default()); + let stream = compressed_input_stream( + Box::new(Cursor::new(compressed)), + compressed_len, + CompressionFormat::Gzip, + metrics, + b"\r\n".to_vec(), + u64::MAX, + None, + ) + .expect("record delimiter should be valid"); + + let error = stream + .try_collect::>() + .await + .expect_err("oversized unterminated record must fail"); + assert_eq!(select_error(&error), Some(SelectError::OverMaxRecordSize)); + } + + struct DropObservedReader { + inner: DuplexStream, + dropped: Arc, + } + + impl AsyncRead for DropObservedReader { + fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_read(cx, buf) + } + } + + impl Drop for DropObservedReader { + fn drop(&mut self) { + self.dropped.store(true, std::sync::atomic::Ordering::Release); + } + } + + #[tokio::test] + async fn dropping_decoder_stream_releases_source_without_background_work() { + let input = b"a\n".repeat(MAX_SELECT_RECORD_BYTES); + for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] { + let compressed = encode_compressed_fixture(format, &input).await; + let compressed_len = u64::try_from(compressed.len() + 1).expect("fixture length should fit in u64"); + let (source, mut peer) = tokio::io::duplex(compressed.len()); + peer.write_all(&compressed).await.expect("write compressed fixture"); + let dropped = Arc::new(std::sync::atomic::AtomicBool::new(false)); + let admission = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = Arc::new( + Arc::clone(&admission) + .try_acquire_owned() + .expect("query admission should be available"), + ); + let mut stream = compressed_input_stream( + Box::new(DropObservedReader { + inner: source, + dropped: Arc::clone(&dropped), + }), + compressed_len, + format, + Arc::new(SelectInputMetrics::default()), + b"\n".to_vec(), + u64::MAX, + Some(permit), + ) + .expect("record delimiter should be valid"); + let first = tokio::time::timeout(std::time::Duration::from_secs(1), stream.next()) + .await + .expect("decoder should produce a chunk before source EOF") + .expect("decoder stream should produce a chunk") + .expect("valid partial decode should succeed"); + assert!(!first.is_empty()); + + drop(stream); + + tokio::time::timeout(std::time::Duration::from_secs(1), async { + while !dropped.load(std::sync::atomic::Ordering::Acquire) { + tokio::task::yield_now().await; + } + }) + .await + .expect("dropping decoded output must cancel the blocked source read"); + let recovered = tokio::time::timeout(std::time::Duration::from_secs(1), Arc::clone(&admission).acquire_owned()) + .await + .expect("decoder exit should release query admission") + .expect("query admission should remain open"); + drop(recovered); + } + } +} diff --git a/crates/s3select-api/src/lib.rs b/crates/s3select-api/src/lib.rs index 7662b4a78..a443a07a1 100644 --- a/crates/s3select-api/src/lib.rs +++ b/crates/s3select-api/src/lib.rs @@ -23,6 +23,7 @@ use datafusion::{ use std::{error::Error as StdError, fmt::Display}; use thiserror::Error; +mod input_stream; mod metrics; pub mod object_store; pub mod query; @@ -79,6 +80,9 @@ pub enum SelectError { #[error("The file is not in a supported compression format. Only GZIP and BZIP2 are supported.")] InvalidCompressionFormat, + #[error("{compression} is not applicable to the queried object. Please correct the request and try again.")] + InvalidCompressionFormatForObject { compression: &'static str }, + #[error("The data source type is not valid. Only CSV, JSON, and Parquet are supported.")] InvalidDataSource, @@ -87,6 +91,9 @@ pub enum SelectError { )] TruncatedInput, + #[error("Scan range queries are not supported on this type of object.")] + UnsupportedScanRangeInput, + #[error("An error occurred while parsing the CSV file. Check the file and try again.")] CsvParsingError, @@ -96,6 +103,9 @@ pub enum SelectError { #[error("An error occurred while parsing the Parquet file. Check the file and try again.")] ParquetParsingError, + #[error("The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB.")] + OverMaxRecordSize, + #[error("{message}")] ParseSelectFailure { message: String }, diff --git a/crates/s3select-api/src/metrics.rs b/crates/s3select-api/src/metrics.rs index d38eefbc8..619f1e895 100644 --- a/crates/s3select-api/src/metrics.rs +++ b/crates/s3select-api/src/metrics.rs @@ -12,7 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::atomic::{AtomicU64, Ordering}; +use arc_swap::ArcSwap; +use std::sync::{ + Arc, + atomic::{AtomicU64, Ordering}, +}; #[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] pub struct SelectInputMetricsSnapshot { @@ -20,33 +24,72 @@ pub struct SelectInputMetricsSnapshot { pub bytes_processed: u64, } -#[derive(Debug, Default)] +#[derive(Debug)] pub struct SelectInputMetrics { + active: ArcSwap, +} + +#[derive(Debug, Default)] +struct SelectInputMetricBank { uncompressed_bytes: AtomicU64, + compressed_bytes_scanned: AtomicU64, + compressed_bytes_processed: AtomicU64, +} + +#[derive(Clone, Debug)] +pub(crate) struct SelectInputMetricsRecorder { + bank: Arc, +} + +impl Default for SelectInputMetrics { + fn default() -> Self { + Self { + active: ArcSwap::from_pointee(SelectInputMetricBank::default()), + } + } } impl SelectInputMetrics { pub fn snapshot(&self) -> SelectInputMetricsSnapshot { - let uncompressed_bytes = self.uncompressed_bytes.load(Ordering::Relaxed); + let bank = self.active.load(); + let uncompressed_bytes = bank.uncompressed_bytes.load(Ordering::Relaxed); SelectInputMetricsSnapshot { - bytes_scanned: uncompressed_bytes, - bytes_processed: uncompressed_bytes, + bytes_scanned: uncompressed_bytes.saturating_add(bank.compressed_bytes_scanned.load(Ordering::Relaxed)), + bytes_processed: uncompressed_bytes.saturating_add(bank.compressed_bytes_processed.load(Ordering::Relaxed)), } } - pub(crate) fn record_uncompressed(&self, bytes: usize) { - let increment = u64::try_from(bytes).unwrap_or(u64::MAX); - let _ = self - .uncompressed_bytes - .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(increment))); + pub(crate) fn recorder(&self) -> SelectInputMetricsRecorder { + SelectInputMetricsRecorder { + bank: self.active.load_full(), + } } - /// Clears planner-only reads before query execution begins. + /// Publishes a fresh bank so late planner writes remain isolated. pub fn reset(&self) { - self.uncompressed_bytes.store(0, Ordering::Relaxed); + self.active.store(Arc::new(SelectInputMetricBank::default())); } } +impl SelectInputMetricsRecorder { + pub(crate) fn record_uncompressed(&self, bytes: usize) { + saturating_add(&self.bank.uncompressed_bytes, bytes); + } + + pub(crate) fn record_scanned(&self, bytes: usize) { + saturating_add(&self.bank.compressed_bytes_scanned, bytes); + } + + pub(crate) fn record_processed(&self, bytes: usize) { + saturating_add(&self.bank.compressed_bytes_processed, bytes); + } +} + +fn saturating_add(counter: &AtomicU64, bytes: usize) { + let increment = u64::try_from(bytes).unwrap_or(u64::MAX); + let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| Some(current.saturating_add(increment))); +} + #[cfg(test)] mod tests { use super::*; @@ -54,7 +97,7 @@ mod tests { #[test] fn records_uncompressed_input_at_both_boundaries() { let metrics = SelectInputMetrics::default(); - metrics.record_uncompressed(7); + metrics.recorder().record_uncompressed(7); assert_eq!( metrics.snapshot(), @@ -68,21 +111,51 @@ mod tests { #[test] fn counters_saturate_instead_of_wrapping() { let metrics = SelectInputMetrics::default(); - metrics.uncompressed_bytes.store(u64::MAX - 1, Ordering::Relaxed); + metrics + .active + .load() + .uncompressed_bytes + .store(u64::MAX - 1, Ordering::Relaxed); - metrics.record_uncompressed(2); + metrics.recorder().record_uncompressed(2); assert_eq!(metrics.snapshot().bytes_scanned, u64::MAX); assert_eq!(metrics.snapshot().bytes_processed, u64::MAX); } + #[test] + fn compressed_boundaries_are_counted_independently() { + let metrics = SelectInputMetrics::default(); + let recorder = metrics.recorder(); + recorder.record_scanned(39); + recorder.record_processed(19); + + assert_eq!( + metrics.snapshot(), + SelectInputMetricsSnapshot { + bytes_scanned: 39, + bytes_processed: 19, + } + ); + } + #[test] fn reset_clears_schema_inference_bytes() { let metrics = SelectInputMetrics::default(); - metrics.record_uncompressed(9); + let planning = metrics.recorder(); + planning.record_uncompressed(9); metrics.reset(); + planning.record_uncompressed(5); + let execution = metrics.recorder(); + execution.record_uncompressed(3); - assert_eq!(metrics.snapshot(), SelectInputMetricsSnapshot::default()); + assert_eq!( + metrics.snapshot(), + SelectInputMetricsSnapshot { + bytes_scanned: 3, + bytes_processed: 3, + } + ); } } diff --git a/crates/s3select-api/src/object_store.rs b/crates/s3select-api/src/object_store.rs index 20d165623..c135000aa 100644 --- a/crates/s3select-api/src/object_store.rs +++ b/crates/s3select-api/src/object_store.rs @@ -13,9 +13,14 @@ // limitations under the License. use crate::{ - PrepareSelectObjectSnapshotError, SELECT_DEFAULT_READ_BUFFER_SIZE, SelectError, SelectGetObjectReader, SelectInputMetrics, - SelectObjectOptions, SelectObjectSnapshot, SelectObjectSnapshotReadError, SelectStorageError, SelectStore, - SnapshotConsistencyError, + PrepareSelectObjectSnapshotError, QueryError, SELECT_DEFAULT_READ_BUFFER_SIZE, SelectError, SelectGetObjectReader, + SelectInputMetrics, SelectObjectOptions, SelectObjectSnapshot, SelectObjectSnapshotReadError, SelectStorageError, + SelectStore, SnapshotConsistencyError, + input_stream::{ + CompressionFormat, MAX_SELECT_RECORD_BYTES, SELECT_DECODE_CHUNK_BYTES, SelectInputReader, compressed_input_reader, + compressed_input_stream, input_io_error, processed_bytes_limit, + }, + metrics::SelectInputMetricsRecorder, query::{ ast::{JsonPathSegment, JsonSource}, parser::RustFsDialect, @@ -52,12 +57,15 @@ use s3s::header::{ X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, }; -use s3s::{S3Error, S3ErrorCode, S3Result, dto::SelectObjectContentInput}; +use s3s::{ + S3Error, S3ErrorCode, S3Result, + dto::{CompressionType, InputSerialization, ScanRange, SelectObjectContentInput}, +}; use std::collections::VecDeque; use std::ops::Range; -use std::sync::Arc; #[cfg(test)] use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, atomic::AtomicBool}; use tokio::{io::AsyncReadExt, sync::OnceCell}; use tokio_util::io::ReaderStream; use transform_stream::AsyncTryStream; @@ -68,6 +76,15 @@ fn select_default_read_buffer_size_u64() -> u64 { u64::try_from(SELECT_DEFAULT_READ_BUFFER_SIZE).unwrap_or(u64::MAX) } +fn compression_format(input: &InputSerialization) -> Result, SelectError> { + match input.compression_type.as_ref().map(|value| value.as_str()) { + None | Some(CompressionType::NONE) => Ok(None), + Some(CompressionType::GZIP) => Ok(Some(CompressionFormat::Gzip)), + Some(CompressionType::BZIP2) => Ok(Some(CompressionFormat::Bzip2)), + Some(_) => Err(SelectError::InvalidCompressionFormat), + } +} + /// Maximum allowed object size for JSON DOCUMENT mode. /// /// JSON DOCUMENT format requires loading the entire file into memory for DOM @@ -89,11 +106,18 @@ fn select_default_read_buffer_size_u64() -> u64 { pub const MAX_JSON_DOCUMENT_BYTES: u64 = 128 * 1024 * 1024; const JSON_DOCUMENT_MEMORY_RESERVATION_MULTIPLIER: usize = 64; const JSON_SCALAR_COLUMN_MEMORY_RESERVATION_MULTIPLIER: usize = 14; +const JSON_CANCELLATION_CHECK_BYTES: usize = 64 * 1024; +const JSON_CANCELLATION_CHECK_KEYS: usize = 1024; pub const INVALID_SCAN_RANGE_MESSAGE: &str = "The value of a parameter in ScanRange element is invalid. Check the service API documentation and try again."; const NORMALIZED_RECORD_DELIMITER: &[u8] = b"\r\n"; const NORMALIZED_FIELD_DELIMITER: &[u8] = &[DEFAULT_DELIMITER]; +/// Returns true for the MinIO-compatible full-scan range marker. +pub fn is_noop_scan_range(scan_range: &ScanRange) -> bool { + scan_range.start == Some(0) && scan_range.end.is_none() +} + pub struct EcObjectStore { input: Arc, need_convert: bool, @@ -312,6 +336,9 @@ impl EcObjectStore { let Some(scan_range) = self.input.request.scan_range.as_ref() else { return Ok(None); }; + if is_noop_scan_range(scan_range) { + return Ok(None); + } scan_range_from_bounds(scan_range.start, scan_range.end, object_size) } @@ -766,6 +793,22 @@ impl ObjectStore for EcObjectStore { // this instance's immutable snapshot; later operations reuse it. let snapshot = self.snapshot(options.version.as_deref()).await?; let original_size = snapshot.logical_size(); + let compression = compression_format(&self.input.request.input_serialization).map_err(|source| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(source), + })?; + let has_effective_request_range = self + .input + .request + .scan_range + .as_ref() + .is_some_and(|scan_range| !is_noop_scan_range(scan_range)); + if compression.is_some() && (options.range.is_some() || has_effective_request_range) { + return Err(o_Error::Generic { + store: "EcObjectStore", + source: Box::new(SelectError::UnsupportedScanRangeInput), + }); + } let object_info = snapshot.object_info(); let meta = ObjectMeta { location: location.clone(), @@ -791,7 +834,7 @@ impl ObjectStore for EcObjectStore { } let record_delimiter = self.record_delimiter_for_conversion(); - let needs_scan_context = options.range.is_none() && self.input.request.scan_range.is_some(); + let needs_scan_context = options.range.is_none() && has_effective_request_range; let scan_context = if needs_scan_context { if let Some(scan_range) = self.scan_range(original_size)? { let delimiter = self.record_delimiter(); @@ -813,7 +856,51 @@ impl ObjectStore for EcObjectStore { }; let meter_input = self.input.request.input_serialization.parquet.is_none(); - let payload = if options.range.is_some() { + let payload = if let Some(compression) = compression { + let max_processed_bytes = processed_bytes_limit(); + let query_guard = match self.query_tracker.as_ref() { + Some(query_tracker) => Some(query_tracker.query_guard().ok_or_else(|| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(QueryError::Cancel), + })?), + None => None, + }; + if self.is_json_document { + let reader = compressed_input_reader( + reader.stream, + original_size, + compression, + Arc::clone(&self.input_metrics), + max_processed_bytes, + query_guard, + ); + let stream = compressed_json_document_ndjson_stream( + reader, + self.json_source.clone(), + Arc::clone(&self.memory_pool), + self.query_tracker.clone(), + ); + GetResultPayload::Stream(stream) + } else { + let input_record_delimiter = if self.input.request.input_serialization.csv.is_some() { + self.record_delimiter() + } else { + b"\n".to_vec() + }; + let stream = compressed_input_stream( + reader.stream, + original_size, + compression, + Arc::clone(&self.input_metrics), + input_record_delimiter, + max_processed_bytes, + query_guard, + )?; + let stream = + convert_csv_delimiter_stream(stream, record_delimiter, self.need_convert.then(|| self.delimiter.clone())); + GetResultPayload::Stream(stream) + } + } else if options.range.is_some() { let size = usize::try_from(result_range.end - result_range.start).map_err(|err| o_Error::Generic { store: "EcObjectStore", source: Box::new(err), @@ -851,8 +938,9 @@ impl ObjectStore for EcObjectStore { let delimiter = self.record_delimiter(); let include_header = self.csv_has_header(); let header = if include_header && read_start > 0 { + let input_metrics = self.input_metrics.recorder(); let header = self.read_header_record(original_size, &delimiter).await?; - self.input_metrics.record_uncompressed(header.len()); + input_metrics.record_uncompressed(header.len()); Some(header) } else { None @@ -1197,7 +1285,7 @@ impl ScanRangeState { /// scalar/object root) is yielded as a separate [`Bytes`] chunk, so /// DataFusion can pipeline row processing as lines arrive. fn json_document_ndjson_stream( - stream: Box, + stream: SelectInputReader, original_size: u64, json_source: JsonSource, input_metrics: Arc, @@ -1206,60 +1294,107 @@ fn json_document_ndjson_stream( ) -> futures_core::stream::BoxStream<'static, Result> { json_document_ndjson_stream_with_parser( stream, - original_size, + JsonDocumentReadMode::Exact { + original_size, + input_metrics: input_metrics.recorder(), + }, json_source, - input_metrics, memory_pool, query_tracker, - |all_bytes, json_source| parse_json_document_to_lines(&all_bytes, &json_source), + |all_bytes, json_source, cancellation| { + parse_json_document_to_lines_cancellable(&all_bytes, &json_source, cancellation.as_ref()) + }, ) } -fn json_document_ndjson_stream_with_parser

( - stream: Box, - original_size: u64, +fn compressed_json_document_ndjson_stream( + stream: SelectInputReader, + json_source: JsonSource, + memory_pool: Arc, + query_tracker: Option, +) -> futures_core::stream::BoxStream<'static, Result> { + json_document_ndjson_stream_with_parser( + stream, + JsonDocumentReadMode::Bounded, + json_source, + memory_pool, + query_tracker, + |all_bytes, json_source, cancellation| { + parse_json_document_to_lines_cancellable(&all_bytes, &json_source, cancellation.as_ref()) + }, + ) +} + +enum JsonDocumentReadMode { + Exact { + original_size: u64, + input_metrics: SelectInputMetricsRecorder, + }, + Bounded, +} + +fn json_document_ndjson_stream_with_parser

( + stream: SelectInputReader, + read_mode: JsonDocumentReadMode, json_source: JsonSource, - input_metrics: Arc, memory_pool: Arc, query_tracker: Option, parser: P, ) -> futures_core::stream::BoxStream<'static, Result> where - P: FnOnce(Vec, JsonSource) -> std::io::Result> + Send + 'static, + P: FnOnce(Vec, JsonSource, Arc) -> std::io::Result> + Send + 'static, { AsyncTryStream::::new(|mut y| async move { - // Compact JSON can expand substantially into a serde_json DOM and - // per-record output buffers, so reserve a conservative upper bound - // before the source buffer is allocated. - let buffer_capacity = usize::try_from(original_size).map_err(|_| o_Error::Generic { - store: "EcObjectStore", - source: Box::new(DataFusionError::ResourcesExhausted(format!( - "JSON DOCUMENT input size {original_size} does not fit in memory" - ))), - })?; - let reservation_bytes = - json_document_memory_reservation_bytes(buffer_capacity, &json_source).map_err(|source| o_Error::Generic { - store: "EcObjectStore", - source: Box::new(source), - })?; let reservation = MemoryConsumer::new("S3 Select JSON document").register(&memory_pool); - reservation.try_resize(reservation_bytes).map_err(|err| o_Error::Generic { - store: "EcObjectStore", - source: Box::new(err), - })?; // ── 1. Read phase (lazy: only runs when the stream is polled) ──── pin_mut!(stream); - let mut all_bytes = Vec::with_capacity(buffer_capacity); - let read_result = stream.take(original_size).read_to_end(&mut all_bytes).await; - input_metrics.record_uncompressed(all_bytes.len()); - read_result.map_err(|e| o_Error::Generic { - store: "EcObjectStore", - source: Box::new(e), - })?; - if all_bytes.len() != buffer_capacity { - return Err(incomplete_object_stream_error(buffer_capacity - all_bytes.len())); - } + let all_bytes = match read_mode { + JsonDocumentReadMode::Exact { + original_size, + input_metrics, + } => { + let buffer_capacity = usize::try_from(original_size).map_err(|_| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(DataFusionError::ResourcesExhausted(format!( + "JSON DOCUMENT input size {original_size} does not fit in memory" + ))), + })?; + resize_json_document_reservation(&reservation, buffer_capacity, &json_source)?; + let mut all_bytes = Vec::with_capacity(buffer_capacity); + let read_result = stream.take(original_size).read_to_end(&mut all_bytes).await; + input_metrics.record_uncompressed(all_bytes.len()); + read_result.map_err(input_io_error)?; + if all_bytes.len() != buffer_capacity { + return Err(incomplete_object_stream_error(buffer_capacity - all_bytes.len())); + } + all_bytes + } + JsonDocumentReadMode::Bounded => { + let mut all_bytes = Vec::new(); + let mut buffer = vec![0; SELECT_DECODE_CHUNK_BYTES]; + loop { + let read = stream.read(&mut buffer).await.map_err(input_io_error)?; + if read == 0 { + break; + } + let new_len = all_bytes.len().checked_add(read).ok_or_else(|| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(json_document_memory_reservation_overflow(all_bytes.len())), + })?; + let new_len_u64 = u64::try_from(new_len).map_err(|_| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(DataFusionError::ResourcesExhausted(format!( + "JSON DOCUMENT input size {new_len} does not fit in the object size type" + ))), + })?; + validate_json_document_size(new_len_u64)?; + grow_json_document_buffer(&mut all_bytes, new_len, &reservation, &json_source)?; + all_bytes.extend_from_slice(&buffer[..read]); + } + all_bytes + } + }; // ── 2. Parse phase (blocking thread pool, non-blocking runtime) ── let queued_query_guard = match query_tracker.as_ref() { @@ -1275,11 +1410,24 @@ where }; let pending_query_guard = PendingQueryExecutionGuard::new(query_tracker); let task_query_guard = pending_query_guard.task_state(); + let cancellation = Arc::new(AtomicBool::new(false)); + let queued_task = Arc::new(Mutex::new(Some(JsonDocumentParseTask { + parser, + all_bytes, + json_source, + task_resources, + }))); + let _cancel_on_drop = JsonDocumentCancellation::new(Arc::clone(&cancellation), Arc::clone(&queued_task)); let (lines, _task_resources) = SpawnedTask::spawn_blocking(move || { + let JsonDocumentParseTask { + parser, + all_bytes, + json_source, + mut task_resources, + } = queued_task.lock().take().ok_or_else(json_document_parse_interrupted_error)?; let query_guard = PendingQueryExecutionGuard::start(&task_query_guard)?; - let mut task_resources = task_resources; task_resources.query_guard = query_guard; - parser(all_bytes, json_source).map(|lines| (lines, task_resources)) + parser(all_bytes, json_source, cancellation).map(|lines| (lines, task_resources)) }) .await .map_err(|e| o_Error::Generic { @@ -1300,12 +1448,116 @@ where .boxed() } +fn grow_json_document_buffer( + buffer: &mut Vec, + required_len: usize, + reservation: &MemoryReservation, + json_source: &JsonSource, +) -> Result<()> { + if required_len <= buffer.capacity() { + return Ok(()); + } + + let max_capacity = usize::try_from(MAX_JSON_DOCUMENT_BYTES).map_err(|_| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(DataFusionError::ResourcesExhausted( + "JSON DOCUMENT size limit does not fit in memory".to_string(), + )), + })?; + let target_capacity = required_len + .checked_next_power_of_two() + .ok_or_else(|| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(DataFusionError::ResourcesExhausted(format!( + "JSON DOCUMENT buffer capacity overflow at {required_len} bytes" + ))), + })? + .min(max_capacity); + if target_capacity < required_len { + return Err(o_Error::Generic { + store: "EcObjectStore", + source: Box::new(DataFusionError::ResourcesExhausted(format!( + "JSON DOCUMENT input size {required_len} exceeds the maximum buffer capacity" + ))), + }); + } + + resize_json_document_reservation(reservation, target_capacity, json_source)?; + buffer + .try_reserve_exact(target_capacity - buffer.len()) + .map_err(|_| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(DataFusionError::ResourcesExhausted(format!( + "JSON DOCUMENT input buffer allocation failed at {target_capacity} bytes" + ))), + })?; + resize_json_document_reservation(reservation, buffer.capacity(), json_source) +} + +fn resize_json_document_reservation(reservation: &MemoryReservation, input_bytes: usize, json_source: &JsonSource) -> Result<()> { + let reservation_bytes = + json_document_memory_reservation_bytes(input_bytes, json_source).map_err(|source| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(source), + })?; + reservation.try_resize(reservation_bytes).map_err(|source| o_Error::Generic { + store: "EcObjectStore", + source: Box::new(source), + }) +} + struct JsonDocumentTaskResources { // Struct fields drop in declaration order, so admission covers the reservation through teardown. _reservation: MemoryReservation, query_guard: Option, } +struct JsonDocumentParseTask

{ + parser: P, + all_bytes: Vec, + json_source: JsonSource, + task_resources: JsonDocumentTaskResources, +} + +struct JsonDocumentCancellation { + cancelled: Arc, + queued: Arc>>, +} + +impl JsonDocumentCancellation { + fn new(cancelled: Arc, queued: Arc>>) -> Self { + Self { cancelled, queued } + } +} + +impl Drop for JsonDocumentCancellation { + fn drop(&mut self) { + self.cancelled.store(true, std::sync::atomic::Ordering::Release); + let queued = self.queued.lock().take(); + drop(queued); + } +} + +struct CancellableJsonReader<'a> { + inner: std::io::Cursor<&'a [u8]>, + cancelled: &'a AtomicBool, +} + +impl std::io::Read for CancellableJsonReader<'_> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + ensure_json_parse_active(self.cancelled)?; + std::io::Read::read(&mut self.inner, buffer) + } +} + +fn ensure_json_parse_active(cancelled: &AtomicBool) -> std::io::Result<()> { + if cancelled.load(std::sync::atomic::Ordering::Acquire) { + Err(std::io::Error::new(std::io::ErrorKind::Interrupted, SelectError::Canceled)) + } else { + Ok(()) + } +} + fn json_document_memory_reservation_bytes(input_bytes: usize, json_source: &JsonSource) -> datafusion::common::Result { let base = input_bytes .checked_mul(JSON_DOCUMENT_MEMORY_RESERVATION_MULTIPLIER) @@ -1409,11 +1661,33 @@ impl Drop for PendingQueryExecutionGuard { /// /// - A JSON array → one line per element. /// - A JSON object or scalar root → one line. +#[cfg(test)] fn parse_json_document_to_lines(bytes: &[u8], json_source: &JsonSource) -> std::io::Result> { - let root: serde_json::Value = - serde_json::from_slice(bytes).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + parse_json_document_to_lines_cancellable(bytes, json_source, &AtomicBool::new(false)) +} + +fn parse_json_document_to_lines_cancellable( + bytes: &[u8], + json_source: &JsonSource, + cancelled: &AtomicBool, +) -> std::io::Result> { + let reader = std::io::BufReader::with_capacity( + JSON_CANCELLATION_CHECK_BYTES, + CancellableJsonReader { + inner: std::io::Cursor::new(bytes), + cancelled, + }, + ); + let root: serde_json::Value = match serde_json::from_reader(reader) { + Ok(root) => root, + Err(error) => { + ensure_json_parse_active(cancelled)?; + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, error)); + } + }; + ensure_json_parse_active(cancelled)?; let json_source_path = json_source.path(); - let values = expand_json_source(root, json_source_path)?; + let values = expand_json_source(root, json_source_path, cancelled)?; // Preserve the two pre-path-AST forms that flattened arrays implicitly. // Explicit indexes and wildcards already identify the intended records // and must not flatten an array-valued result a second time. @@ -1424,19 +1698,25 @@ fn parse_json_document_to_lines(bytes: &[u8], json_source: &JsonSource) -> std:: }); let mut lines: Vec = Vec::new(); for value in values { + ensure_json_parse_active(cancelled)?; match value { serde_json::Value::Array(array) if implicitly_expand_arrays => { for item in array { - lines.push(json_value_to_line(item, scalar_column)?); + ensure_json_parse_active(cancelled)?; + lines.push(json_value_to_line_cancellable(item, scalar_column, cancelled)?); } } - other => lines.push(json_value_to_line(other, scalar_column)?), + other => lines.push(json_value_to_line_cancellable(other, scalar_column, cancelled)?), } } Ok(lines) } -fn expand_json_source(root: serde_json::Value, json_source_path: &[JsonPathSegment]) -> std::io::Result> { +fn expand_json_source( + root: serde_json::Value, + json_source_path: &[JsonPathSegment], + cancelled: &AtomicBool, +) -> std::io::Result> { // S3Object[*] identifies the input record stream. JSON DOCUMENT already // presents the root value as that stream, so the leading marker is not a // lookup against the root object. @@ -1454,11 +1734,13 @@ fn expand_json_source(root: serde_json::Value, json_source_path: &[JsonPathSegme }; for segment in path { + ensure_json_parse_active(cancelled)?; let mut expanded = Vec::new(); for value in values { + ensure_json_parse_active(cancelled)?; match (segment, value) { (JsonPathSegment::Key { name, quoted }, serde_json::Value::Object(mut object)) => { - if let Some(value) = remove_json_source_key(&mut object, name, *quoted)? { + if let Some(value) = remove_json_source_key(&mut object, name, *quoted, cancelled)? { expanded.push(value); } } @@ -1495,21 +1777,69 @@ fn remove_json_source_key( object: &mut serde_json::Map, name: &str, quoted: bool, + cancelled: &AtomicBool, +) -> std::io::Result> { + let mut checkpoint = || ensure_json_parse_active(cancelled); + remove_json_source_key_with_checkpoint(object, name, quoted, &mut checkpoint) +} + +fn remove_json_source_key_with_checkpoint( + object: &mut serde_json::Map, + name: &str, + quoted: bool, + checkpoint: &mut impl FnMut() -> std::io::Result<()>, ) -> std::io::Result> { if quoted { return Ok(object.remove(name)); } - let mut matches = object.keys().filter(|key| key.eq_ignore_ascii_case(name)); - let matched = matches.next().cloned(); - if matches.next().is_some() { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, SelectError::AmbiguousFieldName)); + let mut matched = None; + for (index, key) in object.keys().enumerate() { + if index % JSON_CANCELLATION_CHECK_KEYS == 0 { + checkpoint()?; + } + if json_key_eq_ignore_ascii_case_with_checkpoint(key, name, checkpoint)? { + if matched.is_some() { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, SelectError::AmbiguousFieldName)); + } + matched = Some(key.clone()); + } } - drop(matches); Ok(matched.and_then(|key| object.remove(&key))) } +fn json_key_eq_ignore_ascii_case_with_checkpoint( + key: &str, + expected: &str, + checkpoint: &mut impl FnMut() -> std::io::Result<()>, +) -> std::io::Result { + if key.len() != expected.len() { + return Ok(false); + } + + for (key_chunk, expected_chunk) in key + .as_bytes() + .chunks(JSON_CANCELLATION_CHECK_BYTES) + .zip(expected.as_bytes().chunks(JSON_CANCELLATION_CHECK_BYTES)) + { + checkpoint()?; + if !key_chunk.eq_ignore_ascii_case(expected_chunk) { + return Ok(false); + } + } + Ok(true) +} + +#[cfg(test)] fn json_value_to_line(value: serde_json::Value, scalar_column: &str) -> std::io::Result { + json_value_to_line_cancellable(value, scalar_column, &AtomicBool::new(false)) +} + +fn json_value_to_line_cancellable( + value: serde_json::Value, + scalar_column: &str, + cancelled: &AtomicBool, +) -> std::io::Result { let value = match value { value @ serde_json::Value::Object(_) => value, value => { @@ -1518,11 +1848,62 @@ fn json_value_to_line(value: serde_json::Value, scalar_column: &str) -> std::io: serde_json::Value::Object(row) } }; - let mut line = serde_json::to_vec(&value).map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + let mut line = Vec::new(); + let (serialize_result, limit_exceeded) = { + let mut writer = CancellableJsonWriter { + inner: &mut line, + cancelled, + bytes_since_check: 0, + limit_exceeded: false, + }; + let serialize_result = serde_json::to_writer(&mut writer, &value); + (serialize_result, writer.limit_exceeded) + }; + if limit_exceeded { + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, SelectError::OverMaxRecordSize)); + } + if let Err(error) = serialize_result { + ensure_json_parse_active(cancelled)?; + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, error)); + } + ensure_json_parse_active(cancelled)?; line.push(b'\n'); Ok(Bytes::from(line)) } +struct CancellableJsonWriter<'a> { + inner: &'a mut Vec, + cancelled: &'a AtomicBool, + bytes_since_check: usize, + limit_exceeded: bool, +} + +impl std::io::Write for CancellableJsonWriter<'_> { + fn write(&mut self, buffer: &[u8]) -> std::io::Result { + let Some(new_len) = self.inner.len().checked_add(buffer.len()) else { + self.limit_exceeded = true; + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, SelectError::OverMaxRecordSize)); + }; + if new_len > MAX_SELECT_RECORD_BYTES { + self.limit_exceeded = true; + return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, SelectError::OverMaxRecordSize)); + } + self.bytes_since_check = self.bytes_since_check.saturating_add(buffer.len()); + if self.bytes_since_check >= JSON_CANCELLATION_CHECK_BYTES { + ensure_json_parse_active(self.cancelled)?; + self.bytes_since_check %= JSON_CANCELLATION_CHECK_BYTES; + } + self.inner.extend_from_slice(buffer); + Ok(buffer.len()) + } + + fn flush(&mut self) -> std::io::Result<()> { + ensure_json_parse_active(self.cancelled)?; + self.bytes_since_check = 0; + Ok(()) + } +} + fn invalid_json_source_path(message: &'static str) -> std::io::Error { std::io::Error::new(std::io::ErrorKind::InvalidData, message) } @@ -1552,6 +1933,7 @@ where S: Stream> + Send + 'static, E: Send + 'static, { + let input_metrics = input_metrics.recorder(); stream.inspect_ok(move |bytes| input_metrics.record_uncompressed(bytes.len())) } @@ -1611,14 +1993,16 @@ fn incomplete_object_stream_error(remaining: impl std::fmt::Display) -> o_Error #[cfg(test)] mod test { use super::{ - EcObjectStore, EcObjectStoreBuildError, JSON_DOCUMENT_MEMORY_RESERVATION_MULTIPLIER, OnceCell, + EcObjectStore, EcObjectStoreBuildError, JSON_DOCUMENT_MEMORY_RESERVATION_MULTIPLIER, JsonDocumentReadMode, OnceCell, SELECT_DEFAULT_READ_BUFFER_SIZE, SelectObjectOptions, SelectObjectSnapshot, SelectScanRange, SnapshotConsistencyError, - bytes_stream, convert_csv_delimiter_stream, convert_field_delimiter_stream, convert_record_delimiter_stream, - find_delimiter, flatten_json_document_to_ndjson, http_range_spec_from_get_range, json_document_ndjson_stream, - json_document_ndjson_stream_with_parser, legacy_json_source_from_input, map_storage_error, - meter_uncompressed_input_stream, scan_range_from_bounds, scan_range_stream, select_read_headers, snapshot_last_modified, - validate_json_document_size, + bytes_stream, compressed_json_document_ndjson_stream, convert_csv_delimiter_stream, convert_field_delimiter_stream, + convert_record_delimiter_stream, find_delimiter, flatten_json_document_to_ndjson, grow_json_document_buffer, + http_range_spec_from_get_range, json_document_ndjson_stream, json_document_ndjson_stream_with_parser, + json_key_eq_ignore_ascii_case_with_checkpoint, legacy_json_source_from_input, map_storage_error, + meter_uncompressed_input_stream, remove_json_source_key_with_checkpoint, scan_range_from_bounds, scan_range_stream, + select_read_headers, snapshot_last_modified, validate_json_document_size, }; + use crate::input_stream::{CompressionFormat, MAX_SELECT_RECORD_BYTES, compressed_input_reader, encode_compressed_fixture}; use crate::query::ast::{JsonPathSegment, JsonSource}; use crate::query::session::{QueryExecutionGuard, QueryExecutionOwner, QueryExecutionTracker}; use crate::storage_api::SelectPutObjReader; @@ -1627,7 +2011,7 @@ mod test { use bytes::Bytes; use datafusion::{ common::DataFusionError, - execution::memory_pool::{GreedyMemoryPool, MemoryLimit, MemoryPool, MemoryReservation}, + execution::memory_pool::{GreedyMemoryPool, MemoryConsumer, MemoryLimit, MemoryPool, MemoryReservation}, execution::{config::SessionConfig, context::SessionContext}, object_store::{self, GetOptions, GetRange, GetResultPayload, ObjectStore as _, path::Path}, physical_plan::ExecutionPlanProperties, @@ -1639,8 +2023,8 @@ mod test { use rustfs_test_utils::PutObjectCommitBarrier; use s3s::S3ErrorCode; use s3s::dto::{ - CSVInput, CSVOutput, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput, JSONType, - OutputSerialization, ScanRange, SelectObjectContentInput, SelectObjectContentRequest, + CSVInput, CSVOutput, CompressionType, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONOutput, + JSONType, OutputSerialization, ScanRange, SelectObjectContentInput, SelectObjectContentRequest, }; use s3s::header::{ X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, X_AMZ_SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, @@ -3128,7 +3512,7 @@ mod test { Arc::new(GreedyMemoryPool::new(1024 * 1024)), None, Arc::clone(&input_metrics), - snapshot, + Arc::clone(&snapshot), JsonSource::default(), ) .expect("build metrics-aware object store"); @@ -3175,6 +3559,152 @@ mod test { assert_eq!(input_metrics.snapshot().bytes_processed, 2); } + #[tokio::test] + async fn compressed_object_uses_one_full_stream_and_rejects_internal_ranges() { + const BUCKET: &str = "s3select-compressed-object"; + const OBJECT: &str = "input.csv"; + const DATA: &[u8] = b"id,name\n1,Alice\n2,Bob\n"; + + let compressed = encode_compressed_fixture(CompressionFormat::Gzip, DATA).await; + let env = crate::storage_api::select_test_ecstore_env().await; + env.make_bucket(BUCKET, false).await; + let mut reader = SelectPutObjReader::from_vec(compressed.clone()); + env.ecstore + .put_object(BUCKET, OBJECT, &mut reader, &Default::default()) + .await + .expect("put compressed CSV fixture"); + let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await; + let mut input = (*csv_input(BUCKET, OBJECT)).clone(); + input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP)); + input.request.scan_range = Some(ScanRange { + start: Some(0), + end: None, + }); + let input_metrics = Arc::new(SelectInputMetrics::default()); + let store = EcObjectStore::build_with_snapshot( + Arc::new(input), + Arc::new(GreedyMemoryPool::new(1024 * 1024)), + None, + Arc::clone(&input_metrics), + Arc::clone(&snapshot), + JsonSource::default(), + ) + .expect("build compressed object store"); + + let error = store + .get_opts( + &Path::from(OBJECT), + GetOptions { + range: Some(GetRange::Bounded(0..1)), + ..Default::default() + }, + ) + .await + .expect_err("compressed input must reject DataFusion byte ranges"); + assert_eq!( + QueryError::from(DataFusionError::ObjectStore(Box::new(error))).select_error(), + SelectError::UnsupportedScanRangeInput + ); + assert_eq!(store.reader_open_count.load(Ordering::SeqCst), 0); + + let mut scan_input = (*csv_input(BUCKET, OBJECT)).clone(); + scan_input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP)); + scan_input.request.scan_range = Some(ScanRange { + start: Some(1), + end: None, + }); + let scan_store = EcObjectStore::build_with_snapshot( + Arc::new(scan_input), + Arc::new(GreedyMemoryPool::new(1024 * 1024)), + None, + Arc::new(SelectInputMetrics::default()), + snapshot, + JsonSource::default(), + ) + .expect("build compressed ScanRange object store"); + let error = scan_store + .get_opts(&Path::from(OBJECT), GetOptions::default()) + .await + .expect_err("compressed request ScanRange must fail before object I/O"); + assert_eq!( + QueryError::from(DataFusionError::ObjectStore(Box::new(error))).select_error(), + SelectError::UnsupportedScanRangeInput + ); + assert_eq!(scan_store.reader_open_count.load(Ordering::SeqCst), 0); + + let result = store + .get_opts(&Path::from(OBJECT), GetOptions::default()) + .await + .expect("open full compressed object stream"); + let GetResultPayload::Stream(stream) = result.payload else { + panic!("expected compressed stream payload"); + }; + let body = stream + .try_collect::>() + .await + .expect("decode compressed object") + .concat(); + + assert_eq!(body, DATA); + assert_eq!(store.reader_open_count.load(Ordering::SeqCst), 1); + assert_eq!( + input_metrics.snapshot().bytes_scanned, + u64::try_from(compressed.len()).expect("compressed fixture length should fit in u64") + ); + assert_eq!( + input_metrics.snapshot().bytes_processed, + u64::try_from(DATA.len()).expect("input fixture length should fit in u64") + ); + } + + #[tokio::test] + async fn compressed_stream_throughput_is_independent_of_query_memory_pool() { + const BUCKET: &str = "s3select-compressed-throughput"; + const OBJECT: &str = "input.csv.gz"; + + let data = b"a\n".repeat(1024 * 1024); + let compressed = encode_compressed_fixture(CompressionFormat::Gzip, &data).await; + let env = crate::storage_api::select_test_ecstore_env().await; + env.make_bucket(BUCKET, false).await; + let mut reader = SelectPutObjReader::from_vec(compressed); + env.ecstore + .put_object(BUCKET, OBJECT, &mut reader, &Default::default()) + .await + .expect("put compressed throughput fixture"); + let snapshot = prepare_test_snapshot(BUCKET, OBJECT).await; + let mut input = (*csv_input(BUCKET, OBJECT)).clone(); + input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP)); + let metrics = Arc::new(SelectInputMetrics::default()); + let store = EcObjectStore::build_with_snapshot( + Arc::new(input), + Arc::new(GreedyMemoryPool::new(1)), + None, + Arc::clone(&metrics), + snapshot, + JsonSource::default(), + ) + .expect("build compressed throughput store"); + + let result = store + .get_opts(&Path::from(OBJECT), GetOptions::default()) + .await + .expect("open compressed throughput stream"); + let GetResultPayload::Stream(stream) = result.payload else { + panic!("expected compressed stream payload"); + }; + let decoded = stream + .try_collect::>() + .await + .expect("streamed decoded bytes should not consume the query memory pool") + .concat(); + + assert_eq!(decoded, data); + assert_eq!( + metrics.snapshot().bytes_processed, + u64::try_from(data.len()).expect("fixture length should fit in u64") + ); + } + #[tokio::test] async fn dropping_real_object_stream_counts_only_consumed_bytes() { const BUCKET: &str = "s3select-partial-input-metrics"; @@ -3353,6 +3883,125 @@ mod test { )); } + #[tokio::test] + async fn compressed_json_document_uses_decoded_size_and_metric_boundaries() { + const INPUT: &[u8] = br#"[{"id":1},{"id":2}]"#; + const EXPECTED: &[u8] = b"{\"id\":1}\n{\"id\":2}\n"; + + for format in [CompressionFormat::Gzip, CompressionFormat::Bzip2] { + let compressed = encode_compressed_fixture(format, INPUT).await; + let compressed_len = u64::try_from(compressed.len()).expect("compressed fixture length should fit in u64"); + let input_metrics = Arc::new(SelectInputMetrics::default()); + let reader = compressed_input_reader( + Box::new(std::io::Cursor::new(compressed)), + compressed_len, + format, + Arc::clone(&input_metrics), + u64::MAX, + None, + ); + let output = compressed_json_document_ndjson_stream( + reader, + JsonSource::default(), + Arc::new(GreedyMemoryPool::new(1024 * 1024)), + None, + ) + .try_collect::>() + .await + .expect("compressed JSON DOCUMENT should decode and parse") + .concat(); + + assert_eq!(output, EXPECTED); + assert_eq!(input_metrics.snapshot().bytes_scanned, compressed_len); + assert_eq!( + input_metrics.snapshot().bytes_processed, + u64::try_from(INPUT.len()).expect("JSON fixture length should fit in u64") + ); + } + } + + async fn compressed_json_document_select_error( + format: CompressionFormat, + compressed: Vec, + max_processed_bytes: u64, + ) -> SelectError { + let compressed_len = u64::try_from(compressed.len()).expect("compressed fixture length should fit in u64"); + let reader = compressed_input_reader( + Box::new(std::io::Cursor::new(compressed)), + compressed_len, + format, + Arc::new(SelectInputMetrics::default()), + max_processed_bytes, + None, + ); + let mut output = compressed_json_document_ndjson_stream( + reader, + JsonSource::default(), + Arc::new(GreedyMemoryPool::new(1024 * 1024)), + None, + ); + let source = output + .next() + .await + .expect("decoder failure should produce one stream error") + .expect_err("compressed JSON DOCUMENT decoding must fail"); + assert!(output.next().await.is_none()); + QueryError::from(DataFusionError::ObjectStore(Box::new(source))).select_error() + } + + #[tokio::test] + async fn compressed_json_document_preserves_decoder_select_errors() { + const INPUT: &[u8] = br#"[{"id":1}]"#; + + for (format, compression) in [(CompressionFormat::Gzip, "GZIP"), (CompressionFormat::Bzip2, "BZIP2")] { + assert_eq!( + compressed_json_document_select_error(format, b"not compressed".to_vec(), u64::MAX).await, + SelectError::InvalidCompressionFormatForObject { compression } + ); + + let mut truncated = encode_compressed_fixture(format, INPUT).await; + truncated.truncate(truncated.len() - 1); + assert_eq!( + compressed_json_document_select_error(format, truncated, u64::MAX).await, + SelectError::TruncatedInput + ); + + let compressed = encode_compressed_fixture(format, INPUT).await; + let max_processed_bytes = u64::try_from(INPUT.len() - 1).expect("fixture length should fit in u64"); + assert_eq!( + compressed_json_document_select_error(format, compressed, max_processed_bytes).await, + SelectError::ResourceExhausted + ); + } + } + + #[test] + fn compressed_json_document_buffer_grows_amortized_and_reserves_capacity() { + let memory_pool: Arc = Arc::new(GreedyMemoryPool::new(1024 * 1024)); + let reservation = MemoryConsumer::new("compressed JSON document test").register(&memory_pool); + let json_source = JsonSource::default(); + let mut buffer = Vec::new(); + let mut capacity_growths = 0; + + for _ in 0..1025 { + let old_capacity = buffer.capacity(); + let required_len = buffer.len() + 1; + grow_json_document_buffer(&mut buffer, required_len, &reservation, &json_source) + .expect("bounded JSON buffer should grow"); + if buffer.capacity() != old_capacity { + capacity_growths += 1; + } + buffer.push(0); + assert_eq!( + reservation.size(), + super::json_document_memory_reservation_bytes(buffer.capacity(), &json_source) + .expect("test reservation should fit") + ); + } + + assert!(capacity_growths <= 12, "power-of-two growth should stay logarithmic"); + } + #[tokio::test] async fn scalar_alias_expansion_is_in_the_query_memory_reservation() { let input = b"[0,0]".to_vec(); @@ -3489,12 +4138,14 @@ mod test { Arc::new(GreedyMemoryPool::new(input.len() * JSON_DOCUMENT_MEMORY_RESERVATION_MULTIPLIER)); let mut output = json_document_ndjson_stream_with_parser( Box::new(std::io::Cursor::new(input.clone())), - input.len() as u64, + JsonDocumentReadMode::Exact { + original_size: input.len() as u64, + input_metrics: SelectInputMetrics::default().recorder(), + }, JsonSource::default(), - Arc::new(SelectInputMetrics::default()), memory_pool, None, - |_, _| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, SelectError::AmbiguousFieldName)), + |_, _, _| Err(std::io::Error::new(std::io::ErrorKind::InvalidData, SelectError::AmbiguousFieldName)), ); let source = output @@ -3562,7 +4213,7 @@ mod test { } #[test] - fn test_json_document_queued_parse_retains_query_guard_until_dequeued() { + fn test_json_document_cancelled_queued_parse_releases_before_dequeue() { let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .max_blocking_threads(1) @@ -3614,29 +4265,22 @@ mod test { } drop(output); - assert_eq!(admission.available_permits(), 0); - assert!(memory_pool.reserved() > 0); assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), reservation_released) + .await + .expect("queued parse cancellation should release its memory immediately") + .expect("memory reservation release observer should remain open"), + "query admission must cover the memory reservation through teardown" + ); + assert_eq!(memory_pool.reserved(), 0); + let recovered_permit = tokio::time::timeout(std::time::Duration::from_millis(100), Arc::clone(&admission).acquire_owned()) .await - .is_err(), - "queued parse resources must remain covered by admission" - ); + .expect("queued parse cancellation should release admission before a worker is available") + .expect("query admission should remain open"); + release_blocking_tx.send(()).expect("release blocking worker"); blocker.await.expect("blocking worker should finish"); - assert!( - tokio::time::timeout(std::time::Duration::from_secs(5), reservation_released) - .await - .expect("cancelled JSON parse should release its memory reservation") - .expect("memory reservation release observer should remain open"), - "query admission must cover the memory reservation through task teardown" - ); - let recovered_permit = - tokio::time::timeout(std::time::Duration::from_secs(5), Arc::clone(&admission).acquire_owned()) - .await - .expect("cancelled JSON parse should be dequeued") - .expect("query admission should remain open"); - assert_eq!(memory_pool.reserved(), 0); drop(recovered_permit); assert_eq!(admission.available_permits(), 1); }); @@ -3679,12 +4323,14 @@ mod test { let parser_started_in_task = Arc::clone(&parser_started); let mut output = json_document_ndjson_stream_with_parser( Box::new(std::io::Cursor::new(input.clone())), - input.len() as u64, + JsonDocumentReadMode::Exact { + original_size: input.len() as u64, + input_metrics: SelectInputMetrics::default().recorder(), + }, JsonSource::default(), - Arc::new(SelectInputMetrics::default()), Arc::clone(&memory_pool), Some(query_tracker.clone()), - move |_, _| { + move |_, _, _| { parser_started_in_task.store(true, std::sync::atomic::Ordering::SeqCst); Ok(vec![Bytes::from_static(b"{}\n")]) }, @@ -3756,12 +4402,14 @@ mod test { let parser_started_in_task = Arc::clone(&parser_started); let mut output = json_document_ndjson_stream_with_parser( Box::new(std::io::Cursor::new(input.clone())), - input.len() as u64, + JsonDocumentReadMode::Exact { + original_size: input.len() as u64, + input_metrics: SelectInputMetrics::default().recorder(), + }, JsonSource::default(), - Arc::new(SelectInputMetrics::default()), Arc::clone(&memory_pool), Some(query_tracker), - move |_, _| { + move |_, _, _| { parser_started_in_task.store(true, std::sync::atomic::Ordering::SeqCst); Ok(vec![Bytes::from_static(b"{}\n")]) }, @@ -3787,7 +4435,7 @@ mod test { } #[test] - fn test_json_document_started_parse_retains_query_guard_when_cancelled() { + fn test_json_document_started_parse_cancels_and_releases_query_guard() { let runtime = tokio::runtime::Builder::new_multi_thread() .worker_threads(2) .max_blocking_threads(1) @@ -3812,18 +4460,21 @@ mod test { let memory_pool: Arc = Arc::new(GreedyMemoryPool::new(input.len() * JSON_DOCUMENT_MEMORY_RESERVATION_MULTIPLIER)); let (parse_started_tx, parse_started_rx) = tokio::sync::oneshot::channel(); - let (release_parse_tx, release_parse_rx) = std::sync::mpsc::channel(); let mut output = json_document_ndjson_stream_with_parser( Box::new(std::io::Cursor::new(input.clone())), - input.len() as u64, + JsonDocumentReadMode::Exact { + original_size: input.len() as u64, + input_metrics: SelectInputMetrics::default().recorder(), + }, JsonSource::default(), - Arc::new(SelectInputMetrics::default()), memory_pool, Some(query_tracker), - move |_, _| { + move |_, _, cancellation| { let _ = parse_started_tx.send(()); - release_parse_rx.recv().expect("release JSON parser"); - Ok(vec![Bytes::from_static(b"{}\n")]) + while !cancellation.load(std::sync::atomic::Ordering::Acquire) { + std::thread::yield_now(); + } + Err(std::io::Error::new(std::io::ErrorKind::Interrupted, SelectError::Canceled)) }, ); @@ -3833,14 +4484,13 @@ mod test { assert!(futures::poll!(next.as_mut()).is_pending()); } parse_started_rx.await.expect("JSON parser should start"); + assert!(Arc::clone(&admission).try_acquire_owned().is_err()); drop(output); - assert!(Arc::clone(&admission).try_acquire_owned().is_err()); - release_parse_tx.send(()).expect("release JSON parser"); let recovered_permit = tokio::time::timeout(std::time::Duration::from_secs(5), Arc::clone(&admission).acquire_owned()) .await - .expect("started JSON parse should release the query guard") + .expect("cancelled JSON parse should release the query guard without an external unblock") .expect("query admission should remain open"); drop(recovered_permit); assert_eq!(admission.available_permits(), 1); @@ -3931,6 +4581,32 @@ mod test { assert_eq!(err.kind(), std::io::ErrorKind::InvalidData); } + #[test] + fn json_document_logical_record_enforces_one_megabyte_limit() { + const OBJECT_OVERHEAD: usize = br#"{"v":""}"#.len(); + + let at_limit = serde_json::json!({"v": "x".repeat(MAX_SELECT_RECORD_BYTES - OBJECT_OVERHEAD)}); + let line = super::json_value_to_line(at_limit, "_1").expect("one-megabyte logical record should be accepted"); + assert_eq!(line.len(), MAX_SELECT_RECORD_BYTES + 1); + + let over_limit = serde_json::json!({"v": "x".repeat(MAX_SELECT_RECORD_BYTES + 1 - OBJECT_OVERHEAD)}); + let error = super::json_value_to_line(over_limit, "_1").expect_err("oversized logical record must fail"); + assert_eq!(error.kind(), std::io::ErrorKind::InvalidData); + assert!(error.get_ref().is_some_and(|source| { + source + .downcast_ref::() + .is_some_and(|error| error == &SelectError::OverMaxRecordSize) + })); + + let escaped = serde_json::json!({"v": "\0".repeat(MAX_SELECT_RECORD_BYTES / 6)}); + let error = super::json_value_to_line(escaped, "_1").expect_err("escaped output must be bounded while it is serialized"); + assert!(error.get_ref().is_some_and(|source| { + source + .downcast_ref::() + .is_some_and(|error| error == &SelectError::OverMaxRecordSize) + })); + } + /// Completely empty input returns an error (not valid JSON). #[test] fn test_flatten_empty_input_returns_error() { @@ -4156,6 +4832,55 @@ mod test { ); } + #[test] + fn unquoted_source_key_scan_honors_cancellation() { + let mut object = serde_json::Map::new(); + for index in 0..=super::JSON_CANCELLATION_CHECK_KEYS { + object.insert(format!("field-{index}"), serde_json::Value::Null); + } + let mut checkpoints = 0; + let mut cancel_on_second_checkpoint = || { + checkpoints += 1; + if checkpoints == 2 { + Err(std::io::Error::new(std::io::ErrorKind::Interrupted, SelectError::Canceled)) + } else { + Ok(()) + } + }; + + let error = remove_json_source_key_with_checkpoint(&mut object, "x", false, &mut cancel_on_second_checkpoint) + .expect_err("key scan should stop at its second cancellation checkpoint"); + + assert_eq!(error.kind(), std::io::ErrorKind::Interrupted); + assert_eq!(checkpoints, 2); + assert!( + error + .get_ref() + .and_then(|source| source.downcast_ref::()) + .is_some_and(|error| *error == SelectError::Canceled) + ); + } + + #[test] + fn long_json_key_comparison_honors_cancellation() { + let key = "x".repeat(super::JSON_CANCELLATION_CHECK_BYTES * 2); + let mut checkpoints = 0; + let mut cancel_on_second_checkpoint = || { + checkpoints += 1; + if checkpoints == 2 { + Err(std::io::Error::new(std::io::ErrorKind::Interrupted, SelectError::Canceled)) + } else { + Ok(()) + } + }; + + let error = json_key_eq_ignore_ascii_case_with_checkpoint(&key, &key, &mut cancel_on_second_checkpoint) + .expect_err("long-key comparison should stop at its second cancellation checkpoint"); + + assert_eq!(error.kind(), std::io::ErrorKind::Interrupted); + assert_eq!(checkpoints, 2); + } + #[test] fn missing_source_key_produces_no_records() { let input = br#"{"employees":[]}"#; diff --git a/crates/s3select-api/src/query/session.rs b/crates/s3select-api/src/query/session.rs index dc3a66dc0..7df3cf60a 100644 --- a/crates/s3select-api/src/query/session.rs +++ b/crates/s3select-api/src/query/session.rs @@ -30,6 +30,7 @@ use datafusion::{ prelude::SessionContext, }; use parking_lot::Mutex; +use s3s::dto::CompressionType; use std::sync::{ Arc, Weak, atomic::{AtomicU8, Ordering}, @@ -446,11 +447,19 @@ impl SessionCtxFactory { let scan_range_requires_single_file_scan = context.input.request.scan_range.is_some() && context.input.request.input_serialization.parquet.is_none(); let json_document_requires_single_file_scan = is_json_document_input(&context.input); + let compressed_input_requires_single_file_scan = context + .input + .request + .input_serialization + .compression_type + .as_ref() + .is_some_and(|compression| compression.as_str() != CompressionType::NONE); let metered_input_requires_single_file_scan = input_metrics.is_some() && context.input.request.input_serialization.parquet.is_none(); let config = if custom_two_byte_record_delimiter || scan_range_requires_single_file_scan || json_document_requires_single_file_scan + || compressed_input_requires_single_file_scan || metered_input_requires_single_file_scan { config.with_repartition_file_scans(false) @@ -847,6 +856,21 @@ mod tests { assert!(session.inner().config().options().optimizer.repartition_file_scans); } + #[tokio::test] + async fn compressed_input_disables_file_repartitioning_without_metrics() { + let mut context = test_context(); + Arc::make_mut(&mut context.input).request.input_serialization.compression_type = + Some(CompressionType::from_static(CompressionType::GZIP)); + + let session = SessionCtxFactory::new(true) + .with_target_partitions(2) + .create_session_ctx(&context) + .await + .expect("compressed session should be created"); + + assert!(!session.inner().config().options().optimizer.repartition_file_scans); + } + #[tokio::test] async fn json_document_disables_file_repartitioning() { let mut context = test_context(); diff --git a/crates/s3select-query/src/dispatcher/manager.rs b/crates/s3select-query/src/dispatcher/manager.rs index 72117ce61..0f69949ff 100644 --- a/crates/s3select-query/src/dispatcher/manager.rs +++ b/crates/s3select-query/src/dispatcher/manager.rs @@ -53,7 +53,7 @@ use rustfs_s3select_api::{ }, }, }; -use s3s::dto::{FileHeaderInfo, JSONType, SelectObjectContentInput}; +use s3s::dto::{CompressionType, FileHeaderInfo, JSONType, SelectObjectContentInput}; use std::sync::LazyLock; use tokio::{ sync::Semaphore, @@ -72,6 +72,7 @@ use crate::{ static IGNORE: LazyLock = LazyLock::new(|| FileHeaderInfo::from_static(FileHeaderInfo::IGNORE)); static NONE: LazyLock = LazyLock::new(|| FileHeaderInfo::from_static(FileHeaderInfo::NONE)); static USE: LazyLock = LazyLock::new(|| FileHeaderInfo::from_static(FileHeaderInfo::USE)); +const EXACT_OBJECT_FILE_EXTENSION: &str = ""; #[derive(Clone)] pub struct SimpleQueryDispatcher { @@ -416,6 +417,13 @@ impl SimpleQueryDispatcher { let path = format!("s3://{}/{}", self.input.bucket, self.input.key); let table_path = ListingTableUrl::parse(path)?; + let compressed_input = self + .input + .request + .input_serialization + .compression_type + .as_ref() + .is_some_and(|compression| compression.as_str() != CompressionType::NONE); let (listing_options, need_rename_volume_name, need_ignore_volume_name) = if let Some(csv) = self.input.request.input_serialization.csv.as_ref() { let mut need_rename_volume_name = false; @@ -465,22 +473,30 @@ impl SimpleQueryDispatcher { file_format = file_format.with_quote(quote.as_bytes().first().copied().unwrap_or_default()); } ( - ListingOptions::new(Arc::new(file_format)).with_file_extension(".csv"), + ListingOptions::new(Arc::new(file_format)).with_file_extension(if compressed_input { + EXACT_OBJECT_FILE_EXTENSION + } else { + ".csv" + }), need_rename_volume_name, need_ignore_volume_name, ) } else if self.input.request.input_serialization.json.is_some() { let file_format = JsonFormat::default(); - // Use the actual file extension from the object key so that files stored - // with a `.jsonl` suffix (newline-delimited JSON) are also matched by - // DataFusion's listing/schema-inference logic. Falling back to ".json" - // preserves behaviour for keys that have no extension. - let file_ext = std::path::Path::new(&self.input.key) - .extension() - .and_then(|e| e.to_str()) - .map(|e| format!(".{e}")) - .unwrap_or_else(|| ".json".to_string()); - (ListingOptions::new(Arc::new(file_format)).with_file_extension(file_ext), false, false) + let file_extension = if compressed_input { + EXACT_OBJECT_FILE_EXTENSION.to_string() + } else { + std::path::Path::new(&self.input.key) + .extension() + .and_then(|extension| extension.to_str()) + .map(|extension| format!(".{extension}")) + .unwrap_or_else(|| ".json".to_string()) + }; + ( + ListingOptions::new(Arc::new(file_format)).with_file_extension(file_extension), + false, + false, + ) } else { return Err(SelectError::InvalidDataSource.into()); }; diff --git a/crates/scanner/Cargo.toml b/crates/scanner/Cargo.toml index 904fb0913..68617d6f6 100644 --- a/crates/scanner/Cargo.toml +++ b/crates/scanner/Cargo.toml @@ -106,7 +106,7 @@ hex-simd.workspace = true [dev-dependencies] tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] } serial_test = { workspace = true } -temp-env = { workspace = true } +temp-env = { workspace = true, features = ["async_closure"] } tempfile = { workspace = true } uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] } tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] } diff --git a/crates/scanner/src/data_usage_define.rs b/crates/scanner/src/data_usage_define.rs index 8447c4917..9d72f72f2 100644 --- a/crates/scanner/src/data_usage_define.rs +++ b/crates/scanner/src/data_usage_define.rs @@ -175,6 +175,11 @@ pub static DATA_USAGE_BUCKET: LazyLock = pub static DATA_USAGE_OBJ_NAME_PATH: LazyLock = LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBJECT_NAME}")); +/// Durable evidence for recovery of the exact empty usage fence written by +/// rc.2/rc.3 bucket cleanup before the first authoritative scanner snapshot. +pub static DATA_USAGE_RECOVERY_PATH: LazyLock = + LazyLock::new(|| format!("{}.recovery-pending.json", DATA_USAGE_OBJ_NAME_PATH.as_str())); + pub static DATA_USAGE_OBSERVED_OBJ_NAME_PATH: LazyLock = LazyLock::new(|| format!("{BUCKET_META_PREFIX}{SLASH_SEPARATOR}{DATA_USAGE_OBSERVED_OBJECT_NAME}")); diff --git a/crates/scanner/src/lib.rs b/crates/scanner/src/lib.rs index 3a8cb8d6f..e2992e95a 100644 --- a/crates/scanner/src/lib.rs +++ b/crates/scanner/src/lib.rs @@ -82,8 +82,10 @@ pub use remote_scanner::{ pub use runtime_config::{apply_scanner_runtime_config, scanner_runtime_config_status, validate_scanner_runtime_config}; pub use rustfs_scanner_contracts::last_minute; pub use scanner::{ - ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, init_data_scanner, - reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_topology_digest, + ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus, ScannerCycleScheduleStatus, ScannerPauseBacklogAlertReason, + ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, init_data_scanner, + reset_scanner_cycle_recovery, scanner_cycle_recovery_status, scanner_cycle_schedule_status, scanner_pause_backlog_status, + scanner_topology_digest, }; pub use scanner_io::{ ScannerDirtyUsageAckError, ScannerDirtyUsageState, acknowledge_dirty_usage_generation, clear_dirty_usage_bucket, diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index a83b8a737..b79965d90 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -22,7 +22,8 @@ use std::sync::{Arc, LazyLock, RwLock}; use self::heal_info::{BackgroundHealInfoReadStatus, read_background_heal_info_with_epoch, save_background_heal_info_for_epoch}; use crate::data_usage_define::{ BACKGROUND_HEAL_INFO_PATH, DATA_USAGE_BLOOM_NAME_PATH, DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, - DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, read_config_with_revision, + DATA_USAGE_RECOVERY_PATH, DataUsageCache, DataUsageCacheRevision, LEGACY_DATA_USAGE_OBJ_NAME_PATH, read_config_revision, + read_config_with_revision, }; use crate::runtime_config::{ ScannerRuntimeConfig, ScannerRuntimeConfigSource, refresh_scanner_runtime_config_from_global, scanner_bitrot_cycle, @@ -89,6 +90,144 @@ const EVENT_SCANNER_BACKGROUND_HEAL_STATE: &str = "scanner_background_heal_state const METRIC_SCANNER_LEADER_LOCK_TOTAL: &str = "rustfs_scanner_leader_lock_total"; const CLEAN_IDLE_MAX_INTERVAL: Duration = Duration::from_secs(24 * 60 * 60); const MAX_SCANNER_SCHEDULE_DELAY: Duration = Duration::from_secs(365 * 24 * 60 * 60); + +#[cfg(test)] +static SCANNER_STARTUP_OBSERVED_PROBE: LazyLock>>> = + LazyLock::new(|| StdMutex::new(None)); + +#[cfg(test)] +struct ScannerStartupObservedProbeState { + observed: Notify, + resume: Notify, +} + +#[cfg(test)] +struct ScannerObservedProbeState { + store_key: usize, + paused: bool, + notify: Notify, +} + +#[cfg(test)] +pub(super) struct ScannerStartupObservedProbe { + state: Arc, +} + +#[cfg(test)] +static SCANNER_RUNTIME_OBSERVED_PROBE: LazyLock>>> = + LazyLock::new(|| StdMutex::new(None)); + +#[cfg(test)] +pub(super) struct ScannerRuntimeObservedProbe { + state: Arc, +} + +#[cfg(test)] +impl ScannerStartupObservedProbe { + pub(super) fn install() -> Self { + let state = Arc::new(ScannerStartupObservedProbeState { + observed: Notify::new(), + resume: Notify::new(), + }); + let mut probe = SCANNER_STARTUP_OBSERVED_PROBE + .lock() + .expect("scanner startup observed probe should not be poisoned"); + assert!(probe.is_none(), "scanner startup observed probe must be unique"); + *probe = Some(state.clone()); + Self { state } + } + + pub(super) async fn wait(&self) { + tokio::time::timeout(Duration::from_secs(5), self.state.observed.notified()) + .await + .expect("scanner should complete startup pause-backlog observation"); + } + + pub(super) fn resume(&self) { + self.state.resume.notify_one(); + } +} + +#[cfg(test)] +impl ScannerRuntimeObservedProbe { + pub(super) fn install(storeapi: &Arc, paused: bool) -> Self { + let state = Arc::new(ScannerObservedProbeState { + store_key: scanner_observed_probe_store_key(storeapi), + paused, + notify: Notify::new(), + }); + let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE + .lock() + .expect("scanner runtime observed probe should not be poisoned"); + assert!(probe.is_none(), "scanner runtime observed probe must be unique"); + *probe = Some(state.clone()); + Self { state } + } + + pub(super) async fn wait(&self) { + tokio::time::timeout(Duration::from_secs(10), self.state.notify.notified()) + .await + .expect("scanner should complete runtime pause-backlog observation"); + } +} + +#[cfg(test)] +impl Drop for ScannerStartupObservedProbe { + fn drop(&mut self) { + let mut probe = SCANNER_STARTUP_OBSERVED_PROBE + .lock() + .expect("scanner startup observed probe should not be poisoned"); + if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *probe = None; + } + self.state.resume.notify_one(); + } +} + +#[cfg(test)] +impl Drop for ScannerRuntimeObservedProbe { + fn drop(&mut self) { + let mut probe = SCANNER_RUNTIME_OBSERVED_PROBE + .lock() + .expect("scanner runtime observed probe should not be poisoned"); + if probe.as_ref().is_some_and(|state| Arc::ptr_eq(state, &self.state)) { + *probe = None; + } + } +} + +#[cfg(test)] +async fn notify_scanner_startup_observed_for_test() { + let probe = { + SCANNER_STARTUP_OBSERVED_PROBE + .lock() + .expect("scanner startup observed probe should not be poisoned") + .clone() + }; + if let Some(probe) = probe { + probe.observed.notify_one(); + probe.resume.notified().await; + } +} + +#[cfg(test)] +fn scanner_observed_probe_store_key(storeapi: &Arc) -> usize { + Arc::as_ptr(storeapi).cast::<()>() as usize +} + +#[cfg(test)] +fn notify_scanner_runtime_observed_for_test(storeapi: &Arc, observation: ScannerPauseBacklogObservation) { + if let Some(probe) = SCANNER_RUNTIME_OBSERVED_PROBE + .lock() + .expect("scanner runtime observed probe should not be poisoned") + .clone() + && probe.store_key == scanner_observed_probe_store_key(storeapi) + && probe.paused == observation.paused + { + probe.notify.notify_one(); + } +} + const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2; /// First-retry delay after a scanner cycle cannot publish authoritative usage. /// @@ -294,6 +433,14 @@ fn record_scanner_leader_lock_state(state: &'static str) { .increment(1); } +async fn finish_scanner_leader_iteration(lock_lost: bool, state: &'static str, error: String) { + reset_scanner_cycle_schedule(); + let liveness_already_recorded = lock_lost && !global_metrics().report().await.leader_lock_held_by_this_process; + if !liveness_already_recorded { + global_metrics().record_scanner_leader_liveness(state, false, error).await; + } +} + #[cfg(test)] fn scanner_cycle_max_duration() -> Option { resolve_scanner_runtime_config().cycle_budget.max_duration @@ -742,15 +889,15 @@ fn prepare_cycle_for_usage_floor_bootstrap( cycle_info: &mut CurrentCycle, usage_floor: PersistedUsageFloor, startup: PersistedUsageFloorStartup, -) -> (bool, bool) { +) -> (bool, ScannerCycleResetPolicy) { match startup { - PersistedUsageFloorStartup::Authoritative => (false, false), + PersistedUsageFloorStartup::Authoritative => (false, ScannerCycleResetPolicy::None), PersistedUsageFloorStartup::Missing => { // Cycle progress without its corresponding usage floor cannot // prove namespace coverage. Restart from cycle zero while keeping // the separately fenced leader epoch monotonic. *cycle_info = CurrentCycle::default(); - (true, true) + (true, ScannerCycleResetPolicy::ResetAll) } PersistedUsageFloorStartup::BootstrapPending => { // An unfenced marker may have been written before an upgrade's old @@ -759,7 +906,25 @@ fn prepare_cycle_for_usage_floor_bootstrap( if usage_floor.leader_epoch == 0 { *cycle_info = CurrentCycle::default(); } - (true, usage_floor.leader_epoch == 0) + ( + true, + if usage_floor.leader_epoch == 0 { + ScannerCycleResetPolicy::ResetAll + } else { + ScannerCycleResetPolicy::None + }, + ) + } + PersistedUsageFloorStartup::RecoveredLegacyEmptyFence => { + // The legacy empty fence proves only its leader epoch, not + // namespace coverage. Clear coverage while retaining the durable + // cycle number so surviving caches cannot force a regression. + let next = cycle_info.next; + *cycle_info = CurrentCycle { + next, + ..Default::default() + }; + (true, ScannerCycleResetPolicy::ResetCoveragePreservingNext) } } } @@ -1280,7 +1445,15 @@ where LockLost: Future, { let fence_ctx = ctx.child_token(); - let claim = claim_scanner_leadership(&fence_ctx, storeapi, cycle_info, cycle_revision, leader_epoch, false, false); + let claim = claim_scanner_leadership( + &fence_ctx, + storeapi, + cycle_info, + cycle_revision, + leader_epoch, + false, + ScannerCycleResetPolicy::None, + ); tokio::pin!(claim); tokio::pin!(lock_lost); tokio::select! { @@ -1776,8 +1949,18 @@ async fn run_data_scanner_cycle_with_budget( mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await; return ScannerCycleOutcome::Failed; } - match scanner_cycle_pre_commit_outcome(scan_cycle_result.required_cycle_floor(), &usage_persist_outcome) { + let required_cycle_floor = scan_cycle_result.required_cycle_floor(); + let pre_commit_outcome = scanner_cycle_pre_commit_outcome(required_cycle_floor, &usage_persist_outcome); + update_scanner_cache_cycle_recovery_status( + cycle_info.current, + leader_epoch, + required_cycle_floor, + pre_commit_outcome, + scan_cycle_result.status == ScannerCycleStatus::Complete, + ); + match pre_commit_outcome { Some(ScannerCyclePreCommitOutcome::RecoverCacheCycle(required_cycle)) => { + record_scanner_cache_cycle_recovery_attempt(); warn!( target: "rustfs::scanner", event = EVENT_SCANNER_CYCLE_STATE, @@ -2145,6 +2328,74 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> run_data_scanner_with_maintenance_state(ctx, storeapi, maintenance_features, maintenance_generation).await } +async fn current_scanner_pause_backlog_observation(storeapi: &Arc) -> ScannerPauseBacklogObservation { + let now_unix_secs = scanner_pause_backlog_now(); + let pause = storeapi.scanner_data_movement_pause_status().await; + let metrics = global_metrics().report().await; + ScannerPauseBacklogObservation { + now_unix_secs, + paused: pause.paused, + movement_generation: pause.movement_generation, + movement_work_items: pause.movement_backlog_work_items, + pause_started_at_unix_secs: pause.started_at_unix_secs, + dirty_usage_buckets: metrics.usage_freshness.dirty_pending_buckets, + discovered_expiry_items: metrics + .lifecycle_expiry + .current_queued + .saturating_add(metrics.lifecycle_expiry.current_active), + discovered_transition_items: metrics + .lifecycle_transition + .current_queued + .saturating_add(metrics.lifecycle_transition.current_active) + .saturating_add(metrics.lifecycle_transition.compensation_pending) + .saturating_add(metrics.lifecycle_transition.compensation_running), + } +} + +async fn wait_for_scanner_data_movement_resume( + ctx: &CancellationToken, + storeapi: &Arc, + guard: &NamespaceLockGuard, + pause_backlog: &mut ScannerPauseBacklogController, +) -> bool { + loop { + let observation = current_scanner_pause_backlog_observation(storeapi).await; + pause_backlog.observe(observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(storeapi, observation); + if !observation.paused { + return !ctx.is_cancelled() && !guard.is_lock_lost(); + } + + let movement_changed = storeapi.scanner_data_movement_changed(); + if storeapi.scanner_data_movement_generation() != observation.movement_generation { + continue; + } + tokio::select! { + _ = ctx.cancelled() => return false, + _ = guard.lock_lost_notified() => return false, + _ = movement_changed.notified() => {}, + _ = tokio::time::sleep(SCANNER_CYCLE_RECOVERY_PAUSED_INTERVAL) => {}, + } + } +} + +async fn finish_scanner_pause_backlog_cycle( + pause_backlog: &mut ScannerPauseBacklogController, + storeapi: &Arc, + attempt: ScannerPauseBacklogAttemptDecision, + outcome: ScannerCycleOutcome, +) { + let observation = current_scanner_pause_backlog_observation(storeapi).await; + if let ScannerPauseBacklogAttemptDecision::Tracked(serial) = attempt { + pause_backlog.finish_attempt(serial, outcome, observation).await; + } else { + pause_backlog.observe_cycle_outcome(outcome, observation).await; + } + #[cfg(test)] + notify_scanner_runtime_observed_for_test(storeapi, observation); +} + async fn run_data_scanner_with_maintenance_state( ctx: CancellationToken, storeapi: Arc, @@ -2224,6 +2475,28 @@ async fn run_data_scanner_with_maintenance_state( return Ok(()); } }; + let pause_backlog_now = scanner_pause_backlog_now(); + let mut pause_backlog = match ScannerPauseBacklogController::claim(storeapi.clone(), pause_backlog_now).await { + Ok(controller) => controller, + Err(err) => { + error!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "pause_backlog_claim_failed", + error = %err, + "Scanner pause backlog persistence is unavailable" + ); + ScannerPauseBacklogController::unavailable(storeapi.clone(), err, pause_backlog_now) + } + }; + if !wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await { + global_metrics().set_cycle(None).await; + return Ok(()); + } + #[cfg(test)] + notify_scanner_startup_observed_for_test().await; let single_disk = storeapi.setup_is_erasure_sd().await; let erasure = storeapi.setup_is_erasure().await; let distributed = storeapi.setup_is_dist_erasure().await; @@ -2240,6 +2513,7 @@ async fn run_data_scanner_with_maintenance_state( { let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(false, "stopped", String::new()).await; return Ok(()); }; maintenance_features = features; @@ -2266,16 +2540,19 @@ async fn run_data_scanner_with_maintenance_state( } => (cycle, leader_epoch, revision), ScannerCycleStateStartup::Blocked => { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(false, "stopped", String::new()).await; return Ok(()); } ScannerCycleStateStartup::Transient(err) => { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(false, "stopped", String::new()).await; return Err(err); } }; let (usage_floor, usage_floor_startup) = match persisted_usage_floor_for_startup(storeapi.clone(), true).await { Ok(result) => result, Err(err) => { + let error = err.to_string(); error!( target: "rustfs::scanner", event = EVENT_SCANNER_PERSIST_STATE, @@ -2286,18 +2563,23 @@ async fn run_data_scanner_with_maintenance_state( error = %err, "Scanner stopped because the persisted usage floor could not be loaded" ); + record_scanner_usage_floor_failure(error.clone()); global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(false, "usage_floor_load_failed", error).await; return Ok(()); } }; - let (allow_usage_floor_bootstrap_pending, reset_usage_floor_bootstrap_cycle_on_conflict) = + let (allow_usage_floor_bootstrap_pending, usage_floor_cycle_reset_policy) = prepare_cycle_for_usage_floor_bootstrap(&mut cycle_info, usage_floor, usage_floor_startup); apply_persisted_usage_floor(&mut cycle_info, &mut leader_epoch, usage_floor); match usage_floor_startup { - PersistedUsageFloorStartup::Authoritative | PersistedUsageFloorStartup::BootstrapPending => {} + PersistedUsageFloorStartup::Authoritative + | PersistedUsageFloorStartup::BootstrapPending + | PersistedUsageFloorStartup::RecoveredLegacyEmptyFence => {} PersistedUsageFloorStartup::Missing => { if ctx.is_cancelled() || guard.is_lock_lost() { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; return Ok(()); } @@ -2322,10 +2604,12 @@ async fn run_data_scanner_with_maintenance_state( "Scanner stopped because the usage baseline bootstrap could not be initialized" ); global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(false, "usage_floor_bootstrap_failed", err.to_string()).await; return Ok(()); } None => { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; return Ok(()); } } @@ -2334,6 +2618,7 @@ async fn run_data_scanner_with_maintenance_state( if ctx.is_cancelled() || guard.is_lock_lost() { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; return Ok(()); } let claim_ctx = ctx.child_token(); @@ -2346,7 +2631,7 @@ async fn run_data_scanner_with_maintenance_state( &mut cycle_revision, &mut leader_epoch, allow_usage_floor_bootstrap_pending, - reset_usage_floor_bootstrap_cycle_on_conflict, + usage_floor_cycle_reset_policy, ), guard.lock_lost_notified(), ) @@ -2355,9 +2640,23 @@ async fn run_data_scanner_with_maintenance_state( if guard.is_lock_lost() { record_scanner_leader_lock_lost("Scanner leader lock lost while claiming the leadership epoch").await; global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(true, "lost", String::new()).await; return Ok(()); } if !leadership_claimed { + let observation = current_scanner_pause_backlog_observation(&storeapi).await; + pause_backlog.observe(observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(&storeapi, observation); + if observation.paused { + if wait_for_scanner_data_movement_resume(&ctx, &storeapi, &guard, &mut pause_backlog).await { + return Err(ScannerError::Other( + "scanner startup was fenced by data movement; retrying from durable state".to_string(), + )); + } + global_metrics().set_cycle(None).await; + return Ok(()); + } error!( target: "rustfs::scanner", event = EVENT_SCANNER_LOCK_STATE, @@ -2367,14 +2666,36 @@ async fn run_data_scanner_with_maintenance_state( state = "epoch_claim_failed", "Scanner stopped because the leadership epoch could not be claimed" ); - global_metrics() - .record_scanner_leader_liveness("epoch_claim_failed", false, "leadership epoch claim failed") - .await; global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await; + return Ok(()); + } + if usage_floor_startup == PersistedUsageFloorStartup::RecoveredLegacyEmptyFence + && let Err(err) = complete_legacy_empty_usage_floor_recovery(storeapi.clone(), leader_epoch).await + { + let error = err.to_string(); + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "usage_floor_recovery_cleanup_deferred", + path = %DATA_USAGE_RECOVERY_PATH.as_str(), + error = %err, + "Scanner usage floor recovery marker cleanup was deferred" + ); + global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(false, "usage_floor_recovery_pending", error).await; return Ok(()); } - if !ctx.is_cancelled() { + let initial_pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await; + if !ctx.is_cancelled() + && matches!( + initial_pause_backlog_attempt, + ScannerPauseBacklogAttemptDecision::Untracked | ScannerPauseBacklogAttemptDecision::Tracked(_) + ) + { // Preserve previous behavior: run one cycle immediately after lock acquisition. let dirty_generation_before_cycle = dirty_usage_generation(); let dirty_usage_pending_before_cycle = dirty_usage_buckets_pending(); @@ -2382,6 +2703,7 @@ async fn run_data_scanner_with_maintenance_state( if guard.is_lock_lost() { record_scanner_leader_lock_lost("Scanner leader lock lost before the initial cycle").await; global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(true, "lost", String::new()).await; return Ok(()); } let cycle_ctx = ctx.child_token(); @@ -2405,10 +2727,12 @@ async fn run_data_scanner_with_maintenance_state( ScannerCycleWaitOutcome::LockLost => { record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await; global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(true, "lost", String::new()).await; return Ok(()); } ScannerCycleWaitOutcome::Cancelled => { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; return Ok(()); } ScannerCycleWaitOutcome::Deadline { worker_stopped } => { @@ -2425,15 +2749,18 @@ async fn run_data_scanner_with_maintenance_state( &mut guard, ) .await; + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; return Ok(()); } }; + finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, initial_pause_backlog_attempt, initial_outcome).await; superseded_backoff.record_retryable_cycle(initial_outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(initial_outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; if guard.is_lock_lost() { record_scanner_leader_lock_lost("Scanner leader lock lost during the initial cycle").await; global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(true, "lost", String::new()).await; return Ok(()); } let runtime_config = resolve_scanner_runtime_config(); @@ -2479,6 +2806,10 @@ async fn run_data_scanner_with_maintenance_state( break; } + let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await; + pause_backlog.observe(pause_backlog_observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation); let runtime_config = resolve_scanner_runtime_config(); if clean_idle_topology_supported && scanner_clean_idle_backoff_configured(&runtime_config) { let current_generation = scanner_maintenance_generation(); @@ -2515,11 +2846,16 @@ async fn run_data_scanner_with_maintenance_state( scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for); let superseded_retry_interval = superseded_backoff.retry_interval(runtime_config.cycle_interval); let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval); - let convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval); + let mut convergence_retry_interval = superseded_retry_interval.or(deferred_retry_interval); if let Some(retry_interval) = convergence_retry_interval { wait_plan.effective_interval = retry_interval; wait_plan.delay = randomized_cycle_delay_for(retry_interval).min(retry_interval); } + if let Some(pause_backlog_delay) = pause_backlog.scheduling_delay(scanner_pause_backlog_now()) { + wait_plan.effective_interval = pause_backlog_delay.max(Duration::from_secs(1)); + wait_plan.delay = pause_backlog_delay; + convergence_retry_interval = Some(pause_backlog_delay.max(Duration::from_secs(1))); + } let dirty_generation_before_wait = dirty_usage_generation(); let dirty_usage_pending_before_wait = dirty_usage_buckets_pending(); let maintenance_generation_before_wait = scanner_maintenance_generation(); @@ -2644,6 +2980,20 @@ async fn run_data_scanner_with_maintenance_state( record_scanner_leader_lock_lost("Scanner leader lock lost before starting the next cycle").await; break; } + let pause_backlog_observation = current_scanner_pause_backlog_observation(&storeapi).await; + pause_backlog.observe(pause_backlog_observation).await; + #[cfg(test)] + notify_scanner_runtime_observed_for_test(&storeapi, pause_backlog_observation); + if pause_backlog_observation.paused { + continue; + } + let pause_backlog_attempt = pause_backlog.begin_attempt(scanner_pause_backlog_now()).await; + if matches!( + pause_backlog_attempt, + ScannerPauseBacklogAttemptDecision::RateLimited | ScannerPauseBacklogAttemptDecision::PersistenceUnavailable + ) { + continue; + } let dirty_generation_before_cycle = dirty_usage_generation(); let cycle_ctx = ctx.child_token(); let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config()); @@ -2666,10 +3016,12 @@ async fn run_data_scanner_with_maintenance_state( ScannerCycleWaitOutcome::LockLost => { record_scanner_leader_lock_lost("Scanner leader lock lost during a scanner cycle").await; global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(true, "lost", String::new()).await; return Ok(()); } ScannerCycleWaitOutcome::Cancelled => { global_metrics().set_cycle(None).await; + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; return Ok(()); } ScannerCycleWaitOutcome::Deadline { worker_stopped } => { @@ -2686,9 +3038,11 @@ async fn run_data_scanner_with_maintenance_state( &mut guard, ) .await; + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; return Ok(()); } }; + finish_scanner_pause_backlog_cycle(&mut pause_backlog, &storeapi, pause_backlog_attempt, outcome).await; superseded_backoff.record_retryable_cycle(outcome == ScannerCycleOutcome::Superseded); deferred_backoff.record_retryable_cycle(matches!(outcome, ScannerCycleOutcome::Deferred(_))); dirty_usage_generation_seen = dirty_generation_before_cycle; @@ -2771,10 +3125,7 @@ async fn run_data_scanner_with_maintenance_state( } global_metrics().set_cycle(None).await; - reset_scanner_cycle_schedule(); - if !guard.is_lock_lost() { - global_metrics().record_scanner_leader_liveness("stopped", false, "").await; - } + finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await; debug!( target: "rustfs::scanner", @@ -2852,6 +3203,26 @@ fn scanner_cycle_pre_commit_outcome( } } +fn update_scanner_cache_cycle_recovery_status( + requested_cycle: u64, + leader_epoch: u64, + required_cycle_floor: Option, + pre_commit_outcome: Option, + cache_scope_complete: bool, +) { + match (required_cycle_floor, pre_commit_outcome) { + (Some(required_cycle), _) => { + record_scanner_cache_cycle_ahead(requested_cycle, required_cycle, leader_epoch); + } + (None, Some(ScannerCyclePreCommitOutcome::Deferred(_))) => { + // A deferred scan may not have covered the cache that established + // the existing floor, so it cannot prove recovery is complete. + } + (None, _) if cache_scope_complete => clear_scanner_cache_cycle_ahead(), + (None, _) => {} + } +} + fn scanner_cycle_completion_outcome( scan_status: ScannerCycleStatus, usage_persist_outcome: DataUsagePersistOutcome, @@ -2989,12 +3360,14 @@ fn data_usage_reintroduces_missing_bucket(incoming: &DataUsageInfo, existing: Op /// Store data usage info in backend. Will store all objects sent on the receiver until closed. mod activity; +mod backlog; mod cycle_state; mod heal_info; mod leadership; mod usage_store; use activity::*; +use backlog::*; use cycle_state::*; use leadership::*; use usage_store::*; @@ -3005,6 +3378,10 @@ pub(crate) use activity::{ scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_dirty_usage_acknowledgements, }; pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance}; +pub use backlog::{ + ScannerPauseBacklogAlertReason, ScannerPauseBacklogPhase, ScannerPauseBacklogStatus, ScannerPauseBacklogThresholds, + scanner_pause_backlog_status, +}; #[cfg(test)] pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test; pub use cycle_state::{ diff --git a/crates/scanner/src/scanner/backlog.rs b/crates/scanner/src/scanner/backlog.rs new file mode 100644 index 000000000..2f160a42a --- /dev/null +++ b/crates/scanner/src/scanner/backlog.rs @@ -0,0 +1,2377 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Durable operational accounting for scanner pauses and bounded catch-up. +//! +//! This ledger is replicated outside the authoritative data-usage publication +//! path so it can advance while that path is fenced by data movement. It never +//! grants publication admission; scanner usage still passes the storage-owned +//! movement epoch and final publication fences. + +use super::ScannerCycleOutcome; +use crate::data_usage_define::DataUsageCacheRevision; +use crate::storage_api::owner::ObjectIO as _; +use crate::{ + BUCKET_META_PREFIX, ECStore, EcstoreError, RUSTFS_META_BUCKET, ScannerObjectOptions, SetDisks, save_config_with_preconditions, +}; +use futures::future::join_all; +use http::HeaderMap; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap}; +use std::sync::{Arc, LazyLock, RwLock}; +use std::time::{Duration, SystemTime, UNIX_EPOCH}; +use tokio::io::AsyncReadExt; + +const SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION: u16 = 1; +const SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION: u16 = 1; +const SCANNER_PAUSE_BACKLOG_OBJECT: &str = ".scanner-pause-backlog.json"; +const MAX_SCANNER_PAUSE_BACKLOG_BYTES: u64 = 64 * 1024; +const SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS: u64 = 5 * 60; +const SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS: u64 = 5 * 60; +const SCANNER_CATCH_UP_WINDOW_SECONDS: u64 = 60 * 60; +const SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW: u32 = 4; +const SCANNER_CATCH_UP_FAILURE_LIMIT: u32 = 5; +const SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS: u64 = 60 * 60; +const SCANNER_PAUSE_DURATION_ALERT_SECONDS: u64 = 24 * 60 * 60; +const SCANNER_PAUSE_DEFERRED_CYCLES_ALERT: u64 = 3; +const SCANNER_PAUSE_BACKLOG_ITEMS_ALERT: u64 = 10_000; + +const METRIC_SCANNER_PAUSE_BACKLOG_PHASE: &str = "rustfs_scanner_pause_backlog_phase"; +const METRIC_SCANNER_PAUSE_BACKLOG_PAUSE_DURATION_SECONDS: &str = "rustfs_scanner_pause_backlog_pause_duration_seconds"; +const METRIC_SCANNER_PAUSE_BACKLOG_PENDING_WORK_ITEMS: &str = "rustfs_scanner_pause_backlog_pending_work_items"; +const METRIC_SCANNER_PAUSE_BACKLOG_CONSECUTIVE_FAILURES: &str = "rustfs_scanner_pause_backlog_consecutive_failures"; +const METRIC_SCANNER_PAUSE_BACKLOG_RATE_LIMITED: &str = "rustfs_scanner_pause_backlog_rate_limited"; +const METRIC_SCANNER_PAUSE_BACKLOG_RETRY_EXHAUSTED: &str = "rustfs_scanner_pause_backlog_retry_exhausted"; +const METRIC_SCANNER_PAUSE_BACKLOG_ALERTING: &str = "rustfs_scanner_pause_backlog_alerting"; +const METRIC_SCANNER_PAUSE_BACKLOG_REPLICA_DEGRADED: &str = "rustfs_scanner_pause_backlog_replica_degraded"; + +static SCANNER_PAUSE_BACKLOG_PATH: LazyLock = + LazyLock::new(|| format!("{BUCKET_META_PREFIX}/{SCANNER_PAUSE_BACKLOG_OBJECT}")); +static SCANNER_PAUSE_BACKLOG_RUNTIME_ERROR: LazyLock>> = LazyLock::new(|| RwLock::new(None)); + +#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScannerPauseBacklogPhase { + #[default] + Idle, + Paused, + CatchingUp, + RetryExhausted, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum ScannerPauseBacklogAlertReason { + PauseDurationThreshold, + DeferredCyclesThreshold, + BacklogItemsThreshold, + RetryBudgetExhausted, + CounterExhausted, + ReplicaDegraded, + PersistenceUnavailable, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)] +pub struct ScannerPauseBacklogThresholds { + pub pause_duration_seconds: u64, + pub deferred_cycles: u64, + pub backlog_work_items: u64, + pub catch_up_min_interval_seconds: u64, + pub catch_up_window_seconds: u64, + pub catch_up_max_attempts_per_window: u32, + pub catch_up_failure_limit: u32, + pub retry_exhausted_probe_seconds: u64, +} + +impl Default for ScannerPauseBacklogThresholds { + fn default() -> Self { + Self { + pause_duration_seconds: SCANNER_PAUSE_DURATION_ALERT_SECONDS, + deferred_cycles: SCANNER_PAUSE_DEFERRED_CYCLES_ALERT, + backlog_work_items: SCANNER_PAUSE_BACKLOG_ITEMS_ALERT, + catch_up_min_interval_seconds: SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS, + catch_up_window_seconds: SCANNER_CATCH_UP_WINDOW_SECONDS, + catch_up_max_attempts_per_window: SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW, + catch_up_failure_limit: SCANNER_CATCH_UP_FAILURE_LIMIT, + retry_exhausted_probe_seconds: SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS, + } + } +} + +#[derive(Clone, Debug, Default, Serialize)] +pub struct ScannerPauseBacklogStatus { + pub path: String, + pub persistence_state: String, + pub durable: bool, + pub schema_version: u16, + pub generation: u64, + pub writer_epoch: u64, + pub phase: ScannerPauseBacklogPhase, + pub movement_generation: u64, + pub movement_work_items: u64, + pub pause_started_at_unix_secs: u64, + pub pause_ended_at_unix_secs: u64, + pub pause_duration_seconds: u64, + pub last_updated_at_unix_secs: u64, + pub deferred_cycles: u64, + pub pending_full_scan: bool, + pub dirty_usage_buckets: u64, + pub discovered_expiry_items: u64, + pub discovered_transition_items: u64, + pub pending_work_items: u64, + pub catch_up_attempts: u64, + pub consecutive_failures: u32, + pub attempts_in_current_window: u32, + pub current_window_started_at_unix_secs: u64, + pub last_attempt_at_unix_secs: u64, + pub next_attempt_at_unix_secs: u64, + pub rate_limited: bool, + pub retry_exhausted: bool, + pub replica_count: usize, + pub healthy_replicas: usize, + pub stale_or_unavailable_replicas: usize, + pub alerting: bool, + pub alert_reasons: Vec, + pub thresholds: ScannerPauseBacklogThresholds, + pub error: Option, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +struct ScannerPauseBacklogLedger { + schema_version: u16, + generation: u64, + writer_epoch: u64, + phase: ScannerPauseBacklogPhase, + movement_generation: u64, + movement_work_items: u64, + pause_started_at_unix_secs: u64, + pause_ended_at_unix_secs: u64, + last_updated_at_unix_secs: u64, + deferred_cycles: u64, + pending_full_scan: bool, + dirty_usage_buckets: u64, + discovered_expiry_items: u64, + discovered_transition_items: u64, + catch_up_attempts: u64, + consecutive_failures: u32, + current_window_started_at_unix_secs: u64, + attempts_in_current_window: u32, + last_attempt_at_unix_secs: u64, + next_attempt_at_unix_secs: u64, + current_attempt_serial: u64, + last_finished_attempt_serial: u64, + counter_exhausted: bool, +} + +impl Default for ScannerPauseBacklogLedger { + fn default() -> Self { + Self { + schema_version: SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION, + generation: 0, + writer_epoch: 0, + phase: ScannerPauseBacklogPhase::Idle, + movement_generation: 0, + movement_work_items: 0, + pause_started_at_unix_secs: 0, + pause_ended_at_unix_secs: 0, + last_updated_at_unix_secs: 0, + deferred_cycles: 0, + pending_full_scan: false, + dirty_usage_buckets: 0, + discovered_expiry_items: 0, + discovered_transition_items: 0, + catch_up_attempts: 0, + consecutive_failures: 0, + current_window_started_at_unix_secs: 0, + attempts_in_current_window: 0, + last_attempt_at_unix_secs: 0, + next_attempt_at_unix_secs: 0, + current_attempt_serial: 0, + last_finished_attempt_serial: 0, + counter_exhausted: false, + } + } +} + +impl ScannerPauseBacklogLedger { + fn validate(&self) -> Result<(), String> { + if self.schema_version != SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION { + return Err(format!("unsupported scanner pause backlog schema {}", self.schema_version)); + } + if self.generation == 0 || self.writer_epoch == 0 || self.last_updated_at_unix_secs == 0 { + return Err("scanner pause backlog has an invalid durable fence".to_string()); + } + if self.last_finished_attempt_serial > self.current_attempt_serial { + return Err("scanner pause backlog finished attempt exceeds the current attempt".to_string()); + } + if self.attempts_in_current_window > SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW { + return Err("scanner pause backlog rate window exceeds its attempt limit".to_string()); + } + if self.phase != ScannerPauseBacklogPhase::Idle && self.last_attempt_at_unix_secs > self.next_attempt_at_unix_secs { + return Err("scanner pause backlog next attempt precedes its last attempt".to_string()); + } + if self.phase == ScannerPauseBacklogPhase::Paused && !self.pending_full_scan { + return Err("scanner pause backlog lost its required post-pause scan".to_string()); + } + if self.phase == ScannerPauseBacklogPhase::Paused && self.pause_started_at_unix_secs == 0 { + return Err("scanner pause backlog has no pause start time".to_string()); + } + if matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) && (self.pause_started_at_unix_secs == 0 || self.pause_ended_at_unix_secs == 0) + { + return Err("scanner pause backlog has incomplete catch-up timestamps".to_string()); + } + if self.pause_ended_at_unix_secs != 0 && self.pause_ended_at_unix_secs < self.pause_started_at_unix_secs { + return Err("scanner pause backlog ends before its pause start".to_string()); + } + if self.phase == ScannerPauseBacklogPhase::Idle && self.pending_full_scan { + return Err("idle scanner pause backlog still requires a full scan".to_string()); + } + Ok(()) + } + + fn pending_work_items(&self) -> u64 { + (if self.pending_full_scan { 1_u64 } else { 0 }) + .saturating_add(self.dirty_usage_buckets) + .saturating_add(self.discovered_expiry_items) + .saturating_add(self.discovered_transition_items) + } + + fn pause_duration_seconds(&self, now: u64) -> u64 { + if self.pause_started_at_unix_secs == 0 { + return 0; + } + let end = if self.phase == ScannerPauseBacklogPhase::Paused { + now + } else { + self.pause_ended_at_unix_secs + }; + end.saturating_sub(self.pause_started_at_unix_secs) + } + + fn has_unfinished_attempt(&self) -> bool { + self.current_attempt_serial > self.last_finished_attempt_serial + } + + fn rate_limit_floor(&self) -> u64 { + let mut floor = self.next_attempt_at_unix_secs; + if self.last_attempt_at_unix_secs > 0 { + floor = floor.max( + self.last_attempt_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS), + ); + } + if self.current_window_started_at_unix_secs > 0 + && self.attempts_in_current_window >= SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW + { + floor = floor.max( + self.current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS), + ); + } + floor + } + + fn claim_writer(&mut self, now: u64) -> Result<(), String> { + if self.has_unfinished_attempt() { + self.last_finished_attempt_serial = self.current_attempt_serial; + if matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) { + increment_u32(&mut self.consecutive_failures, &mut self.counter_exhausted); + self.exhaust_retry_budget(now); + } + } + self.writer_epoch = self + .writer_epoch + .checked_add(1) + .ok_or_else(|| "scanner pause backlog writer epoch is exhausted".to_string())?; + Ok(()) + } + + fn apply_observation(&mut self, observation: ScannerPauseBacklogObservation) { + if !observation.paused + && observation.movement_generation != 0 + && observation.movement_generation != self.movement_generation + { + self.apply_observation(ScannerPauseBacklogObservation { + paused: true, + pause_started_at_unix_secs: observation.now_unix_secs, + ..observation + }); + self.apply_observation(observation); + return; + } + if !observation.paused && self.phase == ScannerPauseBacklogPhase::Idle { + self.movement_generation = observation.movement_generation; + self.movement_work_items = 0; + return; + } + self.movement_work_items = observation.movement_work_items; + self.dirty_usage_buckets = observation.dirty_usage_buckets; + self.discovered_expiry_items = observation.discovered_expiry_items; + self.discovered_transition_items = observation.discovered_transition_items; + + if observation.paused { + let new_pause = + self.phase != ScannerPauseBacklogPhase::Paused || self.movement_generation != observation.movement_generation; + if new_pause { + increment_u64(&mut self.deferred_cycles, &mut self.counter_exhausted); + self.pause_started_at_unix_secs = if observation.pause_started_at_unix_secs == 0 { + observation.now_unix_secs + } else { + observation.pause_started_at_unix_secs + }; + self.pause_ended_at_unix_secs = 0; + self.consecutive_failures = 0; + } else if observation.pause_started_at_unix_secs > 0 { + self.pause_started_at_unix_secs = self.pause_started_at_unix_secs.min(observation.pause_started_at_unix_secs); + } + self.phase = ScannerPauseBacklogPhase::Paused; + self.movement_generation = observation.movement_generation; + self.pending_full_scan = true; + self.next_attempt_at_unix_secs = self.rate_limit_floor(); + return; + } + + self.movement_generation = observation.movement_generation; + if self.phase == ScannerPauseBacklogPhase::Paused { + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.pause_ended_at_unix_secs = observation.now_unix_secs.max(self.pause_started_at_unix_secs); + self.pending_full_scan = true; + self.consecutive_failures = 0; + self.next_attempt_at_unix_secs = self.next_attempt_at_unix_secs.max(observation.now_unix_secs); + } else if matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) && !self.pending_full_scan + && self.pending_work_items() == 0 + { + self.phase = ScannerPauseBacklogPhase::Idle; + self.deferred_cycles = 0; + self.consecutive_failures = 0; + self.next_attempt_at_unix_secs = 0; + } + } + + fn begin_attempt(&mut self, now: u64) -> ScannerPauseBacklogAttemptDecision { + if !matches!( + self.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) { + return ScannerPauseBacklogAttemptDecision::Untracked; + } + if !self.pending_full_scan && self.pending_work_items() == 0 { + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + return ScannerPauseBacklogAttemptDecision::RateLimited; + } + if self.has_unfinished_attempt() || now < self.next_attempt_at_unix_secs { + return ScannerPauseBacklogAttemptDecision::RateLimited; + } + + if self.phase == ScannerPauseBacklogPhase::CatchingUp { + let window_end = self + .current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS); + if self.current_window_started_at_unix_secs == 0 || now >= window_end { + self.current_window_started_at_unix_secs = now; + self.attempts_in_current_window = 0; + } + if self.attempts_in_current_window >= SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW { + self.next_attempt_at_unix_secs = self + .current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS); + return ScannerPauseBacklogAttemptDecision::RateLimited; + } + } + + let Some(serial) = self.current_attempt_serial.checked_add(1) else { + self.counter_exhausted = true; + self.phase = ScannerPauseBacklogPhase::RetryExhausted; + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS); + return ScannerPauseBacklogAttemptDecision::RateLimited; + }; + self.current_attempt_serial = serial; + increment_u64(&mut self.catch_up_attempts, &mut self.counter_exhausted); + self.last_attempt_at_unix_secs = now; + + if self.phase == ScannerPauseBacklogPhase::CatchingUp { + increment_u32(&mut self.attempts_in_current_window, &mut self.counter_exhausted); + self.next_attempt_at_unix_secs = if self.attempts_in_current_window >= SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW { + self.current_window_started_at_unix_secs + .saturating_add(SCANNER_CATCH_UP_WINDOW_SECONDS) + } else { + now.saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS) + }; + } else { + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS); + } + self.exhaust_retry_budget(now); + + ScannerPauseBacklogAttemptDecision::Tracked(serial) + } + + fn finish_attempt( + &mut self, + serial: u64, + outcome: ScannerPauseBacklogCycleOutcome, + observation: ScannerPauseBacklogObservation, + ) { + if serial == 0 || serial != self.current_attempt_serial || serial <= self.last_finished_attempt_serial { + return; + } + let movement_generation_advanced = + observation.movement_generation != 0 && observation.movement_generation != self.movement_generation; + let retry_exhausted_probe = self.phase == ScannerPauseBacklogPhase::RetryExhausted; + let successful_probe = matches!( + outcome, + ScannerPauseBacklogCycleOutcome::Completed + | ScannerPauseBacklogCycleOutcome::PendingMaintenance + | ScannerPauseBacklogCycleOutcome::Progressed + ); + let scheduled_retry_at = self.rate_limit_floor(); + let attempt_retry_at = if movement_generation_advanced || outcome == ScannerPauseBacklogCycleOutcome::DataMovementDeferred + { + scheduled_retry_at.max( + observation + .now_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS), + ) + } else { + scheduled_retry_at + }; + self.last_finished_attempt_serial = serial; + self.observe_cycle_outcome(outcome, observation); + let recovered_probe = retry_exhausted_probe && successful_probe && !self.counter_exhausted; + if self.phase == ScannerPauseBacklogPhase::Paused { + if recovered_probe { + self.consecutive_failures = 0; + self.current_window_started_at_unix_secs = observation.now_unix_secs; + self.attempts_in_current_window = 0; + self.next_attempt_at_unix_secs = observation + .now_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS); + } else { + self.next_attempt_at_unix_secs = self.next_attempt_at_unix_secs.max(attempt_retry_at); + } + return; + } + + match outcome { + ScannerPauseBacklogCycleOutcome::Completed => { + self.consecutive_failures = 0; + if movement_generation_advanced { + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.pending_full_scan = true; + } else { + self.pending_full_scan = false; + if self.pending_work_items() == 0 { + self.phase = ScannerPauseBacklogPhase::Idle; + self.deferred_cycles = 0; + self.next_attempt_at_unix_secs = 0; + } else { + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.next_attempt_at_unix_secs = observation + .now_unix_secs + .saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + } + } + } + ScannerPauseBacklogCycleOutcome::Progressed => { + self.consecutive_failures = 0; + self.phase = ScannerPauseBacklogPhase::CatchingUp; + } + ScannerPauseBacklogCycleOutcome::PendingMaintenance => { + self.consecutive_failures = 0; + self.phase = ScannerPauseBacklogPhase::CatchingUp; + self.pending_full_scan = true; + } + ScannerPauseBacklogCycleOutcome::DataMovementDeferred => {} + ScannerPauseBacklogCycleOutcome::RetryableFailure => { + increment_u32(&mut self.consecutive_failures, &mut self.counter_exhausted); + self.exhaust_retry_budget(observation.now_unix_secs); + } + } + if retry_exhausted_probe && !recovered_probe { + self.phase = ScannerPauseBacklogPhase::RetryExhausted; + } + if self.phase != ScannerPauseBacklogPhase::Idle { + if recovered_probe { + self.current_window_started_at_unix_secs = observation.now_unix_secs; + self.attempts_in_current_window = 0; + self.next_attempt_at_unix_secs = observation + .now_unix_secs + .saturating_add(SCANNER_CATCH_UP_MIN_INTERVAL_SECONDS); + } else { + self.next_attempt_at_unix_secs = self.next_attempt_at_unix_secs.max(attempt_retry_at); + } + } + } + + fn observe_cycle_outcome(&mut self, outcome: ScannerPauseBacklogCycleOutcome, observation: ScannerPauseBacklogObservation) { + if outcome == ScannerPauseBacklogCycleOutcome::DataMovementDeferred && !observation.paused { + self.apply_observation(ScannerPauseBacklogObservation { + paused: true, + pause_started_at_unix_secs: observation.now_unix_secs, + ..observation + }); + } + self.apply_observation(observation); + } + + fn exhaust_retry_budget(&mut self, now: u64) { + if self.counter_exhausted || self.consecutive_failures >= SCANNER_CATCH_UP_FAILURE_LIMIT { + self.phase = ScannerPauseBacklogPhase::RetryExhausted; + self.next_attempt_at_unix_secs = now.saturating_add(SCANNER_CATCH_UP_EXHAUSTED_PROBE_SECONDS); + } + } + + fn alert_reasons( + &self, + now: u64, + replica_degraded: bool, + persistence_unavailable: bool, + ) -> Vec { + let mut reasons = Vec::new(); + if self.phase != ScannerPauseBacklogPhase::Idle + && self.pause_duration_seconds(now) >= SCANNER_PAUSE_DURATION_ALERT_SECONDS + { + reasons.push(ScannerPauseBacklogAlertReason::PauseDurationThreshold); + } + if self.phase != ScannerPauseBacklogPhase::Idle && self.deferred_cycles >= SCANNER_PAUSE_DEFERRED_CYCLES_ALERT { + reasons.push(ScannerPauseBacklogAlertReason::DeferredCyclesThreshold); + } + if self.pending_work_items() >= SCANNER_PAUSE_BACKLOG_ITEMS_ALERT { + reasons.push(ScannerPauseBacklogAlertReason::BacklogItemsThreshold); + } + if self.phase == ScannerPauseBacklogPhase::RetryExhausted { + reasons.push(ScannerPauseBacklogAlertReason::RetryBudgetExhausted); + } + if self.counter_exhausted { + reasons.push(ScannerPauseBacklogAlertReason::CounterExhausted); + } + if replica_degraded { + reasons.push(ScannerPauseBacklogAlertReason::ReplicaDegraded); + } + if persistence_unavailable { + reasons.push(ScannerPauseBacklogAlertReason::PersistenceUnavailable); + } + reasons + } +} + +fn increment_u64(value: &mut u64, exhausted: &mut bool) { + if let Some(next) = value.checked_add(1) { + *value = next; + } else { + *exhausted = true; + } +} + +fn increment_u32(value: &mut u32, exhausted: &mut bool) { + if let Some(next) = value.checked_add(1) { + *value = next; + } else { + *exhausted = true; + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) struct ScannerPauseBacklogObservation { + pub(super) now_unix_secs: u64, + pub(super) paused: bool, + pub(super) movement_generation: u64, + pub(super) movement_work_items: u64, + pub(super) pause_started_at_unix_secs: u64, + pub(super) dirty_usage_buckets: u64, + pub(super) discovered_expiry_items: u64, + pub(super) discovered_transition_items: u64, +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum ScannerPauseBacklogCycleOutcome { + Completed, + PendingMaintenance, + Progressed, + DataMovementDeferred, + RetryableFailure, +} + +impl From for ScannerPauseBacklogCycleOutcome { + fn from(outcome: ScannerCycleOutcome) -> Self { + match outcome { + ScannerCycleOutcome::Completed => Self::Completed, + ScannerCycleOutcome::CompletedWithPendingMaintenance => Self::PendingMaintenance, + ScannerCycleOutcome::Partial => Self::Progressed, + ScannerCycleOutcome::Deferred(super::ScannerCycleDeferReason::DataMovement) => Self::DataMovementDeferred, + ScannerCycleOutcome::Superseded | ScannerCycleOutcome::Deferred(_) | ScannerCycleOutcome::Failed => { + Self::RetryableFailure + } + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ScannerPauseBacklogAttemptDecision { + Untracked, + RateLimited, + Tracked(u64), + PersistenceUnavailable, +} + +#[derive(Clone, Copy, Debug, Deserialize, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize)] +struct ScannerPauseBacklogReplicaId { + pool_index: usize, + set_index: usize, +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +struct ScannerPauseBacklogCommitRecord { + ledger: ScannerPauseBacklogLedger, + replicas: Vec, +} + +impl ScannerPauseBacklogCommitRecord { + fn new(ledger: ScannerPauseBacklogLedger, replicas: Vec) -> Self { + Self { ledger, replicas } + } + + fn validate(&self) -> Result<(), String> { + self.ledger.validate()?; + if self.replicas.is_empty() || !self.replicas.windows(2).all(|pair| pair[0] < pair[1]) { + return Err("scanner pause backlog commit has invalid replica membership".to_string()); + } + Ok(()) + } +} + +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +#[serde(deny_unknown_fields)] +struct ScannerPauseBacklogReplicaRecord { + replica_schema_version: u16, + /// Last generation known to be safe without consulting commit records. + stable: Option, + /// Candidate authority only when every surviving set stores it exactly. + committed: Option, +} + +impl ScannerPauseBacklogReplicaRecord { + fn new(stable: Option, committed: Option) -> Self { + Self { + replica_schema_version: SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION, + stable, + committed, + } + } + + fn validate(&self) -> Result<(), String> { + if self.replica_schema_version != SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION { + return Err(format!( + "unsupported scanner pause backlog replica schema {}", + self.replica_schema_version + )); + } + if self.stable.is_none() && self.committed.is_none() { + return Err("scanner pause backlog replica has no durable state".to_string()); + } + if let Some(stable) = &self.stable { + stable.validate()?; + } + if let Some(committed) = &self.committed { + committed.validate()?; + if let Some(stable) = &self.stable { + let stable_key = (stable.writer_epoch, stable.generation); + let committed_key = (committed.ledger.writer_epoch, committed.ledger.generation); + if committed_key < stable_key || (committed_key == stable_key && &committed.ledger != stable) { + return Err("scanner pause backlog commit precedes or diverges from its stable generation".to_string()); + } + } + } + Ok(()) + } +} + +#[derive(Clone)] +enum ScannerPauseBacklogReplicaState { + Missing, + Valid(Box), + Invalid(String), + FutureSchema(u64), + Unavailable(String), +} + +#[derive(Clone)] +struct ScannerPauseBacklogReplica { + id: ScannerPauseBacklogReplicaId, + revision: Option, + state: ScannerPauseBacklogReplicaState, +} + +#[derive(Clone)] +struct LoadedScannerPauseBacklog { + ledger: ScannerPauseBacklogLedger, + durable: bool, + persistence_state: String, + replicas: Vec, + replica_count: usize, + healthy_replicas: usize, + stale_or_unavailable_replicas: usize, + stable_matches_ledger: bool, + authoritative_commit: Option, + requires_reload: bool, +} + +impl LoadedScannerPauseBacklog { + fn status(&self, now: u64, error: Option) -> ScannerPauseBacklogStatus { + let persistence_unavailable = error.is_some(); + let replica_degraded = self.durable && self.stale_or_unavailable_replicas > 0; + status_from_ledger( + &self.ledger, + now, + self.persistence_state.clone(), + self.durable, + self.replica_count, + self.healthy_replicas, + self.stale_or_unavailable_replicas, + replica_degraded, + persistence_unavailable, + error, + ) + } +} + +#[allow(clippy::too_many_arguments)] +fn status_from_ledger( + ledger: &ScannerPauseBacklogLedger, + now: u64, + persistence_state: String, + durable: bool, + replica_count: usize, + healthy_replicas: usize, + stale_or_unavailable_replicas: usize, + replica_degraded: bool, + persistence_unavailable: bool, + error: Option, +) -> ScannerPauseBacklogStatus { + let alert_reasons = ledger.alert_reasons(now, replica_degraded, persistence_unavailable); + let status = ScannerPauseBacklogStatus { + path: SCANNER_PAUSE_BACKLOG_PATH.clone(), + persistence_state, + durable, + schema_version: ledger.schema_version, + generation: ledger.generation, + writer_epoch: ledger.writer_epoch, + phase: ledger.phase, + movement_generation: ledger.movement_generation, + movement_work_items: ledger.movement_work_items, + pause_started_at_unix_secs: ledger.pause_started_at_unix_secs, + pause_ended_at_unix_secs: ledger.pause_ended_at_unix_secs, + pause_duration_seconds: ledger.pause_duration_seconds(now), + last_updated_at_unix_secs: ledger.last_updated_at_unix_secs, + deferred_cycles: ledger.deferred_cycles, + pending_full_scan: ledger.pending_full_scan, + dirty_usage_buckets: ledger.dirty_usage_buckets, + discovered_expiry_items: ledger.discovered_expiry_items, + discovered_transition_items: ledger.discovered_transition_items, + pending_work_items: ledger.pending_work_items(), + catch_up_attempts: ledger.catch_up_attempts, + consecutive_failures: ledger.consecutive_failures, + attempts_in_current_window: ledger.attempts_in_current_window, + current_window_started_at_unix_secs: ledger.current_window_started_at_unix_secs, + last_attempt_at_unix_secs: ledger.last_attempt_at_unix_secs, + next_attempt_at_unix_secs: ledger.next_attempt_at_unix_secs, + rate_limited: ledger.has_unfinished_attempt() + || matches!( + ledger.phase, + ScannerPauseBacklogPhase::CatchingUp | ScannerPauseBacklogPhase::RetryExhausted + ) && now < ledger.next_attempt_at_unix_secs, + retry_exhausted: ledger.phase == ScannerPauseBacklogPhase::RetryExhausted, + replica_count, + healthy_replicas, + stale_or_unavailable_replicas, + alerting: !alert_reasons.is_empty(), + alert_reasons, + thresholds: ScannerPauseBacklogThresholds::default(), + error, + }; + record_scanner_pause_backlog_status(&status); + status +} + +fn record_scanner_pause_backlog_status(status: &ScannerPauseBacklogStatus) { + let phase: u32 = match status.phase { + ScannerPauseBacklogPhase::Idle => 0, + ScannerPauseBacklogPhase::Paused => 1, + ScannerPauseBacklogPhase::CatchingUp => 2, + ScannerPauseBacklogPhase::RetryExhausted => 3, + }; + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_PHASE).set(f64::from(phase)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_PAUSE_DURATION_SECONDS).set(metric_u64(status.pause_duration_seconds)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_PENDING_WORK_ITEMS).set(metric_u64(status.pending_work_items)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_CONSECUTIVE_FAILURES).set(f64::from(status.consecutive_failures)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_RATE_LIMITED).set(bool_metric(status.rate_limited)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_RETRY_EXHAUSTED).set(bool_metric(status.retry_exhausted)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_ALERTING).set(bool_metric(status.alerting)); + metrics::gauge!(METRIC_SCANNER_PAUSE_BACKLOG_REPLICA_DEGRADED) + .set(bool_metric(status.durable && status.stale_or_unavailable_replicas > 0)); +} + +fn metric_u64(value: u64) -> f64 { + f64::from(u32::try_from(value).unwrap_or(u32::MAX)) +} + +fn bool_metric(value: bool) -> f64 { + if value { 1.0 } else { 0.0 } +} + +fn unix_now() -> u64 { + SystemTime::now().duration_since(UNIX_EPOCH).unwrap_or_default().as_secs() +} + +async fn read_scanner_pause_backlog_replica(store: Arc) -> ScannerPauseBacklogReplica { + let id = ScannerPauseBacklogReplicaId { + pool_index: store.pool_index, + set_index: store.set_index, + }; + let reader = match store + .get_object_reader( + RUSTFS_META_BUCKET, + SCANNER_PAUSE_BACKLOG_PATH.as_str(), + None, + HeaderMap::new(), + &ScannerObjectOptions { + no_lock: true, + ..Default::default() + }, + ) + .await + { + Ok(reader) => reader, + Err( + EcstoreError::ConfigNotFound + | EcstoreError::FileNotFound + | EcstoreError::VolumeNotFound + | EcstoreError::ObjectNotFound(_, _) + | EcstoreError::BucketNotFound(_), + ) => { + return ScannerPauseBacklogReplica { + id, + revision: Some(DataUsageCacheRevision::Missing), + state: ScannerPauseBacklogReplicaState::Missing, + }; + } + Err(err) => { + return ScannerPauseBacklogReplica { + id, + revision: None, + state: ScannerPauseBacklogReplicaState::Unavailable(err.to_string()), + }; + } + }; + + let revision = reader + .object_info + .etag + .as_ref() + .filter(|etag| !etag.is_empty()) + .cloned() + .map(DataUsageCacheRevision::Etag); + let max_size = i64::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(i64::MAX); + if revision.is_none() || reader.object_info.is_dir || reader.object_info.size < 0 || reader.object_info.size > max_size { + return ScannerPauseBacklogReplica { + id, + revision, + state: ScannerPauseBacklogReplicaState::Invalid( + "scanner pause backlog replica is oversized or has no revision".to_string(), + ), + }; + } + + let mut data = Vec::new(); + let max_len = usize::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(usize::MAX); + let read_result = reader + .take(MAX_SCANNER_PAUSE_BACKLOG_BYTES.saturating_add(1)) + .read_to_end(&mut data) + .await; + let state = match read_result { + Err(err) => ScannerPauseBacklogReplicaState::Unavailable(err.to_string()), + Ok(_) if data.len() > max_len => { + ScannerPauseBacklogReplicaState::Invalid("scanner pause backlog replica exceeds its size bound".to_string()) + } + Ok(_) => decode_scanner_pause_backlog_ledger(&data), + }; + ScannerPauseBacklogReplica { id, revision, state } +} + +fn decode_scanner_pause_backlog_ledger(data: &[u8]) -> ScannerPauseBacklogReplicaState { + let value = match serde_json::from_slice::(data) { + Ok(value) => value, + Err(err) => return ScannerPauseBacklogReplicaState::Invalid(err.to_string()), + }; + if let Some(version) = value.get("replica_schema_version").and_then(serde_json::Value::as_u64) { + if version > u64::from(SCANNER_PAUSE_BACKLOG_REPLICA_SCHEMA_VERSION) { + return ScannerPauseBacklogReplicaState::FutureSchema(version); + } + let record = match serde_json::from_value::(value) { + Ok(record) => record, + Err(err) => return ScannerPauseBacklogReplicaState::Invalid(err.to_string()), + }; + return match record.validate() { + Ok(()) => ScannerPauseBacklogReplicaState::Valid(Box::new(record)), + Err(err) => ScannerPauseBacklogReplicaState::Invalid(err), + }; + } + + let version = value.get("schema_version").and_then(serde_json::Value::as_u64).unwrap_or(0); + if version > u64::from(SCANNER_PAUSE_BACKLOG_SCHEMA_VERSION) { + return ScannerPauseBacklogReplicaState::FutureSchema(version); + } + let ledger = match serde_json::from_value::(value) { + Ok(ledger) => ledger, + Err(err) => return ScannerPauseBacklogReplicaState::Invalid(err.to_string()), + }; + match ledger.validate() { + Ok(()) => ScannerPauseBacklogReplicaState::Valid(Box::new(ScannerPauseBacklogReplicaRecord::new(Some(ledger), None))), + Err(err) => ScannerPauseBacklogReplicaState::Invalid(err), + } +} + +fn scanner_pause_backlog_consensus<'a, T: Clone + PartialEq + 'a>( + values: impl Iterator>, +) -> Result, ()> { + let mut values = values; + let first = values.next().ok_or(())?; + if values.all(|candidate| candidate == first) { + Ok(first.cloned()) + } else { + Err(()) + } +} + +fn scanner_pause_backlog_replica_ids(replicas: &[ScannerPauseBacklogReplica]) -> Vec { + let mut ids = replicas.iter().map(|replica| replica.id).collect::>(); + ids.sort_unstable(); + ids +} + +fn select_scanner_pause_backlog_commit( + replicas: &[ScannerPauseBacklogReplica], + replica_ids: &[ScannerPauseBacklogReplicaId], +) -> Result, String> { + let current_ids = replica_ids.iter().copied().collect::>(); + let replicas_by_id = replicas + .iter() + .map(|replica| (replica.id, replica)) + .collect::>(); + let mut valid = Vec::::new(); + + for committed in replicas.iter().filter_map(|replica| match &replica.state { + ScannerPauseBacklogReplicaState::Valid(record) => record.committed.as_ref(), + _ => None, + }) { + if valid.contains(committed) + || !committed.replicas.iter().all(|id| { + current_ids.contains(id) + && replicas_by_id.get(id).is_some_and(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(record) + if record.committed.as_ref() == Some(committed) + ) + }) + }) + { + continue; + } + valid.push(committed.clone()); + } + + let Some(max_membership_len) = valid.iter().map(|committed| committed.replicas.len()).max() else { + return Ok(None); + }; + let mut largest = valid + .into_iter() + .filter(|committed| committed.replicas.len() == max_membership_len); + let Some(mut selected) = largest.next() else { + return Ok(None); + }; + for committed in largest { + if committed.ledger != selected.ledger { + return Err("scanner pause backlog has conflicting maximum-membership commit proofs".to_string()); + } + if committed.replicas < selected.replicas { + selected = committed; + } + } + Ok(Some(selected)) +} + +fn select_scanner_pause_backlog_replicas(replicas: Vec) -> Result { + if replicas.is_empty() { + return Err("scanner pause backlog has no storage replicas".to_string()); + } + for replica in &replicas { + if let ScannerPauseBacklogReplicaState::FutureSchema(version) = &replica.state { + return Err(format!( + "scanner pause backlog pool {} set {} uses future schema {version}", + replica.id.pool_index, replica.id.set_index + )); + } + } + + let replica_ids = scanner_pause_backlog_replica_ids(&replicas); + let authoritative_commit = select_scanner_pause_backlog_commit(&replicas, &replica_ids)?; + let stable_consensus = if replicas.iter().all(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(_) | ScannerPauseBacklogReplicaState::Missing + ) + }) { + scanner_pause_backlog_consensus(replicas.iter().map(|replica| match &replica.state { + ScannerPauseBacklogReplicaState::Valid(record) => record.stable.as_ref(), + ScannerPauseBacklogReplicaState::Missing => None, + _ => unreachable!("replica states were checked above"), + })) + } else { + Err(()) + }; + + let selected = match &authoritative_commit { + Some(committed) => Some(committed.ledger.clone()), + None => { + if let Some(replica) = replicas.iter().find(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Invalid(_) | ScannerPauseBacklogReplicaState::Unavailable(_) + ) + }) { + let reason = match &replica.state { + ScannerPauseBacklogReplicaState::Invalid(reason) | ScannerPauseBacklogReplicaState::Unavailable(reason) => { + reason + } + _ => unreachable!("replica state was checked above"), + }; + return Err(format!( + "scanner pause backlog pool {} set {} is unavailable: {reason}", + replica.id.pool_index, replica.id.set_index + )); + } + match &stable_consensus { + Ok(stable) => stable.clone(), + Err(()) => { + return Err( + "scanner pause backlog has neither a surviving membership commit nor a stable rollback point".to_string(), + ); + } + } + } + }; + let Some(selected) = selected else { + let replica_count = replicas.len(); + return Ok(LoadedScannerPauseBacklog { + ledger: ScannerPauseBacklogLedger::default(), + durable: false, + persistence_state: "missing".to_string(), + replica_count, + healthy_replicas: 0, + stale_or_unavailable_replicas: replica_count, + stable_matches_ledger: true, + authoritative_commit: None, + requires_reload: false, + replicas, + }); + }; + + let stable_matches_ledger = matches!(&stable_consensus, Ok(Some(stable)) if stable == &selected); + let healthy_replicas = replicas + .iter() + .filter(|replica| { + matches!( + &replica.state, + ScannerPauseBacklogReplicaState::Valid(record) + if record.stable.as_ref() == Some(&selected) + && match &authoritative_commit { + Some(committed) => record.committed.as_ref() == Some(committed), + None => record.committed.is_none(), + } + ) + }) + .count(); + let replica_count = replicas.len(); + let stale_or_unavailable_replicas = replica_count.saturating_sub(healthy_replicas); + let membership_repair_pending = authoritative_commit + .as_ref() + .is_some_and(|committed| committed.replicas.as_slice() != replica_ids.as_slice()); + Ok(LoadedScannerPauseBacklog { + ledger: selected, + durable: true, + persistence_state: if membership_repair_pending { + "membership_repair_pending" + } else if authoritative_commit.is_some() && !stable_matches_ledger { + "committed_pending_stabilization" + } else if stale_or_unavailable_replicas > 0 { + "rolled_back_partial_commit" + } else { + "healthy" + } + .to_string(), + replica_count, + replicas, + healthy_replicas, + stale_or_unavailable_replicas, + stable_matches_ledger, + authoritative_commit, + requires_reload: false, + }) +} + +async fn load_scanner_pause_backlog(storeapi: Arc) -> Result { + let writable = storeapi.scanner_pause_backlog_writable_set_disks().await; + if writable.is_empty() { + return Err("scanner pause backlog has no surviving storage replicas".to_string()); + } + let replicas = join_all(writable.into_iter().map(read_scanner_pause_backlog_replica)).await; + select_scanner_pause_backlog_replicas(replicas) +} + +async fn write_scanner_pause_backlog_record( + storeapi: Arc, + loaded: &LoadedScannerPauseBacklog, + record: ScannerPauseBacklogReplicaRecord, +) -> Result<(), String> { + let data = serde_json::to_vec(&record).map_err(|err| format!("failed to encode scanner pause backlog: {err}"))?; + if data.len() > usize::try_from(MAX_SCANNER_PAUSE_BACKLOG_BYTES).unwrap_or(usize::MAX) { + return Err("scanner pause backlog exceeds its size bound".to_string()); + } + + let writable = storeapi.scanner_pause_backlog_writable_set_disks().await; + if writable.is_empty() { + return Err("scanner pause backlog has no surviving writable set".to_string()); + } + let loaded_ids = loaded.replicas.iter().map(|replica| replica.id).collect::>(); + let writable_ids = writable + .iter() + .map(|set| ScannerPauseBacklogReplicaId { + pool_index: set.pool_index, + set_index: set.set_index, + }) + .collect::>(); + if writable_ids != loaded_ids { + return Err("scanner pause backlog replica topology changed during commit".to_string()); + } + + let revisions = loaded + .replicas + .iter() + .filter_map(|replica| replica.revision.clone().map(|revision| (replica.id, revision))) + .collect::>(); + let results = join_all(writable.into_iter().map(|set| { + let id = ScannerPauseBacklogReplicaId { + pool_index: set.pool_index, + set_index: set.set_index, + }; + let revision = revisions.get(&id).cloned(); + let data = data.clone(); + async move { + let Some(revision) = revision else { + return (id, Err("replica revision is unavailable".to_string())); + }; + let result = save_config_with_preconditions(set, SCANNER_PAUSE_BACKLOG_PATH.as_str(), data, revision.preconditions()) + .await + .map(|_| ()) + .map_err(|err| err.to_string()); + (id, result) + } + })) + .await; + + let failures = results + .iter() + .filter_map(|(id, result)| { + result + .as_ref() + .err() + .map(|err| format!("pool {} set {}: {err}", id.pool_index, id.set_index)) + }) + .collect::>(); + if !failures.is_empty() { + return Err(format!( + "scanner pause backlog commit did not reach every surviving set ({})", + failures.join("; ") + )); + } + Ok(()) +} + +async fn stabilize_scanner_pause_backlog( + storeapi: Arc, + loaded: &LoadedScannerPauseBacklog, +) -> Result { + let committed = loaded.authoritative_commit.clone().unwrap_or_else(|| { + ScannerPauseBacklogCommitRecord::new(loaded.ledger.clone(), scanner_pause_backlog_replica_ids(&loaded.replicas)) + }); + let record = ScannerPauseBacklogReplicaRecord::new(Some(loaded.ledger.clone()), Some(committed)); + write_scanner_pause_backlog_record(storeapi.clone(), loaded, record).await?; + let stabilized = load_scanner_pause_backlog(storeapi).await?; + if stabilized.ledger != loaded.ledger || !stabilized.stable_matches_ledger { + return Err("scanner pause backlog failed to stabilize its last committed generation".to_string()); + } + Ok(stabilized) +} + +fn committed_scanner_pause_backlog_pending_reload( + ledger: ScannerPauseBacklogLedger, + replica_count: usize, +) -> LoadedScannerPauseBacklog { + LoadedScannerPauseBacklog { + ledger, + durable: true, + persistence_state: "committed_reload_pending".to_string(), + replicas: Vec::new(), + replica_count, + healthy_replicas: 0, + stale_or_unavailable_replicas: replica_count, + stable_matches_ledger: false, + authoritative_commit: None, + requires_reload: true, + } +} + +async fn persist_scanner_pause_backlog( + storeapi: Arc, + loaded: &LoadedScannerPauseBacklog, + ledger: ScannerPauseBacklogLedger, +) -> Result { + let mut base = if loaded.requires_reload { + let reloaded = load_scanner_pause_backlog(storeapi.clone()).await?; + if reloaded.ledger != loaded.ledger { + return Err(format!( + "scanner pause backlog reload advanced to writer epoch {} generation {}", + reloaded.ledger.writer_epoch, reloaded.ledger.generation + )); + } + reloaded + } else { + loaded.clone() + }; + if base.durable && !base.stable_matches_ledger { + base = stabilize_scanner_pause_backlog(storeapi.clone(), &base).await?; + } + + let replicas = scanner_pause_backlog_replica_ids(&base.replicas); + let committed = ScannerPauseBacklogCommitRecord::new(ledger.clone(), replicas); + let record = ScannerPauseBacklogReplicaRecord::new(base.durable.then_some(base.ledger.clone()), Some(committed)); + write_scanner_pause_backlog_record(storeapi.clone(), &base, record).await?; + + match load_scanner_pause_backlog(storeapi.clone()).await { + Ok(committed) if committed.ledger == ledger => match stabilize_scanner_pause_backlog(storeapi, &committed).await { + Ok(stabilized) => Ok(stabilized), + Err(_) => Ok(committed_scanner_pause_backlog_pending_reload(ledger, base.replica_count)), + }, + Ok(_) | Err(_) => Ok(committed_scanner_pause_backlog_pending_reload(ledger, base.replica_count)), + } +} + +pub(super) struct ScannerPauseBacklogController { + storeapi: Arc, + loaded: LoadedScannerPauseBacklog, + persistence_disabled: bool, + persistence_retry_at_unix_secs: u64, +} + +impl ScannerPauseBacklogController { + pub(super) async fn claim(storeapi: Arc, now: u64) -> Result { + let loaded = load_scanner_pause_backlog(storeapi.clone()).await?; + let mut ledger = loaded.ledger.clone(); + ledger.claim_writer(now)?; + prepare_scanner_pause_backlog_persist(&mut ledger, now)?; + let loaded = persist_scanner_pause_backlog(storeapi.clone(), &loaded, ledger).await?; + set_runtime_error(None); + let controller = Self { + storeapi, + loaded, + persistence_disabled: false, + persistence_retry_at_unix_secs: 0, + }; + controller.record_status(now); + Ok(controller) + } + + pub(super) fn unavailable(storeapi: Arc, error: String, now: u64) -> Self { + set_runtime_error(Some(error)); + let loaded = LoadedScannerPauseBacklog { + ledger: ScannerPauseBacklogLedger::default(), + durable: false, + persistence_state: "unavailable".to_string(), + replicas: Vec::new(), + replica_count: 0, + healthy_replicas: 0, + stale_or_unavailable_replicas: 0, + stable_matches_ledger: true, + authoritative_commit: None, + requires_reload: false, + }; + let controller = Self { + storeapi, + loaded, + persistence_disabled: true, + persistence_retry_at_unix_secs: now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS), + }; + controller.record_status(now); + controller + } + + pub(super) fn scheduling_delay(&self, now: u64) -> Option { + if self.persistence_disabled { + return Some(Duration::from_secs(self.persistence_retry_at_unix_secs.saturating_sub(now))); + } + match self.loaded.ledger.phase { + ScannerPauseBacklogPhase::Idle => None, + ScannerPauseBacklogPhase::Paused => Some(Duration::from_secs(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS)), + ScannerPauseBacklogPhase::CatchingUp => { + Some(Duration::from_secs(self.loaded.ledger.next_attempt_at_unix_secs.saturating_sub(now))) + } + ScannerPauseBacklogPhase::RetryExhausted => { + Some(Duration::from_secs(self.loaded.ledger.next_attempt_at_unix_secs.saturating_sub(now))) + } + } + } + + pub(super) async fn observe(&mut self, observation: ScannerPauseBacklogObservation) { + if !self.try_recover_persistence(observation.now_unix_secs).await { + return; + } + self.persist_mutation(observation.now_unix_secs, |ledger| ledger.apply_observation(observation)) + .await; + } + + pub(super) async fn begin_attempt(&mut self, now: u64) -> ScannerPauseBacklogAttemptDecision { + if self.persistence_disabled { + return ScannerPauseBacklogAttemptDecision::PersistenceUnavailable; + } + let mut candidate = self.loaded.ledger.clone(); + let decision = candidate.begin_attempt(now); + if candidate == self.loaded.ledger { + self.record_status(now); + return decision; + } + if let Err(err) = prepare_scanner_pause_backlog_persist(&mut candidate, now) { + self.disable_persistence(err, now); + return ScannerPauseBacklogAttemptDecision::PersistenceUnavailable; + } + match persist_scanner_pause_backlog(self.storeapi.clone(), &self.loaded, candidate).await { + Ok(loaded) => { + self.loaded = loaded; + set_runtime_error(None); + self.record_status(now); + decision + } + Err(err) => { + self.disable_persistence(err, now); + ScannerPauseBacklogAttemptDecision::PersistenceUnavailable + } + } + } + + pub(super) async fn finish_attempt( + &mut self, + serial: u64, + outcome: ScannerCycleOutcome, + observation: ScannerPauseBacklogObservation, + ) { + self.persist_mutation(observation.now_unix_secs, |ledger| { + ledger.finish_attempt(serial, outcome.into(), observation) + }) + .await; + } + + pub(super) async fn observe_cycle_outcome( + &mut self, + outcome: ScannerCycleOutcome, + observation: ScannerPauseBacklogObservation, + ) { + self.persist_mutation(observation.now_unix_secs, |ledger| { + ledger.observe_cycle_outcome(outcome.into(), observation) + }) + .await; + } + + async fn persist_mutation(&mut self, now: u64, mutate: impl FnOnce(&mut ScannerPauseBacklogLedger)) { + if self.persistence_disabled { + return; + } + let mut candidate = self.loaded.ledger.clone(); + mutate(&mut candidate); + if candidate == self.loaded.ledger { + self.record_status(now); + return; + } + if let Err(err) = prepare_scanner_pause_backlog_persist(&mut candidate, now) { + self.disable_persistence(err, now); + return; + } + match persist_scanner_pause_backlog(self.storeapi.clone(), &self.loaded, candidate).await { + Ok(loaded) => { + self.loaded = loaded; + set_runtime_error(None); + self.record_status(now); + } + Err(err) => self.disable_persistence(err, now), + } + } + + fn disable_persistence(&mut self, error: String, now: u64) { + self.persistence_disabled = true; + self.persistence_retry_at_unix_secs = now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + set_runtime_error(Some(error)); + self.record_status(now); + } + + async fn try_recover_persistence(&mut self, now: u64) -> bool { + if !self.persistence_disabled { + return true; + } + if now < self.persistence_retry_at_unix_secs { + self.record_status(now); + return false; + } + match Self::claim(self.storeapi.clone(), now).await { + Ok(controller) => { + *self = controller; + true + } + Err(error) => { + self.persistence_retry_at_unix_secs = now.saturating_add(SCANNER_PAUSE_REFRESH_INTERVAL_SECONDS); + set_runtime_error(Some(error)); + self.record_status(now); + false + } + } + } + + fn record_status(&self, now: u64) { + let _ = self.loaded.status(now, runtime_error()); + } +} + +fn prepare_scanner_pause_backlog_persist(ledger: &mut ScannerPauseBacklogLedger, now: u64) -> Result<(), String> { + ledger.generation = ledger + .generation + .checked_add(1) + .ok_or_else(|| "scanner pause backlog generation is exhausted".to_string())?; + ledger.last_updated_at_unix_secs = now; + ledger.validate() +} + +fn set_runtime_error(error: Option) { + *SCANNER_PAUSE_BACKLOG_RUNTIME_ERROR + .write() + .unwrap_or_else(|poisoned| poisoned.into_inner()) = error; +} + +fn runtime_error() -> Option { + SCANNER_PAUSE_BACKLOG_RUNTIME_ERROR + .read() + .unwrap_or_else(|poisoned| poisoned.into_inner()) + .clone() +} + +pub async fn scanner_pause_backlog_status(storeapi: Arc) -> ScannerPauseBacklogStatus { + let now = unix_now(); + match load_scanner_pause_backlog(storeapi).await { + Ok(loaded) => loaded.status(now, runtime_error()), + Err(error) => { + let error = runtime_error().map_or(error.clone(), |runtime| format!("{error}; {runtime}")); + status_from_ledger( + &ScannerPauseBacklogLedger::default(), + now, + "unavailable".to_string(), + false, + 0, + 0, + 0, + false, + true, + Some(error), + ) + } + } +} + +pub(super) fn scanner_pause_backlog_now() -> u64 { + unix_now() +} + +#[cfg(test)] +mod tests { + use super::*; + + fn observation(now: u64, paused: bool, pending: u64) -> ScannerPauseBacklogObservation { + ScannerPauseBacklogObservation { + now_unix_secs: now, + paused, + movement_generation: 7, + movement_work_items: if paused { 1 } else { 0 }, + pause_started_at_unix_secs: if paused { now } else { 0 }, + dirty_usage_buckets: pending, + discovered_expiry_items: 0, + discovered_transition_items: 0, + } + } + + fn durable_ledger(now: u64) -> ScannerPauseBacklogLedger { + let mut ledger = ScannerPauseBacklogLedger::default(); + ledger.claim_writer(now).expect("writer epoch should be available"); + prepare_scanner_pause_backlog_persist(&mut ledger, now).expect("ledger should become durable"); + ledger + } + + fn decode_valid_ledger(ledger: &ScannerPauseBacklogLedger) -> ScannerPauseBacklogLedger { + let encoded = serde_json::to_vec(ledger).expect("ledger should encode"); + let ScannerPauseBacklogReplicaState::Valid(decoded) = decode_scanner_pause_backlog_ledger(&encoded) else { + panic!("ledger should decode"); + }; + decoded.stable.expect("legacy ledger should become a stable replica") + } + + fn retry_exhausted_ledger() -> ScannerPauseBacklogLedger { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + for now in [120, 420, 720, 1020, 3720] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("failure attempt at {now} should be admitted"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::RetryableFailure, observation(now + 1, false, 0)); + } + ledger + } + + fn replica_id(pool_index: usize, set_index: usize) -> ScannerPauseBacklogReplicaId { + ScannerPauseBacklogReplicaId { pool_index, set_index } + } + + fn decoded_replica( + id: ScannerPauseBacklogReplicaId, + record: &ScannerPauseBacklogReplicaRecord, + ) -> ScannerPauseBacklogReplica { + let encoded = serde_json::to_vec(record).expect("replica should encode"); + ScannerPauseBacklogReplica { + id, + revision: Some(DataUsageCacheRevision::Etag(format!("revision-{}-{}", id.pool_index, id.set_index))), + state: decode_scanner_pause_backlog_ledger(&encoded), + } + } + + fn crash_reload_replicas(replicas: Vec) -> LoadedScannerPauseBacklog { + select_scanner_pause_backlog_replicas(replicas).expect("replicas should have an authoritative rollback point") + } + + fn crash_reload(records: &[ScannerPauseBacklogReplicaRecord]) -> LoadedScannerPauseBacklog { + let replicas = records + .iter() + .enumerate() + .map(|(set_index, record)| decoded_replica(replica_id(0, set_index), record)) + .collect(); + crash_reload_replicas(replicas) + } + + fn replica_record_for_members( + stable: &ScannerPauseBacklogLedger, + committed: &ScannerPauseBacklogLedger, + replicas: &[ScannerPauseBacklogReplicaId], + ) -> ScannerPauseBacklogReplicaRecord { + ScannerPauseBacklogReplicaRecord::new( + Some(stable.clone()), + Some(ScannerPauseBacklogCommitRecord::new(committed.clone(), replicas.to_vec())), + ) + } + + fn replica_record( + stable: &ScannerPauseBacklogLedger, + committed: &ScannerPauseBacklogLedger, + ) -> ScannerPauseBacklogReplicaRecord { + let replicas = (0..3) + .map(|set_index| ScannerPauseBacklogReplicaId { + pool_index: 0, + set_index, + }) + .collect::>(); + replica_record_for_members(stable, committed, &replicas) + } + + #[derive(Clone, Copy)] + enum RejoinedSourceState { + Missing, + OlderCommit, + NewerSmallCommit, + NewerUnprovenCommit, + StaleStable, + } + + fn assert_rejoined_source_is_safely_seeded(source_state: RejoinedSourceState) { + let source_id = replica_id(0, 0); + let target_ids = [replica_id(1, 0), replica_id(1, 1)]; + let full_ids = [source_id, target_ids[0], target_ids[1]]; + let old = durable_ledger(50); + let mut target = old.clone(); + target.claim_writer(100).expect("target membership epoch should advance"); + prepare_scanner_pause_backlog_persist(&mut target, 100).expect("target membership should persist"); + let target_record = replica_record_for_members(&target, &target, &target_ids); + let source = match source_state { + RejoinedSourceState::Missing => ScannerPauseBacklogReplica { + id: source_id, + revision: Some(DataUsageCacheRevision::Missing), + state: ScannerPauseBacklogReplicaState::Missing, + }, + RejoinedSourceState::OlderCommit => { + let stale_record = replica_record_for_members(&old, &old, &[source_id]); + decoded_replica(source_id, &stale_record) + } + RejoinedSourceState::NewerSmallCommit => { + let mut stale = target.clone(); + stale.claim_writer(110).expect("stale source epoch should advance"); + prepare_scanner_pause_backlog_persist(&mut stale, 110).expect("stale source should persist"); + stale.claim_writer(120).expect("stale source epoch should advance again"); + prepare_scanner_pause_backlog_persist(&mut stale, 120).expect("stale source should persist again"); + assert!(stale.writer_epoch > target.writer_epoch); + let stale_record = replica_record_for_members(&stale, &stale, &[source_id]); + decoded_replica(source_id, &stale_record) + } + RejoinedSourceState::NewerUnprovenCommit => { + let mut stale = target.clone(); + stale.claim_writer(110).expect("stale source epoch should advance"); + prepare_scanner_pause_backlog_persist(&mut stale, 110).expect("stale source should persist"); + stale.claim_writer(120).expect("stale source epoch should advance again"); + prepare_scanner_pause_backlog_persist(&mut stale, 120).expect("stale source should persist again"); + assert!(stale.writer_epoch > target.writer_epoch); + let stale_record = replica_record_for_members(&stale, &stale, &full_ids); + decoded_replica(source_id, &stale_record) + } + RejoinedSourceState::StaleStable => { + let stale_record = ScannerPauseBacklogReplicaRecord::new(Some(old), None); + decoded_replica(source_id, &stale_record) + } + }; + + let active = crash_reload_replicas(target_ids.iter().map(|id| decoded_replica(*id, &target_record)).collect()); + assert_eq!(active.ledger, target); + assert_eq!(active.persistence_state, "healthy"); + + let rejoined = crash_reload_replicas(vec![ + source, + decoded_replica(target_ids[0], &target_record), + decoded_replica(target_ids[1], &target_record), + ]); + assert_eq!(rejoined.ledger, target); + assert_eq!(rejoined.persistence_state, "membership_repair_pending"); + assert_eq!(rejoined.healthy_replicas, target_ids.len()); + assert_eq!(rejoined.stale_or_unavailable_replicas, 1); + assert_eq!( + rejoined + .authoritative_commit + .as_ref() + .expect("surviving target proof should remain authoritative") + .replicas + .as_slice(), + target_ids.as_slice() + ); + + let seeded_record = replica_record_for_members(&target, &target, &target_ids); + let seeded = crash_reload_replicas(full_ids.iter().map(|id| decoded_replica(*id, &seeded_record)).collect()); + assert_eq!(seeded.ledger, target); + assert!(seeded.stable_matches_ledger); + assert_eq!(seeded.persistence_state, "membership_repair_pending"); + + let mut replacement = seeded.ledger; + replacement + .claim_writer(200) + .expect("replacement node should claim the surviving ledger"); + prepare_scanner_pause_backlog_persist(&mut replacement, 200).expect("replacement node should persist"); + assert_eq!(replacement.writer_epoch, target.writer_epoch + 1); + let full_commit = replica_record_for_members(&target, &replacement, &full_ids); + + let source_first = crash_reload_replicas(vec![ + decoded_replica(source_id, &full_commit), + decoded_replica(target_ids[0], &seeded_record), + decoded_replica(target_ids[1], &seeded_record), + ]); + assert_eq!(source_first.ledger, target); + assert_eq!(source_first.persistence_state, "membership_repair_pending"); + + let target_first = crash_reload_replicas(vec![ + decoded_replica(source_id, &seeded_record), + decoded_replica(target_ids[0], &full_commit), + decoded_replica(target_ids[1], &seeded_record), + ]); + assert_eq!(target_first.ledger, target); + assert_eq!(target_first.persistence_state, "rolled_back_partial_commit"); + + let committed = crash_reload_replicas(full_ids.iter().map(|id| decoded_replica(*id, &full_commit)).collect()); + assert_eq!(committed.ledger, replacement); + assert_eq!(committed.persistence_state, "committed_pending_stabilization"); + + let stable_full_commit = replica_record_for_members(&replacement, &replacement, &full_ids); + let stable = crash_reload_replicas(full_ids.iter().map(|id| decoded_replica(*id, &stable_full_commit)).collect()); + assert_eq!(stable.ledger, replacement); + assert_eq!(stable.persistence_state, "healthy"); + assert_eq!(stable.healthy_replicas, full_ids.len()); + } + + #[test] + fn restart_recovers_paused_backlog_and_requires_one_full_catch_up_scan() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 3)); + prepare_scanner_pause_backlog_persist(&mut ledger, 110).expect("paused ledger should persist"); + let mut restarted = decode_valid_ledger(&ledger); + + restarted.claim_writer(120).expect("new process should claim a writer epoch"); + restarted.apply_observation(observation(120, false, 3)); + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(restarted.pending_full_scan); + assert_eq!(restarted.next_attempt_at_unix_secs, 120); + } + + #[test] + fn resume_after_clock_rollback_keeps_pause_timestamps_monotonic() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(ScannerPauseBacklogObservation { + now_unix_secs: 200, + pause_started_at_unix_secs: 200, + paused: true, + ..observation(200, true, 0) + }); + + ledger.apply_observation(observation(150, false, 0)); + + assert_eq!(ledger.pause_started_at_unix_secs, 200); + assert_eq!(ledger.pause_ended_at_unix_secs, 200); + prepare_scanner_pause_backlog_persist(&mut ledger, 150).expect("clock rollback should remain persistable"); + } + + #[test] + fn movement_that_starts_and_ends_inside_a_cycle_still_creates_catch_up_work() { + let mut ledger = durable_ledger(100); + ledger.observe_cycle_outcome( + ScannerPauseBacklogCycleOutcome::DataMovementDeferred, + ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(120, false, 0) + }, + ); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.pause_started_at_unix_secs, 120); + assert_eq!(ledger.pause_ended_at_unix_secs, 120); + } + + #[test] + fn restart_recovers_movement_that_completed_before_pause_was_persisted() { + let ledger = durable_ledger(100); + let mut restarted = decode_valid_ledger(&ledger); + + restarted.apply_observation(ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(120, false, 0) + }); + + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(restarted.pending_full_scan); + assert_eq!(restarted.pause_started_at_unix_secs, 120); + assert_eq!(restarted.pause_ended_at_unix_secs, 120); + } + + #[test] + fn generation_advance_reopens_full_scan_while_catch_up_has_only_known_work() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::Completed, observation(130, false, 1)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + + ledger.apply_observation(ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(140, false, 0) + }); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.movement_generation, 8); + assert_eq!(ledger.pause_started_at_unix_secs, 140); + assert_eq!(ledger.pause_ended_at_unix_secs, 140); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + } + + #[test] + fn completed_attempt_observing_generation_advance_keeps_full_scan_debt() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + + ledger.finish_attempt( + serial, + ScannerPauseBacklogCycleOutcome::Completed, + ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(130, false, 0) + }, + ); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.movement_generation, 8); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + prepare_scanner_pause_backlog_persist(&mut ledger, 130).expect("generation advance should remain persistable"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(restarted.pending_full_scan); + assert_eq!(restarted.movement_generation, 8); + } + + #[test] + fn generation_advance_preserves_attempt_rate_fence_for_deferred_and_progressed_cycles() { + for outcome in [ + ScannerPauseBacklogCycleOutcome::DataMovementDeferred, + ScannerPauseBacklogCycleOutcome::Progressed, + ] { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + assert_eq!(ledger.next_attempt_at_unix_secs, 420); + + ledger.finish_attempt( + serial, + outcome, + ScannerPauseBacklogObservation { + movement_generation: 8, + ..observation(130, false, 0) + }, + ); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + assert_eq!(ledger.current_window_started_at_unix_secs, 120); + assert_eq!(ledger.attempts_in_current_window, 1); + assert_eq!(ledger.last_attempt_at_unix_secs, 120); + prepare_scanner_pause_backlog_persist(&mut ledger, 130).expect("rate fence should remain persistable"); + let mut restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.next_attempt_at_unix_secs, 430); + assert_eq!(restarted.current_window_started_at_unix_secs, 120); + assert_eq!(restarted.attempts_in_current_window, 1); + assert_eq!(restarted.last_attempt_at_unix_secs, 120); + assert_eq!(restarted.begin_attempt(429), ScannerPauseBacklogAttemptDecision::RateLimited); + assert!(matches!(restarted.begin_attempt(430), ScannerPauseBacklogAttemptDecision::Tracked(_))); + } + } + + #[test] + fn remote_deferred_cycle_without_generation_change_keeps_durable_rate_state() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + assert_eq!(ledger.next_attempt_at_unix_secs, 420); + + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::DataMovementDeferred, observation(130, false, 0)); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.movement_generation, 7); + assert_eq!(ledger.current_window_started_at_unix_secs, 120); + assert_eq!(ledger.attempts_in_current_window, 1); + assert_eq!(ledger.last_attempt_at_unix_secs, 120); + assert_eq!(ledger.next_attempt_at_unix_secs, 430); + prepare_scanner_pause_backlog_persist(&mut ledger, 130).expect("remote defer fence should persist"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.current_window_started_at_unix_secs, 120); + assert_eq!(restarted.attempts_in_current_window, 1); + assert_eq!(restarted.last_attempt_at_unix_secs, 120); + assert_eq!(restarted.next_attempt_at_unix_secs, 430); + } + + #[test] + fn repeated_short_movements_cannot_reset_the_four_attempt_window() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(first) = ledger.begin_attempt(120) else { + panic!("first catch-up attempt should begin"); + }; + + for (serial, attempt_at, finished_at, movement_generation) in + [(first, 120, 130, 8), (0, 430, 440, 9), (0, 740, 750, 10), (0, 1050, 1060, 11)] + { + let serial = if serial == 0 { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(attempt_at) else { + panic!("short-movement attempt at {attempt_at} should be admitted"); + }; + serial + } else { + serial + }; + ledger.finish_attempt( + serial, + ScannerPauseBacklogCycleOutcome::Progressed, + ScannerPauseBacklogObservation { + movement_generation, + ..observation(finished_at, false, 0) + }, + ); + } + + assert_eq!(ledger.current_window_started_at_unix_secs, 120); + assert_eq!(ledger.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(ledger.last_attempt_at_unix_secs, 1050); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + assert_eq!(ledger.begin_attempt(1360), ScannerPauseBacklogAttemptDecision::RateLimited); + prepare_scanner_pause_backlog_persist(&mut ledger, 1360).expect("movement rate window should persist"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.current_window_started_at_unix_secs, 120); + assert_eq!(restarted.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(restarted.next_attempt_at_unix_secs, 3720); + } + + #[test] + fn node_offline_for_the_whole_movement_epoch_detects_generation_advance() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(ScannerPauseBacklogObservation { + movement_generation: 9, + ..observation(120, false, 0) + }); + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + } + + #[test] + fn node_switch_selects_only_an_all_replica_writer_epoch() { + let old = durable_ledger(100); + let mut current = old.clone(); + current.claim_writer(110).expect("new node should claim a higher epoch"); + prepare_scanner_pause_backlog_persist(&mut current, 110).expect("new node claim should persist"); + let committed = crash_reload(&[ + replica_record(&old, ¤t), + replica_record(&old, ¤t), + replica_record(&old, ¤t), + ]); + assert_eq!(committed.ledger, current); + + let mut divergent = current.clone(); + divergent.deferred_cycles = 9; + let rolled_back = crash_reload(&[ + replica_record(&old, ¤t), + replica_record(&old, &divergent), + replica_record(&old, &old), + ]); + assert_eq!(rolled_back.ledger, old); + assert_eq!(rolled_back.persistence_state, "rolled_back_partial_commit"); + } + + #[test] + fn active_to_failed_rejoins_and_seeds_a_missing_source() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::Missing); + } + + #[test] + fn active_to_canceled_rejoins_from_an_older_source_commit() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::OlderCommit); + } + + #[test] + fn cleared_decommission_rejoins_and_repairs_a_stale_source_after_restart() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::StaleStable); + } + + #[test] + fn rejoined_source_cannot_reverse_overwrite_a_surviving_commit() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::NewerUnprovenCommit); + } + + #[test] + fn one_replica_high_epoch_commit_cannot_override_a_larger_surviving_commit() { + assert_rejoined_source_is_safely_seeded(RejoinedSourceState::NewerSmallCommit); + } + + #[test] + fn partial_begin_attempt_commit_is_not_authoritative_after_crash() { + let mut stable = durable_ledger(100); + stable.apply_observation(observation(110, true, 0)); + stable.apply_observation(observation(120, false, 0)); + prepare_scanner_pause_backlog_persist(&mut stable, 120).expect("catch-up state should persist"); + + let mut begun = stable.clone(); + assert!(matches!(begun.begin_attempt(120), ScannerPauseBacklogAttemptDecision::Tracked(_))); + prepare_scanner_pause_backlog_persist(&mut begun, 120).expect("begun attempt should persist"); + let partial = crash_reload(&[ + replica_record(&stable, &begun), + replica_record(&stable, &stable), + replica_record(&stable, &stable), + ]); + + assert_eq!(partial.ledger, stable); + assert!(!partial.ledger.has_unfinished_attempt()); + let failed_member_absent = crash_reload(&[replica_record(&stable, &begun), replica_record(&stable, &begun)]); + assert_eq!(failed_member_absent.ledger, stable); + assert!(!failed_member_absent.ledger.has_unfinished_attempt()); + let committed = crash_reload(&[ + replica_record(&stable, &begun), + replica_record(&stable, &begun), + replica_record(&stable, &begun), + ]); + assert_eq!(committed.ledger, begun); + assert!(committed.ledger.has_unfinished_attempt()); + } + + #[test] + fn partial_idle_finish_is_rolled_back_and_interrupted_attempt_fails_closed() { + let mut stable = durable_ledger(100); + stable.apply_observation(observation(110, true, 0)); + stable.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = stable.begin_attempt(120) else { + panic!("catch-up attempt should begin"); + }; + prepare_scanner_pause_backlog_persist(&mut stable, 120).expect("begun attempt should persist"); + + let mut finished = stable.clone(); + finished.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::Completed, observation(130, false, 0)); + prepare_scanner_pause_backlog_persist(&mut finished, 130).expect("finished attempt should persist"); + assert_eq!(finished.phase, ScannerPauseBacklogPhase::Idle); + + let partial = crash_reload(&[ + replica_record(&stable, &finished), + replica_record(&stable, &stable), + replica_record(&stable, &stable), + ]); + assert_eq!(partial.ledger, stable); + assert_ne!(partial.ledger.phase, ScannerPauseBacklogPhase::Idle); + assert!(partial.ledger.has_unfinished_attempt()); + let mut replacement = partial.ledger; + replacement + .claim_writer(140) + .expect("replacement node should claim the interrupted attempt"); + assert_eq!(replacement.consecutive_failures, 1); + assert_ne!(replacement.phase, ScannerPauseBacklogPhase::Idle); + + let failed_member_absent = crash_reload(&[replica_record(&stable, &finished), replica_record(&stable, &finished)]); + assert_ne!(failed_member_absent.ledger.phase, ScannerPauseBacklogPhase::Idle); + assert!(failed_member_absent.ledger.has_unfinished_attempt()); + + let committed = crash_reload(&[ + replica_record(&stable, &finished), + replica_record(&stable, &finished), + replica_record(&stable, &finished), + ]); + assert_eq!(committed.ledger.phase, ScannerPauseBacklogPhase::Idle); + } + + #[test] + fn backlog_converges_only_after_full_scan_and_known_ilm_work_clear() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(first) = ledger.begin_attempt(120) else { + panic!("first catch-up attempt should be admitted"); + }; + let mut pending = observation(130, false, 0); + pending.discovered_expiry_items = 2; + ledger.finish_attempt(first, ScannerPauseBacklogCycleOutcome::Completed, pending); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + + ledger.apply_observation(observation(420, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(ledger.pending_work_items(), 0); + prepare_scanner_pause_backlog_persist(&mut ledger, 430).expect("converged ledger should persist"); + assert_eq!(decode_valid_ledger(&ledger).phase, ScannerPauseBacklogPhase::Idle); + } + + #[test] + fn fourth_completed_known_work_attempt_preserves_window_end_before_convergence() { + let mut ledger = durable_ledger(100); + ledger.movement_generation = 7; + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + let ScannerPauseBacklogAttemptDecision::Tracked(full_scan) = ledger.begin_attempt(120) else { + panic!("required full scan should begin"); + }; + let known_work = ScannerPauseBacklogObservation { + dirty_usage_buckets: 1, + discovered_expiry_items: 2, + discovered_transition_items: 3, + ..observation(130, false, 0) + }; + ledger.finish_attempt(full_scan, ScannerPauseBacklogCycleOutcome::Completed, known_work); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 6); + assert_eq!(ledger.begin_attempt(429), ScannerPauseBacklogAttemptDecision::RateLimited); + + for (attempt_at, finished_at, outcome) in [ + (430, 431, ScannerPauseBacklogCycleOutcome::Progressed), + (730, 731, ScannerPauseBacklogCycleOutcome::Progressed), + (1030, 1031, ScannerPauseBacklogCycleOutcome::Completed), + ] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(attempt_at) else { + panic!("known-work attempt at {attempt_at} should be admitted"); + }; + ledger.finish_attempt( + serial, + outcome, + ScannerPauseBacklogObservation { + now_unix_secs: finished_at, + ..known_work + }, + ); + } + assert_eq!(ledger.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(ledger.pending_work_items(), 6); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + assert_eq!(ledger.begin_attempt(1320), ScannerPauseBacklogAttemptDecision::RateLimited); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + + let ScannerPauseBacklogAttemptDecision::Tracked(final_attempt) = ledger.begin_attempt(3720) else { + panic!("next bounded window should admit known work"); + }; + ledger.finish_attempt(final_attempt, ScannerPauseBacklogCycleOutcome::Completed, observation(3721, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(ledger.pending_work_items(), 0); + prepare_scanner_pause_backlog_persist(&mut ledger, 3721).expect("converged known work should persist"); + assert_eq!(decode_valid_ledger(&ledger).phase, ScannerPauseBacklogPhase::Idle); + } + + #[test] + fn catch_up_rate_window_survives_restart() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + for now in [120, 420, 720, 1020] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("attempt at {now} should be admitted"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::RetryableFailure, observation(now + 1, false, 0)); + } + prepare_scanner_pause_backlog_persist(&mut ledger, 1021).expect("rate window should persist"); + let mut restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.begin_attempt(1320), ScannerPauseBacklogAttemptDecision::RateLimited); + assert_eq!(restarted.next_attempt_at_unix_secs, 3720); + } + + #[test] + fn bounded_partial_catch_up_progress_does_not_exhaust_the_retry_budget() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + for now in [120, 420, 720, 1020, 3720] { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("partial attempt at {now} should be admitted"); + }; + ledger.finish_attempt(serial, ScannerPauseBacklogCycleOutcome::Progressed, observation(now + 1, false, 0)); + } + + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.consecutive_failures, 0); + assert_eq!(ledger.next_attempt_at_unix_secs, 4020); + } + + #[test] + fn completed_usage_with_pending_maintenance_stays_durable_until_convergence() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + + let ScannerPauseBacklogAttemptDecision::Tracked(full_scan) = ledger.begin_attempt(120) else { + panic!("full catch-up scan should be admitted"); + }; + ledger.finish_attempt(full_scan, ScannerCycleOutcome::Completed.into(), observation(121, false, 1)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(!ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 1); + assert_eq!(ledger.next_attempt_at_unix_secs, 421); + + for (index, now) in [421, 721, 1021].into_iter().enumerate() { + let ScannerPauseBacklogAttemptDecision::Tracked(serial) = ledger.begin_attempt(now) else { + panic!("pending-maintenance attempt at {now} should be admitted"); + }; + ledger.finish_attempt( + serial, + ScannerCycleOutcome::CompletedWithPendingMaintenance.into(), + observation(now + 1, false, 0), + ); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 1); + if index == 0 { + assert_eq!(ledger.next_attempt_at_unix_secs, 721); + prepare_scanner_pause_backlog_persist(&mut ledger, 422).expect("pending maintenance should remain durable"); + ledger = decode_valid_ledger(&ledger); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(ledger.pending_full_scan); + assert_eq!(ledger.next_attempt_at_unix_secs, 721); + let status = status_from_ledger(&ledger, 422, "healthy".to_string(), true, 1, 1, 0, false, false, None); + assert_eq!(status.phase, ScannerPauseBacklogPhase::CatchingUp); + assert!(status.pending_full_scan); + assert_eq!(status.pending_work_items, 1); + assert!(status.rate_limited); + } + } + + assert_eq!(ledger.attempts_in_current_window, SCANNER_CATCH_UP_MAX_ATTEMPTS_PER_WINDOW); + assert_eq!(ledger.next_attempt_at_unix_secs, 3720); + assert_eq!(ledger.begin_attempt(1320), ScannerPauseBacklogAttemptDecision::RateLimited); + + let ScannerPauseBacklogAttemptDecision::Tracked(final_attempt) = ledger.begin_attempt(3720) else { + panic!("converged maintenance attempt should be admitted in the next window"); + }; + ledger.finish_attempt(final_attempt, ScannerCycleOutcome::Completed.into(), observation(3721, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::Idle); + assert!(!ledger.pending_full_scan); + assert_eq!(ledger.pending_work_items(), 0); + } + + #[test] + fn retry_budget_exhaustion_uses_sparse_probe_and_alerts() { + let mut ledger = retry_exhausted_ledger(); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(ledger.next_attempt_at_unix_secs, 7321); + assert_eq!(ledger.begin_attempt(7000), ScannerPauseBacklogAttemptDecision::RateLimited); + assert!( + ledger + .alert_reasons(7000, false, false) + .contains(&ScannerPauseBacklogAlertReason::RetryBudgetExhausted) + ); + + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = ledger.begin_attempt(7321) else { + panic!("hourly recovery probe should be admitted"); + }; + ledger.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Progressed, observation(7322, false, 0)); + assert_eq!(ledger.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(ledger.consecutive_failures, 0); + assert_eq!(ledger.next_attempt_at_unix_secs, 7622); + prepare_scanner_pause_backlog_persist(&mut ledger, 7322).expect("recovered retry cadence should persist"); + let restarted = decode_valid_ledger(&ledger); + assert_eq!(restarted.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(restarted.current_window_started_at_unix_secs, 7322); + assert_eq!(restarted.attempts_in_current_window, 0); + assert_eq!(restarted.next_attempt_at_unix_secs, 7622); + } + + #[test] + fn retry_exhausted_probe_releases_hourly_floor_only_after_success() { + let exhausted = retry_exhausted_ledger(); + + let mut completed_with_work = exhausted.clone(); + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = completed_with_work.begin_attempt(7321) else { + panic!("completed recovery probe should be admitted"); + }; + completed_with_work.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Completed, observation(7322, false, 1)); + assert_eq!(completed_with_work.phase, ScannerPauseBacklogPhase::CatchingUp); + assert_eq!(completed_with_work.current_window_started_at_unix_secs, 7322); + assert_eq!(completed_with_work.attempts_in_current_window, 0); + assert_eq!(completed_with_work.next_attempt_at_unix_secs, 7622); + + let mut completed = exhausted.clone(); + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = completed.begin_attempt(7321) else { + panic!("final recovery probe should be admitted"); + }; + completed.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Completed, observation(7322, false, 0)); + assert_eq!(completed.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(completed.next_attempt_at_unix_secs, 0); + + let mut deferred = exhausted.clone(); + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = deferred.begin_attempt(7321) else { + panic!("deferred probe should be admitted"); + }; + deferred.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::DataMovementDeferred, observation(7322, false, 0)); + assert_eq!(deferred.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(deferred.next_attempt_at_unix_secs, 10921); + + let mut failed = exhausted; + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = failed.begin_attempt(7321) else { + panic!("failed probe should be admitted"); + }; + failed.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::RetryableFailure, observation(7322, false, 0)); + assert_eq!(failed.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(failed.next_attempt_at_unix_secs, 10922); + + let mut counter_exhausted = retry_exhausted_ledger(); + counter_exhausted.counter_exhausted = true; + let ScannerPauseBacklogAttemptDecision::Tracked(probe) = counter_exhausted.begin_attempt(7321) else { + panic!("counter-exhausted probe should be admitted"); + }; + counter_exhausted.finish_attempt(probe, ScannerPauseBacklogCycleOutcome::Progressed, observation(7322, false, 0)); + assert_eq!(counter_exhausted.phase, ScannerPauseBacklogPhase::RetryExhausted); + assert_eq!(counter_exhausted.next_attempt_at_unix_secs, 10921); + } + + #[test] + fn node_switch_counts_an_interrupted_attempt_and_keeps_the_rate_fence() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(observation(110, true, 0)); + ledger.apply_observation(observation(120, false, 0)); + assert!(matches!(ledger.begin_attempt(120), ScannerPauseBacklogAttemptDecision::Tracked(_))); + ledger.claim_writer(130).expect("replacement node should claim the ledger"); + + assert_eq!(ledger.current_attempt_serial, ledger.last_finished_attempt_serial); + assert_eq!(ledger.consecutive_failures, 1); + assert_eq!(ledger.begin_attempt(130), ScannerPauseBacklogAttemptDecision::RateLimited); + } + + #[test] + fn pause_and_backlog_thresholds_are_visible() { + let mut ledger = durable_ledger(100); + ledger.apply_observation(ScannerPauseBacklogObservation { + now_unix_secs: 110, + paused: true, + movement_generation: 7, + movement_work_items: 1, + pause_started_at_unix_secs: 100, + dirty_usage_buckets: SCANNER_PAUSE_BACKLOG_ITEMS_ALERT, + discovered_expiry_items: 0, + discovered_transition_items: 0, + }); + ledger.deferred_cycles = SCANNER_PAUSE_DEFERRED_CYCLES_ALERT; + let alerts = ledger.alert_reasons(100 + SCANNER_PAUSE_DURATION_ALERT_SECONDS, true, false); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::PauseDurationThreshold)); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::DeferredCyclesThreshold)); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::BacklogItemsThreshold)); + assert!(alerts.contains(&ScannerPauseBacklogAlertReason::ReplicaDegraded)); + } +} diff --git a/crates/scanner/src/scanner/cycle_state.rs b/crates/scanner/src/scanner/cycle_state.rs index 95a42e37f..f8a11c6f3 100644 --- a/crates/scanner/src/scanner/cycle_state.rs +++ b/crates/scanner/src/scanner/cycle_state.rs @@ -14,7 +14,7 @@ /// Scanner cycle-state codec, persisted usage floors, and cycle-state persistence. use super::*; use crate::ScannerGetObjectReader; -use crate::data_usage_define::DATA_USAGE_BLOOM_RECOVERY_PATH; +use crate::data_usage_define::{DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_RECOVERY_PATH}; use crate::storage_api::owner::ObjectIO as _; use tokio::io::AsyncReadExt as _; @@ -23,6 +23,9 @@ const MAX_SCANNER_CYCLE_STATE_BYTES: u64 = 1024 * 1024; pub(super) const MAX_SCANNER_CYCLE_RECOVERY_RETRIES: u32 = 5; const METRIC_SCANNER_CYCLE_RECOVERY_REQUIRED: &str = "rustfs_scanner_cycle_recovery_required"; const METRIC_SCANNER_CYCLE_RECOVERY_RETRY_COUNT: &str = "rustfs_scanner_cycle_recovery_retry_count"; +const USAGE_FLOOR_LOAD_FAILED: &str = "usage_floor_load_failed"; +const LEGACY_EMPTY_USAGE_FLOOR_RECOVERY: &str = "legacy_empty_usage_floor"; +const CACHE_CYCLE_AHEAD: &str = "cache_cycle_ahead"; #[derive(Clone, Debug, Default, Serialize)] pub struct ScannerCycleRecoveryStatus { @@ -38,6 +41,7 @@ pub struct ScannerCycleRecoveryStatus { pub first_detected_at_unix_secs: Option, pub last_attempt_at_unix_secs: Option, pub retry_count: u64, + /// Maximum automatic retries, or zero when the recovery is unbounded. pub max_retries: u32, /// Whether the scanner may retry this state automatically. pub retryable: bool, @@ -62,7 +66,16 @@ pub fn scanner_cycle_recovery_status() -> ScannerCycleRecoveryStatus { } fn set_scanner_cycle_recovery_status(status: ScannerCycleRecoveryStatus) { - let recovery_required = if matches!(status.state.as_str(), "blocked" | "paused" | "recovery-required" | "cleanup-pending") { + let recovery_required = if matches!( + status.state.as_str(), + "blocked" + | "paused" + | "recovery-required" + | "cleanup-pending" + | "usage_floor_load_failed" + | "usage_floor_recovery_pending" + | "cache_cycle_ahead" + ) { 1.0 } else { 0.0 @@ -74,11 +87,120 @@ fn set_scanner_cycle_recovery_status(status: ScannerCycleRecoveryStatus) { .unwrap_or_else(|poisoned| poisoned.into_inner()) = status; } +pub(super) fn record_scanner_usage_floor_failure(reason: String) { + let previous = scanner_cycle_recovery_status(); + let same_failure = previous.classification.as_deref() == Some(USAGE_FLOOR_LOAD_FAILED); + let now = unix_now_secs(); + let (first_detected_at_unix_secs, retry_count) = if same_failure { + (previous.first_detected_at_unix_secs.or(Some(now)), previous.retry_count) + } else { + (Some(now), 0) + }; + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_OBJ_NAME_PATH.clone(), + state: USAGE_FLOOR_LOAD_FAILED.to_string(), + classification: Some(USAGE_FLOOR_LOAD_FAILED.to_string()), + first_detected_at_unix_secs, + last_attempt_at_unix_secs: Some(now), + retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: true, + reason: Some(reason), + ..Default::default() + }); +} + +pub(super) fn clear_scanner_usage_floor_failure() { + if scanner_cycle_recovery_status().classification.as_deref() == Some(USAGE_FLOOR_LOAD_FAILED) { + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + } +} + +pub(super) fn record_scanner_cache_cycle_ahead(requested_cycle: u64, required_cycle: u64, leader_epoch: u64) { + let previous = scanner_cycle_recovery_status(); + let same_floor = previous.classification.as_deref() == Some(CACHE_CYCLE_AHEAD) + && previous.generation == Some(required_cycle) + && previous.leader_epoch == Some(leader_epoch); + let now = unix_now_secs(); + let (first_detected_at_unix_secs, retry_count) = if same_floor { + (previous.first_detected_at_unix_secs.or(Some(now)), previous.retry_count) + } else { + (Some(now), 0) + }; + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_BLOOM_NAME_PATH.clone(), + state: CACHE_CYCLE_AHEAD.to_string(), + classification: Some(CACHE_CYCLE_AHEAD.to_string()), + generation: Some(required_cycle), + leader_epoch: Some(leader_epoch), + first_detected_at_unix_secs, + last_attempt_at_unix_secs: Some(now), + retry_count, + max_retries: 0, + retryable: true, + reason: Some(format!( + "persisted scanner cache cycle {required_cycle} is ahead of requested cycle {requested_cycle}" + )), + ..Default::default() + }); +} + +pub(super) fn record_scanner_cache_cycle_recovery_attempt() { + let mut status = scanner_cycle_recovery_status(); + if status.classification.as_deref() != Some(CACHE_CYCLE_AHEAD) { + return; + } + status.retry_count = status.retry_count.saturating_add(1); + status.last_attempt_at_unix_secs = Some(unix_now_secs()); + set_scanner_cycle_recovery_status(status); +} + +pub(super) fn clear_scanner_cache_cycle_ahead() { + if scanner_cycle_recovery_status().classification.as_deref() == Some(CACHE_CYCLE_AHEAD) { + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + } +} + +pub(super) fn record_legacy_empty_usage_floor_recovery_pending(leader_epoch: u64) { + let previous = scanner_cycle_recovery_status(); + let same_recovery = previous.classification.as_deref() == Some(LEGACY_EMPTY_USAGE_FLOOR_RECOVERY) + && previous.leader_epoch == Some(leader_epoch); + let now = unix_now_secs(); + let (first_detected_at_unix_secs, retry_count) = if same_recovery { + (previous.first_detected_at_unix_secs.or(Some(now)), previous.retry_count) + } else { + (Some(now), 0) + }; + set_scanner_cycle_recovery_status(ScannerCycleRecoveryStatus { + path: DATA_USAGE_OBJ_NAME_PATH.clone(), + quarantine_path: Some(DATA_USAGE_RECOVERY_PATH.clone()), + state: "usage_floor_recovery_pending".to_string(), + classification: Some(LEGACY_EMPTY_USAGE_FLOOR_RECOVERY.to_string()), + leader_epoch: Some(leader_epoch), + first_detected_at_unix_secs, + last_attempt_at_unix_secs: Some(now), + retry_count, + max_retries: MAX_SCANNER_CYCLE_RECOVERY_RETRIES, + retryable: true, + reason: Some("legacy empty usage floor recovery is awaiting a fenced leadership claim".to_string()), + ..Default::default() + }); +} + +pub(super) fn clear_legacy_empty_usage_floor_recovery_status() { + if scanner_cycle_recovery_status().classification.as_deref() == Some(LEGACY_EMPTY_USAGE_FLOOR_RECOVERY) { + set_scanner_cycle_recovery_status(recovery_status("healthy", None, false)); + } +} + pub(super) fn record_scanner_cycle_recovery_retry(attempt: u32) -> bool { let mut status = scanner_cycle_recovery_status(); - status.retry_count = u64::from(attempt); + if status.classification.as_deref() == Some(CACHE_CYCLE_AHEAD) { + return true; + } + status.retry_count = status.retry_count.max(u64::from(attempt)); status.last_attempt_at_unix_secs = Some(unix_now_secs()); - if attempt >= MAX_SCANNER_CYCLE_RECOVERY_RETRIES { + if status.max_retries != 0 && status.retry_count >= u64::from(status.max_retries) { status.state = "paused".to_string(); status.retryable = false; status.reason = Some("scanner cycle recovery retry budget reached; sparse backend probes continue".to_string()); @@ -1185,6 +1307,246 @@ pub(super) enum PersistedUsageFloorStartup { Authoritative, Missing, BootstrapPending, + RecoveredLegacyEmptyFence, +} + +#[derive(Clone, Debug)] +struct LegacyEmptyUsageFloorPrimary { + revision: DataUsageCacheRevision, + epoch: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +struct LegacyEmptyUsageFloorRecoveryMarker { + schema_version: u16, + primary_revision: String, + leader_epoch: u64, +} + +async fn read_legacy_empty_usage_floor_recovery_marker( + storeapi: Arc, +) -> Result, ScannerError> { + let (data, revision) = read_config_with_revision(storeapi, DATA_USAGE_RECOVERY_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to read scanner usage recovery marker: {err}")))?; + let Some(data) = data else { + return Ok(None); + }; + let marker = serde_json::from_slice::(&data) + .map_err(|err| ScannerError::Other(format!("failed to decode scanner usage recovery marker: {err}")))?; + if marker.schema_version != 1 || marker.primary_revision.is_empty() || marker.leader_epoch == 0 { + return Err(ScannerError::Other("scanner usage recovery marker is invalid".to_string())); + } + if !matches!(revision, DataUsageCacheRevision::Etag(_)) { + return Err(ScannerError::Other("scanner usage recovery marker has no revision".to_string())); + } + Ok(Some((marker, revision))) +} + +async fn clear_legacy_empty_usage_floor_recovery_marker( + storeapi: Arc, + marker_revision: &DataUsageCacheRevision, + expected_publication_epoch: u64, +) -> Result<(), ScannerError> { + let delete_result = delete_config_with_publication_admission_for_epoch( + storeapi.clone(), + RUSTFS_META_BUCKET, + DATA_USAGE_RECOVERY_PATH.as_str(), + ScannerObjectOptions { + delete_prefix: false, + http_preconditions: Some(marker_revision.preconditions()), + ..Default::default() + }, + expected_publication_epoch, + ) + .await; + match delete_result { + Ok(_) => Ok(()), + Err(err) => { + let (_, revision) = read_config_with_revision(storeapi, DATA_USAGE_RECOVERY_PATH.as_str()) + .await + .map_err(|read_err| { + ScannerError::Other(format!("failed to reconcile scanner usage recovery cleanup: {read_err}")) + })?; + if matches!(revision, DataUsageCacheRevision::Missing) { + Ok(()) + } else { + Err(ScannerError::Other(format!("failed to clear scanner usage recovery marker: {err}"))) + } + } + } +} + +pub(super) async fn complete_legacy_empty_usage_floor_recovery( + storeapi: Arc, + claimed_epoch: u64, +) -> Result<(), ScannerError> { + let Some((marker, marker_revision)) = read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await? else { + return Ok(()); + }; + if claimed_epoch <= marker.leader_epoch { + return Err(ScannerError::Other("scanner usage recovery did not advance the leader epoch".to_string())); + } + let (primary, _) = read_config_with_revision(storeapi.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to verify recovered scanner usage bootstrap: {err}")))?; + let primary = primary.ok_or_else(|| ScannerError::Other("recovered scanner usage bootstrap is missing".to_string()))?; + let usage = serde_json::from_slice::(&primary) + .map_err(|err| ScannerError::Other(format!("failed to decode recovered scanner usage bootstrap: {err}")))?; + if !data_usage_info_is_bootstrap_pending(&usage) || usage.scanner_epoch != Some(claimed_epoch) { + return Err(ScannerError::Other( + "recovered scanner usage bootstrap does not match the claimed epoch".to_string(), + )); + } + let expected_publication_epoch = scanner_publication_epoch(storeapi.clone()) + .await + .ok_or_else(|| ScannerError::Other("scanner usage recovery cleanup is blocked by data movement".to_string()))?; + clear_legacy_empty_usage_floor_recovery_marker(storeapi, &marker_revision, expected_publication_epoch).await?; + clear_legacy_empty_usage_floor_recovery_status(); + Ok(()) +} + +fn legacy_empty_usage_fence_epoch(data: &[u8], usage: &DataUsageInfo) -> Option> { + if usage.last_update.is_none() || usage.scanner_cycle.is_some() { + return None; + } + if usage.scanner_epoch.is_some_and(|epoch| epoch == 0 || epoch >= u64::MAX - 1) { + return None; + } + let expected = DataUsageInfo { + last_update: usage.last_update, + scanner_epoch: usage.scanner_epoch, + ..Default::default() + }; + if usage != &expected { + return None; + } + + let serde_json::Value::Object(fields) = serde_json::from_slice::(data).ok()? else { + return None; + }; + // RUSTFS_COMPAT_TODO(backlog-2102): accept only the exact empty usage fence serialized by rc.2/rc.3. Remove after those releases are no longer supported direct-upgrade sources. + const REQUIRED_FIELDS: &[&str] = &[ + "total_capacity", + "total_used_capacity", + "total_free_capacity", + "last_update", + "objects_total_count", + "versions_total_count", + "delete_markers_total_count", + "objects_total_size", + "replication_info", + "buckets_count", + "buckets_usage", + "usage_snapshot_complete", + "bucket_sizes", + "disk_usage_status", + ]; + let expected_len = REQUIRED_FIELDS.len() + if usage.scanner_epoch.is_some() { 1 } else { 0 }; + if fields.len() != expected_len + || REQUIRED_FIELDS.iter().any(|field| !fields.contains_key(*field)) + || (usage.scanner_epoch.is_some() != fields.contains_key("scanner_epoch")) + { + return None; + } + Some(usage.scanner_epoch) +} + +async fn recover_legacy_empty_usage_floor( + storeapi: Arc, + primary: LegacyEmptyUsageFloorPrimary, + expected_publication_epoch: u64, +) -> Result<(), ScannerError> { + let DataUsageCacheRevision::Etag(primary_revision) = &primary.revision else { + return Err(ScannerError::Other("legacy empty scanner usage floor has no revision".to_string())); + }; + let marker = LegacyEmptyUsageFloorRecoveryMarker { + schema_version: 1, + primary_revision: primary_revision.clone(), + leader_epoch: primary.epoch, + }; + let marker_data = serde_json::to_vec(&marker) + .map_err(|err| ScannerError::Other(format!("failed to encode scanner usage recovery marker: {err}")))?; + match read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await? { + Some((persisted, _)) if persisted != marker => { + return Err(ScannerError::Other( + "scanner usage recovery marker conflicts with the persisted empty floor".to_string(), + )); + } + Some(_) => {} + None => { + let marker_save = save_config_with_publication_admission_for_epoch( + storeapi.clone(), + DATA_USAGE_RECOVERY_PATH.as_str(), + marker_data.clone(), + DataUsageCacheRevision::Missing.preconditions(), + expected_publication_epoch, + ) + .await; + if !marker_save + .as_ref() + .ok() + .and_then(|info| info.etag.as_deref()) + .is_some_and(|etag| !etag.is_empty()) + { + let persisted = read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await?; + if persisted.as_ref().map(|(persisted, _)| persisted) != Some(&marker) { + return Err(ScannerError::Other(match marker_save { + Ok(_) => "scanner usage recovery marker returned no ETag and could not be confirmed".to_string(), + Err(err) => format!("failed to persist scanner usage recovery marker: {err}"), + })); + } + } + } + } + + let marker = DataUsageInfo { + last_update: Some(std::time::SystemTime::now()), + scanner_epoch: Some(primary.epoch), + usage_snapshot_converged: Some(false), + usage_snapshot_bootstrap_pending: true, + ..Default::default() + }; + let data = serde_json::to_vec(&marker) + .map_err(|err| ScannerError::Other(format!("failed to encode recovered scanner usage bootstrap: {err}")))?; + let save_result = save_config_with_publication_admission_for_epoch( + storeapi.clone(), + DATA_USAGE_OBJ_NAME_PATH.as_str(), + data.clone(), + primary.revision.preconditions(), + expected_publication_epoch, + ) + .await; + if save_result + .as_ref() + .ok() + .and_then(|info| info.etag.as_deref()) + .is_some_and(|etag| !etag.is_empty()) + { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "legacy_empty_usage_floor_recovered", + path = %DATA_USAGE_OBJ_NAME_PATH.as_str(), + scanner_epoch = primary.epoch, + "Scanner recovered a legacy empty usage floor" + ); + return Ok(()); + } + + let (persisted, revision) = read_config_with_revision(storeapi, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .map_err(|err| ScannerError::Other(format!("failed to reconcile recovered scanner usage bootstrap: {err}")))?; + if persisted.as_deref() == Some(data.as_slice()) && matches!(revision, DataUsageCacheRevision::Etag(_)) { + return Ok(()); + } + Err(ScannerError::Other(match save_result { + Ok(_) => "recovered scanner usage bootstrap returned no ETag and could not be confirmed".to_string(), + Err(err) => format!("failed to recover legacy empty scanner usage floor: {err}"), + })) } pub(super) fn encode_scanner_cycle_state( @@ -1324,16 +1686,22 @@ pub(super) async fn persisted_usage_floor_for_startup( let Some(read_epoch) = scanner_publication_epoch(storeapi.clone()).await else { return Err(ScannerError::Other("scanner usage floor read is blocked by data movement".to_string())); }; + let recovery_marker = read_legacy_empty_usage_floor_recovery_marker(storeapi.clone()).await?; let mut floor = PersistedUsageFloor::default(); let mut found_any = false; let mut bootstrap_pending = false; + let mut recovered_bootstrap = false; + let mut bootstrap_epoch = None; // A valid JSON object without a baseline identity is not a floor and must // never be treated as an empty one. It can, however, be a partially // written v2 primary left behind during an upgrade. Keep its epoch as a // fence while looking for a durable companion snapshot; if no companion // is new enough, the caller still fails closed below. let mut invalid_baseline_path: Option = None; - let mut invalid_baseline_epoch: Option = None; + let mut invalid_baseline_epoch = recovery_marker.as_ref().map(|(marker, _)| marker.leader_epoch); + let mut unrecoverable_baseline_path: Option = None; + let mut stale_authoritative_path: Option = None; + let mut legacy_empty_primary: Option = None; let update_floor = |floor: &mut PersistedUsageFloor, usage: &DataUsageInfo, path: &str| -> Result<(), ScannerError> { floor.leader_epoch = floor.leader_epoch.max(usage.scanner_epoch.unwrap_or_default()); if let Some(completed_cycle) = usage.scanner_cycle { @@ -1348,8 +1716,9 @@ pub(super) async fn persisted_usage_floor_for_startup( for primary_path in [DATA_USAGE_OBJ_NAME_PATH.as_str(), LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()] { let backup_path = format!("{primary_path}.bkp"); let is_v2_path = primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str(); + let mut recovered_primary_companion_epoch = None; let primary_epoch = match read_config_with_revision(storeapi.clone(), primary_path).await { - Ok((Some(data), _)) => { + Ok((Some(data), revision)) => { let usage = serde_json::from_slice::(&data).map_err(|err| { ScannerError::Other(format!("failed to decode scanner usage floor from {primary_path}: {err}")) })?; @@ -1358,18 +1727,47 @@ pub(super) async fn persisted_usage_floor_for_startup( return Err(ScannerError::Other("multiple scanner usage bootstrap markers were found".to_string())); } bootstrap_pending = true; + bootstrap_epoch = usage.scanner_epoch; + if let Some((marker, _)) = recovery_marker.as_ref() { + if usage.scanner_epoch.is_none_or(|epoch| epoch < marker.leader_epoch) { + return Err(ScannerError::Other( + "scanner usage bootstrap is older than its recovery marker".to_string(), + )); + } + recovered_bootstrap = true; + } update_floor(&mut floor, &usage, primary_path)?; None } else if !data_usage_info_has_persisted_baseline_identity(&usage) { invalid_baseline_path.get_or_insert_with(|| primary_path.to_string()); invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch); + match legacy_empty_usage_fence_epoch(&data, &usage) { + Some(Some(epoch)) if is_v2_path => { + legacy_empty_primary = Some(LegacyEmptyUsageFloorPrimary { revision, epoch }); + } + Some(_) => {} + None => { + unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string()); + } + } None } else { let epoch = usage.scanner_epoch.unwrap_or_default(); + if recovered_bootstrap && !is_v2_path { + if epoch < floor.leader_epoch { + unrecoverable_baseline_path.get_or_insert_with(|| primary_path.to_string()); + } else { + update_floor(&mut floor, &usage, primary_path)?; + recovered_primary_companion_epoch = Some(epoch); + } + None // A legacy snapshot may be structurally valid but older // than an incomplete v2 snapshot left by a newer leader. // Do not let that candidate regress the startup floor. - if !is_v2_path && invalid_baseline_epoch.is_some_and(|fenced_epoch| epoch < fenced_epoch) { + } else if invalid_baseline_epoch.is_some_and(|fenced_epoch| epoch < fenced_epoch) + && (!is_v2_path || recovery_marker.is_some()) + { + stale_authoritative_path.get_or_insert_with(|| primary_path.to_string()); None } else { update_floor(&mut floor, &usage, primary_path)?; @@ -1387,17 +1785,51 @@ pub(super) async fn persisted_usage_floor_for_startup( let mut any_found = primary_epoch.is_some(); match read_config_with_revision(storeapi.clone(), &backup_path).await { Ok((Some(data), _)) => { + let usage = serde_json::from_slice::(&data).map_err(|err| { + ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}")) + })?; if bootstrap_pending { + if let Some(primary_epoch) = recovered_primary_companion_epoch { + if data_usage_info_has_persisted_baseline_identity(&usage) { + let backup_epoch = usage.scanner_epoch.unwrap_or_default(); + if backup_epoch >= primary_epoch { + update_floor(&mut floor, &usage, &backup_path)?; + } + } else if let Some(epoch) = legacy_empty_usage_fence_epoch(&data, &usage) { + if let Some(epoch) = epoch { + floor.leader_epoch = floor.leader_epoch.max(epoch); + } + } else { + unrecoverable_baseline_path.get_or_insert_with(|| backup_path.clone()); + } + continue; + } + let compatible_empty_fence = legacy_empty_usage_fence_epoch(&data, &usage).is_some_and(|epoch| { + epoch.is_none_or(|epoch| bootstrap_epoch.is_some_and(|bootstrap_epoch| epoch <= bootstrap_epoch)) + }); + if compatible_empty_fence { + continue; + } + if recovered_bootstrap && data_usage_info_has_persisted_baseline_identity(&usage) { + let epoch = usage.scanner_epoch.unwrap_or_default(); + if epoch >= floor.leader_epoch { + update_floor(&mut floor, &usage, &backup_path)?; + if primary_path == DATA_USAGE_OBJ_NAME_PATH.as_str() { + break; + } + continue; + } + } return Err(ScannerError::Other( "scanner usage bootstrap conflicts with a persisted backup".to_string(), )); } - let usage = serde_json::from_slice::(&data).map_err(|err| { - ScannerError::Other(format!("failed to decode scanner usage floor from {backup_path}: {err}")) - })?; if !data_usage_info_has_persisted_baseline_identity(&usage) { invalid_baseline_path.get_or_insert_with(|| backup_path.clone()); invalid_baseline_epoch = invalid_baseline_epoch.max(usage.scanner_epoch); + if legacy_empty_usage_fence_epoch(&data, &usage).is_none() { + unrecoverable_baseline_path.get_or_insert_with(|| backup_path.clone()); + } // This is still persisted state, so it must not enable a // missing-state bootstrap. Continue to a legacy pair in // case it contains a complete, fenced snapshot. @@ -1411,6 +1843,8 @@ pub(super) async fn persisted_usage_floor_for_startup( { update_floor(&mut floor, &usage, &backup_path)?; any_found = true; + } else { + stale_authoritative_path.get_or_insert_with(|| backup_path.clone()); } } } @@ -1433,6 +1867,30 @@ pub(super) async fn persisted_usage_floor_for_startup( } if !found_any && !bootstrap_pending { + if allow_missing_for_bootstrap + && unrecoverable_baseline_path.is_none() + && stale_authoritative_path.is_none() + && let Some(mut primary) = legacy_empty_primary + { + primary.epoch = primary.epoch.max(invalid_baseline_epoch.unwrap_or_default()); + recover_legacy_empty_usage_floor(storeapi.clone(), primary.clone(), read_epoch).await?; + record_legacy_empty_usage_floor_recovery_pending(primary.epoch); + return Ok(( + PersistedUsageFloor { + next_cycle: 0, + leader_epoch: primary.epoch, + }, + PersistedUsageFloorStartup::RecoveredLegacyEmptyFence, + )); + } + if let Some(path) = stale_authoritative_path { + return Err(ScannerError::Other(format!( + "persisted scanner usage floor from {path} is older than the required recovery fence" + ))); + } + if recovery_marker.is_some() { + return Err(ScannerError::Other("scanner usage recovery marker has no matching primary".to_string())); + } if let Some(path) = invalid_baseline_path { return Err(ScannerError::Other(format!( "persisted scanner usage floor from {path} has no authoritative baseline or newer valid backup" @@ -1470,18 +1928,67 @@ pub(super) async fn persisted_usage_floor_for_startup( } drop(publication_admission); } - let Some(_publication_admission) = scanner_publication_admission_for_epoch(storeapi, read_epoch).await else { + let Some(publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await else { return Err(ScannerError::Other( "scanner usage floor changed while its epoch proof was being confirmed".to_string(), )); }; let state = if found_any { PersistedUsageFloorStartup::Authoritative + } else if recovered_bootstrap { + if let Some(path) = unrecoverable_baseline_path { + return Err(ScannerError::Other(format!( + "scanner usage recovery conflicts with persisted usage state at {path}" + ))); + } + let recovery_epoch = recovery_marker + .as_ref() + .map(|(marker, _)| marker.leader_epoch) + .unwrap_or(floor.leader_epoch); + record_legacy_empty_usage_floor_recovery_pending(recovery_epoch); + PersistedUsageFloorStartup::RecoveredLegacyEmptyFence } else if bootstrap_pending { + if let Some(path) = unrecoverable_baseline_path { + return Err(ScannerError::Other(format!( + "scanner usage bootstrap conflicts with persisted usage state at {path}" + ))); + } PersistedUsageFloorStartup::BootstrapPending } else { PersistedUsageFloorStartup::Missing }; + if found_any && let Some((_, marker_revision)) = recovery_marker.as_ref() { + drop(publication_admission); + let marker_cleared = + match clear_legacy_empty_usage_floor_recovery_marker(storeapi.clone(), marker_revision, read_epoch).await { + Ok(()) => true, + Err(err) => { + warn!( + target: "rustfs::scanner", + event = EVENT_SCANNER_PERSIST_STATE, + component = LOG_COMPONENT_SCANNER, + subsystem = LOG_SUBSYSTEM_RUNTIME, + state = "usage_floor_recovery_cleanup_deferred", + path = %DATA_USAGE_RECOVERY_PATH.as_str(), + error = %err, + "Scanner usage floor recovery marker cleanup was deferred" + ); + false + } + }; + let Some(_final_publication_admission) = scanner_publication_admission_for_epoch(storeapi.clone(), read_epoch).await + else { + return Err(ScannerError::Other( + "scanner usage floor changed after recovery marker cleanup".to_string(), + )); + }; + if marker_cleared { + clear_legacy_empty_usage_floor_recovery_status(); + } + clear_scanner_usage_floor_failure(); + return Ok((floor, state)); + } + clear_scanner_usage_floor_failure(); Ok((floor, state)) } diff --git a/crates/scanner/src/scanner/leadership.rs b/crates/scanner/src/scanner/leadership.rs index a99687f33..e728e7420 100644 --- a/crates/scanner/src/scanner/leadership.rs +++ b/crates/scanner/src/scanner/leadership.rs @@ -21,6 +21,29 @@ pub(super) enum ScannerLeadershipClaimReconcile { Unchanged, } +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum ScannerCycleResetPolicy { + None, + ResetAll, + ResetCoveragePreservingNext, +} + +impl ScannerCycleResetPolicy { + fn apply(self, cycle_info: &mut CurrentCycle, attempted_next: u64) { + match self { + Self::None => {} + Self::ResetAll => *cycle_info = CurrentCycle::default(), + Self::ResetCoveragePreservingNext => { + let next = cycle_info.next.max(attempted_next); + *cycle_info = CurrentCycle { + next, + ..Default::default() + }; + } + } + } +} + pub(super) async fn reconcile_scanner_leadership_claim( storeapi: Arc, attempted: &[u8], @@ -329,7 +352,7 @@ pub(super) async fn claim_scanner_leadership( revision: &mut DataUsageCacheRevision, persisted_epoch: &mut u64, allow_bootstrap_pending: bool, - reset_bootstrap_cycle_on_conflict: bool, + cycle_reset_policy: ScannerCycleResetPolicy, ) -> bool { for retry in 0..=SCANNER_PERSIST_CAS_RETRIES { if ctx.is_cancelled() { @@ -347,6 +370,7 @@ pub(super) async fn claim_scanner_leadership( ); return false; }; + let attempted_next = cycle_info.next; let data = match encode_scanner_cycle_state(cycle_info, claimed_epoch) { Ok(data) => data, Err(err) => { @@ -459,9 +483,7 @@ pub(super) async fn claim_scanner_leadership( .await; } Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES => { - if reset_bootstrap_cycle_on_conflict { - *cycle_info = CurrentCycle::default(); - } + cycle_reset_policy.apply(cycle_info, attempted_next); continue; } Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { @@ -517,17 +539,13 @@ pub(super) async fn claim_scanner_leadership( Ok(ScannerLeadershipClaimReconcile::Changed) if retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => { - if reset_bootstrap_cycle_on_conflict { - *cycle_info = CurrentCycle::default(); - } + cycle_reset_policy.apply(cycle_info, attempted_next); continue; } Ok(ScannerLeadershipClaimReconcile::Unchanged) if precondition_failed && retry < SCANNER_PERSIST_CAS_RETRIES && !ctx.is_cancelled() => { - if reset_bootstrap_cycle_on_conflict { - *cycle_info = CurrentCycle::default(); - } + cycle_reset_policy.apply(cycle_info, attempted_next); continue; } Ok(ScannerLeadershipClaimReconcile::Changed | ScannerLeadershipClaimReconcile::Unchanged) => { diff --git a/crates/scanner/src/scanner/tests.rs b/crates/scanner/src/scanner/tests.rs index 5cc87c063..1f2b217b4 100644 --- a/crates/scanner/src/scanner/tests.rs +++ b/crates/scanner/src/scanner/tests.rs @@ -16,10 +16,11 @@ use super::heal_info::{classify_background_heal_read_error, decode_background_he use super::*; use crate::EcstoreResult; use crate::{ - DATA_USAGE_BLOOM_RECOVERY_PATH, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, - ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, - ScannerPutObjReader as PutObjReader, init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, - init_local_disks_with_instance_ctx, + DATA_USAGE_BLOOM_RECOVERY_PATH, DATA_USAGE_CACHE_KEY_FORMAT, DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, + DataUsageCachePrepareOutcome, DataUsageCacheSource, DataUsageEntry, DataUsageScanPlanDigest, Endpoint, EndpointServerPools, + Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, + ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader, init_bucket_metadata_sys_for_scanner_tests, + init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, }; use serial_test::serial; use std::collections::{HashMap, HashSet}; @@ -38,30 +39,46 @@ async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc) { } async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool) -> (tempfile::TempDir, Arc) { + setup_scanner_cycle_store_with_pool_count(seed_usage_baseline, 1).await +} + +async fn setup_scanner_cycle_store_with_pool_count( + seed_usage_baseline: bool, + pool_count: usize, +) -> (tempfile::TempDir, Arc) { init_ecstore_config_for_scanner_tests(); let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created"); - let mut endpoints = Vec::new(); - for disk_index in 0..4 { - let disk_path = temp_dir.path().join(format!("disk{disk_index}")); - tokio::fs::create_dir_all(&disk_path) - .await - .expect("scanner cycle test disk should be created"); - let mut endpoint = - Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); - endpoint.set_pool_index(0); - endpoint.set_set_index(0); - endpoint.set_disk_index(disk_index); - endpoints.push(endpoint); + let mut pools = Vec::with_capacity(pool_count); + for pool_index in 0..pool_count { + let mut endpoints = Vec::new(); + for disk_index in 0..4 { + let disk_path = temp_dir.path().join(format!("pool{pool_index}/disk{disk_index}")); + tokio::fs::create_dir_all(&disk_path) + .await + .expect("scanner cycle test disk should be created"); + let mut endpoint = + Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse"); + endpoint.set_pool_index(pool_index); + endpoint.set_set_index(0); + endpoint.set_disk_index(disk_index); + endpoints.push(endpoint); + } + pools.push(PoolEndpoints { + legacy: false, + set_count: 1, + drives_per_set: 4, + endpoints: Endpoints::from(endpoints), + cmd_line: if pool_count == 1 { + "scanner-cycle-metrics".to_string() + } else { + format!("scanner-cycle-metrics-pool-{pool_index}") + }, + platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), + }); } - let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints { - legacy: false, - set_count: 1, - drives_per_set: 4, - endpoints: Endpoints::from(endpoints), - cmd_line: "scanner-cycle-metrics".to_string(), - platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH), - }]); + let endpoint_pools = EndpointServerPools::from(pools); let instance_ctx = Arc::new(InstanceContext::new()); + instance_ctx.set_endpoints(endpoint_pools.clone()); init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) .await .expect("scanner cycle test disks should initialize"); @@ -88,6 +105,27 @@ async fn setup_scanner_cycle_store_with_usage_baseline(seed_usage_baseline: bool (temp_dir, store) } +async fn restart_scanner_cycle_store_from(store: &Arc) -> Arc { + let endpoint_pools = store + .instance_endpoints() + .expect("scanner restart test store should retain its endpoint topology"); + let instance_ctx = Arc::new(InstanceContext::new()); + instance_ctx.set_endpoints(endpoint_pools.clone()); + init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone()) + .await + .expect("scanner restart test disks should reinitialize"); + let restarted = ECStore::new_with_instance_ctx( + "127.0.0.1:0".parse().expect("test address should parse"), + endpoint_pools, + CancellationToken::new(), + instance_ctx, + ) + .await + .expect("restarted scanner cycle test ECStore should initialize"); + init_bucket_metadata_sys_for_scanner_tests(restarted.clone()).await; + restarted +} + fn assert_run_data_scanner_signature(_run: F) where F: Fn(CancellationToken, Arc) -> Fut, @@ -100,6 +138,190 @@ fn run_data_scanner_keeps_its_two_argument_api() { assert_run_data_scanner_signature(run_data_scanner); } +#[tokio::test] +async fn restarted_main_loop_completes_durable_pause_backlog_catch_up() { + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + global_metrics().set_cycle(None).await; + let (_temp_dir, store) = setup_scanner_cycle_store().await; + + let paused_at = scanner_pause_backlog_now(); + let mut seeded = ScannerPauseBacklogController::claim(store.clone(), paused_at) + .await + .expect("seed writer should claim the durable pause backlog"); + seeded + .observe(ScannerPauseBacklogObservation { + now_unix_secs: paused_at.saturating_add(1), + paused: true, + movement_generation: store.scanner_data_movement_generation().saturating_add(1), + movement_work_items: 1, + pause_started_at_unix_secs: paused_at.saturating_add(1), + dirty_usage_buckets: 0, + discovered_expiry_items: 0, + discovered_transition_items: 0, + }) + .await; + drop(seeded); + + let seeded_status = scanner_pause_backlog_status(store.clone()).await; + assert!(seeded_status.durable, "seeded pause backlog must be set-backed"); + assert_eq!(seeded_status.phase, ScannerPauseBacklogPhase::Paused); + assert!(seeded_status.pending_full_scan); + assert_eq!(seeded_status.catch_up_attempts, 0); + + let restarted = restart_scanner_cycle_store_from(&store).await; + assert!( + restarted.instance_endpoints().is_some(), + "restarted scanner store must retain instance endpoints" + ); + let restarted_status = scanner_pause_backlog_status(restarted.clone()).await; + assert_eq!(restarted_status.phase, ScannerPauseBacklogPhase::Paused); + assert_eq!(restarted_status.generation, seeded_status.generation); + + let ctx = CancellationToken::new(); + let scanner_ctx = ctx.clone(); + let scanner_store = restarted.clone(); + let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await }); + + let final_status = match tokio::time::timeout(Duration::from_secs(30), async { + loop { + let status = scanner_pause_backlog_status(restarted.clone()).await; + if status.phase == ScannerPauseBacklogPhase::Idle + && status.writer_epoch > seeded_status.writer_epoch + && status.catch_up_attempts > seeded_status.catch_up_attempts + { + break status; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + { + Ok(status) => status, + Err(err) => { + ctx.cancel(); + scanner_task.abort(); + panic!("restarted scanner did not complete durable catch-up through the main loop: {err}"); + } + }; + + ctx.cancel(); + tokio::time::timeout(Duration::from_secs(5), scanner_task) + .await + .expect("scanner loop should stop after cancellation") + .expect("scanner task should not panic") + .expect("scanner loop should exit cleanly"); + + assert!(final_status.durable); + assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle); + assert!(!final_status.pending_full_scan); + assert_eq!(final_status.pending_work_items, 0); + assert_eq!(final_status.consecutive_failures, 0); + assert!(final_status.pause_ended_at_unix_secs >= final_status.pause_started_at_unix_secs); + + let usage = read_config(restarted.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("the catch-up scanner cycle should leave an authoritative usage snapshot readable"); + let usage = serde_json::from_slice::(&usage).expect("authoritative usage snapshot should decode"); + assert!( + usage.is_complete_bucket_usage_snapshot(), + "durable catch-up must run a complete scanner cycle before clearing the backlog" + ); + + global_metrics().set_cycle(None).await; + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); +} + +#[tokio::test] +#[serial_test::serial(scanner_runtime_env)] +async fn running_main_loop_catches_up_pause_cleared_after_startup_observe() { + temp_env::async_with_vars([(ENV_SCANNER_CYCLE, Some("1")), (ENV_SCANNER_START_DELAY_SECS, Some("0"))], async { + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + global_metrics().set_cycle(None).await; + let (_temp_dir, store) = setup_scanner_cycle_store_with_pool_count(true, 2).await; + + let ctx = CancellationToken::new(); + let scanner_ctx = ctx.clone(); + let scanner_store = store.clone(); + let startup_probe = ScannerStartupObservedProbe::install(); + let scanner_task = tokio::spawn(async move { run_data_scanner(scanner_ctx, scanner_store).await }); + startup_probe.wait().await; + let ready_probe = ScannerRuntimeObservedProbe::install(&store, false); + startup_probe.resume(); + drop(startup_probe); + ready_probe.wait().await; + drop(ready_probe); + + let paused_probe = ScannerRuntimeObservedProbe::install(&store, true); + let paused_at = time::OffsetDateTime::now_utc(); + { + let mut pool_meta = store.pool_meta.write().await; + pool_meta.pools[0].last_update = paused_at; + pool_meta.pools[0].decommission = Some(crate::storage_api::owner::EcstorePoolDecommissionInfo { + failed: true, + ..Default::default() + }); + } + let pause_status = store.scanner_data_movement_pause_status().await; + assert!(pause_status.paused); + paused_probe.wait().await; + drop(paused_probe); + + let paused_backlog = scanner_pause_backlog_status(store.clone()).await; + assert_eq!(paused_backlog.phase, ScannerPauseBacklogPhase::Paused); + assert!(paused_backlog.pending_full_scan); + + let resumed_probe = ScannerRuntimeObservedProbe::install(&store, false); + store + .clear_decommission(0) + .await + .expect("terminal decommission clear should publish a movement generation"); + resumed_probe.wait().await; + drop(resumed_probe); + + let final_status = match tokio::time::timeout(Duration::from_secs(30), async { + loop { + let status = scanner_pause_backlog_status(store.clone()).await; + if status.phase == ScannerPauseBacklogPhase::Idle + && status.writer_epoch == paused_backlog.writer_epoch + && status.catch_up_attempts > paused_backlog.catch_up_attempts + { + break status; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + }) + .await + { + Ok(status) => status, + Err(err) => { + ctx.cancel(); + scanner_task.abort(); + panic!("running scanner did not complete durable catch-up after a runtime movement clear: {err}"); + } + }; + + ctx.cancel(); + tokio::time::timeout(Duration::from_secs(5), scanner_task) + .await + .expect("scanner loop should stop after cancellation") + .expect("scanner task should not panic") + .expect("scanner loop should exit cleanly"); + + assert!(final_status.durable); + assert_eq!(final_status.phase, ScannerPauseBacklogPhase::Idle); + assert_eq!(final_status.writer_epoch, paused_backlog.writer_epoch); + assert!(!final_status.pending_full_scan); + assert_eq!(final_status.pending_work_items, 0); + assert_eq!(final_status.consecutive_failures, 0); + + global_metrics().set_cycle(None).await; + crate::scanner_io::clear_dirty_usage_buckets_for_tests(); + }) + .await; + crate::runtime_config::refresh_scanner_runtime_config_for_tests(); +} + #[tokio::test] async fn scanner_cycle_lock_fence_cancels_cycle_context() { let cycle_ctx = CancellationToken::new(); @@ -361,6 +583,7 @@ struct MemoryConfigStore { cancel_after_interleaving_puts: Mutex>, cancel_after_successful_puts: Mutex>, replace_after_successful_puts: Mutex)>>, + error_after_commit_deletes: Mutex>, put_counts: Mutex>, publication_admission_blocked: AtomicBool, block_publication_after_admissions: AtomicUsize, @@ -1959,6 +2182,669 @@ async fn scanner_usage_floor_keeps_valid_primary_when_backup_has_no_identity() { ); } +fn rc3_legacy_empty_usage_fence(epoch: Option) -> Vec { + // Pinned field set emitted by rc.3 after DeleteBucket synthesized a + // default v2 usage primary. Leadership added scanner_epoch separately. + const RC3_EMPTY_USAGE_FENCE: &str = r#"{ + "total_capacity":0, + "total_used_capacity":0, + "total_free_capacity":0, + "last_update":{"secs_since_epoch":1,"nanos_since_epoch":0}, + "objects_total_count":0, + "versions_total_count":0, + "delete_markers_total_count":0, + "objects_total_size":0, + "replication_info":{}, + "buckets_count":0, + "buckets_usage":{}, + "usage_snapshot_complete":false, + "bucket_sizes":{}, + "disk_usage_status":[] + }"#; + let mut value = + serde_json::from_str::(RC3_EMPTY_USAGE_FENCE).expect("pinned rc.3 empty usage fence should decode"); + let fields = value + .as_object_mut() + .expect("legacy empty usage fence should be a JSON object"); + if let Some(epoch) = epoch { + fields.insert("scanner_epoch".to_string(), serde_json::Value::from(epoch)); + } + serde_json::to_vec(&value).expect("rc.3 legacy empty usage fence fixture should encode") +} + +#[tokio::test] +async fn scanner_usage_floor_recovers_rc3_empty_fences_and_preserves_cycle_number() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_path = format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup_key = memory_config_key(RUSTFS_META_BUCKET, &backup_path); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store + .objects + .lock() + .await + .insert(backup_key, rc3_legacy_empty_usage_fence(None)); + store.revisions.lock().await.insert(primary_key, 1); + + let (floor, startup) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("rc.3 empty usage fences should enter recovery"); + assert_eq!( + floor, + PersistedUsageFloor { + next_cycle: 0, + leader_epoch: 7, + } + ); + assert_eq!(startup, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence); + + let primary = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("recovered usage bootstrap should be persisted"); + let pending = serde_json::from_slice::(&primary).expect("recovered usage bootstrap should decode"); + assert!(data_usage_info_is_bootstrap_pending(&pending)); + assert_eq!(pending.scanner_epoch, Some(7)); + assert!(read_config(store.clone(), DATA_USAGE_RECOVERY_PATH.as_str()).await.is_ok()); + + let (restart_floor, restart_state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("recovery marker should survive a restart before leadership claim"); + assert_eq!(restart_floor.leader_epoch, 7); + assert_eq!(restart_state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence); + let mut cycle = CurrentCycle { + current: 17_117, + next: 17_118, + cycle_completed: vec![Utc::now()], + started: Utc::now(), + }; + assert_eq!( + prepare_cycle_for_usage_floor_bootstrap(&mut cycle, restart_floor, restart_state), + (true, ScannerCycleResetPolicy::ResetCoveragePreservingNext) + ); + assert_eq!(cycle.current, 0); + assert_eq!(cycle.next, 17_118); + assert!(cycle.cycle_completed.is_empty()); + + let mut revision = DataUsageCacheRevision::Missing; + let mut leader_epoch = restart_floor.leader_epoch; + assert!( + claim_scanner_leadership( + &CancellationToken::new(), + store.clone(), + &mut cycle, + &mut revision, + &mut leader_epoch, + true, + ScannerCycleResetPolicy::ResetCoveragePreservingNext, + ) + .await + ); + assert_eq!(leader_epoch, 8); + assert_eq!(cycle.next, 17_118); + assert_eq!(cycle.current, 0); + assert!(cycle.cycle_completed.is_empty()); + + let source = DataUsageCacheSource::new(0, 0); + let scan_plan_digest = DataUsageScanPlanDigest([7; 32]); + for (cache_path, name) in [ + (DATA_USAGE_CACHE_NAME.to_string(), DATA_USAGE_ROOT), + (format!("photos/{DATA_USAGE_CACHE_NAME}"), "photos"), + ] { + let mut historical = DataUsageCache::default(); + historical.info.name = name.to_string(); + historical.info.next_cycle = 17_118; + historical.info.leader_epoch = 7; + historical.info.source = Some(source); + historical.info.scan_plan_digest = Some(scan_plan_digest); + historical.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT; + historical.info.snapshot_complete = true; + historical.replace(name, "", DataUsageEntry::default()); + historical + .save(store.clone(), &cache_path) + .await + .expect("historical scanner cache should persist through the storage path"); + + let mut recovered = DataUsageCache::default(); + let revisions = recovered + .load_with_revisions(store.clone(), &cache_path) + .await + .expect("historical scanner cache should reload with CAS revisions"); + assert_eq!(recovered.info.name, name); + assert_eq!(recovered.info.next_cycle, 17_118); + assert_eq!(recovered.info.leader_epoch, 7); + assert!(!recovered.cache.is_empty()); + + assert_eq!( + recovered.prepare_for_scan(name, cycle.next, leader_epoch, source, scan_plan_digest, true), + DataUsageCachePrepareOutcome::Reset, + "recovered cache should reset without a cycle regression: {cache_path}" + ); + assert_eq!(recovered.info.next_cycle, 17_118); + assert_eq!(recovered.info.leader_epoch, 8); + assert!(!recovered.info.snapshot_complete); + assert!(recovered.cache.is_empty()); + + recovered + .save_with_revisions(store.clone(), &cache_path, &revisions) + .await + .expect("reset scanner cache should persist with its loaded revisions"); + let mut persisted_reset = DataUsageCache::default(); + persisted_reset + .load(store.clone(), &cache_path) + .await + .expect("persisted reset scanner cache should reload"); + assert_eq!(persisted_reset.info.name, name); + assert_eq!(persisted_reset.info.next_cycle, 17_118); + assert_eq!(persisted_reset.info.leader_epoch, 8); + assert!(!persisted_reset.info.snapshot_complete); + assert!(persisted_reset.cache.is_empty()); + } + complete_legacy_empty_usage_floor_recovery(store.clone(), leader_epoch) + .await + .expect("leadership claim should retire the recovery marker"); + assert!(matches!( + read_config(store.clone(), DATA_USAGE_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); + let (claimed_floor, claimed_state) = persisted_usage_floor_for_startup(store, true) + .await + .expect("claimed bootstrap should remain restartable"); + assert_eq!(claimed_floor.leader_epoch, 8); + assert_eq!(claimed_state, PersistedUsageFloorStartup::BootstrapPending); +} + +#[tokio::test] +async fn scanner_usage_floor_recovery_preserves_newer_authoritative_companion_floor() { + for companion_path in [ + format!("{}.bkp", DATA_USAGE_OBJ_NAME_PATH.as_str()), + LEGACY_DATA_USAGE_OBJ_NAME_PATH.clone(), + ] { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store.revisions.lock().await.insert(primary_key, 1); + persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("legacy empty primary should enter recovery"); + + let mut companion = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + companion.scanner_epoch = Some(8); + companion.scanner_cycle = Some(11); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, &companion_path), + serde_json::to_vec(&companion).expect("authoritative companion should encode"), + ); + if companion_path.ends_with(".bkp") { + let mut stale_legacy = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + stale_legacy.scanner_epoch = Some(6); + stale_legacy.scanner_cycle = Some(10); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&stale_legacy).expect("stale legacy companion should encode"), + ); + } else { + let mut stale_backup = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + stale_backup.scanner_epoch = Some(6); + stale_backup.scanner_cycle = Some(10); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, &format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str())), + serde_json::to_vec(&stale_backup).expect("stale legacy backup should encode"), + ); + } + + let (floor, state) = persisted_usage_floor_for_startup(store, true) + .await + .expect("a newer authoritative companion should advance the recovery floor"); + assert_eq!(floor.leader_epoch, 8, "unexpected companion path: {companion_path}"); + assert_eq!(floor.next_cycle, 12, "unexpected companion path: {companion_path}"); + assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence); + } +} + +#[tokio::test] +async fn scanner_usage_floor_recovery_fences_non_authoritative_legacy_backup() { + for partial_backup in [false, true] { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store.revisions.lock().await.insert(primary_key, 1); + persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("legacy empty primary should enter recovery"); + + let mut legacy_primary = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + legacy_primary.scanner_epoch = Some(8); + legacy_primary.scanner_cycle = Some(11); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&legacy_primary).expect("legacy primary should encode"), + ); + let backup_path = format!("{}.bkp", LEGACY_DATA_USAGE_OBJ_NAME_PATH.as_str()); + let backup = if partial_backup { + serde_json::to_vec(&DataUsageInfo { + last_update: Some(std::time::SystemTime::UNIX_EPOCH), + scanner_epoch: Some(9), + buckets_count: 1, + ..Default::default() + }) + .expect("partial legacy backup should encode") + } else { + rc3_legacy_empty_usage_fence(Some(9)) + }; + store + .objects + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, &backup_path), backup); + + if partial_backup { + let err = persisted_usage_floor_for_startup(store, true) + .await + .expect_err("a partial noncanonical backup must remain fail-closed"); + assert!(err.to_string().contains("conflicts with persisted usage state")); + } else { + let (floor, state) = persisted_usage_floor_for_startup(store, true) + .await + .expect("an exact empty backup should contribute its epoch fence"); + assert_eq!(floor.leader_epoch, 9); + assert_eq!(floor.next_cycle, 12); + assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence); + } + } +} + +#[tokio::test] +async fn scanner_usage_floor_recovery_resumes_after_marker_only_crash_point() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let original = rc3_legacy_empty_usage_fence(Some(7)); + store.objects.lock().await.insert(primary_key.clone(), original.clone()); + store.revisions.lock().await.insert(primary_key.clone(), 1); + store.fail_put_number.lock().await.insert(primary_key, 1); + + persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect_err("injected primary CAS failure should leave recovery pending"); + assert!(read_config(store.clone(), DATA_USAGE_RECOVERY_PATH.as_str()).await.is_ok()); + assert_eq!( + read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("legacy primary should remain after the failed CAS"), + original + ); + + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("the durable marker should resume the primary conversion"); + assert_eq!(floor.leader_epoch, 7); + assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence); + let recovered = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("recovered bootstrap should replace the legacy primary"); + assert!(data_usage_info_is_bootstrap_pending( + &serde_json::from_slice(&recovered).expect("recovered bootstrap should decode") + )); +} + +#[tokio::test] +async fn scanner_usage_floor_recovery_reconciles_marker_post_commit_error() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + let marker_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_RECOVERY_PATH.as_str()); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store.revisions.lock().await.insert(primary_key, 1); + store.error_after_commit_put_number.lock().await.insert(marker_key, 1); + + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("a committed recovery marker should reconcile after an ambiguous error"); + assert_eq!(floor.leader_epoch, 7); + assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence); + assert!(read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await.is_ok()); +} + +#[tokio::test] +async fn scanner_usage_floor_recovery_reconciles_marker_delete_post_commit_error() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store.revisions.lock().await.insert(primary_key, 1); + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("legacy empty primary should enter recovery"); + let mut cycle = CurrentCycle::default(); + let mut revision = DataUsageCacheRevision::Missing; + let mut leader_epoch = floor.leader_epoch; + let (allow_pending, cycle_reset_policy) = prepare_cycle_for_usage_floor_bootstrap(&mut cycle, floor, state); + assert!( + claim_scanner_leadership( + &CancellationToken::new(), + store.clone(), + &mut cycle, + &mut revision, + &mut leader_epoch, + allow_pending, + cycle_reset_policy, + ) + .await + ); + store + .error_after_commit_deletes + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_RECOVERY_PATH.as_str())); + + complete_legacy_empty_usage_floor_recovery(store.clone(), leader_epoch) + .await + .expect("a committed marker delete should reconcile after an ambiguous error"); + assert!(matches!( + read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); + assert_eq!(scanner_cycle_recovery_status().state, "healthy"); +} + +#[tokio::test] +#[serial] +async fn scanner_usage_floor_recovery_retry_budget_uses_marker_epoch_identity() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store.revisions.lock().await.insert(primary_key.clone(), 1); + persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("legacy empty primary should enter recovery"); + assert!(record_scanner_cycle_recovery_retry(3)); + let first_detected = scanner_cycle_recovery_status().first_detected_at_unix_secs; + + let primary = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("recovered bootstrap should exist"); + let mut pending = serde_json::from_slice::(&primary).expect("recovered bootstrap should decode"); + pending.scanner_epoch = Some(8); + store.objects.lock().await.insert( + primary_key.clone(), + serde_json::to_vec(&pending).expect("claimed bootstrap should encode"), + ); + *store.revisions.lock().await.entry(primary_key).or_insert(1) += 1; + + let (floor, state) = persisted_usage_floor_for_startup(store, true) + .await + .expect("claimed bootstrap should retain its recovery identity"); + assert_eq!(floor.leader_epoch, 8); + assert_eq!(state, PersistedUsageFloorStartup::RecoveredLegacyEmptyFence); + let status = scanner_cycle_recovery_status(); + assert_eq!(status.leader_epoch, Some(7)); + assert_eq!(status.retry_count, 3); + assert_eq!(status.first_detected_at_unix_secs, first_detected); + clear_legacy_empty_usage_floor_recovery_status(); +} + +#[tokio::test] +async fn scanner_usage_floor_rejects_noncanonical_empty_fence() { + let store = Arc::new(MemoryConfigStore::default()); + let mut value = serde_json::from_slice::(&rc3_legacy_empty_usage_fence(Some(7))) + .expect("legacy fixture should decode"); + value + .as_object_mut() + .expect("legacy fixture should be an object") + .insert("future_field".to_string(), serde_json::Value::Bool(true)); + store.objects.lock().await.insert( + memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), + serde_json::to_vec(&value).expect("noncanonical fixture should encode"), + ); + + let err = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect_err("unknown legacy fields must not be recovered as an empty baseline"); + assert!(err.to_string().contains("no authoritative baseline")); + assert!(matches!( + read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn scanner_usage_floor_rejects_zero_or_exhausted_empty_fence_epoch() { + for epoch in [0, u64::MAX - 1, u64::MAX] { + let store = Arc::new(MemoryConfigStore::default()); + let primary = rc3_legacy_empty_usage_fence(Some(epoch)); + store + .objects + .lock() + .await + .insert(memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()), primary.clone()); + + persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect_err("an unclaimable legacy epoch must remain fail-closed"); + assert_eq!( + read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("rejected legacy floor should remain unchanged"), + primary + ); + assert!(matches!( + read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); + } +} + +#[tokio::test] +async fn scanner_usage_floor_recovery_does_not_overwrite_concurrent_authoritative_snapshot() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store.revisions.lock().await.insert(primary_key.clone(), 1); + let mut authoritative = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + authoritative.scanner_epoch = Some(8); + authoritative.scanner_cycle = Some(11); + let authoritative = serde_json::to_vec(&authoritative).expect("authoritative usage should encode"); + store + .interleaving_puts + .lock() + .await + .insert(primary_key, (1, authoritative.clone())); + + persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect_err("recovery CAS must lose to a concurrent authoritative snapshot"); + assert_eq!( + read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("concurrent authoritative usage should remain"), + authoritative + ); + + let (floor, state) = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect("the concurrent authoritative snapshot should win on retry"); + assert_eq!(floor.leader_epoch, 8); + assert_eq!(floor.next_cycle, 12); + assert_eq!(state, PersistedUsageFloorStartup::Authoritative); + assert!(matches!( + read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await, + Err(EcstoreError::ConfigNotFound) + )); +} + +#[tokio::test] +async fn scanner_usage_floor_recovery_rejects_concurrent_authoritative_epoch_regression() { + let store = Arc::new(MemoryConfigStore::default()); + let primary_key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_OBJ_NAME_PATH.as_str()); + store + .objects + .lock() + .await + .insert(primary_key.clone(), rc3_legacy_empty_usage_fence(Some(7))); + store.revisions.lock().await.insert(primary_key.clone(), 1); + let mut stale = complete_usage_with_bucket_count(Some(std::time::SystemTime::UNIX_EPOCH), 0); + stale.scanner_epoch = Some(6); + stale.scanner_cycle = Some(11); + let stale = serde_json::to_vec(&stale).expect("stale authoritative usage should encode"); + store.interleaving_puts.lock().await.insert(primary_key, (1, stale.clone())); + + persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect_err("recovery CAS must lose to the concurrent writer"); + let retry_error = persisted_usage_floor_for_startup(store.clone(), true) + .await + .expect_err("the recovery marker must fence an older authoritative winner"); + assert!(retry_error.to_string().contains("older than the required recovery fence")); + assert_eq!( + read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) + .await + .expect("stale concurrent snapshot should not be rewritten without a new scan"), + stale + ); + assert!(read_config(store, DATA_USAGE_RECOVERY_PATH.as_str()).await.is_ok()); +} + +#[test] +#[serial] +fn scanner_usage_floor_failure_is_exposed_and_cleared() { + record_scanner_usage_floor_failure("persisted usage floor is invalid".to_string()); + let blocked = scanner_cycle_recovery_status(); + assert_eq!(blocked.path, DATA_USAGE_OBJ_NAME_PATH.as_str()); + assert_eq!(blocked.state, "usage_floor_load_failed"); + assert_eq!(blocked.classification.as_deref(), Some("usage_floor_load_failed")); + assert!(blocked.retryable); + assert_eq!(blocked.reason.as_deref(), Some("persisted usage floor is invalid")); + let first_detected = blocked.first_detected_at_unix_secs; + + assert!(record_scanner_cycle_recovery_retry(2)); + record_scanner_usage_floor_failure("persisted usage floor remains invalid".to_string()); + let retried = scanner_cycle_recovery_status(); + assert_eq!(retried.retry_count, 2); + assert_eq!(retried.first_detected_at_unix_secs, first_detected); + + clear_scanner_usage_floor_failure(); + let healthy = scanner_cycle_recovery_status(); + assert_eq!(healthy.state, "healthy"); + assert_eq!(healthy.path, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + assert_eq!(healthy.classification, None); +} + +#[test] +#[serial] +fn scanner_usage_floor_recovery_stays_retryable_until_claim_cleanup() { + record_legacy_empty_usage_floor_recovery_pending(7); + let pending = scanner_cycle_recovery_status(); + assert_eq!(pending.state, "usage_floor_recovery_pending"); + assert_eq!(pending.classification.as_deref(), Some("legacy_empty_usage_floor")); + assert_eq!(pending.leader_epoch, Some(7)); + assert!(pending.retryable); + assert_eq!(pending.quarantine_path.as_deref(), Some(DATA_USAGE_RECOVERY_PATH.as_str())); + let first_detected = pending.first_detected_at_unix_secs; + + assert!(record_scanner_cycle_recovery_retry(3)); + record_legacy_empty_usage_floor_recovery_pending(7); + let retried = scanner_cycle_recovery_status(); + assert_eq!(retried.retry_count, 3); + assert_eq!(retried.first_detected_at_unix_secs, first_detected); + + clear_legacy_empty_usage_floor_recovery_status(); + assert_eq!(scanner_cycle_recovery_status().state, "healthy"); +} + +#[test] +#[serial] +fn scanner_cache_cycle_ahead_is_visible_until_a_later_scan_clears_it() { + record_scanner_cache_cycle_ahead(0, 17_118, 8); + let pending = scanner_cycle_recovery_status(); + assert_eq!(pending.state, "cache_cycle_ahead"); + assert_eq!(pending.classification.as_deref(), Some("cache_cycle_ahead")); + assert_eq!(pending.generation, Some(17_118)); + assert_eq!(pending.leader_epoch, Some(8)); + assert!(pending.retryable); + assert_eq!(pending.max_retries, 0); + assert_eq!( + pending.reason.as_deref(), + Some("persisted scanner cache cycle 17118 is ahead of requested cycle 0") + ); + let first_detected = pending.first_detected_at_unix_secs; + + record_scanner_cache_cycle_ahead(0, 17_118, 8); + let observed_again = scanner_cycle_recovery_status(); + assert_eq!(observed_again.retry_count, 0); + assert_eq!(observed_again.first_detected_at_unix_secs, first_detected); + + assert!(record_scanner_cycle_recovery_retry(4)); + assert_eq!(scanner_cycle_recovery_status().retry_count, 0); + + record_scanner_cache_cycle_recovery_attempt(); + record_scanner_cache_cycle_recovery_attempt(); + let retried = scanner_cycle_recovery_status(); + assert_eq!(retried.retry_count, 2); + assert!(retried.retryable); + + update_scanner_cache_cycle_recovery_status( + 0, + 8, + None, + Some(ScannerCyclePreCommitOutcome::Deferred(ScannerCycleDeferReason::DataMovement)), + false, + ); + assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("cache_cycle_ahead")); + + update_scanner_cache_cycle_recovery_status(17_118, 8, None, None, false); + assert_eq!(scanner_cycle_recovery_status().classification.as_deref(), Some("cache_cycle_ahead")); + + update_scanner_cache_cycle_recovery_status(17_118, 8, None, None, true); + assert_eq!(scanner_cycle_recovery_status().state, "healthy"); +} + +#[tokio::test] +#[serial] +async fn scanner_usage_floor_failure_clears_stale_leader_liveness() { + record_scanner_cycle_schedule_role("leader"); + global_metrics().record_scanner_leader_liveness("acquired", true, "").await; + + finish_scanner_leader_iteration(false, "usage_floor_load_failed", "invalid floor".to_string()).await; + + assert_eq!(scanner_cycle_schedule_status().execution_role, "unknown"); + let report = global_metrics().report().await; + assert_eq!(report.leader_lock_state, "usage_floor_load_failed"); + assert!(!report.leader_lock_held_by_this_process); + assert_eq!(report.leader_lock_last_error, "invalid floor"); + + global_metrics().record_scanner_leader_liveness("acquired", true, "").await; + finish_scanner_leader_iteration(true, "stopped", "lock lost before classification".to_string()).await; + let report = global_metrics().report().await; + assert_eq!(report.leader_lock_state, "stopped"); + assert!(!report.leader_lock_held_by_this_process); + assert_eq!(report.leader_lock_last_error, "lock lost before classification"); +} + #[tokio::test] async fn scanner_usage_floor_recovers_from_incomplete_v2_primary_using_fenced_backup() { let store = Arc::new(MemoryConfigStore::default()); @@ -2039,7 +2925,7 @@ async fn scanner_usage_floor_rejects_backup_older_than_incomplete_v2_primary() { let err = persisted_usage_floor_for_startup(store, true) .await .expect_err("an older backup must not cross the incomplete primary epoch fence"); - assert!(err.to_string().contains("no authoritative baseline")); + assert!(err.to_string().contains("older than the required recovery fence")); } #[tokio::test] @@ -2067,7 +2953,7 @@ async fn scanner_usage_floor_rejects_older_legacy_primary_after_incomplete_v2_pr let err = persisted_usage_floor_for_startup(store, true) .await .expect_err("an older legacy baseline must not cross the incomplete v2 epoch fence"); - assert!(err.to_string().contains("no authoritative baseline")); + assert!(err.to_string().contains("older than the required recovery fence")); } #[tokio::test] @@ -2389,7 +3275,7 @@ fn missing_usage_floor_discards_unfenced_cycle_progress() { assert_eq!( prepare_cycle_for_usage_floor_bootstrap(&mut cycle, PersistedUsageFloor::default(), PersistedUsageFloorStartup::Missing,), - (true, true) + (true, ScannerCycleResetPolicy::ResetAll) ); assert_eq!(cycle.next, 0); assert_eq!(cycle.current, 0); @@ -2402,7 +3288,7 @@ fn missing_usage_floor_discards_unfenced_cycle_progress() { PersistedUsageFloor::default(), PersistedUsageFloorStartup::BootstrapPending, ), - (true, true) + (true, ScannerCycleResetPolicy::ResetAll) ); assert_eq!(cycle.next, 0); } @@ -2423,7 +3309,7 @@ fn fenced_usage_bootstrap_retains_partial_cycle_progress() { }, PersistedUsageFloorStartup::BootstrapPending, ), - (true, false) + (true, ScannerCycleResetPolicy::None) ); assert_eq!(cycle.next, 12); @@ -2436,7 +3322,7 @@ fn fenced_usage_bootstrap_retains_partial_cycle_progress() { }, PersistedUsageFloorStartup::Authoritative, ), - (false, false) + (false, ScannerCycleResetPolicy::None) ); assert_eq!(cycle.next, 12); } @@ -2467,7 +3353,7 @@ async fn missing_usage_floor_rebuilds_persisted_cycle_before_leadership_claim() .await .expect("stably missing usage floor should admit a bootstrap marker"); assert_eq!(startup, PersistedUsageFloorStartup::Missing); - let (allow_bootstrap_pending, reset_bootstrap_cycle_on_conflict) = + let (allow_bootstrap_pending, cycle_reset_policy) = prepare_cycle_for_usage_floor_bootstrap(&mut cycle_info, usage_floor, startup); apply_persisted_usage_floor(&mut cycle_info, &mut persisted_epoch, usage_floor); initialize_usage_baseline_bootstrap(store.clone()) @@ -2482,7 +3368,7 @@ async fn missing_usage_floor_rebuilds_persisted_cycle_before_leadership_claim() &mut cycle_revision, &mut persisted_epoch, allow_bootstrap_pending, - reset_bootstrap_cycle_on_conflict, + cycle_reset_policy, ) .await ); @@ -2646,6 +3532,11 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore { } objects.remove(&key); revisions.remove(&key); + drop(revisions); + drop(objects); + if self.error_after_commit_deletes.lock().await.remove(&key) { + return Err(EcstoreError::other("injected delete error after commit")); + } Ok(ObjectInfo::default()) } @@ -2840,7 +3731,18 @@ async fn test_leadership_claim_preserves_usage_epoch_floor_across_old_epoch_conf ); let mut persisted_epoch = 8; - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await); + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) .await @@ -2878,7 +3780,18 @@ async fn unfenced_usage_bootstrap_discards_old_epoch_conflict_progress() { ); let mut persisted_epoch = 1; - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, true, true).await); + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + true, + ScannerCycleResetPolicy::ResetAll, + ) + .await + ); let state = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH) .await @@ -2888,6 +3801,84 @@ async fn unfenced_usage_bootstrap_discards_old_epoch_conflict_progress() { assert_eq!(claimed_epoch, 2); } +#[tokio::test] +async fn recovered_usage_bootstrap_claim_conflicts_preserve_the_highest_cycle_number() { + for winner_next in [42_u64, 20_000] { + let store = Arc::new(MemoryConfigStore::default()); + let ctx = CancellationToken::new(); + let mut revision = DataUsageCacheRevision::Missing; + let mut cycle = CurrentCycle { + next: 12, + ..Default::default() + }; + assert!(persist_scanner_cycle_state(&ctx, store.clone(), &mut cycle, &mut revision, 1).await); + seed_usage_snapshot_for_leadership_claim(&store).await; + + cycle = CurrentCycle { + current: 17_117, + next: 17_118, + cycle_completed: vec![Utc::now()], + started: Utc::now(), + }; + let key = memory_config_key(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str()); + let winner = CurrentCycle { + current: winner_next.saturating_sub(1), + next: winner_next, + cycle_completed: vec![Utc::now()], + started: Utc::now(), + }; + store + .interleaving_puts + .lock() + .await + .insert(key, (2, encode_scanner_cycle_state(&winner, 7).expect("conflict winner should encode"))); + + let mut persisted_epoch = 7; + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + true, + ScannerCycleResetPolicy::ResetCoveragePreservingNext, + ) + .await + ); + + let persisted = read_config(store, DATA_USAGE_BLOOM_NAME_PATH.as_str()) + .await + .expect("recovered leadership claim should remain durable"); + let (persisted_cycle, claimed_epoch) = + decode_scanner_cycle_state(&persisted).expect("recovered leadership claim should decode"); + assert_eq!(persisted_cycle.next, 17_118_u64.max(winner_next)); + assert_eq!(persisted_cycle.current, 0); + assert!(persisted_cycle.cycle_completed.is_empty()); + assert_eq!(claimed_epoch, 8); + } +} + +#[test] +fn recovered_usage_cache_reset_keeps_cycle_and_leader_regression_guards() { + let source = DataUsageCacheSource::new(0, 0); + let digest = DataUsageScanPlanDigest([9; 32]); + let mut newer_cycle = DataUsageCache::default(); + newer_cycle.info.next_cycle = 17_119; + assert_eq!( + newer_cycle.prepare_for_scan(DATA_USAGE_ROOT, 17_118, 8, source, digest, true), + DataUsageCachePrepareOutcome::RejectedNewerCycle + ); + + let mut newer_leader = DataUsageCache::default(); + newer_leader.info.next_cycle = 17_118; + newer_leader.info.leader_epoch = 9; + assert_eq!( + newer_leader.prepare_for_scan(DATA_USAGE_ROOT, 17_118, 8, source, digest, true), + DataUsageCachePrepareOutcome::RejectedNewerLeader + ); +} + #[tokio::test] async fn test_leadership_claim_rejects_terminal_epoch() { let store = Arc::new(MemoryConfigStore::default()); @@ -2899,7 +3890,18 @@ async fn test_leadership_claim_rejects_terminal_epoch() { }; let mut persisted_epoch = u64::MAX - 1; - assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await); + assert!( + !claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); assert_eq!(persisted_epoch, u64::MAX - 1); assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); } @@ -2915,7 +3917,18 @@ async fn scanner_defers_leadership_when_usage_snapshots_are_stably_absent() { }; let mut persisted_epoch = 0; - assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await); + assert!( + !claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); assert!(read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()).await.is_err()); } @@ -2931,9 +3944,31 @@ async fn usage_bootstrap_pending_unblocks_first_leadership_claim() { let mut revision = DataUsageCacheRevision::Missing; let mut cycle = CurrentCycle::default(); let mut persisted_epoch = 0; - assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false,).await); + assert!( + !claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); assert!(read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, true, true).await); + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + true, + ScannerCycleResetPolicy::ResetAll, + ) + .await + ); let usage = read_config(store, DATA_USAGE_OBJ_NAME_PATH.as_str()) .await @@ -3021,7 +4056,18 @@ async fn leadership_claim_defers_on_corrupt_usage_baseline_without_bloom_write() }; let mut persisted_epoch = 0; - assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await); + assert!( + !claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); } @@ -3041,7 +4087,18 @@ async fn leadership_claim_defers_on_unidentified_usage_baseline_without_bloom_wr }; let mut persisted_epoch = 0; - assert!(!claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await); + assert!( + !claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); assert!(read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await.is_err()); } @@ -3063,7 +4120,18 @@ async fn test_leadership_claim_confirms_commit_after_returned_error() { let mut persisted_epoch = 0; seed_usage_snapshot_for_leadership_claim(&store).await; - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await); + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); let state = read_config(store.clone(), &DATA_USAGE_BLOOM_NAME_PATH) .await @@ -3119,7 +4187,18 @@ async fn test_leadership_claim_usage_fence_rejects_old_inflight_writer() { ..Default::default() }; let mut persisted_epoch = 4; - assert!(claim_scanner_leadership(&ctx, store.clone(), &mut cycle, &mut revision, &mut persisted_epoch, false, false).await); + assert!( + claim_scanner_leadership( + &ctx, + store.clone(), + &mut cycle, + &mut revision, + &mut persisted_epoch, + false, + ScannerCycleResetPolicy::None, + ) + .await + ); let (fenced_data, fenced_revision) = read_config_with_revision(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str()) .await @@ -3185,7 +4264,7 @@ async fn cycle_budget_lease_takeover_rejects_old_generation() { &mut replacement_revision, &mut replacement_epoch, false, - false, + ScannerCycleResetPolicy::None, ) .await ); diff --git a/crates/scanner/src/scanner/usage_store.rs b/crates/scanner/src/scanner/usage_store.rs index 9d181d52a..b9dd53271 100644 --- a/crates/scanner/src/scanner/usage_store.rs +++ b/crates/scanner/src/scanner/usage_store.rs @@ -420,7 +420,17 @@ where break 'updates; }; let authoritative = match serde_json::from_slice::(&authoritative_data) { - Ok(info) if data_usage_info_has_persisted_baseline_identity(&info) => info, + // The bootstrap placeholder is a valid baseline identity: on a + // site that has never converged (every cycle superseded by a + // sustained write stream, #6852) it is the only authoritative + // object that will ever exist, and refusing it here means the + // observed snapshot — the only usage data such a site can + // produce — is never published at all. + Ok(info) + if data_usage_info_has_persisted_baseline_identity(&info) || data_usage_info_is_bootstrap_pending(&info) => + { + info + } Ok(_) => { error!( target: "rustfs::scanner", diff --git a/docs/architecture/compat-cleanup-register.md b/docs/architecture/compat-cleanup-register.md index e40a8fade..8b56a5939 100644 --- a/docs/architecture/compat-cleanup-register.md +++ b/docs/architecture/compat-cleanup-register.md @@ -12,6 +12,8 @@ for later deletion. ## Open Items +- `backlog-2102` rc.2/rc.3 empty scanner usage floor recovery: old DeleteBucket cleanup could synthesize an empty incomplete v2 usage primary/backup before leadership added an epoch, while newer scanners require a durable authoritative baseline identity. New scanners recognize only that exact serialized empty-fence shape, preserve its epoch through a CAS-protected recovery marker, and rebuild namespace coverage without treating zero usage as authoritative. Remove this recovery path and marker after rc.2 and rc.3 are no longer supported direct-upgrade sources. +- `s3gate-metadata-xml` persisted bucket XML migration: mixed-version site-replication peers, retained `.metadata.bin` objects, and backup archives can all carry XML written by the s3s codec, so the gateway migration must keep the legacy codec available until every stored form has crossed a verified rewrite boundary. Remove the legacy s3s parser and serializer only after the minimum supported direct-upgrade release reads and writes every persisted XML configuration family through the gateway codec, the four-way D1-D5 gate has remained clean for one full support window, every supported mixed-version site-replication topology has completed its writer upgrade, and migration tooling has verified or rewritten every retained bucket metadata object and restorable backup archive. - `rustfs-6339` legacy bucket policy ID casing: earlier RustFS releases persisted the top-level policy identifier as "ID", while current writes use the S3-compatible "Id" spelling. Readers accept both spellings so retained bucket metadata remains usable after upgrade. Remove the legacy alias after migration tooling has rewritten every retained bucket policy using "ID". - `table-publication-fence-v1` table publication fencing: nodes that predate table and table-bucket publication fences can mutate live files while a new node is publishing a catalog pointer. New nodes retain exact object guards until the operator confirms that every serving node uses the new fences. Fleet confirmation also requires non-overlapping active warehouse prefixes and lifecycle workers that exclude table buckets. Remove the exact live-file fallback and the fleet-confirmation gate after the minimum supported RustFS release acquires table fences for registered-table mutations and table-bucket fences for unresolved-prefix mutations. - `table-catalog-strong-snapshot-v1` durable strong catalog snapshot compatibility: version 1 writes continue during mixed-version rollout until operators confirm that every serving node reads version 2, and version 1 table/view identifier collisions remain available only for cleanup. Remove version 1 writes and collision cleanup after the minimum supported RustFS release reads version 2 and every retained durable strong snapshot is collision-free and has been upgraded to version 2. diff --git a/docs/architecture/s3-tables-support-matrix.md b/docs/architecture/s3-tables-support-matrix.md index 1dfd412a3..309bd1573 100644 --- a/docs/architecture/s3-tables-support-matrix.md +++ b/docs/architecture/s3-tables-support-matrix.md @@ -64,7 +64,7 @@ catalog extension. | Catalog config | Supported | `GET /v1/config` advertises RustFS catalog defaults and only the supported OpenAPI REST paths in `endpoints`. RustFS administration, maintenance, migration, diagnostics, refs, and metadata-location extensions remain available but are not presented as standard Iceberg REST endpoints. | | Table bucket discovery | Supported | `PUT` and `GET /v1/buckets/{warehouse}` enable and inspect table bucket state. | | Namespaces | Supported | Create, list, load, existence check, and drop namespace routes are registered on both catalog prefixes. List responses support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Namespace identifiers are limited to 512 ASCII characters so persisted paths and stateless continuation tokens remain bounded. | -| Tables | Supported | Create, register, list, load, existence check, commit, metadata-location get/update, and drop table routes are registered on both catalog prefixes. Table and view listings support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Commit identifiers must match the URL resource; unknown requirements, updates, and snapshot operations fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. Standard statistics, partition statistics, and schema/spec cleanup updates are accepted. | +| Tables | Supported | Create, register, list, load, existence check, rename, commit, metadata-location get/update, and drop table routes are registered on both catalog prefixes. Object-backed rename uses a bucket-scoped persistent fence, recoverable intent, and conditional publication of the destination, source tombstone, and warehouse index; the source identifier is reusable only through an ETag-conditional tombstone replacement. Table and view listings support Iceberg REST `pageSize`/`pageToken` pagination with context-bound tokens and bounded catalog-store reads. Commit identifiers must match the URL resource; unknown requirements, updates, and snapshot operations fail as bad requests; staged create, register overwrite, purge-on-drop, and v3-only encryption-key updates return an explicit unsupported-operation response. Standard statistics, partition statistics, and schema/spec cleanup updates are accepted. | | Commit CAS | Supported | Single-table commits validate base metadata, expected version token, referenced object existence, warehouse scope, and Iceberg commit requirements before advancing the current metadata pointer. Externally supplied metadata transitions preserve monotonic column, partition, and sequence assignment watermarks and immutable definitions for retained schemas, partition specs, sort orders, and snapshots. Standard commits preserve the normal commit-token file name and use an immutable-table-scoped fallback when rename followed by source-name reuse would otherwise collide at the same generation and commit ID. The catalog does not advertise `idempotency-key-lifetime`; clients must treat standard mutation-wide `Idempotency-Key` semantics as unsupported. | | Commit recovery | Supported | Commit log, idempotency lookup, diagnostics, and recovery routes expose staged/finalization gaps and repair safe idempotency gaps without moving the table pointer. | | Snapshot refs | Supported | Refs can be listed, created or replaced, and deleted through catalog commits. `main` is protected and refs with explicit retention require forced delete. | diff --git a/docs/architecture/unified-object-generation.md b/docs/architecture/unified-object-generation.md index 40656c429..a5bf7b4e5 100644 --- a/docs/architecture/unified-object-generation.md +++ b/docs/architecture/unified-object-generation.md @@ -30,73 +30,120 @@ lifetime**. Left independent, they diverge and punch through one another: different monotonic sources, cannot be compared — a late commit fenced on one plane can still settle quota on the other. -The fix is a single authority with one monotonic source, one persistence +The fix is a single authority with one selected comparison rule, one persistence semantics, and one transport binding, that every consumer references rather than re-derives. -## The authority (single source) +## Target authority and the current bounded token -**The per-object fencing epoch defined by #1312 is the sole generation -authority.** No other monotonic counter, timestamp, or random token may stand in -for generation. +The target contract still requires **one per-object commit identity** consumed +by commit fencing, read leases, cleanup, prepared reads, and quota settlement. +No consumer may mint a second value and call it the same generation. -- The distributed lock grant returns a monotonic `epoch` for the object key. - Acquiring the object write-lock is the only way to mint a new generation. -- The epoch travels down the authoritative commit path (with - `RenameDataRequest` / the local `DiskAPI` call) and is compared at each disk's - atomic `xl.meta` commit point, rejecting stale epochs. It adds no extra - network round trip (#1312 implementation clause 2). -- Every consumer in the table below **binds** this epoch. None defines its own. +The concrete ordering semantics are not settled, however. The original #1326 +proposal requires a total-ordered, monotonic lock-grant epoch. Current main does +not implement that proposal. PR #6077 instead implements an opaque transaction +identity: -### Monotonicity persistence semantics +- `assign_object_transaction_epoch` mints a random non-nil UUID for PUT and + CompleteMultipartUpload when the object-transaction gate is active. +- The UUID is written through `FileInfo::set_object_transaction_epoch` into the + dual internal metadata map. +- The coordinator reads the current UUID (or `Absent`) and revalidates exact + equality immediately before `rename_data`. +- Old-data cleanup receipts carry the committed UUID and reconciliation deletes + only when the receipt UUID still equals the current object UUID. -The epoch must be **monotonic across lock-plane restart and failover** -(#1312 B4). Today the distributed lock entry is in-memory only -(`crates/lock/src/distributed_lock.rs` has no persistence path), so a lock-service -restart resets the counter to zero: a new writer draws epoch 1 while disks have -already observed epoch 100, producing either a permanent write rejection or a -fence *inversion*. To prevent this, the epoch must be one of: +This is a useful **equality-CAS fence and cleanup identity**. It is not a +monotonic epoch, is not minted by the distributed lock grant, and is not +compared atomically at each disk's `xl.meta` commit point. Until the decision +below is made, documents and issue checklists must call it the *object +transaction UUID* rather than use it as proof that the target generation +authority exists. -1. **Quorum-persisted** before it is handed to a writer, or -2. **Derived from a durable monotonic source** — a `(term, counter)` pair where - `term` advances on every lock-service leadership change and is itself durable, - so the composite never regresses even when `counter` resets. +### Ordering decision required -The comparison at the disk commit point is on the full composite; a lower -`(term, counter)` is always rejected. +Before #1313, #1314, or a unified quota binding can consume the authority, one +of these contracts must be selected and tested: + +1. **Total-ordered fencing epoch.** A lock grant returns a durable per-object + `(term, counter)` (or another specified total-order type). Every disk rejects + a lower epoch at the atomic metadata commit point. The value never regresses + across lock-plane restart, failover, or minority recovery. +2. **Opaque commit-generation identity.** Consumers compare only exact identity; + no `<` / `>` semantics are permitted. The authoritative commit must perform + an atomic expected-generation CAS, and all lease, cleanup, prepared-read, and + quota contracts must be rewritten in terms of “references this exact + generation,” not “lower/newer generation.” + +The current UUID implementation proves neither a durable total order nor a +per-disk atomic expected-generation CAS, so it does not by itself decide between +these options. + +### Persistence semantics if total order is selected + +A total-ordered epoch must be **monotonic across lock-plane restart and +failover**. The distributed lock entry remains in-memory; deriving a counter +from that entry alone would reset it after restart. The chosen source therefore +must be either quorum-persisted before grant or derived from a durable term whose +full `(term, counter)` comparison cannot regress. This requirement does not +apply to an opaque UUID as an ordering rule; the opaque alternative instead +requires atomic expected-identity comparison and durable crash recovery. ## Consumer binding contracts +### Current implementation snapshot (2026-08-31, main@9ee7b1221) + +This table separates code that exists on current main from the target contract. +Closing an implementation issue does not imply that its token is already the +unified authority. + +| Surface | Current main | Gap against this contract | +|---|---|---| +| PUT / CompleteMultipartUpload (#1312, PR #6077) | Owned commit tasks retain the relevant guards; an opt-in gate persists a random object transaction UUID and performs a quorum metadata equality recheck before rename | no lock-grant monotonic source; no per-disk atomic epoch/CAS comparison; the live proof is the reused remote-version-state fleet proof, not a dedicated generation capability | +| Old-data cleanup (#1323, PR #6077) | JSON receipt carries transaction UUID, old dir, and committed dir; reconciliation is gated and requires UUID equality | no generation-bound read lease is consulted, so this is crash cleanup fencing rather than the full #1313/#1323 lease lifetime contract | +| Read lease (#1313) | short-term streaming/multipart path holds the namespace read lock through EOF/drop; deterministic part-boundary coverage is tracked by PR #6887 | no cross-node generation-bound lease registry, TTL reclamation, or crash recovery | +| Prepared pool read (#1314) | PR #6889 tracks a pool-local prepared identity and fails closed/refetches when pool state changes | not merged on this snapshot; pool-local identity is not a cross-pool generation authority; black-box mixed-version/rebalance coverage remains open | +| Quota reservation (#1318) | durable per-bucket ledger plus independent snapshot-lease mutation-fence tokens; issue closed after PR #6058 | reservation and settle are not bound to the object transaction UUID; the independent fence must be reconciled with the selected authority or explicitly proven to be a separate, non-generation arbitration domain | +| Internode integrity (#1327, #1541, #1542) | v2/v3 HMAC binds audience, exact method, timestamp, nonce, canonical body digest, and receiver boot epoch; body-bound RPC policy has exact-set coverage | signature/body/replay strict switches remain default-off rollout gates; generation enforcement cannot treat an unrelated fleet-version proof as proof that these strict contracts converged | + | Consumer | How it binds generation | Key invariant | |---|---|---| -| #1312 commit fence | epoch compared at three disk-write points — `rename`, rollback `delete`, and `commit_rename_data_dir` cleanup | stale epoch rejected on **all** disks; an already-ACK'd write is never rolled back | -| #1313 read lease | lease binds the generation observed at read time; GC runs only after every lease referencing that generation is released | lease is visible across nodes; a crashed reader's lease is reclaimed by TTL | -| #1323 old-dir GC | cleanup job carries the committed generation; before deleting `old_dir` it confirms no lease referencing a lower generation still points at it | `old_dir != committed_dir`; a still-referenced directory is never deleted | +| #1312 commit fence | selected generation is checked at `rename`, rollback restore/delete, and cleanup mutation points using the chosen ordered or exact-CAS rule | a stale writer is rejected on **all** disks; an already-ACK'd write is never rolled back | +| #1313 read lease | lease binds the exact generation observed at read time; GC runs only after every lease referencing that generation is released | lease is visible across nodes; a crashed reader's lease is reclaimed by TTL | +| #1323 old-dir GC | cleanup job carries the committed generation; before deleting `old_dir` it confirms that no lease for the generation owning that directory remains | `old_dir != committed_dir`; a still-referenced directory is never deleted | | #1314 prepared pool read | the `PreparedPoolRead` bundle carries the generation resolved during pool lookup; the chosen pool's reader setup reuses it only after a match | generation mismatch forces a fallback to full metadata fanout | -| #1318 quota reservation | reservation / settle token binds the object generation | a late commit holding an old-generation token cannot settle a newer generation | +| #1318 quota reservation | reservation / settle record binds the exact object generation (and an ordered epoch too, if that option is selected) | a late commit cannot settle quota for a different committed generation | ### Fence coverage is three disk-write points, not one (#1312 B2) -Comparing the epoch at the `rename` commit point alone is insufficient. The +Checking the generation only before the `rename` fanout is insufficient. The authoritative commit sequence is `tmp sync → data-dir rename → xl.meta commit → directory sync` in `crates/ecstore/src/disk/local.rs`, and there are two further detachable disk-write points in `crates/ecstore/src/set_disk/core/io_primitives.rs`: -- **Rollback delete** — on quorum failure each disk runs - `delete_version(undo_write=true)`. A fenced old writer's rollback must also - compare epoch, otherwise it deletes the winner's already-committed version. +- **Rollback restore/delete** — on quorum failure each disk can restore backup + metadata or delete the failed version. A stale writer's rollback must compare + the expected generation, otherwise it can overwrite or delete the winner's + already-committed metadata. - **`commit_rename_data_dir`** — a cancel-then-detach disk-write point; the coordinator's "reap all child tasks" must explicitly include it so a cancelled writer cannot bypass fence/lease and keep deleting directories. -If the epoch is validated only at the `xl.meta` commit point, a fenced writer +If generation is validated only after data-dir rename, a fenced writer may already have renamed its data-dir into the object path, leaving a staged orphan. Either move the fence ahead of the data-dir rename, or declare that orphan an acceptable residue accounted for by GC metrics — the white-box acceptance "no background disk write after release" must be rewritten accordingly. +Current PR #6077 performs a quorum metadata equality recheck before rename and +reaps owned commit work. That closes important cancellation windows, but it is +not evidence that every disk mutation above performs the selected generation +comparison atomically. The writer inventory and per-point CAS/ordering proof +remain acceptance work for #1326 even though #1312 is closed. + ### Post-commit convergence is orthogonal to the fence (#1321) The same `SetDisks::rename_data` path already returns a post-commit @@ -125,36 +172,40 @@ internode RPC bodies. Every such flow must be signature-bound. ### RPC signature binding (#1312 B3, #1313, #1318) -**Requirement.** The RPC body digest carrying a generation/epoch/token must be -folded into the RPC HMAC, binding `method + object key + generation`, and the -request must carry a nonce / one-shot identifier inside the 300s replay window. -The nonce is only meaningful if the **receiver enforces it**: each disk keeps a -bounded seen-nonce cache covering the 300s freshness window and rejects any -request whose nonce was already observed. A nonce that is merely transmitted but -not checked provides no replay protection. +**Requirement.** The canonical body carrying a generation or derived token must +be folded into the internode HMAC. The authenticated scope binds the target +audience, exact service/method, timestamp, nonce, canonical body digest, and +receiver replay epoch. The receiver must consume the nonce in a bounded replay +cache; transmitting a nonce without receiver-side consumption is not replay +protection. -This generalizes the existing `walk_dir` pattern: `walk_dir` computes a -`Sha256` of the request body and places it in the signed URL query as -`walk_dir_body_sha256` -(`crates/ecstore/src/cluster/rpc/internode_data_transport.rs:187`), so the body -digest is transitively covered by the URL signature. New generation-bearing RPCs -adopt the same `*_body_sha256` mechanism. +**Current substrate (verified on main).** The original legacy-only description +is obsolete: -**Current gap (verified).** The internode HMAC covers only -`{path_and_query}|{method}|{timestamp}` -(`signature_payload`, `crates/ecstore/src/cluster/rpc/http_auth.rs:75-83`). It -binds neither the request body nor a nonce, and the 300s freshness window has no -one-shot guard. Without the binding above: +- RPC v2 binds target audience, exact method, POST, timestamp, nonce, and body + digest. +- Body-bound policy covers mutating disk RPCs including `RenameData`; its + versioned canonical body includes every `RenameDataRequest` field, so the + `FileInfo` metadata map carrying the transaction UUID is authenticated. +- PR #5425 extended canonical-body enforcement to implemented non-disk mutating + unary RPCs and added an exact policy/handler coverage partition. +- PR #5455 added the receiver boot epoch and rotating replay scope so signatures + captured before a receiver restart are rejected after capability convergence. -- An on-path or replaying attacker can inject a high epoch (e.g. `u32::MAX`) and - **permanently fence out** a key's legitimate writes — monotonicity only - rejects *low/old* epochs, never a forged-high one. -- A captured lease/reservation token can be replayed within 300s to block - old-dir GC (storage-exhaustion DoS) or to double-reserve / prematurely settle - quota. +The rollout switches +`RUSTFS_INTERNODE_RPC_SIGNATURE_STRICT`, +`RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT`, and +`RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT` remain default-off for rolling +compatibility. The compatibility register and fallback/overflow metrics govern +their fleet convergence. Therefore a generation capability may claim strong +transport binding only when the relevant strict modes have converged; the +object-transaction gate's current remote-version-state fleet proof is not, by +itself, proof of RPC signature/body/replay strictness. -Acceptance for each consumer must include: "a replayed old signature to a -different method, and a forged-high-epoch request, are both rejected." +Acceptance for each generation consumer includes method substitution, canonical +body tamper, nonce replay, receiver restart, and stripped-strict-metadata +negative tests. Generation rollout must also record which strict-mode evidence +authorized enforcement. ### Encoding contract (#1312 B1) @@ -167,11 +218,15 @@ The on-disk persistence of generation must not perturb the file format: `xl.meta` unreadable by rolling-upgrade old RustFS nodes and by MinIO — a total read failure, not a graceful downgrade. - **Do not add generation as a `FileInfo` struct field.** The internode RPC layer serializes `FileInfo` with two different msgpack encoders depending on the call site: `encode_msgpack` uses rmp_serde's default **array** (positional) encoding for the `read_version` family, where a new positional field breaks decode across mixed-version nodes; `encode_msgpack_named` uses `.with_struct_map()` (named-map) encoding for `rename_data` (`crates/ecstore/src/cluster/rpc/remote_disk.rs`), which is more tolerant but still requires `#[serde(default)]` and MinIO-side agreement. Because a `FileInfo` field would have to be correct under *both* encoders and under the JSON compatibility twin (see "Wire-encoding migration" below), do not add one — use the metadata map, which rides through every encoder unchanged. -- **Where it may live.** Only inside a version's internal metadata **map** - (MinIO skips unknown internal keys and the map encoding is extensible) or in a - per-disk sidecar outside `xl.meta`. If it goes in the metadata map, it must - obey the dual-key contract (`x-rustfs-internal-*` / `x-minio-internal-*`, see - AGENTS.md "Cross-Cutting Domain Invariants"). +- **Where it lives today.** The object transaction UUID uses the version's + internal metadata map under the dual-key contract + (`x-rustfs-internal-*` / `x-minio-internal-*`) via + `set_object_transaction_epoch`. Missing, malformed, nil, or conflicting dual + values fail closed when fencing is active. +- **Sidecars are not an equivalent alternative.** A future sidecar is admissible + only if it commits atomically with `xl.meta` and has a specified crash-recovery + protocol. No such protocol is implemented, so a sidecar cannot be selected by + an implementation issue merely because this document mentions one. - **Regression guard.** Preserve the #4377 real-MinIO `xl.meta` interop regression (the fixture family around `crates/filemeta/src/filemeta.rs`): objects written by a new node must still be readable by old RustFS nodes and @@ -179,82 +234,151 @@ The on-disk persistence of generation must not perturb the file format: ### Wire-encoding migration (JSON → msgpack) interaction -The internode RPC layer is mid-migration from JSON to msgpack binary, and generation-bearing fields must respect that migration window — this is not optional context, it changes how epoch is transported. +The internode RPC layer retains a JSON/msgpack rolling-compatibility window, and +generation-bearing fields must respect it. - **Dual-field transport.** Each dual-encoded RPC field exists twice in `crates/protos/src/node.proto`: a JSON `string` field and a msgpack `bytes _bin` field (e.g. `file_info` #4 alongside `file_info_bin` #7 on `RenameDataRequest`). Senders emit both; receivers `decode_msgpack_or_json` prefer the `_bin` form and fall back to the JSON string only when `_bin` is empty (`crates/ecstore/src/cluster/rpc/remote_disk.rs`). -- **Capability flags, default off.** `rustfs_protos::internode_rpc_msgpack_only()` only drops the redundant JSON copy when both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true` are deliberately enabled after the `record_msgpack_json_fallback` metric reads zero fleet-wide and the convergence runbook is followed. If only the request flag is set, RustFS keeps dual-writing JSON compatibility fields. **Reuse this exact capability + metric-reads-zero model as the mixed-version gate for generation** rather than inventing a parallel handshake; the section above ("Capability negotiation") is layered on top of it, not instead of it. +- **Capability flags, default off.** `rustfs_protos::internode_rpc_msgpack_only()` only drops the redundant JSON copy when both `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=true` are deliberately enabled after the JSON-fallback metric reads zero fleet-wide and the convergence runbook is followed. Generation follows the same default-off, fleet-confirmed, metric-reads-zero rollout discipline, but a msgpack proof is not itself a generation capability proof. - **Generation must ride both encodings during the window.** If epoch lives in the version's internal metadata map, that map is carried inside `FileInfo`, so it is present in both the msgpack `_bin` and JSON copies automatically — good. But any new *top-level* generation datum must be added to **both** the msgpack and JSON representations (and, for msgpack, be safe under both the array and named-map encoders). A field added to only one encoding is silently lost the moment a peer falls back to the other — exactly the failure the JSON-fallback metric exists to catch. -- **Signature must bind a canonical form.** Because a field is transmitted as both JSON and msgpack and a peer may consume either, the body-digest binding in "RPC signature binding" above must be computed over a single canonical representation (the msgpack `_bin` bytes) — not over whichever copy happened to be decoded. Once the `generation` capability is negotiated for a request, a fenced / generation-bearing request must **reject the JSON fallback path** so a downgrade to the unsigned/loosely-bound JSON copy cannot bypass the epoch check. +- **Signature binds a canonical form.** `RenameDataRequest` now has a versioned, + injective canonical-body encoder that covers both compatibility fields and is + authenticated independently of whichever JSON/msgpack decoder branch a peer + consumes. A generation-capable strict request must reject missing or + mismatched canonical-body metadata; it must not silently downgrade to an + unauthenticated JSON twin. ### Proto evolution -New generation/epoch proto fields use **proto3 `optional`** (explicit presence). -A non-optional field is forbidden: an old coordinator talking to a new disk -decodes an absent field as `0`, which is indistinguishable from a real -`epoch == 0` and silently breaks the "stale epoch rejected" invariant during -upgrade. +No top-level proto field is required by the current metadata-map UUID. If a +future ordered epoch or explicit expected-generation is added to proto, it uses +**proto3 `optional`** (explicit presence). A non-optional scalar is forbidden: +an old coordinator talking to a new disk decodes absence as a plausible zero. ### Mixed-version gate — one direction -When the cluster-level generation capability is **not** negotiated on every -target disk, the behavior **falls back to current semantics** (existing lock + -`is_lock_lost()` check for #1312; degraded-allow read-check for #1318 at -`rustfs/src/app/object/get.rs`; full fanout for #1314). Fail-closed is -**only** an explicit administrator strict mode. Defaulting to fail-closed is -forbidden — it makes writes unavailable for the whole rolling-upgrade window. +When generation enforcement is not explicitly requested, or fleet confirmation +is absent, behavior falls back to current semantics. Fail-closed is reserved for +an explicit administrator-confirmed strict rollout. + +Current object transaction fencing follows that direction: + +- `RUSTFS_OBJECT_TRANSACTION_FENCING_WRITE` and + `RUSTFS_OBJECT_TRANSACTION_FENCING_FLEET_CONFIRMED` both default false. +- With either flag absent, PUT/MPU does not persist or consume the transaction + UUID. +- With both flags enabled, failure to obtain or retain the live fleet proof + rejects the commit before rename. + +This is an opt-in strict gate, not a negotiated generation capability. The +proof is currently borrowed from the remote-version-state writer rollout. It +proves current membership/process-epoch convergence for that feature, but does +not prove an epoch type, per-disk generation CAS support, or RPC strict-mode +convergence. Treating it as the final handshake is forbidden without an +explicit proof mapping for those properties. ## Capability negotiation -Generation enforcement is a **cluster-level handshake**, not a per-request -probe: +Generation enforcement requires one **live fleet proof**, not independent +boolean guesses in each consumer. The proof contract contains at least: -- A node advertises a `generation` capability once it can (a) mint quorum-durable - epochs, (b) compare epochs at all three disk-write points, and (c) verify the - body-digest-bound RPC signature. -- The authoritative writer enables hard enforcement for an object only when - **all** target disks in the set advertise the capability. Any missing - advertisement pins that commit to the mixed-version fallback above. -- The capability is surfaced through the existing runtime capability contract - surface (see [runtime-capability-contracts.md](runtime-capability-contracts.md)), - so consumers read one negotiated flag rather than each re-deriving support. -- Enforcement tracks the current membership rather than latching: it turns on - for a set only while every disk in that set advertises `generation`, and a - single old node rejoining drops the affected sets back to the mixed-version - fallback rather than failing closed. It never regresses the on-disk epoch — - falling back stops *comparing* new epochs, it does not lower any epoch already - persisted. +1. the selected authority version and comparison mode (ordered or exact-CAS), +2. the current membership/topology fingerprint and process epochs, +3. support for every required disk mutation point, +4. RPC signature/body/replay strict convergence, and +5. the on-disk encoding version (the current metadata-map UUID is version 1). + +The authoritative writer enables enforcement only while every target disk in +the set is covered by a current proof. Membership change or an old node rejoin +revokes that proof. Revocation before commit fails an explicitly strict request; +when strict generation was never requested, the request remains on the legacy +path. Revocation never rewrites or lowers an already-persisted generation. + +The existing fleet-proof machinery in `notification_sys` may be reused if its +authenticated statements are extended to cover the properties above. The +runtime capability contract may instead expose the proof. This document does +not choose the storage mechanism; it requires one token whose acquisition and +revalidation semantics are shared by all consumers. ## Implementation order -1. **#1312 first.** It defines the epoch, its persistence, the three fence - points, the RPC signature binding, and the encoding location. Everything - downstream depends on its epoch existing. -2. **#1313** (read lease) reuses the #1312 epoch as the lease generation and - must land before or alongside #1323. -3. **#1323** (old-dir GC) depends on #1313 leases being present and - cross-node-visible; its "no lease references old_dir" check has nothing to - query otherwise. -4. **#1318** (quota reservation) and **#1314** (prepared pool read) bind the - epoch independently; both gate on the same capability handshake. +Some original prerequisites have landed, but not in the originally proposed +form. Remaining work follows this order: -## Open design decisions (pin before implementation) +1. **Resolve the authority mode in #1326.** Select total order or opaque + exact-CAS, specify its atomic commit point, and audit PR #6077 against it. + Do not retrofit ordering semantics onto the existing random UUID. +2. **Define the generation fleet proof.** Map generation enablement to the RPC + signature/body/replay strict proofs delivered by #1327/#1541/#1542 and to + the selected per-disk comparison capability. Keep all strict defaults off + until fallback metrics converge. +3. **Implement #1313 generation-bound read leases.** The lease registry, + cross-node visibility, TTL, and crash recovery must exist before old-dir GC + can claim the full snapshot-lifetime guarantee. #1325 supplies the required + multi-node failure tests. +4. **Bind #1314 prepared reads.** A bundle binds the exact selected generation + within its source pool. Cross-pool ordering is forbidden until a common + authority is demonstrated. Validate rebalance and mixed-version fallback in + the #1325 multi-pool harness. +5. **Reconcile #1318 quota fencing.** Either bind reserve/settle/reconcile to + the selected object generation or document and prove that its independent + snapshot-lease fence is a separate arbitration domain that cannot settle a + different generation. +6. **Re-audit #1323 cleanup.** The existing UUID receipt remains valid crash + cleanup, but full closure against active readers requires the #1313 lease + check and the selected generation semantics. -This document fixes the transport, encoding, proto, and gate constraints, but it is not yet a complete implementable algorithm. The following must be decided and written down before any of the five consumers is coded (per the #1307 maintainer re-review, issuecomment-4992956256): +## Open design decisions (pin before contract closure) -- **Epoch type and total order.** The concrete token type and its total-order rule — a term+counter tuple, its persistence, and overflow behavior. Whether monotonicity is global or strictly per-object. -- **Never-regress on lock-service restart / minority recovery.** The epoch source must survive a lock-service restart or minority-quorum recovery without ever handing out an epoch lower than one already persisted on disk (an in-memory counter reset to zero is a fencing inversion). This is the same requirement as "Monotonicity persistence semantics" above, elevated to a hard, tested acceptance. -- **Complete xl.meta-writer coverage.** Every code path that writes xl.meta (commit rename, rollback delete/metadata restore, old-dir cleanup, heal, transition) must be enumerated and shown to compare or carry the epoch. A single unfenced writer voids the guarantee. -- **Rollback is an expected-generation CAS (#1312 B2).** The quorum-failure rollback at `io_primitives.rs:2646-2691` restores a metadata backup, not just a per-writer tmp delete, so a late rollback by writer A can overwrite writer B's committed xl.meta. Rollback must execute only when `stored_epoch == failed_writer_epoch`; a higher stored epoch must abort the rollback. Task panic / cancel / timeout at `io_primitives.rs:2602-2605` must be reaped into the coordinator's state machine, never bubble out via `?` and skip convergence. +The following decisions remain blockers for calling the contract implemented: + +- **Authority mode.** Choose total order or opaque exact-CAS. If total order is + selected, define the type, per-object scope, persistence, overflow, and + never-regress restart/minority-recovery tests. If opaque identity is selected, + define the atomic expected-generation CAS and remove all ordered wording. +- **Complete xl.meta-writer coverage.** Enumerate commit rename, rollback + restore/delete, cleanup, heal, transition, restore, replication, and data + movement. Each path must compare/carry the selected generation or be proved + incapable of replacing the authoritative object identity. +- **Rollback is an expected-generation CAS (#1312 B2).** The quorum-failure + rollback in `rename_data` can restore backup metadata, not just remove a + writer-private temporary file. It must execute only when the stored generation + still matches the failed writer's expected generation. Panic, cancel, and + timeout outcomes must be reaped into coordinator convergence rather than skip + rollback through an early return. - **Sidecar is excluded unless proven atomic.** An epoch sidecar outside `xl.meta` is only admissible if it commits at the same atomic/CAS point as `xl.meta` with a defined recovery; otherwise it opens a crash gap and must be rejected in favor of the version-internal metadata map. The earlier "metadata map or sidecar" phrasing does not treat the two as equally safe. -- **Read-lease and GC crash recovery.** Lease registry location (local vs cross-node), TTL reclamation, and crash recovery for both the lease holder and the GC executor. -- **Quota reserve → commit → settle idempotency.** The cross-stage reconcile / idempotency story for #1318, including owner-crash reconciliation, so a reservation is neither lost nor double-counted. +- **Generation capability proof.** Decide whether to extend the current + authenticated fleet proof or the runtime capability contract. It must prove + authority version, mutation coverage, topology/process epoch, and RPC strict + convergence in one revalidatable token. +- **Read-lease and GC crash recovery.** Select the cross-node registry, TTL + reclamation, lease-holder crash behavior, and GC-executor recovery. The + current cleanup receipt equality check does not answer these questions. +- **Quota reserve → commit → settle binding.** The durable ledger's idempotency + exists, but its independent mutation tokens must be related to the selected + object generation with a concrete late-settle rejection test. - **PreparedPoolRead is pool-local only.** A #1314 bundle's generation validates freshness only within the pool that produced it. It cannot order commits across different pools unless a cross-pool common authority exists; absent that, the multi-pool wait cannot be short-circuited. -- **Hot-path cost is a blocking metric.** If per-PUT fencing grant, quota reserve, or cleanup journal adds a consensus write / fsync / centralized serialization point, it must be measured under 4KiB and high-concurrency hot-key / hot-bucket A/B as a blocking gate, not accepted by default. +- **Hot-path cost is a blocking metric.** Measure any additional consensus + write, fsync, fleet-proof lookup, lease operation, or centralized serialization + under 4 KiB and high-concurrency hot-key/hot-bucket A/B. +- **Test infrastructure.** #1325 still lacks the complete 4-node × 4-drive, + 2-pool, directed network-fault, and large-object budget needed for restart, + mixed-version, and cross-node lease acceptance. ## Acceptance for this contract -- #1312 / #1313 / #1314 / #1318 / #1323 bodies reference this unified - generation and no longer define their own token. -- The five constraints — transport signature, encoding, proto presence, - mixed-version gate direction, and capability negotiation — are pinned here - once; each implementation sub-issue follows them rather than re-deciding. +- [x] Architecture document exists and is linked from the architecture index. +- [x] Transport signature, encoding, proto presence, mixed-version direction, + and capability-proof requirements are defined once. +- [x] Current implementations are separated from target guarantees; a closed + child issue is not treated as proof of unified generation binding. +- [ ] Authority mode and atomic comparison semantics are selected and tested. +- [ ] #1312 / #1313 / #1314 / #1318 / #1323 bodies reference this document and + use the selected authority terminology. +- [ ] #1313 and #1314 bind the selected generation and pass #1325 multi-node / + multi-pool failure tests. +- [ ] #1318 either binds reserve/settle to the selected generation or provides + an accepted proof that its separate fence cannot cross-settle generations. +- [ ] #1323 reconciliation checks both committed generation and active + generation-bound leases. +- [ ] Generation strict enablement is backed by one live proof that includes RPC + signature/body/replay strict convergence and per-disk comparison support. diff --git a/docs/operations/scanner-runtime-controls.md b/docs/operations/scanner-runtime-controls.md index de54b8f40..b480921ca 100644 --- a/docs/operations/scanner-runtime-controls.md +++ b/docs/operations/scanner-runtime-controls.md @@ -139,6 +139,12 @@ objects: backoff state. - `metrics`: scanner work, pressure, checkpoint, lifecycle, replication, heal, bitrot, and alert counters. +- `data_movement_pause`: the global-pause policy, current movement reason, + operation epoch, start time, duration, and estimated movement work items. +- `pause_backlog`: the replicated durable pause ledger, post-pause catch-up + phase, rate window, retry state, thresholds, and active alert reasons. +- `catch_up_estimate`: movement work plus current dirty-usage and already + discovered lifecycle queues. Example fields to inspect: @@ -163,8 +169,97 @@ metrics.cycle_timeout_total metrics.cycle_last_progress_age metrics.leader_lease_without_progress metrics.cycle_recovery_required_total +data_movement_pause.paused +data_movement_pause.reasons +data_movement_pause.duration_seconds +data_movement_pause.operation_epoch +data_movement_pause.movement_generation +data_movement_pause.movement_backlog_work_items +pause_backlog.persistence_state +pause_backlog.phase +pause_backlog.pause_duration_seconds +pause_backlog.pending_full_scan +pause_backlog.pending_work_items +pause_backlog.next_attempt_at_unix_secs +pause_backlog.alert_reasons +catch_up_estimate.dirty_usage_buckets +catch_up_estimate.discovered_expiry_items +catch_up_estimate.discovered_transition_items ``` +## Data Movement Pauses + +RustFS currently uses a `global_pause` policy while pool decommission or +rebalance can hide scanner metadata. Usage publication, lifecycle discovery, +tier cleanup discovery, scanner-originated heal and bitrot checks, and +replication discovery are deferred together. A failed or canceled +decommission remains a publication barrier until an operator retries or clears +it. + +`data_movement_pause.reasons` combines the in-process decommission worker state +with the durable pool and rebalance operation metadata. Exhausted operation +epochs or movement generations also fail closed and appear as explicit pause +reasons. Its start time, duration, and movement backlog come from the durable +metadata; a worker-only or exhausted-counter snapshot can therefore report +`paused=true` with zero start time and backlog. +`movement_backlog_work_items` counts remaining movement bucket work units, not +expired objects. `catch_up_estimate` combines that estimate with dirty-usage +buckets and lifecycle items that were already discovered before or during the +pause. The API sets `undiscovered_ilm_items_known=false` because a global pause +cannot count newly expired objects without scanning the namespace. Use +`usage_baseline_unix_secs` to judge the age of that estimate. + +The same pause and estimate objects are included in +`GET /v3/ilm/expiry/status`. The gauges +`rustfs_scanner_data_movement_paused`, +`rustfs_scanner_data_movement_pause_duration_seconds`, and +`rustfs_scanner_data_movement_backlog_work_items` expose the local snapshot +without bucket-name labels. + +The scanner persists `.scanner-pause-backlog.json` independently on erasure +sets in every surviving pool. A generation becomes authoritative only after +the identical commit record reaches every set named by its membership marker. +When a failed, canceled, or cleared decommission source rejoins, the last +committed surviving-set ledger seeds it before a new full-membership commit is +allowed; a smaller stale source membership cannot override the largest valid +surviving-set proof, and a membership claim is valid only when every declared +member stores the same proof. This repair appears as +`membership_repair_pending`. A partial commit is +rolled back to the previous stable generation after a crash or leader switch. +The ledger never rewrites pool or rebalance movement state. A new scanner +leader recovers the committed writer epoch and generation, counts an +interrupted attempt as a failure, and requires one successful full namespace +scan after movement clears. Known dirty-usage, expiry, and transition queues +must also reach zero before the ledger returns to `idle`. If the ledger cannot +be read or updated, scanner cycles remain gated and persistence is retried +every five minutes; the management status reports `persistence_unavailable` +until recovery. + +Catch-up attempts remain subject to the normal cycle duration, object, +directory, sleeper, and foreground-read budgets. The additional durable rate +window admits at most four attempts per hour and no more than one attempt per +five minutes. Five consecutive failed or interrupted attempts move the ledger +to `retry_exhausted`; accelerated retries stop and a sparse hourly probe is +used instead. A successful probe can return to bounded catch-up. + +`pause_backlog.thresholds` reports the exact pause-duration, deferred-cycle, +backlog-size, rate, and failure limits used by the running binary. +`pause_backlog.alert_reasons` identifies exceeded thresholds, exhausted +counters or retries, replica degradation, and persistence failures. The +threshold alerts fire after a 24-hour pause, three movement deferrals in one +unconverged pause episode, or 10,000 known pending work items. The +corresponding unlabeled gauges are: + +- `rustfs_scanner_pause_backlog_phase` (`0` idle, `1` paused, `2` catching up, + `3` retry exhausted); +- `rustfs_scanner_pause_backlog_pause_duration_seconds`; +- `rustfs_scanner_pause_backlog_pending_work_items`; +- `rustfs_scanner_pause_backlog_consecutive_failures`; +- `rustfs_scanner_pause_backlog_rate_limited`; +- `rustfs_scanner_pause_backlog_retry_exhausted`; +- `rustfs_scanner_pause_backlog_alerting`; +- `rustfs_scanner_pause_backlog_replica_degraded`. + ## Reading Pacing Pressure `metrics.pacing_pressure.primary_pressure` summarizes the highest-priority diff --git a/docs/testing/e2e-suite-inventory.md b/docs/testing/e2e-suite-inventory.md index 7fbde3908..8a52be5af 100644 --- a/docs/testing/e2e-suite-inventory.md +++ b/docs/testing/e2e-suite-inventory.md @@ -30,7 +30,7 @@ | chaos | 2 | | | checksum_upload_test | 7 | | | cluster_concurrency_test | 3 | 🌙 | -| cluster_multidrive_pool_test | 2 | 🌙 | +| cluster_multidrive_pool_test | 4 | 🌙 | | common | 17 | | | compression_test | 6 | ✅ | | connection_cap_test | 2 | | @@ -103,4 +103,4 @@ | tls_hot_reload_test | 1 | ✅ | | version_id_regression_test | 10 | ✅ | -**Total listed: 619 tests across 86 modules · PR smoke: 165 tests / 36 modules · merge/main full: 495 tests / 77 modules · nightly replication: 56 tests · nightly cluster faults: 29 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-26. +**Total listed: 622 tests across 86 modules · PR smoke: 165 tests / 36 modules · merge/main full: 495 tests / 77 modules · nightly replication: 56 tests · nightly cluster faults: 32 tests / 7 modules · nightly protocols: 16 tests** · updated 2026-08-31. diff --git a/rustfs/src/admin/handlers/account.rs b/rustfs/src/admin/handlers/account.rs index 6378b4bb3..998b1900b 100644 --- a/rustfs/src/admin/handlers/account.rs +++ b/rustfs/src/admin/handlers/account.rs @@ -33,6 +33,7 @@ use super::account_audit::{ }; use super::admin_json_response; use super::iam_error::iam_error_to_s3_error; +use super::site_replication::site_replication_iam_change_hook; use super::supervise_admin_mutation; use crate::admin::auth::validate_admin_request; use crate::admin::router::{AdminOperation, Operation, S3Router}; @@ -48,6 +49,7 @@ use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_iam::mfa::service as mfa_service; use rustfs_madmin::account::{AccountMfaSummary, ChangePasswordRequest, IdentityType, SelfAccountInfo, SetUserSecretKeyRequest}; +use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq, SITE_REPL_API_VERSION, SRIAMItem, SRIAMUser}; use rustfs_policy::auth::is_secret_key_valid; use rustfs_policy::policy::action::{Action, AdminAction}; use rustfs_utils::MaskedAccessKey; @@ -239,7 +241,7 @@ impl Operation for ChangeOwnPasswordHandler { let iam_store = current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?; - iam_store + let (updated_at, status) = iam_store .set_user_secret_key(&access_key, &new_secret_key) .await .map_err(iam_error_to_s3_error)?; @@ -283,6 +285,11 @@ impl Operation for ChangeOwnPasswordHandler { "admin account state" ); + // After the local revocation: peer delivery has no ordering + // dependency on it, and a slow peer must not delay killing the + // old sessions here. + broadcast_secret_key_rotation("change_own_password", &access_key, &new_secret_key, status, updated_at).await; + Ok(revoked) }) .await?; @@ -390,7 +397,19 @@ impl Operation for SetUserSecretKeyHandler { let iam_store = current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?; - iam_store + // Derived credentials live outside the `iam-user` replication + // item: rotating one here would succeed locally and silently skip + // the peer broadcast, leaving the sites permanently diverged. + if let Some(existing) = iam_store.get_user(&target).await + && (existing.credentials.is_temp() || existing.credentials.is_service_account()) + { + return Err(s3::error( + S3ErrorCode::InvalidRequest, + "the target access key is a derived credential; rotate service accounts through update-service-account", + )); + } + + let (updated_at, status) = iam_store .set_user_secret_key(&target, &request.secret_key) .await .map_err(iam_error_to_s3_error)?; @@ -430,6 +449,11 @@ impl Operation for SetUserSecretKeyHandler { "admin account state" ); + // After the local revocation: peer delivery has no ordering + // dependency on it, and a slow peer must not delay killing the + // old sessions here. + broadcast_secret_key_rotation("set_user_secret_key", &target, &request.secret_key, status, updated_at).await; + Ok(revoked) }) .await?; @@ -452,6 +476,58 @@ struct ChangePasswordResult { sessions_revoked: u32, } +/// The `iam-user` item a secret rotation fans out to peer sites. +/// +/// The non-empty secret routes the peer through its create-user path (not the +/// status-only path), so the persisted status must ride along or a disabled +/// account would be re-enabled on the peer. +fn secret_key_rotation_item(access_key: &str, secret_key: &str, status: AccountStatus, updated_at: OffsetDateTime) -> SRIAMItem { + SRIAMItem { + r#type: "iam-user".to_string(), + iam_user: Some(SRIAMUser { + access_key: access_key.to_string(), + is_delete_req: false, + user_req: Some(AddOrUpdateUserReq { + secret_key: secret_key.to_string(), + policy: None, + status, + }), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + }), + updated_at: Some(updated_at), + api_version: Some(SITE_REPL_API_VERSION.to_string()), + ..Default::default() + } +} + +/// Fan a rotated secret out to peer sites. +/// +/// The rotation is already durable locally; a broadcast failure only logs, +/// matching the other IAM site-replication hooks. `status` and `updated_at` +/// come from the persisting write itself, so the item carries exactly the +/// state that was stored. +async fn broadcast_secret_key_rotation( + action: &'static str, + access_key: &str, + secret_key: &str, + status: AccountStatus, + updated_at: OffsetDateTime, +) { + if let Err(err) = site_replication_iam_change_hook(secret_key_rotation_item(access_key, secret_key, status, updated_at)).await + { + warn!( + component = LOG_COMPONENT_ADMIN, + subsystem = LOG_SUBSYSTEM_ACCOUNT, + event = EVENT_ADMIN_ACCOUNT_STATE, + action, + access_key = %MaskedAccessKey(access_key), + result = "site_replication_hook_failed", + error = ?err, + "admin account state" + ); + } +} + /// Reject a new secret that would be useless or a no-op. fn validate_new_secret_key(request: &ChangePasswordRequest) -> S3Result<()> { if !is_secret_key_valid(&request.new_secret_key) { @@ -499,6 +575,49 @@ mod tests { validate_new_secret_key(&change_request("old-secret-key", "new-secret-key")).expect("must accept"); } + #[test] + fn rotation_item_takes_the_peer_create_path_and_preserves_status() { + let ts = OffsetDateTime::now_utc(); + let item = secret_key_rotation_item("rotated-user", "new-secret-key", AccountStatus::Disabled, ts); + + assert_eq!(item.r#type, "iam-user"); + assert_eq!(item.updated_at, Some(ts)); + assert!(item.api_version.is_some()); + + let user = item.iam_user.expect("iam-user payload"); + assert_eq!(user.access_key, "rotated-user"); + assert!(!user.is_delete_req); + + let req = user.user_req.expect("user_req payload"); + // A non-empty secret is what routes the peer through create-user + // instead of the status-only path. + assert_eq!(req.secret_key, "new-secret-key"); + // Policy must stay unset so the peer's policy mapping is untouched. + assert!(req.policy.is_none()); + // A disabled account must stay disabled on the peer. + assert_eq!(req.status, AccountStatus::Disabled); + } + + #[test] + fn both_rotation_handlers_broadcast_after_revoking_sessions() { + let src = include_str!("account.rs"); + for marker in [ + "impl Operation for ChangeOwnPasswordHandler", + "impl Operation for SetUserSecretKeyHandler", + ] { + let start = src.find(marker).expect("handler should exist"); + let block = &src[start..]; + let block = &block[..block.find("\n}\n").expect("handler block end")]; + let revoke = block + .find("revoke_sts_sessions_for_parent") + .expect("handler must revoke sessions"); + let broadcast = block + .find("broadcast_secret_key_rotation(") + .expect("handler must broadcast the rotation to peer sites"); + assert!(revoke < broadcast, "{marker}: peer broadcast must not delay the local session revocation"); + } + } + #[test] fn route_constants_stay_under_the_admin_prefix() { // The constants spell the full path so registration has a single source diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index be7295801..b1c4af3d2 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -68,6 +68,35 @@ const LOG_COMPONENT_ADMIN: &str = "admin"; const LOG_SUBSYSTEM_BUCKET_META: &str = "bucket_meta"; const EVENT_ADMIN_BUCKET_META_STATE: &str = "admin_bucket_meta_state"; +fn export_internal_error(message: impl Into) -> s3s::S3Error { + let message = message.into(); + s3_error!(InternalError, "{message}") +} + +fn checked_raw_xml(validated: &T, raw: Vec, parse: F) -> S3Result> +where + T: PartialEq, + E: std::fmt::Display, + F: FnOnce(&[u8]) -> Result, +{ + let selected = parse(&raw) + .map_err(|e| export_internal_error(format!("persisted bucket metadata changed to invalid XML during export: {e}")))?; + if selected != *validated { + return Err(export_internal_error("bucket metadata changed during export")); + } + Ok(raw) +} + +fn checked_versioning_xml(validated: &VersioningConfiguration, raw: Vec) -> S3Result> { + if raw.is_empty() { + if *validated != VersioningConfiguration::default() { + return Err(export_internal_error("bucket metadata changed during export")); + } + return serialize(validated).map_err(|e| export_internal_error(format!("serialize config failed: {e}"))); + } + checked_raw_xml(validated, raw, deserialize::) +} + #[derive(Debug, Default, serde::Deserialize)] pub struct ExportBucketMetadataQuery { pub bucket: String, @@ -190,8 +219,13 @@ impl Operation for ExportBucketMetadata { Ok(None) => continue, }; + let raw_config = metadata_sys::get(&bucket.name) + .await + .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))? + .notification_config_xml + .clone(); let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + checked_raw_xml(&config, raw_config, deserialize::)?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) @@ -210,8 +244,12 @@ impl Operation for ExportBucketMetadata { return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}")); } }; - let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?; + let raw_config = metadata_sys::get(&bucket.name) + .await + .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))? + .lifecycle_config_xml + .clone(); + let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) @@ -230,8 +268,12 @@ impl Operation for ExportBucketMetadata { return Err(s3_error!(InternalError, "failed to load bucket metadata: {e}")); } }; - let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "failed to serialize config: {e}"))?; + let raw_config = metadata_sys::get(&bucket.name) + .await + .map_err(|e| export_internal_error(format!("failed to load bucket metadata: {e}")))? + .tagging_config_xml + .clone(); + let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) @@ -270,8 +312,12 @@ impl Operation for ExportBucketMetadata { return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); } }; - let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + let raw_config = metadata_sys::get(&bucket.name) + .await + .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))? + .object_lock_config_xml + .clone(); + let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) @@ -290,8 +336,12 @@ impl Operation for ExportBucketMetadata { return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); } }; - let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + let raw_config = metadata_sys::get(&bucket.name) + .await + .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))? + .encryption_config_xml + .clone(); + let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) @@ -310,8 +360,12 @@ impl Operation for ExportBucketMetadata { return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); } }; - let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + let raw_config = metadata_sys::get(&bucket.name) + .await + .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))? + .versioning_config_xml + .clone(); + let config_xml = checked_versioning_xml(&config, raw_config)?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) @@ -330,8 +384,12 @@ impl Operation for ExportBucketMetadata { return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); } }; - let config_xml = - serialize(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; + let raw_config = metadata_sys::get(&bucket.name) + .await + .map_err(|e| export_internal_error(format!("get bucket metadata failed: {e}")))? + .replication_config_xml + .clone(); + let config_xml = checked_raw_xml(&config, raw_config, deserialize::)?; zip_writer .start_file(conf_path, SimpleFileOptions::default()) @@ -1013,6 +1071,66 @@ mod imported_config_apply_tests { } } + #[test] + fn g_d3_005_new_writer_backup_payloads_pass_old_import_validators() { + // Captured from the gateway persistence writers at a16e94426b36c6a454dc200a5639c711b590e1eb. + // Keep these bytes independent of RustFS's old serializer so rollback drift stays visible. + let new_writer_payloads: [(&str, &[u8]); 7] = [ + ( + BUCKET_NOTIFICATION_CONFIG, + b"", + ), + ( + BUCKET_LIFECYCLE_CONFIG, + b"30logs/expireEnabled", + ), + ( + BUCKET_SSECONFIG, + b"AES256", + ), + ( + BUCKET_TAGGING_CONFIG, + b"teamstorage", + ), + ( + OBJECT_LOCK_CONFIG, + b"Enabled", + ), + ( + BUCKET_VERSIONING_CONFIG, + b"Enabled", + ), + ( + BUCKET_REPLICATION_CONFIG, + b"arn:aws:iam::123456789012:role/replicationDisabledarn:aws:s3:::backup1Enabled", + ), + ]; + + let imported_xml_cases = import_cases() + .into_iter() + .filter(|case| case.conf_name != BUCKET_TARGETS_FILE) + .collect::>(); + assert_eq!(new_writer_payloads.len(), imported_xml_cases.len()); + assert!( + imported_xml_cases + .iter() + .all(|case| new_writer_payloads.iter().any(|(name, _)| *name == case.conf_name)) + ); + + let mut metadatas = imported_bucket(); + for (conf_name, payload) in new_writer_payloads { + assert!( + apply_imported_bucket_config(&mut metadatas, BUCKET, conf_name, payload.to_vec(), imported_at()) + .unwrap_or_else(|error| panic!("new-writer {conf_name} must pass the old import validator: {error}")) + ); + let case = imported_xml_cases + .iter() + .find(|case| case.conf_name == conf_name) + .unwrap_or_else(|| panic!("{conf_name} must have an import mapping")); + assert_eq!((case.payload)(&metadatas[BUCKET]), payload); + } + } + #[test] fn a_rejected_payload_leaves_the_field_untouched() { for case in import_cases() { @@ -1215,6 +1333,235 @@ mod import_persist_tests { } } +#[cfg(test)] +mod backup_zip_compatibility_tests { + use super::*; + use crate::admin::runtime_sources::{AppContext, publish_test_app_context}; + use http::{Extensions, Uri}; + use http_body_util::BodyExt as _; + use rustfs_iam::store::{Store as _, object::IAM_CONFIG_PREFIX}; + use std::sync::Arc; + + const ROOT_ACCESS_KEY: &str = "BUCKETMETABACKUPROOT"; + const ROOT_SECRET_KEY: &str = "bucketMetaBackupRootSecret123"; + const BUCKET: &str = "backup-compatibility"; + const NOTIFICATION_XML: &[u8] = b"\n"; + const LIFECYCLE_XML: &[u8] = b"\nexpireEnabledlogs/30\n"; + const SSE_XML: &[u8] = b"\nAES256\n"; + const TAGGING_XML: &[u8] = b"\nteamstorage\n"; + const OBJECT_LOCK_XML: &[u8] = + b"\nEnabled\n"; + const VERSIONING_XML: &[u8] = b"\nEnabled\n"; + const OLD_REPLICATION_XML: &[u8] = b"arn:aws:iam::123456789012:role/replicationpreserve-meEnabled1Disabledarn:aws:s3:::backup"; + const DIFFERENT_REPLICATION_XML: &[u8] = b"arn:aws:iam::123456789012:role/replicationEnabled2Disabledchanged/arn:aws:s3:::replacement"; + + fn persisted_xml_fixtures() -> [(&'static str, &'static [u8]); 7] { + [ + (BUCKET_NOTIFICATION_CONFIG, NOTIFICATION_XML), + (BUCKET_LIFECYCLE_CONFIG, LIFECYCLE_XML), + (BUCKET_SSECONFIG, SSE_XML), + (BUCKET_TAGGING_CONFIG, TAGGING_XML), + (OBJECT_LOCK_CONFIG, OBJECT_LOCK_XML), + (BUCKET_VERSIONING_CONFIG, VERSIONING_XML), + (BUCKET_REPLICATION_CONFIG, OLD_REPLICATION_XML), + ] + } + + fn persisted_xml<'a>(metadata: &'a BucketMetadata, config_file: &str) -> &'a [u8] { + match config_file { + BUCKET_NOTIFICATION_CONFIG => &metadata.notification_config_xml, + BUCKET_LIFECYCLE_CONFIG => &metadata.lifecycle_config_xml, + BUCKET_SSECONFIG => &metadata.encryption_config_xml, + BUCKET_TAGGING_CONFIG => &metadata.tagging_config_xml, + OBJECT_LOCK_CONFIG => &metadata.object_lock_config_xml, + BUCKET_VERSIONING_CONFIG => &metadata.versioning_config_xml, + BUCKET_REPLICATION_CONFIG => &metadata.replication_config_xml, + _ => panic!("unexpected persisted XML config {config_file}"), + } + } + + fn zip_with_entries(bucket: &str, entries: &[(&str, &[u8])]) -> Vec { + let mut writer = ZipWriter::new(Cursor::new(Vec::new())); + for (config_file, payload) in entries { + writer + .start_file(format!("{bucket}/{config_file}"), SimpleFileOptions::default()) + .expect("start compatibility archive entry"); + writer.write_all(payload).expect("write compatibility archive entry"); + } + writer.finish().expect("finish compatibility archive").into_inner() + } + + fn admin_request(method: Method, uri: Uri, body: Vec) -> S3Request { + S3Request { + input: Body::from(body), + method, + uri, + headers: HeaderMap::new(), + extensions: Extensions::new(), + credentials: Some(s3s::auth::Credentials { + access_key: ROOT_ACCESS_KEY.to_string(), + secret_key: s3s::auth::SecretKey::from(ROOT_SECRET_KEY.to_string()), + }), + region: None, + service: None, + trailing_headers: None, + } + } + + async fn import_archive(archive: Vec) { + let response = ImportBucketMetadata {} + .call( + admin_request(Method::PUT, Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), archive), + Params::new(), + ) + .await + .expect("root admin must import the compatibility archive"); + assert_eq!(response.output.0, StatusCode::OK); + } + + #[tokio::test] + #[serial_test::serial] + async fn g_zip_001_002_003_use_real_admin_archive_and_persistence_paths() { + let _ = rustfs_credentials::init_global_action_credentials( + Some(ROOT_ACCESS_KEY.to_string()), + Some(ROOT_SECRET_KEY.to_string()), + ); + let temp = tempfile::tempdir().expect("create bucket metadata backup test root"); + let env = rustfs_test_utils::TestECStoreEnv::builder() + .base_dir(temp.path()) + .disk_count(1) + .build() + .await; + env.make_bucket(BUCKET, false).await; + rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore)) + .save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX)) + .await + .expect("seed IAM format"); + let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore)) + .await + .expect("build test IAM"); + publish_test_app_context(Arc::new(AppContext::with_default_interfaces( + Arc::clone(&env.ecstore), + iam, + Arc::new(rustfs_kms::KmsServiceManager::new()), + ))); + + let corrupt_error = ImportBucketMetadata {} + .call( + admin_request( + Method::PUT, + Uri::from_static("/rustfs/admin/v3/import-bucket-metadata"), + b"not a zip archive".to_vec(), + ), + Params::new(), + ) + .await + .expect_err("a corrupt compatibility archive must be rejected"); + assert!( + corrupt_error + .message() + .is_some_and(|message| message.contains("failed to read import archive")), + "corrupt archive returned the wrong error: {corrupt_error:?}" + ); + + let fixtures = persisted_xml_fixtures(); + import_archive(zip_with_entries(BUCKET, &fixtures)).await; + let imported = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("old archive must persist bucket metadata"); + for (config_file, payload) in fixtures { + assert_eq!(persisted_xml(&imported, config_file), payload, "g-zip-001 changed {config_file} bytes"); + } + import_archive(zip_with_entries(BUCKET, &[(BUCKET_REPLICATION_CONFIG, b"not xml")])).await; + let after_rejected_payload = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("rejected payload must leave persisted metadata readable"); + assert_eq!( + after_rejected_payload.replication_config_xml, OLD_REPLICATION_XML, + "a rejected archive payload must not replace persisted bytes" + ); + + let (validated_replication, _) = metadata_sys::get_replication_config(BUCKET) + .await + .expect("load the revision validated before export"); + checked_raw_xml( + &validated_replication, + DIFFERENT_REPLICATION_XML.to_vec(), + deserialize::, + ) + .expect_err("a different valid revision must not be exported after validating the old revision"); + checked_raw_xml(&validated_replication, b"not xml".to_vec(), deserialize::) + .expect_err("an invalid raw revision must not be exported after validating the old revision"); + let matching_raw = checked_raw_xml( + &validated_replication, + OLD_REPLICATION_XML.to_vec(), + deserialize::, + ) + .expect("the exact validated raw revision must remain exportable"); + assert_eq!(matching_raw, OLD_REPLICATION_XML); + let default_versioning = VersioningConfiguration::default(); + assert!( + !checked_versioning_xml(&default_versioning, Vec::new()) + .expect("empty persisted versioning keeps the existing default fallback") + .is_empty() + ); + let enabled_versioning: VersioningConfiguration = deserialize(VERSIONING_XML).expect("parse enabled versioning fixture"); + checked_versioning_xml(&enabled_versioning, Vec::new()) + .expect_err("an empty raw revision must not export a previously validated enabled revision"); + + let export_response = ExportBucketMetadata {} + .call( + admin_request( + Method::GET, + format!("/rustfs/admin/v3/export-bucket-metadata?bucket={BUCKET}") + .parse() + .expect("export URI"), + Vec::new(), + ), + Params::new(), + ) + .await + .expect("root admin must export persisted bucket metadata"); + assert_eq!(export_response.output.0, StatusCode::OK); + let exported_archive = export_response + .output + .1 + .collect() + .await + .expect("read exported archive body") + .to_bytes() + .to_vec(); + let mut archive = ZipArchive::new(Cursor::new(&exported_archive)).expect("open exported archive"); + for (config_file, payload) in persisted_xml_fixtures() { + let mut exported_payload = Vec::new(); + archive + .by_name(&format!("{BUCKET}/{config_file}")) + .unwrap_or_else(|_| panic!("exported archive must contain {config_file}")) + .read_to_end(&mut exported_payload) + .unwrap_or_else(|_| panic!("read exported {config_file}")); + assert_eq!(exported_payload, payload, "g-zip-003 export must preserve {config_file} byte-for-byte"); + } + drop(archive); + + metadata_sys::update(BUCKET, BUCKET_REPLICATION_CONFIG, DIFFERENT_REPLICATION_XML.to_vec()) + .await + .expect("replace persisted config before rollback import"); + import_archive(exported_archive).await; + let restored = metadata_sys::get_config_from_disk(BUCKET) + .await + .expect("new archive must persist through old import"); + for (config_file, payload) in persisted_xml_fixtures() { + assert_eq!( + persisted_xml(&restored, config_file), + payload, + "g-zip-002 rollback import did not restore {config_file}" + ); + } + let _: ReplicationConfiguration = + deserialize(&restored.replication_config_xml).expect("old parser must read the newly exported archive payload"); + } +} + #[cfg(test)] mod shared_gate_tests { use super::*; diff --git a/rustfs/src/admin/handlers/scanner.rs b/rustfs/src/admin/handlers/scanner.rs index 52fdfc777..47815ae1e 100644 --- a/rustfs/src/admin/handlers/scanner.rs +++ b/rustfs/src/admin/handlers/scanner.rs @@ -18,6 +18,7 @@ use crate::admin::router::{AdminOperation, Operation, S3Router}; use crate::admin::runtime_sources::{ app_context_from_req, current_object_store_handle_for_context, current_scanner_metrics_report, }; +use crate::admin::storage_api::ScannerDataMovementPauseStatus; use crate::module_switches::{ENV_SCANNER_ENABLED, scanner_enabled_from_env}; use crate::server::ADMIN_PREFIX; use chrono::Utc; @@ -27,6 +28,8 @@ use matchit::Params; use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE; use rustfs_credentials::Credentials; use rustfs_policy::policy::action::{Action, AdminAction}; +#[cfg(test)] +use rustfs_scanner_contracts::metrics::ScannerLifecycleTransitionSnapshot; use rustfs_scanner_contracts::metrics::{ ScannerLifecycleExpirySnapshot, ScannerMaintenanceControlSnapshot, ScannerMetricsReport, }; @@ -46,6 +49,9 @@ struct ScannerStatusResponse { cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, cycle_recovery: rustfs_scanner::ScannerCycleRecoveryStatus, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, + catch_up_estimate: ScannerCatchUpEstimate, } #[derive(Debug, Deserialize)] @@ -62,6 +68,17 @@ struct ScannerFreshnessStatus { reason: Option<&'static str>, } +#[derive(Debug, Serialize)] +struct ScannerCatchUpEstimate { + estimated: bool, + movement_work_items: u64, + dirty_usage_buckets: u64, + discovered_expiry_items: u64, + discovered_transition_items: u64, + undiscovered_ilm_items_known: bool, + usage_baseline_unix_secs: u64, +} + #[derive(Debug, Serialize)] struct IlmExpiryStatusResponse { enabled: bool, @@ -71,6 +88,46 @@ struct IlmExpiryStatusResponse { maintenance_control: ScannerMaintenanceControlSnapshot, current_cycle_lifecycle_expiry_actions: u64, last_cycle_lifecycle_expiry_actions: u64, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, + catch_up_estimate: ScannerCatchUpEstimate, +} + +fn scanner_catch_up_estimate( + pause: &ScannerDataMovementPauseStatus, + backlog: &rustfs_scanner::ScannerPauseBacklogStatus, + metrics: &ScannerMetricsReport, +) -> ScannerCatchUpEstimate { + ScannerCatchUpEstimate { + estimated: pause.paused || backlog.phase != rustfs_scanner::ScannerPauseBacklogPhase::Idle, + movement_work_items: pause.movement_backlog_work_items.max(backlog.movement_work_items), + dirty_usage_buckets: metrics.usage_freshness.dirty_pending_buckets.max(backlog.dirty_usage_buckets), + discovered_expiry_items: metrics + .lifecycle_expiry + .current_queued + .saturating_add(metrics.lifecycle_expiry.current_active) + .max(backlog.discovered_expiry_items), + discovered_transition_items: metrics + .lifecycle_transition + .current_queued + .saturating_add(metrics.lifecycle_transition.current_active) + .saturating_add(metrics.lifecycle_transition.compensation_pending) + .saturating_add(metrics.lifecycle_transition.compensation_running) + .max(backlog.discovered_transition_items), + undiscovered_ilm_items_known: !pause.paused && !backlog.pending_full_scan, + usage_baseline_unix_secs: metrics.usage_freshness.last_durable_success_unix_secs, + } +} + +fn unavailable_pause_backlog(error: &str) -> rustfs_scanner::ScannerPauseBacklogStatus { + rustfs_scanner::ScannerPauseBacklogStatus { + persistence_state: "unavailable".to_string(), + alerting: true, + alert_reasons: vec![rustfs_scanner::ScannerPauseBacklogAlertReason::PersistenceUnavailable], + thresholds: rustfs_scanner::ScannerPauseBacklogThresholds::default(), + error: Some(error.to_string()), + ..Default::default() + } } fn scanner_disabled_reason(enabled: bool) -> Option { @@ -122,8 +179,11 @@ fn scanner_status_response( metrics: ScannerMetricsReport, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, ) -> ScannerStatusResponse { let freshness = scanner_freshness_status(&metrics, &runtime_config, cycle_schedule.effective_interval_seconds()); + let catch_up_estimate = scanner_catch_up_estimate(&data_movement_pause, &pause_backlog, &metrics); ScannerStatusResponse { enabled, disabled_reason: scanner_disabled_reason(enabled), @@ -132,6 +192,9 @@ fn scanner_status_response( cycle_schedule, runtime_config, cycle_recovery: rustfs_scanner::scanner::scanner_cycle_recovery_status(), + data_movement_pause, + pause_backlog, + catch_up_estimate, } } @@ -140,8 +203,11 @@ fn ilm_expiry_status_response( metrics: ScannerMetricsReport, runtime_config: rustfs_scanner::runtime_config::ScannerRuntimeConfigStatus, cycle_schedule: rustfs_scanner::ScannerCycleScheduleStatus, + data_movement_pause: ScannerDataMovementPauseStatus, + pause_backlog: rustfs_scanner::ScannerPauseBacklogStatus, ) -> IlmExpiryStatusResponse { let freshness = scanner_freshness_status(&metrics, &runtime_config, cycle_schedule.effective_interval_seconds()); + let catch_up_estimate = scanner_catch_up_estimate(&data_movement_pause, &pause_backlog, &metrics); IlmExpiryStatusResponse { enabled, disabled_reason: scanner_disabled_reason(enabled), @@ -150,6 +216,9 @@ fn ilm_expiry_status_response( maintenance_control: metrics.maintenance_control, current_cycle_lifecycle_expiry_actions: metrics.current_cycle_lifecycle_expiry_actions, last_cycle_lifecycle_expiry_actions: metrics.last_cycle_lifecycle_expiry_actions, + data_movement_pause, + pause_backlog, + catch_up_estimate, } } @@ -208,7 +277,20 @@ impl Operation for ScannerStatusHandler { let metrics = current_scanner_metrics_report().await; let runtime_config = rustfs_scanner::scanner_runtime_config_status(); let cycle_schedule = rustfs_scanner::scanner_cycle_schedule_status(); - let response = scanner_status_response(enabled, metrics, runtime_config, cycle_schedule); + let store = + app_context_from_req(&req).and_then(|context| current_object_store_handle_for_context(Some(context.as_ref()))); + let (data_movement_pause, pause_backlog) = match store { + Some(store) => ( + store.scanner_data_movement_pause_status().await, + rustfs_scanner::scanner_pause_backlog_status(store).await, + ), + None => ( + ScannerDataMovementPauseStatus::default(), + unavailable_pause_backlog("storage layer not initialized"), + ), + }; + let response = + scanner_status_response(enabled, metrics, runtime_config, cycle_schedule, data_movement_pause, pause_backlog); let body = serde_json::to_vec(&response).map_err(|err| { S3Error::with_message(S3ErrorCode::InternalError, format!("failed to encode scanner status: {err}")) })?; @@ -258,7 +340,20 @@ impl Operation for IlmExpiryStatusHandler { let metrics = current_scanner_metrics_report().await; let runtime_config = rustfs_scanner::scanner_runtime_config_status(); let cycle_schedule = rustfs_scanner::scanner_cycle_schedule_status(); - let response = ilm_expiry_status_response(enabled, metrics, runtime_config, cycle_schedule); + let store = + app_context_from_req(&req).and_then(|context| current_object_store_handle_for_context(Some(context.as_ref()))); + let (data_movement_pause, pause_backlog) = match store { + Some(store) => ( + store.scanner_data_movement_pause_status().await, + rustfs_scanner::scanner_pause_backlog_status(store).await, + ), + None => ( + ScannerDataMovementPauseStatus::default(), + unavailable_pause_backlog("storage layer not initialized"), + ), + }; + let response = + ilm_expiry_status_response(enabled, metrics, runtime_config, cycle_schedule, data_movement_pause, pause_backlog); let body = serde_json::to_vec(&response).map_err(|err| { S3Error::with_message(S3ErrorCode::InternalError, format!("failed to encode ILM expiry status: {err}")) })?; @@ -388,6 +483,8 @@ mod tests { ScannerMetricsReport::default(), rustfs_scanner::scanner_runtime_config_status(), rustfs_scanner::ScannerCycleScheduleStatus::default(), + ScannerDataMovementPauseStatus::default(), + rustfs_scanner::ScannerPauseBacklogStatus::default(), ); let encoded = serde_json::to_value(response).expect("scanner status should serialize"); @@ -401,6 +498,23 @@ mod tests { encoded["cycle_recovery"]["quarantine_path"], rustfs_scanner::DATA_USAGE_BLOOM_RECOVERY_PATH.as_str() ); + assert_eq!(encoded["data_movement_pause"]["policy"], "global_pause"); + assert_eq!(encoded["data_movement_pause"]["paused"], false); + assert_eq!(encoded["catch_up_estimate"]["estimated"], false); + assert_eq!(encoded["catch_up_estimate"]["undiscovered_ilm_items_known"], true); + } + + #[test] + fn scanner_status_keeps_an_unavailable_storage_layer_observable() { + let backlog = unavailable_pause_backlog("storage layer not initialized"); + + assert_eq!(backlog.persistence_state, "unavailable"); + assert!(backlog.alerting); + assert_eq!( + backlog.alert_reasons, + vec![rustfs_scanner::ScannerPauseBacklogAlertReason::PersistenceUnavailable] + ); + assert_eq!(backlog.error.as_deref(), Some("storage layer not initialized")); } #[test] @@ -418,6 +532,13 @@ mod tests { scanner_not_enqueued: 13, delete_failed: 19, }, + lifecycle_transition: ScannerLifecycleTransitionSnapshot { + current_queued: 2, + current_active: 3, + compensation_pending: 5, + compensation_running: 7, + ..Default::default() + }, maintenance_control: ScannerMaintenanceControlSnapshot { primary_control: "expiry_backlog".to_string(), ..Default::default() @@ -431,6 +552,18 @@ mod tests { metrics, rustfs_scanner::scanner_runtime_config_status(), rustfs_scanner::ScannerCycleScheduleStatus::default(), + ScannerDataMovementPauseStatus { + paused: true, + movement_backlog_work_items: 31, + movement_backlog_estimated: true, + ..Default::default() + }, + rustfs_scanner::ScannerPauseBacklogStatus { + phase: rustfs_scanner::ScannerPauseBacklogPhase::Paused, + movement_work_items: 31, + pending_full_scan: true, + ..Default::default() + }, ); let encoded = serde_json::to_value(response).expect("ILM expiry status should serialize"); @@ -441,5 +574,11 @@ mod tests { assert_eq!(encoded["maintenance_control"]["primary_control"].as_str(), Some("expiry_backlog")); assert_eq!(encoded["current_cycle_lifecycle_expiry_actions"].as_u64(), Some(23)); assert_eq!(encoded["last_cycle_lifecycle_expiry_actions"].as_u64(), Some(29)); + assert_eq!(encoded["data_movement_pause"]["paused"], true); + assert_eq!(encoded["pause_backlog"]["phase"], "paused"); + assert_eq!(encoded["catch_up_estimate"]["movement_work_items"].as_u64(), Some(31)); + assert_eq!(encoded["catch_up_estimate"]["discovered_expiry_items"].as_u64(), Some(9)); + assert_eq!(encoded["catch_up_estimate"]["discovered_transition_items"].as_u64(), Some(17)); + assert_eq!(encoded["catch_up_estimate"]["undiscovered_ilm_items_known"], false); } } diff --git a/rustfs/src/admin/handlers/table_catalog/mod.rs b/rustfs/src/admin/handlers/table_catalog/mod.rs index d958c8c9f..9882170dc 100644 --- a/rustfs/src/admin/handlers/table_catalog/mod.rs +++ b/rustfs/src/admin/handlers/table_catalog/mod.rs @@ -160,6 +160,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ "GET /v1/{prefix}/namespaces/{namespace}", "HEAD /v1/{prefix}/namespaces/{namespace}", "DELETE /v1/{prefix}/namespaces/{namespace}", + "POST /v1/{prefix}/namespaces/{namespace}/properties", "GET /v1/{prefix}/namespaces/{namespace}/tables", "POST /v1/{prefix}/namespaces/{namespace}/tables", "POST /v1/{prefix}/namespaces/{namespace}/register", @@ -168,6 +169,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ "GET /v1/{prefix}/namespaces/{namespace}/tables/{table}/credentials", "POST /v1/{prefix}/namespaces/{namespace}/tables/{table}", "DELETE /v1/{prefix}/namespaces/{namespace}/tables/{table}", + "POST /v1/{prefix}/tables/rename", "GET /v1/{prefix}/namespaces/{namespace}/views", "POST /v1/{prefix}/namespaces/{namespace}/views", "GET /v1/{prefix}/namespaces/{namespace}/views/{view}", @@ -175,10 +177,7 @@ const TABLE_CATALOG_ENDPOINTS: &[&str] = &[ "POST /v1/{prefix}/namespaces/{namespace}/views/{view}", "DELETE /v1/{prefix}/namespaces/{namespace}/views/{view}", ]; -const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &[ - "POST /v1/{prefix}/namespaces/{namespace}/properties", - "POST /v1/{prefix}/tables/rename", -]; +const TABLE_CATALOG_DURABLE_STRONG_ENDPOINTS: &[&str] = &[]; static GET_CONFIG_HANDLER: GetCatalogConfigHandler = GetCatalogConfigHandler {}; static ENABLE_TABLE_BUCKET_HANDLER: EnableTableBucketHandler = EnableTableBucketHandler {}; @@ -2298,6 +2297,7 @@ fn table_bucket_entry_from_metadata_marker(bucket: &str) -> crate::table_catalog warehouse_root: format!("s3://{bucket}/"), state: crate::table_catalog::TableCatalogEntryState::Active, properties: BTreeMap::new(), + active_rename_id: None, created_at: None, updated_at: None, } diff --git a/rustfs/src/admin/handlers/table_catalog/tests.rs b/rustfs/src/admin/handlers/table_catalog/tests.rs index 7e1520b87..d32239c15 100644 --- a/rustfs/src/admin/handlers/table_catalog/tests.rs +++ b/rustfs/src/admin/handlers/table_catalog/tests.rs @@ -434,11 +434,11 @@ fn catalog_config_response_lists_standard_rest_endpoints() { Some(REST_NAMESPACE_SEPARATOR_URL_ENCODED) ); assert!( - !response + response .endpoints .contains(&"POST /v1/{prefix}/namespaces/{namespace}/properties") ); - assert!(!response.endpoints.contains(&"POST /v1/{prefix}/tables/rename")); + assert!(response.endpoints.contains(&"POST /v1/{prefix}/tables/rename")); assert_eq!(response.admin_discovery.runtime_capabilities, "/rustfs/admin/v4/runtime/capabilities"); assert_eq!(response.admin_discovery.cluster_snapshot, "/rustfs/admin/v4/cluster/snapshot"); assert_eq!(response.admin_discovery.extensions_catalog, "/rustfs/admin/v4/extensions/catalog"); @@ -466,6 +466,7 @@ fn catalog_config_response_reports_durable_strong_backing_override() { .contains(&"POST /v1/{prefix}/namespaces/{namespace}/properties") ); assert!(response.endpoints.contains(&"POST /v1/{prefix}/tables/rename")); + assert_eq!(response.endpoints.as_slice(), TABLE_CATALOG_ENDPOINTS); } #[test] @@ -11120,6 +11121,7 @@ async fn seed_object_table_for_metadata_maintenance( warehouse_root: format!("s3://{bucket}/"), state: crate::table_catalog::TableCatalogEntryState::Active, properties: BTreeMap::new(), + active_rename_id: None, created_at: None, updated_at: None, }) @@ -11272,6 +11274,183 @@ async fn namespace_helpers_call_catalog_store() { assert!(list.namespaces.is_empty()); } +#[tokio::test] +#[serial_test::serial] +async fn namespace_property_handler_updates_object_backed_catalog_and_maps_errors() { + use crate::admin::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions}; + + temp_env::async_with_vars( + [( + crate::table_catalog::ENV_TABLE_CATALOG_BACKING, + Some(crate::table_catalog::TABLE_CATALOG_BACKING_OBJECT), + )], + async { + let (_temp_dir, _disk_paths, object_store) = crate::app::gating_test_env::isolated_multi_pool_ecstore().await; + let bucket = format!("namespace-properties-{}", Uuid::new_v4().simple()); + object_store + .make_bucket(&bucket, &MakeBucketOptions::default()) + .await + .expect("table bucket should be created"); + enable_table_bucket_marker(&object_store, &bucket) + .await + .expect("table bucket marker should be enabled"); + + rustfs_iam::store::object::ObjectStore::new(object_store.clone()) + .save_iam_config( + serde_json::json!({"version": 1}), + format!("{}/format.json", *rustfs_iam::store::object::IAM_CONFIG_PREFIX), + ) + .await + .expect("request IAM format should be seeded"); + let iam = rustfs_iam::build_iam_sys(object_store.clone()) + .await + .expect("request IAM should initialize"); + let context = Arc::new(AppContext::new( + object_store.clone(), + Arc::new(RequestIam { handle: iam }), + Arc::new(RequestKms), + )); + let root_access_key = "namespace-properties-root"; + let root_secret_key = "namespace-properties-root-secret"; + assert!(context.publish_action_credentials(rustfs_credentials::Credentials { + access_key: root_access_key.to_string(), + secret_key: root_secret_key.to_string(), + status: "on".to_string(), + ..Default::default() + })); + let slot = ServerContextSlot::new(); + assert!(slot.install(context.clone())); + + let backend = crate::table_catalog::EcStoreTableCatalogObjectBackend::new_with_strong_runtime( + object_store, + context.table_catalog_strong_runtime(), + ); + let catalog = crate::table_catalog::ConfiguredTableCatalogStore::new_for_test( + backend.clone(), + crate::table_catalog::TableCatalogBackingMode::ObjectBacked, + ); + catalog + .put_table_bucket(table_bucket_entry_from_metadata_marker(&bucket)) + .await + .expect("table bucket catalog entry should be seeded"); + let namespace = crate::table_catalog::Namespace::parse("analytics").expect("namespace should parse"); + let entry = crate::table_catalog::NamespaceEntry { + version: crate::table_catalog::TABLE_CATALOG_ENTRY_VERSION, + table_bucket: bucket.clone(), + namespace: namespace.public_name(), + namespace_id: namespace.storage_id(), + state: crate::table_catalog::TableCatalogEntryState::Active, + properties: BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]), + created_at: None, + updated_at: None, + }; + catalog + .create_namespace(entry.clone()) + .await + .expect("namespace should be seeded"); + + let request = |namespace: &str, body: serde_json::Value| { + let mut extensions = http::Extensions::new(); + extensions.insert(slot.clone()); + S3Request { + input: Body::from(serde_json::to_vec(&body).expect("request body should serialize")), + method: Method::POST, + uri: format!("/iceberg/v1/{bucket}/namespaces/{namespace}/properties") + .parse() + .expect("request URI should parse"), + headers: HeaderMap::new(), + extensions, + credentials: Some(s3s::auth::Credentials { + access_key: root_access_key.to_string(), + secret_key: s3s::auth::SecretKey::from(root_secret_key.to_string()), + }), + region: None, + service: None, + trailing_headers: None, + } + }; + let mut params_router = matchit::Router::new(); + params_router + .insert("/iceberg/v1/{warehouse}/namespaces/{namespace}/properties", ()) + .expect("handler parameter route should register"); + let success_path = format!("/iceberg/v1/{bucket}/namespaces/analytics/properties"); + let params = params_router + .at(&success_path) + .expect("success handler parameters should match") + .params; + let response = RestUpdateNamespacePropertiesHandler {} + .call( + request( + "analytics", + serde_json::json!({ + "removals": ["owner", "missing"], + "updates": {"retention": "30d"} + }), + ), + params, + ) + .await + .expect("handler should update object-backed namespace properties"); + assert_eq!(response.output.0, StatusCode::OK); + let body = http_body_util::BodyExt::collect(response.output.1) + .await + .expect("response body should collect") + .to_bytes(); + assert_eq!( + serde_json::from_slice::(&body).expect("response body should decode"), + serde_json::json!({ + "updated": ["retention"], + "removed": ["owner"], + "missing": ["missing"] + }) + ); + let persisted = catalog + .get_namespace(&bucket, &namespace.public_name()) + .await + .expect("updated namespace should load") + .expect("updated namespace should remain"); + assert_eq!(persisted.properties.get("retention").map(String::as_str), Some("30d")); + assert!(!persisted.properties.contains_key("owner")); + + let missing_path = format!("/iceberg/v1/{bucket}/namespaces/missing/properties"); + let params = params_router + .at(&missing_path) + .expect("missing handler parameters should match") + .params; + let missing = RestUpdateNamespacePropertiesHandler {} + .call(request("missing", serde_json::json!({"updates": {"owner": "platform"}})), params) + .await + .expect_err("missing namespace should fail"); + assert_eq!(missing.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_NO_SUCH_NAMESPACE.into())); + assert_eq!(missing.status_code(), Some(StatusCode::NOT_FOUND)); + + let corrupt = crate::table_catalog::Namespace::parse("corrupt").expect("namespace should parse"); + let corrupt_path = crate::table_catalog::TableCatalogObjectPaths::default().namespace_entry_path(&bucket, &corrupt); + backend + .put_object( + crate::admin::storage_api::RUSTFS_META_BUCKET, + &corrupt_path, + b"{".to_vec(), + crate::table_catalog::TableCatalogPutPrecondition::Any, + ) + .await + .expect("corrupt namespace entry should be seeded"); + let corrupt_request_path = format!("/iceberg/v1/{bucket}/namespaces/corrupt/properties"); + let params = params_router + .at(&corrupt_request_path) + .expect("corrupt handler parameters should match") + .params; + let corrupt = RestUpdateNamespacePropertiesHandler {} + .call(request("corrupt", serde_json::json!({"updates": {"owner": "platform"}})), params) + .await + .expect_err("corrupt namespace should fail"); + assert_eq!(corrupt.code(), &S3ErrorCode::Custom(ICEBERG_ERROR_BAD_REQUEST.into())); + assert_eq!(corrupt.status_code(), Some(StatusCode::BAD_REQUEST)); + }, + ) + .await; +} + #[tokio::test] async fn table_helpers_call_catalog_store() { let store = TestTableCatalogStore::default(); diff --git a/rustfs/src/admin/storage_api.rs b/rustfs/src/admin/storage_api.rs index 5ea5b3f74..200bdc2be 100644 --- a/rustfs/src/admin/storage_api.rs +++ b/rustfs/src/admin/storage_api.rs @@ -85,7 +85,7 @@ mod ecstore_rpc { } mod ecstore_storage { - pub(crate) use crate::storage::storage_api::ecstore_storage::ECStore; + pub(crate) use crate::storage::storage_api::ecstore_storage::{ECStore, ScannerDataMovementPauseStatus}; } mod ecstore_tier { @@ -108,6 +108,7 @@ pub(crate) type RebalanceCleanupWarnings = ecstore_rebalance::RebalanceCleanupWa pub(crate) type RebalanceMeta = ecstore_rebalance::RebalanceMeta; pub(crate) type RebalanceStats = ecstore_rebalance::RebalanceStats; pub(crate) type RebalanceStopPropagationRecord = ecstore_rebalance::RebalanceStopPropagationRecord; +pub(crate) type ScannerDataMovementPauseStatus = ecstore_storage::ScannerDataMovementPauseStatus; pub(crate) type StorageError = ecstore_error::StorageError; pub(crate) type Error = StorageError; pub(crate) type Result = core::result::Result; diff --git a/rustfs/src/app/object/get.rs b/rustfs/src/app/object/get.rs index e02c6528e..c0bdfe859 100644 --- a/rustfs/src/app/object/get.rs +++ b/rustfs/src/app/object/get.rs @@ -164,6 +164,12 @@ impl Drop for GetObjectDiskPermit { } } +fn release_disk_read_permit_if_buffered(disk_permit: &mut Option, buffered_body: Option<&Bytes>) { + if buffered_body.is_some() { + disk_permit.take(); + } +} + const COLD_FILL_HARD_MAX_DURATION: Duration = Duration::from_secs(10 * 60); pub(crate) const MAX_GET_OBJECT_MEMORY_BUFFER_BYTES: i64 = 64 * 1024 * 1024; @@ -2754,7 +2760,7 @@ impl DefaultObjectUsecase { } } - let (io_planning, reader) = if let Some(prepared) = prepared.take() { + let (mut io_planning, reader) = if let Some(prepared) = prepared.take() { let io_planning = metadata_admission .take() .ok_or_else(|| s3_error!(InternalError, "prepared metadata admission is unavailable"))?; @@ -2797,6 +2803,10 @@ impl DefaultObjectUsecase { let read_setup = Self::finish_get_object_read(req, manager, bucket, key, rs, part_number, read_start, reader, cache_fill_allowed) .await?; + // The buffered body has completed storage reads. Release admission + // before output planning so downstream response work cannot occupy a + // disk slot; streaming bodies retain the permit below until EOF/drop. + release_disk_read_permit_if_buffered(&mut io_planning.disk_permit, read_setup.buffered_body.as_ref()); if let Some(read_stage_start) = read_stage_start { rustfs_io_metrics::record_get_object_stage_duration( "s3_handler", @@ -7619,6 +7629,38 @@ mod tests { assert_eq!(semaphore.available_permits(), 1); } + #[tokio::test] + async fn buffered_body_releases_disk_permit_before_output_planning() { + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = semaphore + .clone() + .acquire_owned() + .await + .expect("test semaphore should grant owned permit"); + let mut disk_permit = Some(permit.into()); + release_disk_read_permit_if_buffered(&mut disk_permit, Some(&Bytes::from_static(b"body"))); + assert_eq!(semaphore.available_permits(), 1); + assert!(disk_permit.is_none()); + } + + #[tokio::test] + async fn streaming_body_retains_disk_permit_for_output_planning() { + let semaphore = Arc::new(tokio::sync::Semaphore::new(1)); + let permit = semaphore + .clone() + .acquire_owned() + .await + .expect("test semaphore should grant owned permit"); + let mut disk_permit = Some(permit.into()); + + release_disk_read_permit_if_buffered(&mut disk_permit, None); + assert_eq!(semaphore.available_permits(), 0); + assert!(disk_permit.is_some()); + + drop(disk_permit); + assert_eq!(semaphore.available_permits(), 1); + } + #[tokio::test] #[serial_test::serial(cold_fill_metrics_gate)] async fn cold_fill_follower_disk_permit_metric_tracks_actual_permit_lifetime() { diff --git a/rustfs/src/app/select_object.rs b/rustfs/src/app/select_object.rs index b4aa98408..48ea1ce50 100644 --- a/rustfs/src/app/select_object.rs +++ b/rustfs/src/app/select_object.rs @@ -9,11 +9,17 @@ use super::storage_api::select_object::{ }; use crate::app::runtime_sources::current_s3select_db; use crate::error::ApiError; -use bytes::Bytes; +use bytes::{Bytes, BytesMut}; use datafusion::arrow::{ - csv::{QuoteStyle, WriterBuilder as CsvWriterBuilder, writer::Terminator}, - json::{WriterBuilder as JsonWriterBuilder, writer::LineDelimited}, + array::{Array, ListLikeArray, MapArray, cast::AsArray}, + datatypes::{ + ArrowNativeType, DataType, FieldRef, Int8Type, Int16Type, Int32Type, Int64Type, UInt8Type, UInt16Type, UInt32Type, + UInt64Type, + }, + error::ArrowError, + json::writer::{EncoderOptions, NullableEncoder, make_encoder}, record_batch::RecordBatch, + util::display::{ArrayFormatter, FormatOptions}, }; #[cfg(test)] use datafusion::common::DataFusionError; @@ -22,7 +28,7 @@ use futures::StreamExt; use http::{HeaderMap, StatusCode, header::RANGE}; use rustfs_s3select_api::{ QueryError, SelectError, SelectInputMetrics, - object_store::{INVALID_SCAN_RANGE_MESSAGE, validate_scan_range_bounds}, + object_store::{INVALID_SCAN_RANGE_MESSAGE, is_noop_scan_range, validate_scan_range_bounds}, query::{Context, Query}, }; use rustfs_s3select_query::instance::s3_select_query_timeout; @@ -33,21 +39,42 @@ use s3s::dto::{ StatsEvent, }; use s3s::{S3Error, S3ErrorCode, S3Request, S3Response, S3Result, s3_error}; -use std::sync::Arc; +use std::{ + fmt, + future::poll_fn, + io::{self, Write}, + ops::Range, + pin::Pin, + sync::Arc, + time::Duration, +}; use tokio::sync::mpsc; -use tokio::time::{Instant, timeout_at}; +use tokio::time::{Instant, Interval, MissedTickBehavior, Sleep, timeout_at}; use tokio_stream::wrappers::ReceiverStream; +use tokio_util::sync::PollSender; use tracing::info; const MAX_SELECT_EXPRESSION_BYTES: usize = 256 * 1024; -const RECORDS_CHUNK_TARGET: usize = 128 * 1024; +const MAX_COMPAT_EVENT_STREAM_MESSAGE_BYTES: usize = 128 * 1024 - 256; +const RECORDS_EVENT_STREAM_OVERHEAD_BYTES: usize = 101; +const RECORDS_CHUNK_TARGET: usize = MAX_COMPAT_EVENT_STREAM_MESSAGE_BYTES - RECORDS_EVENT_STREAM_OVERHEAD_BYTES; +const ENCODE_TURN_TARGET_BYTES: usize = 64 * 1024; +const MAX_ENCODE_ROWS_PER_TURN: usize = 1024; +const MAX_SELECT_OUTPUT_RECORD_BYTES: usize = 1024 * 1024; +const RECORDS_FLUSH_INTERVAL: Duration = Duration::from_millis(500); +const CONTINUATION_INTERVAL: Duration = Duration::from_secs(1); +const PROGRESS_INTERVAL: Duration = Duration::from_secs(60); const DATA_SOURCE_PATH_UNSUPPORTED_CODE: &str = "DataSourcePathUnsupported"; const INVALID_QUERY_CODE: &str = "InvalidQuery"; const PARSE_SELECT_FAILURE_CODE: &str = "ParseSelectFailure"; +const INVALID_REQUEST_PARAMETER_MESSAGE: &str = + "The value of a parameter in the SelectRequest element is invalid. Check the service API documentation and try again."; const BUSY_MESSAGE: &str = "The service is unavailable. Try again later."; const EMPTY_SELECT_EXPRESSION_MESSAGE: &str = "empty SQL expression"; const SLOW_DOWN_MESSAGE: &str = "Reduce your request rate."; const UNSUPPORTED_SQL_STRUCTURE_MESSAGE: &str = "We encountered an unsupported SQL structure. Check the SQL Reference."; +const OVER_MAX_RECORD_SIZE_MESSAGE: &str = + "The length of a record in the input or result is greater than the maxCharsPerRecord limit of 1 MB."; #[derive(Clone, Debug)] struct SelectValidation { @@ -67,8 +94,14 @@ enum SelectProducerOutcome { ReceiverClosed, } +enum TerminalRecordsMode { + Complete, + PrefixBeforeError, +} + struct SelectEventChannel { tx: mpsc::Sender>, + terminal_records_permit: Option>>, terminal_permit: mpsc::OwnedPermit>, } @@ -103,7 +136,11 @@ pub async fn execute_select_object_content( ) .await .map_err(|_| select_query_timeout_error(query_timeout.as_secs()))??; - validate_scan_range_for_object_size(&input.request, snapshot.logical_size())?; + let object_size = snapshot.logical_size(); + validate_scan_range_for_object_size(&input.request, object_size)?; + if object_size == 0 && is_compressed_input(&input.request.input_serialization) { + return Err(map_select_error_to_s3(&SelectError::TruncatedInput)); + } let snapshot = Arc::new(snapshot); let query = Query::new_with_snapshot(Context { input: input.clone() }, input.request.expression.clone(), Arc::clone(&snapshot)); @@ -117,7 +154,11 @@ pub async fn execute_select_object_content( .into_record_batch_stream() .map_err(map_query_error_to_s3)?; - let (tx, rx) = mpsc::channel::>(9); + let (tx, rx) = mpsc::channel::>(10); + let terminal_records_permit = tx + .clone() + .try_reserve_owned() + .map_err(|_| map_select_error_to_s3(&SelectError::InternalError))?; let terminal_permit = tx .clone() .try_reserve_owned() @@ -129,7 +170,11 @@ pub async fn execute_select_object_content( spawn_traced(async move { send_select_events_until_deadline( output, - SelectEventChannel { tx, terminal_permit }, + SelectEventChannel { + tx, + terminal_records_permit: Some(terminal_records_permit), + terminal_permit, + }, validation, input_metrics, query_deadline, @@ -144,28 +189,25 @@ pub async fn execute_select_object_content( async fn send_select_events_until_deadline( output: SendableRecordBatchStream, - event_channel: SelectEventChannel, + mut event_channel: SelectEventChannel, validation: SelectValidation, input_metrics: Arc, deadline: Instant, timeout_seconds: u64, snapshot_lease: L, ) { - let outcome = match timeout_at( + let outcome = send_select_events( + output, + &mut event_channel, + validation, + input_metrics, deadline, - send_select_events(output, &event_channel.tx, validation, input_metrics, &snapshot_lease), + timeout_seconds, + &snapshot_lease, ) - .await - { - Ok(outcome) => outcome, - Err(_) => SelectProducerOutcome::Terminal(Err(map_query_error_to_s3( - SelectError::QueryTimeout { - seconds: timeout_seconds, - } - .into(), - ))), - }; + .await; if let SelectProducerOutcome::Terminal(event) = outcome { + drop(event_channel.terminal_records_permit.take()); event_channel.terminal_permit.send(event); } drop(snapshot_lease); @@ -173,81 +215,382 @@ async fn send_select_events_until_deadline( async fn send_select_events( mut output: SendableRecordBatchStream, - tx: &mpsc::Sender>, + event_channel: &mut SelectEventChannel, validation: SelectValidation, input_metrics: Arc, + deadline: Instant, + timeout_seconds: u64, snapshot_fence: &impl SelectSnapshotFence, ) -> SelectProducerOutcome { - let mut encoder = SelectOutputEncoder::new(validation.output_format); - let mut progress = SelectProgress::new(validation.reports_input_metrics.then_some(input_metrics)); - - if tx - .send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) - .await - .is_err() - { - return SelectProducerOutcome::ReceiverClosed; - } - + let SelectValidation { + output_format, + progress_enabled, + reports_input_metrics, + } = validation; + let mut encoder = SelectOutputEncoder::new(output_format); + let mut progress = SelectProgress::new(reports_input_metrics.then_some(input_metrics)); + let started_at = Instant::now(); + let records_flush = tokio::time::sleep_until(deadline); + tokio::pin!(records_flush); + let mut records_flush_armed = false; + let mut continuation = delayed_select_interval(started_at, CONTINUATION_INTERVAL); + let mut progress_interval = progress_enabled.then(|| delayed_select_interval(started_at, PROGRESS_INTERVAL)); + let deadline_sleep = tokio::time::sleep_until(deadline); + tokio::pin!(deadline_sleep); + let tx = event_channel.tx.clone(); + let mut periodic_sender = PollSender::new(tx.clone()); let receiver_closed = tx.closed(); tokio::pin!(receiver_closed); - while let Some(result) = tokio::select! { - biased; - _ = &mut receiver_closed => return SelectProducerOutcome::ReceiverClosed, - result = output.next() => result, - } { - let batch = match result { - Ok(batch) => batch, - Err(err) => { - return SelectProducerOutcome::Terminal(Err(map_query_error_to_s3(err.into()))); - } - }; - match encoder.encode_batch(&batch) { - Ok(payloads) => { - for payload in payloads { - let payload_len = payload.len(); - if tx - .send(Ok(SelectObjectContentEvent::Records(RecordsEvent { payload: Some(payload) }))) - .await + let mut records_buffer = BytesMut::new(); + let mut pending_event: Option = None; + let mut pending_batch: Option = None; + let mut pending_batch_offset = 0; + let mut continuation_due = false; + let mut progress_due = false; + + loop { + if pending_event.is_some() { + periodic_sender.abort_send(); + } + let periodic_due = progress_due || continuation_due; + let finishing_success = matches!(pending_event.as_ref(), Some(SelectObjectContentEvent::Stats(_))); + let progress_armed = progress_interval.is_some(); + + tokio::select! { + biased; + + _ = &mut receiver_closed => return SelectProducerOutcome::ReceiverClosed, + + _ = &mut deadline_sleep => { + return finish_select_with_error( + select_query_timeout_error(timeout_seconds), + event_channel, + &mut pending_event, + &mut records_buffer, + &mut progress, + ); + } + + _ = tick_optional_interval(&mut progress_interval), if progress_armed && !progress_due && !finishing_success => { + progress_due = true; + } + + _ = continuation.tick(), if !continuation_due && !finishing_success => { + continuation_due = true; + } + + permit = tx.reserve(), if pending_event.is_some() => { + let permit = match permit { + Ok(permit) => permit, + Err(_) => return SelectProducerOutcome::ReceiverClosed, + }; + let Some(event) = pending_event.take() else { + return SelectProducerOutcome::Terminal(Err(map_select_error_to_s3(&SelectError::InternalError))); + }; + let finishes_successfully = matches!(&event, SelectObjectContentEvent::Stats(_)); + if finishes_successfully + && let Err(error) = snapshot_fence.ensure_snapshot_valid() + { + return SelectProducerOutcome::Terminal(Err(error)); + } + let returned = records_payload_len(&event); + permit.send(Ok(event)); + if let Some(returned) = returned { + progress.add_returned(returned); + } + if finishes_successfully { + return SelectProducerOutcome::Terminal(Ok(SelectObjectContentEvent::End(EndEvent::default()))); + } + if !periodic_due { + schedule_buffered_records( + &mut records_buffer, + records_flush.as_mut(), + &mut records_flush_armed, + &mut pending_event, + deadline, + ); + } + } + + _ = &mut records_flush, if records_flush_armed && !finishing_success && pending_event.is_none() => { + records_flush_armed = false; + records_flush.as_mut().reset(deadline); + pending_event = take_records_payload(&mut records_buffer).map(records_event); + } + + permit = poll_fn(|cx| periodic_sender.poll_reserve(cx)), if periodic_due && pending_event.is_none() => { + if permit.is_err() { + return SelectProducerOutcome::ReceiverClosed; + } + if progress_due { + if periodic_sender + .send_item(Ok(SelectObjectContentEvent::Progress(ProgressEvent { + details: Some(progress.to_progress()), + }))) .is_err() { return SelectProducerOutcome::ReceiverClosed; } - progress.add_returned(payload_len); - if validation.progress_enabled - && tx - .send(Ok(SelectObjectContentEvent::Progress(ProgressEvent { - details: Some(progress.to_progress()), - }))) - .await - .is_err() + progress_due = false; + if let Some(interval) = progress_interval.as_mut() { + interval.reset(); + } + } else { + if periodic_sender + .send_item(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) + .is_err() { return SelectProducerOutcome::ReceiverClosed; } + continuation_due = false; + continuation.reset(); + } + if !progress_due && !continuation_due { + schedule_buffered_records( + &mut records_buffer, + records_flush.as_mut(), + &mut records_flush_armed, + &mut pending_event, + deadline, + ); } } - Err(err) => { - return SelectProducerOutcome::Terminal(Err(err)); + + _ = tokio::task::yield_now(), if pending_batch.is_some() && pending_event.is_none() && !periodic_due => { + let Some(batch) = pending_batch.as_ref() else { + return SelectProducerOutcome::Terminal(Err(map_select_error_to_s3(&SelectError::InternalError))); + }; + let remaining_rows = batch.num_rows().saturating_sub(pending_batch_offset); + if remaining_rows == 0 { + pending_batch = None; + pending_batch_offset = 0; + continue; + } + let encoded_rows = match encode_batch_turn( + &mut encoder, + batch, + pending_batch_offset, + &mut records_buffer, + ) { + Ok(encoded_rows) => encoded_rows, + Err(error) => { + return finish_select_with_error( + error, + event_channel, + &mut pending_event, + &mut records_buffer, + &mut progress, + ); + } + }; + if encoded_rows == 0 { + return SelectProducerOutcome::Terminal(Err(map_select_error_to_s3(&SelectError::InternalError))); + } + pending_batch_offset += encoded_rows; + if pending_batch_offset == batch.num_rows() { + pending_batch = None; + pending_batch_offset = 0; + } + schedule_buffered_records( + &mut records_buffer, + records_flush.as_mut(), + &mut records_flush_armed, + &mut pending_event, + deadline, + ); } + + result = output.next(), if !finishing_success && pending_event.is_none() && pending_batch.is_none() && !periodic_due => { + match result { + Some(Ok(batch)) => { + pending_batch = Some(batch); + pending_batch_offset = 0; + } + Some(Err(error)) => { + return finish_select_with_error( + map_query_error_to_s3(error.into()), + event_channel, + &mut pending_event, + &mut records_buffer, + &mut progress, + ); + } + None => { + if let Err(error) = snapshot_fence.ensure_snapshot_valid() { + return finish_select_with_error( + error, + event_channel, + &mut pending_event, + &mut records_buffer, + &mut progress, + ); + } + if let Err(error) = flush_terminal_records( + event_channel, + &mut pending_event, + &mut records_buffer, + &mut progress, + TerminalRecordsMode::Complete, + ) { + return SelectProducerOutcome::Terminal(Err(error)); + } + pending_event = Some(SelectObjectContentEvent::Stats(StatsEvent { + details: Some(progress.to_stats()), + })); + } + } + } + + } + } +} + +fn delayed_select_interval(started_at: Instant, period: Duration) -> Interval { + let mut interval = tokio::time::interval_at(started_at + period, period); + interval.set_missed_tick_behavior(MissedTickBehavior::Delay); + interval +} + +fn encode_batch_turn( + encoder: &mut SelectOutputEncoder, + batch: &RecordBatch, + offset: usize, + buffer: &mut BytesMut, +) -> S3Result { + let remaining_rows = batch.num_rows().saturating_sub(offset); + if remaining_rows == 0 { + return Ok(0); + } + + let original_len = buffer.len(); + let candidate_rows = remaining_rows.min(MAX_ENCODE_ROWS_PER_TURN); + let output_limit = if candidate_rows == 1 { + MAX_SELECT_OUTPUT_RECORD_BYTES + } else { + ENCODE_TURN_TARGET_BYTES + }; + match encoder.encode_batch_limited(batch, offset..offset + candidate_rows, buffer, output_limit) { + Ok(encoded_rows) if encoded_rows > 0 => return Ok(encoded_rows), + Ok(_) if candidate_rows > 1 => {} + Ok(_) => return Err(over_max_record_size_error()), + Err(error) => { + buffer.truncate(original_len); + return Err(error); } } - if let Err(error) = snapshot_fence.ensure_snapshot_valid() { - return SelectProducerOutcome::Terminal(Err(error)); + match encoder.encode_batch_limited(batch, offset..offset + 1, buffer, MAX_SELECT_OUTPUT_RECORD_BYTES) { + Ok(1) => Ok(1), + Ok(_) => { + buffer.truncate(original_len); + Err(over_max_record_size_error()) + } + Err(error) => { + buffer.truncate(original_len); + Err(error) + } } - let stats = SelectObjectContentEvent::Stats(StatsEvent { - details: Some(progress.to_stats()), - }); - let stats_permit = match tx.reserve().await { - Ok(permit) => permit, - Err(_) => return SelectProducerOutcome::ReceiverClosed, +} + +async fn tick_optional_interval(interval: &mut Option) { + match interval { + Some(interval) => { + interval.tick().await; + } + None => std::future::pending().await, + } +} + +fn schedule_buffered_records( + buffer: &mut BytesMut, + mut flush: Pin<&mut Sleep>, + flush_armed: &mut bool, + pending_event: &mut Option, + idle_deadline: Instant, +) { + if pending_event.is_some() || buffer.is_empty() { + return; + } + if buffer.len() >= RECORDS_CHUNK_TARGET { + *flush_armed = false; + flush.as_mut().reset(idle_deadline); + *pending_event = Some(records_event(buffer.split_to(RECORDS_CHUNK_TARGET).freeze())); + } else if !*flush_armed { + flush.as_mut().reset(Instant::now() + RECORDS_FLUSH_INTERVAL); + *flush_armed = true; + } +} + +fn take_records_payload(buffer: &mut BytesMut) -> Option { + (!buffer.is_empty()).then(|| buffer.split().freeze()) +} + +fn records_event(payload: Bytes) -> SelectObjectContentEvent { + SelectObjectContentEvent::Records(RecordsEvent { payload: Some(payload) }) +} + +fn records_payload_len(event: &SelectObjectContentEvent) -> Option { + match event { + SelectObjectContentEvent::Records(records) => records.payload.as_ref().map(Bytes::len), + _ => None, + } +} + +fn flush_terminal_records( + event_channel: &mut SelectEventChannel, + pending_event: &mut Option, + records_buffer: &mut BytesMut, + progress: &mut SelectProgress, + mode: TerminalRecordsMode, +) -> S3Result<()> { + let pending = pending_event.take(); + let pending_payload = match pending { + Some(SelectObjectContentEvent::Records(records)) => records.payload, + _ => None, }; - if let Err(error) = snapshot_fence.ensure_snapshot_valid() { + let payload = pending_payload.or_else(|| take_records_payload(records_buffer)); + if matches!(mode, TerminalRecordsMode::PrefixBeforeError) { + records_buffer.clear(); + } + let permit = event_channel.terminal_records_permit.take(); + let Some(payload) = payload else { + drop(permit); + return Ok(()); + }; + let payload = match mode { + TerminalRecordsMode::Complete if payload.len() > RECORDS_CHUNK_TARGET => { + return Err(map_select_error_to_s3(&SelectError::InternalError)); + } + TerminalRecordsMode::PrefixBeforeError if payload.len() > RECORDS_CHUNK_TARGET => payload.slice(..RECORDS_CHUNK_TARGET), + _ => payload, + }; + let Some(permit) = permit else { + return Err(map_select_error_to_s3(&SelectError::InternalError)); + }; + let returned = payload.len(); + permit.send(Ok(records_event(payload))); + progress.add_returned(returned); + Ok(()) +} + +fn finish_select_with_error( + error: S3Error, + event_channel: &mut SelectEventChannel, + pending_event: &mut Option, + records_buffer: &mut BytesMut, + progress: &mut SelectProgress, +) -> SelectProducerOutcome { + if let Err(error) = flush_terminal_records( + event_channel, + pending_event, + records_buffer, + progress, + TerminalRecordsMode::PrefixBeforeError, + ) { return SelectProducerOutcome::Terminal(Err(error)); } - stats_permit.send(Ok(stats)); - SelectProducerOutcome::Terminal(Ok(SelectObjectContentEvent::End(EndEvent::default()))) + SelectProducerOutcome::Terminal(Err(error)) } fn validate_select_request(headers: &http::HeaderMap, input: &mut SelectObjectContentInput) -> S3Result { @@ -262,7 +605,14 @@ fn validate_select_request(headers: &http::HeaderMap, input: &mut SelectObjectCo } normalize_input_serialization(&mut input.request.input_serialization)?; + let compressed_input = is_compressed_input(&input.request.input_serialization); + if compressed_input && input.request.scan_range.as_ref().is_some_and(is_noop_scan_range) { + input.request.scan_range = None; + } validate_scan_range(&input.request)?; + if compressed_input && input.request.scan_range.is_some() { + return Err(map_select_error_to_s3(&SelectError::UnsupportedScanRangeInput)); + } let output_format = normalize_output_serialization(&mut input.request.output_serialization)?; if input.request.expression.trim().is_empty() { @@ -284,6 +634,13 @@ fn validate_select_request(headers: &http::HeaderMap, input: &mut SelectObjectCo }) } +fn is_compressed_input(input: &InputSerialization) -> bool { + input + .compression_type + .as_ref() + .is_some_and(|compression| compression.as_str() != CompressionType::NONE) +} + fn normalize_input_serialization(input: &mut InputSerialization) -> S3Result<()> { let format_count = usize::from(input.csv.is_some()) + usize::from(input.json.is_some()) + usize::from(input.parquet.is_some()); @@ -298,15 +655,19 @@ fn normalize_input_serialization(input: &mut InputSerialization) -> S3Result<()> match compression.as_str() { CompressionType::NONE => {} CompressionType::GZIP | CompressionType::BZIP2 => { - return Err(s3_error!( - NotImplemented, - "SelectObjectContent currently supports only uncompressed input" - )); + if input.parquet.is_some() { + return Err(S3Error::with_message( + S3ErrorCode::InvalidRequestParameter, + INVALID_REQUEST_PARAMETER_MESSAGE, + )); + } } _ => return Err(map_select_error_to_s3(&SelectError::InvalidCompressionFormat)), } } - input.compression_type = Some(CompressionType::from_static(CompressionType::NONE)); + input + .compression_type + .get_or_insert_with(|| CompressionType::from_static(CompressionType::NONE)); if let Some(csv) = input.csv.as_mut() { if csv.allow_quoted_record_delimiter.unwrap_or(false) { @@ -531,91 +892,624 @@ impl SelectOutputEncoder { Self { format } } - fn encode_batch(&mut self, batch: &RecordBatch) -> S3Result> { - let bytes = match &self.format { - SelectOutputFormat::Csv(config) => encode_csv_batch(batch, config)?, - SelectOutputFormat::Json(config) => encode_json_batch(batch, config)?, - }; - Ok(split_records_payload(bytes)) - } -} - -fn encode_csv_batch(batch: &RecordBatch, config: &CSVOutput) -> S3Result> { - let mut buffer = Vec::new(); - let mut builder = CsvWriterBuilder::new().with_header(false); - if let Some(delimiter) = config.field_delimiter.as_deref() { - builder = builder.with_delimiter(delimiter.as_bytes()[0]); - } - if let Some(quote) = config.quote_character.as_deref() { - builder = builder.with_quote(quote.as_bytes()[0]); - } - if let Some(escape) = config.quote_escape_character.as_deref() { - builder = builder.with_escape(escape.as_bytes()[0]); - } - if let Some(record_delimiter) = config.record_delimiter.as_deref() { - builder = builder.with_line_terminator(csv_terminator(record_delimiter)); - } - if let Some(quote_fields) = config.quote_fields.as_ref() - && quote_fields.as_str() == QuoteFields::ALWAYS - { - builder = builder.with_quote_style(QuoteStyle::Always); - } - - let mut writer = builder.build(&mut buffer); - writer.write(batch).map_err(internal_select_error)?; - drop(writer); - Ok(buffer) -} - -fn csv_terminator(value: &str) -> Terminator { - if value == "\r\n" { - Terminator::CRLF - } else { - Terminator::Any(value.as_bytes()[0]) - } -} - -fn encode_json_batch(batch: &RecordBatch, config: &JSONOutput) -> S3Result> { - let mut buffer = Vec::new(); - let mut writer = JsonWriterBuilder::new() - .with_explicit_nulls(true) - .build::<_, LineDelimited>(&mut buffer); - writer.write(batch).map_err(internal_select_error)?; - writer.finish().map_err(internal_select_error)?; - drop(writer); - - if let Some(delimiter) = config.record_delimiter.as_deref() - && delimiter != "\n" - { - return Ok(replace_json_record_delimiter(&buffer, delimiter.as_bytes())); - } - Ok(buffer) -} - -fn replace_json_record_delimiter(buffer: &[u8], delimiter: &[u8]) -> Vec { - let mut output = Vec::with_capacity(buffer.len()); - for byte in buffer { - if *byte == b'\n' { - output.extend_from_slice(delimiter); - } else { - output.push(*byte); + fn encode_batch_limited( + &mut self, + batch: &RecordBatch, + rows: Range, + buffer: &mut BytesMut, + max_bytes: usize, + ) -> S3Result { + match &self.format { + SelectOutputFormat::Csv(config) => encode_csv_batch_limited(batch, rows, config, buffer, max_bytes), + SelectOutputFormat::Json(config) => encode_json_batch_limited(batch, rows, config, buffer, max_bytes), } } - output } -fn split_records_payload(bytes: Vec) -> Vec { - if bytes.is_empty() { - return Vec::new(); +#[cfg(test)] +fn encode_csv_batch(batch: &RecordBatch, config: &CSVOutput, buffer: &mut BytesMut) -> S3Result<()> { + if encode_csv_batch_limited(batch, 0..batch.num_rows(), config, buffer, usize::MAX)? == batch.num_rows() { + Ok(()) + } else { + Err(internal_select_error(io::Error::other("S3 Select output length overflow"))) } - let bytes = Bytes::from(bytes); - if bytes.len() <= RECORDS_CHUNK_TARGET { - return vec![bytes]; +} + +fn encode_csv_batch_limited( + batch: &RecordBatch, + rows: Range, + config: &CSVOutput, + buffer: &mut BytesMut, + max_bytes: usize, +) -> S3Result { + let options = FormatOptions::default(); + let mut formatters = Vec::new(); + let field_delimiter = config.field_delimiter.as_deref().unwrap_or(",").as_bytes()[0]; + let quote = config.quote_character.as_deref().unwrap_or("\"").as_bytes()[0]; + let quote_escape = config.quote_escape_character.as_deref().unwrap_or("\"").as_bytes()[0]; + let record_delimiter = config.record_delimiter.as_deref().unwrap_or("\n").as_bytes(); + let quote_all = config + .quote_fields + .as_ref() + .is_some_and(|quote_fields| quote_fields.as_str() == QuoteFields::ALWAYS); + let mut output = LimitedBytesWriter::new(buffer, max_bytes); + let mut field = BoundedText::default(); + let mut encoded_rows = 0; + for row in rows { + let row_start = output.checkpoint(); + for column in 0..batch.num_columns() { + if column > 0 && output.write_all(&[field_delimiter]).is_err() { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + if formatters.len() == column { + let array = batch.column(column); + if array.data_type().is_nested() { + return Err(internal_select_error(datafusion::arrow::error::ArrowError::CsvError(format!( + "Nested type {} is not supported in CSV", + array.data_type() + )))); + } + formatters.push(ArrayFormatter::try_new(array.as_ref(), &options).map_err(internal_select_error)?); + } + let formatter = formatters + .get(column) + .ok_or_else(|| internal_select_error(io::Error::other("missing S3 Select CSV formatter")))?; + field.reset(output.remaining()); + let result = formatter.value(row).write(&mut field); + if field.limit_exceeded { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + result.map_err(internal_select_error)?; + if write_csv_field(&mut output, field.value.as_bytes(), field_delimiter, quote, quote_escape, quote_all).is_err() { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + } + if output.write_all(record_delimiter).is_err() { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + encoded_rows += 1; + } + Ok(encoded_rows) +} + +fn write_csv_field( + output: &mut LimitedBytesWriter<'_>, + field: &[u8], + delimiter: u8, + quote: u8, + quote_escape: u8, + quote_all: bool, +) -> io::Result<()> { + let quote_field = quote_all || csv_field_needs_quotes(field, delimiter, quote); + if !quote_field { + return output.write_all(field); + } + + output.write_all(&[quote])?; + let mut start = 0; + while let Some(relative) = field[start..].iter().position(|byte| *byte == quote) { + let position = start + relative; + output.write_all(&field[start..position])?; + output.write_all(&[quote_escape, quote])?; + start = position + 1; + } + output.write_all(&field[start..])?; + output.write_all(&[quote]) +} + +fn csv_field_needs_quotes(field: &[u8], delimiter: u8, quote: u8) -> bool { + if field.is_empty() { + return false; + } + if field == b"\\." + || field + .iter() + .copied() + .any(|byte| matches!(byte, b'\r' | b'\n') || byte == delimiter || byte == quote) + { + return true; + } + std::str::from_utf8(field) + .ok() + .and_then(|value| value.chars().next()) + .is_some_and(char::is_whitespace) +} + +#[derive(Default)] +struct BoundedText { + value: String, + max_bytes: usize, + limit_exceeded: bool, +} + +impl BoundedText { + fn reset(&mut self, max_bytes: usize) { + self.value.clear(); + self.max_bytes = max_bytes; + self.limit_exceeded = false; + } +} + +impl fmt::Write for BoundedText { + fn write_str(&mut self, value: &str) -> fmt::Result { + let Some(len) = self.value.len().checked_add(value.len()) else { + self.limit_exceeded = true; + return Err(fmt::Error); + }; + if len > self.max_bytes { + self.limit_exceeded = true; + return Err(fmt::Error); + } + self.value.push_str(value); + Ok(()) + } +} + +struct LimitedJsonValueEncoder<'a> { + array: &'a dyn Array, + options: &'a EncoderOptions, + kind: LimitedJsonValueKind<'a>, +} + +enum LimitedJsonValueKind<'a> { + Scalar(NullableEncoder<'a>), + List { + array: &'a dyn ListLikeArray, + value_field: &'a FieldRef, + value_array: &'a dyn Array, + values: Option>>, + }, + Struct { + fields: &'a [FieldRef], + arrays: &'a [datafusion::arrow::array::ArrayRef], + values: Vec>, + }, + Dictionary { + value_index: Box usize + 'a>, + value_field: &'a FieldRef, + value_array: &'a dyn Array, + values: Option>>, + }, + Indexed { + value_index: Box usize + 'a>, + value_field: &'a FieldRef, + value_array: &'a dyn Array, + values: Option>>, + }, + Map { + array: &'a MapArray, + field: &'a FieldRef, + keys: Option>>, + values: Option>>, + }, +} + +impl<'a> LimitedJsonValueEncoder<'a> { + fn try_new(field: &'a FieldRef, array: &'a dyn Array, options: &'a EncoderOptions) -> Result { + macro_rules! dictionary { + ($key:ty) => {{ + let dictionary = array.as_dictionary::<$key>(); + LimitedJsonValueKind::Dictionary { + value_index: Box::new(move |row| dictionary.keys().value(row).as_usize()), + value_field: field, + value_array: dictionary.values().as_ref(), + values: None, + } + }}; + } + macro_rules! run_end_encoded { + ($run_end:ty) => {{ + let run = array.as_run::<$run_end>(); + LimitedJsonValueKind::Indexed { + value_index: Box::new(move |row| run.get_physical_index(row)), + value_field: field, + value_array: run.values().as_ref(), + values: None, + } + }}; + } + + let kind = match array.data_type() { + DataType::List(value_field) => { + let list = array.as_list::(); + LimitedJsonValueKind::List { + array: list, + value_field, + value_array: list.values().as_ref(), + values: None, + } + } + DataType::LargeList(value_field) => { + let list = array.as_list::(); + LimitedJsonValueKind::List { + array: list, + value_field, + value_array: list.values().as_ref(), + values: None, + } + } + DataType::ListView(value_field) => { + let list = array.as_list_view::(); + LimitedJsonValueKind::List { + array: list, + value_field, + value_array: list.values().as_ref(), + values: None, + } + } + DataType::LargeListView(value_field) => { + let list = array.as_list_view::(); + LimitedJsonValueKind::List { + array: list, + value_field, + value_array: list.values().as_ref(), + values: None, + } + } + DataType::FixedSizeList(value_field, _) => { + let list = array.as_fixed_size_list(); + LimitedJsonValueKind::List { + array: list, + value_field, + value_array: list.values().as_ref(), + values: None, + } + } + DataType::Struct(fields) => LimitedJsonValueKind::Struct { + fields: fields.as_ref(), + arrays: array.as_struct().columns(), + values: Vec::new(), + }, + DataType::Dictionary(key_type, _) => match key_type.as_ref() { + DataType::Int8 => dictionary!(Int8Type), + DataType::Int16 => dictionary!(Int16Type), + DataType::Int32 => dictionary!(Int32Type), + DataType::Int64 => dictionary!(Int64Type), + DataType::UInt8 => dictionary!(UInt8Type), + DataType::UInt16 => dictionary!(UInt16Type), + DataType::UInt32 => dictionary!(UInt32Type), + DataType::UInt64 => dictionary!(UInt64Type), + key_type => { + return Err(ArrowError::JsonError(format!( + "Unsupported dictionary key type for JSON encoding: {key_type:?}" + ))); + } + }, + DataType::RunEndEncoded(run_ends, _) => match run_ends.data_type() { + DataType::Int16 => run_end_encoded!(Int16Type), + DataType::Int32 => run_end_encoded!(Int32Type), + DataType::Int64 => run_end_encoded!(Int64Type), + run_end_type => { + return Err(ArrowError::JsonError(format!( + "Unsupported run-end type for JSON encoding: {run_end_type:?}" + ))); + } + }, + DataType::Map(_, _) => { + let map = array.as_map(); + if !matches!(map.keys().data_type(), DataType::Utf8 | DataType::LargeUtf8 | DataType::Utf8View) { + return Err(ArrowError::JsonError(format!( + "Only UTF8 keys supported by JSON MapArray Writer: got {:?}", + map.keys().data_type() + ))); + } + if map.keys().null_count() != 0 { + return Err(ArrowError::InvalidArgumentError("Encountered nulls in MapArray keys".to_string())); + } + if map.entries().nulls().is_some_and(|nulls| nulls.null_count() != 0) { + return Err(ArrowError::InvalidArgumentError("Encountered nulls in MapArray entries".to_string())); + } + LimitedJsonValueKind::Map { + array: map, + field, + keys: None, + values: None, + } + } + _ => LimitedJsonValueKind::Scalar(make_encoder(field, array, options)?), + }; + Ok(Self { array, options, kind }) + } + + fn encode(&mut self, row: usize, output: &mut LimitedBytesWriter<'_>, scratch: &mut Vec) -> Result { + if self.array.is_null(row) { + return Ok(write_limited(output, b"null")); + } + self.encode_non_null(row, output, scratch) + } + + fn encode_non_null( + &mut self, + row: usize, + output: &mut LimitedBytesWriter<'_>, + scratch: &mut Vec, + ) -> Result { + let array = self.array; + // NullArray has no physical null buffer, including behind a dictionary index. + if matches!(array.data_type(), DataType::Null) { + return Ok(write_limited(output, b"null")); + } + let options = self.options; + match &mut self.kind { + LimitedJsonValueKind::Scalar(encoder) => Ok(write_json_scalar_limited(array, encoder, row, output, scratch)), + LimitedJsonValueKind::List { + array, + value_field, + value_array, + values, + } => { + if !write_limited(output, b"[") { + return Ok(false); + } + for (index, value_row) in array.element_range(row).enumerate() { + if index > 0 && !write_limited(output, b",") { + return Ok(false); + } + let values = Self::lazy_value_encoder(values, value_field, *value_array, options)?; + if !values.encode(value_row, output, scratch)? { + return Ok(false); + } + } + Ok(write_limited(output, b"]")) + } + LimitedJsonValueKind::Struct { fields, arrays, values } => { + if !write_limited(output, b"{") { + return Ok(false); + } + for (index, field) in fields.iter().enumerate() { + if (index > 0 && !write_limited(output, b",")) + || !write_json_string_limited(output, field.name()) + || !write_limited(output, b":") + { + return Ok(false); + } + if values.len() == index { + values.push(Self::try_new(field, arrays[index].as_ref(), options)?); + } + let value = values + .get_mut(index) + .ok_or_else(|| ArrowError::JsonError("S3 Select JSON encoder state is inconsistent".to_string()))?; + if !value.encode(row, output, scratch)? { + return Ok(false); + } + } + Ok(write_limited(output, b"}")) + } + // Arrow checks dictionary key nulls but delegates value nulls to the value encoder. + LimitedJsonValueKind::Dictionary { + value_index, + value_field, + value_array, + values, + } => Self::lazy_value_encoder(values, value_field, *value_array, options)?.encode_non_null( + value_index(row), + output, + scratch, + ), + LimitedJsonValueKind::Indexed { + value_index, + value_field, + value_array, + values, + } => Self::lazy_value_encoder(values, value_field, *value_array, options)?.encode(value_index(row), output, scratch), + LimitedJsonValueKind::Map { + array, + field, + keys, + values, + } => { + if !write_limited(output, b"{") { + return Ok(false); + } + let offsets = array.value_offsets(); + let start = offsets[row].as_usize(); + let end = offsets[row + 1].as_usize(); + for (index, value_row) in (start..end).enumerate() { + if index > 0 && !write_limited(output, b",") { + return Ok(false); + } + let keys = Self::lazy_value_encoder(keys, field, array.keys(), options)?; + if !keys.encode(value_row, output, scratch)? || !write_limited(output, b":") { + return Ok(false); + } + let values = Self::lazy_value_encoder(values, field, array.values(), options)?; + if !values.encode(value_row, output, scratch)? { + return Ok(false); + } + } + Ok(write_limited(output, b"}")) + } + } + } + + fn lazy_value_encoder<'b>( + slot: &'b mut Option>>, + field: &'a FieldRef, + array: &'a dyn Array, + options: &'a EncoderOptions, + ) -> Result<&'b mut LimitedJsonValueEncoder<'a>, ArrowError> { + if slot.is_none() { + *slot = Some(Box::new(Self::try_new(field, array, options)?)); + } + slot.as_deref_mut() + .ok_or_else(|| ArrowError::JsonError("S3 Select JSON encoder state is inconsistent".to_string())) + } +} + +fn write_json_scalar_limited( + array: &dyn Array, + encoder: &mut NullableEncoder<'_>, + row: usize, + output: &mut LimitedBytesWriter<'_>, + scratch: &mut Vec, +) -> bool { + match array.data_type() { + DataType::Utf8 => write_json_string_limited(output, array.as_string::().value(row)), + DataType::LargeUtf8 => write_json_string_limited(output, array.as_string::().value(row)), + DataType::Utf8View => write_json_string_limited(output, array.as_string_view().value(row)), + DataType::Binary => write_json_binary_limited(output, array.as_binary::().value(row), scratch), + DataType::LargeBinary => write_json_binary_limited(output, array.as_binary::().value(row), scratch), + DataType::BinaryView => write_json_binary_limited(output, array.as_binary_view().value(row), scratch), + DataType::FixedSizeBinary(_) => write_json_binary_limited(output, array.as_fixed_size_binary().value(row), scratch), + _ => { + scratch.clear(); + encoder.encode(row, scratch); + write_limited(output, scratch) + } + } +} + +fn write_json_string_limited(output: &mut LimitedBytesWriter<'_>, value: &str) -> bool { + if value.len().checked_add(2).is_none_or(|minimum| minimum > output.remaining()) { + return false; + } + serde_json::to_writer(output, value).is_ok() +} + +fn write_json_binary_limited(output: &mut LimitedBytesWriter<'_>, value: &[u8], scratch: &mut Vec) -> bool { + let Some(encoded_len) = value.len().checked_mul(2).and_then(|bytes| bytes.checked_add(2)) else { + return false; + }; + if encoded_len > output.remaining() || !write_limited(output, b"\"") { + return false; + } + const HEX: &[u8; 16] = b"0123456789abcdef"; + const INPUT_CHUNK_BYTES: usize = 2048; + for chunk in value.chunks(INPUT_CHUNK_BYTES) { + scratch.clear(); + scratch.reserve(chunk.len() * 2); + for byte in chunk { + scratch.push(HEX[(byte >> 4) as usize]); + scratch.push(HEX[(byte & 0x0f) as usize]); + } + if !write_limited(output, scratch) { + return false; + } + } + write_limited(output, b"\"") +} + +fn write_limited(output: &mut LimitedBytesWriter<'_>, bytes: &[u8]) -> bool { + output.write_all(bytes).is_ok() +} + +fn json_record_delimiter(config: &JSONOutput) -> &[u8] { + if let Some(delimiter) = config.record_delimiter.as_deref() { + delimiter.as_bytes() + } else { + b"\n" + } +} + +#[cfg(test)] +fn encode_json_batch(batch: &RecordBatch, config: &JSONOutput, buffer: &mut BytesMut) -> S3Result<()> { + if encode_json_batch_limited(batch, 0..batch.num_rows(), config, buffer, usize::MAX)? == batch.num_rows() { + Ok(()) + } else { + Err(internal_select_error(io::Error::other("S3 Select output length overflow"))) + } +} + +fn encode_json_batch_limited( + batch: &RecordBatch, + rows: Range, + config: &JSONOutput, + buffer: &mut BytesMut, + max_bytes: usize, +) -> S3Result { + let options = EncoderOptions::default().with_explicit_nulls(true); + let schema = batch.schema(); + let fields = schema.fields(); + let arrays = batch.columns(); + let mut values = Vec::new(); + let delimiter = json_record_delimiter(config); + let mut output = LimitedBytesWriter::new(buffer, max_bytes); + let mut scratch = Vec::new(); + let mut encoded_rows = 0; + for row in rows { + let row_start = output.checkpoint(); + if !write_limited(&mut output, b"{") { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + for (index, field) in fields.iter().enumerate() { + if (index > 0 && !write_limited(&mut output, b",")) + || !write_json_string_limited(&mut output, field.name()) + || !write_limited(&mut output, b":") + { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + if values.len() == index { + values.push( + LimitedJsonValueEncoder::try_new(field, arrays[index].as_ref(), &options).map_err(internal_select_error)?, + ); + } + let value = values + .get_mut(index) + .ok_or_else(|| internal_select_error(io::Error::other("S3 Select JSON encoder state is inconsistent")))?; + if !value.encode(row, &mut output, &mut scratch).map_err(internal_select_error)? { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + } + if !write_limited(&mut output, b"}") || !write_limited(&mut output, delimiter) { + output.rollback_to(row_start); + return Ok(encoded_rows); + } + encoded_rows += 1; + } + Ok(encoded_rows) +} + +struct LimitedBytesWriter<'a> { + buffer: &'a mut BytesMut, + max_bytes: usize, + written: usize, +} + +impl<'a> LimitedBytesWriter<'a> { + fn new(buffer: &'a mut BytesMut, max_bytes: usize) -> Self { + Self { + buffer, + max_bytes, + written: 0, + } + } + + fn remaining(&self) -> usize { + self.max_bytes.saturating_sub(self.written) + } + + fn checkpoint(&self) -> usize { + self.buffer.len() + } + + fn rollback_to(&mut self, checkpoint: usize) { + self.buffer.truncate(checkpoint); + } +} + +impl Write for LimitedBytesWriter<'_> { + fn write(&mut self, bytes: &[u8]) -> io::Result { + let Some(written) = self.written.checked_add(bytes.len()) else { + return Err(io::Error::other("S3 Select output length overflow")); + }; + if written > self.max_bytes { + return Err(io::Error::other("S3 Select encode turn limit exceeded")); + } + self.buffer.extend_from_slice(bytes); + self.written = written; + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) } - (0..bytes.len()) - .step_by(RECORDS_CHUNK_TARGET) - .map(|start| bytes.slice(start..(start + RECORDS_CHUNK_TARGET).min(bytes.len()))) - .collect() } struct SelectProgress { @@ -667,11 +1561,16 @@ fn map_query_error_to_s3(err: QueryError) -> S3Error { fn map_select_error_to_s3(err: &SelectError) -> S3Error { match err { SelectError::InvalidCompressionFormat => S3Error::with_message(S3ErrorCode::InvalidCompressionFormat, err.to_string()), + SelectError::InvalidCompressionFormatForObject { .. } => { + S3Error::with_message(S3ErrorCode::InvalidCompressionFormat, err.to_string()) + } SelectError::InvalidDataSource => S3Error::with_message(S3ErrorCode::InvalidDataSource, err.to_string()), SelectError::TruncatedInput => S3Error::with_message(S3ErrorCode::TruncatedInput, err.to_string()), + SelectError::UnsupportedScanRangeInput => S3Error::with_message(S3ErrorCode::UnsupportedScanRangeInput, err.to_string()), SelectError::CsvParsingError => S3Error::with_message(S3ErrorCode::CSVParsingError, err.to_string()), SelectError::JsonParsingError => S3Error::with_message(S3ErrorCode::JSONParsingError, err.to_string()), SelectError::ParquetParsingError => S3Error::with_message(S3ErrorCode::ParquetParsingError, err.to_string()), + SelectError::OverMaxRecordSize => S3Error::with_message(S3ErrorCode::OverMaxRecordSize, err.to_string()), SelectError::ParseSelectFailure { message } => custom_bad_request(PARSE_SELECT_FAILURE_CODE, message.clone()), SelectError::InvalidQuery => custom_bad_request(INVALID_QUERY_CODE, err.to_string()), SelectError::InvalidDataType => S3Error::with_message(S3ErrorCode::InvalidDataType, err.to_string()), @@ -706,6 +1605,10 @@ fn internal_select_error(_error: impl std::error::Error + Send + Sync + 'static) map_select_error_to_s3(&SelectError::InternalError) } +fn over_max_record_size_error() -> S3Error { + S3Error::with_message(S3ErrorCode::OverMaxRecordSize, OVER_MAX_RECORD_SIZE_MESSAGE) +} + fn custom_bad_request(code: &'static str, message: String) -> S3Error { let mut err = S3Error::with_message(S3ErrorCode::Custom(code.into()), message); err.set_status_code(StatusCode::BAD_REQUEST); @@ -727,9 +1630,15 @@ mod tests { use super::*; use datafusion::{ arrow::{ - array::{Array, ListArray, StringArray}, + array::{ + Array, ArrayRef, BinaryArray, BinaryViewArray, DictionaryArray, FixedSizeBinaryArray, Int32Array, + LargeBinaryArray, LargeListArray, LargeListViewArray, LargeStringArray, ListArray, ListViewArray, MapArray, + NullArray, RunArray, StringArray, StringDictionaryBuilder, StringViewArray, StructArray, + builder::{BooleanBuilder, FixedSizeListBuilder, Int32Builder, ListBuilder}, + }, datatypes::{DataType, Field, Int32Type, Schema}, error::ArrowError, + json::writer::{LineDelimited, WriterBuilder}, }, physical_plan::stream::RecordBatchStreamAdapter, sql::sqlparser::parser::ParserError, @@ -882,6 +1791,25 @@ mod tests { } } + fn pending_output() -> SendableRecordBatchStream { + Box::pin(RecordBatchStreamAdapter::new( + Arc::new(Schema::empty()), + futures::stream::pending::>(), + )) + } + + fn large_pending_output(chunks: usize) -> SendableRecordBatchStream { + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let value = "x".repeat(RECORDS_CHUNK_TARGET * chunks); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(StringArray::from(vec![value]))]) + .expect("test record batch should be valid"); + Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { Ok::<_, DataFusionError>(batch) }) + .chain(futures::stream::pending::>()), + )) + } + fn spawn_test_producer( output: SendableRecordBatchStream, channel_capacity: usize, @@ -890,24 +1818,53 @@ mod tests { mpsc::Receiver>, tokio::sync::oneshot::Receiver<()>, ) { - let (tx, rx) = mpsc::channel(channel_capacity); - let terminal_permit = tx - .clone() - .try_reserve_owned() - .expect("test channel should reserve terminal capacity"); + spawn_test_producer_with(output, channel_capacity, csv_validation(), Duration::from_secs(300)) + } + + fn spawn_test_producer_with( + output: SendableRecordBatchStream, + channel_capacity: usize, + validation: SelectValidation, + deadline_after: Duration, + ) -> ( + tokio::task::JoinHandle<()>, + mpsc::Receiver>, + tokio::sync::oneshot::Receiver<()>, + ) { + let (event_channel, rx) = test_event_channel(channel_capacity); let (lease, lease_released) = lease_drop_signal(); let producer = tokio::spawn(send_select_events_until_deadline( output, - SelectEventChannel { tx, terminal_permit }, - csv_validation(), + event_channel, + validation, Arc::new(SelectInputMetrics::default()), - Instant::now() + std::time::Duration::from_secs(1), + Instant::now() + deadline_after, 300, lease, )); (producer, rx, lease_released) } + fn test_event_channel(channel_capacity: usize) -> (SelectEventChannel, mpsc::Receiver>) { + let (tx, rx) = mpsc::channel(channel_capacity); + let terminal_records_permit = tx + .clone() + .try_reserve_owned() + .expect("test channel should reserve terminal Records capacity"); + let terminal_permit = tx + .clone() + .try_reserve_owned() + .expect("test channel should reserve terminal capacity"); + ( + SelectEventChannel { + tx, + terminal_records_permit: Some(terminal_records_permit), + terminal_permit, + }, + rx, + ) + } + #[test] fn validate_rejects_http_range() { let mut input = base_input(); @@ -995,8 +1952,20 @@ mod tests { S3ErrorCode::InvalidCompressionFormat, StatusCode::BAD_REQUEST, ), + ( + SelectError::InvalidCompressionFormatForObject { + compression: CompressionType::GZIP, + }, + S3ErrorCode::InvalidCompressionFormat, + StatusCode::BAD_REQUEST, + ), (SelectError::InvalidDataSource, S3ErrorCode::InvalidDataSource, StatusCode::BAD_REQUEST), (SelectError::TruncatedInput, S3ErrorCode::TruncatedInput, StatusCode::BAD_REQUEST), + ( + SelectError::UnsupportedScanRangeInput, + S3ErrorCode::UnsupportedScanRangeInput, + StatusCode::BAD_REQUEST, + ), (SelectError::CsvParsingError, S3ErrorCode::CSVParsingError, StatusCode::BAD_REQUEST), (SelectError::JsonParsingError, S3ErrorCode::JSONParsingError, StatusCode::BAD_REQUEST), ( @@ -1004,6 +1973,7 @@ mod tests { S3ErrorCode::ParquetParsingError, StatusCode::BAD_REQUEST, ), + (SelectError::OverMaxRecordSize, S3ErrorCode::OverMaxRecordSize, StatusCode::BAD_REQUEST), ( SelectError::ParseSelectFailure { message: "invalid SELECT expression".to_string(), @@ -1108,22 +2078,17 @@ mod tests { #[tokio::test(start_paused = true)] async fn producer_deadline_cancels_backpressured_send() { - let output = Box::pin(RecordBatchStreamAdapter::new( - Arc::new(Schema::empty()), - futures::stream::pending::>(), - )); - let (tx, mut rx) = mpsc::channel(2); - let terminal_permit = tx - .clone() - .try_reserve_owned() - .expect("test channel should reserve terminal capacity"); - tx.send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) + let output = pending_output(); + let (event_channel, mut rx) = test_event_channel(3); + event_channel + .tx + .send(Ok(SelectObjectContentEvent::Cont(ContinuationEvent::default()))) .await .expect("test channel should accept the prefilled event"); let (lease, lease_released) = lease_drop_signal(); let producer = tokio::spawn(send_select_events_until_deadline( output, - SelectEventChannel { tx, terminal_permit }, + event_channel, csv_validation(), Arc::new(SelectInputMetrics::default()), Instant::now() + std::time::Duration::from_secs(1), @@ -1147,6 +2112,784 @@ mod tests { assert!(lease_released.await.is_ok(), "timeout should release the snapshot lease"); } + #[tokio::test(start_paused = true)] + async fn deadline_preempts_multi_slice_batch_encoding() { + let value = "x".repeat(64 * 1024); + let mut builder = StringDictionaryBuilder::::new(); + for _ in 0..(MAX_ENCODE_ROWS_PER_TURN + 1) { + builder.append(&value).expect("dictionary value should append"); + } + let values = builder.finish(); + let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(values)]).expect("test record batch should be valid"); + let output = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { Ok::<_, DataFusionError>(batch) }), + )); + let (event_channel, mut rx) = test_event_channel(4); + let (lease, lease_released) = lease_drop_signal(); + let producer = send_select_events_until_deadline( + output, + event_channel, + csv_validation(), + Arc::new(SelectInputMetrics::default()), + Instant::now() + Duration::from_secs(1), + 1, + lease, + ); + tokio::pin!(producer); + + assert!(futures::poll!(producer.as_mut()).is_pending()); + assert!(futures::poll!(producer.as_mut()).is_pending()); + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + tokio::time::advance(Duration::from_secs(1)).await; + assert!(futures::poll!(producer.as_mut()).is_ready()); + + let Some(Ok(SelectObjectContentEvent::Records(records))) = rx.recv().await else { + panic!("the first encoded slice should flush before the timeout"); + }; + assert_eq!(records.payload.as_ref().map(Bytes::len), Some(value.len() + 1)); + let timeout = rx + .recv() + .await + .expect("deadline should send one terminal error") + .expect_err("deadline terminal event should be an error"); + assert_eq!(timeout.code(), &S3ErrorCode::Busy); + assert!(rx.recv().await.is_none()); + assert!(lease_released.await.is_ok(), "deadline should release the snapshot lease"); + } + + #[test] + fn skewed_dictionary_batch_stays_within_one_encode_turn() { + let value = "x".repeat(64 * 1024); + let mut builder = StringDictionaryBuilder::::new(); + builder.append("").expect("empty dictionary value should append"); + for _ in 0..MAX_ENCODE_ROWS_PER_TURN { + builder.append(&value).expect("large dictionary value should append"); + } + let values = builder.finish(); + let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default())); + let mut buffer = BytesMut::new(); + + let encoded_rows = + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("dictionary rows should encode successfully"); + + assert_eq!(encoded_rows, 1, "shared dictionary values must be charged before batching rows"); + assert_eq!(buffer.as_ref(), b"\n"); + } + + #[test] + fn unused_large_dictionary_value_does_not_force_per_row_encoding() { + let row_count = MAX_ENCODE_ROWS_PER_TURN; + let keys = Int32Array::from(vec![0; row_count]); + let unused = "x".repeat(64 * 1024); + let dictionary = Arc::new(StringArray::from(vec!["", unused.as_str()])); + let values = DictionaryArray::::try_new(keys, dictionary).expect("test dictionary should be valid"); + let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default())); + let mut buffer = BytesMut::new(); + + let encoded_rows = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("referenced empty values should batch"); + + assert_eq!(encoded_rows, row_count); + assert_eq!(buffer.as_ref(), "\n".repeat(row_count).as_bytes()); + } + + #[test] + fn skewed_json_dictionary_keeps_the_complete_bounded_prefix() { + let row_count = MAX_ENCODE_ROWS_PER_TURN; + let mut keys = vec![0; row_count]; + keys[row_count - 1] = 1; + let large = "x".repeat(70 * 1024); + let dictionary = Arc::new(StringArray::from(vec!["a", large.as_str()])); + let values = + DictionaryArray::::try_new(Int32Array::from(keys), dictionary).expect("test dictionary should be valid"); + let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + let encoded_rows = + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("the bounded dictionary prefix should encode"); + + assert_eq!(encoded_rows, row_count - 1); + assert_eq!(buffer.iter().filter(|byte| **byte == b'\n').count(), encoded_rows); + } + + #[test] + fn short_rows_share_an_encode_turn() { + let row_count = MAX_ENCODE_ROWS_PER_TURN; + let values = StringArray::from(vec!["x"; row_count]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default())); + let mut buffer = BytesMut::new(); + + let encoded_rows = + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("short rows should encode successfully"); + + assert!(encoded_rows > 1, "small rows should not construct one Arrow writer per row"); + assert!(encoded_rows <= MAX_ENCODE_ROWS_PER_TURN); + assert_eq!(buffer.as_ref(), "x\n".repeat(encoded_rows).as_bytes()); + } + + #[test] + fn small_nested_json_rows_share_an_encode_turn() { + let values = ListArray::from_iter_primitive::([Some([Some(1)]), Some([Some(2)])]); + let schema = Arc::new(Schema::new(vec![Field::new("items", values.data_type().clone(), false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + let encoded_rows = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("small nested JSON rows should encode"); + + assert_eq!(encoded_rows, 2); + assert_eq!(buffer.as_ref(), b"{\"items\":[1]}\n{\"items\":[2]}\n"); + } + + #[test] + fn large_second_nested_json_row_stops_after_the_complete_prefix() { + let mut builder = ListBuilder::new(BooleanBuilder::new()); + builder.values().append_value(true); + builder.append(true); + for _ in 0..(ENCODE_TURN_TARGET_BYTES / b"true,".len() + 8) { + builder.values().append_value(true); + } + builder.append(true); + let values = builder.finish(); + let schema = Arc::new(Schema::new(vec![Field::new("items", values.data_type().clone(), false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + let encoded_rows = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("the complete first row should encode"); + + assert_eq!(encoded_rows, 1); + assert_eq!(buffer.as_ref(), b"{\"items\":[true]}\n"); + + buffer.clear(); + let encoded_rows = + encode_batch_turn(&mut encoder, &batch, 1, &mut buffer).expect("the larger second row should encode alone"); + assert_eq!(encoded_rows, 1); + assert!(buffer.len() > ENCODE_TURN_TARGET_BYTES); + assert!(buffer.len() <= MAX_SELECT_OUTPUT_RECORD_BYTES); + } + + #[test] + fn csv_turn_rolls_back_a_partially_escaped_second_row() { + let quoted = "\"".repeat(40 * 1024); + let values = StringArray::from(vec!["ok", quoted.as_str()]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default())); + let mut buffer = BytesMut::new(); + + let encoded_rows = + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("the complete first row should remain staged"); + + assert_eq!(encoded_rows, 1); + assert_eq!(buffer.as_ref(), b"ok\n"); + } + + #[test] + fn csv_encoder_error_discards_the_partial_current_turn() { + let values = StringArray::from(vec!["ok"]); + let nested = ListArray::from_iter_primitive::([Some([Some(1)])]); + let schema = Arc::new(Schema::new(vec![ + Field::new("value", DataType::Utf8, false), + Field::new("nested", nested.data_type().clone(), false), + ])); + let batch = + RecordBatch::try_new(schema, vec![Arc::new(values), Arc::new(nested)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default())); + let mut buffer = BytesMut::from(b"staged\n".as_slice()); + + let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect_err("nested CSV output should fail after the first column"); + + assert_eq!(error.code(), &S3ErrorCode::InternalError); + assert_eq!(buffer.as_ref(), b"staged\n"); + } + + #[test] + fn oversized_nested_json_stops_at_the_output_budget() { + let value_count = MAX_SELECT_OUTPUT_RECORD_BYTES / b"true,".len() + 1; + let mut builder = ListBuilder::new(BooleanBuilder::new()); + for _ in 0..value_count { + builder.values().append_value(true); + } + builder.append(true); + let values = builder.finish(); + let schema = Arc::new(Schema::new(vec![Field::new("items", values.data_type().clone(), false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect_err("nested JSON larger than one MiB must stop at the output budget"); + + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + + #[test] + fn nested_json_struct_map_dictionary_and_run_end_match_arrow_semantics() { + let profile = StructArray::from(vec![ + ( + Arc::new(Field::new("name", DataType::Utf8, false)), + Arc::new(StringArray::from(vec!["a\n"])) as ArrayRef, + ), + ( + Arc::new(Field::new("count", DataType::Int32, true)), + Arc::new(Int32Array::from(vec![None])) as ArrayRef, + ), + ]); + let labels = + MapArray::from_vec_of_maps::(vec![Some(vec![("a", Some(1)), ("b", None)])], true); + let mut dictionary = StringDictionaryBuilder::::new(); + dictionary.append("small").expect("dictionary value should append"); + let dictionary = dictionary.finish(); + let run_ends = Int32Array::from(vec![1]); + let run_values = Arc::new(StringArray::from(vec!["run"])) as ArrayRef; + let run = RunArray::::try_new(&run_ends, &run_values).expect("run-end encoded value should be valid"); + let schema = Arc::new(Schema::new(vec![ + Field::new("profile", profile.data_type().clone(), false), + Field::new("labels", labels.data_type().clone(), false), + Field::new("code", dictionary.data_type().clone(), false), + Field::new("run", run.data_type().clone(), false), + ])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(profile), Arc::new(labels), Arc::new(dictionary), Arc::new(run)]) + .expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("nested JSON values should encode"), + 1 + ); + assert_eq!( + buffer.as_ref(), + b"{\"profile\":{\"name\":\"a\\n\",\"count\":null},\"labels\":{\"a\":1,\"b\":null},\"code\":\"small\",\"run\":\"run\"}\n" + ); + } + + #[test] + fn json_null_projection_encodes_without_invoking_arrows_null_encoder() { + let values = NullArray::new(1); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Null, true)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("NULL projection should encode"), + 1 + ); + assert_eq!(buffer.as_ref(), b"{\"value\":null}\n"); + } + + #[test] + fn json_dictionary_null_value_matches_arrow_writer_semantics() { + let keys = Int32Array::from(vec![0]); + let dictionary = Arc::new(StringArray::from(vec![None::<&str>])); + let values = DictionaryArray::::try_new(keys, dictionary).expect("test dictionary should be valid"); + let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), true)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("dictionary null value should encode"), + 1 + ); + assert_eq!(buffer.as_ref(), b"{\"value\":\"\"}\n"); + } + + #[test] + fn json_dictionary_of_null_type_does_not_invoke_arrows_null_encoder() { + let values = DictionaryArray::::try_new(Int32Array::from(vec![0]), Arc::new(NullArray::new(1))) + .expect("test dictionary should be valid"); + let schema = Arc::new(Schema::new(vec![Field::new("value", values.data_type().clone(), true)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("dictionary Null value should encode"), + 1 + ); + assert_eq!(buffer.as_ref(), b"{\"value\":null}\n"); + } + + #[test] + fn json_escaped_field_name_stops_at_the_output_budget() { + let field_name = "\n".repeat(MAX_SELECT_OUTPUT_RECORD_BYTES / 2 + 1); + let values = NullArray::new(1); + let schema = Arc::new(Schema::new(vec![Field::new(field_name, DataType::Null, true)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect_err("an escaped field name larger than one MiB must fail at the output budget"); + + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + + #[test] + fn null_json_struct_does_not_materialize_its_child_encoder_plan() { + let child_name = "\n".repeat(MAX_SELECT_OUTPUT_RECORD_BYTES); + let child = Arc::new(Field::new(child_name, DataType::Utf8, true)); + let values = StructArray::new_null(vec![child].into(), 1); + let schema = Arc::new(Schema::new(vec![Field::new("nested", values.data_type().clone(), true)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("a null struct should encode without its children"), + 1 + ); + assert_eq!(buffer.as_ref(), b"{\"nested\":null}\n"); + } + + #[test] + fn json_record_delimiter_is_in_the_encode_budget() { + let delimiter = "x".repeat(64 * 1024); + let values = StringArray::from(vec!["a", "b"]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput { + record_delimiter: Some(delimiter.clone()), + })); + let mut buffer = BytesMut::new(); + + let encoded_rows = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("bounded JSON delimiter should encode"); + + assert_eq!(encoded_rows, 1); + assert_eq!(buffer.len(), br#"{"value":"a"}"#.len() + delimiter.len()); + } + + #[test] + fn oversized_output_record_fails_before_growing_records_buffer() { + let value = "x".repeat(MAX_SELECT_OUTPUT_RECORD_BYTES + 1); + let values = StringArray::from(vec![value.as_str()]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default())); + let mut buffer = BytesMut::new(); + + let error = + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect_err("result records larger than one MiB must fail"); + + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + + #[test] + fn repeated_json_projection_is_bounded_per_field() { + let value = "x".repeat(64 * 1024); + let values = Arc::new(StringArray::from(vec![value.as_str()])); + let fields = (0..1024) + .map(|index| Field::new(format!("value_{index}"), DataType::Utf8, false)) + .collect::>(); + let columns = (0..fields.len()) + .map(|_| Arc::clone(&values) as datafusion::arrow::array::ArrayRef) + .collect::>(); + let batch = RecordBatch::try_new(Arc::new(Schema::new(fields)), columns).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect_err("a repeated projection larger than one MiB must fail"); + + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + + #[test] + fn csv_output_limit_counts_the_record_delimiter() { + for excess in 0..=1 { + let value = "x".repeat(MAX_SELECT_OUTPUT_RECORD_BYTES - 1 + excess); + let values = StringArray::from(vec![value.as_str()]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Csv(CSVOutput::default())); + let mut buffer = BytesMut::new(); + + if excess == 0 { + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect("a complete one MiB CSV record should encode"), + 1 + ); + assert_eq!(buffer.len(), MAX_SELECT_OUTPUT_RECORD_BYTES); + } else { + let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect_err("a CSV record exceeding one MiB by its delimiter must fail"); + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + } + } + + #[test] + fn json_output_limit_counts_the_record_delimiter() { + let overhead = br#"{"value":""}"#.len() + 1; + for excess in 0..=1 { + let value = "x".repeat(MAX_SELECT_OUTPUT_RECORD_BYTES - overhead + excess); + let values = StringArray::from(vec![value.as_str()]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + if excess == 0 { + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect("a complete one MiB JSON record should encode"), + 1 + ); + assert_eq!(buffer.len(), MAX_SELECT_OUTPUT_RECORD_BYTES); + } else { + let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect_err("a JSON record exceeding one MiB by its delimiter must fail"); + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + } + } + + #[test] + fn json_binary_hex_encoding_honors_the_output_limit() { + let overhead = br#"{"value":""}"#.len() + 1; + let max_value_bytes = (MAX_SELECT_OUTPUT_RECORD_BYTES - overhead) / 2; + for excess in 0..=1 { + let value = vec![0xab; max_value_bytes + excess]; + let values = BinaryArray::from(vec![value.as_slice()]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Binary, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + if excess == 0 { + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect("the largest binary value within the output limit should encode"), + 1 + ); + assert_eq!(buffer.len(), overhead + value.len() * 2); + assert_eq!(buffer.len(), MAX_SELECT_OUTPUT_RECORD_BYTES - 1); + } else { + let error = + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer).expect_err("binary JSON exceeding one MiB must fail"); + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + } + } + + #[test] + fn json_string_escaping_honors_the_exact_output_limit() { + let overhead = br#"{"value":""}"#.len() + 1; + let escaped_budget = MAX_SELECT_OUTPUT_RECORD_BYTES - overhead; + let escaped_newlines = escaped_budget / 2; + let mut exact_value = "\n".repeat(escaped_newlines); + exact_value.push_str(&"x".repeat(escaped_budget % 2)); + + for excess in 0..=1 { + let mut value = exact_value.clone(); + value.push_str(&"x".repeat(excess)); + let values = StringArray::from(vec![value.as_str()]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut encoder = SelectOutputEncoder::new(SelectOutputFormat::Json(JSONOutput::default())); + let mut buffer = BytesMut::new(); + + if excess == 0 { + assert_eq!( + encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect("escaped JSON record of exactly one MiB should encode"), + 1 + ); + assert_eq!(buffer.len(), MAX_SELECT_OUTPUT_RECORD_BYTES); + } else { + let error = encode_batch_turn(&mut encoder, &batch, 0, &mut buffer) + .expect_err("escaped JSON record exceeding one MiB must fail"); + assert_eq!(error.code(), &S3ErrorCode::OverMaxRecordSize); + assert!(buffer.is_empty()); + } + } + } + + #[tokio::test(start_paused = true)] + async fn deadline_preempts_writable_multi_chunk_records() { + const PAYLOAD_CHUNKS: usize = 8; + + let (event_channel, mut rx) = test_event_channel(4); + let (lease, lease_released) = lease_drop_signal(); + let producer = send_select_events_until_deadline( + large_pending_output(PAYLOAD_CHUNKS), + event_channel, + csv_validation(), + Arc::new(SelectInputMetrics::default()), + Instant::now() + Duration::from_secs(1), + 1, + lease, + ); + tokio::pin!(producer); + + for _ in 0..3 { + assert!(futures::poll!(producer.as_mut()).is_pending()); + } + for _ in 0..2 { + assert!(matches!(rx.try_recv(), Ok(Ok(SelectObjectContentEvent::Records(_))))); + } + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + + tokio::time::advance(Duration::from_secs(1)).await; + assert!(futures::poll!(producer.as_mut()).is_ready()); + assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Records(_))))); + let timeout = rx + .recv() + .await + .expect("deadline should send one terminal error") + .expect_err("deadline terminal event should be an error"); + assert_eq!(timeout.code(), &S3ErrorCode::Busy); + assert!(rx.recv().await.is_none()); + assert!(lease_released.await.is_ok(), "timeout should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn continuation_starts_at_one_second_without_query_output() { + let (producer, mut rx, lease_released) = spawn_test_producer(pending_output(), 4); + tokio::task::yield_now().await; + + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + tokio::time::advance(Duration::from_millis(999)).await; + tokio::task::yield_now().await; + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + + tokio::time::advance(Duration::from_millis(1)).await; + assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn progress_starts_at_sixty_seconds_only_when_enabled() { + let mut validation = csv_validation(); + validation.progress_enabled = true; + let (producer, mut rx, lease_released) = + spawn_test_producer_with(pending_output(), 8, validation, Duration::from_secs(300)); + tokio::task::yield_now().await; + + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + tokio::time::advance(Duration::from_millis(59_999)).await; + assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + + tokio::time::advance(Duration::from_millis(1)).await; + let Some(Ok(SelectObjectContentEvent::Progress(progress))) = rx.recv().await else { + panic!("enabled progress should fire at sixty seconds"); + }; + let details = progress.details.expect("Progress should contain details"); + assert_eq!(details.bytes_scanned, Some(0)); + assert_eq!(details.bytes_processed, Some(0)); + assert_eq!(details.bytes_returned, Some(0)); + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + + let (producer, mut rx, lease_released) = spawn_test_producer(pending_output(), 8); + tokio::task::yield_now().await; + tokio::time::advance(Duration::from_secs(60)).await; + assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); + for _ in 0..3 { + tokio::task::yield_now().await; + } + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn small_records_flush_at_five_hundred_milliseconds() { + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(StringArray::from(vec!["row"]))]) + .expect("test record batch should be valid"); + let output = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { Ok::<_, DataFusionError>(batch) }) + .chain(futures::stream::pending::>()), + )); + let (producer, mut rx, lease_released) = spawn_test_producer(output, 4); + for _ in 0..3 { + tokio::task::yield_now().await; + } + + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + tokio::time::advance(Duration::from_millis(499)).await; + tokio::task::yield_now().await; + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + + tokio::time::advance(Duration::from_millis(1)).await; + let Some(Ok(SelectObjectContentEvent::Records(records))) = rx.recv().await else { + panic!("small Records payload should flush at five hundred milliseconds"); + }; + assert_eq!(records.payload.as_deref(), Some(b"row\n".as_slice())); + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn full_records_payload_flushes_without_advancing_time() { + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let value = "x".repeat(RECORDS_CHUNK_TARGET); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(StringArray::from(vec![value]))]) + .expect("test record batch should be valid"); + let output = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { Ok::<_, DataFusionError>(batch) }) + .chain(futures::stream::pending::>()), + )); + let (producer, mut rx, lease_released) = spawn_test_producer(output, 4); + + let Some(Ok(SelectObjectContentEvent::Records(records))) = rx.recv().await else { + panic!("a full Records payload should flush without waiting for the timer"); + }; + assert_eq!(records.payload.as_ref().map(Bytes::len), Some(RECORDS_CHUNK_TARGET)); + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn delayed_intervals_do_not_burst_after_time_jump() { + let (producer, mut rx, lease_released) = spawn_test_producer(pending_output(), 16); + tokio::task::yield_now().await; + + tokio::time::advance(Duration::from_secs(10)).await; + assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); + for _ in 0..5 { + tokio::task::yield_now().await; + } + assert!(matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty))); + + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn continuous_records_do_not_starve_continuation() { + let (producer, mut rx, lease_released) = spawn_test_producer(large_pending_output(8), 16); + tokio::task::yield_now().await; + + tokio::time::advance(CONTINUATION_INTERVAL).await; + let mut saw_continuation = false; + for _ in 0..10 { + let event = rx + .recv() + .await + .expect("scheduler should emit an event") + .expect("event should not fail"); + if matches!(event, SelectObjectContentEvent::Cont(_)) { + saw_continuation = true; + break; + } + } + assert!(saw_continuation, "continuous Records must not starve the continuation ticker"); + + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn continuous_records_do_not_starve_progress() { + let mut validation = csv_validation(); + validation.progress_enabled = true; + let (producer, mut rx, lease_released) = + spawn_test_producer_with(large_pending_output(8), 16, validation, Duration::from_secs(300)); + tokio::task::yield_now().await; + + tokio::time::advance(PROGRESS_INTERVAL).await; + let mut saw_progress = false; + let mut saw_continuation = false; + for _ in 0..12 { + let event = rx + .recv() + .await + .expect("scheduler should emit an event") + .expect("event should not fail"); + match event { + SelectObjectContentEvent::Progress(progress) => { + assert!( + progress + .details + .is_some_and(|details| details.bytes_returned.is_some_and(|bytes| bytes > 0)) + ); + saw_progress = true; + } + SelectObjectContentEvent::Cont(_) => saw_continuation = true, + _ => {} + } + if saw_progress && saw_continuation { + break; + } + } + assert!(saw_progress, "continuous Records must not starve the progress ticker"); + assert!(saw_continuation, "continuous Records must not starve the continuation ticker"); + + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn backpressured_records_cannot_starve_periodic_events() { + let (producer, mut rx, lease_released) = spawn_test_producer(large_pending_output(4), 3); + tokio::task::yield_now().await; + + tokio::time::advance(CONTINUATION_INTERVAL).await; + let mut records_before_continuation = 0; + loop { + let event = rx + .recv() + .await + .expect("scheduler should emit an event") + .expect("event should not fail"); + match event { + SelectObjectContentEvent::Records(_) => { + records_before_continuation += 1; + assert!( + records_before_continuation <= 2, + "only the queued and already-pending Records events may precede a due continuation" + ); + } + SelectObjectContentEvent::Cont(_) => break, + _ => panic!("unexpected event before the due continuation"), + } + } + + let Some(Ok(SelectObjectContentEvent::Records(records))) = rx.recv().await else { + panic!("buffered Records must resume immediately after the due continuation"); + }; + assert_eq!(records.payload.as_ref().map(Bytes::len), Some(RECORDS_CHUNK_TARGET)); + + drop(rx); + producer.await.expect("producer should stop after the receiver closes"); + assert!(lease_released.await.is_ok(), "receiver close should release the snapshot lease"); + } + #[tokio::test(start_paused = true)] async fn producer_preserves_finite_stream_terminal_events() { let schema = Arc::new(datafusion::arrow::datatypes::Schema::new(vec![datafusion::arrow::datatypes::Field::new( @@ -1164,13 +2907,10 @@ mod tests { producer.await.expect("producer should finish at query EOF"); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); - for expected in [b"a\n".as_slice(), b"b\n".as_slice()] { - let Some(Ok(SelectObjectContentEvent::Records(records))) = rx.recv().await else { - panic!("producer should emit a records event for each batch"); - }; - assert_eq!(records.payload.as_deref(), Some(expected)); - } + let Some(Ok(SelectObjectContentEvent::Records(records))) = rx.recv().await else { + panic!("producer should flush buffered records at query EOF"); + }; + assert_eq!(records.payload.as_deref(), Some(b"a\nb\n".as_slice())); let Some(Ok(SelectObjectContentEvent::Stats(stats))) = rx.recv().await else { panic!("producer should emit final stats"); }; @@ -1180,6 +2920,73 @@ mod tests { assert!(lease_released.await.is_ok(), "End should release the snapshot lease"); } + #[tokio::test(start_paused = true)] + async fn multi_slice_multi_chunk_output_is_complete_and_ordered() { + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let values = (0..(MAX_ENCODE_ROWS_PER_TURN * 2 + 1)) + .map(|index| format!("{index:04}-{}", "x".repeat(72))) + .collect::>(); + let expected = values.iter().map(|value| format!("{value}\n")).collect::(); + assert!(expected.len() > RECORDS_CHUNK_TARGET); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(StringArray::from(values))]) + .expect("test record batch should be valid"); + let output = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::once(async move { Ok::<_, DataFusionError>(batch) }), + )); + let (producer, mut rx, lease_released) = spawn_test_producer(output, 8); + + producer.await.expect("producer should finish successfully"); + + let mut records = Vec::new(); + let mut stats_returned = None; + let mut saw_end = false; + while let Some(event) = rx.recv().await { + match event.expect("successful stream should not emit an error") { + SelectObjectContentEvent::Records(event) => { + records.extend_from_slice(event.payload.expect("Records should contain a payload").as_ref()); + } + SelectObjectContentEvent::Stats(event) => { + assert!(stats_returned.is_none(), "Stats should be emitted once"); + stats_returned = event.details.and_then(|details| details.bytes_returned); + } + SelectObjectContentEvent::End(_) => { + assert!(stats_returned.is_some(), "End must follow Stats"); + saw_end = true; + } + _ => panic!("finite query should emit only Records, Stats, and End"), + } + } + + assert_eq!(records, expected.as_bytes()); + assert_eq!( + stats_returned, + Some(i64::try_from(records.len()).expect("test output length should fit in i64")) + ); + assert!(saw_end); + assert!(lease_released.await.is_ok(), "End should release the snapshot lease"); + } + + #[tokio::test(start_paused = true)] + async fn empty_stream_emits_only_stats_then_end() { + let schema = Arc::new(Schema::empty()); + let output = Box::pin(RecordBatchStreamAdapter::new( + schema, + futures::stream::empty::>(), + )); + let (producer, mut rx, lease_released) = spawn_test_producer(output, 4); + + producer.await.expect("empty producer should finish successfully"); + + let Some(Ok(SelectObjectContentEvent::Stats(stats))) = rx.recv().await else { + panic!("empty result should start with Stats"); + }; + assert_eq!(stats.details.and_then(|details| details.bytes_returned), Some(0)); + assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::End(_))))); + assert!(rx.recv().await.is_none()); + assert!(lease_released.await.is_ok(), "End should release the snapshot lease"); + } + #[tokio::test(start_paused = true)] async fn successful_stream_serializes_records_stats_and_end_without_error() { let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); @@ -1206,7 +3013,7 @@ mod tests { .find_map(|(name, value)| (name == ":event-type").then_some(value.as_str())) }) .collect::>(); - assert_eq!(event_types, ["Cont", "Records", "Stats", "End"]); + assert_eq!(event_types, ["Records", "Stats", "End"]); assert!(!messages.iter().flatten().any(|(name, value)| { (name == ":message-type" && value == "error") || name == ":error-code" || name == ":error-message" })); @@ -1214,7 +3021,7 @@ mod tests { } #[tokio::test(start_paused = true)] - async fn eof_at_deadline_uses_reserved_slot_for_stats_then_end() { + async fn deadline_wins_when_eof_becomes_ready_at_same_instant() { let output = Box::pin(RecordBatchStreamAdapter::new( Arc::new(Schema::empty()), futures::stream::unfold((), |_| async { @@ -1222,26 +3029,24 @@ mod tests { None::<(Result, ())> }), )); - let (producer, mut rx, lease_released) = spawn_test_producer(output, 3); + let (producer, mut rx, lease_released) = spawn_test_producer_with(output, 3, csv_validation(), Duration::from_secs(1)); tokio::task::yield_now().await; tokio::time::advance(std::time::Duration::from_secs(1)).await; producer.await.expect("producer should finish at the shared deadline"); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); - let stats = rx + let timeout = rx .recv() .await - .expect("successful Select should send stats") - .expect("stats event should not be an error"); - assert!(matches!(stats, SelectObjectContentEvent::Stats(_))); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::End(_))))); + .expect("deadline should send one terminal error") + .expect_err("deadline terminal event should be an error"); + assert_eq!(timeout.code(), &S3ErrorCode::Busy); assert!(rx.recv().await.is_none()); - assert!(lease_released.await.is_ok(), "EOF should release the snapshot lease"); + assert!(lease_released.await.is_ok(), "deadline should release the snapshot lease"); } #[tokio::test(start_paused = true)] - async fn stream_error_at_deadline_uses_reserved_terminal_slot() { + async fn deadline_wins_when_stream_error_becomes_ready_at_same_instant() { let output = Box::pin(RecordBatchStreamAdapter::new( Arc::new(Schema::empty()), futures::stream::once(async { @@ -1249,21 +3054,20 @@ mod tests { Err(DataFusionError::External(Box::new(SelectError::QueryConcurrencyLimit))) }), )); - let (producer, mut rx, lease_released) = spawn_test_producer(output, 2); + let (producer, mut rx, lease_released) = spawn_test_producer_with(output, 3, csv_validation(), Duration::from_secs(1)); tokio::task::yield_now().await; tokio::time::advance(std::time::Duration::from_secs(1)).await; producer.await.expect("producer should finish at the shared deadline"); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); - let stream_error = rx + let timeout = rx .recv() .await - .expect("stream failure should send one terminal error") + .expect("deadline should send one terminal error") .expect_err("terminal event should be an error"); - assert_eq!(stream_error.code(), &S3ErrorCode::SlowDown); + assert_eq!(timeout.code(), &S3ErrorCode::Busy); assert!(rx.recv().await.is_none()); - assert!(lease_released.await.is_ok(), "stream error should release the snapshot lease"); + assert!(lease_released.await.is_ok(), "deadline should release the snapshot lease"); } #[tokio::test(start_paused = true)] @@ -1372,6 +3176,12 @@ mod tests { assert_eq!(compression_status, StatusCode::BAD_REQUEST); assert!(compression_body.contains("InvalidCompressionFormat")); assert!(compression_body.contains("")); + + let scan_range_error = map_select_error_to_s3(&SelectError::UnsupportedScanRangeInput); + let (scan_range_status, scan_range_body) = http_xml_error(scan_range_error).await; + assert_eq!(scan_range_status, StatusCode::BAD_REQUEST); + assert!(scan_range_body.contains("UnsupportedScanRangeInput")); + assert!(scan_range_body.contains("Scan range queries are not supported on this type of object.")); } #[tokio::test(start_paused = true)] @@ -1410,7 +3220,7 @@ mod tests { .find_map(|(name, value)| (name == ":event-type").then_some(value.as_str())) }) .collect::>(), - ["Cont", "Records"] + ["Records"] ); let terminal_headers = messages.last().expect("stream should contain a terminal error"); assert!( @@ -1443,11 +3253,8 @@ mod tests { )); let (producer, mut rx, lease_released) = spawn_test_producer(output, 2); - tokio::task::yield_now().await; - tokio::time::advance(std::time::Duration::from_secs(1)).await; producer.await.expect("producer should not block on a terminal encoder error"); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); let encoder_error = rx .recv() .await @@ -1470,25 +3277,20 @@ mod tests { futures::future::pending::>().await }), )); - let (tx, mut rx) = mpsc::channel(2); - let terminal_permit = tx - .clone() - .try_reserve_owned() - .expect("test channel should reserve terminal capacity"); + let (event_channel, rx) = test_event_channel(2); let (lease, lease_released) = lease_drop_signal(); let producer = send_select_events_until_deadline( output, - SelectEventChannel { tx, terminal_permit }, + event_channel, csv_validation(), Arc::new(SelectInputMetrics::default()), - Instant::now() + std::time::Duration::from_secs(1), + Instant::now() + Duration::from_secs(300), 300, lease, ); tokio::pin!(producer); assert!(futures::poll!(producer.as_mut()).is_pending()); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); drop(rx); assert!( @@ -1514,14 +3316,20 @@ mod tests { Ok(RecordBatch::new_empty(Arc::new(Schema::empty()))) }), )); - let (tx, mut rx) = mpsc::channel(2); + let (mut event_channel, rx) = test_event_channel(2); let snapshot_fence = LeaseDropSignal(None); - let producer = - send_select_events(output, &tx, csv_validation(), Arc::new(SelectInputMetrics::default()), &snapshot_fence); + let producer = send_select_events( + output, + &mut event_channel, + csv_validation(), + Arc::new(SelectInputMetrics::default()), + Instant::now() + Duration::from_secs(300), + 300, + &snapshot_fence, + ); tokio::pin!(producer); assert!(futures::poll!(producer.as_mut()).is_pending()); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); drop(rx); ready_tx.send(()).expect("test should make the query stream ready"); @@ -1551,13 +3359,15 @@ mod tests { schema, futures::stream::once(async move { Ok::<_, DataFusionError>(batch) }), )); - let (tx, mut rx) = mpsc::channel(4); + let (mut event_channel, mut rx) = test_event_channel(4); let outcome = send_select_events( output, - &tx, + &mut event_channel, csv_validation(), Arc::new(SelectInputMetrics::default()), + Instant::now() + Duration::from_secs(300), + 300, &FailingSnapshotFence, ) .await; @@ -1566,30 +3376,36 @@ mod tests { panic!("failed final snapshot fence must produce a terminal error"); }; assert_eq!(error.code(), &S3ErrorCode::InternalError); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Records(_))))); assert!(rx.try_recv().is_err(), "failed final fence must not enqueue Stats or End"); } #[tokio::test] async fn producer_rechecks_snapshot_after_stats_backpressure() { + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(StringArray::from(vec!["row"]))]) + .expect("test record batch should be valid"); let output = Box::pin(RecordBatchStreamAdapter::new( - Arc::new(Schema::empty()), - futures::stream::empty::>(), + schema, + futures::stream::once(async move { Ok::<_, DataFusionError>(batch) }), )); - let (tx, mut rx) = mpsc::channel(2); - let _terminal_permit = tx - .clone() - .try_reserve_owned() - .expect("test channel should reserve terminal capacity"); + let (mut event_channel, mut rx) = test_event_channel(2); let snapshot_fence = FailsAfterFirstSnapshotFence(std::sync::atomic::AtomicUsize::new(0)); - let producer = - send_select_events(output, &tx, csv_validation(), Arc::new(SelectInputMetrics::default()), &snapshot_fence); + let producer = send_select_events( + output, + &mut event_channel, + csv_validation(), + Arc::new(SelectInputMetrics::default()), + Instant::now() + Duration::from_secs(300), + 300, + &snapshot_fence, + ); tokio::pin!(producer); + assert!(futures::poll!(producer.as_mut()).is_pending()); assert!(futures::poll!(producer.as_mut()).is_pending()); assert_eq!(snapshot_fence.0.load(std::sync::atomic::Ordering::Relaxed), 1); - assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Cont(_))))); + assert!(matches!(rx.recv().await, Some(Ok(SelectObjectContentEvent::Records(_))))); let SelectProducerOutcome::Terminal(Err(error)) = producer.await else { panic!("snapshot loss during Stats backpressure must reject successful End"); @@ -1625,6 +3441,89 @@ mod tests { ); } + #[test] + fn validate_preserves_supported_compression_for_csv_and_json_lines() { + for compression in [CompressionType::GZIP, CompressionType::BZIP2] { + let mut csv_input = base_input(); + csv_input.request.input_serialization.compression_type = Some(CompressionType::from_static(compression)); + validate_select_request(&HeaderMap::new(), &mut csv_input).expect("compressed CSV should be accepted"); + assert_eq!( + csv_input + .request + .input_serialization + .compression_type + .as_ref() + .map(|value| value.as_str()), + Some(compression) + ); + + let mut json_input = base_input(); + json_input.request.input_serialization.csv = None; + json_input.request.input_serialization.json = Some(JSONInput { + type_: Some(JSONType::from_static(JSONType::LINES)), + }); + json_input.request.input_serialization.compression_type = Some(CompressionType::from_static(compression)); + validate_select_request(&HeaderMap::new(), &mut json_input).expect("compressed JSON LINES should be accepted"); + assert_eq!( + json_input + .request + .input_serialization + .compression_type + .as_ref() + .map(|value| value.as_str()), + Some(compression) + ); + } + } + + #[test] + fn validate_rejects_parquet_compression_with_select_request_error() { + let mut input = base_input(); + input.request.input_serialization.csv = None; + input.request.input_serialization.parquet = Some(ParquetInput {}); + input.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP)); + + let error = validate_select_request(&HeaderMap::new(), &mut input).expect_err("compressed Parquet must fail"); + + assert_eq!(error.code(), &S3ErrorCode::InvalidRequestParameter); + assert_eq!(error.message(), Some(INVALID_REQUEST_PARAMETER_MESSAGE)); + } + + #[test] + fn validate_normalizes_noop_compressed_scan_range_and_rejects_real_ranges() { + let mut noop = base_input(); + noop.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP)); + noop.request.scan_range = Some(ScanRange { + start: Some(0), + end: None, + }); + validate_select_request(&HeaderMap::new(), &mut noop).expect("zero-start full scan should be normalized"); + assert!(noop.request.scan_range.is_none()); + + let mut ranged = base_input(); + ranged.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP)); + ranged.request.scan_range = Some(ScanRange { + start: Some(1), + end: None, + }); + let error = validate_select_request(&HeaderMap::new(), &mut ranged) + .expect_err("compressed input with an effective ScanRange must fail before object I/O"); + assert_eq!(error.code(), &S3ErrorCode::UnsupportedScanRangeInput); + assert_eq!(error.status_code(), Some(StatusCode::BAD_REQUEST)); + assert_eq!(error.message(), Some("Scan range queries are not supported on this type of object.")); + + let mut malformed = base_input(); + malformed.request.input_serialization.compression_type = Some(CompressionType::from_static(CompressionType::GZIP)); + malformed.request.scan_range = Some(ScanRange { + start: Some(10), + end: Some(1), + }); + let error = validate_select_request(&HeaderMap::new(), &mut malformed) + .expect_err("malformed ScanRange must fail before compression compatibility validation"); + assert_eq!(error.code(), &S3ErrorCode::InvalidRequestParameter); + assert_eq!(error.message(), Some(INVALID_SCAN_RANGE_MESSAGE)); + } + #[test] fn validate_rejects_unknown_csv_header_mode_before_streaming() { let mut input = base_input(); @@ -1760,6 +3659,22 @@ mod tests { assert_eq!(err.code(), &S3ErrorCode::InvalidRequestParameter); } + fn assert_json_encoder_matches_arrow(array: ArrayRef) { + let schema = Arc::new(Schema::new(vec![Field::new("value", array.data_type().clone(), true)])); + let batch = RecordBatch::try_new(schema, vec![array]).expect("test record batch should be valid"); + let mut expected = Vec::new(); + { + let mut writer = WriterBuilder::new() + .with_explicit_nulls(true) + .build::<_, LineDelimited>(&mut expected); + writer.write(&batch).expect("Arrow JSON reference should encode"); + writer.finish().expect("Arrow JSON reference should finish"); + } + let mut actual = BytesMut::new(); + encode_json_batch(&batch, &JSONOutput::default(), &mut actual).expect("S3 Select JSON should encode"); + assert_eq!(actual.as_ref(), expected.as_slice(), "type: {}", batch.column(0).data_type()); + } + #[test] fn json_encoder_outputs_line_delimited_records() { let schema = @@ -1776,9 +3691,9 @@ mod tests { ) .unwrap(); - let bytes = encode_json_batch(&batch, &JSONOutput::default()).unwrap(); - let output = String::from_utf8(bytes).unwrap(); - assert_eq!(output, "{\"name\":\"a\"}\n{\"name\":\"b\"}\n"); + let mut bytes = BytesMut::new(); + encode_json_batch(&batch, &JSONOutput::default(), &mut bytes).unwrap(); + assert_eq!(bytes.as_ref(), b"{\"name\":\"a\"}\n{\"name\":\"b\"}\n"); } #[test] @@ -1797,15 +3712,44 @@ mod tests { ) .unwrap(); - let bytes = encode_json_batch( + let mut bytes = BytesMut::new(); + encode_json_batch( &batch, &JSONOutput { record_delimiter: Some("|".to_string()), }, + &mut bytes, ) .unwrap(); - let output = String::from_utf8(bytes).unwrap(); - assert_eq!(output, "{\"name\":\"a\"}|{\"name\":\"b\"}|"); + assert_eq!(bytes.as_ref(), b"{\"name\":\"a\"}|{\"name\":\"b\"}|"); + } + + #[test] + fn json_encoder_matches_arrow_for_limited_encoder_variants() { + let mut fixed_list = FixedSizeListBuilder::new(Int32Builder::new(), 2); + fixed_list.values().append_value(1); + fixed_list.values().append_null(); + fixed_list.append(true); + + let arrays: Vec = vec![ + Arc::new(LargeStringArray::from(vec!["a\n"])), + Arc::new(StringViewArray::from(vec!["a\n"])), + Arc::new(BinaryArray::from(vec![b"\xab".as_slice()])), + Arc::new(LargeBinaryArray::from(vec![b"\xab".as_slice()])), + Arc::new(BinaryViewArray::from(vec![b"\xab".as_slice()])), + Arc::new( + FixedSizeBinaryArray::try_from_iter([b"\xab".as_slice()].into_iter()) + .expect("fixed binary test array should be valid"), + ), + Arc::new(LargeListArray::from_iter_primitive::([Some([Some(1), None])])), + Arc::new(ListViewArray::from_iter_primitive::([Some([Some(1), None])])), + Arc::new(LargeListViewArray::from_iter_primitive::([Some([Some(1), None])])), + Arc::new(fixed_list.finish()), + ]; + + for array in arrays { + assert_json_encoder_matches_arrow(array); + } } #[test] @@ -1823,27 +3767,149 @@ mod tests { ) .unwrap(); - let bytes = encode_csv_batch( + let mut bytes = BytesMut::new(); + encode_csv_batch( &batch, &CSVOutput { field_delimiter: Some("|".to_string()), record_delimiter: Some("\r\n".to_string()), ..Default::default() }, + &mut bytes, ) .unwrap(); - assert_eq!(String::from_utf8(bytes).unwrap(), "a|1\r\nb|2\r\n"); + assert_eq!(bytes.as_ref(), b"a|1\r\nb|2\r\n"); } #[test] - fn split_records_payload_uses_exact_returned_bytes() { - let payloads = split_records_payload(vec![b'x'; RECORDS_CHUNK_TARGET + 7]); + fn csv_encoder_matches_select_as_needed_quote_rules() { + let values = StringArray::from(vec!["", "\\.", "\u{00a0}value", "line\rbreak", "a\"b", "a|b"]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut bytes = BytesMut::new(); + + encode_csv_batch( + &batch, + &CSVOutput { + quote_escape_character: Some("\\".to_string()), + quote_fields: Some(QuoteFields::from_static(QuoteFields::ASNEEDED)), + record_delimiter: Some("|".to_string()), + ..Default::default() + }, + &mut bytes, + ) + .expect("CSV output should encode"); + + let expected = [ + b"|".as_slice(), + br#""\.""#, + b"|", + "\"\u{00a0}value\"|".as_bytes(), + b"\"line\rbreak\"|", + br#""a\"b"|"#, + b"a|b|", + ] + .concat(); + assert_eq!(bytes.as_ref(), expected); + } + + #[test] + fn csv_encoder_honors_always_and_custom_quote_characters() { + let values = StringArray::from(vec!["plain", "a'b"]); + let schema = Arc::new(Schema::new(vec![Field::new("value", DataType::Utf8, false)])); + let batch = RecordBatch::try_new(schema, vec![Arc::new(values)]).expect("test record batch should be valid"); + let mut bytes = BytesMut::new(); + + encode_csv_batch( + &batch, + &CSVOutput { + quote_character: Some("'".to_string()), + quote_escape_character: Some("\\".to_string()), + quote_fields: Some(QuoteFields::from_static(QuoteFields::ALWAYS)), + record_delimiter: Some("|".to_string()), + ..Default::default() + }, + &mut bytes, + ) + .expect("CSV output should encode"); + + assert_eq!(bytes.as_ref(), br#"'plain'|'a\'b'|"#); + } + + #[tokio::test(start_paused = true)] + async fn records_staging_preserves_payload_limit_and_returned_bytes() { + let mut buffer = BytesMut::from(vec![b'x'; RECORDS_CHUNK_TARGET + 7].as_slice()); + let flush = tokio::time::sleep(Duration::from_secs(300)); + tokio::pin!(flush); + let mut flush_armed = false; + let mut pending = None; + schedule_buffered_records( + &mut buffer, + flush.as_mut(), + &mut flush_armed, + &mut pending, + Instant::now() + Duration::from_secs(300), + ); + let Some(SelectObjectContentEvent::Records(records)) = pending else { + panic!("full payload should flush immediately"); + }; + let first = records.payload.expect("Records should contain a payload"); + assert_eq!(first.len(), RECORDS_CHUNK_TARGET); + let second = take_records_payload(&mut buffer).expect("remaining payload should stay staged"); + assert_eq!(second.len(), 7); + let mut progress = SelectProgress::new(Some(Arc::new(SelectInputMetrics::default()))); - for payload in &payloads { - progress.add_returned(payload.len()); - } + progress.add_returned(first.len()); + progress.add_returned(second.len()); assert_eq!(progress.to_stats().bytes_returned, Some((RECORDS_CHUNK_TARGET + 7) as i64)); - assert!(payloads.len() > 1); + } + + #[tokio::test] + async fn terminal_error_sends_at_most_one_compat_records_chunk() { + let (mut event_channel, mut rx) = test_event_channel(2); + let mut pending_event = None; + let mut records_buffer = BytesMut::from(vec![b'x'; RECORDS_CHUNK_TARGET + 17].as_slice()); + let mut progress = SelectProgress::new(Some(Arc::new(SelectInputMetrics::default()))); + + flush_terminal_records( + &mut event_channel, + &mut pending_event, + &mut records_buffer, + &mut progress, + TerminalRecordsMode::PrefixBeforeError, + ) + .expect("the reserved terminal Records slot should be available"); + + let Some(Ok(SelectObjectContentEvent::Records(records))) = rx.recv().await else { + panic!("the terminal prefix should be sent as Records"); + }; + assert_eq!(records.payload.as_ref().map(Bytes::len), Some(RECORDS_CHUNK_TARGET)); + assert!(records_buffer.is_empty(), "failed output after the terminal prefix must be discarded"); + assert_eq!(progress.to_stats().bytes_returned, Some(RECORDS_CHUNK_TARGET as i64)); + } + + #[tokio::test] + async fn maximum_records_payload_stays_within_compat_message_limit() { + let (tx, rx) = mpsc::channel(1); + tx.send(Ok(records_event(Bytes::from(vec![b'x'; RECORDS_CHUNK_TARGET])))) + .await + .expect("test channel should accept Records"); + drop(tx); + + let mut byte_stream = SelectObjectContentEventStream::new(ReceiverStream::new(rx)).into_byte_stream(); + let mut encoded = Vec::new(); + while let Some(chunk) = byte_stream.next().await { + encoded.extend_from_slice(&chunk.expect("Records event should serialize")); + } + + let total_len = usize::try_from(u32::from_be_bytes( + encoded[0..4] + .try_into() + .expect("event-stream message should contain a prelude"), + )) + .expect("event-stream message length should fit in usize"); + assert_eq!(total_len, encoded.len()); + assert_eq!(total_len, MAX_COMPAT_EVENT_STREAM_MESSAGE_BYTES); } #[test] diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index 687d18f11..c471457c0 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -86,6 +86,14 @@ impl ApiError { } } + pub fn service_unavailable() -> Self { + ApiError { + code: S3ErrorCode::ServiceUnavailable, + message: Self::error_code_to_message(&S3ErrorCode::ServiceUnavailable), + source: None, + } + } + pub fn invalid_request(message: impl std::fmt::Display) -> Self { ApiError { code: S3ErrorCode::InvalidRequest, diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index 42775eabc..65125c03d 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -1598,7 +1598,11 @@ async fn table_data_plane_resource_for_request( error = %err, "failed to resolve table data-plane resource" ); - s3_error!(AccessDenied, "Access Denied") + if matches!(err, crate::table_catalog::TableCatalogStoreError::Unavailable(_)) { + S3Error::from(ApiError::service_unavailable()) + } else { + s3_error!(AccessDenied, "Access Denied") + } })?; let bucket_fence_key = (bucket.to_string(), crate::table_catalog::default_table_bucket_publication_lock_path()); let mut state = retained.state.lock(); diff --git a/rustfs/src/storage/rpc/node_service.rs b/rustfs/src/storage/rpc/node_service.rs index c6b7a740f..83ea962e7 100644 --- a/rustfs/src/storage/rpc/node_service.rs +++ b/rustfs/src/storage/rpc/node_service.rs @@ -31,6 +31,7 @@ use crate::storage::storage_api::rpc_consumer::node_service::{ use crate::storage::storage_api::runtime_sources_consumer::{EndpointServerPools, runtime_sources}; use crate::storage::storage_api::{ sign_tonic_rpc_response_proof, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, + verify_tonic_mutation_body_digest_reject_unsigned, }; use bytes::Bytes; use futures::Stream; @@ -123,6 +124,15 @@ fn verify_node_mutation_body(request: &Request, ope .map_err(|err| Status::permission_denied(format!("{operation} authentication failed: {err}"))) } +fn verify_node_signal_body(request: &Request, operation: &'static str) -> Result<(), Status> { + let canonical_body = request + .get_ref() + .canonical_body() + .map_err(|_| Status::invalid_argument(format!("{operation} request length cannot be represented")))?; + verify_tonic_mutation_body_digest_reject_unsigned(request, &canonical_body) + .map_err(|err| Status::permission_denied(format!("{operation} authentication failed: {err}"))) +} + fn start_decommission_failure_response(err: Error) -> StartDecommissionResponse { match err { Error::InvalidArgument(_, _, reason) => StartDecommissionResponse { @@ -1839,7 +1849,7 @@ impl Node for NodeService { } async fn signal_service(&self, request: Request) -> Result, Status> { - verify_node_mutation_body(&request, "signal service")?; + verify_node_signal_body(&request, "signal service")?; let request = request.into_inner(); let vars = match request.vars { Some(vars) => vars.value, @@ -4744,6 +4754,34 @@ mod tests { assert!(refresh_response.error_info.is_some()); } + #[tokio::test] + async fn lock_rolling_unsigned_v2_remains_compatible_for_unknown_peer() { + let service = create_test_node_service(); + let unsigned_request = || { + let mut request = Request::new(GenerallyLockRequest { + args: "invalid json".to_string(), + }); + request + .metadata_mut() + .insert("x-rustfs-rpc-auth-version", "2".parse().expect("valid metadata value")); + request + .metadata_mut() + .insert("x-rustfs-content-sha256", "UNSIGNED-PAYLOAD".parse().expect("valid metadata value")); + request + }; + + let lock = service + .lock(unsigned_request()) + .await + .expect("unsigned lock must pass the rolling body gate"); + assert!(!lock.into_inner().success, "invalid test lock args should fail in the lock handler"); + let unlock = service + .un_lock(unsigned_request()) + .await + .expect("unsigned unlock must pass the rolling body gate"); + assert!(!unlock.into_inner().success, "invalid test unlock args should fail in the unlock handler"); + } + /// Premise guard for the no-object-layer RPC tests (backlog#1830): they /// assert the error surface returned while the global object layer is /// absent. Under nextest — the authoritative runner — every test owns its @@ -5561,6 +5599,54 @@ mod tests { assert_eq!(response.error_info.as_deref(), Some("unsupported service signal: 99")); } + #[tokio::test] + async fn signal_service_rejects_explicitly_unsigned_v2_body() { + let service = create_test_node_service(); + let request = SignalServiceRequest { + vars: Some(Mss { + value: HashMap::from([(PEER_RESTSIGNAL.to_string(), "99".to_string())]), + }), + }; + let mut request = Request::new(request); + request + .metadata_mut() + .insert("x-rustfs-rpc-auth-version", "2".parse().expect("valid metadata value")); + request + .metadata_mut() + .insert("x-rustfs-content-sha256", "UNSIGNED-PAYLOAD".parse().expect("valid metadata value")); + + let error = service + .signal_service(request) + .await + .expect_err("an explicitly unsigned v2 signal must fail before handler logic"); + assert_eq!(error.code(), tonic::Code::PermissionDenied); + } + + #[tokio::test] + async fn signal_service_accepts_historical_unsigned_v2_marker_during_rollout() { + let service = create_test_node_service(); + let mut request = Request::new(SignalServiceRequest { + vars: Some(Mss { + value: HashMap::from([(PEER_RESTSIGNAL.to_string(), "99".to_string())]), + }), + }); + request + .metadata_mut() + .insert("x-rustfs-rpc-auth-version", "2".parse().expect("valid metadata value")); + request + .metadata_mut() + .insert("x-rustfs-content-sha256", "UNSIGNED-PAYLOAD".parse().expect("valid metadata value")); + request + .metadata_mut() + .insert("x-rustfs-rpc-nonce", "unsigned".parse().expect("valid metadata value")); + + let response = service + .signal_service(request) + .await + .expect("historical unsigned v2 marker must remain compatible during rollout"); + assert!(!response.into_inner().success, "invalid signal fixture should reach handler validation"); + } + #[tokio::test] async fn every_non_disk_mutation_rejects_a_mismatched_body_digest() { let service = create_test_node_service(); diff --git a/rustfs/src/storage/storage_api.rs b/rustfs/src/storage/storage_api.rs index 81c0040ca..0288ac712 100644 --- a/rustfs/src/storage/storage_api.rs +++ b/rustfs/src/storage/storage_api.rs @@ -539,7 +539,8 @@ pub(crate) mod ecstore_rpc { sign_ns_scanner_capability_with_tier_registry_generation, sign_put_file_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_canonical_body_digest, - verify_tonic_mutation_body_digest, verify_tonic_rpc_signature_with_bootstrap, + verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned, + verify_tonic_rpc_signature_with_bootstrap, }; #[cfg(test)] pub(crate) use rustfs_ecstore::api::rpc::{ @@ -593,8 +594,9 @@ pub(crate) mod ecstore_storage { #[cfg(test)] pub(crate) use rustfs_ecstore::api::storage::init_local_disks; pub(crate) use rustfs_ecstore::api::storage::{ - ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, all_local_disk, all_local_disk_path, find_local_disk_by_ref, - init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map_with_instance_ctx, + ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk, all_local_disk_path, + find_local_disk_by_ref, init_local_disks_with_instance_ctx, init_lock_clients, + prewarm_local_disk_id_map_with_instance_ctx, }; } @@ -1903,6 +1905,13 @@ pub(crate) fn verify_tonic_mutation_body_digest(request: &tonic::Request, ecstore_rpc::verify_tonic_mutation_body_digest(request, canonical_body) } +pub(crate) fn verify_tonic_mutation_body_digest_reject_unsigned( + request: &tonic::Request, + canonical_body: &[u8], +) -> std::io::Result<()> { + ecstore_rpc::verify_tonic_mutation_body_digest_reject_unsigned(request, canonical_body) +} + #[cfg(test)] pub(crate) fn set_tonic_canonical_body_digest(request: &mut tonic::Request, canonical_body: &[u8]) -> std::io::Result<()> { ecstore_rpc::set_tonic_canonical_body_digest(request, canonical_body) diff --git a/rustfs/src/table_catalog/mod.rs b/rustfs/src/table_catalog/mod.rs index e82c9e825..dfdf982e9 100644 --- a/rustfs/src/table_catalog/mod.rs +++ b/rustfs/src/table_catalog/mod.rs @@ -101,6 +101,7 @@ pub(crate) const TABLE_RESOURCE_MARKER_VERSION: u16 = 1; )] pub(crate) const TABLE_METADATA_POINTER_VERSION: u16 = 1; pub(crate) const TABLE_CATALOG_ENTRY_VERSION: u16 = 1; +pub(crate) const TABLE_RENAME_INTENT_VERSION: u16 = 1; pub(crate) const TABLE_WAREHOUSE_INDEX_STATE_VERSION: u16 = 2; pub(crate) const TABLE_MAINTENANCE_CONFIG_VERSION: u16 = 1; pub(crate) const TABLE_EXTERNAL_CATALOG_BRIDGE_VERSION: u16 = 1; @@ -166,6 +167,7 @@ const COMMIT_LOG_ROOT: &str = "commits"; const COMMIT_IDEMPOTENCY_ROOT: &str = "commit-idempotency"; const WAREHOUSE_INDEX_ROOT: &str = "warehouse-index"; const WAREHOUSE_INDEX_STATE_FILE: &str = "state.json"; +const TABLE_RENAME_ROOT: &str = "renames"; const WAREHOUSE_INDEX_MAX_PREFIX_DEPTH: usize = 64; const EXTERNAL_CATALOG_ROOT: &str = "external-catalog"; const EXTERNAL_CATALOG_BRIDGE_FILE: &str = "bridge.json"; diff --git a/rustfs/src/table_catalog/model.rs b/rustfs/src/table_catalog/model.rs index d091dd849..6fdc605e6 100644 --- a/rustfs/src/table_catalog/model.rs +++ b/rustfs/src/table_catalog/model.rs @@ -39,6 +39,8 @@ pub(crate) fn table_bucket_marker_json() -> Result, serde_json::Error> { #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub(crate) enum TableCatalogEntryState { Active, + /// Persisted only behind a rename intent so older readers reject the unknown state and fail closed. + Renaming, Deleting, Deleted, } @@ -53,10 +55,40 @@ pub(crate) struct TableBucketEntry { pub state: TableCatalogEntryState, #[serde(default)] pub properties: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub active_rename_id: Option, pub created_at: Option, pub updated_at: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +pub(crate) enum TableRenameIntentState { + Prepared, + SourceFenced, + DestinationWritten, + SourceTombstoned, + IndexPublished, + DestinationPublished, + Completed, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub(crate) struct TableRenameIntent { + pub version: u16, + pub rename_id: String, + pub table_bucket: String, + pub source: TableEntry, + pub destination: TableEntry, + pub source_etag: String, + pub destination_etag: Option, + pub warehouse_index_etag: String, + pub state: TableRenameIntentState, + pub created_at: String, + pub updated_at: String, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub(crate) struct NamespaceEntry { @@ -1225,6 +1257,7 @@ pub(crate) enum TableCatalogBackingMigrationStep { #[derive(Debug, Clone, PartialEq, Eq, Serialize)] #[serde(rename_all = "SCREAMING_SNAKE_CASE")] pub(crate) enum TableCatalogBackingMigrationBlocker { + TableRenameRecoveryRequired, CommitRecoveryRequired, CommitManualReviewRequired, WarehouseIndexBackfillRequired, diff --git a/rustfs/src/table_catalog/store/migration.rs b/rustfs/src/table_catalog/store/migration.rs index 398285f88..3bd77e2aa 100644 --- a/rustfs/src/table_catalog/store/migration.rs +++ b/rustfs/src/table_catalog/store/migration.rs @@ -301,6 +301,11 @@ where return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); }; validate_table_bucket_entry_object(&self.paths, &bucket_path, &table_bucket_entry)?; + if table_bucket_entry.active_rename_id.is_some() { + return Err(TableCatalogStoreError::Conflict(format!( + "table bucket {table_bucket} has a table rename requiring recovery" + ))); + } let mut namespaces = Vec::new(); let mut tables = Vec::new(); @@ -343,6 +348,9 @@ where ))); }; validate_table_entry_object(&self.paths, table_object, &table_entry)?; + if table_entry.state != TableCatalogEntryState::Active { + continue; + } for commit_object in self .backend @@ -595,9 +603,10 @@ where &self, table_bucket: &str, ) -> TableCatalogStoreResult { - if self.get_table_bucket(table_bucket).await?.is_none() { + let Some(table_bucket_entry) = self.get_table_bucket(table_bucket).await? else { return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); - } + }; + let rename_recovery_required = table_bucket_entry.active_rename_id.is_some(); if let Some((global_fence, _)) = self .read_entry::( self.catalog_bucket(), @@ -653,18 +662,19 @@ where continue; }; validate_table_entry_object(&self.paths, &object, &table)?; + if table.state != TableCatalogEntryState::Active { + continue; + } table_count = table_count.saturating_add(1); if !table_ids.insert(table.table_id.clone()) { duplicate_table_identity = true; } - if table.state == TableCatalogEntryState::Active { - active_table_identifiers.insert((table.namespace.clone(), table.table.clone())); - let warehouse_prefix = table_warehouse_object_prefix(&table)?; - warehouse_prefix_owners - .entry(warehouse_prefix) - .and_modify(|count| *count = count.saturating_add(1)) - .or_insert(1); - } + active_table_identifiers.insert((table.namespace.clone(), table.table.clone())); + let warehouse_prefix = table_warehouse_object_prefix(&table)?; + warehouse_prefix_owners + .entry(warehouse_prefix) + .and_modify(|count| *count = count.saturating_add(1)) + .or_insert(1); let recovery = self.table_commit_recovery_report_for_entry(&table, 0).await?; commit_log_count = commit_log_count.saturating_add(recovery.commits.len()); @@ -711,33 +721,41 @@ where let table_view_identifier_collision_count = active_table_identifiers.intersection(&active_view_identifiers).count(); let mut blockers = Vec::new(); let mut recommended_actions = Vec::new(); + if rename_recovery_required { + blockers.push(TableCatalogBackingMigrationBlocker::TableRenameRecoveryRequired); + recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery); + } if recovery_required_count > 0 { blockers.push(TableCatalogBackingMigrationBlocker::CommitRecoveryRequired); } if manual_review_count > 0 { blockers.push(TableCatalogBackingMigrationBlocker::CommitManualReviewRequired); } - if recovery_required_count > 0 || manual_review_count > 0 { + if (recovery_required_count > 0 || manual_review_count > 0) + && !recommended_actions.contains(&TableCatalogBackingMigrationAction::RunCatalogRecovery) + { recommended_actions.push(TableCatalogBackingMigrationAction::RunCatalogRecovery); } if !warehouse_index_ready { blockers.push(TableCatalogBackingMigrationBlocker::WarehouseIndexBackfillRequired); recommended_actions.push(TableCatalogBackingMigrationAction::BackfillWarehouseIndex); } - if conflicting_warehouse_prefix { + if conflicting_warehouse_prefix && !rename_recovery_required { blockers.push(TableCatalogBackingMigrationBlocker::DuplicateWarehousePrefix); recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateWarehousePrefixes); } - if duplicate_table_identity { + if duplicate_table_identity && !rename_recovery_required { blockers.push(TableCatalogBackingMigrationBlocker::DuplicateTableIdentity); recommended_actions.push(TableCatalogBackingMigrationAction::ReviewDuplicateTableIdentities); } - if table_view_identifier_collision_count > 0 { + if table_view_identifier_collision_count > 0 && !rename_recovery_required { blockers.push(TableCatalogBackingMigrationBlocker::TableViewIdentifierCollision); recommended_actions.push(TableCatalogBackingMigrationAction::ReviewTableViewIdentifierCollisions); } - let mut status = if manual_review_count > 0 + let mut status = if rename_recovery_required { + TableCatalogBackingMigrationStatus::RecoveryRequired + } else if manual_review_count > 0 || conflicting_warehouse_prefix || duplicate_table_identity || table_view_identifier_collision_count > 0 diff --git a/rustfs/src/table_catalog/store/mod.rs b/rustfs/src/table_catalog/store/mod.rs index 2dba0ad9a..22f75765a 100644 --- a/rustfs/src/table_catalog/store/mod.rs +++ b/rustfs/src/table_catalog/store/mod.rs @@ -57,6 +57,9 @@ fn validate_table_bucket_entry(entry: &TableBucketEntry) -> TableCatalogStoreRes if entry.catalog_type != TABLE_BUCKET_CATALOG_TYPE { return Err(TableCatalogStoreError::Invalid("unsupported table bucket catalog type".to_string())); } + if entry.active_rename_id.as_ref().is_some_and(String::is_empty) { + return Err(TableCatalogStoreError::Invalid("active table rename id cannot be empty".to_string())); + } Ok(()) } @@ -984,6 +987,15 @@ impl TableCatalogObjectPaths { ) } + pub fn table_rename_intent_path(&self, table_bucket: &str, rename_id: &str) -> String { + format!( + "{}{}/{}.json", + self.table_bucket_root_prefix(table_bucket), + TABLE_RENAME_ROOT, + table_catalog_path_hash(rename_id) + ) + } + pub fn backing_migration_fence_path(&self, table_bucket: &str) -> String { format!( "{}{}/{}", @@ -1159,9 +1171,7 @@ where update: NamespacePropertiesUpdate, ) -> TableCatalogStoreResult { match self { - Self::ObjectBacked(_) => Err(TableCatalogStoreError::Unsupported( - "namespace property updates require durable-strong catalog backing".to_string(), - )), + Self::ObjectBacked(store) => store.update_namespace_properties(table_bucket, namespace, update).await, Self::DurableStrong(store) => store.update_namespace_properties(table_bucket, namespace, update).await, } } @@ -1241,9 +1251,11 @@ where destination_table: &str, ) -> TableCatalogStoreResult<()> { match self { - Self::ObjectBacked(_) => Err(TableCatalogStoreError::Unsupported( - "table rename requires durable-strong catalog backing".to_string(), - )), + Self::ObjectBacked(store) => { + store + .rename_table(table_bucket, source_namespace, source_table, destination_namespace, destination_table) + .await + } Self::DurableStrong(store) => { store .rename_table(table_bucket, source_namespace, source_table, destination_namespace, destination_table) diff --git a/rustfs/src/table_catalog/store/object.rs b/rustfs/src/table_catalog/store/object.rs index bade7b02d..e2c7249ee 100644 --- a/rustfs/src/table_catalog/store/object.rs +++ b/rustfs/src/table_catalog/store/object.rs @@ -58,6 +58,65 @@ pub(super) fn validate_table_entry_object( Ok(namespace) } +fn validate_table_rename_intent_object( + paths: &TableCatalogObjectPaths, + object: &str, + intent: &TableRenameIntent, +) -> TableCatalogStoreResult<()> { + if intent.version != TABLE_RENAME_INTENT_VERSION + || intent.rename_id.is_empty() + || intent.source_etag.is_empty() + || intent.destination_etag.as_deref().is_some_and(str::is_empty) + || intent.warehouse_index_etag.is_empty() + || intent.created_at.is_empty() + || intent.updated_at.is_empty() + { + return Err(TableCatalogStoreError::Invalid( + "catalog table rename intent has invalid required fields".to_string(), + )); + } + if paths.table_rename_intent_path(&intent.table_bucket, &intent.rename_id) != object { + return Err(TableCatalogStoreError::Invalid( + "catalog table rename intent identity does not match its object path".to_string(), + )); + } + validate_table_entry_version_and_id(&intent.source)?; + validate_table_entry_version_and_id(&intent.destination)?; + if intent.source.table_bucket != intent.table_bucket + || intent.destination.table_bucket != intent.table_bucket + || intent.source.state != TableCatalogEntryState::Active + || intent.destination.state != TableCatalogEntryState::Active + { + return Err(TableCatalogStoreError::Invalid( + "catalog table rename intent has invalid table ownership or state".to_string(), + )); + } + let mut expected_destination = intent.source.clone(); + expected_destination.namespace.clone_from(&intent.destination.namespace); + expected_destination.table.clone_from(&intent.destination.table); + expected_destination.updated_at.clone_from(&intent.destination.updated_at); + if expected_destination != intent.destination + || intent.destination.updated_at.as_deref() != Some(intent.created_at.as_str()) + || (intent.source.namespace == intent.destination.namespace && intent.source.table == intent.destination.table) + { + return Err(TableCatalogStoreError::Invalid( + "catalog table rename intent changes fields other than the table identifier and update time".to_string(), + )); + } + Ok(()) +} + +fn next_table_catalog_update_time(previous: Option<&str>) -> String { + let now = OffsetDateTime::now_utc(); + let update_time = previous + .and_then(|value| OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339).ok()) + .filter(|previous| *previous >= now) + .map_or(now, |previous| previous.saturating_add(Duration::nanoseconds(1))); + update_time + .format(&time::format_description::well_known::Rfc3339) + .unwrap_or_else(|_| update_time.to_string()) +} + pub(super) fn validate_view_entry_object( paths: &TableCatalogObjectPaths, object: &str, @@ -670,6 +729,437 @@ where self.backend.put_object_unlocked(bucket, object, data, precondition).await } + async fn write_exact_entry_unlocked( + &self, + bucket: &str, + object: &str, + entry: &T, + precondition: TableCatalogPutPrecondition, + ) -> TableCatalogStoreResult + where + T: DeserializeOwned + PartialEq + Serialize, + { + let write_result = self.write_entry_unlocked(bucket, object, entry, precondition).await; + let current = self.read_entry_unlocked::(bucket, object).await?; + match current { + Some((current, Some(etag))) if current == *entry => Ok(etag), + Some((current, None)) if current == *entry => Err(TableCatalogStoreError::Internal(format!( + "catalog entry has no etag after write: {object}" + ))), + _ => match write_result { + Ok(()) => Err(TableCatalogStoreError::Internal(format!( + "catalog entry does not match the completed write: {object}" + ))), + Err(err) => Err(err), + }, + } + } + + async fn read_table_bucket_with_etag_unlocked( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult> { + let object = self.paths.table_bucket_entry_path(table_bucket); + let Some((entry, etag)) = self + .read_entry_unlocked::(self.catalog_bucket(), &object) + .await? + else { + return Ok(None); + }; + validate_table_bucket_entry_object(&self.paths, &object, &entry)?; + let Some(etag) = etag else { + return Err(TableCatalogStoreError::Internal(format!( + "catalog table bucket entry has no etag: {object}" + ))); + }; + Ok(Some((entry, etag))) + } + + async fn ensure_no_active_table_rename(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { + self.table_rename_read_version(table_bucket).await.map(|_| ()) + } + + async fn table_rename_read_version(&self, table_bucket: &str) -> TableCatalogStoreResult> { + self.table_rename_read_snapshot(table_bucket) + .await + .map(|snapshot| snapshot.map(|(_, etag)| etag)) + } + + async fn table_rename_read_snapshot( + &self, + table_bucket: &str, + ) -> TableCatalogStoreResult> { + let object = self.paths.table_bucket_entry_path(table_bucket); + let Some((entry, etag)) = self.read_entry::(self.catalog_bucket(), &object).await? else { + return Ok(None); + }; + validate_table_bucket_entry_object(&self.paths, &object, &entry)?; + if let Some(rename_id) = entry.active_rename_id { + return Err(TableCatalogStoreError::Unavailable(format!( + "table bucket {table_bucket} has an active table rename {rename_id}" + ))); + } + let etag = + etag.ok_or_else(|| TableCatalogStoreError::Internal(format!("catalog table bucket entry has no etag: {object}")))?; + Ok(Some((entry, etag))) + } + + async fn finish_table_rename_read(&self, table_bucket: &str, expected_version: Option<&str>) -> TableCatalogStoreResult<()> { + let object = self.paths.table_bucket_entry_path(table_bucket); + let current_version = + match self.backend.object_metadata(self.catalog_bucket(), &object).await? { + None => None, + Some(metadata) => Some(metadata.etag.ok_or_else(|| { + TableCatalogStoreError::Internal(format!("catalog table bucket entry has no etag: {object}")) + })?), + }; + if current_version.as_deref() != expected_version { + return Err(TableCatalogStoreError::Unavailable(format!( + "table bucket {table_bucket} changed while reading the table catalog" + ))); + } + Ok(()) + } + + async fn acquire_catalog_write_locks(&self, mut objects: Vec) -> TableCatalogStoreResult> { + objects.sort_unstable(); + objects.dedup(); + let mut guards = Vec::with_capacity(objects.len()); + for object in objects { + guards.push(self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?); + } + Ok(guards) + } + + async fn advance_table_rename_intent_unlocked( + &self, + object: &str, + intent: &mut TableRenameIntent, + etag: String, + state: TableRenameIntentState, + ) -> TableCatalogStoreResult { + if intent.state >= state { + return Ok(etag); + } + intent.state = state; + intent.updated_at = OffsetDateTime::now_utc().to_string(); + self.write_exact_entry_unlocked(self.catalog_bucket(), object, intent, TableCatalogPutPrecondition::IfMatch(etag)) + .await + } + + async fn recover_active_table_rename( + &self, + table_bucket: &str, + publication: &(dyn TableCommitPublication + Sync), + ) -> TableCatalogStoreResult<()> { + if !publication.holds_table_bucket(table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table rename recovery requires a table-bucket publication fence".to_string(), + )); + } + let bucket_object = self.paths.table_bucket_entry_path(table_bucket); + let Some((observed_bucket, _)) = self + .read_entry::(self.catalog_bucket(), &bucket_object) + .await? + else { + // Recovery only owns an active rename advertised by the bucket + // entry. Callers retain their existing validation when no entry + // exists. + return Ok(()); + }; + validate_table_bucket_entry_object(&self.paths, &bucket_object, &observed_bucket)?; + if observed_bucket.active_rename_id.is_none() { + return Ok(()); + } + let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_object).await?; + let Some((bucket_entry, _)) = self.read_table_bucket_with_etag_unlocked(table_bucket).await? else { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + }; + let Some(rename_id) = bucket_entry.active_rename_id.clone() else { + return Ok(()); + }; + let intent_object = self.paths.table_rename_intent_path(table_bucket, &rename_id); + let _intent_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &intent_object).await?; + let Some((mut intent, mut intent_etag)) = self + .read_entry_unlocked::(self.catalog_bucket(), &intent_object) + .await? + else { + return Err(TableCatalogStoreError::Unavailable(format!( + "active table rename is missing its durable intent: {intent_object}" + ))); + }; + let Some(mut intent_etag) = intent_etag.take() else { + return Err(TableCatalogStoreError::Internal(format!( + "catalog table rename intent has no etag: {intent_object}" + ))); + }; + validate_table_rename_intent_object(&self.paths, &intent_object, &intent)?; + if intent.table_bucket != table_bucket || intent.rename_id != rename_id { + return Err(TableCatalogStoreError::Invalid( + "active table rename does not belong to its table bucket fence".to_string(), + )); + } + if bucket_entry.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::Conflict(format!( + "table bucket {table_bucket} became inactive during table rename recovery" + ))); + } + + let source_namespace = parse_namespace_for_store(&intent.source.namespace)?; + let source_table = parse_table_for_store(&intent.source.table)?; + let destination_namespace = parse_namespace_for_store(&intent.destination.namespace)?; + let destination_table = parse_table_for_store(&intent.destination.table)?; + let source_object = self.paths.table_entry_path(table_bucket, &source_namespace, &source_table); + let destination_object = self + .paths + .table_entry_path(table_bucket, &destination_namespace, &destination_table); + let destination_view_object = self + .paths + .view_entry_path(table_bucket, &destination_namespace, &destination_table); + let index = table_warehouse_index_entry(&intent.source)?; + let index_object = self + .paths + .warehouse_index_entry_path(table_bucket, &index.warehouse_object_prefix); + let _catalog_guards = self + .acquire_catalog_write_locks(vec![ + self.paths.namespace_entry_path(table_bucket, &source_namespace), + self.paths.namespace_entry_path(table_bucket, &destination_namespace), + source_object.clone(), + destination_object.clone(), + destination_view_object.clone(), + index_object.clone(), + ]) + .await?; + if !publication.holds_table_bucket(table_bucket) { + return Err(TableCatalogStoreError::Unavailable( + "table-bucket publication fence was lost during table rename recovery".to_string(), + )); + } + self.require_active_namespace_unlocked( + table_bucket, + &source_namespace, + &self.paths.namespace_entry_path(table_bucket, &source_namespace), + ) + .await + .map_err(|err| match err { + TableCatalogStoreError::NotFound(_) => { + TableCatalogStoreError::Conflict("table rename source namespace disappeared during recovery".to_string()) + } + err => err, + })?; + self.require_active_namespace_unlocked( + table_bucket, + &destination_namespace, + &self.paths.namespace_entry_path(table_bucket, &destination_namespace), + ) + .await + .map_err(|err| match err { + TableCatalogStoreError::NotFound(_) => { + TableCatalogStoreError::Conflict("table rename destination namespace disappeared during recovery".to_string()) + } + err => err, + })?; + if self + .read_entry_unlocked::(self.catalog_bucket(), &destination_view_object) + .await? + .is_some() + { + return Err(TableCatalogStoreError::Conflict( + "table rename destination became a view during recovery".to_string(), + )); + } + + let mut source_fence = intent.source.clone(); + source_fence.state = TableCatalogEntryState::Renaming; + // Keep the source object as a conditional-replacement tombstone instead of relying on an unconditional delete. + let mut source_tombstone = intent.source.clone(); + source_tombstone.state = TableCatalogEntryState::Deleted; + source_tombstone.updated_at = Some(intent.created_at.clone()); + match self + .read_table_with_etag_unlocked(table_bucket, &source_namespace, &source_table) + .await? + { + Some((current, _)) if current == source_fence || current == source_tombstone => {} + Some((current, current_etag)) if current == intent.source && current_etag == intent.source_etag => { + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &source_object, + &source_fence, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await?; + } + _ => { + return Err(TableCatalogStoreError::Conflict(format!( + "table rename source changed during recovery: {table_bucket}/{}/{}", + source_namespace.public_name(), + source_table.as_str() + ))); + } + } + intent_etag = self + .advance_table_rename_intent_unlocked(&intent_object, &mut intent, intent_etag, TableRenameIntentState::SourceFenced) + .await?; + + let mut destination_fence = intent.destination.clone(); + destination_fence.state = TableCatalogEntryState::Renaming; + match self + .read_table_with_etag_unlocked(table_bucket, &destination_namespace, &destination_table) + .await? + { + Some((current, _)) if current == destination_fence || current == intent.destination => {} + None if intent.destination_etag.is_none() => { + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &destination_object, + &destination_fence, + TableCatalogPutPrecondition::IfAbsent, + ) + .await?; + } + Some((current, current_etag)) + if current.state == TableCatalogEntryState::Deleted + && Some(current_etag.as_str()) == intent.destination_etag.as_deref() => + { + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &destination_object, + &destination_fence, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await?; + } + _ => { + return Err(TableCatalogStoreError::Conflict(format!( + "table rename destination changed during recovery: {table_bucket}/{}/{}", + destination_namespace.public_name(), + destination_table.as_str() + ))); + } + } + intent_etag = self + .advance_table_rename_intent_unlocked( + &intent_object, + &mut intent, + intent_etag, + TableRenameIntentState::DestinationWritten, + ) + .await?; + + match self + .read_table_with_etag_unlocked(table_bucket, &source_namespace, &source_table) + .await? + { + Some((current, _)) if current == source_tombstone => {} + Some((current, current_etag)) if current == source_fence => { + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &source_object, + &source_tombstone, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await?; + } + _ => { + return Err(TableCatalogStoreError::Conflict(format!( + "table rename source changed during recovery: {table_bucket}/{}/{}", + source_namespace.public_name(), + source_table.as_str() + ))); + } + } + intent_etag = self + .advance_table_rename_intent_unlocked( + &intent_object, + &mut intent, + intent_etag, + TableRenameIntentState::SourceTombstoned, + ) + .await?; + + let destination_index = table_warehouse_index_entry(&intent.destination)?; + let Some((current_index, current_index_etag)) = self + .read_entry_unlocked::(self.catalog_bucket(), &index_object) + .await? + else { + return Err(TableCatalogStoreError::Conflict( + "table rename warehouse index disappeared during recovery".to_string(), + )); + }; + validate_table_warehouse_index_entry_object(&self.paths, &index_object, ¤t_index)?; + if current_index != destination_index { + if current_index != index || current_index_etag.as_deref() != Some(intent.warehouse_index_etag.as_str()) { + return Err(TableCatalogStoreError::Conflict( + "table rename warehouse index changed during recovery".to_string(), + )); + } + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &index_object, + &destination_index, + TableCatalogPutPrecondition::IfMatch(intent.warehouse_index_etag.clone()), + ) + .await?; + } + intent_etag = self + .advance_table_rename_intent_unlocked( + &intent_object, + &mut intent, + intent_etag, + TableRenameIntentState::IndexPublished, + ) + .await?; + match self + .read_table_with_etag_unlocked(table_bucket, &destination_namespace, &destination_table) + .await? + { + Some((current, _)) if current == intent.destination => {} + Some((current, current_etag)) if current == destination_fence => { + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &destination_object, + &intent.destination, + TableCatalogPutPrecondition::IfMatch(current_etag), + ) + .await?; + } + _ => { + return Err(TableCatalogStoreError::Conflict(format!( + "table rename destination changed during recovery: {table_bucket}/{}/{}", + destination_namespace.public_name(), + destination_table.as_str() + ))); + } + } + intent_etag = self + .advance_table_rename_intent_unlocked( + &intent_object, + &mut intent, + intent_etag, + TableRenameIntentState::DestinationPublished, + ) + .await?; + self.advance_table_rename_intent_unlocked(&intent_object, &mut intent, intent_etag, TableRenameIntentState::Completed) + .await?; + + let Some((mut bucket_entry, bucket_etag)) = self.read_table_bucket_with_etag_unlocked(table_bucket).await? else { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + }; + if bucket_entry.active_rename_id.as_deref() != Some(rename_id.as_str()) { + return Err(TableCatalogStoreError::Conflict("table rename fence changed during recovery".to_string())); + } + bucket_entry.active_rename_id = None; + bucket_entry.updated_at = Some(next_table_catalog_update_time(bucket_entry.updated_at.as_deref())); + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &bucket_object, + &bucket_entry, + TableCatalogPutPrecondition::IfMatch(bucket_etag), + ) + .await?; + Ok(()) + } + async fn write_warehouse_index_state_unlocked(&self, table_bucket: &str) -> TableCatalogStoreResult<()> { let state = TableWarehouseIndexStateEntry { version: TABLE_WAREHOUSE_INDEX_STATE_VERSION, @@ -796,6 +1286,9 @@ where let candidate = table_warehouse_index_entry(entry)?; validate_table_entry_version_and_id(entry)?; for existing in self.list_all_table_entries(&candidate.table_bucket).await? { + if existing.state != TableCatalogEntryState::Active { + continue; + } if existing.table_id == candidate.table_id { if existing.namespace != candidate.namespace || existing.table != candidate.table { return Err(TableCatalogStoreError::Conflict( @@ -804,9 +1297,6 @@ where } continue; } - if existing.state != TableCatalogEntryState::Active { - continue; - } let existing_prefix = table_warehouse_object_prefix(&existing)?; if warehouse_object_prefixes_overlap(&existing_prefix, &candidate.warehouse_object_prefix) { return Err(TableCatalogStoreError::Conflict(format!( @@ -1141,7 +1631,12 @@ where if self.read_warehouse_index_state_unlocked(table_bucket).await? { return Ok(()); } - let tables = self.list_all_table_entries(table_bucket).await?; + let tables = self + .list_all_table_entries(table_bucket) + .await? + .into_iter() + .filter(|table| table.state == TableCatalogEntryState::Active) + .collect::>(); let mut table_ids = BTreeSet::new(); if let Some(table) = tables.iter().find(|table| !table_ids.insert(table.table_id.as_str())) { return Err(TableCatalogStoreError::Conflict(format!( @@ -1150,7 +1645,7 @@ where ))); } let mut active_prefixes = Vec::new(); - for table in tables.iter().filter(|table| table.state == TableCatalogEntryState::Active) { + for table in &tables { active_prefixes.push((table_warehouse_object_prefix(table)?, table.table_id.as_str())); } active_prefixes.sort_unstable_by(|left, right| left.0.cmp(&right.0)); @@ -1164,9 +1659,6 @@ where ))); } for table in tables { - if table.state != TableCatalogEntryState::Active { - continue; - } self.backfill_active_table_warehouse_index(&table.table_bucket, &table.namespace, &table.table) .await?; } @@ -1307,6 +1799,7 @@ where let _publication_completion = TableCommitPublicationCompletion::new(publication); self.require_table_bucket(&entry.table_bucket).await?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; + self.recover_active_table_rename(&entry.table_bucket, publication).await?; let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); let _namespace_guard = self .backend @@ -1327,6 +1820,23 @@ where entry.table_bucket, entry.namespace, entry.table ))); } + let mut precondition = precondition; + if matches!(precondition, TableCatalogPutPrecondition::IfAbsent) + && let Some((current, etag)) = self + .read_entry_unlocked::(self.catalog_bucket(), &table_path) + .await? + { + validate_table_entry_object(&self.paths, &table_path, ¤t)?; + if current.state != TableCatalogEntryState::Deleted { + return Err(TableCatalogStoreError::Conflict(format!( + "catalog object already exists: table {}/{}/{}", + entry.table_bucket, entry.namespace, entry.table + ))); + } + precondition = etag + .map(TableCatalogPutPrecondition::IfMatch) + .ok_or_else(|| TableCatalogStoreError::Internal(format!("catalog table entry has no etag: {table_path}")))?; + } // Preserve catalog -> publication -> object lock order across rolling upgrades. publication .prepare(&entry.table_bucket, &entry.namespace, &entry.table) @@ -1382,6 +1892,7 @@ where let view = parse_table_for_store(&entry.view)?; validate_view_warehouse_location(&entry.table_bucket, &entry.warehouse_location)?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; + self.recover_active_table_rename(&entry.table_bucket, publication).await?; let namespace_path = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); let _namespace_guard = self .backend @@ -1620,7 +2131,11 @@ where ) -> TableCatalogStoreResult { let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; + let publication = TableCommitLockPublication::new(&self.backend); + publication.begin_table_bucket(table_bucket).await?; + let _publication_completion = TableCommitPublicationCompletion::new(&publication); let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + self.recover_active_table_rename(table_bucket, &publication).await?; let table_path = self.paths.table_entry_path(table_bucket, &namespace, &table); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; let Some((entry, _)) = self.read_table_with_etag_unlocked(table_bucket, &namespace, &table).await? else { @@ -4006,12 +4521,21 @@ where Ok(Some(entry)) } - async fn put_table_bucket(&self, entry: TableBucketEntry) -> TableCatalogStoreResult<()> { + async fn put_table_bucket(&self, mut entry: TableBucketEntry) -> TableCatalogStoreResult<()> { validate_table_bucket_entry(&entry)?; let _registry_guard = self.acquire_table_bucket_registry_write_permit().await?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; let object = self.paths.table_bucket_entry_path(&entry.table_bucket); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; + if let Some((current, _)) = self.read_table_bucket_with_etag_unlocked(&entry.table_bucket).await? { + if current.active_rename_id.is_some() { + return Err(TableCatalogStoreError::Unavailable(format!( + "table bucket {} has an active table rename", + entry.table_bucket + ))); + } + entry.updated_at = Some(next_table_catalog_update_time(current.updated_at.as_deref())); + } self.write_entry_unlocked(self.catalog_bucket(), &object, &entry, TableCatalogPutPrecondition::Any) .await } @@ -4023,6 +4547,16 @@ where let _migration_guard = self.acquire_object_backed_catalog_write_permit(&entry.table_bucket).await?; let bucket_path = self.paths.table_bucket_entry_path(&entry.table_bucket); let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?; + if self + .read_table_bucket_with_etag_unlocked(&entry.table_bucket) + .await? + .is_some_and(|(current, _)| current.active_rename_id.is_some()) + { + return Err(TableCatalogStoreError::Unavailable(format!( + "table bucket {} has an active table rename", + entry.table_bucket + ))); + } let object = self.paths.namespace_entry_path(&entry.table_bucket, &namespace); let _namespace_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &object).await?; let precondition = match self @@ -4102,6 +4636,66 @@ where Ok(None) } + async fn update_namespace_properties( + &self, + table_bucket: &str, + namespace: &str, + update: NamespacePropertiesUpdate, + ) -> TableCatalogStoreResult { + let namespace = parse_namespace_for_store(namespace)?; + self.require_table_bucket(table_bucket).await?; + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + let bucket_path = self.paths.table_bucket_entry_path(table_bucket); + let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?; + let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); + let _namespace_guard = self + .backend + .acquire_write_lock(self.catalog_bucket(), &namespace_path) + .await?; + + let current = self + .read_entry_unlocked::(self.catalog_bucket(), &namespace_path) + .await?; + let (mut next, precondition) = match current { + Some((entry, etag)) => { + validate_namespace_entry_object(&self.paths, &namespace_path, &entry)?; + validate_namespace_properties(&entry.properties)?; + if entry.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {table_bucket}/{}", + namespace.public_name() + ))); + } + let etag = etag.ok_or_else(|| { + TableCatalogStoreError::Internal(format!("catalog namespace entry has no etag: {namespace_path}")) + })?; + (entry, TableCatalogPutPrecondition::IfMatch(etag)) + } + None => { + if !self.has_active_namespace_object(table_bucket, &namespace).await? + && !self.has_active_namespace_descendant(table_bucket, &namespace).await? + { + return Err(TableCatalogStoreError::NotFound(format!( + "namespace {table_bucket}/{}", + namespace.public_name() + ))); + } + (synthetic_namespace_entry(table_bucket, &namespace), TableCatalogPutPrecondition::IfAbsent) + } + }; + + let before = next.clone(); + let result = update.apply_to(&mut next); + validate_namespace_entry_object(&self.paths, &namespace_path, &next)?; + validate_namespace_properties(&next.properties)?; + if before == next { + return Ok(result); + } + self.write_entry_unlocked(self.catalog_bucket(), &namespace_path, &next, precondition) + .await?; + Ok(result) + } + async fn list_namespaces_under(&self, table_bucket: &str, parent: &str) -> TableCatalogStoreResult> { let parent = parse_namespace_for_store(parent)?; let prefix = format!("{}{}/", self.paths.namespace_entries_prefix(table_bucket), parent.storage_id()); @@ -4150,6 +4744,15 @@ where let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; let bucket_path = self.paths.table_bucket_entry_path(table_bucket); let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_path).await?; + if self + .read_table_bucket_with_etag_unlocked(table_bucket) + .await? + .is_some_and(|(current, _)| current.active_rename_id.is_some()) + { + return Err(TableCatalogStoreError::Unavailable(format!( + "table bucket {table_bucket} has an active table rename" + ))); + } let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); // Match create_namespace and migration lock order while draining table/view creation. let _namespace_guard = self @@ -4209,6 +4812,7 @@ where } async fn list_tables(&self, table_bucket: &str, namespace: &str) -> TableCatalogStoreResult> { + let read_version = self.table_rename_read_version(table_bucket).await?; let namespace = parse_namespace_for_store(namespace)?; let mut entries = Vec::new(); for object in self @@ -4228,16 +4832,20 @@ where } } entries.sort_by(|left, right| left.table.cmp(&right.table)); + self.finish_table_rename_read(table_bucket, read_version.as_deref()).await?; Ok(entries) } async fn list_all_tables(&self, table_bucket: &str) -> TableCatalogStoreResult> { - self.list_all_table_entries(table_bucket).await.map(|entries| { + let read_version = self.table_rename_read_version(table_bucket).await?; + let entries = self.list_all_table_entries(table_bucket).await.map(|entries| { entries .into_iter() .filter(|entry| entry.state == TableCatalogEntryState::Active) .collect() - }) + })?; + self.finish_table_rename_read(table_bucket, read_version.as_deref()).await?; + Ok(entries) } async fn list_tables_page( @@ -4247,22 +4855,236 @@ where cursor: Option<&str>, limit: NonZeroUsize, ) -> TableCatalogStoreResult> { + let read_version = self.table_rename_read_version(table_bucket).await?; let namespace = parse_namespace_for_store(namespace)?; - self.list_entry_page( - &self.paths.table_entries_prefix(table_bucket, &namespace), - TABLE_ENTRY_FILE, - cursor, - limit, - |entry: &TableEntry| entry.state == TableCatalogEntryState::Active, - |object, entry: &TableEntry| validate_table_entry_object(&self.paths, object, entry).map(|_| ()), - ) - .await + let page = self + .list_entry_page( + &self.paths.table_entries_prefix(table_bucket, &namespace), + TABLE_ENTRY_FILE, + cursor, + limit, + |entry: &TableEntry| entry.state == TableCatalogEntryState::Active, + |object, entry: &TableEntry| validate_table_entry_object(&self.paths, object, entry).map(|_| ()), + ) + .await?; + self.finish_table_rename_read(table_bucket, read_version.as_deref()).await?; + Ok(page) } async fn load_table(&self, table_bucket: &str, namespace: &str, table: &str) -> TableCatalogStoreResult> { - self.load_table_entry(table_bucket, namespace, table) + let read_version = self.table_rename_read_version(table_bucket).await?; + let entry = self + .load_table_entry(table_bucket, namespace, table) .await - .map(|entry| entry.filter(|table| table.state == TableCatalogEntryState::Active)) + .map(|entry| entry.filter(|table| table.state == TableCatalogEntryState::Active))?; + self.finish_table_rename_read(table_bucket, read_version.as_deref()).await?; + Ok(entry) + } + + async fn rename_table( + &self, + table_bucket: &str, + source_namespace: &str, + source_table: &str, + destination_namespace: &str, + destination_table: &str, + ) -> TableCatalogStoreResult<()> { + let source_namespace = parse_namespace_for_store(source_namespace)?; + let source_table = parse_table_for_store(source_table)?; + let destination_namespace = parse_namespace_for_store(destination_namespace)?; + let destination_table = parse_table_for_store(destination_table)?; + let publication = TableCommitLockPublication::new(&self.backend); + publication.begin_table_bucket(table_bucket).await?; + if !publication.holds_table_bucket(table_bucket) { + return Err(TableCatalogStoreError::Internal( + "table rename requires a table-bucket publication fence".to_string(), + )); + } + let _publication_completion = TableCommitPublicationCompletion::new(&publication); + let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + self.recover_active_table_rename(table_bucket, &publication).await?; + + { + let bucket_object = self.paths.table_bucket_entry_path(table_bucket); + let _bucket_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &bucket_object).await?; + let Some((mut bucket_entry, bucket_etag)) = self.read_table_bucket_with_etag_unlocked(table_bucket).await? else { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + }; + if bucket_entry.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::NotFound(format!("table bucket {table_bucket}"))); + } + if bucket_entry.active_rename_id.is_some() { + return Err(TableCatalogStoreError::Unavailable(format!( + "table bucket {table_bucket} has an active table rename" + ))); + } + + let rename_id = Uuid::new_v4().to_string(); + let intent_object = self.paths.table_rename_intent_path(table_bucket, &rename_id); + let _intent_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &intent_object).await?; + let source_object = self.paths.table_entry_path(table_bucket, &source_namespace, &source_table); + let destination_object = self + .paths + .table_entry_path(table_bucket, &destination_namespace, &destination_table); + let _catalog_guards = self + .acquire_catalog_write_locks(vec![ + self.paths.namespace_entry_path(table_bucket, &source_namespace), + self.paths.namespace_entry_path(table_bucket, &destination_namespace), + source_object.clone(), + destination_object.clone(), + self.paths + .view_entry_path(table_bucket, &destination_namespace, &destination_table), + ]) + .await?; + self.require_active_namespace_unlocked( + table_bucket, + &source_namespace, + &self.paths.namespace_entry_path(table_bucket, &source_namespace), + ) + .await + .map_err(|err| match err { + TableCatalogStoreError::NotFound(_) => TableCatalogStoreError::TableNotFound(format!( + "{table_bucket}/{}/{}", + source_namespace.public_name(), + source_table.as_str() + )), + err => err, + })?; + self.require_active_namespace_unlocked( + table_bucket, + &destination_namespace, + &self.paths.namespace_entry_path(table_bucket, &destination_namespace), + ) + .await + .map_err(|err| match err { + TableCatalogStoreError::NotFound(_) => { + TableCatalogStoreError::NamespaceNotFound(format!("{table_bucket}/{}", destination_namespace.public_name())) + } + err => err, + })?; + let Some((source, source_etag)) = self + .read_table_with_etag_unlocked(table_bucket, &source_namespace, &source_table) + .await? + else { + return Err(TableCatalogStoreError::TableNotFound(format!( + "{table_bucket}/{}/{}", + source_namespace.public_name(), + source_table.as_str() + ))); + }; + if source.state != TableCatalogEntryState::Active { + return Err(TableCatalogStoreError::TableNotFound(format!( + "{table_bucket}/{}/{}", + source_namespace.public_name(), + source_table.as_str() + ))); + } + if !is_valid_table_metadata_location_for_entry(&source, &source.metadata_location) { + return Err(TableCatalogStoreError::Invalid( + "current metadata location must be inside the table metadata directory".to_string(), + )); + } + let destination_etag = match self + .read_table_with_etag_unlocked(table_bucket, &destination_namespace, &destination_table) + .await? + { + None => None, + Some((current, etag)) if current.state == TableCatalogEntryState::Deleted => Some(etag), + Some(_) => { + return Err(TableCatalogStoreError::AlreadyExists(format!( + "destination table already exists: {table_bucket}/{}/{}", + destination_namespace.public_name(), + destination_table.as_str() + ))); + } + }; + if self + .read_entry_unlocked::( + self.catalog_bucket(), + &self + .paths + .view_entry_path(table_bucket, &destination_namespace, &destination_table), + ) + .await? + .is_some() + { + return Err(TableCatalogStoreError::AlreadyExists(format!( + "destination table already exists: {table_bucket}/{}/{}", + destination_namespace.public_name(), + destination_table.as_str() + ))); + } + + let now = next_table_catalog_update_time(bucket_entry.updated_at.as_deref()); + let mut destination = source.clone(); + destination.namespace = destination_namespace.public_name(); + destination.table = destination_table.as_str().to_string(); + destination.updated_at = Some(now.clone()); + let source_index = table_warehouse_index_entry(&source)?; + let index_object = self + .paths + .warehouse_index_entry_path(table_bucket, &source_index.warehouse_object_prefix); + let _index_guard = self.backend.acquire_write_lock(self.catalog_bucket(), &index_object).await?; + let warehouse_index_etag = match self + .read_entry_unlocked::(self.catalog_bucket(), &index_object) + .await? + { + Some((current, Some(etag))) if current == source_index => etag, + Some((current, None)) if current == source_index => { + return Err(TableCatalogStoreError::Internal(format!( + "catalog warehouse index entry has no etag: {index_object}" + ))); + } + Some(_) => { + return Err(TableCatalogStoreError::Conflict( + "table warehouse index does not match the rename source".to_string(), + )); + } + None => { + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &index_object, + &source_index, + TableCatalogPutPrecondition::IfAbsent, + ) + .await? + } + }; + let intent = TableRenameIntent { + version: TABLE_RENAME_INTENT_VERSION, + rename_id: rename_id.clone(), + table_bucket: table_bucket.to_string(), + source, + destination, + source_etag, + destination_etag, + warehouse_index_etag, + state: TableRenameIntentState::Prepared, + created_at: now.clone(), + updated_at: now.clone(), + }; + validate_table_rename_intent_object(&self.paths, &intent_object, &intent)?; + + // An orphan intent has no catalog effect; a published bucket fence without its intent cannot be recovered safely. + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &intent_object, + &intent, + TableCatalogPutPrecondition::IfAbsent, + ) + .await?; + bucket_entry.active_rename_id = Some(rename_id); + bucket_entry.updated_at = Some(now); + self.write_exact_entry_unlocked( + self.catalog_bucket(), + &bucket_object, + &bucket_entry, + TableCatalogPutPrecondition::IfMatch(bucket_etag), + ) + .await?; + } + + self.recover_active_table_rename(table_bucket, &publication).await } async fn resolve_table_data_plane_resource( @@ -4273,7 +5095,7 @@ where if table_bucket.is_empty() || object.is_empty() { return Ok(None); } - let Some(table_bucket_entry) = self.get_table_bucket(table_bucket).await? else { + let Some((table_bucket_entry, read_version)) = self.table_rename_read_snapshot(table_bucket).await? else { return Err(TableCatalogStoreError::Internal(format!( "object-backed catalog has no entry for table-enabled bucket {table_bucket}" ))); @@ -4284,34 +5106,36 @@ where ))); } - if self.warehouse_index_ready(table_bucket).await? { - return match self + let resource = if self.warehouse_index_ready(table_bucket).await? { + match self .resolve_table_data_plane_resource_from_index(table_bucket, object) .await? { Some(resource) => Ok(Some(resource)), None => scan_table_data_plane_resource_for_object(self, table_bucket, object).await, - }; - } - - match self.backfill_table_warehouse_index(table_bucket).await { - Ok(()) => match self - .resolve_table_data_plane_resource_from_index(table_bucket, object) - .await? - { - Some(resource) => Ok(Some(resource)), - None => scan_table_data_plane_resource_for_object(self, table_bucket, object).await, - }, - Err(err @ TableCatalogStoreError::Internal(_)) => { - tracing::warn!( - table_bucket = %table_bucket, - error = %err, - "failed to backfill table warehouse index; falling back to catalog scan" - ); - scan_table_data_plane_resource_for_object(self, table_bucket, object).await } - Err(err) => Err(err), - } + } else { + match self.backfill_table_warehouse_index(table_bucket).await { + Ok(()) => match self + .resolve_table_data_plane_resource_from_index(table_bucket, object) + .await? + { + Some(resource) => Ok(Some(resource)), + None => scan_table_data_plane_resource_for_object(self, table_bucket, object).await, + }, + Err(err @ TableCatalogStoreError::Internal(_)) => { + tracing::warn!( + table_bucket = %table_bucket, + error = %err, + "failed to backfill table warehouse index; falling back to catalog scan" + ); + scan_table_data_plane_resource_for_object(self, table_bucket, object).await + } + Err(err) => Err(err), + } + }?; + self.finish_table_rename_read(table_bucket, Some(&read_version)).await?; + Ok(resource) } async fn commit_table(&self, request: TableCommitRequest) -> TableCatalogStoreResult { @@ -4330,6 +5154,11 @@ where let namespace = parse_namespace_for_store(&request.namespace)?; let table = parse_table_for_store(&request.table)?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; + if publication.holds_table_bucket(&request.table_bucket) { + self.recover_active_table_rename(&request.table_bucket, publication).await?; + } else { + self.ensure_no_active_table_rename(&request.table_bucket).await?; + } let table_path = self.paths.table_entry_path(&request.table_bucket, &namespace, &table); let _guard = self.backend.acquire_write_lock(self.catalog_bucket(), &table_path).await?; // Preserve catalog -> publication -> object lock order across rolling upgrades. @@ -4749,6 +5578,7 @@ where let namespace = parse_namespace_for_store(namespace)?; let table = parse_table_for_store(table)?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + self.recover_active_table_rename(table_bucket, &publication).await?; let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); let _namespace_guard = self .backend @@ -4902,6 +5732,11 @@ where } } let _migration_guard = self.acquire_object_backed_catalog_write_permit(&request.table_bucket).await?; + if publication.holds_table_bucket(&request.table_bucket) { + self.recover_active_table_rename(&request.table_bucket, publication).await?; + } else { + self.ensure_no_active_table_rename(&request.table_bucket).await?; + } let namespace_path = self.paths.namespace_entry_path(&request.table_bucket, &namespace); let _namespace_guard = self .backend @@ -5020,9 +5855,13 @@ where } async fn drop_view(&self, table_bucket: &str, namespace: &str, view: &str) -> TableCatalogStoreResult<()> { + let publication = TableCommitLockPublication::new(&self.backend); + publication.begin_table_bucket(table_bucket).await?; + let _publication_completion = TableCommitPublicationCompletion::new(&publication); let namespace = parse_namespace_for_store(namespace)?; let view = parse_table_for_store(view)?; let _migration_guard = self.acquire_object_backed_catalog_write_permit(table_bucket).await?; + self.recover_active_table_rename(table_bucket, &publication).await?; let namespace_path = self.paths.namespace_entry_path(table_bucket, &namespace); let _namespace_guard = self .backend diff --git a/rustfs/src/table_catalog/test_support.rs b/rustfs/src/table_catalog/test_support.rs index 1daeeb7a0..5967e91e6 100644 --- a/rustfs/src/table_catalog/test_support.rs +++ b/rustfs/src/table_catalog/test_support.rs @@ -642,12 +642,26 @@ impl TestCatalogObjectBackend { let mut state = self.state.lock().await; let key = (bucket.to_string(), object.to_string()); let next_attempt = state.put_attempts.get(&key).copied().unwrap_or_default() + 1; + Self::pause_put_attempt_unlocked(&mut state, key, next_attempt) + } + + pub(crate) async fn pause_put_attempt(&self, bucket: &str, object: &str, attempt: usize) -> TestCatalogObjectPause { + let mut state = self.state.lock().await; + let key = (bucket.to_string(), object.to_string()); + Self::pause_put_attempt_unlocked(&mut state, key, attempt) + } + + fn pause_put_attempt_unlocked( + state: &mut TestCatalogObjectState, + key: (String, String), + attempt: usize, + ) -> TestCatalogObjectPause { let pause = TestCatalogObjectPause::default(); state .pause_put_attempts .entry(key) .or_default() - .insert(next_attempt, pause.clone()); + .insert(attempt, pause.clone()); pause } diff --git a/rustfs/src/table_catalog/tests.rs b/rustfs/src/table_catalog/tests.rs index 8c1363407..40f3d7c88 100644 --- a/rustfs/src/table_catalog/tests.rs +++ b/rustfs/src/table_catalog/tests.rs @@ -144,6 +144,7 @@ fn catalog_entry_structures_serialize_stable_fields() { warehouse_root: "s3://analytics/".to_string(), state: TableCatalogEntryState::Active, properties: BTreeMap::from([("owner".to_string(), "platform".to_string())]), + active_rename_id: None, created_at: Some("2026-05-23T00:00:00Z".to_string()), updated_at: Some("2026-05-23T00:00:00Z".to_string()), }; @@ -3441,6 +3442,7 @@ fn test_bucket_entry(bucket: &str) -> TableBucketEntry { warehouse_root: format!("s3://{bucket}/"), state: TableCatalogEntryState::Active, properties: BTreeMap::new(), + active_rename_id: None, created_at: None, updated_at: None, } @@ -5107,7 +5109,8 @@ async fn object_catalog_pagination_bounds_reads_and_covers_rest_resources() { .await .expect("first table page should load"); assert_eq!(table_page.entries[0].table, "alpha"); - assert_eq!(backend.read_call_count().await, 1); + // One read snapshots the bucket rename fence and one loads the page entry. + assert_eq!(backend.read_call_count().await, 2); let table_page = store .list_tables_page(bucket, &namespace_name, table_page.next_cursor.as_deref(), one) .await @@ -16614,13 +16617,16 @@ fn namespace_property_update_and_limits_reject_ambiguous_or_oversized_state() { } #[tokio::test] -async fn configured_object_catalog_rejects_namespace_property_update_without_mutation() { +async fn configured_object_catalog_updates_namespace_properties() { let backend = TestCatalogObjectBackend::default(); let store = ConfiguredTableCatalogStore::new_for_test(backend, TableCatalogBackingMode::ObjectBacked); let bucket = "analytics"; let namespace = Namespace::parse("sales").expect("namespace should parse"); let mut entry = test_namespace_entry(bucket, &namespace); - entry.properties = BTreeMap::from([("owner".to_string(), "lakehouse".to_string())]); + entry.properties = BTreeMap::from([ + ("obsolete".to_string(), "true".to_string()), + ("owner".to_string(), "lakehouse".to_string()), + ]); store .put_table_bucket(test_bucket_entry(bucket)) .await @@ -16631,18 +16637,354 @@ async fn configured_object_catalog_rejects_namespace_property_update_without_mut .update_namespace_properties( bucket, "sales", - NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "platform".to_string())])) - .expect("namespace update should validate"), + NamespacePropertiesUpdate::try_new( + vec!["obsolete".to_string(), "missing".to_string()], + BTreeMap::from([ + ("owner".to_string(), "platform".to_string()), + ("retention".to_string(), "30d".to_string()), + ]), + ) + .expect("namespace update should validate"), ) .await - .expect_err("object-backed namespace property update should be unsupported"); - assert_matches!(result, TableCatalogStoreError::Unsupported(_)); + .expect("object-backed namespace properties should update"); + assert_eq!(result.updated, vec!["owner".to_string(), "retention".to_string()]); + assert_eq!(result.removed, vec!["obsolete".to_string()]); + assert_eq!(result.missing, vec!["missing".to_string()]); let stored = store .get_namespace(bucket, "sales") .await .expect("namespace lookup should succeed") .expect("namespace should remain"); - assert_eq!(stored.properties.get("owner").map(String::as_str), Some("lakehouse")); + assert_eq!( + stored.properties, + BTreeMap::from([ + ("owner".to_string(), "platform".to_string()), + ("retention".to_string(), "30d".to_string()), + ]) + ); +} + +#[tokio::test] +async fn object_catalog_namespace_property_update_materializes_implicit_parent() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let parent = Namespace::parse("sales").expect("parent namespace should parse"); + let child = Namespace::parse("sales.daily").expect("child namespace should parse"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket entry should be seeded"); + let child_entry = test_namespace_entry(bucket, &child); + store + .create_namespace(child_entry.clone()) + .await + .expect("child namespace should be created"); + + let no_change = store + .update_namespace_properties( + bucket, + &parent.public_name(), + NamespacePropertiesUpdate::try_new(vec!["missing".to_string()], BTreeMap::new()) + .expect("namespace update should validate"), + ) + .await + .expect("implicit parent no-op should succeed"); + assert_eq!(no_change.missing, vec!["missing".to_string()]); + let parent_path = store.paths.namespace_entry_path(bucket, &parent); + assert!( + store + .read_entry::(store.catalog_bucket(), &parent_path) + .await + .expect("implicit parent lookup should succeed") + .is_none() + ); + + let result = store + .update_namespace_properties( + bucket, + &parent.public_name(), + NamespacePropertiesUpdate::try_new( + vec!["missing".to_string()], + BTreeMap::from([("owner".to_string(), "platform".to_string())]), + ) + .expect("namespace update should validate"), + ) + .await + .expect("implicit parent should materialize"); + + assert_eq!(result.updated, vec!["owner".to_string()]); + assert!(result.removed.is_empty()); + assert_eq!(result.missing, vec!["missing".to_string()]); + let (materialized, _) = store + .read_entry::(store.catalog_bucket(), &parent_path) + .await + .expect("materialized parent should load") + .expect("parent should have an explicit entry"); + assert_eq!(materialized.properties.get("owner").map(String::as_str), Some("platform")); + assert_eq!( + store + .get_namespace(bucket, &child.public_name()) + .await + .expect("child lookup should succeed"), + Some(child_entry) + ); +} + +#[tokio::test] +async fn object_catalog_namespace_property_update_materializes_resource_only_parents() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let table_namespace = Namespace::parse("table_only").expect("table namespace should parse"); + let view_namespace = Namespace::parse("view_only").expect("view namespace should parse"); + let table = IdentifierSegment::parse("orders").expect("table should parse"); + let view = IdentifierSegment::parse("recent_orders").expect("view should parse"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket entry should be seeded"); + + let table_entry = test_table_entry( + bucket, + &table_namespace, + &table, + default_table_metadata_file_path(&table_namespace, &table, "00001.metadata.json"), + ); + backend + .seed_object( + RUSTFS_META_BUCKET, + &store.paths.table_entry_path(bucket, &table_namespace, &table), + serde_json::to_vec(&table_entry).expect("table entry should serialize"), + ) + .await; + let view_entry = test_view_entry( + bucket, + &view_namespace, + &view, + default_view_metadata_file_path(&view_namespace, &view, "00001.view.json"), + ); + backend + .seed_object( + RUSTFS_META_BUCKET, + &store.paths.view_entry_path(bucket, &view_namespace, &view), + serde_json::to_vec(&view_entry).expect("view entry should serialize"), + ) + .await; + + let table_before = store + .load_table(bucket, &table_namespace.public_name(), table.as_str()) + .await + .expect("table should load before materializing its namespace"); + let view_before = store + .load_view(bucket, &view_namespace.public_name(), view.as_str()) + .await + .expect("view should load before materializing its namespace"); + + for namespace in [&table_namespace, &view_namespace] { + let result = store + .update_namespace_properties( + bucket, + &namespace.public_name(), + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "platform".to_string())])) + .expect("namespace update should validate"), + ) + .await + .expect("active resource should prove the implicit namespace"); + assert_eq!(result.updated, vec!["owner".to_string()]); + let materialized = store + .get_namespace(bucket, &namespace.public_name()) + .await + .expect("materialized namespace should load") + .expect("materialized namespace should exist"); + assert_eq!(materialized.properties.get("owner").map(String::as_str), Some("platform")); + } + + assert_eq!( + store + .load_table(bucket, &table_namespace.public_name(), table.as_str()) + .await + .expect("table should load after materializing its namespace"), + table_before + ); + assert_eq!( + store + .load_view(bucket, &view_namespace.public_name(), view.as_str()) + .await + .expect("view should load after materializing its namespace"), + view_before + ); +} + +#[tokio::test] +async fn object_catalog_namespace_property_update_requires_etag_and_skips_noop_writes() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket entry should be seeded"); + + let etagless = Namespace::parse("etagless").expect("namespace should parse"); + let mut etagless_entry = test_namespace_entry(bucket, &etagless); + etagless_entry.properties.insert("owner".to_string(), "lakehouse".to_string()); + store + .create_namespace(etagless_entry) + .await + .expect("etagless namespace should be seeded"); + let etagless_path = store.paths.namespace_entry_path(bucket, &etagless); + backend.omit_etag_for_object(RUSTFS_META_BUCKET, &etagless_path).await; + let etagless_puts = backend.put_attempt_count(RUSTFS_META_BUCKET, &etagless_path).await; + + assert_matches!( + store + .update_namespace_properties( + bucket, + &etagless.public_name(), + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "platform".to_string())]),) + .expect("namespace update should validate"), + ) + .await, + Err(TableCatalogStoreError::Internal(_)) + ); + assert_eq!(backend.put_attempt_count(RUSTFS_META_BUCKET, &etagless_path).await, etagless_puts); + let unchanged = store + .get_namespace(bucket, &etagless.public_name()) + .await + .expect("etagless namespace should still load") + .expect("etagless namespace should remain"); + assert_eq!(unchanged.properties.get("owner").map(String::as_str), Some("lakehouse")); + + let no_op = Namespace::parse("no_op").expect("namespace should parse"); + let mut no_op_entry = test_namespace_entry(bucket, &no_op); + no_op_entry.properties.insert("owner".to_string(), "lakehouse".to_string()); + store + .create_namespace(no_op_entry) + .await + .expect("no-op namespace should be seeded"); + let no_op_path = store.paths.namespace_entry_path(bucket, &no_op); + backend.fail_next_put(RUSTFS_META_BUCKET, &no_op_path).await; + let puts_before_no_op = backend.put_attempt_count(RUSTFS_META_BUCKET, &no_op_path).await; + + let result = store + .update_namespace_properties( + bucket, + &no_op.public_name(), + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "lakehouse".to_string())])) + .expect("namespace update should validate"), + ) + .await + .expect("unchanged namespace properties should not write"); + assert_eq!(result.updated, vec!["owner".to_string()]); + assert_eq!(backend.put_attempt_count(RUSTFS_META_BUCKET, &no_op_path).await, puts_before_no_op); + + assert_matches!( + store + .update_namespace_properties( + bucket, + &no_op.public_name(), + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "platform".to_string())]),) + .expect("namespace update should validate"), + ) + .await, + Err(TableCatalogStoreError::Internal(_)) + ); + let unchanged = store + .get_namespace(bucket, &no_op.public_name()) + .await + .expect("namespace should load after failed write") + .expect("namespace should remain"); + assert_eq!(unchanged.properties.get("owner").map(String::as_str), Some("lakehouse")); +} + +#[tokio::test] +async fn object_catalog_namespace_property_update_rejects_missing_inactive_and_corrupt_entries() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket entry should be seeded"); + let update = || { + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "platform".to_string())])) + .expect("namespace update should validate") + }; + + assert_matches!( + store.update_namespace_properties(bucket, "missing", update()).await, + Err(TableCatalogStoreError::NotFound(_)) + ); + + let inactive = Namespace::parse("inactive").expect("inactive namespace should parse"); + let mut inactive_entry = test_namespace_entry(bucket, &inactive); + inactive_entry.state = TableCatalogEntryState::Deleted; + backend + .seed_object( + RUSTFS_META_BUCKET, + &store.paths.namespace_entry_path(bucket, &inactive), + serde_json::to_vec(&inactive_entry).expect("inactive namespace should encode"), + ) + .await; + assert_matches!( + store + .update_namespace_properties(bucket, &inactive.public_name(), update()) + .await, + Err(TableCatalogStoreError::NotFound(_)) + ); + + let corrupt = Namespace::parse("corrupt").expect("corrupt namespace should parse"); + backend + .seed_object(RUSTFS_META_BUCKET, &store.paths.namespace_entry_path(bucket, &corrupt), b"{".to_vec()) + .await; + assert_matches!( + store + .update_namespace_properties(bucket, &corrupt.public_name(), update()) + .await, + Err(TableCatalogStoreError::Invalid(_)) + ); + + let semantically_corrupt = Namespace::parse("semantically_corrupt").expect("corrupt namespace should parse"); + let mut semantically_corrupt_entry = test_namespace_entry(bucket, &semantically_corrupt); + semantically_corrupt_entry.properties = (0..=NAMESPACE_PROPERTIES_MAX_ENTRIES) + .map(|index| (format!("key{index}"), "value".to_string())) + .collect(); + let semantically_corrupt_path = store.paths.namespace_entry_path(bucket, &semantically_corrupt); + backend + .seed_object( + RUSTFS_META_BUCKET, + &semantically_corrupt_path, + serde_json::to_vec(&semantically_corrupt_entry).expect("corrupt namespace should encode"), + ) + .await; + let put_attempts = backend + .put_attempt_count(RUSTFS_META_BUCKET, &semantically_corrupt_path) + .await; + let repair_update = + NamespacePropertiesUpdate::try_new(vec![format!("key{NAMESPACE_PROPERTIES_MAX_ENTRIES}")], BTreeMap::new()) + .expect("repair request should validate structurally"); + + assert_matches!( + store + .update_namespace_properties(bucket, &semantically_corrupt.public_name(), repair_update,) + .await, + Err(TableCatalogStoreError::Invalid(_)) + ); + assert_eq!( + backend + .put_attempt_count(RUSTFS_META_BUCKET, &semantically_corrupt_path) + .await, + put_attempts + ); + let persisted = store + .read_entry::(RUSTFS_META_BUCKET, &semantically_corrupt_path) + .await + .expect("corrupt namespace lookup should succeed") + .expect("corrupt namespace should remain") + .0; + assert_eq!(persisted.properties.len(), NAMESPACE_PROPERTIES_MAX_ENTRIES + 1); } #[tokio::test] @@ -16981,6 +17323,59 @@ async fn object_catalog_namespace_replacement_is_fenced_by_observed_etag() { assert_eq!(stored.properties.get("owner").map(String::as_str), Some("winner")); } +#[tokio::test] +async fn object_catalog_namespace_property_update_is_fenced_by_observed_etag() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let namespace = Namespace::parse("sales").expect("namespace should parse"); + store + .put_table_bucket(test_bucket_entry(bucket)) + .await + .expect("table bucket should be created"); + store + .create_namespace(test_namespace_entry(bucket, &namespace)) + .await + .expect("namespace should be created"); + + let namespace_path = store.paths.namespace_entry_path(bucket, &namespace); + let pause = backend.pause_next_put(RUSTFS_META_BUCKET, &namespace_path).await; + let stale_store = store.clone(); + let stale_update = tokio::spawn(async move { + stale_store + .update_namespace_properties( + bucket, + "sales", + NamespacePropertiesUpdate::try_new(Vec::new(), BTreeMap::from([("owner".to_string(), "stale".to_string())])) + .expect("namespace update should validate"), + ) + .await + }); + pause.wait_started().await; + + let mut winner = test_namespace_entry(bucket, &namespace); + winner.properties.insert("owner".to_string(), "winner".to_string()); + backend + .seed_object( + RUSTFS_META_BUCKET, + &namespace_path, + serde_json::to_vec(&winner).expect("winning namespace should encode"), + ) + .await; + pause.release(); + + assert_matches!( + stale_update.await.expect("stale namespace update task should finish"), + Err(TableCatalogStoreError::Conflict(_)) + ); + let stored = store + .get_namespace(bucket, &namespace.public_name()) + .await + .expect("winning namespace should load") + .expect("winning namespace should remain"); + assert_eq!(stored.properties.get("owner").map(String::as_str), Some("winner")); +} + async fn assert_direct_namespace_child_contract(store: &S, bucket: &str, cursor_prefix: &str) where S: TableCatalogStore + ?Sized, @@ -17752,7 +18147,461 @@ async fn strong_catalog_table_rename_returns_success_after_committed_snapshot_re } #[tokio::test] -async fn configured_object_catalog_rejects_table_rename() { +async fn object_catalog_table_rename_preserves_identity_index_and_reuses_source_tombstone() { + let backend = TestCatalogObjectBackend { + content_addressed_etags: true, + ..Default::default() + }; + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let source_namespace = Namespace::parse("sales").expect("source namespace should parse"); + let destination_namespace = Namespace::parse("curated").expect("destination namespace should parse"); + let source_table = IdentifierSegment::parse("orders").expect("source table should parse"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &source_namespace)) + .await + .unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &destination_namespace)) + .await + .unwrap(); + let source = test_table_entry( + bucket, + &source_namespace, + &source_table, + default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"), + ); + store.create_table(source.clone()).await.unwrap(); + let bucket_object = store.paths.table_bucket_entry_path(bucket); + let bucket_etag_before = store + .read_entry::(RUSTFS_META_BUCKET, &bucket_object) + .await + .unwrap() + .unwrap() + .1 + .expect("table bucket should have an etag"); + + store + .rename_table(bucket, "sales", "orders", "curated", "orders_v2") + .await + .expect("object-backed table rename should complete"); + let bucket_etag_after = store + .read_entry::(RUSTFS_META_BUCKET, &bucket_object) + .await + .unwrap() + .unwrap() + .1 + .expect("table bucket should have an etag"); + assert_ne!(bucket_etag_after, bucket_etag_before); + + assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none()); + let destination = store + .load_table(bucket, "curated", "orders_v2") + .await + .unwrap() + .expect("destination table should exist"); + let mut expected_destination = source.clone(); + expected_destination.namespace = "curated".to_string(); + expected_destination.table = "orders_v2".to_string(); + assert_ne!(destination.updated_at, source.updated_at); + expected_destination.updated_at.clone_from(&destination.updated_at); + assert_eq!(destination, expected_destination); + + let source_object = store.paths.table_entry_path(bucket, &source_namespace, &source_table); + let source_tombstone = store + .read_entry::(RUSTFS_META_BUCKET, &source_object) + .await + .unwrap() + .expect("source tombstone should remain") + .0; + assert_eq!(source_tombstone.state, TableCatalogEntryState::Deleted); + assert_eq!(source_tombstone.table_id, destination.table_id); + assert!( + store + .get_table_bucket(bucket) + .await + .unwrap() + .expect("table bucket should exist") + .active_rename_id + .is_none() + ); + + let destination_index = table_warehouse_index_entry(&destination).unwrap(); + let index_object = store + .paths + .warehouse_index_entry_path(bucket, &destination_index.warehouse_object_prefix); + let persisted_index = store + .read_entry::(RUSTFS_META_BUCKET, &index_object) + .await + .unwrap() + .expect("warehouse index should exist") + .0; + assert_eq!(persisted_index, destination_index); + let resource = store + .resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet") + .await + .unwrap() + .expect("renamed table should resolve its stable warehouse prefix"); + assert_eq!(resource.namespace, "curated"); + assert_eq!(resource.table, "orders_v2"); + store + .backfill_table_warehouse_index(bucket) + .await + .expect("retained source tombstone should not make the warehouse index ambiguous"); + let migration = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + assert_eq!(migration.table_count, 1); + assert!( + !migration + .blockers + .contains(&TableCatalogBackingMigrationBlocker::DuplicateTableIdentity) + ); + + store + .rename_table(bucket, "curated", "orders_v2", "sales", "orders") + .await + .expect("rename should conditionally replace the retained source tombstone"); + store + .rename_table(bucket, "sales", "orders", "curated", "orders_v2") + .await + .expect("rename should conditionally replace a destination tombstone"); + + let mut replacement = test_table_entry( + bucket, + &source_namespace, + &source_table, + default_table_metadata_file_path(&source_namespace, &source_table, "00002.metadata.json"), + ); + replacement.table_id = "replacement-table-id".to_string(); + replacement.table_uuid = "replacement-table-uuid".to_string(); + replacement.warehouse_location = "s3://analytics/tables/replacement-table-id".to_string(); + store + .create_table(replacement.clone()) + .await + .expect("create should conditionally replace the source tombstone"); + assert_eq!( + store + .load_table(bucket, "sales", "orders") + .await + .unwrap() + .expect("source identifier should be reusable"), + replacement + ); +} + +#[tokio::test] +async fn object_catalog_table_rename_rejects_missing_and_conflicting_destinations() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend); + let bucket = "analytics"; + let source_namespace = Namespace::parse("sales").unwrap(); + let destination_namespace = Namespace::parse("curated").unwrap(); + let source_table = IdentifierSegment::parse("orders").unwrap(); + let destination_table = IdentifierSegment::parse("orders_v2").unwrap(); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &source_namespace)) + .await + .unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &destination_namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry( + bucket, + &source_namespace, + &source_table, + default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"), + )) + .await + .unwrap(); + + assert_matches!( + store.rename_table(bucket, "sales", "orders", "missing", "orders_v2").await, + Err(TableCatalogStoreError::NamespaceNotFound(_)) + ); + assert_matches!( + store.rename_table(bucket, "sales", "missing", "curated", "orders_v2").await, + Err(TableCatalogStoreError::TableNotFound(_)) + ); + let mut existing = test_table_entry( + bucket, + &destination_namespace, + &destination_table, + default_table_metadata_file_path(&destination_namespace, &destination_table, "00001.metadata.json"), + ); + existing.table_id = "destination-table-id".to_string(); + existing.table_uuid = "destination-table-uuid".to_string(); + existing.warehouse_location = "s3://analytics/tables/destination-table-id".to_string(); + store.create_table(existing).await.unwrap(); + assert_matches!( + store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await, + Err(TableCatalogStoreError::AlreadyExists(_)) + ); + let destination_view = IdentifierSegment::parse("orders_view").unwrap(); + store + .create_view(test_view_entry( + bucket, + &destination_namespace, + &destination_view, + default_view_metadata_file_path(&destination_namespace, &destination_view, "00001.metadata.json"), + )) + .await + .unwrap(); + assert_matches!( + store.rename_table(bucket, "sales", "orders", "curated", "orders_view").await, + Err(TableCatalogStoreError::AlreadyExists(_)) + ); + assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_some()); +} + +#[tokio::test] +async fn object_catalog_table_rename_fails_closed_around_durable_fence_creation() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let source_namespace = Namespace::parse("sales").unwrap(); + let destination_namespace = Namespace::parse("curated").unwrap(); + let source_table = IdentifierSegment::parse("orders").unwrap(); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &source_namespace)) + .await + .unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &destination_namespace)) + .await + .unwrap(); + store + .create_table(test_table_entry( + bucket, + &source_namespace, + &source_table, + default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"), + )) + .await + .unwrap(); + + let bucket_object = store.paths.table_bucket_entry_path(bucket); + backend.fail_next_put(RUSTFS_META_BUCKET, &bucket_object).await; + assert_matches!( + store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await, + Err(TableCatalogStoreError::Internal(_)) + ); + assert!( + store + .get_table_bucket(bucket) + .await + .unwrap() + .expect("table bucket should remain") + .active_rename_id + .is_none() + ); + assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_some()); + assert!(store.load_table(bucket, "curated", "orders_v2").await.unwrap().is_none()); + + let mut fenced_bucket = store.get_table_bucket(bucket).await.unwrap().unwrap(); + fenced_bucket.active_rename_id = Some("missing-intent".to_string()); + backend + .seed_object( + RUSTFS_META_BUCKET, + &bucket_object, + serde_json::to_vec(&fenced_bucket).expect("fenced bucket should serialize"), + ) + .await; + assert_matches!( + store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await, + Err(TableCatalogStoreError::Unavailable(_)) + ); + assert_eq!( + store + .get_table_bucket(bucket) + .await + .unwrap() + .expect("table bucket should remain fail-closed") + .active_rename_id + .as_deref(), + Some("missing-intent") + ); + assert_matches!( + store + .resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet") + .await, + Err(TableCatalogStoreError::Unavailable(_)) + ); +} + +#[tokio::test] +async fn object_catalog_table_rename_recovers_after_destination_publish_and_fences_concurrent_mutations() { + let backend = TestCatalogObjectBackend::default(); + let store = ObjectTableCatalogStore::new(backend.clone()); + let bucket = "analytics"; + let source_namespace = Namespace::parse("sales").expect("source namespace should parse"); + let destination_namespace = Namespace::parse("curated").expect("destination namespace should parse"); + let source_table = IdentifierSegment::parse("orders").expect("source table should parse"); + store.put_table_bucket(test_bucket_entry(bucket)).await.unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &source_namespace)) + .await + .unwrap(); + store + .create_namespace(test_namespace_entry(bucket, &destination_namespace)) + .await + .unwrap(); + let source = test_table_entry( + bucket, + &source_namespace, + &source_table, + default_table_metadata_file_path(&source_namespace, &source_table, "00001.metadata.json"), + ); + store.create_table(source.clone()).await.unwrap(); + + let source_object = store.paths.table_entry_path(bucket, &source_namespace, &source_table); + let source_tombstone_attempt = backend.put_attempt_count(RUSTFS_META_BUCKET, &source_object).await + 2; + let source_tombstone_pause = backend + .pause_put_attempt(RUSTFS_META_BUCKET, &source_object, source_tombstone_attempt) + .await; + let rename_store = store.clone(); + let rename = tokio::spawn(async move { + rename_store + .rename_table(bucket, "sales", "orders", "curated", "orders_v2") + .await + }); + source_tombstone_pause.wait_started().await; + + let active_rename_id = store + .get_table_bucket(bucket) + .await + .unwrap() + .expect("table bucket should exist") + .active_rename_id + .expect("rename fence should be durable before destination publication"); + assert_matches!( + store.load_table(bucket, "sales", "orders").await, + Err(TableCatalogStoreError::Unavailable(_)) + ); + assert_matches!(store.list_tables(bucket, "sales").await, Err(TableCatalogStoreError::Unavailable(_))); + assert_matches!( + store + .resolve_table_data_plane_resource(bucket, "tables/table-id/data/part.parquet") + .await, + Err(TableCatalogStoreError::Unavailable(_)) + ); + let migration = store.plan_durable_strong_backing_migration(bucket).await.unwrap(); + assert_eq!(migration.status, TableCatalogBackingMigrationStatus::RecoveryRequired); + assert!( + migration + .blockers + .contains(&TableCatalogBackingMigrationBlocker::TableRenameRecoveryRequired) + ); + + let publication_lock = default_table_bucket_publication_lock_path(); + let publication_attempts = backend.write_lock_acquisition_count(bucket, &publication_lock).await; + let commit_store = store.clone(); + let commit = tokio::spawn(async move { + commit_store + .commit_table(TableCommitRequest { + table_bucket: bucket.to_string(), + namespace: "sales".to_string(), + table: "orders".to_string(), + commit_id: "concurrent-commit".to_string(), + idempotency_key: None, + operation: "append".to_string(), + expected_version_token: source.version_token, + expected_metadata_location: source.metadata_location, + new_metadata_location: "unused.metadata.json".to_string(), + requirements: Vec::new(), + writer: Some("rename-test".to_string()), + }) + .await + }); + let drop_store = store.clone(); + let drop_table = tokio::spawn(async move { drop_store.drop_table(bucket, "sales", "orders").await }); + let create_store = store.clone(); + let mut replacement = test_table_entry( + bucket, + &source_namespace, + &source_table, + default_table_metadata_file_path(&source_namespace, &source_table, "00002.metadata.json"), + ); + replacement.table_id = "replacement-table-id".to_string(); + replacement.table_uuid = "replacement-table-uuid".to_string(); + replacement.warehouse_location = "s3://analytics/tables/replacement-table-id".to_string(); + let create = tokio::spawn(async move { create_store.create_table(replacement).await }); + tokio::time::timeout(TABLE_CATALOG_TEST_TIMEOUT, async { + while backend.write_lock_acquisition_count(bucket, &publication_lock).await < publication_attempts + 3 { + tokio::task::yield_now().await; + } + }) + .await + .expect("concurrent mutations should reach the table-bucket publication fence"); + assert!(!commit.is_finished()); + assert!(!drop_table.is_finished()); + assert!(!create.is_finished()); + commit.abort(); + drop_table.abort(); + create.abort(); + + rename.abort(); + source_tombstone_pause.release(); + let _ = rename.await; + let destination_object = store.paths.table_entry_path( + bucket, + &destination_namespace, + &IdentifierSegment::parse("orders_v2").expect("destination table should parse"), + ); + let source_fence = store + .read_entry::(RUSTFS_META_BUCKET, &source_object) + .await + .unwrap() + .expect("source rename fence should be durable") + .0; + let destination_fence = store + .read_entry::(RUSTFS_META_BUCKET, &destination_object) + .await + .unwrap() + .expect("destination rename fence should be durable") + .0; + assert_eq!(source_fence.state, TableCatalogEntryState::Renaming); + assert_eq!(destination_fence.state, TableCatalogEntryState::Renaming); + assert_matches!( + store.load_table(bucket, "curated", "orders_v2").await, + Err(TableCatalogStoreError::Unavailable(_)) + ); + assert_matches!( + store.rename_table(bucket, "sales", "orders", "curated", "orders_v2").await, + Err(TableCatalogStoreError::TableNotFound(_)) + ); + + assert!(store.load_table(bucket, "sales", "orders").await.unwrap().is_none()); + let destination = store + .load_table(bucket, "curated", "orders_v2") + .await + .unwrap() + .expect("recovery should finish the destination publication"); + assert_eq!(destination.table_id, "table-id"); + assert!( + store + .get_table_bucket(bucket) + .await + .unwrap() + .expect("table bucket should exist") + .active_rename_id + .is_none() + ); + let intent_object = store.paths.table_rename_intent_path(bucket, &active_rename_id); + let intent = store + .read_entry::(RUSTFS_META_BUCKET, &intent_object) + .await + .unwrap() + .expect("completed rename intent should be retained as a recovery record") + .0; + assert_eq!(intent.state, TableRenameIntentState::Completed); +} + +#[tokio::test] +async fn configured_object_catalog_dispatches_table_rename() { let store = ConfiguredTableCatalogStore::new_for_test(TestCatalogObjectBackend::default(), TableCatalogBackingMode::ObjectBacked); @@ -17760,6 +18609,6 @@ async fn configured_object_catalog_rejects_table_rename() { store .rename_table("analytics", "sales", "orders", "curated", "orders_v2") .await, - Err(TableCatalogStoreError::Unsupported(_)) + Err(TableCatalogStoreError::NotFound(_)) ); } diff --git a/scripts/run_get_1mib_abba_stage_metrics.sh b/scripts/run_get_1mib_abba_stage_metrics.sh index 48b3341fa..8a55541e0 100755 --- a/scripts/run_get_1mib_abba_stage_metrics.sh +++ b/scripts/run_get_1mib_abba_stage_metrics.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -# Dedicated exact-1MiB GET attribution harness for rustfs/backlog#1434. +# Dedicated exact-1MiB GET attribution harness for rustfs/backlog#2093. # # The heavy lifting stays in run_get_codec_streaming_smoke.sh. This wrapper only # fixes the experiment matrix so a reviewer can reproduce the isolated-host @@ -59,7 +59,7 @@ Usage: scripts/run_get_1mib_abba_stage_metrics.sh [options] Purpose: - Run the rustfs/backlog#1434 exact-1MiB isolated-host GET attribution matrix: + Run the rustfs/backlog#2093 exact-1MiB isolated-host GET attribution matrix: - object size fixed to 1MiB / 1048576 bytes - legacy and codec-legacy read-path profiles - normal and reverse profile ordering for ABBA order-bias checks @@ -256,7 +256,7 @@ rustc_version="$(rustc --version 2>/dev/null || echo unavailable)" cargo_version="$(cargo --version 2>/dev/null || echo unavailable)" cat >"${OUT_DIR}/manifest.env" </dev/null -rg -qx 'issue=rustfs/backlog#1434' "${OUT_DIR}/manifest.env" +rg -qx 'issue=rustfs/backlog#2093' "${OUT_DIR}/manifest.env" rg -qx 'exact_size=1MiB' "${OUT_DIR}/manifest.env" rg -qx 'exact_size_bytes=1048576' "${OUT_DIR}/manifest.env" rg -qx 'read_path_profiles=legacy,codec-legacy' "${OUT_DIR}/manifest.env" @@ -41,6 +41,11 @@ rg -qx 'diagnostic_obs_metric_endpoint=http://127.0.0.1:4318/v1/metrics' "${OUT_ rg -qx 'diagnostic_obs_meter_interval=1' "${OUT_DIR}/manifest.env" rg -qx 'compressed_fallback_probe=true' "${OUT_DIR}/manifest.env" rg -qx 'performance_conclusion=not_encoded_by_harness_collect_raw_abba_stage_metrics_first' "${OUT_DIR}/manifest.env" +rg -qx 'get_seek_buffer_enable=unset' "${OUT_DIR}/manifest.env" +rg -qx 'get_small_body_once_enable=unset' "${OUT_DIR}/manifest.env" +rg -qx 'get_lockstep_data_shards_only_enable=unset' "${OUT_DIR}/manifest.env" +rg -qx 'get_metadata_read_version_coalesce=unset' "${OUT_DIR}/manifest.env" +rg -qx 'object_data_cache_mode=unset' "${OUT_DIR}/manifest.env" rg -Fq '("service.name", "service_name", "job", "otel_scope_name")' "${SCRIPT_DIR}/run_get_codec_streaming_smoke.sh" rg -Fq '("service_name", "service.name", "job", "otel_scope_name")' "${SCRIPT_DIR}/run_get_codec_streaming_smoke.sh" rg -Fq 'compressed_size = max(object_size, codec_min_size, 128 * 1024)' "${SCRIPT_DIR}/run_get_codec_streaming_smoke.sh"