mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-12 05:49:01 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 xiaomage
|
f6ebc9182b |
@@ -1,267 +0,0 @@
|
||||
# RustFS Fault-Tolerance (degradation) Test
|
||||
#
|
||||
# Scenario suite for the 2026-09 degradation report: verifies read/write
|
||||
# behavior under drive and node loss against the erasure-coding contract and
|
||||
# snapshots health-endpoint responses at every tier.
|
||||
#
|
||||
# A single-node 4 drives (SNMD): hide 1/2/3 drives, restore
|
||||
# B multi-node 4x1 (one drive per node): stop 1/2/3 nodes, restore
|
||||
# C multi-node 4x4 (16 drives, EC:4): stop 1 node (read-quorum boundary),
|
||||
# stop 2 nodes, restore
|
||||
# C2 multi-node 4x4 with EC:8: 2 nodes down puts 8 drives online -- reads
|
||||
# satisfy the EC read quorum while the lock majority is broken (the
|
||||
# reported divergence window: reads 503 with lock_quorum_unavailable)
|
||||
#
|
||||
# Expectations come from product source (default_parity_count, erasure set
|
||||
# sizing). By default a "reads refused although the read quorum is met"
|
||||
# observation is reported as known-divergence without failing the suite; the
|
||||
# strict input turns those into failures once the product behavior changes.
|
||||
|
||||
name: RustFS Fault-Tolerance Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
package_url:
|
||||
description: 'Direct .deb URL. Required unless the nightly default is wanted.'
|
||||
required: false
|
||||
type: string
|
||||
strict:
|
||||
description: 'Fail the suite when reads are refused despite a met read quorum'
|
||||
type: boolean
|
||||
default: false
|
||||
cleanup_before:
|
||||
description: 'Reset the nodes before the test (DESTROYS existing data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
cleanup_after:
|
||||
description: 'Reset the nodes after the test (DESTROYS test data/config)'
|
||||
type: boolean
|
||||
default: true
|
||||
repository_dispatch:
|
||||
# Chain handoff: dispatched when the replication suite finishes, ahead of
|
||||
# the performance suite.
|
||||
types: [rustfs-chain-fault-tolerance]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
# The suite stops services and hides drive dirs on the shared fleet; only one
|
||||
# functional suite may touch the environment at a time.
|
||||
concurrency:
|
||||
group: rustfs-shared-functional-tests
|
||||
cancel-in-progress: false
|
||||
|
||||
defaults:
|
||||
run:
|
||||
shell: bash
|
||||
|
||||
env:
|
||||
RUSTFS_ACCESS_KEY: ${{ secrets.RUSTFS_ACCESS_KEY }}
|
||||
RUSTFS_SECRET_KEY: ${{ secrets.RUSTFS_SECRET_KEY }}
|
||||
RUSTFS_NODES: ${{ secrets.RUSTFS_NODES || vars.RUSTFS_NODES }}
|
||||
RUSTFS_SSH_USER: ${{ secrets.RUSTFS_SSH_USER || vars.RUSTFS_SSH_USER }}
|
||||
PF_TESTING_GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
RUSTFS_NIGHTLY_PACKAGE_URL: ${{ vars.RUSTFS_NIGHTLY_PACKAGE_URL || 'https://dl.rustfs.com/artifacts/rustfs/packages/nightly/rustfs-nightly-latest.deb' }}
|
||||
|
||||
jobs:
|
||||
fault-tolerance-test:
|
||||
runs-on: smoke-testing
|
||||
timeout-minutes: 480
|
||||
if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }}
|
||||
steps:
|
||||
- name: Initialize functional evidence
|
||||
id: evidence
|
||||
run: |
|
||||
set -euo pipefail
|
||||
umask 077
|
||||
FUNCTIONAL_ARTIFACTS_DIR="${RUNNER_TEMP}/rustfs-ft-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
|
||||
mkdir -- "${FUNCTIONAL_ARTIFACTS_DIR}" "${FUNCTIONAL_ARTIFACTS_DIR}/evidence"
|
||||
{
|
||||
printf 'FUNCTIONAL_ARTIFACTS_DIR=%s\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'LOG_FILE=%s/suite.log\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'REPORT_FILE=%s/report.md\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
printf 'EVIDENCE_DIR=%s/evidence\n' "${FUNCTIONAL_ARTIFACTS_DIR}"
|
||||
} >> "${GITHUB_ENV}"
|
||||
|
||||
# auto-testing is private: clone it with the dedicated PF token (not
|
||||
# GITHUB_TOKEN) and retry transient GitHub/network failures.
|
||||
- name: Checkout auto-testing scripts (with retry)
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
rm -rf auto-testing
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if gh repo clone rustfs/auto-testing auto-testing -- --depth 1 --quiet; then
|
||||
echo "auto-testing cloned (attempt ${attempt})"
|
||||
exit 0
|
||||
fi
|
||||
rm -rf auto-testing
|
||||
echo "clone attempt ${attempt} failed; retrying in $((attempt * 15))s" >&2
|
||||
sleep $((attempt * 15))
|
||||
done
|
||||
echo "ERROR: unable to clone rustfs/auto-testing after 5 attempts" >&2
|
||||
exit 1
|
||||
|
||||
- name: Show environment
|
||||
run: |
|
||||
uname -a
|
||||
jq --version
|
||||
aws --version
|
||||
df -h /data | tail -1
|
||||
|
||||
- name: Cleanup environment (before)
|
||||
if: ${{ inputs.cleanup_before != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}"
|
||||
|
||||
- name: Run fault-tolerance scenarios (A, B, C, C2)
|
||||
id: test
|
||||
run: |
|
||||
ARGS=(--all -y --package-url "${{ inputs.package_url || env.RUSTFS_NIGHTLY_PACKAGE_URL }}" --log-file "${LOG_FILE}")
|
||||
if [ "${{ inputs.strict }}" = "true" ]; then
|
||||
ARGS+=(--strict)
|
||||
fi
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh "${ARGS[@]}"
|
||||
|
||||
- name: Generate report
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
{
|
||||
echo "# RustFS fault-tolerance test report"
|
||||
echo ""
|
||||
echo "- Run: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Package: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "- Strict mode: ${{ inputs.strict || 'false' }}"
|
||||
echo ""
|
||||
echo "## Per-probe results"
|
||||
echo ""
|
||||
echo '```'
|
||||
grep -E '^FT-(CASE|SUMMARY|REPORT)' "${LOG_FILE}" || echo "(no FT-CASE lines found)"
|
||||
echo '```'
|
||||
echo ""
|
||||
echo "## Health snapshots"
|
||||
echo ""
|
||||
for f in "${FUNCTIONAL_ARTIFACTS_DIR}"/evidence/*.code; do
|
||||
[ -e "${f}" ] || continue
|
||||
printf '%s -> %s\n' "$(basename "${f}" .code)" "$(cat "${f}")"
|
||||
done
|
||||
} > "${REPORT_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Signal-based lifecycle: dedups against open issues by label
|
||||
# (fault-tolerance + FT case), labels new issues (functional-test,
|
||||
# category, case IDs, env), and closes fixed issues after a fully
|
||||
# green run. Never acts on cancelled runs. Logic lives in
|
||||
# auto-testing/scripts/issue_manager.py, which parses the
|
||||
# FT-CASE verdict lines from the suite log.
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'fault-tolerance'
|
||||
SUITE_LABEL: 'Fault-Tolerance'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
|
||||
- name: Upload test logs & evidence
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: rustfs-fault-tolerance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/report.md
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/suite.log
|
||||
${{ env.FUNCTIONAL_ARTIFACTS_DIR }}/evidence/
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Cleanup environment (after)
|
||||
if: ${{ always() && inputs.cleanup_after != 'false' }}
|
||||
run: |
|
||||
./auto-testing/rustfs-fault-tolerance-test.sh --cleanup -y --log-file "${LOG_FILE}" || true
|
||||
|
||||
- name: "Continue functional chain (next: Performance)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
# workflow_dispatch runs stop after their own cleanup. A failed
|
||||
# handoff retries, then files an alert issue in rustfs/backlog.
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
run: |
|
||||
set -uo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; cannot dispatch the next suite" >&2
|
||||
exit 1
|
||||
fi
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=fault-tolerance'; then
|
||||
echo "dispatched next suite Performance (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
echo "dispatch attempt ${attempt} failed; retrying in ${attempt}0s" >&2
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after fault-tolerance (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The functional chain could not hand off from **fault-tolerance** to **Performance** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-performance'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test \
|
||||
|| gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}" \
|
||||
|| echo "could not file the stall alert issue either; check the token" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Notify on failure
|
||||
if: failure()
|
||||
run: |
|
||||
echo "RustFS fault-tolerance test failed"
|
||||
echo "Package source: ${{ inputs.package_url || 'nightly (R2 latest)' }}"
|
||||
echo "See the uploaded log artifact and FT-CASE lines for details."
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
# Functional chain driver: runs the ten functional suites in a fixed order
|
||||
# (upgrade -> s3 -> kms -> tier -> storage -> heal -> pool -> security ->
|
||||
# replication -> fault-tolerance -> performance). Each suite attempts the next handoff even
|
||||
# replication -> performance). Each suite attempts the next handoff even
|
||||
# when its tests fail.
|
||||
#
|
||||
# Each suite workflow can still be dispatched standalone (workflow_dispatch);
|
||||
|
||||
@@ -280,51 +280,65 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'heal'
|
||||
SUITE_LABEL: 'Heal'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload test logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
|
||||
@@ -4,8 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
@@ -230,55 +231,65 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'kms'
|
||||
SUITE_LABEL: 'KMS'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
|
||||
@@ -237,48 +237,65 @@ jobs:
|
||||
echo "created ${REPORT_PATH} in rustfs/dashboard"
|
||||
fi
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Signal-based lifecycle: dedups against open issues by label,
|
||||
# labels new issues (functional-test, category, case IDs, env),
|
||||
# closes fixed issues after a fully green run. Never acts on
|
||||
# cancelled runs. Logic lives in auto-testing/scripts/.
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.benchmark.outcome == 'failure' || steps.benchmark.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'performance'
|
||||
SUITE_LABEL: 'Performance'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.benchmark.outcome }}" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload test logs & results
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
|
||||
@@ -556,54 +556,64 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'pool'
|
||||
SUITE_LABEL: 'Pool expansion'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-report.md'
|
||||
LOG_FILE: '${{ env.POOL_ARTIFACT_DIR }}/pool-test.log'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.pool_test.outcome }}" \
|
||||
--report-file "${POOL_ARTIFACT_DIR}/pool-report.md" \
|
||||
--log "${POOL_ARTIFACT_DIR}/pool-test.log" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload test logs
|
||||
if: always()
|
||||
|
||||
@@ -18,8 +18,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
@@ -238,55 +239,65 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'replication'
|
||||
SUITE_LABEL: 'Replication (bucket + site)'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
@@ -318,7 +329,7 @@ jobs:
|
||||
'
|
||||
done
|
||||
|
||||
- name: "Continue functional chain (next: Fault tolerance)"
|
||||
- name: "Continue functional chain (next: Performance)"
|
||||
if: ${{ always() && github.event_name == 'repository_dispatch' }}
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
@@ -331,9 +342,9 @@ jobs:
|
||||
DISPATCHED=0
|
||||
for attempt in 1 2 3; do
|
||||
if gh api --method POST repos/rustfs/rustfs/dispatches \
|
||||
-f event_type='rustfs-chain-fault-tolerance' \
|
||||
-f event_type='rustfs-chain-performance' \
|
||||
-F 'client_payload[from_suite]=replication'; then
|
||||
echo "dispatched next suite Fault tolerance (attempt ${attempt})"
|
||||
echo "dispatched next suite Performance (attempt ${attempt})"
|
||||
DISPATCHED=1
|
||||
break
|
||||
fi
|
||||
@@ -341,19 +352,19 @@ jobs:
|
||||
sleep "${attempt}0"
|
||||
done
|
||||
if [ "${DISPATCHED:-0}" -ne 1 ]; then
|
||||
echo "ERROR: functional chain stalled: could not dispatch Fault tolerance after 3 attempts" >&2
|
||||
echo "ERROR: functional chain stalled: could not dispatch Performance after 3 attempts" >&2
|
||||
TITLE="[functional][chain] stalled after replication (run ${GITHUB_RUN_ID})"
|
||||
BODY_FILE="$(mktemp)"
|
||||
trap 'rm -f "${BODY_FILE}"' EXIT
|
||||
{
|
||||
echo "The functional chain could not hand off from **replication** to **Fault tolerance** after 3 attempts."
|
||||
echo "The functional chain could not hand off from **replication** to **Performance** after 3 attempts."
|
||||
echo ""
|
||||
echo "- Failed suite job: ${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}"
|
||||
echo "- Expected next event: 'rustfs-chain-fault-tolerance'"
|
||||
echo "- Expected next event: 'rustfs-chain-performance'"
|
||||
echo "- Likely cause: PF_TESTING_GH_TOKEN lacks contents:write on rustfs/rustfs, or the GitHub API was unavailable."
|
||||
echo "- Recovery: re-dispatch manually with"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-fault-tolerance'"
|
||||
echo " gh api --method POST repos/rustfs/rustfs/dispatches -f event_type='rustfs-chain-performance'"
|
||||
FENCE="$(printf "\x60\x60\x60")"; echo " ${FENCE}"
|
||||
} > "${BODY_FILE}"
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
|
||||
@@ -4,8 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
@@ -207,55 +208,65 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 's3'
|
||||
SUITE_LABEL: 'S3 compatibility'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
|
||||
@@ -18,8 +18,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
@@ -239,54 +240,65 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'security'
|
||||
SUITE_LABEL: 'Security'
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/report.md
|
||||
LOG_FILE: ''
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report-file "${SECURITY_ARTIFACTS_DIR}/report.md" \
|
||||
--log "${SECURITY_ARTIFACTS_DIR}/suite.log" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
|
||||
@@ -4,8 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
@@ -222,55 +223,65 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'storage'
|
||||
SUITE_LABEL: 'Storage engine'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
|
||||
@@ -4,8 +4,9 @@ on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
rustfs_version:
|
||||
description: 'RustFS release tag to test (leave empty to use the latest nightly deb)'
|
||||
description: 'RustFS release tag to test (e.g. 1.0.0-rc.4-preview.1)'
|
||||
required: false
|
||||
default: '1.0.0-rc.4-preview.1'
|
||||
package_url:
|
||||
description: 'Direct .deb URL (nightly/R2/dev). Overrides rustfs_version.'
|
||||
required: false
|
||||
@@ -494,55 +495,75 @@ jobs:
|
||||
fi
|
||||
[ "${failed}" -eq 0 ]
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled' || steps.evidence_verify.outcome == 'failure' || steps.evidence_verify.outcome == 'cancelled' || steps.gate.outcome == 'failure' || steps.gate.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
SUITE: 'tier'
|
||||
SUITE_LABEL: 'Tier'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
EVIDENCE_DIR: ${{ env.TIER_ARTIFACTS_DIR }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
VERIFY_OUTCOME: ${{ steps.evidence_verify.outcome }}
|
||||
GATE_OUTCOME: ${{ steps.gate.outcome }}
|
||||
REPORT_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier-report.md
|
||||
LOG_FILE: ${{ env.TIER_ARTIFACTS_DIR }}/rustfs-tier.log
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
PACKAGE_URL='${{ inputs.package_url }}'
|
||||
RUSTFS_VERSION='${{ inputs.rustfs_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${PACKAGE_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_URL}"
|
||||
elif [ -n "${RUSTFS_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="version ${RUSTFS_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo "- Evidence initialization: ${EVIDENCE_OUTCOME}"
|
||||
echo "- Evidence verification: ${VERIFY_OUTCOME}"
|
||||
echo "- Final gate: ${GATE_OUTCOME}"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" != "success" ]; then
|
||||
echo "(the run evidence directory was rejected; its contents were not read)"
|
||||
elif [ ! -d "${EVIDENCE_DIR}" ] || [ -L "${EVIDENCE_DIR}" ]; then
|
||||
echo "(the run evidence directory is missing or unsafe; its contents were not read)"
|
||||
elif [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${TIER_ARTIFACTS_DIR}/rustfs-tier-cases.md" \
|
||||
--report-file "${TIER_ARTIFACTS_DIR}/rustfs-tier-report.md" \
|
||||
--log "${TIER_ARTIFACTS_DIR}/rustfs-tier.log" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: "Continue functional chain (next: Storage engine)"
|
||||
# Only chain-triggered runs forward to the next suite; standalone
|
||||
|
||||
@@ -306,62 +306,65 @@ jobs:
|
||||
fi
|
||||
rm -f "${B64_FILE}"
|
||||
|
||||
- name: Manage backlog issues (dedup / label / auto-close)
|
||||
# Replaces the old per-run failure filing. One entry point that:
|
||||
# - dedups by signal: failing cases are matched against open backlog
|
||||
# issues by label (category + case ID); covered cases become a
|
||||
# comment on the existing issue, only uncovered cases file a new one
|
||||
# - labels new issues (functional-test, category, case IDs, env)
|
||||
# - closes fixed issues after a fully green run
|
||||
# - never files or closes on cancelled runs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
- name: File failure issue in rustfs/backlog
|
||||
if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}
|
||||
continue-on-error: true
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.PF_TESTING_GH_TOKEN }}
|
||||
EVIDENCE_OUTCOME: ${{ steps.evidence.outcome }}
|
||||
SUITE: 'upgrade'
|
||||
SUITE_LABEL: 'Upgrade compatibility'
|
||||
RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [ -z "${GH_TOKEN:-}" ]; then
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue management"
|
||||
echo "PF_TESTING_GH_TOKEN is not configured; skipping backlog issue"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -f auto-testing/scripts/issue_manager.py ]; then
|
||||
echo "issue_manager.py not found in auto-testing checkout; skipping"
|
||||
TITLE="[functional][${SUITE}] ${SUITE_LABEL} suite failed (run ${GITHUB_RUN_ID})"
|
||||
EXISTING="$(gh issue list -R rustfs/backlog --state all \
|
||||
--search "in:title \"run ${GITHUB_RUN_ID}\"" \
|
||||
--json number --jq '.[].number' || true)"
|
||||
if [ -n "${EXISTING}" ]; then
|
||||
echo "backlog issue already exists for run ${GITHUB_RUN_ID}; skipping"
|
||||
exit 0
|
||||
fi
|
||||
FROM_URL='${{ inputs.from_url }}'
|
||||
FROM_VERSION='${{ inputs.from_version }}'
|
||||
TO_URL='${{ inputs.to_url }}'
|
||||
TO_VERSION='${{ inputs.to_version }}'
|
||||
PACKAGE_SOURCE=""
|
||||
if [ -n "${TO_URL}" ]; then
|
||||
PACKAGE_SOURCE="to ${TO_URL}"
|
||||
elif [ -n "${TO_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="to version ${TO_VERSION}"
|
||||
else
|
||||
PACKAGE_SOURCE="to ${RUSTFS_NIGHTLY_PACKAGE_URL}"
|
||||
redact() {
|
||||
sed -E \
|
||||
-e 's/(RUSTFS_(ACCESS_KEY|SECRET_KEY)[=: ]+)[^[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/(Authorization:).*/\1 [REDACTED]/Ig' \
|
||||
-e 's/(X-Amz-Signature=)[^&[:space:]]+/\1[REDACTED]/Ig' \
|
||||
-e 's/^.*(password|secret|token)[=: ].*/[REDACTED SENSITIVE LINE]/Ig'
|
||||
}
|
||||
BODY_FILE="$(mktemp)"
|
||||
{
|
||||
echo "The **${SUITE_LABEL}** functional suite failed."
|
||||
echo ""
|
||||
echo "- Suite: \`${SUITE}\`"
|
||||
echo "- Run: ${RUN_URL}"
|
||||
echo "- Attempt: ${GITHUB_RUN_ATTEMPT}"
|
||||
echo "- Workflow Commit: ${GITHUB_SHA}"
|
||||
echo "- Trigger: ${GITHUB_EVENT_NAME}"
|
||||
echo "- Date: $(date -u +%Y-%m-%d)"
|
||||
echo ""
|
||||
echo "## Report (errors and symptoms)"
|
||||
echo ""
|
||||
if [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${REPORT_FILE}" ]; then
|
||||
redact < "${REPORT_FILE}"
|
||||
elif [ "${EVIDENCE_OUTCOME}" = "success" ] && [ -s "${LOG_FILE:-}" ]; then
|
||||
echo "(report file missing; log tail below)"
|
||||
echo ""
|
||||
tail -n 200 "${LOG_FILE}" | redact
|
||||
else
|
||||
echo "(no report or log file was produced)"
|
||||
fi
|
||||
} | head -c 55000 > "${BODY_FILE}"
|
||||
gh label create functional-test -R rustfs/backlog --color d73a4a 2>/dev/null || true
|
||||
if ! gh issue create -R rustfs/backlog --title "${TITLE}" \
|
||||
--body-file "${BODY_FILE}" --label functional-test; then
|
||||
gh issue create -R rustfs/backlog --title "${TITLE}" --body-file "${BODY_FILE}"
|
||||
fi
|
||||
if [ -n "${FROM_URL}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_SOURCE}; from ${FROM_URL}"
|
||||
elif [ -n "${FROM_VERSION}" ]; then
|
||||
PACKAGE_SOURCE="${PACKAGE_SOURCE}; from version ${FROM_VERSION}"
|
||||
fi
|
||||
python3 auto-testing/scripts/issue_manager.py handle \
|
||||
--repo rustfs/backlog \
|
||||
--suite "${SUITE}" --category "${SUITE}" --suite-label "${SUITE_LABEL}" \
|
||||
--outcome "${{ steps.test.outcome }}" \
|
||||
--report "${FUNCTIONAL_ARTIFACTS_DIR}/cases.md" \
|
||||
--report-file "${REPORT_FILE}" \
|
||||
--log "${LOG_FILE}" \
|
||||
--run-url "${RUN_URL}" \
|
||||
--run-id "${GITHUB_RUN_ID}" \
|
||||
--attempt "${GITHUB_RUN_ATTEMPT}" \
|
||||
--commit "${GITHUB_SHA}" \
|
||||
--trigger "${{ github.event_name }}" \
|
||||
--package-source "${PACKAGE_SOURCE}" \
|
||||
--date "$(date -u +%Y-%m-%d)"
|
||||
echo "filed backlog issue for suite ${SUITE}"
|
||||
|
||||
- name: Upload report and logs
|
||||
if: ${{ always() && steps.evidence.outcome == 'success' }}
|
||||
|
||||
@@ -17609,17 +17609,6 @@ impl ECStore {
|
||||
}
|
||||
self.persist_decommission_durable_ilm_receipt(source_pool_idx, target_pool_idx, &receipt)
|
||||
.await?;
|
||||
self.decommission_durable_ilm_receipt_path_for_test(source_pool_idx, source_path, record)
|
||||
.await
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "test-util"))]
|
||||
pub(crate) async fn decommission_durable_ilm_receipt_path_for_test(
|
||||
&self,
|
||||
source_pool_idx: usize,
|
||||
source_path: &str,
|
||||
record: &ValidatedDurableIlmRecord,
|
||||
) -> Result<String> {
|
||||
let run_token = self.durable_ilm_receipt_run_token(source_pool_idx).await?;
|
||||
Ok(decommission_durable_ilm_receipt_path(&run_token, source_path, record.id_kind, &record.id))
|
||||
}
|
||||
|
||||
@@ -865,8 +865,9 @@ mod tests {
|
||||
io::ErrorKind::PermissionDenied,
|
||||
"staging rejected",
|
||||
)));
|
||||
let cloned = marked.clone();
|
||||
assert!(marked.is_conditional_file_not_committed());
|
||||
assert!(marked.is_conditional_file_not_committed());
|
||||
assert!(cloned.is_conditional_file_not_committed());
|
||||
assert!(!DiskError::Timeout.is_conditional_file_not_committed());
|
||||
assert!(
|
||||
!DiskError::Io(io::Error::new(io::ErrorKind::PermissionDenied, "rename rejected"))
|
||||
|
||||
@@ -18604,75 +18604,71 @@ mod put_object_tmp_cleanup_tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn cancelled_rename_keeps_namespace_lock_until_publication() {
|
||||
temp_env::async_with_vars([(ENV_RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE, Some("false"))], async {
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-commit-lock-cancelled-rename";
|
||||
let object = "commit-lock-cancelled-rename-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks(4).await;
|
||||
let bucket = "put-commit-lock-cancelled-rename";
|
||||
let object = "commit-lock-cancelled-rename-object";
|
||||
for disk in &disk_stores {
|
||||
disk.make_volume(bucket).await.expect("bucket volume should be created");
|
||||
}
|
||||
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_store = Arc::clone(&set_disks);
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
first_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
let rename_tasks = rename_fanout_barrier::observe_tasks(object);
|
||||
let rename_barrier = rename_fanout_barrier::arm(object, 0, rename_fanout_barrier::PHASE_RENAME);
|
||||
let first_store = Arc::clone(&set_disks);
|
||||
let first = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'1'; TEST_OBJECT_SIZE]);
|
||||
first_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("first PUT should pause during the authoritative rename");
|
||||
|
||||
let second_namespace_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
|
||||
let second_store = Arc::clone(&set_disks);
|
||||
let second = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
second_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
});
|
||||
second_namespace_barrier.release_and_wait_until_namespace_pending().await;
|
||||
|
||||
first.abort();
|
||||
assert!(
|
||||
first
|
||||
.await
|
||||
.expect_err("the first request should be cancelled while rename is parked")
|
||||
.is_cancelled()
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
!second.is_finished(),
|
||||
"the second writer must remain blocked by the cancelled commit owner"
|
||||
);
|
||||
|
||||
rename_barrier.release();
|
||||
drop(rename_barrier);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
});
|
||||
tokio::time::timeout(Duration::from_secs(30), rename_barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("the cancelled owner's rename fanout should drain");
|
||||
second
|
||||
.await
|
||||
.expect("second overwrite task should join")
|
||||
.expect("second overwrite should commit after the cancelled owner reaches publication");
|
||||
.expect("first PUT should pause during the authoritative rename");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
let second_namespace_barrier = PutObjectCommitBarrier::install(bucket, object, PutObjectCommitPause::BeforeNamespace);
|
||||
let second_store = Arc::clone(&set_disks);
|
||||
let second = tokio::spawn(async move {
|
||||
let mut reader = PutObjReader::from_vec(vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
second_store
|
||||
.put_object(bucket, object, &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the latest overwrite should be readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
});
|
||||
second_namespace_barrier.release_and_wait_until_namespace_pending().await;
|
||||
|
||||
first.abort();
|
||||
assert!(
|
||||
first
|
||||
.await
|
||||
.expect_err("the first request should be cancelled while rename is parked")
|
||||
.is_cancelled()
|
||||
);
|
||||
tokio::task::yield_now().await;
|
||||
assert!(
|
||||
!second.is_finished(),
|
||||
"the second writer must remain blocked by the cancelled commit owner"
|
||||
);
|
||||
|
||||
rename_barrier.release();
|
||||
drop(rename_barrier);
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
while rename_tasks.running() != 0 {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
})
|
||||
.await;
|
||||
.await
|
||||
.expect("the cancelled owner's rename fanout should drain");
|
||||
second
|
||||
.await
|
||||
.expect("second overwrite task should join")
|
||||
.expect("second overwrite should commit after the cancelled owner reaches publication");
|
||||
|
||||
let mut reader = set_disks
|
||||
.get_object_reader(bucket, object, None, HeaderMap::new(), &ObjectOptions::default())
|
||||
.await
|
||||
.expect("the latest overwrite should be readable");
|
||||
let mut body = Vec::new();
|
||||
reader.stream.read_to_end(&mut body).await.expect("latest body should drain");
|
||||
assert_eq!(body, vec![b'2'; TEST_OBJECT_SIZE]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -9643,13 +9643,7 @@ mod tests {
|
||||
com::save_config(store.pools[1].clone(), &manual_task_path, manual_task_bytes.clone())
|
||||
.await
|
||||
.expect("target task rewrite should invalidate cached metadata before the quorum check");
|
||||
let manual_task_record =
|
||||
validate_durable_ilm_record(&manual_task_path, &manual_task_bytes).expect("manual task should validate");
|
||||
let manual_task_receipt_path = store
|
||||
.decommission_durable_ilm_receipt_path_for_test(0, &manual_task_path, &manual_task_record)
|
||||
.await
|
||||
.expect("manual task receipt path should resolve");
|
||||
let target_task_set = store.pools[1].get_disks_by_key(&manual_task_receipt_path);
|
||||
let target_task_set = store.pools[1].get_disks_by_key(&manual_task_path);
|
||||
let original_target_task_disks = {
|
||||
let mut disks = target_task_set.disks.write().await;
|
||||
let original = disks.clone();
|
||||
|
||||
+27
-189
@@ -2133,7 +2133,7 @@ impl DefaultObjectUsecase {
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn build_reader_blob<R>(
|
||||
fn build_reader_blob<R>(
|
||||
reader: R,
|
||||
response_content_length: i64,
|
||||
request_id: &str,
|
||||
@@ -2144,12 +2144,10 @@ impl DefaultObjectUsecase {
|
||||
key: &str,
|
||||
lifecycle: GetObjectBodyLifecycle,
|
||||
resume: Option<GetObjectResumeControl<R>>,
|
||||
) -> S3Result<StreamingBlob>
|
||||
) -> StreamingBlob
|
||||
where
|
||||
R: AsyncRead + Send + Sync + Unpin + 'static,
|
||||
{
|
||||
use tokio::io::AsyncReadExt as _;
|
||||
|
||||
let streaming_blob_start = rustfs_io_metrics::get_stage_metrics_enabled().then(std::time::Instant::now);
|
||||
let expected = usize::try_from(response_content_length.max(0)).unwrap_or(usize::MAX);
|
||||
let tuned_stream_buffer_size =
|
||||
@@ -2165,7 +2163,7 @@ impl DefaultObjectUsecase {
|
||||
);
|
||||
}
|
||||
let handoff_start = get_stage_metrics_enabled.then(std::time::Instant::now);
|
||||
let mut reader = GetObjectStreamingReader::new(
|
||||
let reader = GetObjectStreamingReader::new(
|
||||
reader,
|
||||
bucket,
|
||||
key,
|
||||
@@ -2176,17 +2174,6 @@ impl DefaultObjectUsecase {
|
||||
lifecycle,
|
||||
resume,
|
||||
);
|
||||
let mut prefix = [0_u8; 1];
|
||||
let prefix_len = if expected == 0 {
|
||||
0
|
||||
} else {
|
||||
reader
|
||||
.read_exact(&mut prefix)
|
||||
.await
|
||||
.map_err(|error| map_get_object_reader_error(StorageError::from(error)))?;
|
||||
1
|
||||
};
|
||||
let reader = std::io::Cursor::new(prefix).take(prefix_len).chain(reader);
|
||||
let stream = GetObjectReaderStream::new(reader, stream_buffer_size, expected, stream_strategy.as_str(), buffer_source)
|
||||
.with_diagnostics(bucket, key, request_id);
|
||||
let blob = StreamingBlob::new(stream);
|
||||
@@ -2200,7 +2187,7 @@ impl DefaultObjectUsecase {
|
||||
);
|
||||
}
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAMING_BLOB, streaming_blob_start);
|
||||
Ok(blob)
|
||||
blob
|
||||
}
|
||||
|
||||
fn init_get_object_bootstrap(&self, bucket: &str, key: &str, request_id: &str) -> S3Result<GetObjectBootstrap> {
|
||||
@@ -3174,7 +3161,7 @@ impl DefaultObjectUsecase {
|
||||
let (stream_buffer_size, stream_strategy) =
|
||||
Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start);
|
||||
return Self::build_reader_blob(
|
||||
return Ok(Self::build_reader_blob(
|
||||
final_stream,
|
||||
response_content_length,
|
||||
request_id,
|
||||
@@ -3185,8 +3172,7 @@ impl DefaultObjectUsecase {
|
||||
key,
|
||||
lifecycle,
|
||||
resume(info),
|
||||
)
|
||||
.await;
|
||||
));
|
||||
}
|
||||
|
||||
if let Some(buffered_body) = buffered_body {
|
||||
@@ -3253,7 +3239,7 @@ impl DefaultObjectUsecase {
|
||||
let (stream_buffer_size, stream_strategy) =
|
||||
Self::select_stream_buffer_strategy(response_content_length, optimal_buffer_size, enable_readahead, has_range);
|
||||
record_get_object_s3_handler_stage_duration(GET_OBJECT_STAGE_BODY_STREAM_STRATEGY, stream_strategy_start);
|
||||
Self::build_reader_blob(
|
||||
Ok(Self::build_reader_blob(
|
||||
final_stream,
|
||||
response_content_length,
|
||||
request_id,
|
||||
@@ -3264,8 +3250,7 @@ impl DefaultObjectUsecase {
|
||||
key,
|
||||
lifecycle,
|
||||
resume(info),
|
||||
)
|
||||
.await
|
||||
))
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
@@ -5862,11 +5847,8 @@ mod tests {
|
||||
}
|
||||
|
||||
impl AsyncRead for ReadProbeReader {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
fn poll_read(self: Pin<&mut Self>, _cx: &mut Context<'_>, _buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
self.reads.fetch_add(1, AtomicOrdering::Relaxed);
|
||||
if buf.remaining() > 0 {
|
||||
buf.put_slice(b"x");
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
}
|
||||
@@ -6466,11 +6448,12 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("reservation bypass must construct the normal streaming fallback");
|
||||
let mut received = Vec::new();
|
||||
while let Some(chunk) = fallback_body.next().await {
|
||||
received.extend_from_slice(&chunk.expect("fallback stream must not fail"));
|
||||
}
|
||||
assert_eq!(received, b"body");
|
||||
let chunk = fallback_body
|
||||
.next()
|
||||
.await
|
||||
.expect("fallback stream must yield a body chunk")
|
||||
.expect("fallback stream must not fail");
|
||||
assert_eq!(chunk, Bytes::from_static(b"body"));
|
||||
assert!(fallback_reads.load(AtomicOrdering::Relaxed) > 0);
|
||||
assert_eq!(readers.load(AtomicOrdering::Relaxed), 0, "cold-fill materialization must remain unopened");
|
||||
assert_eq!(coordinator.active_session_count_for_test(), 0);
|
||||
@@ -7790,151 +7773,6 @@ mod tests {
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_reader_blob_rejects_quorum_failure_before_handoff() {
|
||||
let result = DefaultObjectUsecase::build_reader_blob(
|
||||
FailAtEndReader::new(
|
||||
b"",
|
||||
Some(std::io::Error::other(StorageError::InsufficientReadQuorum(
|
||||
"test-bucket".to_string(),
|
||||
"unavailable-object".to_string(),
|
||||
))),
|
||||
),
|
||||
5,
|
||||
"req-preheader-quorum",
|
||||
None,
|
||||
64,
|
||||
GetObjectStreamStrategy::Standard,
|
||||
"test-bucket",
|
||||
"unavailable-object",
|
||||
GetObjectBodyLifecycle::disabled(),
|
||||
None,
|
||||
)
|
||||
.await;
|
||||
|
||||
let error = result.expect_err("a read quorum failure before the first byte must reject response construction");
|
||||
assert_eq!(error.code(), &S3ErrorCode::Custom("SlowDownRead".into()));
|
||||
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(error.message(), Some("Resource requested is unreadable, please reduce your request rate"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_reader_blob_preserves_primed_byte_for_full_and_range() {
|
||||
for (request_id, content_range) in [("req-preheader-full", None), ("req-preheader-range", Some("bytes 10-14/100"))] {
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let mut body = DefaultObjectUsecase::build_reader_blob(
|
||||
DataProbeReader {
|
||||
reads: Arc::clone(&reads),
|
||||
data: std::io::Cursor::new(b"hello".to_vec()),
|
||||
},
|
||||
5,
|
||||
request_id,
|
||||
content_range,
|
||||
64,
|
||||
GetObjectStreamStrategy::Standard,
|
||||
"test-bucket",
|
||||
"test-object",
|
||||
GetObjectBodyLifecycle::disabled(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("the first byte should be available before response handoff");
|
||||
|
||||
assert_eq!(reads.load(AtomicOrdering::Relaxed), 1, "response construction must prime one byte");
|
||||
let mut received = Vec::new();
|
||||
while let Some(chunk) = body.next().await {
|
||||
received.extend_from_slice(&chunk.expect("the primed body should remain readable"));
|
||||
}
|
||||
assert_eq!(received, b"hello", "the primed byte must be delivered exactly once");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_reader_blob_does_not_poll_empty_object() {
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let mut body = DefaultObjectUsecase::build_reader_blob(
|
||||
ReadProbeReader {
|
||||
reads: Arc::clone(&reads),
|
||||
},
|
||||
0,
|
||||
"req-empty-object",
|
||||
None,
|
||||
64,
|
||||
GetObjectStreamStrategy::Standard,
|
||||
"test-bucket",
|
||||
"empty-object",
|
||||
GetObjectBodyLifecycle::disabled(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("an empty response should not require a storage read");
|
||||
|
||||
assert_eq!(reads.load(AtomicOrdering::Relaxed), 0);
|
||||
assert!(body.next().await.is_none());
|
||||
assert_eq!(reads.load(AtomicOrdering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_reader_blob_leaves_later_failure_in_body_stream() {
|
||||
let mut body = DefaultObjectUsecase::build_reader_blob(
|
||||
FailAtEndReader::new(b"h", Some(std::io::Error::other("failure after handoff"))),
|
||||
5,
|
||||
"req-postheader-failure",
|
||||
None,
|
||||
64,
|
||||
GetObjectStreamStrategy::Standard,
|
||||
"test-bucket",
|
||||
"later-failure-object",
|
||||
GetObjectBodyLifecycle::disabled(),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("the available first byte should allow response handoff");
|
||||
|
||||
let first = body
|
||||
.next()
|
||||
.await
|
||||
.expect("the primed byte must be present")
|
||||
.expect("the primed byte must be successful");
|
||||
assert_eq!(first, Bytes::from_static(b"h"));
|
||||
let error = body
|
||||
.next()
|
||||
.await
|
||||
.expect("the later read must produce a body result")
|
||||
.expect_err("a failure after the first byte must stay in the body stream");
|
||||
assert!(error.to_string().contains("failure after handoff"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_reader_blob_resume_offset_includes_primed_byte() {
|
||||
let reopen_count = Arc::new(AtomicUsize::new(0));
|
||||
let control = counting_resume_control(Arc::clone(&reopen_count), |emitted| {
|
||||
assert_eq!(emitted, 1, "resume must start after the byte consumed before response handoff");
|
||||
Ok(FailAtEndReader::new(b"ello", None))
|
||||
});
|
||||
let mut body = DefaultObjectUsecase::build_reader_blob(
|
||||
FailAtEndReader::new(b"h", Some(relocation_read_error())),
|
||||
5,
|
||||
"req-preheader-resume-offset",
|
||||
None,
|
||||
64,
|
||||
GetObjectStreamStrategy::Standard,
|
||||
"test-bucket",
|
||||
"relocated-object",
|
||||
GetObjectBodyLifecycle::disabled(),
|
||||
Some(control),
|
||||
)
|
||||
.await
|
||||
.expect("the first byte should permit response handoff before relocation");
|
||||
|
||||
let mut received = Vec::new();
|
||||
while let Some(chunk) = body.next().await {
|
||||
received.extend_from_slice(&chunk.expect("resume should complete the body"));
|
||||
}
|
||||
assert_eq!(received, b"hello");
|
||||
assert_eq!(reopen_count.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn get_object_streaming_reader_resumes_after_relocation_error() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
@@ -9203,7 +9041,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_get_object_body_primes_large_stream_before_handoff() {
|
||||
async fn build_get_object_body_keeps_large_objects_on_streaming_path_without_preread() {
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let reader = ReadProbeReader {
|
||||
reads: Arc::clone(&reads),
|
||||
@@ -9236,13 +9074,13 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
reads.load(AtomicOrdering::Relaxed),
|
||||
1,
|
||||
"large-object response construction should prime exactly one byte"
|
||||
0,
|
||||
"large-object response construction should not pre-read object data"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn build_get_object_body_primes_large_encrypted_stream_before_handoff() {
|
||||
async fn build_get_object_body_keeps_large_encrypted_objects_on_streaming_path_without_preread() {
|
||||
let reads = Arc::new(AtomicUsize::new(0));
|
||||
let reader = ReadProbeReader {
|
||||
reads: Arc::clone(&reads),
|
||||
@@ -9275,8 +9113,8 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
reads.load(AtomicOrdering::Relaxed),
|
||||
1,
|
||||
"large encrypted object response construction should prime exactly one byte"
|
||||
0,
|
||||
"large encrypted object response construction should not pre-read object data"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -9446,8 +9284,8 @@ mod tests {
|
||||
assert_eq!(fill, rustfs_object_data_cache::ObjectDataCacheFillResult::SkippedSizeMismatch);
|
||||
assert_eq!(
|
||||
reads.load(AtomicOrdering::Relaxed),
|
||||
1,
|
||||
"size-mismatched rejected fill should prime the fallback stream before handoff"
|
||||
0,
|
||||
"size-mismatched rejected fill should construct the fallback stream without pre-reading"
|
||||
);
|
||||
assert!(
|
||||
matches!(lookup_after_mismatch, rustfs_object_data_cache::ObjectDataCacheLookup::Miss),
|
||||
@@ -10155,8 +9993,8 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
reads.load(AtomicOrdering::Relaxed),
|
||||
1,
|
||||
"too-large materialize-fill candidate must prime the streaming fallback"
|
||||
0,
|
||||
"too-large materialize-fill candidate must not pre-read the fallback reader"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -10194,8 +10032,8 @@ mod tests {
|
||||
|
||||
assert_eq!(
|
||||
reads.load(AtomicOrdering::Relaxed),
|
||||
1,
|
||||
"default GetObject response construction should prime exactly one byte"
|
||||
0,
|
||||
"default GetObject response construction should not pre-read small plain object data"
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -465,7 +465,7 @@ mod bucket_default_sse_lookup_tests {
|
||||
let err = classify_bucket_default_sse_lookup("bucket", Err(StorageError::ErasureReadQuorum))
|
||||
.expect_err("an unreadable metadata subsystem must never degrade to plaintext");
|
||||
|
||||
assert_eq!(err.code(), &S3ErrorCode::Custom("SlowDownRead".into()));
|
||||
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
+6
-25
@@ -20,8 +20,6 @@ use s3s::{S3Error, S3ErrorCode};
|
||||
|
||||
const MAX_VERSIONS_EXCEEDED_CODE: &str = "MaxVersionsExceeded";
|
||||
const MAX_VERSIONS_EXCEEDED_MESSAGE: &str = "You've exceeded the limit on the number of versions you can create on this object";
|
||||
const SLOW_DOWN_READ_CODE: &str = "SlowDownRead";
|
||||
const SLOW_DOWN_READ_MESSAGE: &str = "Resource requested is unreadable, please reduce your request rate";
|
||||
|
||||
/// S3 error code for a request that names a KMS key the KMS does not hold.
|
||||
pub const KMS_KEY_NOT_FOUND_ERROR_CODE: &str = "KMS.NotFoundException";
|
||||
@@ -107,7 +105,6 @@ fn custom_error_status(code: &S3ErrorCode) -> Option<StatusCode> {
|
||||
S3ErrorCode::Custom(custom) if &**custom == KMS_KEY_NOT_FOUND_ERROR_CODE || &**custom == MAX_VERSIONS_EXCEEDED_CODE => {
|
||||
Some(StatusCode::BAD_REQUEST)
|
||||
}
|
||||
S3ErrorCode::Custom(custom) if &**custom == SLOW_DOWN_READ_CODE => Some(StatusCode::SERVICE_UNAVAILABLE),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -406,7 +403,6 @@ impl ApiError {
|
||||
S3ErrorCode::Custom(code) if &**code == MAX_VERSIONS_EXCEEDED_CODE => {
|
||||
MAX_VERSIONS_EXCEEDED_MESSAGE.to_string()
|
||||
}
|
||||
S3ErrorCode::Custom(code) if &**code == SLOW_DOWN_READ_CODE => SLOW_DOWN_READ_MESSAGE.to_string(),
|
||||
_ => code.as_str().to_string(),
|
||||
}
|
||||
}
|
||||
@@ -581,10 +577,10 @@ impl From<StorageError> for ApiError {
|
||||
| StorageError::FaultyRemoteDisk
|
||||
| StorageError::DiskNotFound
|
||||
| StorageError::TooManyOpenFiles => S3ErrorCode::ServiceUnavailable,
|
||||
StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => {
|
||||
S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into())
|
||||
}
|
||||
StorageError::ErasureWriteQuorum | StorageError::InsufficientWriteQuorum(_, _) => S3ErrorCode::ServiceUnavailable,
|
||||
StorageError::ErasureReadQuorum
|
||||
| StorageError::InsufficientReadQuorum(_, _)
|
||||
| StorageError::ErasureWriteQuorum
|
||||
| StorageError::InsufficientWriteQuorum(_, _) => S3ErrorCode::ServiceUnavailable,
|
||||
StorageError::NamespaceLockQuorumUnavailable { .. } => S3ErrorCode::ServiceUnavailable,
|
||||
StorageError::QuotaExceeded { .. } => S3ErrorCode::InvalidRequest,
|
||||
StorageError::MaxVersionsExceeded => S3ErrorCode::Custom(MAX_VERSIONS_EXCEEDED_CODE.into()),
|
||||
@@ -1292,10 +1288,10 @@ mod tests {
|
||||
(StorageError::FaultyRemoteDisk, S3ErrorCode::ServiceUnavailable),
|
||||
(StorageError::DiskNotFound, S3ErrorCode::ServiceUnavailable),
|
||||
(StorageError::TooManyOpenFiles, S3ErrorCode::ServiceUnavailable),
|
||||
(StorageError::ErasureReadQuorum, S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into())),
|
||||
(StorageError::ErasureReadQuorum, S3ErrorCode::ServiceUnavailable),
|
||||
(
|
||||
StorageError::InsufficientReadQuorum("test".into(), "test".into()),
|
||||
S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()),
|
||||
S3ErrorCode::ServiceUnavailable,
|
||||
),
|
||||
(StorageError::ErasureWriteQuorum, S3ErrorCode::ServiceUnavailable),
|
||||
(
|
||||
@@ -1433,21 +1429,6 @@ mod tests {
|
||||
assert_eq!(s3_error.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn read_quorum_failure_matches_minio_slow_down_read_response() {
|
||||
for error in [
|
||||
StorageError::ErasureReadQuorum,
|
||||
StorageError::InsufficientReadQuorum("bucket".into(), "object".into()),
|
||||
] {
|
||||
let api_error = ApiError::from(error);
|
||||
assert_eq!(api_error.code, S3ErrorCode::Custom(SLOW_DOWN_READ_CODE.into()));
|
||||
assert_eq!(api_error.message, SLOW_DOWN_READ_MESSAGE);
|
||||
|
||||
let s3_error: S3Error = api_error.into();
|
||||
assert_eq!(s3_error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn quota_exceeded_preserves_existing_s3_error_contract() {
|
||||
let api_error: ApiError = StorageError::QuotaExceeded { current: 5, limit: 10 }.into();
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use crate::server::RPC_PREFIX;
|
||||
use crate::storage::request_context::spawn_traced;
|
||||
use crate::storage::storage_api::DiskError;
|
||||
use crate::storage::storage_api::rpc_consumer::http_service::{
|
||||
DEFAULT_READ_BUFFER_SIZE, DeleteOptions, DiskStore, NS_SCANNER_PROTOCOL_VERSION, NsScannerCapabilityResponse,
|
||||
PUT_FILE_AUTH_TRAILER_LEN, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_VERSION, PutFileCapabilityResponse, StorageDiskRpcExt as _,
|
||||
@@ -30,7 +31,6 @@ use crate::storage::storage_api::rpc_consumer::http_service::{
|
||||
};
|
||||
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
|
||||
use crate::storage::storage_api::tonic_rpc_auth_failure_reason;
|
||||
use crate::storage::storage_api::{DiskError, FileReader};
|
||||
use bytes::{Bytes, BytesMut};
|
||||
use futures_util::{Stream, StreamExt, TryStreamExt, stream};
|
||||
use http::{HeaderMap, HeaderValue, Method, Request, Response, StatusCode, Uri};
|
||||
@@ -67,8 +67,6 @@ const LOG_SUBSYSTEM_NAMESPACE_SCANNER: &str = "namespace_scanner";
|
||||
const LOG_SUBSYSTEM_ROUTING: &str = "routing";
|
||||
const EVENT_RPC_REQUEST_REJECTED: &str = "rpc_request_rejected";
|
||||
const EVENT_RPC_REQUEST_FAILED: &str = "rpc_request_failed";
|
||||
const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
|
||||
const MIGRATING_META_BUCKET: &str = ".minio.sys";
|
||||
const EVENT_RPC_BACKGROUND_TASK_FAILED: &str = "rpc_background_task_failed";
|
||||
const RPC_OPERATION_UNKNOWN: &str = "unknown";
|
||||
const READ_FILE_STREAM_PATH: &str = "/rustfs/rpc/read_file_stream";
|
||||
@@ -635,32 +633,34 @@ async fn handle_read_file(req: Request<Incoming>) -> Response<Body> {
|
||||
return response_with_status(StatusCode::BAD_REQUEST, "disk not found");
|
||||
};
|
||||
|
||||
let file =
|
||||
match read_file_stream_with_legacy_meta_fallback(&disk, &query.volume, &query.path, query.offset, query.length).await {
|
||||
Ok(file) => file,
|
||||
Err(e) => {
|
||||
let message = format!("read file err {e}");
|
||||
error!(
|
||||
event = EVENT_RPC_REQUEST_FAILED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_FILE_TRANSFER,
|
||||
operation = INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
result = "failed",
|
||||
status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
|
||||
rpc_path = req.uri().path(),
|
||||
method = %req.method(),
|
||||
reason = "read_file_failed",
|
||||
disk = %query.disk,
|
||||
volume = %query.volume,
|
||||
path = %query.path,
|
||||
offset = query.offset,
|
||||
length = query.length,
|
||||
error = %e,
|
||||
"internode rpc request failed"
|
||||
);
|
||||
return response_with_disk_error(&e, message);
|
||||
}
|
||||
};
|
||||
let file = match disk
|
||||
.read_file_stream(&query.volume, &query.path, query.offset, query.length)
|
||||
.await
|
||||
{
|
||||
Ok(file) => file,
|
||||
Err(e) => {
|
||||
let message = format!("read file err {e}");
|
||||
error!(
|
||||
event = EVENT_RPC_REQUEST_FAILED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_FILE_TRANSFER,
|
||||
operation = INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
result = "failed",
|
||||
status_code = StatusCode::INTERNAL_SERVER_ERROR.as_u16(),
|
||||
rpc_path = req.uri().path(),
|
||||
method = %req.method(),
|
||||
reason = "read_file_failed",
|
||||
disk = %query.disk,
|
||||
volume = %query.volume,
|
||||
path = %query.path,
|
||||
offset = query.offset,
|
||||
length = query.length,
|
||||
error = %e,
|
||||
"internode rpc request failed"
|
||||
);
|
||||
return response_with_disk_error(&e, message);
|
||||
}
|
||||
};
|
||||
|
||||
runtime_sources::current_internode_metrics().record_incoming_request_for_operation_and_backend(
|
||||
INTERNODE_OPERATION_READ_FILE_STREAM,
|
||||
@@ -800,30 +800,28 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
|
||||
let log_disk_id = args.disk_id.clone();
|
||||
let log_skip_total_timeout = args.skip_total_timeout;
|
||||
let body = walk_dir_response_body(propagate_completion_errors, move |mut writer| async move {
|
||||
walk_dir_with_legacy_meta_fallback(&disk, args, &mut writer)
|
||||
.await
|
||||
.map_err(|e| {
|
||||
warn!(
|
||||
event = EVENT_RPC_BACKGROUND_TASK_FAILED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_DIRECTORY_WALK,
|
||||
operation = INTERNODE_OPERATION_WALK_DIR,
|
||||
result = "failed",
|
||||
disk = %log_disk,
|
||||
bucket = %log_bucket,
|
||||
base_dir = %log_base_dir,
|
||||
recursive = log_recursive,
|
||||
report_notfound = log_report_notfound,
|
||||
filter_prefix = ?log_filter_prefix,
|
||||
forward_to = ?log_forward_to,
|
||||
limit = log_limit,
|
||||
disk_id = %log_disk_id,
|
||||
skip_total_timeout = log_skip_total_timeout,
|
||||
error = %e,
|
||||
"internode rpc background task failed"
|
||||
);
|
||||
io::Error::other("remote walk_dir failed")
|
||||
})
|
||||
disk.walk_dir(args, &mut writer).await.map_err(|e| {
|
||||
warn!(
|
||||
event = EVENT_RPC_BACKGROUND_TASK_FAILED,
|
||||
component = LOG_COMPONENT_INTERNODE_RPC,
|
||||
subsystem = LOG_SUBSYSTEM_DIRECTORY_WALK,
|
||||
operation = INTERNODE_OPERATION_WALK_DIR,
|
||||
result = "failed",
|
||||
disk = %log_disk,
|
||||
bucket = %log_bucket,
|
||||
base_dir = %log_base_dir,
|
||||
recursive = log_recursive,
|
||||
report_notfound = log_report_notfound,
|
||||
filter_prefix = ?log_filter_prefix,
|
||||
forward_to = ?log_forward_to,
|
||||
limit = log_limit,
|
||||
disk_id = %log_disk_id,
|
||||
skip_total_timeout = log_skip_total_timeout,
|
||||
error = %e,
|
||||
"internode rpc background task failed"
|
||||
);
|
||||
io::Error::other("remote walk_dir failed")
|
||||
})
|
||||
});
|
||||
|
||||
runtime_sources::current_internode_metrics()
|
||||
@@ -835,54 +833,6 @@ async fn handle_walk_dir(req: Request<Incoming>) -> Response<Body> {
|
||||
.expect("failed to build walk dir response")
|
||||
}
|
||||
|
||||
fn legacy_meta_bucket_alias(volume: &str) -> Option<String> {
|
||||
if volume == MIGRATING_META_BUCKET {
|
||||
return Some(RUSTFS_META_BUCKET.to_string());
|
||||
}
|
||||
volume
|
||||
.strip_prefix(MIGRATING_META_BUCKET)
|
||||
.filter(|rest| rest.starts_with('/'))
|
||||
.map(|rest| format!("{RUSTFS_META_BUCKET}{rest}"))
|
||||
}
|
||||
|
||||
fn legacy_meta_alias_can_retry(error: &DiskError, volume: &str) -> bool {
|
||||
matches!(error, DiskError::FileNotFound | DiskError::VolumeNotFound) && legacy_meta_bucket_alias(volume).is_some()
|
||||
}
|
||||
|
||||
async fn read_file_stream_with_legacy_meta_fallback(
|
||||
disk: &DiskStore,
|
||||
volume: &str,
|
||||
path: &str,
|
||||
offset: usize,
|
||||
length: usize,
|
||||
) -> Result<FileReader, DiskError> {
|
||||
match disk.read_file_stream(volume, path, offset, length).await {
|
||||
Ok(file) => Ok(file),
|
||||
Err(error) if legacy_meta_alias_can_retry(&error, volume) => {
|
||||
let alias = legacy_meta_bucket_alias(volume).expect("legacy meta alias checked before retry");
|
||||
disk.read_file_stream(&alias, path, offset, length).await
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
async fn walk_dir_with_legacy_meta_fallback<W: tokio::io::AsyncWrite + Unpin + Send>(
|
||||
disk: &DiskStore,
|
||||
args: WalkDirOptions,
|
||||
writer: &mut W,
|
||||
) -> Result<(), DiskError> {
|
||||
let original_bucket = args.bucket.clone();
|
||||
match disk.walk_dir(args.clone(), writer).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if legacy_meta_alias_can_retry(&error, &original_bucket) => {
|
||||
let mut retry_args = args;
|
||||
retry_args.bucket = legacy_meta_bucket_alias(&original_bucket).expect("legacy meta alias checked before retry");
|
||||
disk.walk_dir(retry_args, writer).await
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
async fn handle_ns_scanner(req: Request<Incoming>) -> Response<Body> {
|
||||
let query = match parse_query::<NsScannerQuery>(&req) {
|
||||
Ok(query) => query,
|
||||
@@ -1749,14 +1699,13 @@ mod tests {
|
||||
NS_SCANNER_SESSION_SEQUENCE_QUERY, NS_SCANNER_TIER_REGISTRY_GENERATION_QUERY, NsScannerCapabilityResponse,
|
||||
NsScannerQuery, PUT_FILE_AUTH_STREAM_PATH, PUT_FILE_CAPABILITY_PATH, PUT_FILE_STREAM_PATH, PutFileQuery,
|
||||
READ_FILE_STREAM_PATH, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_PATH, WalkDirQuery, append_walk_dir_completion,
|
||||
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, legacy_meta_bucket_alias,
|
||||
ns_scanner_response_body, ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_auth_nonce,
|
||||
put_file_capability_response, put_file_server_epoch_accepted, put_file_server_epoch_matches,
|
||||
put_file_stage_error_message, put_file_target_lock, read_file_body_stream, read_file_stream_buffer_size,
|
||||
remote_scanner_claim_rejection, response_with_disk_error, supports_walk_dir_stream_completion,
|
||||
validate_walk_dir_completion_request, verify_internode_rpc_signature, verify_ns_scanner_body_digest,
|
||||
verify_walk_dir_body_digest, walk_dir_response_body, write_authenticated_put_file, write_body_chunks_to_writer,
|
||||
write_put_file_body_chunks_to_writer,
|
||||
internode_http_operation, internode_rpc_subsystem, is_internode_rpc_path, ns_scanner_response_body,
|
||||
ns_scanner_server_epoch_matches, put_body_size_mismatch, put_file_auth_nonce, put_file_capability_response,
|
||||
put_file_server_epoch_accepted, put_file_server_epoch_matches, put_file_stage_error_message, put_file_target_lock,
|
||||
read_file_body_stream, read_file_stream_buffer_size, remote_scanner_claim_rejection, response_with_disk_error,
|
||||
supports_walk_dir_stream_completion, validate_walk_dir_completion_request, verify_internode_rpc_signature,
|
||||
verify_ns_scanner_body_digest, verify_walk_dir_body_digest, walk_dir_response_body, write_authenticated_put_file,
|
||||
write_body_chunks_to_writer, write_put_file_body_chunks_to_writer,
|
||||
};
|
||||
use crate::storage::storage_api::ecstore_rpc::{build_put_file_auth_trailer, gen_signature_headers};
|
||||
use crate::storage::storage_api::rpc_consumer::http_service::{DiskAPI as _, DiskOption, DiskStore, Endpoint, new_disk};
|
||||
@@ -3064,15 +3013,4 @@ mod tests {
|
||||
let response = response_with_disk_error(&DiskError::DiskAccessDenied, "permission denied");
|
||||
assert!(response.headers().get(rustfs_rio::INTERNODE_DISK_ERROR_HEADER).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_meta_bucket_alias_maps_only_legacy_system_metadata() {
|
||||
assert_eq!(legacy_meta_bucket_alias(".minio.sys").as_deref(), Some(".rustfs.sys"));
|
||||
assert_eq!(
|
||||
legacy_meta_bucket_alias(".minio.sys/config/iam").as_deref(),
|
||||
Some(".rustfs.sys/config/iam")
|
||||
);
|
||||
assert_eq!(legacy_meta_bucket_alias(".minio.sys-lookalike"), None);
|
||||
assert_eq!(legacy_meta_bucket_alias("user-bucket"), None);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,10 +46,6 @@ REQUIRED_GATES = {
|
||||
}
|
||||
|
||||
|
||||
def is_sha(value: Any) -> bool:
|
||||
return isinstance(value, str) and len(value) == 40 and all(char in "0123456789abcdef" for char in value)
|
||||
|
||||
|
||||
def command(*parts: str) -> list[str]:
|
||||
return list(parts)
|
||||
|
||||
@@ -78,14 +74,6 @@ def load_registry() -> dict[str, Any]:
|
||||
return registry
|
||||
|
||||
|
||||
def read_json_object(path: Path) -> dict[str, Any]:
|
||||
with path.open() as stream:
|
||||
payload = json.load(stream)
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"expected JSON object: {path}")
|
||||
return payload
|
||||
|
||||
|
||||
def validate_registry(registry: dict[str, Any]) -> None:
|
||||
gates = {item["gate"] for item in registry.get("release_requirements", [])}
|
||||
missing = sorted(REQUIRED_GATES - gates)
|
||||
@@ -616,89 +604,6 @@ def build_status(plan: dict[str, Any], run_root: Path) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def descriptor_gate_names(payload: dict[str, Any]) -> list[str]:
|
||||
gates = payload.get("gates")
|
||||
if not isinstance(gates, dict):
|
||||
return []
|
||||
return sorted(gate for gate in gates if isinstance(gate, str))
|
||||
|
||||
|
||||
def build_descriptor_ledger(descriptor_paths: list[Path], revision: str) -> dict[str, Any]:
|
||||
entries = []
|
||||
same_head_gates: set[str] = set()
|
||||
old_head_gates: set[str] = set()
|
||||
duplicate_gates: dict[str, list[str]] = {}
|
||||
gate_sources: dict[str, list[str]] = {}
|
||||
for raw_path in descriptor_paths:
|
||||
path = raw_path.resolve()
|
||||
entry: dict[str, Any] = {
|
||||
"path": str(raw_path),
|
||||
"file_name": path.name,
|
||||
}
|
||||
try:
|
||||
if not path.is_file():
|
||||
raise ValueError("descriptor is missing")
|
||||
if path.stat().st_size <= 0:
|
||||
raise ValueError("descriptor is empty")
|
||||
payload = read_json_object(path)
|
||||
evidence = payload.get("evidence")
|
||||
descriptor_revision = payload.get("source_revision")
|
||||
gates = descriptor_gate_names(payload)
|
||||
if evidence != "measured":
|
||||
classification = "case-level only"
|
||||
elif not is_sha(descriptor_revision):
|
||||
classification = "invalid"
|
||||
elif not gates:
|
||||
classification = "case-level only"
|
||||
elif descriptor_revision == revision:
|
||||
classification = "same-head verified"
|
||||
same_head_gates.update(gates)
|
||||
else:
|
||||
classification = "old-head measured, drift-readable"
|
||||
old_head_gates.update(gates)
|
||||
for gate in gates:
|
||||
gate_sources.setdefault(gate, []).append(path.name)
|
||||
entry.update({
|
||||
"status": "present",
|
||||
"classification": classification,
|
||||
"evidence": evidence,
|
||||
"source_revision": descriptor_revision,
|
||||
"gates": gates,
|
||||
})
|
||||
except (ValueError, OSError, json.JSONDecodeError) as error:
|
||||
entry.update({
|
||||
"status": "invalid",
|
||||
"classification": "invalid",
|
||||
"error": str(error),
|
||||
"gates": [],
|
||||
})
|
||||
entries.append(entry)
|
||||
for gate, sources in sorted(gate_sources.items()):
|
||||
if len(sources) > 1:
|
||||
duplicate_gates[gate] = sorted(sources)
|
||||
measured_gates = same_head_gates | old_head_gates
|
||||
return {
|
||||
"schema": 1,
|
||||
"kind": "scanner-heal-descriptor-ledger",
|
||||
"source_revision": revision,
|
||||
"release_approved": False,
|
||||
"entries": entries,
|
||||
"same_head_verified_gates": sorted(same_head_gates),
|
||||
"old_head_measured_gates": sorted(old_head_gates - same_head_gates),
|
||||
"missing_measured_gates": sorted(REQUIRED_GATES - measured_gates),
|
||||
"missing_current_head_gates": sorted(REQUIRED_GATES - same_head_gates),
|
||||
"duplicate_gates": duplicate_gates,
|
||||
"totals": {
|
||||
"descriptors": len(entries),
|
||||
"same_head_verified_gates": len(same_head_gates),
|
||||
"old_head_measured_gates": len(old_head_gates - same_head_gates),
|
||||
"missing_measured_gates": len(REQUIRED_GATES - measured_gates),
|
||||
"missing_current_head_gates": len(REQUIRED_GATES - same_head_gates),
|
||||
"invalid_descriptors": sum(1 for entry in entries if entry["status"] == "invalid"),
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def run_preflight(plan: dict[str, Any]) -> int:
|
||||
commands = iter_preflight_commands(plan)
|
||||
if not commands:
|
||||
@@ -767,7 +672,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser.add_argument("--format", choices=("text", "json"), default="text")
|
||||
parser.add_argument("--run-preflight", action="store_true")
|
||||
parser.add_argument("--status-root", type=Path)
|
||||
parser.add_argument("--descriptor-ledger", type=Path, nargs="+")
|
||||
parser.add_argument("--self-test", action="store_true")
|
||||
return parser.parse_args(argv)
|
||||
|
||||
@@ -783,15 +687,9 @@ def main(argv: list[str] | None = None) -> int:
|
||||
phases = set(args.phase or ["all"])
|
||||
if "all" in phases and len(phases) > 1:
|
||||
raise ValueError("--phase all cannot be combined with another phase")
|
||||
if args.status_root is not None and (args.write_plan or args.run_preflight or args.descriptor_ledger):
|
||||
raise ValueError("--status-root cannot be combined with --write-plan, --run-preflight, or --descriptor-ledger")
|
||||
if args.descriptor_ledger and (args.write_plan or args.run_preflight):
|
||||
raise ValueError("--descriptor-ledger cannot be combined with --write-plan or --run-preflight")
|
||||
revision = source_revision(args.source_revision)
|
||||
if args.descriptor_ledger:
|
||||
print(json.dumps(build_descriptor_ledger(args.descriptor_ledger, revision), indent=2, sort_keys=True))
|
||||
return 0
|
||||
plan = build_plan(registry, revision, phases)
|
||||
if args.status_root is not None and (args.write_plan or args.run_preflight):
|
||||
raise ValueError("--status-root cannot be combined with --write-plan or --run-preflight")
|
||||
plan = build_plan(registry, source_revision(args.source_revision), phases)
|
||||
if args.status_root is not None:
|
||||
status = build_status(plan, args.status_root)
|
||||
print(json.dumps(status, indent=2, sort_keys=True))
|
||||
|
||||
@@ -108,60 +108,6 @@ assert status["release_approved"] is False
|
||||
assert status["artifact_totals"]["missing"] == 0
|
||||
PY
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" - "$TMP_DIR" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
root = pathlib.Path(sys.argv[1])
|
||||
same_head = root / "same-head.json"
|
||||
old_head = root / "old-head.json"
|
||||
case_level = root / "case-level.json"
|
||||
same_head.write_text(json.dumps({
|
||||
"schema": 1,
|
||||
"evidence": "measured",
|
||||
"source_revision": "a" * 40,
|
||||
"gates": {"G01": {}, "G02": {}},
|
||||
}) + "\n")
|
||||
old_head.write_text(json.dumps({
|
||||
"schema": 1,
|
||||
"evidence": "measured",
|
||||
"source_revision": "b" * 40,
|
||||
"gates": {"G03": {}, "G09": {}},
|
||||
}) + "\n")
|
||||
case_level.write_text(json.dumps({
|
||||
"schema": 1,
|
||||
"evidence": "case",
|
||||
"source_revision": "a" * 40,
|
||||
"gates": {"G14": {}},
|
||||
}) + "\n")
|
||||
PY
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--source-revision aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
|
||||
--descriptor-ledger "$TMP_DIR/same-head.json" "$TMP_DIR/old-head.json" "$TMP_DIR/case-level.json" \
|
||||
"$TMP_DIR/missing.json" >"$TMP_DIR/ledger.json"
|
||||
|
||||
"${RUSTFS_PYTHON_BIN:-python3}" - "$TMP_DIR/ledger.json" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
ledger = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
assert ledger["kind"] == "scanner-heal-descriptor-ledger"
|
||||
assert ledger["release_approved"] is False
|
||||
assert ledger["same_head_verified_gates"] == ["G01", "G02"]
|
||||
assert ledger["old_head_measured_gates"] == ["G03", "G09"]
|
||||
assert "G14" in ledger["missing_measured_gates"]
|
||||
assert "G03" in ledger["missing_current_head_gates"]
|
||||
assert ledger["totals"]["invalid_descriptors"] == 1
|
||||
classifications = {entry["file_name"]: entry["classification"] for entry in ledger["entries"]}
|
||||
assert classifications["same-head.json"] == "same-head verified"
|
||||
assert classifications["old-head.json"] == "old-head measured, drift-readable"
|
||||
assert classifications["case-level.json"] == "case-level only"
|
||||
assert classifications["missing.json"] == "invalid"
|
||||
PY
|
||||
|
||||
if "${RUSTFS_PYTHON_BIN:-python3}" "$RUNNER" \
|
||||
--phase performance \
|
||||
--run-preflight >/dev/null 2>"$TMP_DIR/no-preflight.err"; then
|
||||
|
||||
@@ -140,9 +140,9 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
|
||||
self.assertNotIn(" continue-on-error: true", self.steps[name])
|
||||
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps["Generate report"])
|
||||
self.assertNotIn("/tmp/rustfs-security", self.source)
|
||||
for name in ("Upload functional report to dashboard", "Manage backlog issues (dedup / label / auto-close)"):
|
||||
expected = "${{ env.SECURITY_ARTIFACTS_DIR }}/report.md" if name.startswith("Upload") else '--report-file "${SECURITY_ARTIFACTS_DIR}/report.md"'
|
||||
self.assertIn(expected, "\n".join(self.steps[name]))
|
||||
for name in ("Upload functional report to dashboard", "File failure issue in rustfs/backlog"):
|
||||
report = next(line for line in self.steps[name] if line.strip().startswith("REPORT_FILE:"))
|
||||
self.assertIn("${{ env.SECURITY_ARTIFACTS_DIR }}/report.md", report)
|
||||
for name in ("Upload functional report to dashboard", "Upload report and logs"):
|
||||
self.assertIn(" if: ${{ always() && steps.evidence.outcome == 'success' }}", self.steps[name])
|
||||
artifact_settings = yaml_block(self.steps["Upload report and logs"], "with", 8)
|
||||
@@ -268,12 +268,10 @@ class SecurityWorkflowTests(WorkflowSteps, unittest.TestCase):
|
||||
gh.chmod(0o755)
|
||||
body = self.directory / "issue-body.md"
|
||||
self.env.update(PATH=f"{fake_bin}{os.pathsep}{os.environ['PATH']}", CAPTURE_BODY=str(body))
|
||||
result = self.run_step("Manage backlog issues (dedup / label / auto-close)")
|
||||
result = self.run_step("File failure issue in rustfs/backlog")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
# The manager lives in the private auto-testing checkout; without it
|
||||
# the step must skip without publishing anything.
|
||||
self.assertIn("issue_manager.py not found", result.stdout + result.stderr)
|
||||
self.assertFalse(body.exists())
|
||||
self.assertNotIn("OLD RUN REPORT", body.read_text())
|
||||
self.assertIn("https://github.com/rustfs/rustfs/actions/runs/314159", body.read_text())
|
||||
|
||||
def test_all_ten_suites_hold_the_shared_lock_for_manual_and_chain_runs(self) -> None:
|
||||
for suite in ("upgrade", "s3-compat", "kms", "tier", "storage", "heal", "pool-expand", "security", "replication", "performance"):
|
||||
@@ -353,7 +351,7 @@ fi
|
||||
job = yaml_block(replication.splitlines(), "replication-test", 2)
|
||||
self.assertFalse(any(line.startswith(" continue-on-error:") for line in job))
|
||||
self.steps = named_steps(job)
|
||||
handoff = "Continue functional chain (next: Fault tolerance)"
|
||||
handoff = "Continue functional chain (next: Performance)"
|
||||
self.assertIn(" if: ${{ always() && github.event_name == 'repository_dispatch' }}", self.steps[handoff])
|
||||
self.assertFalse(any(line.strip().startswith("continue-on-error:") for line in self.steps[handoff]))
|
||||
self.assertIn(" if: always()", self.steps["Cleanup environment (after)"])
|
||||
@@ -373,11 +371,11 @@ fi
|
||||
self.assertEqual(forwarded.returncode == 0, bool(token) and failed_attempts < 3, forwarded.stderr)
|
||||
calls = dispatches.read_text().splitlines() if dispatches.exists() else []
|
||||
self.assertEqual(calls, [
|
||||
"api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-fault-tolerance -F client_payload[from_suite]=replication",
|
||||
"api --method POST repos/rustfs/rustfs/dispatches -f event_type=rustfs-chain-performance -F client_payload[from_suite]=replication",
|
||||
] * (min(failed_attempts + 1, 3) if token else 0))
|
||||
if failed_attempts == 3:
|
||||
self.assertIn("could not hand off from **replication** to **Fault tolerance**", body.read_text())
|
||||
self.assertIn("rustfs-chain-fault-tolerance", body.read_text())
|
||||
self.assertIn("could not hand off from **replication** to **Performance**", body.read_text())
|
||||
self.assertIn("rustfs-chain-performance", body.read_text())
|
||||
self.assertEqual(executed.read_text().splitlines().count("issue"), 2 if issue_exit else 1)
|
||||
self.assertFalse(Path(body_path.read_text().strip()).exists())
|
||||
|
||||
@@ -589,10 +587,11 @@ class FunctionalEvidenceTests(WorkflowSteps, unittest.TestCase):
|
||||
initialized = self.run_step("Initialize functional evidence")
|
||||
self.assertNotEqual(initialized.returncode, 0)
|
||||
self.assertFalse(Path(self.env["GITHUB_ENV"]).exists())
|
||||
manager = self.run_step("Manage backlog issues (dedup / label / auto-close)")
|
||||
self.assertEqual(manager.returncode, 0, manager.stderr)
|
||||
self.assertIn("issue_manager.py not found", manager.stdout + manager.stderr)
|
||||
self.assertFalse(Path(self.env["CAPTURE_BODY"]).exists())
|
||||
issue = self.run_step("File failure issue in rustfs/backlog")
|
||||
self.assertEqual(issue.returncode, 0, issue.stderr)
|
||||
body = Path(self.env["CAPTURE_BODY"]).read_text()
|
||||
self.assertNotIn("OLD RUN EVIDENCE", body)
|
||||
self.assertIn("no report or log file was produced", body)
|
||||
self.assertEqual((existing / "report.md").read_text(), "OLD RUN EVIDENCE")
|
||||
|
||||
def test_reports_use_only_current_complete_suite_evidence(self):
|
||||
|
||||
Reference in New Issue
Block a user