diff --git a/.github/workflows/rustfs-heal-test.yml b/.github/workflows/rustfs-heal-test.yml index 666c83b4c..efbe2a09c 100644 --- a/.github/workflows/rustfs-heal-test.yml +++ b/.github/workflows/rustfs-heal-test.yml @@ -235,17 +235,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }} diff --git a/.github/workflows/rustfs-kms-test.yml b/.github/workflows/rustfs-kms-test.yml index 8647c9268..5c3a2b6b1 100644 --- a/.github/workflows/rustfs-kms-test.yml +++ b/.github/workflows/rustfs-kms-test.yml @@ -236,17 +236,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }} diff --git a/.github/workflows/rustfs-pool-expand-test.yml b/.github/workflows/rustfs-pool-expand-test.yml index 3c9836b2d..c700df7c8 100644 --- a/.github/workflows/rustfs-pool-expand-test.yml +++ b/.github/workflows/rustfs-pool-expand-test.yml @@ -539,17 +539,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.pool_test.outcome == 'failure' || steps.pool_test.outcome == 'cancelled') }} diff --git a/.github/workflows/rustfs-replication-test.yml b/.github/workflows/rustfs-replication-test.yml index 74c57b6d5..839d1de7f 100644 --- a/.github/workflows/rustfs-replication-test.yml +++ b/.github/workflows/rustfs-replication-test.yml @@ -245,17 +245,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }} diff --git a/.github/workflows/rustfs-s3-compat-test.yml b/.github/workflows/rustfs-s3-compat-test.yml index 80856194c..875db70bb 100644 --- a/.github/workflows/rustfs-s3-compat-test.yml +++ b/.github/workflows/rustfs-s3-compat-test.yml @@ -216,17 +216,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }} diff --git a/.github/workflows/rustfs-security-test.yml b/.github/workflows/rustfs-security-test.yml index 16e10a80c..ee37e7d5d 100644 --- a/.github/workflows/rustfs-security-test.yml +++ b/.github/workflows/rustfs-security-test.yml @@ -77,10 +77,14 @@ jobs: timeout-minutes: 360 if: ${{ github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch' }} steps: + # Checkout the repository into its own subdirectory. Checking out at + # the workspace root would wipe the auto-testing clone above (that is + # exactly how run 33934141181 lost rustfs-security-test.sh). - name: Checkout repository (for the OIDC live gate script) uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: persist-credentials: false + path: rustfs-repo - name: Initialize security evidence id: evidence @@ -145,7 +149,7 @@ jobs: env: REPORT_FILE: ${{ env.SECURITY_ARTIFACTS_DIR }}/suite-report.md TMPDIR: ${{ env.SECURITY_ARTIFACTS_DIR }} - RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/scripts/test/oidc_keycloak_live.sh + RUSTFS_SECURITY_OIDC_LIVE_SCRIPT: ${{ github.workspace }}/rustfs-repo/scripts/test/oidc_keycloak_live.sh run: | set -euo pipefail chmod +x auto-testing/rustfs-security-test.sh @@ -219,17 +223,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }} diff --git a/.github/workflows/rustfs-storage-test.yml b/.github/workflows/rustfs-storage-test.yml index 767f734dc..1e99dce4e 100644 --- a/.github/workflows/rustfs-storage-test.yml +++ b/.github/workflows/rustfs-storage-test.yml @@ -231,17 +231,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }} diff --git a/.github/workflows/rustfs-tier-test.yml b/.github/workflows/rustfs-tier-test.yml index 5d9a2c1d7..abe09f101 100644 --- a/.github/workflows/rustfs-tier-test.yml +++ b/.github/workflows/rustfs-tier-test.yml @@ -377,17 +377,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: Verify required tier evidence id: evidence_verify diff --git a/.github/workflows/rustfs-upgrade-test.yml b/.github/workflows/rustfs-upgrade-test.yml index 0b4c19af1..612765874 100644 --- a/.github/workflows/rustfs-upgrade-test.yml +++ b/.github/workflows/rustfs-upgrade-test.yml @@ -330,17 +330,22 @@ jobs: fi DATE="$(date -u +%Y-%m-%d)" REPORT_PATH="functional-reports/${SUITE}/${DATE}.md" - CONTENT="$(python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}")" + # Base64-encode the report into a temp file and feed it to jq via + # --rawfile: large reports (e.g. pool) exceed the OS argv limit and + # make `jq --arg content "${CONTENT}"` fail with "Argument list too long". + B64_FILE="$(mktemp)" + python3 -c 'import base64,sys;print(base64.b64encode(open(sys.argv[1],"rb").read()).decode())' "${REPORT_FILE}" > "${B64_FILE}" SHA="$(gh api "repos/rustfs/dashboard/contents/${REPORT_PATH}" -q '.sha' 2>/dev/null || true)" if [ -n "${SHA}" ]; then - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" --arg sha "${SHA}" \ - '{message:$msg, content:$content, sha:$sha}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" --arg sha "${SHA}" \ + '{message:$msg, content:($content|rtrimstr("\n")), sha:$sha}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null else - jq -n --arg msg "report(${SUITE}): ${DATE}" --arg content "${CONTENT}" \ - '{message:$msg, content:$content}' \ + jq -n --arg msg "report(${SUITE}): ${DATE}" --rawfile content "${B64_FILE}" \ + '{message:$msg, content:($content|rtrimstr("\n"))}' \ | gh api --method PUT "repos/rustfs/dashboard/contents/${REPORT_PATH}" --input - >/dev/null fi + rm -f "${B64_FILE}" - name: File failure issue in rustfs/backlog if: ${{ always() && (failure() || steps.test.outcome == 'failure' || steps.test.outcome == 'cancelled') }}