diff --git a/.config/make/tests.mak b/.config/make/tests.mak index d1297e9ad..726ca04cf 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -41,6 +41,7 @@ script-tests: ## Run shell script tests $(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/check_scheduled_validation_freshness.py --self-test $(RUSTFS_PYTHON_BIN) ./scripts/test_security_workflow.py + $(RUSTFS_PYTHON_BIN) ./scripts/test_nightly_candidate.py $(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py bash -n ./scripts/validate_object_data_cache_cold_stampede.sh $(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test diff --git a/.github/actions/quick-checks/action.yml b/.github/actions/quick-checks/action.yml index e6cc59775..18c6cb64d 100644 --- a/.github/actions/quick-checks/action.yml +++ b/.github/actions/quick-checks/action.yml @@ -100,6 +100,7 @@ runs: python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_scheduled_validation_freshness.py --self-test python3 ./scripts/test_security_workflow.py + python3 ./scripts/test_nightly_candidate.py python3 ./scripts/check_test_wiring.py - name: Check no planning docs committed diff --git a/.github/workflows/nightly-gnu.yml b/.github/workflows/nightly-gnu.yml index 0dedbf465..1864a9715 100644 --- a/.github/workflows/nightly-gnu.yml +++ b/.github/workflows/nightly-gnu.yml @@ -166,8 +166,9 @@ jobs: # e.g. https://dl.rustfs.com/artifacts/rustfs/packages/nightly/... . # Skipped when the R2 secrets are not configured (artifact-only mode). - name: Upload DEB to Cloudflare R2 - if: env.R2_ACCESS_KEY_ID != '' + id: publish env: + DEB_FILE: ${{ steps.deb.outputs.deb_file }} R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ENDPOINT: ${{ secrets.R2_ENDPOINT }} @@ -182,28 +183,70 @@ jobs: exit 0 fi - if ! command -v aws >/dev/null 2>&1; then - sudo apt-get update && sudo apt-get install -y -qq awscli - fi - export AWS_ACCESS_KEY_ID="$R2_ACCESS_KEY_ID" export AWS_SECRET_ACCESS_KEY="$R2_SECRET_ACCESS_KEY" export AWS_DEFAULT_REGION="auto" - DEB_FILE="${{ steps.deb.outputs.deb_file }}" + SOURCE_SHA="$(git rev-parse HEAD)" + if [[ "${SOURCE_SHA}" != "${GITHUB_SHA}" ]]; then + echo "Checkout SHA does not match the nightly build run" >&2 + exit 1 + fi + DEB_SHA256="$(sha256sum "${DEB_FILE}" | cut -d ' ' -f 1)" + CANDIDATE_KEY="artifacts/rustfs/packages/nightly/runs/${GITHUB_RUN_ID}/${GITHUB_RUN_ATTEMPT}/${DEB_SHA256}/rustfs.deb" + CANDIDATE_URL="https://dl.rustfs.com/${CANDIDATE_KEY}" + + # Old AWS CLI models lack conditional PutObject support. Never fall + # back to an overwriting upload for a candidate. + AWS_CLI=aws + if ! "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null; then + sudo apt-get update + sudo apt-get install -y -qq python3-venv + AWS_CLI_DIR="$(mktemp -d "${RUNNER_TEMP}/nightly-awscli.XXXXXX")" + trap 'rm -rf "${AWS_CLI_DIR}"' EXIT + python3 -m venv "${AWS_CLI_DIR}" + "${AWS_CLI_DIR}/bin/python" -m pip install --disable-pip-version-check 'awscli==1.44.79' + AWS_CLI="${AWS_CLI_DIR}/bin/aws" + fi + "${AWS_CLI}" s3api put-object --generate-cli-skeleton input | jq -e 'has("IfNoneMatch")' >/dev/null + "${AWS_CLI}" --version + "${AWS_CLI}" s3api put-object --bucket "${R2_BUCKET}" --key "${CANDIDATE_KEY}" \ + --body "${DEB_FILE}" --if-none-match '*' --endpoint-url "${R2_ENDPOINT}" + PUBLISHED_SHA256="$(curl -fsSL --retry 3 --connect-timeout 15 --max-time 300 "${CANDIDATE_URL}" | sha256sum | cut -d ' ' -f 1)" + if [[ "${PUBLISHED_SHA256}" != "${DEB_SHA256}" ]]; then + echo "Published candidate checksum does not match the built package" >&2 + exit 1 + fi + R2_PREFIX="s3://${R2_BUCKET}/artifacts/rustfs/packages/nightly/" echo "📤 Uploading ${DEB_FILE} to ${R2_PREFIX}" - aws s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors + "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}" --endpoint-url "$R2_ENDPOINT" --only-show-errors # Stable "latest" alias so tests can fetch the newest nightly # without knowing today's date. echo "📤 Uploading latest alias" - aws s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \ + "${AWS_CLI}" s3 cp "${DEB_FILE}" "${R2_PREFIX}rustfs-nightly-latest.deb" \ --endpoint-url "$R2_ENDPOINT" --only-show-errors echo "✅ R2 upload complete" + CANDIDATE_FILE="${RUNNER_TEMP}/nightly-candidate-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}.json" + jq -n --arg source_sha "${SOURCE_SHA}" \ + --argjson build_run_id "${GITHUB_RUN_ID}" --argjson build_run_attempt "${GITHUB_RUN_ATTEMPT}" \ + --arg package_url "${CANDIDATE_URL}" --arg package_sha256 "${DEB_SHA256}" \ + '{schema: 1, source_sha: $source_sha, build_run_id: $build_run_id, build_run_attempt: $build_run_attempt, package_url: $package_url, package_sha256: $package_sha256}' \ + > "${CANDIDATE_FILE}" + echo "candidate_file=${CANDIDATE_FILE}" >> "${GITHUB_OUTPUT}" + + - name: Upload nightly candidate manifest + if: ${{ steps.publish.outputs.candidate_file != '' }} + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.publish.outputs.candidate_file }} + if-no-files-found: error + # Live-Vault lane for the rustfs-kms suite (rustfs/backlog#1774). # # RUSTFS_KMS_VAULT_TOKEN is the single switch that adds the Vault KV2 and diff --git a/scripts/test_nightly_candidate.py b/scripts/test_nightly_candidate.py new file mode 100644 index 000000000..3e5a50401 --- /dev/null +++ b/scripts/test_nightly_candidate.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Exercise the nightly publication step without AWS, network or package builds.""" + +import hashlib +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +from check_test_wiring import yaml_block + + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/nightly-gnu.yml" + + +class NightlyCandidateTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.package = self.root / "rustfs-nightly-2026-09-06.deb" + self.package.write_bytes(b"built package bytes\x00\xff") + for command in (["git", "init", "-q"], ["git", "add", self.package.name], + ["git", "-c", "user.name=Test", "-c", "user.email=test@example.invalid", "commit", "-qm", "fixture"]): + subprocess.run(command, cwd=self.root, check=True, capture_output=True) + self.sha = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=self.root, text=True).strip() + self.digest = hashlib.sha256(self.package.read_bytes()).hexdigest() + self.output = self.root / "github-output" + self.store = self.root / "store" + self.shims = self.root / "fake-tools.sh" + self.shims.write_text(r'''aws() { + printf '%s\n' "$*" >> "$FAKE_AWS_LOG" + if [[ "$1" == --version ]]; then printf 'aws-cli/1.44.79 fixture\n'; return; fi + if [[ "$*" == *--generate-cli-skeleton* ]]; then + if [[ "$FAKE_MODE" == broken-install || "$FAKE_MODE" =~ ^(old-cli|bootstrap-failure|install-failure)$ && ! -e "$FAKE_INSTALLED" ]]; then + printf '{}\n' + else + printf '{"IfNoneMatch":""}\n' + fi + return + fi + if [[ "$1 $2" == 's3api put-object' ]]; then + [[ "$FAKE_MODE" != upload-failure ]] || return 42 + shift 2 + local key="" body="" condition="" + while [[ $# -gt 0 ]]; do + case "$1" in + --key) key="$2";; + --body) body="$2";; + --if-none-match) condition="$2";; + esac + shift 2 + done + [[ -z "$condition" || "$condition" == '*' ]] || return 43 + if [[ "$condition" == '*' && -e "$FAKE_STORE/$key" ]]; then return 44; fi + mkdir -p "$(dirname "$FAKE_STORE/$key")" + cp "$body" "$FAKE_STORE/$key" + elif [[ "$1 $2" == 's3 cp' ]]; then + [[ "$FAKE_MODE" != alias-failure ]] || return 45 + local destination="${4#s3://test-bucket/}" + [[ "$destination" != */ ]] || destination+="$(basename "$3")" + mkdir -p "$(dirname "$FAKE_STORE/$destination")" + cp "$3" "$FAKE_STORE/$destination" + else + return 46 + fi +} +curl() { + local url="${!#}" + printf '%s\n' "$url" >> "$FAKE_CURL_LOG" + [[ "$url" == https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/* ]] || return 22 + [[ "$FAKE_MODE" != missing-public-url ]] || return 22 + if [[ "$FAKE_MODE" == wrong-public-bytes ]]; then printf 'different package'; return; fi + cat "$FAKE_STORE/${url#https://dl.rustfs.com/}" || return 22 + [[ "$FAKE_MODE" != incomplete-download ]] || return 47 +} +sudo() { + [[ "$*" == 'apt-get update' || "$*" == 'apt-get install -y -qq python3-venv' ]] || return 49 + [[ "$FAKE_MODE" != bootstrap-failure ]] || return 48 +} +python3() { + [[ "$1 $2" == '-m venv' ]] || return 50 + mkdir -p "$3/bin" + cat > "$3/bin/python" <<'SH' +#!/usr/bin/env bash +[[ "$*" == '-m pip install --disable-pip-version-check awscli==1.44.79' ]] || exit 51 +[[ "$FAKE_MODE" != install-failure ]] || exit 52 +: > "$FAKE_INSTALLED" +SH + printf '#!/usr/bin/env bash\naws "$@"\n' > "$3/bin/aws" + chmod +x "$3/bin/python" "$3/bin/aws" +} +''') + self.env = dict(os.environ, BASH_ENV=str(self.shims), DEB_FILE=self.package.name, + R2_ACCESS_KEY_ID="fake-access", R2_SECRET_ACCESS_KEY="fake-secret", R2_ENDPOINT="https://r2.example.invalid", R2_BUCKET="test-bucket", + RUNNER_TEMP=str(self.root), GITHUB_SHA=self.sha, GITHUB_RUN_ID="12345", GITHUB_RUN_ATTEMPT="1", GITHUB_OUTPUT=str(self.output), + FAKE_STORE=str(self.store), FAKE_AWS_LOG=str(self.root / "aws.log"), FAKE_CURL_LOG=str(self.root / "curl.log"), FAKE_INSTALLED=str(self.root / "installed"), FAKE_MODE="success") + source = WORKFLOW.read_text() + job = yaml_block(source.splitlines(), "build", 2) + starts = [i for i, line in enumerate(job) if line.startswith(" - name: ")] + self.steps = { + job[start].split(": ", 1)[1]: job[start:end] + for start, end in zip(starts, starts[1:] + [len(job)]) + } + self.publish = self.steps["Upload DEB to Cloudflare R2"] + start = self.publish.index(" run: |") + 1 + self.shell = "\n".join(line[10:] for line in self.publish[start:] if not line.strip() or line.startswith(" ")) + + def run_publish(self, **overrides): + self.output.unlink(missing_ok=True) + return subprocess.run(["bash", "--noprofile", "--norc", "-e", "-o", "pipefail", "-c", self.shell], + cwd=self.root, env=dict(self.env, **overrides), capture_output=True, text=True) + + def manifest(self): + output = self.output.read_text().strip() + self.assertTrue(output.startswith("candidate_file="), output) + return json.loads(Path(output.split("=", 1)[1]).read_text()) + + def test_success_binds_actual_package_checkout_and_attempt(self): + result = self.run_publish() + self.assertEqual(result.returncode, 0, result.stderr) + manifest = self.manifest() + self.assertEqual(manifest, {"schema": 1, "source_sha": self.sha, "build_run_id": 12345, "build_run_attempt": 1, + "package_sha256": self.digest, "package_url": f"https://dl.rustfs.com/artifacts/rustfs/packages/nightly/runs/12345/1/{self.digest}/rustfs.deb"}) + for path in (f"runs/12345/1/{self.digest}/rustfs.deb", self.package.name, "rustfs-nightly-latest.deb"): + self.assertEqual((self.store / "artifacts/rustfs/packages/nightly" / path).read_bytes(), self.package.read_bytes()) + self.assertEqual((self.root / "curl.log").read_text().strip(), manifest["package_url"]) + + def test_missing_credentials_remain_artifact_only(self): + for key in ("R2_ACCESS_KEY_ID", "R2_SECRET_ACCESS_KEY", "R2_ENDPOINT", "R2_BUCKET"): + with self.subTest(missing=key): + result = self.run_publish(**{key: ""}) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(self.output.exists()) + self.assertFalse((self.root / "aws.log").exists()) + self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), []) + + def test_publication_failures_never_emit_a_candidate(self): + for index, mode in enumerate(("upload-failure", "missing-public-url", "wrong-public-bytes", "incomplete-download", "alias-failure")): + with self.subTest(mode=mode): + result = self.run_publish(FAKE_MODE=mode, GITHUB_RUN_ID=str(20000 + index)) + self.assertNotEqual(result.returncode, 0, result.stdout) + self.assertFalse(self.output.exists()) + self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), []) + + def test_old_cli_is_upgraded_in_an_isolated_temporary_environment(self): + result = self.run_publish(FAKE_MODE="old-cli") + self.assertEqual(result.returncode, 0, result.stderr) + self.assertTrue((self.root / "installed").exists()) + self.assertEqual(self.manifest()["package_sha256"], self.digest) + self.assertEqual(list(self.root.glob("nightly-awscli.*")), []) + + def test_failed_cli_bootstrap_cannot_publish(self): + for mode in ("bootstrap-failure", "install-failure", "broken-install"): + with self.subTest(mode=mode): + (self.root / "installed").unlink(missing_ok=True) + result = self.run_publish(FAKE_MODE=mode) + self.assertNotEqual(result.returncode, 0) + self.assertFalse(self.output.exists()) + self.assertFalse(self.store.exists()) + self.assertEqual(list(self.root.glob("nightly-awscli.*")), []) + + def test_checkout_sha_mismatch_fails_before_upload(self): + result = self.run_publish(GITHUB_SHA="f" * 40) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Checkout SHA", result.stderr) + self.assertFalse(self.output.exists()) + self.assertFalse((self.root / "aws.log").exists()) + + def test_same_date_builds_and_reruns_keep_distinct_candidates(self): + urls = [] + for run, attempt in (("12345", "1"), ("54321", "1"), ("12345", "2")): + result = self.run_publish(GITHUB_RUN_ID=run, GITHUB_RUN_ATTEMPT=attempt) + self.assertEqual(result.returncode, 0, result.stderr) + urls.append(self.manifest()["package_url"]) + self.assertEqual(len(set(urls)), 3) + self.assertEqual(len(list(self.root.glob("nightly-candidate-*.json"))), 3) + + def test_duplicate_key_is_not_overwritten_or_recertified(self): + result = self.run_publish() + self.assertEqual(result.returncode, 0, result.stderr) + key = self.manifest()["package_url"].removeprefix("https://dl.rustfs.com/") + stored = self.store / key + stored.write_bytes(b"preexisting conflicting object") + (self.root / "nightly-candidate-12345-1.json").unlink() + result = self.run_publish() + self.assertNotEqual(result.returncode, 0) + self.assertEqual(stored.read_bytes(), b"preexisting conflicting object") + self.assertFalse(self.output.exists()) + self.assertEqual(list(self.root.glob("nightly-candidate-*.json")), []) + + def test_manifest_upload_requires_publication_output(self): + upload = self.steps["Upload nightly candidate manifest"] + self.assertIn(" id: publish", self.publish) + self.assertIn(" DEB_FILE: ${{ steps.deb.outputs.deb_file }}", self.publish) + self.assertIn(" if: ${{ steps.publish.outputs.candidate_file != '' }}", upload) + self.assertIn(" name: nightly-candidate-${{ github.run_id }}-${{ github.run_attempt }}", upload) + self.assertIn(" path: ${{ steps.publish.outputs.candidate_file }}", upload) + self.assertIn(" if-no-files-found: error", upload) + self.assertNotIn(" continue-on-error: true", self.publish) + self.assertNotIn(" overwrite: true", upload) + + +if __name__ == "__main__": + unittest.main()