ci: repair e2e selection and verify tool downloads via API (#9240)

* ci: align full e2e selection with retired catalog test

* ci: verify pinned tools through GitHub release asset API

* ci: fetch pinned typos through the release asset API

* ci: download pinned actionlint through the release asset API

* ci: ensure GitHub CLI exists before pinned mc download

* ci: budget full quick checks after tool installation

.config/e2e-full-selection.txt

@@ -1,2 +1,2 @@
-sha256-darwin=219ac11e720b3f54dfee4d5d5e95171785ba159b2fe01db3037a8f5061c1ad32
-sha256-linux=2d34698cc8ec52d5b6c37ac612919a1d21f36cd56a1af539a7522a4eea518e26
+sha256-darwin=9901a4950099b94d3d2618524d540f1ed4bf6023044903886f98b6c4bf5f30fb
+sha256-linux=09fd5f48f5556c9fc576357222dccd7ebb0b90b790798f1834db2ec5d67b4952

.config/make/tests.mak

@@ -54,6 +54,7 @@ script-tests: ## Run shell script tests
 	./scripts/check_embedded_secrets.sh --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
+	$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_inspect_btf_prototypes.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_build_workflow.py
 	$(RUSTFS_PYTHON_BIN) ./scripts/test_release_glibc.py

.github/actions/quick-checks/action.yml

@@ -28,11 +28,12 @@ runs:
 
     - name: Install actionlint
       shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+        ASSET_DOWNLOADER: ${{ github.action_path }}/../setup/download_release_asset.py
       run: |
         actionlint_dir="$(mktemp -d "${RUNNER_TEMP}/actionlint.XXXXXX")"
-        curl --fail --location --silent --show-error \
-          --output "$actionlint_dir/actionlint.tar.gz" \
-          https://github.com/rhysd/actionlint/releases/download/v1.7.12/actionlint_1.7.12_linux_amd64.tar.gz
+        python3 "$ASSET_DOWNLOADER" --repository rhysd/actionlint --tag v1.7.12 --asset actionlint_1.7.12_linux_amd64.tar.gz --output "$actionlint_dir/actionlint.tar.gz" --sha256 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
         echo "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8  $actionlint_dir/actionlint.tar.gz" | sha256sum --check --status
         tar -xzf "$actionlint_dir/actionlint.tar.gz" -C "$actionlint_dir" actionlint
         rm "$actionlint_dir/actionlint.tar.gz"

.github/actions/setup/action.yml

@@ -66,10 +66,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    # protobuf-compiler is deliberately absent: the setup-protoc step below
-    # installs 35.1 into the tool cache and prepends it to PATH, so the apt
-    # build (older, and never version-matched) was shadowed on every run and
-    # simply never used.
+    # Use the pinned code generators rather than Ubuntu's older packages.
     - name: Install system dependencies (Ubuntu)
       if: runner.os == 'Linux'
       shell: bash
@@ -79,7 +76,9 @@ runs:
           build-essential \
           pkg-config \
           libssl-dev \
-          ripgrep
+          ripgrep \
+          python3 \
+          gh
 
     # musl-gcc is needed by the native musl release leg, and zip/unzip by the
     # release packaging steps. No CI test lane touches any of them.
@@ -88,13 +87,32 @@ runs:
       shell: bash
       run: sudo apt-get install -y musl-tools zip unzip
 
+    - name: Install pinned code generators through GitHub API (Linux x64)
+      if: runner.os == 'Linux' && runner.arch == 'X64'
+      shell: bash
+      env:
+        GH_TOKEN: ${{ github.token }}
+        ASSET_DOWNLOADER: ${{ github.action_path }}/download_release_asset.py
+      run: |
+        python3 "$ASSET_DOWNLOADER" --repository protocolbuffers/protobuf --tag v35.1 --asset protoc-35.1-linux-x86_64.zip --output "$RUNNER_TEMP/protoc.zip"
+        python3 "$ASSET_DOWNLOADER" --repository google/flatbuffers --tag v25.12.19 --asset 'Linux.flatc.binary.g++-13.zip' --output "$RUNNER_TEMP/flatc.zip"
+        python3 -m zipfile -e "$RUNNER_TEMP/protoc.zip" "$RUNNER_TEMP/protoc"
+        python3 -m zipfile -e "$RUNNER_TEMP/flatc.zip" "$RUNNER_TEMP/flatc"
+        chmod +x "$RUNNER_TEMP/protoc/bin/protoc" "$RUNNER_TEMP/flatc/flatc"
+        "$RUNNER_TEMP/protoc/bin/protoc" --version
+        "$RUNNER_TEMP/flatc/flatc" --version
+        echo "$RUNNER_TEMP/protoc/bin" >> "$GITHUB_PATH"
+        echo "$RUNNER_TEMP/flatc" >> "$GITHUB_PATH"
+
     - name: Install protoc
+      if: runner.os != 'Linux' || runner.arch != 'X64'
       uses: rustfs/setup-protoc@a3705324d8f9bf5b6c3573fb6cf8ae421db55dd6 # v3.0.1
       with:
         version: "35.1"
         repo-token: ${{ github.token }}
 
     - name: Install flatc
+      if: runner.os != 'Linux' || runner.arch != 'X64'
       uses: Nugine/setup-flatc@698800de72a96bfb22cf60431dc21a2ff9a7e07b # v1
       with:
         version: "25.12.19"

.github/actions/setup/download_release_asset.py

@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Download a named GitHub release asset through its API and verify its digest."""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import re
+import subprocess
+import sys
+import tempfile
+from urllib.parse import quote
+
+
+def download(repository, tag, name, output, expected_sha256=None):
+    if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*", repository):
+        raise ValueError("invalid release repository")
+    if not tag or not name or "/" in name or "\\" in name:
+        raise ValueError("invalid release tag or asset name")
+    if expected_sha256 is not None and not re.fullmatch(r"[0-9a-f]{64}", expected_sha256):
+        raise ValueError("invalid pinned asset digest")
+    result = subprocess.run(
+        ["gh", "api", "--hostname", "github.com", f"repos/{repository}/releases/tags/{quote(tag, safe='')}"],
+        check=True, capture_output=True, text=True, timeout=60,
+    )
+    release = json.loads(result.stdout)
+    if not isinstance(release, dict) or release.get("tag_name") != tag:
+        raise ValueError("release metadata does not match the requested tag")
+    assets = release.get("assets")
+    if not isinstance(assets, list) or any(not isinstance(asset, dict) for asset in assets):
+        raise ValueError("invalid release asset inventory")
+    matches = [asset for asset in assets if asset.get("name") == name]
+    if len(matches) != 1:
+        raise ValueError("release must contain exactly one named asset")
+    asset = matches[0]
+    asset_id, size, digest = asset.get("id"), asset.get("size"), asset.get("digest")
+    if type(asset_id) is not int or asset_id <= 0 or type(size) is not int or not 0 < size <= 512 * 1024 * 1024:
+        raise ValueError("invalid release asset identity or size")
+    if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
+        raise ValueError("release asset has no valid SHA-256 digest")
+    sha256 = digest.removeprefix("sha256:")
+    if expected_sha256 is not None and expected_sha256 != sha256:
+        raise ValueError("release digest differs from the pinned digest")
+    output = Path(output)
+    output.parent.mkdir(parents=True, exist_ok=True)
+    temporary = None
+    try:
+        with tempfile.NamedTemporaryFile(dir=output.parent, prefix=".release-asset-", delete=False) as stream:
+            temporary = Path(stream.name)
+            # gh authenticates to the API; its HTTP client strips credentials on
+            # cross-host redirects. No browser URL or token is supplied in argv.
+            subprocess.run(
+                ["gh", "api", "--hostname", "github.com", "--header", "Accept: application/octet-stream", f"repos/{repository}/releases/assets/{asset_id}"],
+                check=True, stdout=stream, timeout=300,
+            )
+        with temporary.open("rb") as stream:
+            actual = hashlib.file_digest(stream, "sha256").hexdigest()
+        if temporary.stat().st_size != size or actual != sha256:
+            raise ValueError("downloaded release asset failed size or SHA-256 verification")
+        temporary.replace(output)
+        print(f"Verified release asset: {repository} {tag} {name} sha256={actual}")
+    finally:
+        if temporary is not None:
+            temporary.unlink(missing_ok=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--repository", required=True)
+    parser.add_argument("--tag", required=True)
+    parser.add_argument("--asset", required=True)
+    parser.add_argument("--output", required=True, type=Path)
+    parser.add_argument("--sha256")
+    args = parser.parse_args()
+    try:
+        download(args.repository, args.tag, args.asset, args.output, args.sha256)
+    except (ValueError, KeyError, TypeError, json.JSONDecodeError, subprocess.SubprocessError, OSError) as error:
+        print(f"Release asset download failed: {error}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

.github/workflows/ci.yml

@@ -142,15 +142,23 @@ jobs:
       - uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
         with:
           persist-credentials: false
+      - name: Install pinned typos through GitHub API
+        env:
+          GH_TOKEN: ${{ github.token }}
+          TYPOS_VERSION: 1.50.2
+        run: |
+          python3 .github/actions/setup/download_release_asset.py --repository crate-ci/typos --tag "v${TYPOS_VERSION}" --asset "typos-v${TYPOS_VERSION}-x86_64-unknown-linux-musl.tar.gz" --output "$RUNNER_TEMP/typos.tar.gz"
+          mkdir -p "$RUNNER_TEMP/typos"
+          tar -xzf "$RUNNER_TEMP/typos.tar.gz" -C "$RUNNER_TEMP/typos"
       - name: Typos check with custom config file
-        uses: crate-ci/typos@512fc24f32f44ab01972217aaaf3dc86ec234d53 # v1.50.2
+        run: '"$RUNNER_TEMP/typos/typos" .'
 
   # Fail early with compile-free checks for every pull request.
   quick-checks:
     name: Quick Checks
     if: github.event_name != 'pull_request' || github.event.action != 'closed'
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 20
     steps:
       - name: Checkout repository
         uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -1137,11 +1145,16 @@ jobs:
 
       - name: Install mc
         env:
+          GH_TOKEN: ${{ github.token }}
           MC_VERSION: RELEASE.2025-08-13T08-35-41Z
           MC_SHA256: 01f866e9c5f9b87c2b09116fa5d7c06695b106242d829a8bb32990c00312e891
         run: |
+          if ! command -v gh >/dev/null 2>&1; then
+            sudo apt-get update
+            sudo apt-get install -y gh
+          fi
           MC_BINARY="mc.linux-amd64.${MC_VERSION}"
-          curl -fsSLo "$RUNNER_TEMP/mc" "https://github.com/minio/mc/releases/download/${MC_VERSION}/${MC_BINARY}"
+          python3 .github/actions/setup/download_release_asset.py --repository minio/mc --tag "$MC_VERSION" --asset "$MC_BINARY" --output "$RUNNER_TEMP/mc" --sha256 "$MC_SHA256"
           echo "${MC_SHA256}  $RUNNER_TEMP/mc" | sha256sum --check --status
           chmod +x "$RUNNER_TEMP/mc"
           echo "$RUNNER_TEMP" >> "$GITHUB_PATH"

scripts/test_download_release_asset.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Exercise the release download entry point without fetching external tools."""
+
+import hashlib
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+
+SCRIPT = Path(__file__).resolve().parents[1] / ".github/actions/setup/download_release_asset.py"
+
+
+class ReleaseAssetTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.body = b"verified tool fixture"
+        self.sha256 = hashlib.sha256(self.body).hexdigest()
+        self.asset = {"name": "tool.zip", "id": 17, "size": len(self.body), "digest": "sha256:" + self.sha256}
+        self.metadata = {"tag_name": "v1.0", "assets": [self.asset]}
+        self.output = self.root / "tool.zip"
+        self.output.write_bytes(b"previous file")
+        self.calls = self.root / "calls.jsonl"
+        gh = self.root / "gh"
+        gh.write_text(f"#!{sys.executable}\n" + '''import json,os,pathlib,sys
+args=sys.argv[1:]
+with pathlib.Path(os.environ['CALLS']).open('a') as out:out.write(json.dumps(args)+'\\n')
+if '/releases/tags/' in args[-1]:
+    if os.environ.get('FAIL_METADATA'):raise SystemExit(19)
+    print(os.environ['METADATA'])
+else:
+    if os.environ.get('FAIL_DOWNLOAD'):raise SystemExit(23)
+    sys.stdout.buffer.write(bytes.fromhex(os.environ['BODY']))
+''')
+        gh.chmod(0o755)
+        self.env = dict(os.environ, PATH=str(self.root) + os.pathsep + os.environ["PATH"], CALLS=str(self.calls), GH_TOKEN="synthetic-do-not-log-token")
+
+    def invoke(self, extra=(), **env):
+        environment = dict(self.env, METADATA=json.dumps(self.metadata), BODY=self.body.hex(), **env)
+        return subprocess.run([sys.executable, str(SCRIPT), "--repository", "owner/repo", "--tag", "v1.0", "--asset", "tool.zip", "--output", str(self.output), *extra], env=environment, capture_output=True, text=True)
+
+    def assert_failed_atomically(self, result):
+        self.assertNotEqual(result.returncode, 0)
+        self.assertEqual(self.output.read_bytes(), b"previous file")
+        self.assertEqual(list(self.root.glob(".release-asset-*")), [])
+        self.assertNotIn(self.env["GH_TOKEN"], result.stdout + result.stderr)
+
+    def test_valid_asset_uses_api_and_installs_only_verified_bytes(self):
+        result = self.invoke(("--sha256", self.sha256))
+        self.assertEqual(result.returncode, 0, result.stderr)
+        self.assertEqual(self.output.read_bytes(), self.body)
+        calls = [json.loads(line) for line in self.calls.read_text().splitlines()]
+        self.assertEqual(calls[0], ["api", "--hostname", "github.com", "repos/owner/repo/releases/tags/v1.0"])
+        self.assertEqual(calls[1], ["api", "--hostname", "github.com", "--header", "Accept: application/octet-stream", "repos/owner/repo/releases/assets/17"])
+        self.assertNotIn(self.env["GH_TOKEN"], json.dumps(calls) + result.stdout + result.stderr)
+
+    def test_download_failure_preserves_destination(self):
+        self.assert_failed_atomically(self.invoke(FAIL_DOWNLOAD="1"))
+
+    def test_missing_gh_fails_without_replacing_destination(self):
+        (self.root / "gh").unlink()
+        self.env["PATH"] = str(self.root)
+        self.assert_failed_atomically(self.invoke())
+
+    def test_metadata_failure_preserves_destination(self):
+        self.assert_failed_atomically(self.invoke(FAIL_METADATA="1"))
+
+    def test_corrupt_download_fails_digest_validation(self):
+        self.body = b"modified tool fixture"
+        self.assert_failed_atomically(self.invoke())
+
+    def test_truncated_download_fails_size_validation(self):
+        self.asset["size"] += 1
+        self.assert_failed_atomically(self.invoke())
+
+    def test_pinned_digest_cannot_be_replaced_by_release_metadata(self):
+        self.assert_failed_atomically(self.invoke(("--sha256", "0" * 64)))
+        self.assertEqual(len(self.calls.read_text().splitlines()), 1)
+
+    def test_ambiguous_or_missing_assets_fail_before_download(self):
+        for assets in ([], [self.asset, self.asset]):
+            with self.subTest(assets=len(assets)):
+                self.metadata["assets"] = assets
+                self.assert_failed_atomically(self.invoke())
+
+    def test_metadata_tag_must_match_requested_version(self):
+        self.metadata["tag_name"] = "v2.0"
+        self.assert_failed_atomically(self.invoke())
+
+    def test_malformed_release_inventory_fails_before_download(self):
+        for metadata in ([], {"tag_name": "v1.0", "assets": None}, {"tag_name": "v1.0", "assets": [None]}):
+            with self.subTest(metadata=metadata):
+                self.metadata = metadata
+                self.assert_failed_atomically(self.invoke())
+
+    def test_missing_digest_or_invalid_identity_fails_before_download(self):
+        for field, value in [("digest", None), ("digest", "sha256:invalid"), ("id", True), ("id", 0), ("size", 0), ("size", 513 * 1024 * 1024)]:
+            with self.subTest(field=field, value=value):
+                self.asset[field] = value
+                self.assert_failed_atomically(self.invoke())
+                self.asset = {"name": "tool.zip", "id": 17, "size": len(self.body), "digest": "sha256:" + self.sha256}
+                self.metadata["assets"] = [self.asset]
+
+
+if __name__ == "__main__":
+    unittest.main()