diff --git a/.github/workflows/connect-top-disk-acceptance.yml b/.github/workflows/connect-top-disk-acceptance.yml index 7843ad0bd..e6b32ce43 100644 --- a/.github/workflows/connect-top-disk-acceptance.yml +++ b/.github/workflows/connect-top-disk-acceptance.yml @@ -12,21 +12,21 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Connect top.disk artifact acceptance +name: Connect top.disk service-job acceptance on: workflow_dispatch: inputs: build_run_id: - description: Successful main-branch Build and Release workflow run ID + description: Successful main-branch Build and Release run ID required: true type: string artifact_id: - description: Linux x86_64 GNU artifact ID from that run + description: Linux x86_64 GNU artifact from that run required: true type: string source_sha: - description: Exact 40-character source commit + description: Exact RustFS source commit required: true type: string artifact_digest: @@ -34,7 +34,11 @@ on: required: true type: string binary_sha256: - description: Expected rustfs binary SHA-256 + description: Independently verified rustfs binary SHA-256 + required: true + type: string + connect_sha: + description: Exact Connect top.disk service-job acceptance harness commit required: true type: string @@ -44,16 +48,67 @@ permissions: jobs: top-disk: - name: Verify native Linux x86_64 top.disk artifact - runs-on: sm-standard-2 - timeout-minutes: 15 + name: Verify native top.disk service job over mTLS + runs-on: dind-sm-standard-2 + timeout-minutes: 45 steps: - - name: Checkout acceptance harness + - name: Checkout exact RustFS source uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 with: + path: rustfs-source persist-credentials: false + ref: ${{ inputs.source_sha }} - - name: Verify source run and artifact identity + - name: Checkout exact Connect harness + uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7 + with: + repository: rustfs/connect + path: connect-harness + persist-credentials: false + ref: ${{ inputs.connect_sha }} + token: ${{ secrets.PF_TESTING_GH_TOKEN }} + + - name: Set up Node.js + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 25.8.1 + cache: npm + cache-dependency-path: connect-harness/web/package-lock.json + + - name: Ensure GitHub CLI + shell: bash + run: | + if command -v gh >/dev/null 2>&1; then + gh --version + exit 0 + fi + + gh_version="2.101.0" + case "$(uname -m)" in + x86_64) gh_arch="amd64" ;; + aarch64|arm64) gh_arch="arm64" ;; + *) + echo "Unsupported Linux architecture for GitHub CLI: $(uname -m)" >&2 + exit 1 + ;; + esac + + gh_install_root="${RUNNER_TEMP:-$PWD}/gh-cli" + rm -rf "$gh_install_root" + mkdir -p "$gh_install_root/bin" + archive="$gh_install_root/gh.tar.gz" + curl --fail --location --retry 3 \ + "https://github.com/cli/cli/releases/download/v${gh_version}/gh_${gh_version}_linux_${gh_arch}.tar.gz" \ + --output "$archive" + tar -xzf "$archive" -C "$gh_install_root" + install -m 0755 \ + "$gh_install_root/gh_${gh_version}_linux_${gh_arch}/bin/gh" \ + "$gh_install_root/bin/gh" + echo "$gh_install_root/bin" >> "$GITHUB_PATH" + export PATH="$gh_install_root/bin:$PATH" + gh --version + + - name: Verify official source and artifact identity shell: bash env: GH_TOKEN: ${{ github.token }} @@ -61,82 +116,136 @@ jobs: ARTIFACT_ID: ${{ inputs.artifact_id }} SOURCE_SHA: ${{ inputs.source_sha }} ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + BINARY_SHA256: ${{ inputs.binary_sha256 }} + CONNECT_SHA: ${{ inputs.connect_sha }} run: | set -euo pipefail - [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]] - [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]] - [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$GITHUB_REPOSITORY" == rustfs/rustfs ]] + [[ "$BUILD_RUN_ID" =~ ^[1-9][0-9]*$ && "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]] + [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ && "$CONNECT_SHA" =~ ^[0-9a-f]{40}$ ]] [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + [[ "$BINARY_SHA256" =~ ^[0-9a-f]{64}$ ]] + [[ $(git -C rustfs-source rev-parse HEAD) == "$SOURCE_SHA" ]] + [[ $(git -C connect-harness rev-parse HEAD) == "$CONNECT_SHA" ]] + [[ $(git -C rustfs-source remote get-url origin) == https://github.com/rustfs/rustfs ]] + [[ $(git -C connect-harness remote get-url origin) == https://github.com/rustfs/connect ]] + run=$(gh api "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}") + jq -e --arg source "$SOURCE_SHA" ' + .head_sha == $source and .head_branch == "main" + and .head_repository.full_name == "rustfs/rustfs" + and .name == "Build and Release" and .path == ".github/workflows/build.yml" + and .status == "completed" and .conclusion == "success" + ' <<<"$run" >/dev/null + jobs=$(gh api --paginate --slurp "repos/rustfs/rustfs/actions/runs/${BUILD_RUN_ID}/jobs?per_page=100") + jq -e ' + [.[].jobs[] | select(.name | test("^Build RustFS \\(linux-x86_64-gnu, [a-z0-9-]+, x86_64-unknown-linux-gnu, false, linux, pyroscope\\)$"))] as $matches + | ($matches | length) == 1 and $matches[0].conclusion == "success" + ' <<<"$jobs" >/dev/null + artifact=$(gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}") + jq -e --argjson run "$BUILD_RUN_ID" --arg source "$SOURCE_SHA" --arg digest "$ARTIFACT_DIGEST" ' + .workflow_run.id == $run and .workflow_run.head_sha == $source + and .name == ("rustfs-linux-x86_64-gnu-dev-" + $source[0:7]) + and .digest == $digest and .expired == false + and (.expires_at | fromdateiso8601) > now + and .size_in_bytes > 0 and .size_in_bytes <= 2147483648 + ' <<<"$artifact" >/dev/null + jq -r '.expires_at' <<<"$artifact" > artifact-expires-at + gh api "repos/rustfs/rustfs/actions/artifacts/${ARTIFACT_ID}/zip" > official-artifact.zip + [[ $(stat --format=%s official-artifact.zip) == $(jq -r '.size_in_bytes' <<<"$artifact") ]] + printf '%s official-artifact.zip\n' "${ARTIFACT_DIGEST#sha256:}" | sha256sum --check --strict - run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}") - [[ $(jq -r '.conclusion' <<<"$run") == success ]] - [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]] - [[ $(jq -r '.head_branch' <<<"$run") == main ]] - [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]] - [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]] - - artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}") - [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]] - [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]] - [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]] - [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]] - [[ $(jq -r '.expired' <<<"$artifact") == false ]] - - - name: Download exact build artifact - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 - with: - artifact-ids: ${{ inputs.artifact_id }} - path: artifact - run-id: ${{ inputs.build_run_id }} - github-token: ${{ github.token }} - - - name: Extract exact RustFS binary - shell: bash - env: - SOURCE_SHA: ${{ inputs.source_sha }} - run: | - set -euo pipefail - short_sha=${SOURCE_SHA:0:7} - package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit) - [[ -n "$package" ]] - mkdir -p binary - unzip -qq "$package" rustfs -d binary - chmod +x binary/rustfs - - - name: Run top.disk acceptance + - name: Extract only the verified official service binary shell: bash env: SOURCE_SHA: ${{ inputs.source_sha }} BINARY_SHA256: ${{ inputs.binary_sha256 }} run: | set -euo pipefail - scripts/ci/check_connect_top_disk_artifact.sh \ - binary/rustfs "$SOURCE_SHA" "$BINARY_SHA256" \ - top-disk-runtime-evidence.json top-disk.zip + python3 - <<'PY' + import io + import os + import pathlib + import shutil + import stat + import zipfile - - name: Bind workflow and artifact provenance + def regular(entry): + mode = entry.external_attr >> 16 + return not entry.is_dir() and stat.S_IFMT(mode) in (0, stat.S_IFREG) + + package = f"rustfs-linux-x86_64-gnu-dev-{os.environ['SOURCE_SHA'][:7]}.zip" + with zipfile.ZipFile('official-artifact.zip') as outer: + entries = outer.infolist() + assert 1 <= len(entries) <= 16 + assert len({entry.filename for entry in entries}) == len(entries) + assert all(regular(entry) and pathlib.PurePosixPath(entry.filename).name == entry.filename for entry in entries) + assert sum(entry.file_size for entry in entries) <= 2147483648 + assert outer.testzip() is None + with zipfile.ZipFile(io.BytesIO(outer.read(package))) as inner: + entries = inner.infolist() + expected = {'rustfs', 'rustfs-cli', 'rustfs.cpu-symbol-catalog.json', 'rustfs.cpu-symbol-catalog.sha256'} + assert {entry.filename for entry in entries} == expected and len(entries) == len(expected) + assert all(regular(entry) and 0 < entry.file_size <= 1073741824 for entry in entries) + assert inner.testzip() is None + pathlib.Path('binary').mkdir() + with inner.open('rustfs') as source, open('binary/rustfs', 'xb') as destination: + header = source.read(20) + assert header[:7] == b'\x7fELF\x02\x01\x01' and header[18:20] == b'\x3e\x00', 'Expected a little-endian ELF64 x86-64 binary' + destination.write(header) + shutil.copyfileobj(source, destination) + PY + chmod 0755 binary/rustfs + printf '%s binary/rustfs\n' "$BINARY_SHA256" | sha256sum --check --strict + + - name: Run the real top.disk service job + shell: bash + env: + SOURCE_SHA: ${{ inputs.source_sha }} + CONNECT_E2E_PRODUCER_SOURCE: ${{ inputs.source_sha }} + CONNECT_E2E_PRODUCER_SHA256: ${{ inputs.binary_sha256 }} + CONNECT_E2E_PRODUCER_ARTIFACT_ID: ${{ inputs.artifact_id }} + run: | + set -euo pipefail + git -C connect-harness diff --exit-code + printf '%s\n' "$SOURCE_SHA" >connect-harness/tests/e2e/connected/rustfs-ref + changed=$(git -C connect-harness diff --name-only) + [[ -z "$changed" || "$changed" == tests/e2e/connected/rustfs-ref ]] + npm --prefix connect-harness/web ci + connect-harness/web/node_modules/.bin/playwright install --with-deps chromium + make -C connect-harness e2e-connected-dispatch-check + RUSTFS_BINARY="$GITHUB_WORKSPACE/binary/rustfs" \ + RUSTFS_WORKTREE="$GITHUB_WORKSPACE/rustfs-source" \ + CONNECT_E2E_TOP_DISK_EVIDENCE="$GITHUB_WORKSPACE/top-disk-service-job-evidence.json" \ + make -C connect-harness e2e-connected E2E_SCENARIO=top-disk + + - name: Bind and validate sanitized evidence shell: bash env: BUILD_RUN_ID: ${{ inputs.build_run_id }} ARTIFACT_ID: ${{ inputs.artifact_id }} ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + BINARY_SHA256: ${{ inputs.binary_sha256 }} + CONNECT_SHA: ${{ inputs.connect_sha }} + SOURCE_SHA: ${{ inputs.source_sha }} run: | set -euo pipefail - jq \ - --arg workflowRunId "$GITHUB_RUN_ID" \ - --arg buildRunId "$BUILD_RUN_ID" \ - --arg artifactId "$ARTIFACT_ID" \ - --arg artifactDigest "$ARTIFACT_DIGEST" \ - '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest}' \ - top-disk-runtime-evidence.json >top-disk-runtime-evidence.bound.json - mv top-disk-runtime-evidence.bound.json top-disk-runtime-evidence.json + jq -e --arg source "$SOURCE_SHA" --arg binary "$BINARY_SHA256" --arg artifact "$ARTIFACT_ID" ' + .status == "PASS" and .officialArtifactId == $artifact and .producerSource == $source and .producerSha256 == $binary + and .result.provenance.sourceCommit == $source + and .result.provenance.executableSha256 == $binary + ' top-disk-service-job-evidence.json >/dev/null + jq --arg run "$GITHUB_RUN_ID" --arg build "$BUILD_RUN_ID" \ + --arg artifact "$ARTIFACT_ID" --arg digest "$ARTIFACT_DIGEST" \ + --arg connect "$CONNECT_SHA" --arg source "$SOURCE_SHA" --arg binary "$BINARY_SHA256" \ + --arg expiry "$(cat artifact-expires-at)" ' + . + {workflowRunId:$run,buildRunId:$build,artifactId:$artifact,artifactDigest:$digest,artifactExpiresAt:$expiry,connectSha:$connect,sourceSha:$source,binarySha256:$binary} + ' top-disk-service-job-evidence.json > top-disk-service-job-evidence.bound.json + mv top-disk-service-job-evidence.bound.json top-disk-service-job-evidence.json - - name: Upload acceptance evidence + - name: Upload only verified sanitized evidence uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: connect-top-disk-evidence-${{ github.run_id }} - path: | - top-disk-runtime-evidence.json - top-disk.zip + name: connect-top-disk-service-job-evidence-${{ github.run_id }} + path: top-disk-service-job-evidence.json retention-days: 14 if-no-files-found: error