From dfcee45b986efb3dfecbcb68e75683c1922093db Mon Sep 17 00:00:00 2001 From: Chris Date: Mon, 14 Sep 2026 10:21:37 +0800 Subject: [PATCH] test(connect): verify top.disk release artifacts (#7808) --- .../workflows/connect-top-disk-acceptance.yml | 139 ++++++++++++++ scripts/ci/check_connect_top_disk_artifact.sh | 171 ++++++++++++++++++ 2 files changed, 310 insertions(+) create mode 100644 .github/workflows/connect-top-disk-acceptance.yml create mode 100755 scripts/ci/check_connect_top_disk_artifact.sh diff --git a/.github/workflows/connect-top-disk-acceptance.yml b/.github/workflows/connect-top-disk-acceptance.yml new file mode 100644 index 000000000..a9024080e --- /dev/null +++ b/.github/workflows/connect-top-disk-acceptance.yml @@ -0,0 +1,139 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Connect top.disk artifact acceptance + +on: + workflow_dispatch: + inputs: + build_run_id: + description: Successful main-branch Build and Release workflow run ID + required: true + type: string + artifact_id: + description: Linux x86_64 GNU artifact ID from that run + required: true + type: string + source_sha: + description: Exact 40-character source commit + required: true + type: string + artifact_digest: + description: GitHub artifact digest including sha256 prefix + required: true + type: string + binary_sha256: + description: Expected rustfs binary SHA-256 + required: true + type: string + +permissions: + actions: read + contents: read + +jobs: + top-disk: + name: Verify native Linux x86_64 top.disk artifact + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - name: Checkout acceptance harness + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Verify source run and artifact identity + shell: bash + env: + GH_TOKEN: ${{ github.token }} + BUILD_RUN_ID: ${{ inputs.build_run_id }} + ARTIFACT_ID: ${{ inputs.artifact_id }} + SOURCE_SHA: ${{ inputs.source_sha }} + ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + run: | + set -euo pipefail + [[ "$BUILD_RUN_ID" =~ ^[0-9]+$ ]] + [[ "$ARTIFACT_ID" =~ ^[0-9]+$ ]] + [[ "$SOURCE_SHA" =~ ^[0-9a-f]{40}$ ]] + [[ "$ARTIFACT_DIGEST" =~ ^sha256:[0-9a-f]{64}$ ]] + + run=$(gh api "repos/${GITHUB_REPOSITORY}/actions/runs/${BUILD_RUN_ID}") + [[ $(jq -r '.conclusion' <<<"$run") == success ]] + [[ $(jq -r '.head_sha' <<<"$run") == "$SOURCE_SHA" ]] + [[ $(jq -r '.head_branch' <<<"$run") == main ]] + [[ $(jq -r '.head_repository.full_name' <<<"$run") == "$GITHUB_REPOSITORY" ]] + [[ $(jq -r '.name' <<<"$run") == "Build and Release" ]] + + artifact=$(gh api "repos/${GITHUB_REPOSITORY}/actions/artifacts/${ARTIFACT_ID}") + [[ $(jq -r '.workflow_run.id' <<<"$artifact") == "$BUILD_RUN_ID" ]] + [[ $(jq -r '.workflow_run.head_sha' <<<"$artifact") == "$SOURCE_SHA" ]] + [[ $(jq -r '.name' <<<"$artifact") == rustfs-linux-x86_64-gnu-* ]] + [[ $(jq -r '.digest' <<<"$artifact") == "$ARTIFACT_DIGEST" ]] + [[ $(jq -r '.expired' <<<"$artifact") == false ]] + + - name: Download exact build artifact + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7 + with: + artifact-ids: ${{ inputs.artifact_id }} + path: artifact + run-id: ${{ inputs.build_run_id }} + github-token: ${{ github.token }} + + - name: Extract exact RustFS binary + shell: bash + env: + SOURCE_SHA: ${{ inputs.source_sha }} + run: | + set -euo pipefail + short_sha=${SOURCE_SHA:0:7} + package=$(find artifact -type f -name "rustfs-linux-x86_64-gnu-dev-${short_sha}.zip" -print -quit) + [[ -n "$package" ]] + mkdir -p binary + unzip -qq "$package" rustfs -d binary + chmod +x binary/rustfs + + - name: Run top.disk acceptance + shell: bash + env: + SOURCE_SHA: ${{ inputs.source_sha }} + BINARY_SHA256: ${{ inputs.binary_sha256 }} + run: | + set -euo pipefail + scripts/ci/check_connect_top_disk_artifact.sh \ + binary/rustfs "$SOURCE_SHA" "$BINARY_SHA256" top-disk-runtime-evidence.json + + - name: Bind workflow and artifact provenance + shell: bash + env: + BUILD_RUN_ID: ${{ inputs.build_run_id }} + ARTIFACT_ID: ${{ inputs.artifact_id }} + ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} + run: | + set -euo pipefail + jq \ + --arg workflowRunId "$GITHUB_RUN_ID" \ + --arg buildRunId "$BUILD_RUN_ID" \ + --arg artifactId "$ARTIFACT_ID" \ + --arg artifactDigest "$ARTIFACT_DIGEST" \ + '. + {workflowRunId: $workflowRunId, buildRunId: $buildRunId, artifactId: $artifactId, artifactDigest: $artifactDigest}' \ + top-disk-runtime-evidence.json >top-disk-runtime-evidence.bound.json + mv top-disk-runtime-evidence.bound.json top-disk-runtime-evidence.json + + - name: Upload acceptance evidence + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: connect-top-disk-evidence-${{ github.run_id }} + path: top-disk-runtime-evidence.json + retention-days: 14 + if-no-files-found: error diff --git a/scripts/ci/check_connect_top_disk_artifact.sh b/scripts/ci/check_connect_top_disk_artifact.sh new file mode 100755 index 000000000..98e4eabeb --- /dev/null +++ b/scripts/ci/check_connect_top_disk_artifact.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +if [[ $# -ne 4 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +binary=$(realpath "$1") +expected_source_sha=$2 +expected_binary_sha256=$3 +evidence_json=$4 +work_dir=$(mktemp -d) +trap 'rm -rf "$work_dir"' EXIT + +actual_binary_sha256=$(sha256sum "$binary" | awk '{print $1}') +[[ "$actual_binary_sha256" == "$expected_binary_sha256" ]] + +version=$($binary --version) +grep -Fq "git commit : $expected_source_sha" <<<"$version" +grep -Fq "build profile: release" <<<"$version" +grep -Fq "build os : linux-x86_64" <<<"$version" + +mkdir -p "$work_dir/state/identity" "$work_dir/archive" +openssl genpkey -algorithm EC -pkeyopt ec_paramgen_curve:P-256 -out "$work_dir/device.pem" >/dev/null 2>&1 +openssl pkcs8 -topk8 -nocrypt -in "$work_dir/device.pem" -outform DER -out "$work_dir/state/identity/device.key" +chmod 600 "$work_dir/state/identity/device.key" + +now=$(date +%s) +expires_at=$((now + 600)) +organization=organizations/019e3ae0-0000-7000-8000-000000000010 +cluster=$organization/clusters/019e3ae0-0000-7000-8000-000000000011 +device=$cluster/clusterDevices/019e3ae0-0000-7000-8000-000000000012 +run_uid=019e3ae0-0000-7000-8000-000000000001 +artifact_uid=019e3ae0-0000-7000-8000-000000000013 +consent_uid=019e3ae0-0000-7000-8000-000000000014 + +common_args=( + connect top disk + --state-dir "$work_dir/state" + --organization "$organization" + --cluster "$cluster" + --device "$device" + --run-uid "$run_uid" + --artifact-uid "$artifact_uid" + --consent-uid "$consent_uid" + --policy-revision 1 + --export-validity-seconds 300 +) + +output=$work_dir/top-disk.zip +success_log=$work_dir/success.log +"$binary" "${common_args[@]}" --output "$output" --consent-expires-at "$expires_at" --run-expires-at "$expires_at" --window-millis 500 --acknowledge-l3 >"$success_log" +result_json=$(sed -n 's/^result=//p' "$success_log") +[[ $(jq -r '.outcome' <<<"$result_json") == SUCCEEDED ]] +[[ $(jq -r '.reasonCode' <<<"$result_json") == COMPLETE ]] +[[ $(jq -r '.provenance.sourceCommit' <<<"$result_json") == "$expected_source_sha" ]] +[[ $(jq -r '.provenance.executableSha256' <<<"$result_json") == "$expected_binary_sha256" ]] +[[ $(jq -r '.provenance.osFamily' <<<"$result_json") == LINUX ]] +[[ $(jq -r '.provenance.architecture' <<<"$result_json") == x86_64 ]] +[[ $(jq -r '.data.resourceAlias' <<<"$result_json") == resource-1 ]] +counter_total=$(jq '[.data.readBytes, .data.writeBytes, .data.ioCount] | add' <<<"$result_json") +((counter_total > 0)) + +reported_archive_sha256=$(sed -n 's/^artifact=[^ ]* bytes=[^ ]* sha256=//p' "$success_log") +actual_archive_sha256=$(sha256sum "$output" | awk '{print $1}') +[[ "$actual_archive_sha256" == "$reported_archive_sha256" ]] +unzip -qq "$output" -d "$work_dir/archive" +mapfile -t archive_members < <(find "$work_dir/archive" -maxdepth 1 -type f -printf '%f\n' | sort) +[[ "${archive_members[*]}" == "envelope.json envelope.sig result.json" ]] +cmp -s <(jq -cS . <<<"$result_json") <(jq -cS . "$work_dir/archive/result.json") +result_sha256=$(sha256sum "$work_dir/archive/result.json" | awk '{print $1}') +[[ $(jq -r '.payload.sha256' "$work_dir/archive/envelope.json") == "$result_sha256" ]] +[[ $(jq -r '.algorithm' "$work_dir/archive/envelope.sig") == ES256 ]] + +python3 - "$work_dir/archive/envelope.sig" "$work_dir/signature.der" <<'PY' +import base64 +import json +import sys + +signature = json.load(open(sys.argv[1], encoding="utf-8"))["value"] +raw = base64.urlsafe_b64decode(signature + "=" * (-len(signature) % 4)) +if len(raw) != 64: + raise SystemExit("ES256 signature is not 64 bytes") + +def integer(value: bytes) -> bytes: + value = value.lstrip(b"\0") or b"\0" + if value[0] & 0x80: + value = b"\0" + value + return b"\x02" + bytes([len(value)]) + value + +body = integer(raw[:32]) + integer(raw[32:]) +open(sys.argv[2], "wb").write(b"\x30" + bytes([len(body)]) + body) +PY +printf 'rustfs-diagnostic-envelope-v1\0' >"$work_dir/signed-input" +cat "$work_dir/archive/envelope.json" >>"$work_dir/signed-input" +openssl pkey -in "$work_dir/device.pem" -pubout -out "$work_dir/device.pub" >/dev/null 2>&1 +openssl dgst -sha256 -verify "$work_dir/device.pub" -signature "$work_dir/signature.der" "$work_dir/signed-input" + +if grep -R -F "$work_dir" "$work_dir/archive" || grep -R -F 'device.key' "$work_dir/archive"; then + echo "top.disk export leaked a local path" >&2 + exit 1 +fi + +set +e +"$binary" "${common_args[@]}" --output "$work_dir/no-consent.zip" --consent-expires-at "$expires_at" --run-expires-at "$expires_at" --window-millis 500 >"$work_dir/no-consent.log" 2>&1 +no_consent_status=$? +set -e +((no_consent_status != 0)) +[[ ! -e "$work_dir/no-consent.zip" ]] + +set +e +"$binary" "${common_args[@]}" --output "$work_dir/expired.zip" --consent-expires-at "$((now - 1))" --run-expires-at "$((now - 1))" --window-millis 500 --acknowledge-l3 >"$work_dir/expired.log" 2>&1 +expired_status=$? +set -e +((expired_status != 0)) +[[ ! -e "$work_dir/expired.zip" ]] + +set +e +"$binary" "${common_args[@]}" --output "$work_dir/over-limit.zip" --consent-expires-at "$expires_at" --run-expires-at "$expires_at" --window-millis 30001 --acknowledge-l3 >"$work_dir/over-limit.log" 2>&1 +limit_status=$? +set -e +((limit_status != 0)) +[[ ! -e "$work_dir/over-limit.zip" ]] + +printf 'keep-existing\n' >"$work_dir/existing.zip" +set +e +"$binary" "${common_args[@]}" --output "$work_dir/existing.zip" --consent-expires-at "$expires_at" --run-expires-at "$expires_at" --window-millis 500 --acknowledge-l3 >"$work_dir/existing.log" 2>&1 +existing_status=$? +set -e +((existing_status != 0)) +[[ $(cat "$work_dir/existing.zip") == keep-existing ]] + +interrupt_output=$work_dir/interrupted.zip +"$binary" "${common_args[@]}" --output "$interrupt_output" --consent-expires-at "$expires_at" --run-expires-at "$expires_at" --window-millis 30000 --acknowledge-l3 >"$work_dir/interrupted.log" 2>&1 & +interrupt_pid=$! +sleep 1 +kill -INT "$interrupt_pid" +set +e +wait "$interrupt_pid" +interrupt_status=$? +set -e +((interrupt_status != 0)) +[[ ! -e "$interrupt_output" ]] +if find "$work_dir" -name '*.partial' -print -quit | grep -q .; then + echo "interrupted capture left a partial file" >&2 + exit 1 +fi + +jq -n \ + --arg sourceSha "$expected_source_sha" \ + --arg binarySha256 "$actual_binary_sha256" \ + --arg archiveSha256 "$actual_archive_sha256" \ + --argjson result "$result_json" \ + '{sourceSha: $sourceSha, binarySha256: $binarySha256, archiveSha256: $archiveSha256, invocation: "scripts/ci/check_connect_top_disk_artifact.sh ", result: $result, controls: {signature: "VERIFIED", consent: "REJECTED_WITHOUT_ACKNOWLEDGEMENT", expiry: "REJECTED", limits: "REJECTED", sigint: "CANCELLED_WITHOUT_OUTPUT", noClobber: "PRESERVED", redaction: "VERIFIED"}}' >"$evidence_json" + +jq '{sourceSha, binarySha256, archiveSha256, result: {outcome: .result.outcome, reasonCode: .result.reasonCode, durationMillis: .result.durationMillis, data: .result.data}, controls}' "$evidence_json"