diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bd026d55c..b90529bad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -231,26 +231,95 @@ jobs: - name: Download static console assets shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} run: | mkdir -p ./rustfs/static - if [[ "${{ matrix.platform }}" == "windows" ]]; then - if curl.exe --fail -L "https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" -o console.zip --retry 3 --retry-delay 5 --max-time 300; then - unzip -o console.zip -d ./rustfs/static - rm console.zip + + verify_sha256() { + local expected="$1" + local file="$2" + + if command -v sha256sum >/dev/null 2>&1; then + printf '%s %s\n' "$expected" "$file" | sha256sum -c - + elif command -v shasum >/dev/null 2>&1; then + printf '%s %s\n' "$expected" "$file" | shasum -a 256 -c - else - echo "Warning: Failed to download console assets, continuing without them" - echo "// Static assets not available" > ./rustfs/static/empty.txt + echo "No SHA-256 verification tool found" >&2 + return 1 fi + } + + download_console_assets() { + local curl_bin="curl" + local python_bin="python3" + local console_api="https://api.github.com/repos/rustfs/console/releases/latest" + local console_json="console-release.json" + local console_url + local console_sha256 + local curl_auth_args=() + + if [[ "${{ matrix.platform }}" == "windows" ]]; then + curl_bin="curl.exe" + else + chmod +w ./rustfs/static/LICENSE || true + fi + + if ! command -v "$python_bin" >/dev/null 2>&1; then + python_bin="python" + fi + if ! command -v "$python_bin" >/dev/null 2>&1; then + echo "No Python interpreter found for release metadata parsing" >&2 + return 1 + fi + + if [[ -n "${GITHUB_TOKEN:-}" ]]; then + curl_auth_args=(-H "Authorization: Bearer ${GITHUB_TOKEN}" -H "X-GitHub-Api-Version: 2022-11-28") + fi + + "$curl_bin" "${curl_auth_args[@]}" --fail -L "$console_api" \ + -o "$console_json" --retry 3 --retry-delay 5 --max-time 300 || return 1 + + read -r console_url console_sha256 < <("$python_bin" - "$console_json" <<'PY' + import json + import sys + + with open(sys.argv[1], encoding="utf-8") as handle: + release = json.load(handle) + + for asset in release.get("assets", []): + name = asset.get("name", "") + digest = asset.get("digest", "") + url = asset.get("browser_download_url", "") + if name.startswith("rustfs-console-") and name.endswith(".zip") and digest.startswith("sha256:") and url: + print(url, digest.split(":", 1)[1]) + break + else: + raise SystemExit("no console zip asset with sha256 digest found") + PY + ) || return 1 + + if [[ -z "$console_url" || -z "$console_sha256" ]]; then + echo "Console release metadata is missing URL or SHA-256 digest" >&2 + return 1 + fi + + "$curl_bin" --fail -L "$console_url" -o console.zip --retry 3 --retry-delay 5 --max-time 300 || return 1 + verify_sha256 "$console_sha256" console.zip || return 2 + unzip -o console.zip -d ./rustfs/static || return 2 + } + + if download_console_assets; then + rm -f console.zip console-release.json else - chmod +w ./rustfs/static/LICENSE || true - if curl --fail -L "https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" \ - -o console.zip --retry 3 --retry-delay 5 --max-time 300; then - unzip -o console.zip -d ./rustfs/static - rm console.zip - else - echo "Warning: Failed to download console assets, continuing without them" - echo "// Static assets not available" > ./rustfs/static/empty.txt + status=$? + rm -f console.zip console-release.json + if [[ "$status" -eq 2 ]]; then + echo "Console asset integrity verification failed" >&2 + exit 1 fi + echo "Warning: Failed to download verified console assets, continuing without them" + echo "// Static assets not available" > ./rustfs/static/empty.txt fi - name: Build RustFS @@ -765,8 +834,10 @@ jobs: OSSUTIL_VERSION="2.1.1" OSSUTIL_ZIP="ossutil-${OSSUTIL_VERSION}-linux-amd64.zip" OSSUTIL_DIR="ossutil-${OSSUTIL_VERSION}-linux-amd64" + OSSUTIL_SHA256="60f3a808cbbdfd4b5269b8f967c6a66f564c9dacff52d93a5d21a588976ab5a2" - curl -o "$OSSUTIL_ZIP" "https://gosspublic.alicdn.com/ossutil/v2/${OSSUTIL_VERSION}/${OSSUTIL_ZIP}" + curl --fail -L -o "$OSSUTIL_ZIP" "https://gosspublic.alicdn.com/ossutil/v2/${OSSUTIL_VERSION}/${OSSUTIL_ZIP}" + printf '%s %s\n' "$OSSUTIL_SHA256" "$OSSUTIL_ZIP" | sha256sum -c - unzip "$OSSUTIL_ZIP" OSSUTIL_BIN="${RUNNER_TEMP}/ossutil" mv "${OSSUTIL_DIR}/ossutil" "$OSSUTIL_BIN" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 06e79e018..ff2bda880 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -406,9 +406,8 @@ jobs: RELEASE=${{ steps.meta.outputs.docker_release }} CHANNEL=${{ steps.meta.outputs.docker_channel }} BUILDKIT_INLINE_CACHE=1 - # Enable advanced BuildKit features for better performance - provenance: false - sbom: false + provenance: true + sbom: true # Add retry mechanism by splitting the build process no-cache: false pull: true diff --git a/Dockerfile b/Dockerfile index a8da4fecb..5156c6aaa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,7 +17,7 @@ FROM alpine:3.23.4 AS build ARG TARGETARCH ARG RELEASE=latest -RUN apk add --no-cache ca-certificates curl unzip +RUN apk add --no-cache ca-certificates curl jq unzip WORKDIR /build RUN set -eux; \ @@ -27,9 +27,8 @@ RUN set -eux; \ *) echo "Unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \ esac; \ if [ "$RELEASE" = "latest" ]; then \ - TAG="$(curl -fsSL https://api.github.com/repos/rustfs/rustfs/releases \ - | grep -o '"tag_name": "[^"]*"' | cut -d'"' -f4 | head -n 1)"; \ - RELEASE_JSON="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG")"; \ + RELEASE_JSON="$(curl -fsSL https://api.github.com/repos/rustfs/rustfs/releases | jq -c '.[0]')"; \ + TAG="$(printf '%s' "$RELEASE_JSON" | jq -r '.tag_name // empty')"; \ else \ TAG="$RELEASE"; \ RELEASE_JSON="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG" 2>/dev/null || true)"; \ @@ -44,14 +43,19 @@ RUN set -eux; \ TAG="$ALT_TAG"; \ fi; \ fi; \ + if [ -z "$TAG" ] || [ -z "$RELEASE_JSON" ]; then echo "Failed to resolve release metadata" >&2; exit 1; fi; \ echo "Using tag: $TAG (arch pattern: $ARCH_SUBSTR)"; \ - # Find download URL in assets list for this tag that contains arch substring and ends with .zip - URL="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG" \ - | grep -o "\"browser_download_url\": \"[^\"]*${ARCH_SUBSTR}[^\"]*\\.zip\"" \ - | cut -d'"' -f4 | head -n 1)"; \ - if [ -z "$URL" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \ + ASSET_JSON="$(printf '%s' "$RELEASE_JSON" | jq -c --arg arch "$ARCH_SUBSTR" '.assets[] | select((.name | contains($arch)) and (.name | endswith(".zip"))) | {name: .name, url: .browser_download_url, digest: .digest}' | head -n 1)"; \ + if [ -z "$ASSET_JSON" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \ + ASSET_NAME="$(printf '%s' "$ASSET_JSON" | jq -r '.name // empty')"; \ + URL="$(printf '%s' "$ASSET_JSON" | jq -r '.url // empty')"; \ + DIGEST="$(printf '%s' "$ASSET_JSON" | jq -r '.digest // empty')"; \ + SHA256="$(printf '%s' "$DIGEST" | sed 's/^sha256://')"; \ + if [ -z "$URL" ] || [ -z "$ASSET_NAME" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \ + if [ -z "$SHA256" ] || [ "$SHA256" = "$DIGEST" ]; then echo "Release asset $ASSET_NAME is missing a sha256 digest" >&2; exit 1; fi; \ echo "Downloading: $URL"; \ curl -fL "$URL" -o rustfs.zip; \ + printf '%s rustfs.zip\n' "$SHA256" | sha256sum -c -; \ unzip -q rustfs.zip -d /build; \ # If binary is not in root directory, try to locate and move from zip to /build/rustfs if [ ! -x /build/rustfs ]; then \ diff --git a/Dockerfile.glibc b/Dockerfile.glibc index 20005d983..e837b8570 100644 --- a/Dockerfile.glibc +++ b/Dockerfile.glibc @@ -20,6 +20,7 @@ ARG RELEASE=latest RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ + jq \ unzip \ && rm -rf /var/lib/apt/lists/* @@ -32,9 +33,8 @@ RUN set -eux; \ *) echo "Unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \ esac; \ if [ "$RELEASE" = "latest" ]; then \ - TAG="$(curl -fsSL https://api.github.com/repos/rustfs/rustfs/releases \ - | grep -o '"tag_name": "[^"]*"' | cut -d'"' -f4 | head -n 1)"; \ - RELEASE_JSON="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG")"; \ + RELEASE_JSON="$(curl -fsSL https://api.github.com/repos/rustfs/rustfs/releases | jq -c '.[0]')"; \ + TAG="$(printf '%s' "$RELEASE_JSON" | jq -r '.tag_name // empty')"; \ else \ TAG="$RELEASE"; \ RELEASE_JSON="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG" 2>/dev/null || true)"; \ @@ -49,14 +49,20 @@ RUN set -eux; \ TAG="$ALT_TAG"; \ fi; \ fi; \ + if [ -z "$TAG" ] || [ -z "$RELEASE_JSON" ]; then echo "Failed to resolve release metadata" >&2; exit 1; fi; \ echo "Using tag: $TAG (arch pattern: $ARCH_SUBSTR)"; \ - URL="$(curl -fsSL "https://api.github.com/repos/rustfs/rustfs/releases/tags/$TAG" \ - | grep -o "\"browser_download_url\": \"[^\"]*${ARCH_SUBSTR}[^\"]*\\.zip\"" \ - | cut -d'"' -f4 | head -n 1)"; \ + ASSET_JSON="$(printf '%s' "$RELEASE_JSON" | jq -c --arg arch "$ARCH_SUBSTR" '.assets[] | select((.name | contains($arch)) and (.name | endswith(".zip"))) | {name: .name, url: .browser_download_url, digest: .digest}' | head -n 1)"; \ + if [ -z "$ASSET_JSON" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \ + ASSET_NAME="$(printf '%s' "$ASSET_JSON" | jq -r '.name // empty')"; \ + URL="$(printf '%s' "$ASSET_JSON" | jq -r '.url // empty')"; \ + DIGEST="$(printf '%s' "$ASSET_JSON" | jq -r '.digest // empty')"; \ + SHA256="$(printf '%s' "$DIGEST" | sed 's/^sha256://')"; \ \ - if [ -z "$URL" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \ + if [ -z "$URL" ] || [ -z "$ASSET_NAME" ]; then echo "Failed to locate release asset for $ARCH_SUBSTR at tag $TAG" >&2; exit 1; fi; \ + if [ -z "$SHA256" ] || [ "$SHA256" = "$DIGEST" ]; then echo "Release asset $ASSET_NAME is missing a sha256 digest" >&2; exit 1; fi; \ \ curl -fL "$URL" -o rustfs.zip; \ + printf '%s rustfs.zip\n' "$SHA256" | sha256sum -c -; \ unzip -q rustfs.zip -d /build; \ \ if [ ! -x /build/rustfs ]; then \