diff --git a/.github/workflows/minio-interop.yml b/.github/workflows/minio-interop.yml new file mode 100644 index 000000000..585dca543 --- /dev/null +++ b/.github/workflows/minio-interop.yml @@ -0,0 +1,70 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# MinIO on-disk interop: prove RustFS reads MinIO-written erasure-coded SSE +# objects with byte-identical data and correct logical size. +# +# This is NOT a PR gate. The fixtures are real MinIO backend trees generated on +# the fly (they are gitignored, never committed), so the job regenerates them +# each run with Docker and then runs the `#[ignore]` reader tests in +# crates/ecstore/tests/minio_generated_read_test.rs. +# +# Runner: GitHub-hosted `ubuntu-latest`. It reliably ships Docker + Python, +# unlike the self-hosted fleet, whose pods drift in Docker/pip availability +# (see the infra note in e2e-s3tests.yml). Nightly + manual only. +name: minio-interop + +on: + workflow_dispatch: + schedule: + # Nightly at 03:17 UTC (offset from other nightly jobs). + - cron: "17 3 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + minio-interop: + name: MinIO interop (EC + SSE read parity) + # Skip on forks: needs the repo's runners and is not a contributor gate. + if: github.repository == 'rustfs/rustfs' + runs-on: ubuntu-latest + timeout-minutes: 40 + env: + # Fixed 32-byte test KMS key baked into the fixture lab; not a secret. + RUSTFS_MINIO_STATIC_KMS_KEY_B64: IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + rust-version: stable + cache-shared-key: ci-minio-interop + github-token: ${{ secrets.GITHUB_TOKEN }} + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + + - name: Generate real MinIO fixtures via Docker + run: bash crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh + + - name: Run MinIO interop reader tests + run: | + cargo nextest run --run-ignored ignored-only \ + -p rustfs-ecstore --features rio-v2 \ + -E 'binary(minio_generated_read_test)' diff --git a/crates/ecstore/tests/minio_generated_read_test.rs b/crates/ecstore/tests/minio_generated_read_test.rs index e2d11bcc2..875aa9be9 100644 --- a/crates/ecstore/tests/minio_generated_read_test.rs +++ b/crates/ecstore/tests/minio_generated_read_test.rs @@ -280,14 +280,20 @@ async fn rejects_minio_generated_sse_s3_fixture_with_truncated_ciphertext() { async fn assert_fixture_round_trip(case_id: &str, expected_size: i64) { let (object_info, encrypted, expected_sha256) = load_fixture_reader_input(case_id).await; - let object_size = object_info.size; + // `ObjectInfo.size` is the on-disk size. For SSE objects that is the + // DARE-encrypted size (plaintext + 32 bytes per 64 KiB block), which is + // deliberately larger than the logical object size. The size a client sees + // (and what MinIO records via `x-*-internal-actual-size`) comes from + // `decrypted_size()`/`get_actual_size()`, so assert against that — the raw + // `size` field would never equal the plaintext length for encrypted objects. + let decrypted_size = object_info.decrypted_size().expect("decrypted size from MinIO metadata"); let kms_key_b64 = minio_static_kms_key_b64(); let plaintext = read_fixture_plaintext(encrypted, object_info, kms_key_b64) .await .expect("fixture must restore with the configured KMS key"); - assert_eq!(object_size, expected_size); + assert_eq!(decrypted_size, expected_size); assert_eq!(plaintext.len(), expected_size as usize); assert_eq!(sha256_hex(&plaintext), expected_sha256); } diff --git a/crates/rio-v2/tests/minio_fixture_lab/Dockerfile b/crates/rio-v2/tests/minio_fixture_lab/Dockerfile new file mode 100644 index 000000000..8e6e50069 --- /dev/null +++ b/crates/rio-v2/tests/minio_fixture_lab/Dockerfile @@ -0,0 +1,17 @@ +# Throwaway image for generating real MinIO on-disk fixtures without installing +# a MinIO server on the host. Multi-stage: pull the official MinIO server binary +# from the published image (linux, no dl.min.io download), then drop it into a +# small Python image that runs the fixture lab (lab.py needs only python3 + +# openssl; it drives MinIO's S3 API directly, so no `mc` is required). +# +# The MinIO release is pinned so the captured fixture format is reproducible; +# this is the release the interop tests were validated against. +FROM minio/minio:RELEASE.2025-09-07T16-13-09Z AS minio + +FROM python:3.12-slim +RUN apt-get update \ + && apt-get install -y --no-install-recommends openssl ca-certificates \ + && rm -rf /var/lib/apt/lists/* +COPY --from=minio /usr/bin/minio /usr/local/bin/minio +RUN chmod +x /usr/local/bin/minio && /usr/local/bin/minio --version +WORKDIR /repo diff --git a/crates/rio-v2/tests/minio_fixture_lab/README.md b/crates/rio-v2/tests/minio_fixture_lab/README.md index 4f5f96243..f20abe205 100644 --- a/crates/rio-v2/tests/minio_fixture_lab/README.md +++ b/crates/rio-v2/tests/minio_fixture_lab/README.md @@ -121,6 +121,30 @@ uv run python D:\Github\rustfs\crates\rio-v2\tests\minio_fixture_lab\lab.py capt That is a runner smoke path only. Real compatibility fixtures should still prefer the intended multi-disk backend layout when the local environment can support it. +## Via Docker (Linux/macOS, no local `minio` install) + +When you don't have a `minio` binary on the host, `capture_via_docker.sh` +generates the fixtures the ignored round-trip tests consume using Docker only. +It builds a throwaway image (the official MinIO server binary pulled from +`minio/minio` plus this lab on a small Python base — `lab.py` drives MinIO's S3 +API directly, so no `mc` is needed) and runs `capture-matrix` inside it, writing +fixtures under `crates/rio-v2/tests/fixtures/minio-generated/` (the root the Rust +tests read): + +```bash +# Default: the two 8 MiB multipart cases the round-trip tests need. +# Pass case ids to override, or "all" for the full default matrix. +./capture_via_docker.sh + +RUSTFS_MINIO_STATIC_KMS_KEY_B64=IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= \ + cargo test -p rustfs-ecstore --features rio-v2 --test minio_generated_read_test -- --ignored +``` + +This is exactly what the nightly `minio-interop` GitHub Actions workflow runs +(`.github/workflows/minio-interop.yml`), so the local and CI paths stay in sync. +SSE-C cases still need the host-`minio` + TLS path above; the Docker helper +targets the SSE-S3 / SSE-KMS multipart cases the interop tests assert on. + ## Capture Guidance For each case, preserve these inputs when possible: diff --git a/crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh b/crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh new file mode 100755 index 000000000..77f6e4687 --- /dev/null +++ b/crates/rio-v2/tests/minio_fixture_lab/capture_via_docker.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Generate real MinIO on-disk fixtures via Docker — no host `minio`/`mc` install. +# +# Builds a throwaway image carrying the official MinIO server binary plus the +# fixture lab, runs `lab.py capture-matrix` inside it, and writes the captured +# backend fixtures under crates/rio-v2/tests/fixtures/minio-generated (which is +# gitignored — regenerate as needed). The same script is used locally and by the +# nightly `minio-interop` CI workflow so both paths stay in sync. +# +# Usage: +# ./capture_via_docker.sh # the two multipart cases the +# # ignored interop tests consume +# ./capture_via_docker.sh sse-s3-singlepart-64k # specific case id(s) +# ./capture_via_docker.sh all # full SSE/size matrix +set -euo pipefail + +IMAGE="${MINIO_LAB_IMAGE:-rustfs-minio-lab:latest}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# repo root is four levels up: crates/rio-v2/tests/minio_fixture_lab -> repo +REPO_ROOT="$(cd "$SCRIPT_DIR/../../../.." && pwd)" +LAB_REL="crates/rio-v2/tests/minio_fixture_lab" +FIXTURE_REL="crates/rio-v2/tests/fixtures/minio-generated" + +# Default to the two multipart cases the ignored round-trip tests consume; pass +# case ids to override, or "all" for the full default matrix. +cases=("$@") +if [ "${#cases[@]}" -eq 0 ]; then + cases=(sse-s3-multipart-8m sse-kms-multipart-8m) +fi +case_args=() +if [ "${cases[0]}" != "all" ]; then + for c in "${cases[@]}"; do + case_args+=(--case-id "$c") + done +fi + +echo ">> building ${IMAGE}" +docker build -f "${SCRIPT_DIR}/Dockerfile" -t "${IMAGE}" "${SCRIPT_DIR}" + +echo ">> capturing fixtures into ${FIXTURE_REL}" +docker run --rm -v "${REPO_ROOT}:/repo" "${IMAGE}" \ + python3 "/repo/${LAB_REL}/lab.py" capture-matrix \ + --root "/repo/${FIXTURE_REL}" \ + --work-root /tmp/minio-lab-work \ + --minio-binary /usr/local/bin/minio \ + "${case_args[@]}" + +echo ">> done — fixtures under ${REPO_ROOT}/${FIXTURE_REL}/cases/"