diff --git a/.config/e2e-odm-interop-selection.txt b/.config/e2e-odm-interop-selection.txt new file mode 100644 index 000000000..36c133d20 --- /dev/null +++ b/.config/e2e-odm-interop-selection.txt @@ -0,0 +1 @@ +sha256=87c05c46d611ea7ed3feb5f7276bda8e5a0f70d72165d305d73a457907e7ba79 diff --git a/.config/nextest.toml b/.config/nextest.toml index cc40c78f1..c6eaf1908 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -526,6 +526,34 @@ path = "junit.xml" filter = 'package(e2e_test)' test-group = 'e2e-cluster-nightly' +# --------------------------------------------------------------------------- +# e2e-odm-interop profile — on-demand migration provider interop lane (ODM-20) +# --------------------------------------------------------------------------- +# backlog#2167. Report-only, scheduled, never a required check; wired by +# .github/workflows/on-demand-migration-interop.yml. +# +# The four cases in `on_demand_migration::interop_test` take their source from +# the environment (`RUSTFS_ODM_INTEROP_*`, documented on the constants in +# `crates/e2e_test/src/on_demand_migration/common.rs`), so the same bodies run +# against the in-process fake source locally and against a MinIO container or a +# real cloud provider in the lane. The cloud jobs narrow this profile with their +# own `-E` filter to the three-case minimum (GET miss, HEAD miss, merged list +# pagination) and pass `--no-tests=fail` so a rename cannot silently select +# nothing; the MinIO job runs the whole profile, backfill included. +# +# These cases are deliberately absent from every other lane: without an +# interop source they only re-prove what `get_basic_test` and +# `list_through_test` already cover in e2e-smoke and e2e-full. The committed +# selection digest is the guard against a rename dropping one of them. +[profile.e2e-odm-interop] +default-filter = 'package(e2e_test) & test(/^on_demand_migration::interop_test::/)' +fail-fast = false + +[profile.e2e-odm-interop.junit] +# Emitted to target/nextest/e2e-odm-interop/junit.xml; the lane uploads it and +# reconciles it against the per-case JSON report entries. +path = "junit.xml" + # --------------------------------------------------------------------------- # e2e-protocols profile — serial protocol lane # --------------------------------------------------------------------------- @@ -558,6 +586,11 @@ path = "junit.xml" # cluster-fault lane. heal_erasure_disk_rebuild is intentionally not # excluded here because backlog#2213 promotes core heal rebuild coverage to # this merge/main lane while retaining nightly coverage. +# * on_demand_migration::interop_test — the ODM-20 provider interoperability +# cases, which are meaningless without a source: they run in the dedicated +# [profile.e2e-odm-interop] lane below, where the workflow points them at a +# MinIO container or a real cloud provider. Excluding them here also keeps +# this profile's committed selection digest stable. # * replication_extension_test — repl-1 already splits it into the PR # `e2e-smoke` (20 fast) and `e2e-repl-nightly` (56 slow) lanes and reserves # it for those, so e2e-full does not double-run it. @@ -576,7 +609,7 @@ default-filter = """ & !test(/^(admin_timeout_regression_test|cluster_concurrency_test|cluster_multidrive_pool_test|degraded_listing_availability_test|namespace_lock_quorum_test|object_lambda_test|stale_multipart_cleanup_cluster_test)::/) & !test(/^replication_extension_test::/) & !test(/^replication_target_matrix_test::/) - & !test(/^on_demand_migration::(concurrency_test|fault_test|real_source_test)::/) + & !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/) """ fail-fast = false diff --git a/.github/actions/odm-interop-report/action.yml b/.github/actions/odm-interop-report/action.yml new file mode 100644 index 000000000..18954fac7 --- /dev/null +++ b/.github/actions/odm-interop-report/action.yml @@ -0,0 +1,100 @@ +name: On-demand migration interop report +description: >- + Merge the per-case JSON entries an on-demand-migration interop run wrote with + the nextest JUnit result into one provider report, and summarise it. + +inputs: + provider: + description: Provider the run addressed (minio, aws, r2, gcs). + required: true + cases-dir: + description: Directory the cases wrote their JSON entries into. + required: true + junit: + description: nextest JUnit XML of the run. + required: true + output: + description: Path of the merged JSON report to write. + required: true + +runs: + using: composite + steps: + # The JUnit file is authoritative for which cases ran and how they ended: + # a case that fails or panics never reaches its own report entry, so + # trusting the entries alone would silently shorten the report exactly when + # something went wrong. The entries only add what JUnit cannot know — the + # source request accounting and the bucket's migration counters. + - name: Merge interop case reports + shell: bash + env: + ODM_REPORT_PROVIDER: ${{ inputs.provider }} + ODM_REPORT_CASES_DIR: ${{ inputs.cases-dir }} + ODM_REPORT_JUNIT: ${{ inputs.junit }} + ODM_REPORT_OUTPUT: ${{ inputs.output }} + run: | + python3 - <<'PY' + import json + import os + import pathlib + import xml.etree.ElementTree as ElementTree + + provider = os.environ["ODM_REPORT_PROVIDER"] + cases_dir = pathlib.Path(os.environ["ODM_REPORT_CASES_DIR"]) + junit = pathlib.Path(os.environ["ODM_REPORT_JUNIT"]) + output = pathlib.Path(os.environ["ODM_REPORT_OUTPUT"]) + + entries = {} + if cases_dir.is_dir(): + for path in sorted(cases_dir.glob("*.json")): + entry = json.loads(path.read_text()) + entries[entry["case"]] = entry + + cases = [] + for case in ElementTree.parse(junit).getroot().iter("testcase"): + name = case.get("name", "") + failed = [child for child in case if child.tag in ("failure", "error")] + skipped = [child for child in case if child.tag == "skipped"] + outcome = "failed" if failed else "skipped" if skipped else "passed" + entry = entries.get(name.rsplit("::", 1)[-1], {}) + cases.append( + { + "name": name, + "outcome": outcome, + "junit_duration_ms": round(float(case.get("time", "0")) * 1000), + "case_duration_ms": entry.get("duration_ms"), + "source_requests": entry.get("source_requests"), + "odm_counters": entry.get("odm_counters"), + } + ) + + report = { + "provider": provider, + "repository": os.environ.get("GITHUB_REPOSITORY", ""), + "sha": os.environ.get("GITHUB_SHA", ""), + "run_id": os.environ.get("GITHUB_RUN_ID", ""), + "cases": cases, + "totals": { + "cases": len(cases), + "passed": sum(1 for case in cases if case["outcome"] == "passed"), + "failed": sum(1 for case in cases if case["outcome"] == "failed"), + }, + } + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(json.dumps(report, indent=2, sort_keys=True) + "\n") + + summary = [f"### On-demand migration interop: `{provider}`", "", "| Case | Outcome | Duration | Source requests |", "|---|---|---|---|"] + for case in cases: + requests = case["source_requests"] + counted = f"{requests['total']} ({requests['counted_by']})" if requests else "not reported" + summary.append(f"| `{case['name']}` | {case['outcome']} | {case['junit_duration_ms']} ms | {counted} |") + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as handle: + handle.write("\n".join(summary) + "\n\n") + + # A passed case with no entry of its own means the harness stopped + # writing one: the report would keep looking complete while silently + # losing its request accounting. + unreported = [case["name"] for case in cases if case["outcome"] == "passed" and case["source_requests"] is None] + if unreported: + raise SystemExit(f"passed cases wrote no interop report entry: {', '.join(unreported)}") + PY diff --git a/.github/workflows/on-demand-migration-interop.yml b/.github/workflows/on-demand-migration-interop.yml new file mode 100644 index 000000000..7b285fd25 --- /dev/null +++ b/.github/workflows/on-demand-migration-interop.yml @@ -0,0 +1,315 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# On-demand migration provider interop (rustfs/backlog#2167, ODM-20). +# +# The in-process fake source that the merge-gate ODM suite runs against covers +# the protocol semantics, but real implementations differ in path-style vs +# virtual-host addressing, region handling, ETag shape, list pagination and +# rate limiting. This lane runs the same case bodies +# (crates/e2e_test/src/on_demand_migration/interop_test.rs) against real +# sources; the source is injected through RUSTFS_ODM_INTEROP_* environment +# variables, so nothing about the cases is duplicated per provider. +# +# Report-only and scheduled. It is never a required check and must not be +# promoted to one: it depends on third-party endpoints and on repository +# secrets that a fork does not have. +# +# Jobs: +# * minio-source runs the whole e2e-odm-interop profile — read-through, +# HEAD passthrough, merged list pagination and a backfill — against a +# pinned MinIO container. The backfill is sized at 5,000 objects here: the +# fake source retains at most 4,096 object versions and 4,096 journal +# entries, so the merge-gate backfill coverage cannot go past that, and a +# real source is where a production-shaped batch belongs. +# * cloud-source runs the three-case minimum (GET miss, HEAD miss, merged +# list pagination) against AWS S3, Cloudflare R2 and the GCS XML +# interoperability API. Each provider is skipped with a summary note when +# its ODM_INTEROP_* repository secrets are absent, which is the normal +# state on a fork and in any clone of this repository. +# +# Every job uploads one JSON report per provider naming the cases, their +# timings and the source request accounting. +name: on-demand-migration-interop + +on: + workflow_dispatch: + schedule: + # Nightly at 05:23 UTC, offset from the other nightly lanes. + - cron: "23 5 * * *" + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + # The three cases a cloud provider is asked for. Named individually rather + # than by module so adding a fourth case does not silently start billing a + # cloud account for it. + CLOUD_CASE_FILTER: >- + package(e2e_test) & test(/^on_demand_migration::interop_test::(interop_get_miss_pulls_from_the_source_and_serves_locally|interop_head_miss_answers_from_the_source_without_persisting|interop_list_through_pages_the_source_namespace)$/) + CLOUD_CASE_COUNT: "3" + +jobs: + minio-source: + name: MinIO source (read-through, list-through, backfill) + # Skip on forks: needs this repository's runners and is not a contributor + # gate. + if: github.repository == 'rustfs/rustfs' + runs-on: ubuntu-latest + timeout-minutes: 90 + env: + NO_PROXY: 127.0.0.1,localhost + # Fixed credentials of the container this job starts and throws away; + # not a secret and deliberately not read from one, so the lane runs + # unattended in any clone that enables it. + MINIO_ROOT_USER: rustfsodminterop + MINIO_ROOT_PASSWORD: rustfsodminteropsecret + RUSTFS_ODM_INTEROP_PROVIDER: minio + RUSTFS_ODM_INTEROP_ENDPOINT: http://127.0.0.1:9100 + RUSTFS_ODM_INTEROP_REGION: auto + RUSTFS_ODM_INTEROP_BUCKET: odm-interop-source + RUSTFS_ODM_INTEROP_PATH_STYLE: path + RUSTFS_ODM_INTEROP_ACCESS_KEY: rustfsodminterop + RUSTFS_ODM_INTEROP_SECRET_KEY: rustfsodminteropsecret + RUSTFS_ODM_INTEROP_BACKFILL_OBJECTS: "5000" + RUSTFS_ODM_INTEROP_REPORT_DIR: ${{ github.workspace }}/artifacts/odm-interop/minio/cases + NEXTEST_LISTING: ${{ github.workspace }}/artifacts/odm-interop/minio/selection.json + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + cache-shared-key: ci-odm-interop + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + install-build-packaging-tools: 'false' + + - name: Start MinIO source + run: | + set -euo pipefail + mkdir -p artifacts/odm-interop/minio + docker run -d --name rustfs-odm-interop-minio \ + -e "MINIO_ROOT_USER=${MINIO_ROOT_USER}" \ + -e "MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}" \ + -p 9100:9000 \ + minio/minio:RELEASE.2025-09-07T16-13-09Z server /data + for _ in $(seq 1 120); do + curl -fsS http://127.0.0.1:9100/minio/health/live >/dev/null 2>&1 && break + sleep 1 + done + curl -fsS http://127.0.0.1:9100/minio/health/live + + # The harness never creates a bucket, so that pointing it at a cloud + # account cannot create one there either. The source bucket for the + # container is created here instead. + - name: Create the MinIO source bucket + env: + AWS_ACCESS_KEY_ID: ${{ env.MINIO_ROOT_USER }} + AWS_SECRET_ACCESS_KEY: ${{ env.MINIO_ROOT_PASSWORD }} + AWS_DEFAULT_REGION: us-east-1 + run: | + aws --endpoint-url "${RUSTFS_ODM_INTEROP_ENDPOINT}" \ + s3api create-bucket --bucket "${RUSTFS_ODM_INTEROP_BUCKET}" + + - name: Build the RustFS binary under test + run: cargo build --locked -p rustfs --bins + + # The lane selects tests by module, so a rename would quietly shrink it. + # The committed digest in .config/e2e-odm-interop-selection.txt fails + # closed on that. + - name: Verify interop lane membership + run: | + cargo nextest list --profile e2e-odm-interop -p e2e_test --message-format json > "${NEXTEST_LISTING}" + python3 ./scripts/check_test_wiring.py --check-profile e2e-odm-interop "${NEXTEST_LISTING}" + + - name: Run the interop cases against MinIO + run: cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail + + - name: Build the MinIO interop report + if: always() + uses: ./.github/actions/odm-interop-report + with: + provider: minio + cases-dir: ${{ env.RUSTFS_ODM_INTEROP_REPORT_DIR }} + junit: target/nextest/e2e-odm-interop/junit.xml + output: artifacts/odm-interop/minio/report.json + + - name: Collect MinIO logs + if: always() + run: | + docker logs --tail 500 rustfs-odm-interop-minio \ + > artifacts/odm-interop/minio/minio.log 2>&1 || true + + - name: Stop MinIO source + if: always() + run: docker rm -f rustfs-odm-interop-minio >/dev/null 2>&1 || true + + - name: Upload the MinIO interop report + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }} + path: | + artifacts/odm-interop/minio + target/nextest/e2e-odm-interop/junit.xml + retention-days: 14 + + # Unlike a production migration source, which needs read access only, the + # credentials here also seed the objects each case reads back, so they need + # write and delete on the interop bucket. Every run seeds under + # `odm-interop///` and deletes what it seeded when the case + # passes; give the bucket an expiration lifecycle rule so the prefixes a + # failing case leaves behind cannot accumulate. + cloud-source: + name: ${{ matrix.provider }} source (three-case minimum) + if: github.repository == 'rustfs/rustfs' + runs-on: ubuntu-latest + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + - provider: aws + secret_prefix: AWS + path_style: virtual + - provider: r2 + secret_prefix: R2 + path_style: virtual + - provider: gcs + secret_prefix: GCS_HMAC + path_style: virtual + env: + RUSTFS_ODM_INTEROP_PROVIDER: ${{ matrix.provider }} + RUSTFS_ODM_INTEROP_PATH_STYLE: ${{ matrix.path_style }} + RUSTFS_ODM_INTEROP_ENDPOINT: ${{ secrets[format('ODM_INTEROP_{0}_ENDPOINT', matrix.secret_prefix)] }} + RUSTFS_ODM_INTEROP_REGION: ${{ secrets[format('ODM_INTEROP_{0}_REGION', matrix.secret_prefix)] }} + RUSTFS_ODM_INTEROP_BUCKET: ${{ secrets[format('ODM_INTEROP_{0}_BUCKET', matrix.secret_prefix)] }} + RUSTFS_ODM_INTEROP_ACCESS_KEY: ${{ secrets[format('ODM_INTEROP_{0}_ACCESS_KEY_ID', matrix.secret_prefix)] }} + RUSTFS_ODM_INTEROP_SECRET_KEY: ${{ secrets[format('ODM_INTEROP_{0}_SECRET_ACCESS_KEY', matrix.secret_prefix)] }} + RUSTFS_ODM_INTEROP_REPORT_DIR: ${{ github.workspace }}/artifacts/odm-interop/${{ matrix.provider }}/cases + NEXTEST_LISTING: ${{ github.workspace }}/artifacts/odm-interop/${{ matrix.provider }}/selection.json + steps: + # Absent secrets are the normal state, not a failure: the lane reports + # which providers it could reach and skips the rest. An empty value is + # what an unset repository secret expands to, so it is checked, not the + # secret's existence. + - name: Check for provider credentials + id: credentials + run: | + set -euo pipefail + if [ -z "${RUSTFS_ODM_INTEROP_ENDPOINT}" ] \ + || [ -z "${RUSTFS_ODM_INTEROP_REGION}" ] \ + || [ -z "${RUSTFS_ODM_INTEROP_BUCKET}" ] \ + || [ -z "${RUSTFS_ODM_INTEROP_ACCESS_KEY}" ] \ + || [ -z "${RUSTFS_ODM_INTEROP_SECRET_KEY}" ]; then + echo "present=false" >> "$GITHUB_OUTPUT" + { + echo "### On-demand migration interop: \`${{ matrix.provider }}\`" + echo + echo "Skipped: the \`ODM_INTEROP_${{ matrix.secret_prefix }}_*\` repository secrets" + echo "(\`_ENDPOINT\`, \`_REGION\`, \`_BUCKET\`, \`_ACCESS_KEY_ID\`, \`_SECRET_ACCESS_KEY\`)" + echo "are not configured, so no real \`${{ matrix.provider }}\` source was reached." + echo + } >> "$GITHUB_STEP_SUMMARY" + else + echo "present=true" >> "$GITHUB_OUTPUT" + fi + + - name: Checkout repository + if: steps.credentials.outputs.present == 'true' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup Rust environment + if: steps.credentials.outputs.present == 'true' + uses: ./.github/actions/setup + with: + cache-shared-key: ci-odm-interop + cache-save-if: 'false' + install-build-packaging-tools: 'false' + + - name: Build the RustFS binary under test + if: steps.credentials.outputs.present == 'true' + run: cargo build --locked -p rustfs --bins + + # A filterset that matches nothing is valid, so the count is asserted + # rather than inferred from a green run. + - name: Verify the three-case minimum still selects three cases + if: steps.credentials.outputs.present == 'true' + run: | + set -euo pipefail + mkdir -p "$(dirname "${NEXTEST_LISTING}")" + cargo nextest list --profile e2e-odm-interop -p e2e_test \ + -E "${CLOUD_CASE_FILTER}" --message-format json > "${NEXTEST_LISTING}" + selected="$(python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(sum(1 for suite in d.get("rust-suites", {}).values() for test in suite.get("testcases", {}).values() if test.get("filter-match", {}).get("status") == "matches"))' "${NEXTEST_LISTING}")" + echo "cloud interop cases selected: ${selected}" + if [ "${selected}" != "${CLOUD_CASE_COUNT}" ]; then + echo "::error::CLOUD_CASE_FILTER selected ${selected} cases, expected ${CLOUD_CASE_COUNT}; the interop cases were renamed or moved. Context: rustfs/backlog#2167." + exit 1 + fi + + - name: Run the three-case minimum + if: steps.credentials.outputs.present == 'true' + run: | + cargo nextest run --profile e2e-odm-interop -p e2e_test \ + -E "${CLOUD_CASE_FILTER}" --no-tests=fail + + - name: Build the ${{ matrix.provider }} interop report + if: always() && steps.credentials.outputs.present == 'true' + uses: ./.github/actions/odm-interop-report + with: + provider: ${{ matrix.provider }} + cases-dir: ${{ env.RUSTFS_ODM_INTEROP_REPORT_DIR }} + junit: target/nextest/e2e-odm-interop/junit.xml + output: artifacts/odm-interop/${{ matrix.provider }}/report.json + + - name: Upload the ${{ matrix.provider }} interop report + if: always() && steps.credentials.outputs.present == 'true' + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: odm-interop-${{ matrix.provider }}-${{ github.run_number }}-${{ github.run_attempt }} + path: | + artifacts/odm-interop/${{ matrix.provider }} + target/nextest/e2e-odm-interop/junit.xml + retention-days: 14 + + alert-on-failure: + name: Alert on scheduled failure + needs: [minio-source, cloud-source] + if: >- + always() && github.event_name == 'schedule' && + (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/crates/e2e_test/src/on_demand_migration/common.rs b/crates/e2e_test/src/on_demand_migration/common.rs index 1afe50499..ed2925fa0 100644 --- a/crates/e2e_test/src/on_demand_migration/common.rs +++ b/crates/e2e_test/src/on_demand_migration/common.rs @@ -30,6 +30,7 @@ use aws_sdk_s3::Client; use aws_sdk_s3::config::{Credentials, Region}; use aws_smithy_http_client::Builder as SmithyHttpClientBuilder; use bytes::Bytes; +use futures::stream::{StreamExt, TryStreamExt}; use serde::Serialize; use std::fmt; use std::time::{Duration, Instant}; @@ -627,6 +628,37 @@ impl OdmTestEnv { } } + /// Waits until the migration runtime for `bucket` is live, whatever the + /// source is. [`Self::wait_until_source_consulted`] proves the same thing + /// from the fake source's journal, which a real source does not have; this + /// one reads the bucket's own `requests_total.get` counters instead, which + /// only start moving once the state has been built. The probe is a GET of a + /// key that exists nowhere and is unique per call, so nothing is pulled and + /// only that key enters the negative cache. + pub async fn wait_until_odm_engaged(&self, bucket: &str) -> Result<(), BoxError> { + let probe_key = format!("_odm-engaged-probe-{}", uuid::Uuid::new_v4()); + let deadline = Instant::now() + Duration::from_secs(60); + loop { + let _ = self.raw_get(bucket, &probe_key).await?; + let counted: u64 = self + .status_json(bucket) + .await + .ok() + .as_ref() + .and_then(|status| status.pointer("/counters/requests_total/get")) + .and_then(serde_json::Value::as_object) + .map(|by_outcome| by_outcome.values().filter_map(serde_json::Value::as_u64).sum()) + .unwrap_or(0); + if counted > 0 { + return Ok(()); + } + if Instant::now() >= deadline { + return Err(format!("on-demand migration runtime for {bucket} did not engage in time").into()); + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + } + /// Creates `bucket` unless it already exists, installs `spec` on it and /// returns once the runtime consults the source. Scenarios with a second /// bucket, a bucket created with non-default options, or a reinstalled @@ -821,3 +853,400 @@ pub async fn start_configured_env_with( env.configure_and_wait(bucket, &spec).await?; Ok(env) } + +// --------------------------------------------------------------------------- +// Provider interoperability lane (ODM-20, rustfs/backlog#2167) +// --------------------------------------------------------------------------- +// +// `interop_test.rs` has one body per case; which source that body runs against +// is decided here, from the environment. Unset means the in-process fake +// source, which is what a local run gets; the scheduled interop workflow sets +// the variables below to a MinIO container or a real cloud provider. Nothing +// else about the cases changes, so a provider difference shows up as the same +// assertion failing rather than as a separate, drifting test file. + +/// Provider preset of the interop source (`minio`, `aws`, `r2`, `gcs`, `s3`). +/// Unset selects the in-process fake source. +pub const INTEROP_PROVIDER_ENV: &str = "RUSTFS_ODM_INTEROP_PROVIDER"; +/// `http(s)://host[:port]`. Required for every provider including `aws`, where +/// the runtime could derive it from the region: the lane pins exactly one +/// endpoint per provider so its report names what was actually reached. +pub const INTEROP_ENDPOINT_ENV: &str = "RUSTFS_ODM_INTEROP_ENDPOINT"; +pub const INTEROP_REGION_ENV: &str = "RUSTFS_ODM_INTEROP_REGION"; +/// Source bucket, which must already exist: the harness never creates a bucket +/// on a real provider account. +pub const INTEROP_BUCKET_ENV: &str = "RUSTFS_ODM_INTEROP_BUCKET"; +pub const INTEROP_ACCESS_KEY_ENV: &str = "RUSTFS_ODM_INTEROP_ACCESS_KEY"; +pub const INTEROP_SECRET_KEY_ENV: &str = "RUSTFS_ODM_INTEROP_SECRET_KEY"; +pub const INTEROP_SESSION_TOKEN_ENV: &str = "RUSTFS_ODM_INTEROP_SESSION_TOKEN"; +/// `auto` (default), `path` or `virtual`. +pub const INTEROP_PATH_STYLE_ENV: &str = "RUSTFS_ODM_INTEROP_PATH_STYLE"; +/// Objects the backfill case seeds. +pub const INTEROP_BACKFILL_OBJECTS_ENV: &str = "RUSTFS_ODM_INTEROP_BACKFILL_OBJECTS"; +/// Directory each case writes its JSON report entry into. Unset means no +/// report, which is what a local run wants. +pub const INTEROP_REPORT_DIR_ENV: &str = "RUSTFS_ODM_INTEROP_REPORT_DIR"; + +/// Backfill objects when [`INTEROP_BACKFILL_OBJECTS_ENV`] is unset. The fake +/// source retains at most 4,096 object versions and 4,096 journal entries, and +/// one pull is a HEAD plus a GET, so the default has to stay far below that. +/// A real source has no such cap and the workflow raises the count there. +pub const INTEROP_DEFAULT_BACKFILL_OBJECTS: usize = 200; +/// In-flight requests while seeding or cleaning a real source. High enough to +/// hide the round-trip on a few thousand tiny objects, low enough not to look +/// like a burst to a cloud provider. +const INTEROP_SOURCE_CONCURRENCY: usize = 32; + +/// A real S3-compatible endpoint acting as the migration source. +#[derive(Clone)] +pub struct InteropRemoteSource { + pub provider: String, + pub endpoint: String, + pub region: String, + pub bucket: String, + pub path_style: String, + pub access_key: String, + pub secret_key: String, + pub session_token: Option, +} + +impl fmt::Debug for InteropRemoteSource { + /// The interop lane uploads its logs as an artifact; keep the credentials + /// out of them. + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("InteropRemoteSource") + .field("provider", &self.provider) + .field("endpoint", &self.endpoint) + .field("region", &self.region) + .field("bucket", &self.bucket) + .field("path_style", &self.path_style) + .field("access_key", &self.access_key) + .field("secret_key", &"REDACTED") + .field("session_token", &self.session_token.as_ref().map(|_| "REDACTED")) + .finish() + } +} + +impl InteropRemoteSource { + /// Enabled configuration pointing at this source. + fn spec(&self) -> OdmSourceSpec { + let mut spec = OdmSourceSpec::new( + &self.provider, + &self.endpoint, + &self.region, + self.bucket.clone(), + &self.access_key, + &self.secret_key, + ); + spec.source.path_style = self.path_style.clone(); + spec.source.credentials = Some(OdmCredentials { + access_key: self.access_key.clone(), + secret_key: self.secret_key.clone(), + session_token: self.session_token.clone(), + }); + spec + } + + /// S3 client for seeding and cleaning the source bucket. Retries are off + /// so a provider-side failure surfaces as itself instead of being masked + /// by a second attempt. + fn client(&self) -> Result { + let credentials = + Credentials::new(&self.access_key, &self.secret_key, self.session_token.clone(), None, "odm-interop-source"); + let mut config = aws_sdk_s3::Config::builder() + .credentials_provider(credentials) + .region(Region::new(self.region.clone())) + .endpoint_url(&self.endpoint) + .force_path_style(self.path_style == "path") + .behavior_version_latest() + .retry_config(RetryConfig::standard().with_max_attempts(1)); + // The default connector is HTTPS-only; a container source is plain HTTP. + if self.endpoint.starts_with("http://") { + config = config.http_client(SmithyHttpClientBuilder::new().build_http()); + } + Ok(Client::from_conf(config.build())) + } +} + +/// Where an interop case gets its source objects from. +#[derive(Debug, Clone)] +pub enum InteropSource { + /// The in-process fake source of the [`OdmTestEnv`]. + Fake, + /// A real S3-compatible endpoint described by the environment. + Remote(InteropRemoteSource), +} + +impl InteropSource { + /// Reads the source description from the environment. An unset + /// [`INTEROP_PROVIDER_ENV`] means the fake source; a named provider with + /// any required variable missing is an error rather than a silent + /// fallback, so a misconfigured CI secret can never pass as a green + /// real-source run. + pub fn from_env() -> Result { + let Some(provider) = interop_env(INTEROP_PROVIDER_ENV) else { + return Ok(Self::Fake); + }; + let mut missing = Vec::new(); + let mut required = |name: &'static str| { + interop_env(name).unwrap_or_else(|| { + missing.push(name); + String::new() + }) + }; + let endpoint = required(INTEROP_ENDPOINT_ENV); + let region = required(INTEROP_REGION_ENV); + let bucket = required(INTEROP_BUCKET_ENV); + let access_key = required(INTEROP_ACCESS_KEY_ENV); + let secret_key = required(INTEROP_SECRET_KEY_ENV); + if !missing.is_empty() { + return Err(format!("{INTEROP_PROVIDER_ENV}={provider} needs {}", missing.join(", ")).into()); + } + Ok(Self::Remote(InteropRemoteSource { + provider, + endpoint, + region, + bucket, + path_style: interop_env(INTEROP_PATH_STYLE_ENV).unwrap_or_else(|| "auto".to_string()), + access_key, + secret_key, + session_token: interop_env(INTEROP_SESSION_TOKEN_ENV), + })) + } + + /// Name the report uses for this source. + pub fn provider(&self) -> &str { + match self { + Self::Fake => "fake", + Self::Remote(remote) => &remote.provider, + } + } +} + +/// A set but empty variable is the shape a missing GitHub secret takes, so it +/// reads the same as unset here. +fn interop_env(name: &str) -> Option { + std::env::var(name) + .ok() + .map(|value| value.trim().to_string()) + .filter(|value| !value.is_empty()) +} + +/// Objects the backfill case seeds, from [`INTEROP_BACKFILL_OBJECTS_ENV`]. +pub fn interop_backfill_objects() -> Result { + match interop_env(INTEROP_BACKFILL_OBJECTS_ENV) { + Some(value) => Ok(value + .parse::() + .map_err(|error| format!("{INTEROP_BACKFILL_OBJECTS_ENV}: {error}"))?), + None => Ok(INTEROP_DEFAULT_BACKFILL_OBJECTS), + } +} + +/// One interop case: a RustFS under test migrating `bucket` from whichever +/// source [`InteropSource::from_env`] resolved. +/// +/// Every source key of a run lives under a unique `filter.source_prefix`, so +/// a shared real bucket can host concurrent runs, a backfill lists only this +/// run's objects, and [`Self::finish`] can delete exactly what it seeded. +pub struct OdmInteropEnv { + pub env: OdmTestEnv, + pub source: InteropSource, + pub bucket: String, + case: &'static str, + source_bucket: String, + source_prefix: String, + remote: Option, + /// Source-side keys this run created, for cleanup. + seeded: std::sync::Mutex>, + /// Start of the case body, after the fixture is up. The report keeps this + /// next to the JUnit wall time so a provider's own latency is readable + /// without the constant cost of starting a RustFS server drowning it. + started: Instant, +} + +impl OdmInteropEnv { + /// Starts the pair and installs the configuration `adjust` tweaked, + /// returning once the migration runtime is live. + pub async fn start(case: &'static str, bucket: &str, adjust: impl FnOnce(&mut OdmSourceSpec)) -> Result { + let source = InteropSource::from_env()?; + let env = OdmTestEnv::start().await?; + env.rustfs.create_test_bucket(bucket).await?; + + let source_prefix = format!("odm-interop/{case}/{}/", uuid::Uuid::new_v4()); + let (mut spec, remote, source_bucket) = match &source { + InteropSource::Fake => { + let source_bucket = format!("{bucket}-source"); + env.source.create_bucket_with_mode(&source_bucket, BucketMode::Unversioned); + (env.fake_source_spec(&source_bucket), None, source_bucket) + } + InteropSource::Remote(remote) => { + let client = remote.client()?; + client + .head_bucket() + .bucket(&remote.bucket) + .send() + .await + .map_err(|error| format!("interop source bucket {} is not reachable: {error}", remote.bucket))?; + (remote.spec(), Some(client), remote.bucket.clone()) + } + }; + spec.filter.source_prefix = Some(source_prefix.clone()); + adjust(&mut spec); + + let response = env.configure_source(bucket, &spec).await?; + if response.status != 200 { + return Err(format!("configure on-demand migration for {bucket}: {} {}", response.status, response.body).into()); + } + env.wait_until_odm_engaged(bucket).await?; + Ok(Self { + env, + source, + bucket: bucket.to_string(), + case, + source_bucket, + source_prefix, + remote, + seeded: std::sync::Mutex::new(Vec::new()), + started: Instant::now(), + }) + } + + /// Source-side key for a local key, the same mapping the runtime applies. + fn source_key(&self, local_key: &str) -> String { + format!("{}{local_key}", self.source_prefix) + } + + /// Stores `objects` in the source under this run's prefix and returns + /// their ETags, unquoted, in input order. + pub async fn seed(&self, objects: &[SeedObject]) -> Result, BoxError> { + let etags = match &self.remote { + None => objects + .iter() + .map(|object| { + self.env.source.put_seed_object( + &self.source_bucket, + self.source_key(&object.key), + object.body.clone(), + &object.metadata, + ) + }) + .collect(), + Some(client) => { + futures::stream::iter(objects.iter().map(|object| { + let key = self.source_key(&object.key); + let body = object.body.clone(); + async move { + let output = client + .put_object() + .bucket(&self.source_bucket) + .key(&key) + .body(aws_sdk_s3::primitives::ByteStream::from(body)) + .send() + .await + .map_err(|error| format!("seeding {}/{key}: {error}", self.source_bucket))?; + Ok::(unquote_etag(output.e_tag().unwrap_or_default())) + } + })) + .buffered(INTEROP_SOURCE_CONCURRENCY) + .try_collect::>() + .await? + } + }; + self.seeded + .lock() + .expect("interop seed ledger is not poisoned") + .extend(objects.iter().map(|object| self.source_key(&object.key))); + Ok(etags) + } + + /// Records the case in the lane's report and removes everything it seeded + /// from the source. Call it at the end of every case: a case that fails + /// before this point leaves no report entry, which is why the workflow + /// reconciles the entries against the JUnit case list rather than trusting + /// them to be complete. + pub async fn finish(self) -> Result<(), BoxError> { + let duration = self.started.elapsed(); + let counters = self + .env + .status_json(&self.bucket) + .await + .ok() + .and_then(|status| status.get("counters").cloned()); + self.write_report(duration, counters).await?; + self.clean_source().await + } + + async fn write_report(&self, duration: Duration, counters: Option) -> Result<(), BoxError> { + let Some(dir) = interop_env(INTEROP_REPORT_DIR_ENV) else { + return Ok(()); + }; + // The fake source journals every wire request, so its count is exact. + // A real provider has no such journal, so the report falls back to the + // bucket's own counters, which count client requests that entered + // migration rather than requests that left for the source; the + // `counted_by` field says which of the two a reader is looking at. + let (counted_by, total) = match self.remote { + None => ("fake_source_journal", self.env.source.requests().len() as u64), + Some(_) => ( + "odm_status_counters", + counters + .as_ref() + .and_then(|counters| counters.pointer("/requests_total")) + .and_then(serde_json::Value::as_object) + .map(|by_op| { + by_op + .values() + .filter_map(serde_json::Value::as_object) + .flat_map(|by_outcome| by_outcome.values().filter_map(serde_json::Value::as_u64)) + .sum() + }) + .unwrap_or(0), + ), + }; + let entry = serde_json::json!({ + "case": self.case, + "provider": self.source.provider(), + "source_bucket": self.source_bucket, + "source_prefix": self.source_prefix, + "duration_ms": duration.as_millis() as u64, + "source_requests": {"counted_by": counted_by, "total": total}, + "odm_counters": counters, + }); + tokio::fs::create_dir_all(&dir).await?; + tokio::fs::write( + format!("{dir}/{}-{}.json", self.source.provider(), self.case), + serde_json::to_vec_pretty(&entry)?, + ) + .await?; + Ok(()) + } + + /// Deletes this run's source keys one by one: the multi-object delete is + /// not available on every provider the lane targets (the GCS XML API has + /// no equivalent), and the object counts here are small enough that the + /// portable form costs nothing worth saving. + async fn clean_source(&self) -> Result<(), BoxError> { + let Some(client) = &self.remote else { + return Ok(()); + }; + let keys = std::mem::take(&mut *self.seeded.lock().expect("interop seed ledger is not poisoned")); + futures::stream::iter(keys.into_iter().map(|key| async move { + client + .delete_object() + .bucket(&self.source_bucket) + .key(&key) + .send() + .await + .map_err(|error| format!("deleting {}/{key}: {error}", self.source_bucket))?; + Ok::<(), BoxError>(()) + })) + .buffered(INTEROP_SOURCE_CONCURRENCY) + .try_collect::>() + .await?; + Ok(()) + } +} + +fn unquote_etag(etag: &str) -> String { + etag.trim_matches('"').to_string() +} diff --git a/crates/e2e_test/src/on_demand_migration/interop_test.rs b/crates/e2e_test/src/on_demand_migration/interop_test.rs new file mode 100644 index 000000000..d9033be0b --- /dev/null +++ b/crates/e2e_test/src/on_demand_migration/interop_test.rs @@ -0,0 +1,243 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Provider interoperability cases (ODM-20, rustfs/backlog#2167). +//! +//! One body per case, run against whichever source the environment names: +//! the in-process fake source locally, a MinIO container or a real cloud +//! provider under `.github/workflows/on-demand-migration-interop.yml`. The +//! source is resolved by [`OdmInteropEnv`], so a provider difference in +//! path-style addressing, region handling, ETag shape or list pagination +//! shows up as one of these assertions failing rather than as a second, +//! drifting copy of the suite. +//! +//! Consequently these cases assert only on what every S3 implementation has +//! to agree on — what the client receives and what RustFS stored — never on +//! the fake source's request journal, which a real provider does not have. +//! The journal-backed expectations stay in `get_basic_test.rs` and +//! `interaction_test.rs`. +//! +//! The first three cases are the minimum a cloud provider is asked for (GET +//! miss, HEAD miss, merged list pagination); the backfill case runs against +//! the MinIO container, whose object count the lane raises well past the fake +//! source's caps. + +use super::common::{BackfillRequest, BoxError, OdmInteropEnv, SeedObject, interop_backfill_objects}; +use bytes::Bytes; +use std::time::Duration; + +type TestResult = Result<(), BoxError>; + +const ODM_RESPONSE_HEADER: &str = "x-rustfs-on-demand-migration"; +/// Background pulls land after the response that triggered them; generous for +/// a loaded runner talking to a container. +const SETTLE: Duration = Duration::from_secs(90); + +/// Position-dependent payload so a misaligned or truncated copy is caught. +fn payload(len: usize) -> Bytes { + (0..len).map(|index| (index % 251) as u8).collect::>().into() +} + +/// A GET miss is answered from the source with the source's own ETag, and the +/// object it stored serves every later read locally. +#[tokio::test] +async fn interop_get_miss_pulls_from_the_source_and_serves_locally() -> TestResult { + let case = + OdmInteropEnv::start("interop_get_miss_pulls_from_the_source_and_serves_locally", "odm-interop-get", |_| {}).await?; + let key = "interop/report.bin"; + let body = payload(200 * 1024); + let etag = case.seed(&[SeedObject::new(key, body.clone())]).await?.remove(0); + let quoted_etag = format!("\"{etag}\""); + + let first = case.env.raw_get(&case.bucket, key).await?; + assert_eq!(first.status, 200, "{}", String::from_utf8_lossy(&first.body)); + assert_eq!(first.header(ODM_RESPONSE_HEADER), Some("source"), "a source answer is marked"); + assert_eq!(first.header("content-length"), Some(body.len().to_string().as_str())); + assert_eq!( + first.header("etag"), + Some(quoted_etag.as_str()), + "the source ETag is passed through unchanged" + ); + assert_eq!(first.body, body, "the client receives the source bytes"); + + assert!( + case.env.wait_local_listed(&case.bucket, key, SETTLE).await?, + "the inline pull must store the object locally" + ); + let second = case.env.raw_get(&case.bucket, key).await?; + assert_eq!(second.status, 200, "{}", String::from_utf8_lossy(&second.body)); + assert_eq!(second.header(ODM_RESPONSE_HEADER), None, "a local hit carries no source marker"); + assert_eq!(second.body, body, "the local copy is the source bytes"); + assert_eq!( + second.header("etag"), + Some(quoted_etag.as_str()), + "preserve_etag keeps the source ETag on the stored object" + ); + case.finish().await +} + +/// A HEAD miss is proxied with the source's size and ETag and stores nothing. +#[tokio::test] +async fn interop_head_miss_answers_from_the_source_without_persisting() -> TestResult { + let case = + OdmInteropEnv::start("interop_head_miss_answers_from_the_source_without_persisting", "odm-interop-head", |_| {}).await?; + let key = "interop/head-only.bin"; + let body = payload(9_000); + let etag = case.seed(&[SeedObject::new(key, body.clone())]).await?.remove(0); + + let head = case + .env + .raw_object_request(http::Method::HEAD, &case.bucket, key, &[]) + .await?; + assert_eq!(head.status, 200, "HEAD must be answered from the source"); + assert_eq!(head.header(ODM_RESPONSE_HEADER), Some("source")); + assert_eq!(head.header("content-length"), Some(body.len().to_string().as_str())); + assert_eq!(head.header("etag"), Some(format!("\"{etag}\"").as_str())); + assert!(head.body.is_empty(), "a HEAD carries no body"); + case.env.assert_local_absent(&case.bucket, key).await; + + // A key the source does not hold is a plain 404, not a source error. + let missing = case + .env + .raw_object_request(http::Method::HEAD, &case.bucket, "interop/absent.bin", &[]) + .await?; + assert_eq!(missing.status, 404, "a source miss is a 404"); + case.finish().await +} + +/// The merged `ListObjectsV2` pages the source namespace in byte order, keeps +/// every page within `max_keys`, and lets a local object win a shared key. +#[tokio::test] +async fn interop_list_through_pages_the_source_namespace() -> TestResult { + const SOURCE_KEYS: usize = 120; + const PAGE_SIZE: i32 = 50; + const SOURCE_BODY_LEN: usize = 3; + const LOCAL_BODY_LEN: usize = 11; + + let case = OdmInteropEnv::start("interop_list_through_pages_the_source_namespace", "odm-interop-list", |spec| { + spec.policy.list_through = true + }) + .await?; + let keys: Vec = (0..SOURCE_KEYS).map(|index| format!("page/obj-{index:05}")).collect(); + let seeds: Vec = keys + .iter() + .map(|key| SeedObject::new(key.clone(), payload(SOURCE_BODY_LEN))) + .collect(); + case.seed(&seeds).await?; + + // Five keys the local bucket also holds, with a body length that tells the + // two sides apart in the listing. + let shared: Vec = keys.iter().step_by(25).cloned().collect(); + for key in &shared { + case.env + .client + .put_object() + .bucket(&case.bucket) + .key(key) + .body(aws_sdk_s3::primitives::ByteStream::from(payload(LOCAL_BODY_LEN))) + .send() + .await?; + } + + let mut listed: Vec<(String, i64)> = Vec::new(); + let mut token: Option = None; + let mut completed = false; + for _ in 0..SOURCE_KEYS { + let page = case + .env + .client + .list_objects_v2() + .bucket(&case.bucket) + .prefix("page/") + .max_keys(PAGE_SIZE) + .set_continuation_token(token.take()) + .send() + .await?; + assert!(page.contents().len() <= PAGE_SIZE as usize, "a merged page must not exceed max_keys"); + for object in page.contents() { + listed.push((object.key().unwrap_or_default().to_string(), object.size().unwrap_or_default())); + } + if !page.is_truncated().unwrap_or(false) { + completed = true; + break; + } + token = Some( + page.next_continuation_token() + .ok_or("truncated merged page without a continuation token")? + .to_string(), + ); + } + assert!(completed, "the merged listing did not terminate"); + + let listed_keys: Vec = listed.iter().map(|(key, _)| key.clone()).collect(); + assert_eq!(listed_keys, keys, "the merged listing is the source namespace in byte order"); + for (key, size) in &listed { + let expected = if shared.contains(key) { + LOCAL_BODY_LEN + } else { + SOURCE_BODY_LEN + }; + assert_eq!(*size, expected as i64, "{key} must be reported by the side that wins it"); + } + case.finish().await +} + +/// A backfill pulls every object under the run's source prefix. The count +/// comes from the environment: the fake source caps out around 4,096 stored +/// versions, while the MinIO lane runs the full production-shaped batch. +#[tokio::test] +async fn interop_backfill_pulls_every_source_object() -> TestResult { + const KEY_PREFIX: &str = "cold/"; + let count = interop_backfill_objects()?; + assert!(count > 0, "the backfill case needs at least one source object"); + let case = OdmInteropEnv::start("interop_backfill_pulls_every_source_object", "odm-interop-backfill", |_| {}).await?; + + let objects: Vec = (0..count) + .map(|index| SeedObject::new(format!("{KEY_PREFIX}{index:06}"), Bytes::from(format!("object-{index:06}")))) + .collect(); + case.seed(&objects).await?; + + let started = case.env.start_backfill(&case.bucket, BackfillRequest::default()).await?; + assert_eq!(started.status, 200, "start backfill: {}", started.body); + + // One pull is a HEAD plus a GET plus a local write; the ceiling scales + // with the object count so raising it in the workflow does not need a + // second knob here. + let timeout = Duration::from_secs(180 + count as u64 / 5); + let done = case + .env + .wait_for_backfill(&case.bucket, timeout, |job| job["state"] == "completed") + .await?; + for (name, expected) in [ + ("listed", count as u64), + ("enqueued", count as u64), + ("pulled", count as u64), + ("failed", 0), + ] { + assert_eq!( + done[name].as_u64().unwrap_or_else(|| panic!("{name} missing in {done}")), + expected, + "backfill {name}" + ); + } + assert_eq!( + case.env.local_key_count(&case.bucket, KEY_PREFIX).await?, + count, + "every source object must be stored locally" + ); + case.env + .assert_local_present(&case.bucket, &objects[count - 1].key, &objects[count - 1].body) + .await; + case.finish().await +} diff --git a/crates/e2e_test/src/on_demand_migration/mod.rs b/crates/e2e_test/src/on_demand_migration/mod.rs index aa898ab17..887d2608b 100644 --- a/crates/e2e_test/src/on_demand_migration/mod.rs +++ b/crates/e2e_test/src/on_demand_migration/mod.rs @@ -22,7 +22,10 @@ //! optional merged `ListObjectsV2` (ODM-17, rustfs/backlog#2164). The fault, concurrency, //! interaction and real-source matrix is rustfs/backlog#2158; its lane split //! lives in `.config/nextest.toml` (fault / concurrency / real source run -//! nightly, the rest in the merge lane). +//! nightly, the rest in the merge lane). `interop_test` is the provider +//! interoperability lane (ODM-20, rustfs/backlog#2167): the same case bodies +//! against the fake source locally and against a real provider named by the +//! environment in `.github/workflows/on-demand-migration-interop.yml`. pub mod common; @@ -32,5 +35,6 @@ mod fault_test; mod get_basic_test; mod harness_self_test; mod interaction_test; +mod interop_test; mod list_through_test; mod real_source_test; diff --git a/docs/operations/on-demand-migration.md b/docs/operations/on-demand-migration.md index 0ff868b60..7a29db72c 100644 --- a/docs/operations/on-demand-migration.md +++ b/docs/operations/on-demand-migration.md @@ -125,17 +125,19 @@ Validation also rejects two shapes outright: a source whose endpoint and bucket ## Provider presets and source permissions -| Provider | Endpoint | Addressing | `region` | Notes | -|---|---|---|---|---| -| `aws` | Optional; derived as `https://s3..amazonaws.com` | Virtual-host | Real region required (`auto` rejected) | The derived form only accepts `[A-Za-z0-9-]` in `region` | -| `s3` | Required | Path-style | Real region | Generic S3-compatible endpoint (Wasabi, Backblaze B2 S3 API, Ceph RGW, …) | -| `minio` | Required | Path-style | `auto` allowed | | -| `rustfs` | Required | Path-style | `auto` allowed | A RustFS source answers the migration request locally thanks to the anti-loop marker | -| `r2` | `https://.r2.cloudflarestorage.com` | Virtual-host | `auto` allowed (signed as `us-east-1`) | | -| `gcs` | `https://storage.googleapis.com` | Virtual-host | Real region required | Uses the GCS XML interoperability API with an HMAC key pair, not a service-account JSON key | +| Provider | Endpoint | Addressing | `region` | Notes | Interop evidence | +|---|---|---|---|---|---| +| `aws` | Optional; derived as `https://s3..amazonaws.com` | Virtual-host | Real region required (`auto` rejected) | The derived form only accepts `[A-Za-z0-9-]` in `region` | `cloud-source (aws)`, only while `ODM_INTEROP_AWS_*` are configured; no difference recorded yet | +| `s3` | Required | Path-style | Real region | Generic S3-compatible endpoint (Wasabi, Backblaze B2 S3 API, Ceph RGW, …) | No lane of its own; the preset is the same code path the `minio` job exercises | +| `minio` | Required | Path-style | `auto` allowed | | `minio-source`, nightly: read-through, HEAD passthrough, merged list pagination and a 5,000-object backfill; no difference recorded yet | +| `rustfs` | Required | Path-style | `auto` allowed | A RustFS source answers the migration request locally thanks to the anti-loop marker | `real_source_test.rs` in the `e2e-nightly` lane | +| `r2` | `https://.r2.cloudflarestorage.com` | Virtual-host | `auto` allowed (signed as `us-east-1`) | | `cloud-source (r2)`, only while `ODM_INTEROP_R2_*` are configured; no difference recorded yet | +| `gcs` | `https://storage.googleapis.com` | Virtual-host | Real region required | Uses the GCS XML interoperability API with an HMAC key pair, not a service-account JSON key | `cloud-source (gcs)`, only while `ODM_INTEROP_GCS_HMAC_*` are configured; no difference recorded yet | Azure Blob has no preset; a native provider is deferred (rustfs/backlog#2166). +The "Interop evidence" column names the job in `.github/workflows/on-demand-migration-interop.yml` (rustfs/backlog#2167) that last exercised the preset against a real implementation, and is where a provider difference belongs once the lane finds one. That lane is report-only and scheduled: it runs `crates/e2e_test/src/on_demand_migration/interop_test.rs` — the same case bodies as the merge-gate suite, with the source injected through `RUSTFS_ODM_INTEROP_*` — against a pinned MinIO container, and against each cloud provider whose repository secrets are configured. A provider without secrets is skipped with a note in the run summary rather than failing, so "no difference recorded yet" means exactly that and not "verified clean"; see [ci-gates.md](../testing/ci-gates.md) for the row. + The credentials only ever need read access to the source bucket: - `s3:ListBucket` on the bucket — used by the admin probe, by the backfill listing, and by every merged listing under `policy.list_through`. diff --git a/docs/testing/ci-gates.md b/docs/testing/ci-gates.md index 117fd1e37..256b41bd4 100644 --- a/docs/testing/ci-gates.md +++ b/docs/testing/ci-gates.md @@ -77,6 +77,7 @@ Scheduled lanes never block a PR. Their workflow-local gate fails the run, sched | `e2e-s3tests.yml` (weekly) | `s3tests` (single and distributed, four shards each), `upstream-head-canary` | compatibility gate; report, JUnit, node IDs, server logs | yes | `scripts/s3-tests/run.sh` against an existing single or distributed target | | `fuzz.yml` (nightly) | `nightly-fuzz-corpus` per target | gate; corpus and crash artifacts | yes | `MAX_TOTAL_TIME= ./scripts/fuzz/run.sh` | | `minio-interop.yml` (nightly) | `minio-interop` | EC + SSE read-parity gate | yes, with `never_ran_grace_until` | pinned Docker fixture steps in the workflow | +| `on-demand-migration-interop.yml` (nightly) | `minio-source`, `cloud-source` (`aws`, `r2`, `gcs`) | report-only provider interop; one JSON report per provider naming cases, timings and source request counts, plus JUnit and MinIO logs. A cloud provider whose `ODM_INTEROP_*` secrets are absent is skipped with a summary note, not failed | no | start the pinned MinIO container as in the job, export the `RUSTFS_ODM_INTEROP_*` variables, then `cargo nextest run --profile e2e-odm-interop -p e2e_test` | | `performance-ab.yml` (nightly) | `warp-ab` | regression-budget gate; A/B summaries and server logs | yes | `bash scripts/run_hotpath_warp_abba.sh --help` | | `nightly-gnu.yml` (nightly) | `build`, `kms-vault-lane`, `kms-vault-ha-failover` | build, live Vault, and HA failover gates | yes | commands and pinned Vault images in the workflow | | `audit.yml` (nightly) | `cargo-deny`, `workflow-pin-report` | dependency and workflow-pin gates | yes | `cargo deny check`; `scripts/security/check_workflow_pins.sh` |