diff --git a/.config/make/tests.mak b/.config/make/tests.mak index 993f27347..3e485058a 100644 --- a/.config/make/tests.mak +++ b/.config/make/tests.mak @@ -34,6 +34,7 @@ script-tests: ## Run shell script tests ./scripts/test_exact_1mib_handoff_abba.sh ./scripts/test_pinned_paired_abba_bench.sh ./scripts/test_manual_transition_runbooks.sh + ./scripts/test_fuzz_runner.sh ./scripts/check_embedded_secrets.sh --self-test python3 ./scripts/check_test_wiring.py --self-test python3 ./scripts/check_security_coverage.py --self-test diff --git a/.docker/observability/prometheus.yml.bak-issue2007-node-exporter-20260823T172248Z b/.docker/observability/prometheus.yml.bak-issue2007-node-exporter-20260823T172248Z new file mode 100644 index 000000000..1039e2972 --- /dev/null +++ b/.docker/observability/prometheus.yml.bak-issue2007-node-exporter-20260823T172248Z @@ -0,0 +1,84 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +global: + scrape_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + evaluation_interval: 15s + external_labels: + cluster: 'rustfs-dev' # Label to identify the cluster + replica: '1' # Replica identifier + +rule_files: + - /etc/prometheus/rules/*.yml + +scrape_configs: + - job_name: 'otel-collector' + static_configs: + - targets: [ 'otel-collector:8888' ] # Scrape metrics from Collector + scrape_interval: 10s + + - job_name: 'rustfs-app-metrics' + static_configs: + - targets: [ 'otel-collector:8889' ] # Application indicators + scrape_interval: 15s + metric_relabel_configs: + - source_labels: [ __name__ ] + regex: 'go_.*' + action: drop # Drop Go runtime metrics if not needed + + - job_name: 'tempo' + static_configs: + - targets: [ 'tempo:3200' ] # Scrape metrics from Tempo + + - job_name: 'jaeger' + static_configs: + - targets: [ 'jaeger:14269' ] # Jaeger admin port (14269 is standard for admin/metrics) + + - job_name: 'loki' + static_configs: + - targets: [ 'loki:3100' ] + + - job_name: 'prometheus' + static_configs: + - targets: [ 'localhost:9090' ] + + - job_name: 'vulture' + static_configs: + - targets: + - 'vulture:8080' + +otlp: + promote_resource_attributes: + - service.instance.id + - service.name + - service.namespace + - cloud.availability_zone + - cloud.region + - container.name + - deployment.environment.name + - k8s.cluster.name + - k8s.container.name + - k8s.cronjob.name + - k8s.daemonset.name + - k8s.deployment.name + - k8s.job.name + - k8s.namespace.name + - k8s.pod.name + - k8s.replicaset.name + - k8s.statefulset.name + translation_strategy: NoUTF8EscapingWithSuffixes + +storage: + tsdb: + out_of_order_time_window: 30m diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c596ed1d5..664ef7952 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -181,22 +181,14 @@ jobs: needs: [ quick-checks ] runs-on: sm-standard-4 timeout-minutes: 90 - # Both lines are required. Job-level `permissions` replaces the workflow - # block rather than merging with it, so declaring only `actions: write` - # would drop `contents: read` and break this job's checkout and the - # repo-token the setup action hands to setup-protoc. - permissions: - contents: read - actions: write env: FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" steps: - name: Checkout repository uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 with: - # This job's token can cancel runs and delete Actions caches. Checkout - # otherwise writes it into .git/config, where a PR's own build.rs or - # proc-macro could read it back out. + # Checkout otherwise writes the token into .git/config, where a PR's + # own build.rs or proc-macro could read it back out. persist-credentials: false - name: Setup Rust environment @@ -347,41 +339,36 @@ jobs: - name: Run rebalance/decommission migration proofs run: ./scripts/check_migration_gate_count.sh - # Early stop. Once this job has failed the PR cannot merge, so the sibling - # lanes are burning runners on a result nobody can act on: on run - # 30674613104 three lanes had already failed while Test and Lint and the - # rio-v2 variant kept going past 70 minutes. - # - # Only this job may cancel. The lanes that are NOT required checks - # (protocols, ILM, e2e, s3-tests) must never hold that power: a flake in - # one of them would turn the required "Test and Lint" into `cancelled`, - # which blocks the merge. Today a maintainer can merge with sftp red, and - # that has to stay true. - # - # These steps run last so the `if: always()` artifact upload above still - # captures logs and diagnostics before the run goes away. + # Record the reason before this job completes as FAILURE. A separate + # dependent job cancels sibling lanes only after GitHub has preserved this + # required check's failure verdict. - name: Annotate early-stop reason - if: failure() && github.event_name == 'pull_request' - run: | - { - echo "## CI early-stop" - echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; cancelling run ${GITHUB_RUN_ID} to free runners." - echo "Sibling jobs showing **cancelled** were stopped by this job, not by their own failure." - } >> "$GITHUB_STEP_SUMMARY" - - # curl rather than `gh`: every existing `gh` call in this repo runs on - # ubuntu-latest, and the sm-standard-* images are custom and trimmed (they - # ship no C toolchain, see the e2e job below), so `gh` is not known to - # exist here. - # - # Fork PRs are excluded explicitly instead of relying on the error path: - # their GITHUB_TOKEN is forced read-only and job-level permissions cannot - # raise it, so the call would always 403. Skipping keeps their logs clean. - - name: Cancel run on failure (same-repo PR only) if: >- failure() && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository - continue-on-error: true + run: | + { + echo "## CI early-stop" + echo "Job \`${GITHUB_JOB}\` (Test and Lint) failed; a follow-up job will cancel sibling lanes to free runners." + echo "Sibling jobs showing **cancelled** were stopped by the early-stop follow-up, not by their own failure." + } >> "$GITHUB_STEP_SUMMARY" + + # Preserve the required Test and Lint FAILURE verdict before stopping sibling + # lanes. Cancelling from inside test-and-lint changed its own conclusion to + # CANCELLED and hid the actionable failure in the PR checks UI. + cancel-after-test-and-lint-failure: + name: Cancel siblings after Test and Lint failure + if: >- + failure() && needs.test-and-lint.result == 'failure' + && github.event_name == 'pull_request' + && github.event.pull_request.head.repo.full_name == github.repository + needs: [ test-and-lint ] + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + actions: write + steps: + - name: Cancel remaining jobs env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -389,7 +376,7 @@ jobs: -H "Authorization: Bearer ${GH_TOKEN}" \ -H "Accept: application/vnd.github+json" \ -H "X-GitHub-Api-Version: 2022-11-28" \ - "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" || true + "${GITHUB_API_URL}/repos/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/cancel" # Dedicated serial lane for the ILM / lifecycle integration tests. These tests # drive the object layer through process-global singletons (the GLOBAL_ENV diff --git a/.github/workflows/e2e-upgrade.yml b/.github/workflows/e2e-upgrade.yml new file mode 100644 index 000000000..27ce41f2d --- /dev/null +++ b/.github/workflows/e2e-upgrade.yml @@ -0,0 +1,105 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Upgrade Compatibility + +on: + pull_request: + paths: + - ".github/workflows/e2e-upgrade.yml" + - "crates/e2e_test/src/common.rs" + - "crates/e2e_test/src/lib.rs" + - "crates/e2e_test/src/upgrade_compatibility_test.rs" + - "crates/ecstore/**" + - "crates/filemeta/**" + - "crates/kms/**" + - "crates/storage-api/**" + - "rustfs/**" + - "Cargo.lock" + push: + tags: + - "[0-9]*.[0-9]*.[0-9]*" + schedule: + - cron: "17 3 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +env: + CARGO_TERM_COLOR: always + RUST_BACKTRACE: 1 + UPGRADE_SOURCE_VERSION: 1.0.0-rc.2 + UPGRADE_SOURCE_ASSET: rustfs-linux-x86_64-gnu-v1.0.0-rc.2.zip + UPGRADE_SOURCE_SHA256: 7c789386bf85278f865b8e0d359bf4edb84d5aa408cc3fa54a18c25ca74cd6e7 + +jobs: + direct-upgrade: + name: Direct upgrade from rc.2 + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + cache-shared-key: e2e-direct-upgrade + cache-save-if: ${{ github.ref == 'refs/heads/main' }} + install-build-packaging-tools: "false" + + - name: Download pinned previous release + env: + SOURCE_DIR: ${{ runner.temp }}/rustfs-upgrade-source + run: | + set -euo pipefail + mkdir -p "$SOURCE_DIR" + archive="$SOURCE_DIR/$UPGRADE_SOURCE_ASSET" + curl --fail --location --retry 3 --output "$archive" \ + "https://github.com/${GITHUB_REPOSITORY}/releases/download/${UPGRADE_SOURCE_VERSION}/${UPGRADE_SOURCE_ASSET}" + echo "$UPGRADE_SOURCE_SHA256 $archive" | sha256sum --check --strict + unzip -q "$archive" -d "$SOURCE_DIR" + chmod +x "$SOURCE_DIR/rustfs" + test -x "$SOURCE_DIR/rustfs" + echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV" + echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV" + + - name: Build current RustFS binary + run: | + cargo build --locked -p rustfs --bin rustfs + : > target/debug/rustfs.features + + - name: Run direct-upgrade compatibility test + run: | + cargo test --locked -p e2e_test \ + upgrade_compatibility_test::direct_upgrade_from_rc2_preserves_object_contracts \ + -- --ignored --exact --nocapture + + - name: Upload server logs + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: direct-upgrade-server-logs-${{ github.run_number }} + path: ${{ runner.temp }}/rustfs-upgrade-logs + if-no-files-found: warn + retention-days: 14 diff --git a/.github/workflows/fuzz.yml b/.github/workflows/fuzz.yml index 6aa780fc6..395dc1848 100644 --- a/.github/workflows/fuzz.yml +++ b/.github/workflows/fuzz.yml @@ -173,7 +173,7 @@ jobs: path: | fuzz/artifacts/** fuzz/corpus/${{ matrix.target }}/** - if-no-files-found: ignore + if-no-files-found: error retention-days: 7 # ────────────────────────────────────────────────────────────── @@ -227,7 +227,7 @@ jobs: path: | fuzz/artifacts/** fuzz/corpus/${{ matrix.target }}/** - if-no-files-found: ignore + if-no-files-found: error retention-days: 30 # ────────────────────────────────────────────────────────────── diff --git a/.github/workflows/oidc-keycloak.yml b/.github/workflows/oidc-keycloak.yml new file mode 100644 index 000000000..31a57efbc --- /dev/null +++ b/.github/workflows/oidc-keycloak.yml @@ -0,0 +1,110 @@ +# Copyright 2024 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: OIDC Keycloak Live + +on: + pull_request: + paths: + - ".github/workflows/oidc-keycloak.yml" + - "crates/config/src/constants/oidc.rs" + - "crates/iam/src/federation/**" + - "crates/iam/src/oidc.rs" + - "rustfs/src/admin/handlers/oidc.rs" + - "rustfs/src/admin/handlers/sts.rs" + - "scripts/test/oidc_keycloak_live.sh" + - "scripts/test/fixtures/keycloak-rustfs-ci-realm.json" + push: + branches: [main] + paths: + - ".github/workflows/oidc-keycloak.yml" + - "crates/config/src/constants/oidc.rs" + - "crates/iam/src/federation/**" + - "crates/iam/src/oidc.rs" + - "rustfs/src/admin/handlers/oidc.rs" + - "rustfs/src/admin/handlers/sts.rs" + - "scripts/test/oidc_keycloak_live.sh" + - "scripts/test/fixtures/keycloak-rustfs-ci-realm.json" + schedule: + - cron: "23 2 * * 1" + timezone: "Asia/Shanghai" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: oidc-keycloak-live-${{ github.ref }} + cancel-in-progress: true + +jobs: + oidc-keycloak-live: + name: OIDC Keycloak live gate + runs-on: ubuntu-latest + timeout-minutes: 60 + env: + FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true" + steps: + - name: Checkout repository + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + + - name: Setup Rust environment + uses: ./.github/actions/setup + with: + cache-shared-key: oidc-keycloak-live + cache-save-if: "true" + install-build-packaging-tools: "false" + install-test-tools: "false" + + - name: Build RustFS + run: cargo build --locked -p rustfs --bin rustfs + + - name: Install pinned request signer + run: | + python3 -m pip install --user --upgrade pip "awscurl==0.44" + echo "${HOME}/.local/bin" >> "${GITHUB_PATH}" + + - name: Run live Keycloak discovery, JWT and STS checks + run: bash scripts/test/oidc_keycloak_live.sh ./target/debug/rustfs + + - name: Upload service logs + if: failure() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: oidc-keycloak-live-${{ github.run_number }} + path: ${{ runner.temp }}/rustfs-keycloak-live-*/**/*.log + if-no-files-found: ignore + retention-days: 3 + + alert-on-failure: + name: Alert on scheduled failure + needs: oidc-keycloak-live + if: >- + always() && github.event_name == 'schedule' && + (needs.oidc-keycloak-live.result == 'failure' || needs.oidc-keycloak-live.result == 'cancelled') + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + issues: write + steps: + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7 + with: + persist-credentials: false + - name: Open or update failure-tracking issue + uses: ./.github/actions/schedule-failure-issue + with: + github-token: ${{ secrets.GITHUB_TOKEN }} diff --git a/crates/config/src/constants/runtime.rs b/crates/config/src/constants/runtime.rs index 36fc33141..3e0203a71 100644 --- a/crates/config/src/constants/runtime.rs +++ b/crates/config/src/constants/runtime.rs @@ -103,7 +103,7 @@ pub const ENV_ALLOCATOR_RECLAIM_ENABLED: &str = "RUSTFS_ALLOCATOR_RECLAIM_ENABLE pub const ENV_ALLOCATOR_RECLAIM_INTERVAL_SECS: &str = "RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS"; pub const ENV_ALLOCATOR_RECLAIM_FORCE: &str = "RUSTFS_ALLOCATOR_RECLAIM_FORCE"; pub const ENV_ALLOCATOR_RECLAIM_IDLE_INTERVALS: &str = "RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS"; -pub const DEFAULT_ALLOCATOR_RECLAIM_ENABLED: bool = false; +pub const DEFAULT_ALLOCATOR_RECLAIM_ENABLED: bool = true; pub const DEFAULT_ALLOCATOR_RECLAIM_INTERVAL_SECS: u64 = 30; pub const DEFAULT_ALLOCATOR_RECLAIM_FORCE: bool = true; pub const DEFAULT_ALLOCATOR_RECLAIM_IDLE_INTERVALS: u64 = 3; diff --git a/crates/e2e_test/README.md b/crates/e2e_test/README.md index 443cd9fa1..334936a68 100644 --- a/crates/e2e_test/README.md +++ b/crates/e2e_test/README.md @@ -27,6 +27,7 @@ Registered in [`src/lib.rs`](src/lib.rs). Grouped by concern: | **reliant** | [`src/reliant/`](src/reliant) | Tests that reuse an **externally started** server (SQL/select, conditional writes, lifecycle, deleted-object reads, node-interact). Run via [`scripts/run_e2e_tests.sh`](../../scripts/run_e2e_tests.sh); see [`src/reliant/README.md`](src/reliant/README.md) | | **cluster** | `cluster_concurrency_test`, `stale_multipart_cleanup_cluster_test`, `namespace_lock_quorum_test`, `admin_timeout_regression_test`, `object_lambda_test`, `replication_extension_test` | Multi-node scenarios via `RustFSTestClusterEnvironment` | | **chaos / reliability** | [`src/chaos.rs`](src/chaos.rs), `reliability_disk_fault_test`, `heal_erasure_disk_rebuild_test`, `server_startup_failfast_test` | Disk offline/replace/corrupt, EC rebuild, heal, fail-fast startup | +| **upgrade compatibility** | `upgrade_compatibility_test` | Pinned previous-release writes followed by current-build reads on the same data directory | ## How to run @@ -168,6 +169,7 @@ the same profile for membership and execution with one nightly worker. | `s3s-e2e` black-box | `e2e-tests` + `e2e-tests-rio-v2` jobs | **Active** (external conformance tool) | | ILM / lifecycle (ignored) | `test-ilm-integration-serial` lane, `-j1` | **Active** (backlog#1148 ilm-1) | | KMS suite | `e2e-full` job, merge queue + main | **Active** | +| Direct upgrade from pinned previous release | `e2e-upgrade.yml`, storage-sensitive PRs + release tags + weekly | **Active** | | Cluster faults (`e2e-nightly` profile) | consolidated nightly workflow | **Active** (backlog#1149 ci-7) | | Protocols (FTPS/WebDAV/SFTP) | consolidated nightly workflow, serial | **Active** (backlog#1149 ci-7) | | Replication (fast subset) | `e2e-smoke` profile, `e2e-tests` job, every PR | **Active** (backlog#1147 repl-1) | diff --git a/crates/e2e_test/src/bucket_logging_test.rs b/crates/e2e_test/src/bucket_logging_test.rs index e60483097..f76eedc15 100644 --- a/crates/e2e_test/src/bucket_logging_test.rs +++ b/crates/e2e_test/src/bucket_logging_test.rs @@ -16,8 +16,10 @@ #[cfg(test)] mod tests { + use std::borrow::Borrow; + use crate::common::{RustFSTestEnvironment, init_logging, signed_s3_request}; - use aws_sdk_s3::error::ProvideErrorMetadata; + use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError}; use aws_sdk_s3::types::{ AccelerateConfiguration, BucketAccelerateStatus, BucketLoggingStatus, IndexDocument, LoggingEnabled, Payer, RequestPaymentConfiguration, WebsiteConfiguration, @@ -26,6 +28,26 @@ mod tests { use http::header::CONTENT_TYPE; use tracing::info; + fn assert_s3_error(result: Result, expected_status: u16, expected_code: &str, context: &str) + where + T: std::fmt::Debug, + E: ProvideErrorMetadata + std::fmt::Debug, + R: Borrow> + std::fmt::Debug, + { + let error = result.expect_err(context); + let sdk_error = error.borrow(); + assert_eq!( + sdk_error.raw_response().map(|response| response.status().as_u16()), + Some(expected_status), + "{context}: expected HTTP {expected_status}, got: {error:?}" + ); + assert_eq!( + sdk_error.as_service_error().and_then(ProvideErrorMetadata::code), + Some(expected_code), + "{context}: expected {expected_code}, got: {error:?}" + ); + } + #[tokio::test] async fn test_dummy_bucket_compatibility_endpoints() { init_logging(); @@ -217,17 +239,11 @@ mod tests { .expect("DeleteBucketWebsite should return success"); let website_after_delete = client.get_bucket_website().bucket(bucket).send().await; - assert!( - website_after_delete.is_err(), - "GetBucketWebsite should return NoSuchWebsiteConfiguration after deletion" - ); - let website_err = website_after_delete.err().unwrap(); - let website_code = website_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(website_code, Some("NoSuchWebsiteConfiguration")), - "Unexpected GetBucketWebsite error code: {:?}, err: {:?}", - website_code, - website_err + assert_s3_error( + website_after_delete, + 404, + "NoSuchWebsiteConfiguration", + "GetBucketWebsite after deleting the website configuration", ); env.stop_server(); @@ -245,15 +261,7 @@ mod tests { let missing_bucket = "test-dummy-bucket-missing"; let get_logging = client.get_bucket_logging().bucket(missing_bucket).send().await; - assert!(get_logging.is_err(), "GetBucketLogging should fail for missing bucket"); - let get_logging_err = get_logging.err().unwrap(); - let get_logging_code = get_logging_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(get_logging_code, Some("NoSuchBucket")), - "Unexpected GetBucketLogging error code: {:?}, err: {:?}", - get_logging_code, - get_logging_err - ); + assert_s3_error(get_logging, 404, "NoSuchBucket", "GetBucketLogging for a missing bucket"); let put_logging = client .put_bucket_logging() @@ -261,41 +269,22 @@ mod tests { .bucket_logging_status(BucketLoggingStatus::builder().build()) .send() .await; - assert!(put_logging.is_err(), "PutBucketLogging should fail for missing bucket"); - let put_logging_err = put_logging.err().unwrap(); - let put_logging_code = put_logging_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(put_logging_code, Some("NoSuchBucket")), - "Unexpected PutBucketLogging error code: {:?}, err: {:?}", - put_logging_code, - put_logging_err - ); + assert_s3_error(put_logging, 404, "NoSuchBucket", "PutBucketLogging for a missing bucket"); let get_accelerate = client .get_bucket_accelerate_configuration() .bucket(missing_bucket) .send() .await; - assert!(get_accelerate.is_err(), "GetBucketAccelerateConfiguration should fail for missing bucket"); - let get_accelerate_err = get_accelerate.err().unwrap(); - let get_accelerate_code = get_accelerate_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(get_accelerate_code, Some("NoSuchBucket")), - "Unexpected GetBucketAccelerateConfiguration error code: {:?}, err: {:?}", - get_accelerate_code, - get_accelerate_err + assert_s3_error( + get_accelerate, + 404, + "NoSuchBucket", + "GetBucketAccelerateConfiguration for a missing bucket", ); let get_request_payment = client.get_bucket_request_payment().bucket(missing_bucket).send().await; - assert!(get_request_payment.is_err(), "GetBucketRequestPayment should fail for missing bucket"); - let get_request_payment_err = get_request_payment.err().unwrap(); - let get_request_payment_code = get_request_payment_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(get_request_payment_code, Some("NoSuchBucket")), - "Unexpected GetBucketRequestPayment error code: {:?}, err: {:?}", - get_request_payment_code, - get_request_payment_err - ); + assert_s3_error(get_request_payment, 404, "NoSuchBucket", "GetBucketRequestPayment for a missing bucket"); let put_accelerate = client .put_bucket_accelerate_configuration() @@ -307,14 +296,11 @@ mod tests { ) .send() .await; - assert!(put_accelerate.is_err(), "PutBucketAccelerateConfiguration should fail for missing bucket"); - let put_accelerate_err = put_accelerate.err().unwrap(); - let put_accelerate_code = put_accelerate_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(put_accelerate_code, Some("NoSuchBucket")), - "Unexpected PutBucketAccelerateConfiguration error code: {:?}, err: {:?}", - put_accelerate_code, - put_accelerate_err + assert_s3_error( + put_accelerate, + 404, + "NoSuchBucket", + "PutBucketAccelerateConfiguration for a missing bucket", ); let put_request_payment = client @@ -328,15 +314,7 @@ mod tests { ) .send() .await; - assert!(put_request_payment.is_err(), "PutBucketRequestPayment should fail for missing bucket"); - let put_request_payment_err = put_request_payment.err().unwrap(); - let put_request_payment_code = put_request_payment_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(put_request_payment_code, Some("NoSuchBucket")), - "Unexpected PutBucketRequestPayment error code: {:?}, err: {:?}", - put_request_payment_code, - put_request_payment_err - ); + assert_s3_error(put_request_payment, 404, "NoSuchBucket", "PutBucketRequestPayment for a missing bucket"); let put_website = client .put_bucket_website() @@ -353,37 +331,13 @@ mod tests { ) .send() .await; - assert!(put_website.is_err(), "PutBucketWebsite should fail for missing bucket"); - let put_website_err = put_website.err().unwrap(); - let put_website_code = put_website_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(put_website_code, Some("NoSuchBucket")), - "Unexpected PutBucketWebsite error code: {:?}, err: {:?}", - put_website_code, - put_website_err - ); + assert_s3_error(put_website, 404, "NoSuchBucket", "PutBucketWebsite for a missing bucket"); let get_website = client.get_bucket_website().bucket(missing_bucket).send().await; - assert!(get_website.is_err(), "GetBucketWebsite should fail for missing bucket"); - let get_website_err = get_website.err().unwrap(); - let get_website_code = get_website_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(get_website_code, Some("NoSuchBucket")), - "Unexpected GetBucketWebsite error code: {:?}, err: {:?}", - get_website_code, - get_website_err - ); + assert_s3_error(get_website, 404, "NoSuchBucket", "GetBucketWebsite for a missing bucket"); let delete_website = client.delete_bucket_website().bucket(missing_bucket).send().await; - assert!(delete_website.is_err(), "DeleteBucketWebsite should fail for missing bucket"); - let delete_website_err = delete_website.err().unwrap(); - let delete_website_code = delete_website_err.as_service_error().and_then(|e| e.code()); - assert!( - matches!(delete_website_code, Some("NoSuchBucket")), - "Unexpected DeleteBucketWebsite error code: {:?}, err: {:?}", - delete_website_code, - delete_website_err - ); + assert_s3_error(delete_website, 404, "NoSuchBucket", "DeleteBucketWebsite for a missing bucket"); env.stop_server(); } diff --git a/crates/e2e_test/src/bucket_policy_check_test.rs b/crates/e2e_test/src/bucket_policy_check_test.rs index 0b9345f06..51800e195 100644 --- a/crates/e2e_test/src/bucket_policy_check_test.rs +++ b/crates/e2e_test/src/bucket_policy_check_test.rs @@ -17,6 +17,7 @@ use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::config::{Credentials, Region}; +use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::{Client, Config}; use tracing::info; @@ -73,10 +74,14 @@ async fn test_bucket_policy_authenticated_user() -> Result<(), Box, extra_env: &[(&str, &str)], cleanup_existing: bool, + ) -> Result<(), Box> { + let binary_path = rustfs_binary_path(); + self.start_rustfs_server_inner_with_binary(&binary_path, extra_args, extra_env, cleanup_existing) + .await + } + + async fn start_rustfs_server_inner_with_binary( + &mut self, + binary_path: &Path, + extra_args: Vec<&str>, + extra_env: &[(&str, &str)], + cleanup_existing: bool, ) -> Result<(), Box> { if cleanup_existing { self.cleanup_existing_processes().await?; @@ -647,8 +659,7 @@ impl RustFSTestEnvironment { info!("Starting RustFS server with args: {:?}", args); - let binary_path = rustfs_binary_path(); - let mut command = Command::new(&binary_path); + let mut command = Command::new(binary_path); command.env("RUST_LOG", "rustfs=info,rustfs_notify=debug"); // The embedded console would bind the fixed default port :9001, which // collides with unrelated local services (e.g. Docker Desktop). Tests @@ -668,6 +679,19 @@ impl RustFSTestEnvironment { Ok(()) } + /// Start a specific RustFS binary against this environment's isolated + /// data directory. Upgrade tests use this to seed an old on-disk format + /// before restarting the same environment with the workspace binary. + pub async fn start_rustfs_server_from_binary( + &mut self, + binary_path: &Path, + extra_args: Vec<&str>, + extra_env: &[(&str, &str)], + ) -> Result<(), Box> { + self.start_rustfs_server_inner_with_binary(binary_path, extra_args, extra_env, true) + .await + } + /// Start RustFS server with basic configuration pub async fn start_rustfs_server(&mut self, extra_args: Vec<&str>) -> Result<(), Box> { self.start_rustfs_server_inner(extra_args, &[], true).await diff --git a/crates/e2e_test/src/kms/common.rs b/crates/e2e_test/src/kms/common.rs index 3a6c2d364..b551a9849 100644 --- a/crates/e2e_test/src/kms/common.rs +++ b/crates/e2e_test/src/kms/common.rs @@ -24,6 +24,7 @@ use crate::common::{RustFSTestEnvironment, awscurl_get, awscurl_post, init_logging as common_init_logging, local_http_client}; use aws_sdk_s3::Client; +use aws_sdk_s3::error::{ProvideErrorMetadata, SdkError}; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::ServerSideEncryption; use base64::{Engine, engine::general_purpose::STANDARD as BASE64}; @@ -50,6 +51,9 @@ pub const VAULT_TOKEN: &str = "dev-root-token"; pub const VAULT_TRANSIT_PATH: &str = "transit"; pub const VAULT_KEY_NAME: &str = "rustfs-master-key"; pub const ENV_TEST_VAULT_BIN: &str = "RUSTFS_TEST_VAULT_BIN"; +pub const SSE_C_KEY_MISMATCH_MESSAGE: &str = + "The provided encryption parameters did not match the ones used originally to encrypt the object."; +pub const SSE_C_MISSING_PARAMETERS_MESSAGE: &str = "The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object."; /// Initialize tracing for KMS tests with KMS-specific log levels pub fn init_logging() { @@ -63,6 +67,24 @@ pub fn sse_customer_key_md5_base64(key: &str) -> String { BASE64.encode(hasher.finalize()) } +pub fn assert_s3_error(result: Result>, status: u16, code: &str, message: &str, context: &str) +where + T: std::fmt::Debug, + E: ProvideErrorMetadata + std::fmt::Debug, +{ + let error = result.expect_err(context); + assert_eq!( + error.raw_response().map(|response| response.status().as_u16()), + Some(status), + "{context}: unexpected HTTP status: {error:?}" + ); + let service_error = error + .as_service_error() + .expect("request failure should retain an S3 service error"); + assert_eq!(service_error.code(), Some(code), "{context}: unexpected error code: {error:?}"); + assert_eq!(service_error.message(), Some(message), "{context}: unexpected error message: {error:?}"); +} + pub async fn kms_admin_request( base_url: &str, method: http::Method, @@ -559,7 +581,13 @@ pub async fn test_error_scenarios(s3_client: &Client, bucket: &str) -> Result<() .send() .await; - assert!(wrong_key_result.is_err(), "Download with wrong SSE-C key should fail"); + assert_s3_error( + wrong_key_result, + 400, + "InvalidRequest", + SSE_C_KEY_MISMATCH_MESSAGE, + "download with a wrong SSE-C key must be rejected", + ); info!("✅ Correctly rejected download with wrong SSE-C key"); info!("Error scenario tests completed successfully"); diff --git a/crates/e2e_test/src/kms/kms_comprehensive_test.rs b/crates/e2e_test/src/kms/kms_comprehensive_test.rs index 9ec1087a3..d4bbf604a 100644 --- a/crates/e2e_test/src/kms/kms_comprehensive_test.rs +++ b/crates/e2e_test/src/kms/kms_comprehensive_test.rs @@ -19,9 +19,9 @@ //! complex workflows. use super::common::{ - EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, create_sse_c_config, sse_customer_key_md5_base64, - test_all_multipart_encryption_types, test_kms_key_management, test_multipart_upload_with_config, test_sse_c_encryption, - test_sse_kms_encryption, test_sse_s3_encryption, + EncryptionType, LocalKMSTestEnvironment, MultipartTestConfig, SSE_C_KEY_MISMATCH_MESSAGE, assert_s3_error, + create_sse_c_config, sse_customer_key_md5_base64, test_all_multipart_encryption_types, test_kms_key_management, + test_multipart_upload_with_config, test_sse_c_encryption, test_sse_kms_encryption, test_sse_s3_encryption, }; use crate::common::{TEST_BUCKET, init_logging}; use tracing::info; @@ -191,7 +191,13 @@ async fn test_comprehensive_key_isolation() -> Result<(), Box) -> String { - let mut hasher = Md5::new(); - hasher.update(input.as_ref()); - hex::encode(hasher.finalize()) -} - /// Test encryption of zero-byte files (empty files) #[tokio::test] async fn test_kms_zero_byte_file_encryption() -> Result<(), Box> { @@ -295,7 +288,7 @@ async fn test_kms_invalid_key_scenarios() -> Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box Result<(), Box info!("✅ Multipart upload aborted successfully"), - Err(e) => warn!("⚠️ Failed to abort multipart upload: {}", e), - } + .await?; + info!("✅ Multipart upload aborted successfully"); // Try to complete the aborted upload - this should fail info!("🔍 Attempting to complete aborted upload"); @@ -310,18 +340,38 @@ async fn test_kms_multipart_upload_interruption() -> Result<(), Box Result<(), Box current-build on-disk compatibility. +#[cfg(test)] +mod upgrade_compatibility_test; + // Receiver-side replication LWW (rustfs/backlog#1953): stale inbound // replication metadata must not overwrite a newer local category state. #[cfg(test)] diff --git a/crates/e2e_test/src/negative_sigv4_test.rs b/crates/e2e_test/src/negative_sigv4_test.rs index a6a440383..2a2a02fd8 100644 --- a/crates/e2e_test/src/negative_sigv4_test.rs +++ b/crates/e2e_test/src/negative_sigv4_test.rs @@ -34,6 +34,7 @@ //! rejected header-SigV4 requests. use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; +use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use rustfs_signer::constants::UNSIGNED_PAYLOAD; use rustfs_signer::request_signature_v4::{SIGN_V4_ALGORITHM, get_scope, get_signature, get_signing_key}; @@ -280,7 +281,8 @@ async fn tampered_payload_is_rejected() -> Result<(), Box Result<(), Box { let status = resp.status(); let body = resp.text().await.unwrap_or_default(); - assert_ne!(status.as_u16(), 200, "payload mismatch must not succeed, body:\n{body}"); assert!( - status.is_client_error() || status.is_server_error(), - "payload mismatch must be an error status, got {status}, body:\n{body}" + status.is_client_error(), + "payload mismatch must be rejected with a client error, got {status}, body:\n{body}" ); info!(%status, "tampered payload rejected with error status"); } // A mid-stream hash-mismatch abort surfacing as a transport error is // also a valid rejection (definitely not a 200 success). - Err(err) => info!(%err, "tampered payload rejected via transport error"), + Err(err) => { + assert!(!err.is_connect(), "connection failure is not proof of payload rejection: {err}"); + assert!(!err.is_timeout(), "request timeout is not proof of payload rejection: {err}"); + info!(%err, "tampered payload rejected via mid-stream transport error"); + } } + + let absent = env + .create_s3_client() + .get_object() + .bucket(BUCKET) + .key(key) + .send() + .await + .expect_err("a tampered payload must not publish an object"); + assert_eq!(absent.raw_response().map(|response| response.status().as_u16()), Some(404)); + assert_eq!(absent.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey")); Ok(()) } diff --git a/crates/e2e_test/src/object_lock/common.rs b/crates/e2e_test/src/object_lock/common.rs index f63fefd3b..b402fa687 100644 --- a/crates/e2e_test/src/object_lock/common.rs +++ b/crates/e2e_test/src/object_lock/common.rs @@ -23,6 +23,7 @@ use aws_sdk_s3::Client; use aws_sdk_s3::error::SdkError; use aws_sdk_s3::operation::delete_object::DeleteObjectError; +use aws_sdk_s3::operation::put_object_retention::PutObjectRetentionError; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{ DefaultRetention, ObjectLockConfiguration, ObjectLockEnabled, ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockMode, @@ -182,11 +183,8 @@ pub async fn put_object_retention( mode: ObjectLockRetentionMode, retain_until: DateTime, bypass_governance: bool, -) -> Result<(), Box> { - // AWS SDK requires UTC time without timezone offset (e.g., "2026-01-24T11:20:14Z") - let retain_until_str = retain_until.format("%Y-%m-%dT%H:%M:%SZ").to_string(); - let retain_until_datetime = - aws_sdk_s3::primitives::DateTime::from_str(&retain_until_str, aws_sdk_s3::primitives::DateTimeFormat::DateTime)?; +) -> Result<(), Box>> { + let retain_until_datetime = aws_sdk_s3::primitives::DateTime::from_secs(retain_until.timestamp()); let retention = ObjectLockRetention::builder() .mode(mode.clone()) @@ -204,7 +202,7 @@ pub async fn put_object_retention( request = request.version_id(vid); } - request.send().await?; + request.send().await.map_err(Box::new)?; info!("Put object retention on {} with mode {:?}", key, mode); Ok(()) } diff --git a/crates/e2e_test/src/object_lock/object_lock_test.rs b/crates/e2e_test/src/object_lock/object_lock_test.rs index ddc733fbc..64792567e 100644 --- a/crates/e2e_test/src/object_lock/object_lock_test.rs +++ b/crates/e2e_test/src/object_lock/object_lock_test.rs @@ -1475,7 +1475,7 @@ async fn test_put_retention_compliance_cannot_shorten() { ) .await; - assert!(shorten_result.is_err(), "Shortening COMPLIANCE retention should fail"); + assert_access_denied(shorten_result, "Shortening COMPLIANCE retention should fail"); info!("✅ Test passed: Cannot shorten COMPLIANCE retention"); } @@ -1598,10 +1598,7 @@ async fn test_put_retention_governance_shorten_requires_bypass() { ) .await; - assert!( - shorten_without_bypass.is_err(), - "Shortening GOVERNANCE retention without bypass should fail" - ); + assert_access_denied(shorten_without_bypass, "Shortening GOVERNANCE retention without bypass should fail"); // Shorten with bypass - should succeed let shorten_with_bypass = put_object_retention( diff --git a/crates/e2e_test/src/quota_test.rs b/crates/e2e_test/src/quota_test.rs index d6f5b6e8e..7acb51e02 100644 --- a/crates/e2e_test/src/quota_test.rs +++ b/crates/e2e_test/src/quota_test.rs @@ -14,6 +14,7 @@ use crate::common::{RustFSTestEnvironment, admin_request, awscurl_delete, awscurl_get, awscurl_post, awscurl_put, init_logging}; use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; use http::{Method, StatusCode}; use tokio::time::{Duration, sleep, timeout}; use tracing::{debug, info}; @@ -132,19 +133,13 @@ impl QuotaTestEnv { pub async fn object_exists(&self, key: &str) -> Result> { match self.client.head_object().bucket(&self.bucket_name).key(key).send().await { Ok(_) => Ok(true), - Err(e) => { - // Check for any 404-related errors and return false instead of propagating - let error_str = e.to_string(); - if error_str.contains("404") || error_str.contains("Not Found") || error_str.contains("NotFound") { + Err(error) => { + let status = error.raw_response().map(|response| response.status().as_u16()); + let code = error.as_service_error().and_then(ProvideErrorMetadata::code); + if status == Some(404) && matches!(code, Some("NotFound" | "NoSuchKey")) { Ok(false) } else { - // Also check the error code directly - if let Some(service_err) = e.as_service_error() - && service_err.is_not_found() - { - return Ok(false); - } - Err(e.into()) + Err(error.into()) } } } @@ -278,7 +273,46 @@ impl QuotaTestEnv { #[cfg(test)] mod integration_tests { use super::*; - use aws_sdk_s3::error::ProvideErrorMetadata; + + fn assert_error_response(status: StatusCode, body: &str, expected_status: StatusCode, expected_code: &str) { + assert_eq!(status, expected_status, "unexpected error status: {status} {body}"); + assert!( + body.contains(&format!("{expected_code}")), + "expected {expected_code}, got: {body}" + ); + } + + fn assert_quota_rejection(status: Option, service_error: Option<&E>, error: &impl std::fmt::Debug) + where + E: ProvideErrorMetadata + std::fmt::Debug, + { + assert_eq!(status, Some(400), "quota rejection must return HTTP 400: {error:?}"); + let service_error = service_error.expect("quota rejection must be an S3 service error"); + assert_eq!(service_error.code(), Some("InvalidRequest"), "unexpected quota error: {error:?}"); + assert!( + service_error + .message() + .is_some_and(|message| message.starts_with("Bucket quota exceeded")), + "operation must fail specifically at quota admission: {error:?}" + ); + } + + async fn assert_put_rejected_by_quota(env: &QuotaTestEnv, key: &str, size_bytes: usize) { + let error = env + .client + .put_object() + .bucket(&env.bucket_name) + .key(key) + .body(aws_sdk_s3::primitives::ByteStream::from(vec![0u8; size_bytes])) + .send() + .await + .expect_err("PUT above quota must be rejected"); + assert_quota_rejection( + error.raw_response().map(|response| response.status().as_u16()), + error.as_service_error(), + &error, + ); + } #[tokio::test] async fn test_quota_basic_operations() -> Result<(), Box> { @@ -304,8 +338,7 @@ mod integration_tests { assert!(env.object_exists("test2.txt").await?); // Try to upload 1KB more (should fail due to quota) - let upload_result = env.upload_object("test3.txt", 1024).await; - assert!(upload_result.is_err()); + assert_put_rejected_by_quota(&env, "test3.txt", 1024).await; assert!(!env.object_exists("test3.txt").await?); // Clean up @@ -356,10 +389,10 @@ mod integration_tests { let err = put_aws_chunked("over-quota.bin", 16 * 1024) .await .expect_err("declared aws-chunked PUT over quota must be rejected"); - let err_debug = format!("{err:?}"); - assert!( - !err_debug.contains("UnexpectedContent"), - "over-quota rejection must be the quota error, not UnexpectedContent: {err_debug}" + assert_quota_rejection( + err.raw_response().map(|response| response.status().as_u16()), + err.as_service_error(), + &err, ); assert!(!env.object_exists("over-quota.bin").await?); @@ -581,24 +614,35 @@ mod integration_tests { env.create_bucket().await?; // Test invalid quota type - let url = format!("{}/rustfs/admin/v3/quota/{}", env.env.url, env.bucket_name); + let quota_path = format!("/rustfs/admin/v3/quota/{}", env.bucket_name); let invalid_config = serde_json::json!({ "quota": 1024, "quota_type": "SOFT" // Invalid type }); - let response = awscurl_put(&url, &invalid_config.to_string(), &env.env.access_key, &env.env.secret_key).await; - assert!(response.is_err()); - let error_msg = response.unwrap_err().to_string(); - assert!(error_msg.contains("InvalidArgument")); + let (status, body) = admin_request( + &env.env.url, + Method::PUT, + "a_path, + Some(invalid_config.to_string()), + &env.env.access_key, + &env.env.secret_key, + ) + .await?; + assert_error_response(status, &body, StatusCode::BAD_REQUEST, "InvalidArgument"); // Test operations on non-existent bucket - let url = format!("{}/rustfs/admin/v3/quota/non-existent-bucket", env.env.url); - let response = awscurl_get(&url, &env.env.access_key, &env.env.secret_key).await; - assert!(response.is_err()); - let error_msg = response.unwrap_err().to_string(); - assert!(error_msg.contains("NoSuchBucket")); + let (status, body) = admin_request( + &env.env.url, + Method::GET, + "/rustfs/admin/v3/quota/non-existent-bucket", + None, + &env.env.access_key, + &env.env.secret_key, + ) + .await?; + assert_error_response(status, &body, StatusCode::NOT_FOUND, "NoSuchBucket"); env.cleanup_bucket().await?; @@ -652,10 +696,16 @@ mod integration_tests { "quota": 1024, "quota_type": "SOFT" }); - let response = awscurl_put(&url, &invalid_config.to_string(), &env.env.access_key, &env.env.secret_key).await; - assert!(response.is_err()); - let error_msg = response.unwrap_err().to_string(); - assert!(error_msg.contains("InvalidArgument")); + let (status, body) = admin_request( + &env.env.url, + Method::PUT, + &format!("/rustfs/admin/v3/quota/{}", env.bucket_name), + Some(invalid_config.to_string()), + &env.env.access_key, + &env.env.secret_key, + ) + .await?; + assert_error_response(status, &body, StatusCode::BAD_REQUEST, "InvalidArgument"); env.cleanup_bucket().await?; @@ -698,26 +748,21 @@ mod integration_tests { assert!(resp.contains("quota_limit")); // Normal user sets quota — should be denied - let set_error = awscurl_put( - &get_url, - &serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string(), + let quota_path = format!("/rustfs/admin/v3/quota/{}", env.bucket_name); + let (status, body) = admin_request( + &env.env.url, + Method::PUT, + "a_path, + Some(serde_json::json!({"quota": 2048, "quota_type": "HARD"}).to_string()), normal_ak, normal_sk, ) - .await - .expect_err("normal user should not be able to set quota") - .to_string(); - assert!(set_error.contains("AccessDenied"), "quota denial must return AccessDenied: {set_error}"); + .await?; + assert_error_response(status, &body, StatusCode::FORBIDDEN, "AccessDenied"); // Normal user clears quota — should be denied - let delete_error = awscurl_delete(&get_url, normal_ak, normal_sk) - .await - .expect_err("normal user should not be able to clear quota") - .to_string(); - assert!( - delete_error.contains("AccessDenied"), - "quota deletion denial must return AccessDenied: {delete_error}" - ); + let (status, body) = admin_request(&env.env.url, Method::DELETE, "a_path, None, normal_ak, normal_sk).await?; + assert_error_response(status, &body, StatusCode::FORBIDDEN, "AccessDenied"); env.cleanup_bucket().await?; Ok(()) @@ -757,7 +802,12 @@ mod integration_tests { .send() .await; - assert!(copy_result.is_err()); + let copy_error = copy_result.expect_err("copy above quota must be rejected"); + assert_quota_rejection( + copy_error.raw_response().map(|response| response.status().as_u16()), + copy_error.as_service_error(), + ©_error, + ); assert!(!env.object_exists("copy2.txt").await?); env.cleanup_bucket().await?; @@ -780,8 +830,7 @@ mod integration_tests { env.upload_object("file2.txt", 1024 * 1024).await?; // Verify quota is full - let upload_result = env.upload_object("file3.txt", 1024).await; - assert!(upload_result.is_err()); + assert_put_rejected_by_quota(&env, "file3.txt", 1024).await; // Delete multiple objects using batch delete let objects = vec![ @@ -881,9 +930,7 @@ mod integration_tests { // Test 2: Multipart upload exceeds quota (should fail) // Upload 6MB filler (total now: 5MB + 6MB = 11MB > 10MB quota) - let upload_filler = env.upload_object("filler.txt", 6 * 1024 * 1024).await; - // This should fail due to quota - assert!(upload_filler.is_err()); + assert_put_rejected_by_quota(&env, "filler.txt", 6 * 1024 * 1024).await; // Verify filler doesn't exist assert!(!env.object_exists("filler.txt").await?); @@ -939,7 +986,11 @@ mod integration_tests { .await; let complete_error = complete_result.expect_err("multipart completion above quota must be rejected"); - assert_eq!(complete_error.as_service_error().and_then(|error| error.code()), Some("InvalidRequest")); + assert_quota_rejection( + complete_error.raw_response().map(|response| response.status().as_u16()), + complete_error.as_service_error(), + &complete_error, + ); assert!(!env.object_exists("over_quota.txt").await?); let staged_parts = env diff --git a/crates/e2e_test/src/replication_extension_test.rs b/crates/e2e_test/src/replication_extension_test.rs index 206bd50e2..479fbeeed 100644 --- a/crates/e2e_test/src/replication_extension_test.rs +++ b/crates/e2e_test/src/replication_extension_test.rs @@ -20,7 +20,10 @@ use crate::fake_s3_target::{ FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, RequestRecord, }; -use crate::kms::common::{create_key_with_specific_id, sse_customer_key_md5_base64}; +use crate::kms::common::{ + SSE_C_KEY_MISMATCH_MESSAGE, SSE_C_MISSING_PARAMETERS_MESSAGE, assert_s3_error, create_key_with_specific_id, + sse_customer_key_md5_base64, +}; use crate::storage_api::replication_extension::BucketTargetSys; use aws_sdk_s3::config::{Credentials, Region}; use aws_sdk_s3::error::ProvideErrorMetadata; @@ -4375,7 +4378,13 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult { // Without the customer key the replica must not be readable — the direct // detection point for a silent-plaintext replica (backlog#1291). let plain_read = target_client.get_object().bucket(&target_bucket).key(key).send().await; - assert!(plain_read.is_err(), "SSE-C replica must not be readable without the customer key"); + assert_s3_error( + plain_read, + 400, + "InvalidRequest", + SSE_C_MISSING_PARAMETERS_MESSAGE, + "SSE-C replica must not be readable without the customer key", + ); // A wrong customer key must fail too. let wrong_key = BASE64_STANDARD.encode("99999999999999999999999999999999"); @@ -4389,7 +4398,13 @@ async fn test_bucket_replication_sse_c_contract() -> TestResult { .sse_customer_key_md5(&wrong_key_md5) .send() .await; - assert!(wrong_read.is_err(), "SSE-C replica must reject a wrong customer key"); + assert_s3_error( + wrong_read, + 400, + "InvalidRequest", + SSE_C_KEY_MISMATCH_MESSAGE, + "SSE-C replica must reject a wrong customer key", + ); Ok(()) } @@ -4486,9 +4501,12 @@ async fn test_bucket_replication_sse_c_multipart_passthrough() -> TestResult { assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), payload.as_slice()); let plain_read = target_client.get_object().bucket(&target_bucket).key(key).send().await; - assert!( - plain_read.is_err(), - "SSE-C multipart replica must not be readable without the customer key" + assert_s3_error( + plain_read, + 400, + "InvalidRequest", + SSE_C_MISSING_PARAMETERS_MESSAGE, + "SSE-C multipart replica must not be readable without the customer key", ); // Stability across scanner cycles: convergence must hold for passthrough. @@ -4892,15 +4910,12 @@ async fn test_bucket_replication_sse_c_existing_object_resync() -> TestResult { assert_eq!(replica.body.collect().await?.into_bytes().as_ref(), body.as_slice()); // No plaintext leak: the replica stays unreadable without the key. - assert!( - target_client - .get_object() - .bucket(target_bucket) - .key(key) - .send() - .await - .is_err(), - "SSE-C replica must not be readable without the customer key" + assert_s3_error( + target_client.get_object().bucket(target_bucket).key(key).send().await, + 400, + "InvalidRequest", + SSE_C_MISSING_PARAMETERS_MESSAGE, + "SSE-C resynced replica must not be readable without the customer key", ); Ok(()) diff --git a/crates/e2e_test/src/special_chars_test.rs b/crates/e2e_test/src/special_chars_test.rs index 18d929668..d72ba7d2c 100644 --- a/crates/e2e_test/src/special_chars_test.rs +++ b/crates/e2e_test/src/special_chars_test.rs @@ -28,6 +28,7 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging, local_http_client}; use aws_sdk_s3::Client; + use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use http::StatusCode; use http::header::HOST; @@ -731,12 +732,7 @@ mod tests { create_bucket(&client, bucket).await.expect("Failed to create bucket"); // Test that control characters are rejected - let invalid_keys = vec![ - "file\0with\0null.txt", - "file\nwith\nnewline.txt", - "file\rwith\rcarriage.txt", - "file\twith\ttab.txt", // Tab might be allowed, but let's test - ]; + let invalid_keys = ["file\0with\0null.txt", "file\nwith\nnewline.txt", "file\rwith\rcarriage.txt"]; for key in invalid_keys { info!("Testing rejection of control character in key: {:?}", key); @@ -747,18 +743,28 @@ mod tests { .key(key) .body(ByteStream::from_static(b"test")) .send() - .await; - - // Note: The validation happens on the server side, so we expect an error - // For null byte, newline, and carriage return - if key.contains('\0') || key.contains('\n') || key.contains('\r') { - assert!(result.is_err(), "Control character should be rejected for key: {key:?}"); - if let Err(e) = result { - info!("✅ Control character correctly rejected: {:?}", e); - } - } + .await + .expect_err("invalid control characters must be rejected by the server"); + assert_eq!( + result.raw_response().map(|response| response.status().as_u16()), + Some(400), + "control character must return HTTP 400 for key {key:?}: {result:?}" + ); + assert_eq!( + result.as_service_error().and_then(ProvideErrorMetadata::code), + Some("InvalidArgument"), + "control character must return InvalidArgument for key {key:?}: {result:?}" + ); } + let listed = client + .list_objects_v2() + .bucket(bucket) + .send() + .await + .expect("server must remain healthy after rejected requests"); + assert!(listed.contents().is_empty(), "rejected requests must not create objects"); + // Cleanup env.stop_server(); info!("Test completed successfully"); diff --git a/crates/e2e_test/src/upgrade_compatibility_test.rs b/crates/e2e_test/src/upgrade_compatibility_test.rs new file mode 100644 index 000000000..3485f9593 --- /dev/null +++ b/crates/e2e_test/src/upgrade_compatibility_test.rs @@ -0,0 +1,254 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use crate::common::{RustFSTestEnvironment, init_logging}; +use aws_sdk_s3::Client; +use aws_sdk_s3::error::ProvideErrorMetadata; +use aws_sdk_s3::primitives::ByteStream; +use aws_sdk_s3::types::{ + BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, ServerSideEncryption, VersioningConfiguration, +}; +use std::path::PathBuf; + +type TestResult = Result<(), Box>; + +const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY"; +const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY"; +const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI="; +const PLAIN_BUCKET: &str = "upgrade-plain-data"; +const VERSIONED_BUCKET: &str = "upgrade-versioned-data"; + +fn source_binary() -> Result> { + let path = std::env::var_os(SOURCE_BINARY_ENV) + .map(PathBuf::from) + .ok_or("RUSTFS_UPGRADE_SOURCE_BINARY must point to the pinned previous release binary")?; + if !path.is_file() { + return Err(format!("upgrade source binary does not exist: {}", path.display()).into()); + } + Ok(path) +} + +async fn enable_versioning(client: &Client, bucket: &str) -> TestResult { + let configuration = VersioningConfiguration::builder() + .status(BucketVersioningStatus::Enabled) + .build(); + client + .put_bucket_versioning() + .bucket(bucket) + .versioning_configuration(configuration) + .send() + .await?; + Ok(()) +} + +async fn read_object( + client: &Client, + bucket: &str, + key: &str, + version_id: Option<&str>, +) -> Result<(Option, Vec), Box> { + let mut request = client.get_object().bucket(bucket).key(key); + if let Some(version_id) = version_id { + request = request.version_id(version_id); + } + let response = request.send().await?; + let encryption = response.server_side_encryption().cloned(); + let body = response.body.collect().await?.into_bytes().to_vec(); + Ok((encryption, body)) +} + +async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec]) -> TestResult { + let created = client.create_multipart_upload().bucket(bucket).key(key).send().await?; + let upload_id = created.upload_id().ok_or("CreateMultipartUpload omitted upload ID")?; + let mut completed_parts = Vec::with_capacity(parts.len()); + + for (index, part) in parts.iter().enumerate() { + let part_number = i32::try_from(index + 1)?; + let uploaded = client + .upload_part() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .part_number(part_number) + .body(ByteStream::from(part.clone())) + .send() + .await?; + completed_parts.push( + CompletedPart::builder() + .part_number(part_number) + .e_tag(uploaded.e_tag().ok_or("UploadPart omitted ETag")?) + .build(), + ); + } + + client + .complete_multipart_upload() + .bucket(bucket) + .key(key) + .upload_id(upload_id) + .multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed_parts)).build()) + .send() + .await?; + Ok(()) +} + +#[tokio::test] +#[ignore = "requires a pinned previous RustFS release binary"] +async fn direct_upgrade_from_rc2_preserves_object_contracts() -> TestResult { + init_logging(); + let previous_binary = source_binary()?; + let mut env = RustFSTestEnvironment::new().await?; + let server_env = [(SSE_MASTER_KEY_ENV, SSE_MASTER_KEY)]; + env.start_rustfs_server_from_binary(&previous_binary, vec![], &server_env) + .await?; + + let old_client = env.create_s3_client(); + env.create_test_bucket(PLAIN_BUCKET).await?; + env.create_test_bucket(VERSIONED_BUCKET).await?; + enable_versioning(&old_client, VERSIONED_BUCKET).await?; + + let plain_key = "plain-object"; + let plain_bytes = b"written by the previous RustFS release"; + old_client + .put_object() + .bucket(PLAIN_BUCKET) + .key(plain_key) + .body(ByteStream::from_static(plain_bytes)) + .send() + .await?; + + let encrypted_key = "sse-s3-object"; + let encrypted_bytes = b"encrypted by the previous RustFS release"; + old_client + .put_object() + .bucket(PLAIN_BUCKET) + .key(encrypted_key) + .server_side_encryption(ServerSideEncryption::Aes256) + .body(ByteStream::from_static(encrypted_bytes)) + .send() + .await?; + + let multipart_key = "multipart-object"; + let multipart_parts = vec![vec![b'a'; 5 * 1024 * 1024], b"final multipart bytes".to_vec()]; + let multipart_bytes = multipart_parts.concat(); + write_multipart(&old_client, PLAIN_BUCKET, multipart_key, &multipart_parts).await?; + + let versioned_key = "versioned-object"; + let version1_bytes = b"version one from the previous release"; + let version1 = old_client + .put_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .body(ByteStream::from_static(version1_bytes)) + .send() + .await? + .version_id() + .ok_or("first versioned PUT omitted version ID")? + .to_string(); + let version2_bytes = b"version two from the previous release"; + let version2 = old_client + .put_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .body(ByteStream::from_static(version2_bytes)) + .send() + .await? + .version_id() + .ok_or("second versioned PUT omitted version ID")? + .to_string(); + let deleted = old_client + .delete_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .send() + .await?; + assert_eq!(deleted.delete_marker(), Some(true)); + let delete_marker = deleted + .version_id() + .ok_or("versioned DELETE omitted delete marker version ID")? + .to_string(); + + env.restart_server_preserving_data(vec![], &server_env).await?; + let current_client = env.create_s3_client(); + + assert_eq!(read_object(¤t_client, PLAIN_BUCKET, plain_key, None).await?.1, plain_bytes); + + let (encryption, upgraded_encrypted_bytes) = read_object(¤t_client, PLAIN_BUCKET, encrypted_key, None).await?; + assert_eq!(encryption, Some(ServerSideEncryption::Aes256)); + assert_eq!(upgraded_encrypted_bytes, encrypted_bytes); + + assert_eq!(read_object(¤t_client, PLAIN_BUCKET, multipart_key, None).await?.1, multipart_bytes); + + assert_eq!( + read_object(¤t_client, VERSIONED_BUCKET, versioned_key, Some(&version1)) + .await? + .1, + version1_bytes + ); + assert_eq!( + read_object(¤t_client, VERSIONED_BUCKET, versioned_key, Some(&version2)) + .await? + .1, + version2_bytes + ); + + let current_read = current_client + .get_object() + .bucket(VERSIONED_BUCKET) + .key(versioned_key) + .send() + .await + .expect_err("the previous release's delete marker must remain current after upgrade"); + assert_eq!(current_read.raw_response().map(|response| response.status().as_u16()), Some(404)); + assert_eq!(current_read.as_service_error().and_then(ProvideErrorMetadata::code), Some("NoSuchKey")); + + let listed = current_client + .list_object_versions() + .bucket(VERSIONED_BUCKET) + .prefix(versioned_key) + .send() + .await?; + assert_eq!(listed.versions().len(), 2); + assert!( + listed + .versions() + .iter() + .any(|version| version.version_id() == Some(version1.as_str())) + ); + assert!( + listed + .versions() + .iter() + .any(|version| version.version_id() == Some(version2.as_str())) + ); + assert_eq!(listed.delete_markers().len(), 1); + assert_eq!(listed.delete_markers()[0].version_id(), Some(delete_marker.as_str())); + assert_eq!(listed.delete_markers()[0].is_latest(), Some(true)); + + let post_upgrade_key = "written-after-upgrade"; + let post_upgrade_bytes = b"written by the current RustFS build"; + current_client + .put_object() + .bucket(PLAIN_BUCKET) + .key(post_upgrade_key) + .body(ByteStream::from_static(post_upgrade_bytes)) + .send() + .await?; + assert_eq!( + read_object(¤t_client, PLAIN_BUCKET, post_upgrade_key, None).await?.1, + post_upgrade_bytes + ); + + Ok(()) +} diff --git a/crates/e2e_test/src/version_id_regression_test.rs b/crates/e2e_test/src/version_id_regression_test.rs index 75d19e5db..f75fcad87 100644 --- a/crates/e2e_test/src/version_id_regression_test.rs +++ b/crates/e2e_test/src/version_id_regression_test.rs @@ -25,6 +25,7 @@ mod tests { use crate::common::{RustFSTestEnvironment, init_logging}; use aws_sdk_s3::Client; + use aws_sdk_s3::error::ProvideErrorMetadata; use aws_sdk_s3::primitives::ByteStream; use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration}; use tracing::info; @@ -285,7 +286,10 @@ mod tests { let output = result.unwrap(); info!("📥 PutObject response - version_id: {:?}", output.version_id); - // version_id can be None or Some("null") for non-versioned buckets + assert!( + output.version_id().is_none() || output.version_id() == Some("null"), + "non-versioned PUT must omit version ID or return the S3 null version" + ); info!("✅ PASSED: PutObject works correctly without versioning"); } @@ -317,7 +321,11 @@ mod tests { .send() .await; assert!(put_result.is_ok(), "PUT operation failed"); - let _version_id = put_result.unwrap().version_id; + let version_id = put_result + .unwrap() + .version_id() + .expect("versioned PUT should return a version ID") + .to_string(); // Test GET info!("📥 Testing GET operation"); @@ -341,16 +349,46 @@ mod tests { // Test DELETE info!("🗑️ Testing DELETE operation"); - let delete_result = client.delete_object().bucket(bucket).key(key).send().await; - assert!(delete_result.is_ok(), "DELETE operation failed"); + let delete_result = client + .delete_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect("DELETE operation failed"); + assert_eq!(delete_result.delete_marker(), Some(true)); + let delete_marker_version_id = delete_result + .version_id() + .expect("versioned DELETE should return a delete marker version ID") + .to_string(); - // Verify object is deleted (should return NoSuchKey or version marker) - let get_after_delete = client.get_object().bucket(bucket).key(key).send().await; - assert!( - get_after_delete.is_err() || get_after_delete.unwrap().delete_marker == Some(true), - "Object should be deleted or have delete marker" + let get_after_delete = client + .get_object() + .bucket(bucket) + .key(key) + .send() + .await + .expect_err("the current delete marker must hide the object"); + assert_eq!(get_after_delete.raw_response().map(|response| response.status().as_u16()), Some(404)); + assert_eq!( + get_after_delete.as_service_error().and_then(ProvideErrorMetadata::code), + Some("NoSuchKey") ); + let versions = client + .list_object_versions() + .bucket(bucket) + .prefix(key) + .send() + .await + .expect("ListObjectVersions failed after DELETE"); + assert_eq!(versions.versions().len(), 1); + assert_eq!(versions.versions()[0].version_id(), Some(version_id.as_str())); + assert_eq!(versions.versions()[0].is_latest(), Some(false)); + assert_eq!(versions.delete_markers().len(), 1); + assert_eq!(versions.delete_markers()[0].version_id(), Some(delete_marker_version_id.as_str())); + assert_eq!(versions.delete_markers()[0].is_latest(), Some(true)); + info!("✅ PASSED: All basic S3 operations work correctly"); } @@ -417,31 +455,59 @@ mod tests { let client = env.create_s3_client(); env.create_test_bucket(bucket).await?; + enable_versioning(&client, bucket).await?; let key = "terraform.tfstate"; - let response = client + let first_version = client .put_object() .bucket(bucket) .key(key) .body(ByteStream::from(b"v1".to_vec())) .send() - .await; - assert!(response.is_ok()); + .await? + .version_id() + .ok_or("first Terraform state PUT omitted version ID")? + .to_string(); - client.delete_object().bucket(bucket).key(key).send().await?; + let deleted = client.delete_object().bucket(bucket).key(key).send().await?; + assert_eq!(deleted.delete_marker(), Some(true)); + let delete_marker = deleted + .version_id() + .ok_or("Terraform state DELETE omitted delete marker version ID")? + .to_string(); - let response = client + let second_version = client .put_object() .bucket(bucket) .key(key) - .body(ByteStream::from(b"v1".to_vec())) + .body(ByteStream::from(b"v2".to_vec())) .send() - .await; + .await? + .version_id() + .ok_or("second Terraform state PUT omitted version ID")? + .to_string(); - assert!(response.is_ok()); + let get_response = client.get_object().bucket(bucket).key(key).send().await?; + let current_body = get_response.body.collect().await?.into_bytes(); + assert_eq!(current_body.as_ref(), b"v2"); - let get_response = client.get_object().bucket(bucket).key(key).send().await; - assert!(get_response.is_ok(), "Object should exist after PUT"); + let listed = client.list_object_versions().bucket(bucket).prefix(key).send().await?; + assert_eq!(listed.versions().len(), 2); + assert!( + listed + .versions() + .iter() + .any(|version| version.version_id() == Some(first_version.as_str()) && version.is_latest() == Some(false)) + ); + assert!( + listed + .versions() + .iter() + .any(|version| version.version_id() == Some(second_version.as_str()) && version.is_latest() == Some(true)) + ); + assert_eq!(listed.delete_markers().len(), 1); + assert_eq!(listed.delete_markers()[0].version_id(), Some(delete_marker.as_str())); + assert_eq!(listed.delete_markers()[0].is_latest(), Some(false)); Ok(()) } diff --git a/crates/ecstore/src/erasure/codec/buffer_pool.rs b/crates/ecstore/src/erasure/codec/buffer_pool.rs new file mode 100644 index 000000000..23c68edb6 --- /dev/null +++ b/crates/ecstore/src/erasure/codec/buffer_pool.rs @@ -0,0 +1,94 @@ +//! General-purpose buffer pool for reducing Vec allocations. +//! +//! This pool reuses Vec buffers to avoid repeated heap allocations +//! in hot paths like EC encoding/decoding and data read/write. +//! +//! Current integration: bitrot.rs (bitrot_verify path) +//! Future integration: decode.rs, encode.rs + +use std::sync::Mutex; + +/// A thread-safe pool of reusable Vec buffers. +pub(crate) struct BufferPool { + buckets: Mutex>>>, + max_per_bucket: usize, +} + +impl BufferPool { + pub(crate) fn with_limits(max_per_bucket: usize) -> Self { + let buckets = (0..32).map(|_| Vec::new()).collect(); + Self { + buckets: Mutex::new(buckets), + max_per_bucket, + } + } + + pub(crate) fn get(&self, min_capacity: usize) -> Vec { + let bucket = self.bucket_for_capacity(min_capacity); + let mut buckets = self.buckets.lock().unwrap(); + if let Some(buf) = buckets[bucket].pop() { + return buf; + } + drop(buckets); + Vec::with_capacity(min_capacity.next_power_of_two().max(min_capacity)) + } + + pub(crate) fn put(&self, mut buf: Vec) { + if buf.is_empty() { + return; + } + let bucket = self.bucket_for_capacity(buf.capacity()); + buf.clear(); + let mut buckets = self.buckets.lock().unwrap(); + if buckets[bucket].len() < self.max_per_bucket { + buckets[bucket].push(buf); + } + } + + fn bucket_for_capacity(&self, capacity: usize) -> usize { + if capacity == 0 { + return 0; + } + let rounded = capacity.next_power_of_two(); + (usize::BITS - rounded.leading_zeros() - 1) as usize + } +} + +static EC_BUFFER_POOL: std::sync::LazyLock = std::sync::LazyLock::new(|| BufferPool::with_limits(16)); + +pub(crate) fn get_ec_buffer(min_capacity: usize) -> Vec { + EC_BUFFER_POOL.get(min_capacity) +} + +pub(crate) fn return_ec_buffer(buf: Vec) { + EC_BUFFER_POOL.put(buf); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_buffer_pool_basic() { + let pool = BufferPool::with_limits(16); + let buf = pool.get(1024); + assert!(buf.capacity() >= 1024); + pool.put(buf); + let buf2 = pool.get(1024); + assert!(buf2.capacity() >= 1024); + } + + #[test] + fn test_buffer_pool_different_sizes() { + let pool = BufferPool::with_limits(16); + let buf1 = pool.get(100); + let buf2 = pool.get(1000); + let buf3 = pool.get(10000); + pool.put(buf1); + pool.put(buf2); + pool.put(buf3); + let _ = pool.get(100); + let _ = pool.get(1000); + let _ = pool.get(10000); + } +} diff --git a/crates/ecstore/src/erasure/codec/mod.rs b/crates/ecstore/src/erasure/codec/mod.rs index cdd2dd043..e8dde3bdc 100644 --- a/crates/ecstore/src/erasure/codec/mod.rs +++ b/crates/ecstore/src/erasure/codec/mod.rs @@ -13,4 +13,5 @@ // limitations under the License. pub(crate) mod bridge; +pub(crate) mod buffer_pool; pub(crate) mod workspace; diff --git a/crates/ecstore/src/erasure/coding/bitrot.rs b/crates/ecstore/src/erasure/coding/bitrot.rs index 947bbb68e..e950644ef 100644 --- a/crates/ecstore/src/erasure/coding/bitrot.rs +++ b/crates/ecstore/src/erasure/coding/bitrot.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::erasure::codec::buffer_pool::{get_ec_buffer, return_ec_buffer}; use pin_project_lite::pin_project; use rustfs_utils::HashAlgorithm; use std::future::poll_fn; @@ -635,11 +636,15 @@ pub async fn bitrot_verify( shard_size = left; } - let mut buf = vec![0; shard_size]; + let mut buf = get_ec_buffer(shard_size); + buf.resize(shard_size, 0); let read = r.read_exact(&mut buf).await?; let actual_hash = algo.hash_encode(&buf); - if actual_hash.as_ref() != &hash_buf[0..n] { + let hash_ok = actual_hash.as_ref() == &hash_buf[0..n]; + drop(actual_hash); // 释放借用 + return_ec_buffer(buf); + if !hash_ok { return Err(std::io::Error::other("bitrot hash mismatch")); } diff --git a/crates/ecstore/src/memory_observability.rs b/crates/ecstore/src/memory_observability.rs new file mode 100644 index 000000000..bebf7fdce --- /dev/null +++ b/crates/ecstore/src/memory_observability.rs @@ -0,0 +1,33 @@ + +/// Check mimalloc arena configuration and log diagnostics +pub fn log_mimalloc_diagnostics() { + #[cfg(feature = "mimalloc")] + { + use rustfs_mimalloc::MiMalloc; + + // Check arena_max_object_size + let arena_max_obj_size = MiMalloc::option_get_size( + rustfs_mimalloc_sys::mi_option_t::mi_option_arena_max_object_size + ); + tracing::info!( + arena_max_object_size_bytes = arena_max_obj_size, + "mimalloc arena_max_object_size" + ); + + // Check if pagemap is enabled + let pagemap_commit = MiMalloc::option_is_enabled( + rustfs_mimalloc_sys::mi_option_t::mi_option_pagemap_commit + ); + tracing::info!( + pagemap_commit = pagemap_commit, + "mimalloc pagemap_commit" + ); + + // Log version + let version = MiMalloc::version(); + tracing::info!( + mimalloc_version = version, + "mimalloc version" + ); + } +} diff --git a/crates/ecstore/src/set_disk/ops/object.rs b/crates/ecstore/src/set_disk/ops/object.rs index f2203aaf1..d23b85e92 100644 --- a/crates/ecstore/src/set_disk/ops/object.rs +++ b/crates/ecstore/src/set_disk/ops/object.rs @@ -7332,6 +7332,8 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks { commit_opts.no_lock = true; commit_opts.metadata_cache_safe = false; commit_opts.include_part_checksums = true; + // Note: Using clone() here is necessary because ObjectOptions has 124 fields. + // Future optimization: Consider using Cow or a builder pattern. let transition_lock_guard = if opts.no_lock { None } else { diff --git a/crates/heal/tests/heal_b920_subquorum_union_test.rs b/crates/heal/tests/heal_b920_subquorum_union_test.rs index d2af21e33..b38a90620 100644 --- a/crates/heal/tests/heal_b920_subquorum_union_test.rs +++ b/crates/heal/tests/heal_b920_subquorum_union_test.rs @@ -234,6 +234,7 @@ mod serial_tests { create_versioned_bucket(&ecstore, bucket).await; let v1 = put_versioned(&ecstore, bucket, object, &versioned_test_data(1)).await; + wait_for_object_copies(&disk_paths, bucket, object).await; // Wipe the object entirely on disks 1..4, leaving it on ONLY disk[0] // (1/4 disks < read-quorum 2). effective listing_quorum for 4 drives is @@ -387,6 +388,7 @@ mod serial_tests { let data_v1 = versioned_test_data(9); let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; + wait_for_object_copies(&disk_paths, bucket, object).await; // EC4+4: wipe the object ENTIRELY on 4 disks, leaving full copies on the // other 4 (== data_blocks). Meta quorum (4) still holds, so this heals via @@ -500,6 +502,7 @@ mod serial_tests { let data_v1 = versioned_test_data(11); let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; + wait_for_object_copies(&disk_paths, bucket, object).await; // Drop the object entirely on ONE disk (its shard + meta gone), leaving it // on 3/4 (>= data_blocks 2). The union must still enumerate it. @@ -545,6 +548,7 @@ mod serial_tests { let data_v1 = versioned_test_data(12); let v1 = put_versioned(&ecstore, bucket, object, &data_v1).await; + wait_for_object_copies(&disk_paths, bucket, object).await; std::fs::remove_dir_all(object_dir(&disk_paths[3], bucket, object)).expect("wipe object on disk3"); for disk in &disk_paths[..3] { diff --git a/crates/heal/tests/heal_integration_test.rs b/crates/heal/tests/heal_integration_test.rs index 21a0ab78b..940a69f73 100644 --- a/crates/heal/tests/heal_integration_test.rs +++ b/crates/heal/tests/heal_integration_test.rs @@ -56,6 +56,32 @@ async fn wait_for_path_exists(path: &Path, timeout: Duration, interval: Duration } } +fn find_part_file(obj_dir: &Path) -> Option { + WalkDir::new(obj_dir) + .min_depth(2) + .max_depth(2) + .into_iter() + .filter_map(Result::ok) + .find(|entry| entry.file_type().is_file() && entry.file_name().to_str().is_some_and(|name| name.starts_with("part."))) + .map(|entry| entry.into_path()) +} + +async fn wait_for_object_copies(disks: &[PathBuf], bucket: &str, object: &str) { + tokio::time::timeout(HEAL_FORMAT_WAIT_TIMEOUT, async { + loop { + if disks.iter().all(|disk| { + let obj_dir = disk.join(bucket).join(object); + obj_dir.join("xl.meta").exists() && find_part_file(&obj_dir).is_some() + }) { + break; + } + tokio::time::sleep(HEAL_FORMAT_WAIT_INTERVAL).await; + } + }) + .await + .expect("PUT rename tails must converge before corrupting the disk fixture"); +} + /// Test helper: build the shared 4-disk temp-dir ECStore environment /// (rustfs-test-utils, backlog#1153 infra-1) and wrap it in the heal storage /// layer. Port 0 + uuid temp dirs keep this parallel-safe under nextest. @@ -103,18 +129,10 @@ mod serial_tests { create_test_bucket(&ecstore, bucket_name).await; upload_test_object(&ecstore, bucket_name, object_name, &test_data).await; - let _obj_dir = disk_paths[0].join(bucket_name).join(object_name); + wait_for_object_copies(&disk_paths, bucket_name, object_name).await; // ─── 1️⃣ delete single data shard file ───────────────────────────────────── let obj_dir = disk_paths[0].join(bucket_name).join(object_name); - // find part file at depth 2, e.g. ...//part.1 - let target_part = WalkDir::new(&obj_dir) - .min_depth(2) - .max_depth(2) - .into_iter() - .filter_map(Result::ok) - .find(|e| e.file_type().is_file() && e.file_name().to_str().map(|n| n.starts_with("part.")).unwrap_or(false)) - .map(|e| e.into_path()) - .expect("Failed to locate part file to delete"); + let target_part = find_part_file(&obj_dir).expect("converged fixture must contain a part file"); std::fs::remove_file(&target_part).expect("failed to delete part file"); assert!(!target_part.exists()); @@ -171,6 +189,7 @@ mod serial_tests { create_test_bucket(&ecstore, bucket_name).await; upload_test_object(&ecstore, bucket_name, object_name, &test_data).await; + wait_for_object_copies(&disk_paths, bucket_name, object_name).await; // Plant a leaked, unreferenced UUID data dir under the object on every disk // that actually holds the object (i.e. has an `xl.meta`). Planting on a disk @@ -381,15 +400,9 @@ mod serial_tests { create_test_bucket(&ecstore, bucket_name).await; upload_test_object(&ecstore, bucket_name, object_name, &test_data).await; + wait_for_object_copies(&disk_paths, bucket_name, object_name).await; let obj_dir = disk_paths[0].join(bucket_name).join(object_name); - let target_part = WalkDir::new(&obj_dir) - .min_depth(2) - .max_depth(2) - .into_iter() - .filter_map(Result::ok) - .find(|e| e.file_type().is_file() && e.file_name().to_str().map(|n| n.starts_with("part.")).unwrap_or(false)) - .map(|e| e.into_path()) - .expect("Failed to locate part file to delete"); + let target_part = find_part_file(&obj_dir).expect("converged fixture must contain a part file"); // ─── 1️⃣ delete format.json on one disk ────────────── let format_path = disk_paths[0].join(".rustfs.sys").join("format.json"); diff --git a/crates/iam/src/oidc.rs b/crates/iam/src/oidc.rs index 05e8f33f2..fc287bb4a 100644 --- a/crates/iam/src/oidc.rs +++ b/crates/iam/src/oidc.rs @@ -2273,6 +2273,7 @@ pub(crate) fn test_config(id: &str) -> OidcProviderConfig { #[cfg(test)] mod tests { use super::*; + use jsonwebtoken::{Algorithm, EncodingKey, Header}; use rustfs_utils::egress::OutboundDnsPolicyRejection; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -2641,12 +2642,15 @@ mod tests { ) } - fn start_mock_oidc_discovery_server( + fn start_mock_oidc_discovery_server_with_jwks( build_discovery_issuer: F, max_requests: usize, + signing_alg: &'static str, + jwks_response: J, ) -> Option<(String, std::thread::JoinHandle<()>)> where F: Fn(&str) -> (String, String, String) + Send + 'static, + J: Fn(usize) -> String + Send + 'static, { use std::io::Write; use std::net::{Shutdown, TcpListener}; @@ -2674,10 +2678,9 @@ mod tests { "response_types_supported": ["code"], "response_modes_supported": ["query"], "subject_types_supported": ["public"], - "id_token_signing_alg_values_supported": ["RS256"], + "id_token_signing_alg_values_supported": [signing_alg], }) .to_string(); - let jwks_body = r#"{"keys":[]}"#; let (ready_tx, ready_rx) = mpsc::channel(); let handle = std::thread::spawn(move || { @@ -2687,6 +2690,7 @@ mod tests { let _ = ready_tx.send(()); let mut seen = 0usize; + let mut jwks_fetches = 0usize; let start = Instant::now(); let mut last_completed = Instant::now(); @@ -2719,7 +2723,11 @@ mod tests { .expect("failed to set discovery mock read timeout"); let path = read_mock_oidc_request_path(&mut stream); - let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, jwks_body); + let jwks_body = jwks_response(jwks_fetches); + if path == expected_jwks_path { + jwks_fetches += 1; + } + let response = mock_oidc_response(&path, &discovery_body, &expected_jwks_path, &jwks_body); let _ = stream.write_all(response.as_bytes()); let _ = stream.flush(); let _ = stream.shutdown(Shutdown::Both); @@ -2737,6 +2745,94 @@ mod tests { Some((base, handle)) } + fn start_mock_oidc_discovery_server( + build_discovery_issuer: F, + max_requests: usize, + ) -> Option<(String, std::thread::JoinHandle<()>)> + where + F: Fn(&str) -> (String, String, String) + Send + 'static, + { + start_mock_oidc_discovery_server_with_jwks(build_discovery_issuer, max_requests, "RS256", |_| { + r#"{"keys":[]}"#.to_string() + }) + } + + fn oidc_es256_key_and_jwk(kid: &str) -> (EncodingKey, serde_json::Value) { + let certified = + rcgen::generate_simple_self_signed(vec![format!("{kid}.invalid")]).expect("OIDC signing key should generate"); + let encoding_key = + EncodingKey::from_ec_pem(certified.signing_key.serialize_pem().as_bytes()).expect("OIDC signing key should encode"); + let mut jwk = jsonwebtoken::jwk::Jwk::from_encoding_key(&encoding_key, Algorithm::ES256) + .expect("OIDC public JWK should derive from signing key"); + jwk.common.key_id = Some(kid.to_string()); + jwk.common.public_key_use = Some(jsonwebtoken::jwk::PublicKeyUse::Signature); + (encoding_key, serde_json::to_value(jwk).expect("OIDC JWK should serialize")) + } + + #[tokio::test] + async fn web_identity_verification_refreshes_rotated_jwks() { + let (_, initial_jwk) = oidc_es256_key_and_jwk("initial"); + let (rotated_key, rotated_jwk) = oidc_es256_key_and_jwk("rotated"); + let initial_jwks = serde_json::json!({ "keys": [initial_jwk] }).to_string(); + let rotated_jwks = serde_json::json!({ "keys": [rotated_jwk] }).to_string(); + let Some((base, handle)) = start_mock_oidc_discovery_server_with_jwks( + |base| (base.to_string(), format!("{base}/jwks"), "/jwks".to_string()), + 4, + "ES256", + move |fetch| { + if fetch == 0 { + initial_jwks.clone() + } else { + rotated_jwks.clone() + } + }, + ) else { + return; + }; + + let config = build_mocked_oidc_provider_config("rotating", &base); + let policy = OutboundPolicy::from_allowed_origins(&base).expect("loopback origin should be allowed"); + let http_client = ReqwestHttpClient::with_policy(policy); + let state = OidcSys::discover_provider(&config, &http_client) + .await + .expect("initial OIDC discovery should succeed"); + let sys = OidcSys { + configs: HashMap::from([(config.id.clone(), config.clone())]), + provider_states: RwLock::new(HashMap::from([(config.id.clone(), state)])), + state_store: OidcStateStore::new(), + http_client, + }; + + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .expect("system clock should be after Unix epoch") + .as_secs(); + let mut header = Header::new(Algorithm::ES256); + header.kid = Some("rotated".to_string()); + let token = jsonwebtoken::encode( + &header, + &serde_json::json!({ + "iss": base, + "sub": "rotated-user", + "aud": config.client_id, + "iat": now, + "exp": now + 300, + "groups": ["readwrite"], + }), + &rotated_key, + ) + .expect("rotated OIDC token should sign"); + + let (claims, provider_id) = sys + .verify_web_identity_token(&token) + .await + .expect("verification should refresh JWKS and accept the rotated key"); + assert_eq!(provider_id, "rotating"); + assert_eq!(claims.sub, "rotated-user"); + assert_eq!(claims.groups, vec!["readwrite"]); + handle.join().expect("rotating JWKS mock server should exit cleanly"); + } + fn start_mock_oidc_tls_discovery_server( build_discovery_issuer: F, max_requests: usize, diff --git a/deploy/config/rustfs.env b/deploy/config/rustfs.env index c16ecd95e..95dfb6ff5 100644 --- a/deploy/config/rustfs.env +++ b/deploy/config/rustfs.env @@ -20,6 +20,13 @@ RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 # RUSTFS_SERVER_DOMAINS=s3.example.com # Optional RustFS license content # RUSTFS_LICENSE=REPLACE_WITH_LICENSE_CONTENT +# Allocator reclaim is enabled by default to return freed allocator pages to +# the OS after idle samples. Set to false only for latency-sensitive profiles +# that have measured a benefit from keeping allocator pages resident. +# RUSTFS_ALLOCATOR_RECLAIM_ENABLED=false +# RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS=30 +# RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS=3 +# RUSTFS_ALLOCATOR_RECLAIM_FORCE=true # Observability configuration endpoint: RUSTFS_OBS_ENDPOINT RUSTFS_OBS_ENDPOINT=http://localhost:4318 # Optional TLS certificates directory path: deploy/certs diff --git a/fuzz/README.md b/fuzz/README.md index 9d24efed5..2698ba256 100644 --- a/fuzz/README.md +++ b/fuzz/README.md @@ -68,10 +68,18 @@ FUZZ_TARGET=path_containment ./scripts/fuzz/run.sh # Nightly-style: 300s per target MAX_TOTAL_TIME=300 ./scripts/fuzz/run.sh +# Replay a recorded libFuzzer seed +FUZZ_TARGET=path_containment FUZZ_SEED=123456789 ./scripts/fuzz/run.sh + # Skip build (use pre-built harness) SKIP_BUILD=1 FUZZ_TARGET=local_metadata ./scripts/fuzz/run.sh ``` +Each run writes `fuzz/artifacts//run-manifest.txt` with the target, +libFuzzer seed, time budget, Git revision and dirty state, and runner mode. CI +uploads that manifest with the corpus and any crash input so the exact run can +be replayed. + ## CI Workflow The GitHub Actions workflow (`.github/workflows/fuzz.yml`) uses a **build/run separation** pattern: diff --git a/rustfs/src/allocator_reclaim.rs b/rustfs/src/allocator_reclaim.rs index 5c2853de2..f2983e9ac 100644 --- a/rustfs/src/allocator_reclaim.rs +++ b/rustfs/src/allocator_reclaim.rs @@ -18,8 +18,8 @@ //! GET, scanner, and heal workloads, mimalloc can retain freed pages in process //! heaps for later reuse instead of immediately returning them to the OS. That //! behavior is usually good for latency, but it can make process RSS look high -//! after a workload has gone idle. This module provides an opt-in background -//! loop that waits for a configurable idle window and then asks the allocator to +//! after a workload has gone idle. This module provides a configurable +//! background loop that waits for an idle window and then asks the allocator to //! collect retained memory. //! //! The loop is intentionally conservative: @@ -41,7 +41,7 @@ use metrics::{counter, gauge, histogram}; use serde::Serialize; use std::time::Duration; use tokio_util::sync::CancellationToken; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; const ALLOCATOR_RECLAIM_SERVICE_NAME: &str = "allocator_reclaim"; @@ -238,8 +238,9 @@ fn reclaimable_work_snapshot() -> ReclaimableWorkSnapshot { /// Read the startup enablement switch. /// -/// The code default is disabled. Local developer scripts may choose to export -/// the variable as enabled for their own launch profile. +/// The code default is enabled so direct binary, container, Helm, and local +/// script launches share the same reclaim contract. Operators can still set +/// `RUSTFS_ALLOCATOR_RECLAIM_ENABLED=false` for latency-sensitive deployments. fn configured_allocator_reclaim_enabled() -> bool { rustfs_utils::get_env_bool( rustfs_config::ENV_ALLOCATOR_RECLAIM_ENABLED, @@ -421,15 +422,27 @@ pub fn init_allocator_reclaim(ctx: CancellationToken) { gauge!("rustfs_memory_allocator_reclaim_enabled").set(if enabled { 1.0 } else { 0.0 }); counter!("rustfs_memory_allocator_backend_info", "backend" => backend.to_string()).increment(1); - if !enabled { - debug!("allocator reclaim loop disabled"); - return; - } - let configured_force = configured_allocator_reclaim_force(); let force = effective_allocator_reclaim_force(backend, configured_force); let idle_intervals = configured_allocator_reclaim_idle_intervals(); let interval = Duration::from_secs(configured_allocator_reclaim_interval_secs()); + info!( + event = "allocator_reclaim_configured", + component = "runtime", + subsystem = "memory", + state = if enabled { "enabled" } else { "disabled" }, + backend, + configured_force, + effective_force = force, + idle_intervals, + interval_secs = interval.as_secs(), + "allocator reclaim configured" + ); + + if !enabled { + debug!("allocator reclaim loop disabled"); + return; + } tokio::spawn(async move { let mut ticker = tokio::time::interval(interval); diff --git a/rustfs/src/cgroup_resources.rs b/rustfs/src/cgroup_resources.rs index cce2aa150..4dc8aaf19 100644 --- a/rustfs/src/cgroup_resources.rs +++ b/rustfs/src/cgroup_resources.rs @@ -62,6 +62,8 @@ pub struct ContainerResources { pub cgroup_detected: bool, /// Whether values were overridden by environment variables. pub overridden: bool, + /// Pre-computed basis string for metrics ("cgroup" or "host"). + pub basis: &'static str, } impl Default for ContainerResources { @@ -71,6 +73,7 @@ impl Default for ContainerResources { memory_bytes: 0, cgroup_detected: false, overridden: false, + basis: "host", } } } @@ -205,7 +208,7 @@ mod cgroup { /// Detection priority: /// 1. Environment variable overrides /// 2. cgroup v1/v2 limits (Linux only) -/// 3. Host values from sysinfo +/// 3. Host values from sysinfo (single System instance for both CPU and memory) fn detect_container_resources() -> ContainerResources { // Check if cgroup detection is disabled let cgroup_disabled = std::env::var(ENV_DISABLE_CGROUP_DETECTION) @@ -234,30 +237,25 @@ fn detect_container_resources() -> ContainerResources { let overridden = override_cores.is_some() || override_memory.is_some(); - // Get host values for fallback - let host_cores = { - let mut sys = - sysinfo::System::new_with_specifics(sysinfo::RefreshKind::everything().without_memory().without_processes()); + // Get host values from a single sysinfo::System instance (avoids double init) + let (host_cores, host_memory) = { + let mut sys = sysinfo::System::new_with_specifics(sysinfo::RefreshKind::everything().without_processes()); sys.refresh_cpu_all(); - sys.cpus().len().max(1) - }; - - let host_memory = { - let mut sys = sysinfo::System::new(); sys.refresh_memory(); - sys.total_memory() + (sys.cpus().len().max(1), sys.total_memory()) }; // Determine effective values: override > cgroup > host let cpu_cores = override_cores.or(cgroup_cpus).unwrap_or(host_cores).max(1); - let memory_bytes = override_memory.or(cgroup_memory).unwrap_or(host_memory); + let basis = if cgroup_detected { "cgroup" } else { "host" }; ContainerResources { cpu_cores, memory_bytes, cgroup_detected, overridden, + basis, } } @@ -273,12 +271,13 @@ pub fn container_resources() -> &'static ContainerResources { /// Should be called once during startup to help operators verify detection. pub fn log_container_resources() { let res = container_resources(); + let memory_mib = res.memory_bytes / (1024 * 1024); if res.overridden { tracing::info!( cpu_cores = res.cpu_cores, memory_bytes = res.memory_bytes, - memory_mib = res.memory_bytes / (1024 * 1024), + memory_mib, cgroup_detected = res.cgroup_detected, "container resources (overridden by environment variables)" ); @@ -286,7 +285,7 @@ pub fn log_container_resources() { tracing::info!( cpu_cores = res.cpu_cores, memory_bytes = res.memory_bytes, - memory_mib = res.memory_bytes / (1024 * 1024), + memory_mib, "container resources (detected from cgroup)" ); } else { @@ -298,12 +297,6 @@ pub fn log_container_resources() { } } -/// Get the memory basis string for metrics. -pub fn memory_basis() -> &'static str { - let res = container_resources(); - if res.cgroup_detected { "cgroup" } else { "host" } -} - // ============================================================================ // Tests // ============================================================================ @@ -319,6 +312,7 @@ mod tests { assert_eq!(resources.memory_bytes, 0); assert!(!resources.cgroup_detected); assert!(!resources.overridden); + assert_eq!(resources.basis, "host"); } #[test] diff --git a/rustfs/src/config/cli.rs b/rustfs/src/config/cli.rs index 172101911..91bb098b4 100644 --- a/rustfs/src/config/cli.rs +++ b/rustfs/src/config/cli.rs @@ -302,6 +302,11 @@ pub struct TlsInspectOpts { /// Server subcommand options #[derive(Args, Clone)] +#[command(after_help = "Allocator reclaim environment: + RUSTFS_ALLOCATOR_RECLAIM_ENABLED=true|false Enable allocator page reclaim after idle samples (default: true) + RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS=30 Sampling interval in seconds + RUSTFS_ALLOCATOR_RECLAIM_FORCE=true|false Request forceful collection when supported + RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS=3 Consecutive idle samples required before reclaim")] pub struct ServerOpts { /// DIR points to a directory on a filesystem. #[arg( @@ -612,4 +617,23 @@ mod tests { assert_eq!(help.kind(), ErrorKind::DisplayHelp); assert!(help.to_string().contains("Unix only")); } + + #[test] + fn server_help_lists_allocator_reclaim_environment() { + let result = Cli::try_parse_from(["rustfs", "server", "--help"]); + let Err(help) = result else { + panic!("help exits without parsing server options"); + }; + + assert_eq!(help.kind(), ErrorKind::DisplayHelp); + let help = help.to_string(); + for env in [ + "RUSTFS_ALLOCATOR_RECLAIM_ENABLED", + "RUSTFS_ALLOCATOR_RECLAIM_INTERVAL_SECS", + "RUSTFS_ALLOCATOR_RECLAIM_FORCE", + "RUSTFS_ALLOCATOR_RECLAIM_IDLE_INTERVALS", + ] { + assert!(help.contains(env), "server help should mention {env}"); + } + } } diff --git a/rustfs/src/connect/config.rs b/rustfs/src/connect/config.rs index 391d4ef45..8ecc6f04b 100644 --- a/rustfs/src/connect/config.rs +++ b/rustfs/src/connect/config.rs @@ -14,6 +14,7 @@ use std::env; use std::ffi::OsString; +#[cfg(target_os = "linux")] use std::fs; use std::path::PathBuf; use std::time::Duration; @@ -63,16 +64,37 @@ impl HeartbeatConfig { credential_store: CredentialStore, state_path: impl Into, ) -> Self { + let state_path = state_path.into(); Self { endpoint: endpoint.into(), root_ca_pem: root_ca_pem.into(), identity_store, credential_store, - state_path: state_path.into(), + state_path, schedule: HeartbeatSchedule::default(), } } + #[cfg(any(target_os = "linux", test))] + pub(crate) fn state_only(state_root: PathBuf) -> Self { + Self { + endpoint: String::new(), + root_ca_pem: Vec::new(), + identity_store: IdentityStore::new(state_root.join("identity")), + credential_store: CredentialStore::new(state_root.join("credential")), + state_path: state_root.join("heartbeat/state.json"), + schedule: HeartbeatSchedule::default(), + } + } + + pub(crate) fn transport_enabled(&self) -> bool { + !self.endpoint.is_empty() + } + + pub(crate) fn state_root(&self) -> Option<&std::path::Path> { + self.state_path.parent().and_then(std::path::Path::parent) + } + pub fn from_env() -> Result, HeartbeatConfigError> { Self::from_env_values( env::var_os(ENV_CONNECT_ENDPOINT), @@ -90,19 +112,33 @@ impl HeartbeatConfig { if !configured { return Ok(None); } - let (Some(endpoint), Some(root_ca_file), Some(state_dir)) = (endpoint, root_ca_file, state_dir) else { + let Some(state_dir) = state_dir else { return Err(HeartbeatConfigError::Partial); }; - let endpoint = endpoint.into_string().map_err(|_| HeartbeatConfigError::EndpointEncoding)?; - let root_ca_file = PathBuf::from(root_ca_file); let state_dir = PathBuf::from(state_dir); - if endpoint.is_empty() || root_ca_file.as_os_str().is_empty() || state_dir.as_os_str().is_empty() { + if state_dir.as_os_str().is_empty() || endpoint.is_some() != root_ca_file.is_some() { return Err(HeartbeatConfigError::Partial); } + #[cfg(not(target_os = "linux"))] + return Err(HeartbeatConfigError::PlatformSecurity); + #[cfg(target_os = "linux")] + let (Some(endpoint), Some(root_ca_file)) = (endpoint, root_ca_file) else { + return Ok(Some(Self::state_only(state_dir))); + }; + #[cfg(target_os = "linux")] + let endpoint = endpoint.into_string().map_err(|_| HeartbeatConfigError::EndpointEncoding)?; + #[cfg(target_os = "linux")] + let root_ca_file = PathBuf::from(root_ca_file); + #[cfg(target_os = "linux")] + if endpoint.is_empty() || root_ca_file.as_os_str().is_empty() { + return Err(HeartbeatConfigError::Partial); + } + #[cfg(target_os = "linux")] let root_ca_pem = fs::read(&root_ca_file).map_err(|source| HeartbeatConfigError::RootCertificate { path: root_ca_file, source, })?; + #[cfg(target_os = "linux")] Ok(Some(Self::new( endpoint, root_ca_pem, @@ -116,17 +152,19 @@ impl HeartbeatConfig { #[derive(Debug, thiserror::Error)] pub enum HeartbeatConfigError { #[error( - "Connect heartbeat configuration requires RUSTFS_CONNECT_ENDPOINT, RUSTFS_CONNECT_ROOT_CA_FILE, and RUSTFS_CONNECT_STATE_DIR" + "Connect requires RUSTFS_CONNECT_STATE_DIR and either both or neither of RUSTFS_CONNECT_ENDPOINT and RUSTFS_CONNECT_ROOT_CA_FILE" )] Partial, #[error("RUSTFS_CONNECT_ENDPOINT is not valid UTF-8")] EndpointEncoding, - #[error("failed to read the Connect root CA at {path}: {source}")] + #[error("Connect root CA could not be read")] RootCertificate { path: PathBuf, #[source] source: std::io::Error, }, + #[error("Connect inventory persistence requires Linux filesystem security guarantees")] + PlatformSecurity, } #[cfg(test)] @@ -149,9 +187,34 @@ mod tests { HeartbeatConfig::from_env_values(Some(OsString::from("https://connect.example/agent/")), None, None), Err(HeartbeatConfigError::Partial) )); + assert!(matches!( + HeartbeatConfig::from_env_values( + Some(OsString::from("https://connect.example/agent/")), + Some(OsString::from("root.pem")), + None, + ), + Err(HeartbeatConfigError::Partial) + )); + assert!(matches!( + HeartbeatConfig::from_env_values(None, Some(OsString::from("root.pem")), Some(OsString::from("state"))), + Err(HeartbeatConfigError::Partial) + )); } #[test] + #[cfg(target_os = "linux")] + fn state_directory_alone_enables_local_inventory_without_transport() { + let state = tempfile::tempdir().expect("tempdir").keep(); + let config = HeartbeatConfig::from_env_values(None, None, Some(state.clone().into_os_string())) + .expect("state-only config") + .expect("enabled config"); + + assert_eq!(config.state_root(), Some(state.as_path())); + assert!(!config.transport_enabled()); + } + + #[test] + #[cfg(target_os = "linux")] fn complete_environment_builds_the_durable_paths() { let temp = tempfile::tempdir().expect("tempdir"); let root = temp.path().join("root.pem"); @@ -168,6 +231,24 @@ mod tests { assert_eq!(config.endpoint, "https://connect.example/agent/"); assert_eq!(config.root_ca_pem, b"root certificate"); assert_eq!(config.state_path, state.join("heartbeat/state.json")); + assert_eq!(config.state_root(), Some(state.as_path())); assert!(!state.exists(), "parsing configuration must not create state"); } + + #[test] + #[cfg(not(target_os = "linux"))] + fn configured_inventory_fails_without_linux_filesystem_guarantees() { + assert!(matches!( + HeartbeatConfig::from_env_values(None, None, Some(OsString::from("state"))), + Err(HeartbeatConfigError::PlatformSecurity) + )); + assert!(matches!( + HeartbeatConfig::from_env_values( + Some(OsString::from("https://connect.example/agent/")), + Some(OsString::from("missing-root.pem")), + Some(OsString::from("state")), + ), + Err(HeartbeatConfigError::PlatformSecurity) + )); + } } diff --git a/rustfs/src/connect/inventory.rs b/rustfs/src/connect/inventory.rs index 5a5943d2f..6238a581a 100644 --- a/rustfs/src/connect/inventory.rs +++ b/rustfs/src/connect/inventory.rs @@ -14,8 +14,16 @@ use std::collections::BTreeSet; use std::fs; -use std::io::{self, Write as _}; -use std::path::{Path, PathBuf}; +#[cfg(any(target_os = "linux", test))] +use std::io; +#[cfg(target_os = "linux")] +use std::io::Read as _; +#[cfg(target_os = "linux")] +use std::io::Write as _; +use std::path::Path; +#[cfg(target_os = "linux")] +use std::sync::Arc; +#[cfg(target_os = "linux")] use std::sync::atomic::{AtomicU64, Ordering}; use std::time::Duration; @@ -37,10 +45,34 @@ const RUSTFS_VERSION: &str = concat!( const HASH_PREFIX: &[u8] = b"rustfs-connect/agent/v1/inventory-snapshot\n"; const MAX_SEQUENCE: u64 = 9_007_199_254_740_991; const MAX_SAFE_INTEGER: u64 = 9_007_199_254_740_991; -#[cfg(unix)] +const ENVELOPE_FORMAT_VERSION: &str = "v1"; +const ENVELOPE_HASH_PREFIX: &[u8] = b"rustfs-connect-inventory-envelope-v1"; +#[cfg(target_os = "linux")] +const MAX_PERSISTED_BYTES: usize = 16 * 1024; +#[allow(dead_code)] // Kept for the crate-private stopped-server reader consumed by R06. +const MAX_FUTURE_SKEW: Duration = Duration::from_secs(5 * 60); +#[cfg(target_os = "linux")] const FILE_MODE: u32 = 0o600; +#[cfg(target_os = "linux")] static STAGING_SEQUENCE: AtomicU64 = AtomicU64::new(0); +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields, rename_all = "camelCase")] +struct InventoryEnvelope { + format_version: String, + captured_at: String, + snapshot: InventorySnapshot, + envelope_hash: String, +} + +#[derive(Debug, PartialEq, Eq)] +#[allow(dead_code)] // This is the intentionally narrow handoff to R06. +pub(crate) struct PersistedInventory { + pub(crate) snapshot: InventorySnapshot, + pub(crate) captured_at: String, + pub(crate) age: Duration, +} + #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct InventorySchedule { pub cadence: Duration, @@ -257,6 +289,7 @@ impl PendingInventory { } } + #[cfg(target_os = "linux")] fn is_valid(&self) -> bool { self.protocol_version == PROTOCOL_VERSION && self.sequence <= MAX_SEQUENCE @@ -268,13 +301,17 @@ impl PendingInventory { fn content_hash(&self) -> Result { self.snapshot.content_hash() } + + pub(crate) fn snapshot(&self) -> &InventorySnapshot { + &self.snapshot + } } pub(crate) enum InventoryDelivery { Accepted { content_hash: String, received_at: String }, Retry { retry_after: Option }, - AuthenticationStopped { status: u16, reason: Option }, - Rejected { status: u16, reason: Option }, + AuthenticationStopped { status: u16 }, + Rejected { status: u16 }, } pub(crate) struct InventorySender { @@ -318,17 +355,50 @@ impl InventorySender { }) } TelemetryDelivery::Retry { retry_after } => Ok(InventoryDelivery::Retry { retry_after }), - TelemetryDelivery::AuthenticationStopped { status, reason } => { - Ok(InventoryDelivery::AuthenticationStopped { status, reason }) - } - TelemetryDelivery::Rejected { status, reason } => Ok(InventoryDelivery::Rejected { status, reason }), + TelemetryDelivery::AuthenticationStopped { status, .. } => Ok(InventoryDelivery::AuthenticationStopped { status }), + TelemetryDelivery::Rejected { status, .. } => Ok(InventoryDelivery::Rejected { status }), } } } #[derive(Clone)] pub(crate) struct InventoryStateStore { - path: PathBuf, + #[cfg(target_os = "linux")] + directory: Arc, + #[cfg(target_os = "linux")] + state_root: Arc, +} + +#[cfg(target_os = "linux")] +struct StateRootAnchor { + root: fs::File, + components: Vec<(std::ffi::OsString, fs::File)>, +} + +#[cfg(target_os = "linux")] +impl StateRootAnchor { + fn state_root(&self) -> Result<&fs::File, InventoryError> { + self.components + .last() + .map(|(_, directory)| directory) + .ok_or(InventoryError::StatePath) + } + + fn validate(&self) -> Result<(), InventoryError> { + validate_directory(&self.root, false)?; + let mut current = None; + for (index, (component, expected)) in self.components.iter().enumerate() { + let parent = current.as_ref().unwrap_or(&self.root); + let resolved = open_directory_component_at(parent, component)?; + let dedicated = index + 1 == self.components.len(); + validate_directory(&resolved, dedicated)?; + if file_identity(expected)? != file_identity(&resolved)? { + return Err(InventoryError::PersistenceSecurity); + } + current = Some(resolved); + } + Ok(()) + } } #[derive(Default, Serialize, Deserialize)] @@ -340,49 +410,63 @@ struct InventoryState { } impl InventoryStateStore { - pub(crate) fn from_heartbeat_path(path: &Path) -> Result { - let root = path.parent().and_then(Path::parent).ok_or(InventoryError::StatePath)?; - Ok(Self { - path: root.join("inventory/state.json"), - }) + pub(crate) fn from_state_root(path: &Path) -> Result { + #[cfg(not(target_os = "linux"))] + { + let _ = path; + Err(InventoryError::PlatformSecurity) + } + #[cfg(target_os = "linux")] + { + let (state_root, directory) = open_inventory_directory(path)?; + Ok(Self { + directory: Arc::new(directory), + state_root: Arc::new(state_root), + }) + } } pub(crate) fn try_runtime_lock(&self) -> Result { - let directory = parent(&self.path)?; - prepare_inventory_directory(directory)?; - let name = filename(&self.path)?; - let path = directory.join(format!(".{name}.lock")); - let mut options = fs::OpenOptions::new(); - options.create(true).truncate(false).read(true).write(true); - #[cfg(unix)] + #[cfg(not(target_os = "linux"))] { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(FILE_MODE); + Err(InventoryError::PlatformSecurity) + } + #[cfg(target_os = "linux")] + { + self.validate_anchor()?; + let lock = open_file_at(&self.directory, ".state.json.lock", true, true)?; + validate_regular_file(&lock)?; + lock.try_lock().map_err(|_| InventoryError::AlreadyRunning)?; + self.validate_anchor()?; + Ok(lock) } - let lock = options.open(&path).map_err(|source| state_io(&path, source))?; - check_mode(&path)?; - lock.try_lock().map_err(|_| InventoryError::AlreadyRunning)?; - Ok(lock) } - pub(crate) async fn pending(&self) -> Result, InventoryError> { + pub(crate) async fn pending(&self) -> Result, InventoryError> { let store = self.clone(); tokio::task::spawn_blocking(move || { - let state = store.read()?; + let (state, persisted_at) = store.read_with_persisted_at()?; if state.pending.is_none() && state.next_sequence > MAX_SEQUENCE { return Err(InventoryError::SequenceExhausted); } - Ok(state.pending) + state + .pending + .map(|pending| { + persisted_at + .map(|persisted_at| (pending, persisted_at)) + .ok_or(InventoryError::StateCorrupt) + }) + .transpose() }) .await - .map_err(|source| state_io(&self.path, io::Error::other(source)))? + .map_err(|_| InventoryError::StateIo)? } pub(crate) async fn prepare(&self, snapshot: InventorySnapshot) -> Result, InventoryError> { let store = self.clone(); tokio::task::spawn_blocking(move || store.prepare_sync(snapshot)) .await - .map_err(|source| state_io(&self.path, io::Error::other(source)))? + .map_err(|_| InventoryError::StateIo)? } pub(crate) async fn mark_accepted(&self, accepted: &PendingInventory) -> Result<(), InventoryError> { @@ -390,7 +474,94 @@ impl InventoryStateStore { let accepted = accepted.clone(); tokio::task::spawn_blocking(move || store.mark_accepted_sync(&accepted)) .await - .map_err(|source| state_io(&self.path, io::Error::other(source)))? + .map_err(|_| InventoryError::StateIo)? + } + + pub(crate) async fn publish_latest( + &self, + snapshot: InventorySnapshot, + captured_at: String, + shutdown: tokio_util::sync::CancellationToken, + ) -> Result<(), InventoryError> { + let store = self.clone(); + tokio::task::spawn_blocking(move || store.publish_latest_sync(snapshot, captured_at, &shutdown)) + .await + .map_err(|_| InventoryError::StateIo)? + } + + pub(crate) async fn ensure_latest( + &self, + snapshot: InventorySnapshot, + captured_at: String, + shutdown: tokio_util::sync::CancellationToken, + ) -> Result<(), InventoryError> { + let store = self.clone(); + tokio::task::spawn_blocking(move || match store.read_latest(chrono::Utc::now()) { + Ok(_) => Ok(()), + Err(InventoryError::StateMissing) => store.publish_latest_sync(snapshot, captured_at, &shutdown), + Err(error) => Err(error), + }) + .await + .map_err(|_| InventoryError::StateIo)? + } + + #[allow(dead_code)] // R06 reads this after the server has stopped. + pub(crate) fn read_latest(&self, now: chrono::DateTime) -> Result { + self.read_latest_inner(now, || {}) + } + + #[cfg(all(test, target_os = "linux"))] + fn read_latest_after_open( + &self, + now: chrono::DateTime, + after_open: impl FnOnce(), + ) -> Result { + self.read_latest_inner(now, after_open) + } + + fn read_latest_inner( + &self, + now: chrono::DateTime, + after_open: impl FnOnce(), + ) -> Result { + #[cfg(not(target_os = "linux"))] + { + let _ = (now, after_open); + Err(InventoryError::PlatformSecurity) + } + #[cfg(target_os = "linux")] + { + self.validate_anchor()?; + let mut file = open_file_at(&self.directory, "latest.json", false, false)?; + validate_regular_file(&file)?; + let before = file_identity(&file)?; + after_open(); + let bytes = read_bounded(&mut file)?; + validate_regular_file(&file)?; + if before != file_identity(&file)? { + return Err(InventoryError::PersistenceSecurity); + } + let current = open_file_at(&self.directory, "latest.json", false, false)?; + validate_regular_file(¤t)?; + if before != file_identity(¤t)? { + return Err(InventoryError::PersistenceSecurity); + } + self.validate_anchor()?; + decode_envelope(&bytes, now) + } + } + + #[cfg(target_os = "linux")] + fn validate_anchor(&self) -> Result<(), InventoryError> { + self.state_root.validate()?; + let state_root = self.state_root.state_root()?; + validate_directory(&self.directory, true)?; + let current = open_directory_at(state_root, "inventory")?; + validate_directory(¤t, true)?; + if file_identity(&self.directory)? != file_identity(¤t)? { + return Err(InventoryError::PersistenceSecurity); + } + Ok(()) } fn prepare_sync(&self, snapshot: InventorySnapshot) -> Result, InventoryError> { @@ -424,46 +595,223 @@ impl InventoryStateStore { } fn read(&self) -> Result { - let bytes = match fs::read(&self.path) { - Ok(bytes) => bytes, - Err(source) if source.kind() == io::ErrorKind::NotFound => return Ok(InventoryState::default()), - Err(source) => return Err(state_io(&self.path, source)), - }; - check_mode(&self.path)?; - let state: InventoryState = serde_json::from_slice(&bytes).map_err(|source| InventoryError::StateInvalid { - path: self.path.clone(), - source, - })?; - let last_hash_valid = state.last_accepted_content_hash.as_deref().is_none_or(valid_content_hash); - let pending_valid = state.pending.as_ref().is_none_or(|pending| { - pending.sequence == state.next_sequence - && pending.is_valid() - && pending - .content_hash() - .is_ok_and(|hash| state.last_accepted_content_hash.as_deref() != Some(&hash)) - }); - if state.next_sequence > MAX_SEQUENCE + 1 || !last_hash_valid || !pending_valid { - return Err(InventoryError::StateCorrupt { path: self.path.clone() }); + self.read_with_persisted_at().map(|(state, _)| state) + } + + fn read_with_persisted_at(&self) -> Result<(InventoryState, Option), InventoryError> { + #[cfg(not(target_os = "linux"))] + { + Err(InventoryError::PlatformSecurity) + } + #[cfg(target_os = "linux")] + { + self.validate_anchor()?; + let mut file = match open_file_at(&self.directory, "state.json", false, false) { + Ok(file) => file, + Err(InventoryError::StateMissing) => return Ok((InventoryState::default(), None)), + Err(error) => return Err(error), + }; + validate_regular_file(&file)?; + let bytes = read_bounded(&mut file)?; + let modified = file + .metadata() + .and_then(|metadata| metadata.modified()) + .map_err(|_| InventoryError::StateIo)?; + self.validate_anchor()?; + let state: InventoryState = serde_json::from_slice(&bytes).map_err(|_| InventoryError::StateInvalid)?; + let last_hash_valid = state.last_accepted_content_hash.as_deref().is_none_or(valid_content_hash); + let pending_valid = state.pending.as_ref().is_none_or(|pending| { + pending.sequence == state.next_sequence + && pending.is_valid() + && pending + .content_hash() + .is_ok_and(|hash| state.last_accepted_content_hash.as_deref() != Some(&hash)) + }); + if state.next_sequence > MAX_SEQUENCE + 1 || !last_hash_valid || !pending_valid { + return Err(InventoryError::StateCorrupt); + } + let modified = chrono::DateTime::::from(modified); + if modified > chrono::Utc::now() + MAX_FUTURE_SKEW { + return Err(InventoryError::StateCorrupt); + } + let persisted_at = modified.format("%Y-%m-%dT%H:%M:%SZ").to_string(); + Ok((state, Some(persisted_at))) } - Ok(state) } fn write(&self, state: &InventoryState) -> Result<(), InventoryError> { - let bytes = serde_json::to_vec(state).map_err(|source| InventoryError::StateInvalid { - path: self.path.clone(), - source, - })?; - let directory = parent(&self.path)?; - prepare_inventory_directory(directory)?; - let temp = stage(directory, &self.path, &bytes)?; - let result = fs::rename(&temp, &self.path) - .map_err(|source| state_io(&self.path, source)) - .and_then(|()| fsync_dir(directory).map_err(|source| state_io(directory, source))); - if result.is_err() { - let _ = fs::remove_file(temp); - } - result + let bytes = serde_json::to_vec(state).map_err(|_| InventoryError::StateInvalid)?; + self.replace_file("state.json", &bytes, || false) } + + fn publish_latest_sync( + &self, + snapshot: InventorySnapshot, + captured_at: String, + shutdown: &tokio_util::sync::CancellationToken, + ) -> Result<(), InventoryError> { + snapshot.validate()?; + let bytes = encode_envelope(snapshot, captured_at)?; + self.replace_file("latest.json", &bytes, || shutdown.is_cancelled()) + } + + fn replace_file(&self, destination: &str, bytes: &[u8], cancelled: impl FnOnce() -> bool) -> Result<(), InventoryError> { + self.replace_file_inner( + destination, + bytes, + cancelled, + #[cfg(all(test, target_os = "linux"))] + None, + #[cfg(all(test, target_os = "linux"))] + || {}, + ) + } + + fn replace_file_inner( + &self, + destination: &str, + bytes: &[u8], + cancelled: impl FnOnce() -> bool, + #[cfg(all(test, target_os = "linux"))] fault: Option, + #[cfg(all(test, target_os = "linux"))] before_commit: impl FnOnce(), + ) -> Result<(), InventoryError> { + #[cfg(not(target_os = "linux"))] + { + let _ = (destination, bytes, cancelled); + Err(InventoryError::PlatformSecurity) + } + #[cfg(target_os = "linux")] + { + self.validate_anchor()?; + match open_file_at(&self.directory, destination, false, false) { + Ok(existing) => validate_regular_file(&existing)?, + Err(InventoryError::StateMissing) => {} + Err(error) => return Err(error), + } + let (temp_name, mut temp) = stage_at(&self.directory, destination)?; + #[cfg(test)] + let injected_write = matches!(fault.as_ref(), Some(PersistFault::Write)); + #[cfg(not(test))] + let injected_write = false; + let staged = if injected_write { + Err(InventoryError::StateIo) + } else { + temp.write_all(bytes).map_err(|_| InventoryError::StateIo) + } + .and_then(|()| { + #[cfg(test)] + if matches!(fault.as_ref(), Some(PersistFault::TempSync)) { + return Err(InventoryError::StateIo); + } + temp.sync_all().map_err(|_| InventoryError::StateIo) + }); + if staged.is_err() || cancelled() { + let _ = unlink_at(&self.directory, &temp_name); + return staged.and(Err(InventoryError::Cancelled)); + } + if let Err(error) = validate_regular_file(&temp) { + let _ = unlink_at(&self.directory, &temp_name); + return Err(error); + } + #[cfg(test)] + before_commit(); + if let Err(error) = self.validate_anchor() { + let _ = unlink_at(&self.directory, &temp_name); + return Err(error); + } + #[cfg(test)] + if let Some(PersistFault::CancelDuringCommit(token)) = fault.as_ref() { + token.cancel(); + } + #[cfg(test)] + let rename_failed = matches!(fault.as_ref(), Some(PersistFault::Rename)); + #[cfg(not(test))] + let rename_failed = false; + if rename_failed || rename_at(&self.directory, &temp_name, destination).is_err() { + let _ = unlink_at(&self.directory, &temp_name); + return Err(InventoryError::StateIo); + } + #[cfg(test)] + if matches!(fault.as_ref(), Some(PersistFault::DirectorySync)) { + return Err(InventoryError::DurabilityAfterCommit); + } + self.directory.sync_all().map_err(|_| InventoryError::DurabilityAfterCommit)?; + self.validate_anchor() + } + } +} + +#[cfg(all(test, target_os = "linux"))] +enum PersistFault { + Write, + TempSync, + Rename, + DirectorySync, + CancelDuringCommit(tokio_util::sync::CancellationToken), +} + +fn encode_envelope(snapshot: InventorySnapshot, captured_at: String) -> Result, InventoryError> { + if !is_exact_utc_seconds(&captured_at) { + return Err(InventoryError::EnvelopeTimestamp); + } + let envelope_hash = envelope_hash(ENVELOPE_FORMAT_VERSION, &captured_at, &snapshot)?; + serde_json::to_vec(&InventoryEnvelope { + format_version: ENVELOPE_FORMAT_VERSION.to_owned(), + captured_at, + snapshot, + envelope_hash, + }) + .map_err(|_| InventoryError::StateInvalid) +} + +fn envelope_hash(format_version: &str, captured_at: &str, snapshot: &InventorySnapshot) -> Result { + #[derive(Serialize)] + #[serde(rename_all = "camelCase")] + struct Canonical<'a> { + format_version: &'a str, + captured_at: &'a str, + snapshot: &'a InventorySnapshot, + } + let canonical = serde_json::to_vec(&Canonical { + format_version, + captured_at, + snapshot, + }) + .map_err(|_| InventoryError::StateInvalid)?; + let mut digest = Sha256::new(); + digest.update(ENVELOPE_HASH_PREFIX); + digest.update([0]); + digest.update(canonical); + Ok(hex_simd::encode_to_string(digest.finalize(), hex_simd::AsciiCase::Lower)) +} + +#[allow(dead_code)] // Used by the crate-private stopped-server reader. +fn decode_envelope(bytes: &[u8], now: chrono::DateTime) -> Result { + let envelope: InventoryEnvelope = serde_json::from_slice(bytes).map_err(|_| InventoryError::EnvelopeInvalid)?; + if envelope.format_version != ENVELOPE_FORMAT_VERSION { + return Err(InventoryError::EnvelopeVersion); + } + if !is_exact_utc_seconds(&envelope.captured_at) { + return Err(InventoryError::EnvelopeTimestamp); + } + envelope.snapshot.validate()?; + let expected = envelope_hash(&envelope.format_version, &envelope.captured_at, &envelope.snapshot)?; + if envelope.envelope_hash != expected || !valid_content_hash(&envelope.envelope_hash) { + return Err(InventoryError::EnvelopeHash); + } + let captured_at = chrono::DateTime::parse_from_rfc3339(&envelope.captured_at) + .map_err(|_| InventoryError::EnvelopeTimestamp)? + .with_timezone(&chrono::Utc); + let future = captured_at.signed_duration_since(now); + if future > chrono::Duration::from_std(MAX_FUTURE_SKEW).map_err(|_| InventoryError::EnvelopeTimestamp)? { + return Err(InventoryError::EnvelopeFuture); + } + let age = now.signed_duration_since(captured_at).to_std().unwrap_or_default(); + Ok(PersistedInventory { + snapshot: envelope.snapshot, + captured_at: envelope.captured_at, + age, + }) } fn valid_content_hash(value: &str) -> bool { @@ -473,277 +821,975 @@ fn valid_content_hash(value: &str) -> bool { .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) } -fn parent(path: &Path) -> Result<&Path, InventoryError> { - path.parent() - .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state path has no parent"))) -} - -fn filename(path: &Path) -> Result<&str, InventoryError> { - path.file_name() - .and_then(|name| name.to_str()) - .ok_or_else(|| state_io(path, io::Error::new(io::ErrorKind::InvalidInput, "state filename is invalid"))) -} - -fn prepare_inventory_directory(directory: &Path) -> Result<(), InventoryError> { - prepare_inventory_directory_with(directory, create_inventory_directory, fsync_dir) -} - -fn prepare_inventory_directory_with( - directory: &Path, - create: impl FnOnce(&Path) -> io::Result<()>, - mut sync: impl FnMut(&Path) -> io::Result<()>, -) -> Result<(), InventoryError> { - let root = parent(directory)?; - let root_metadata = fs::symlink_metadata(root).map_err(|source| state_io(root, source))?; - if root_metadata.file_type().is_symlink() || !root_metadata.is_dir() { - return Err(state_io( - root, - io::Error::new(io::ErrorKind::InvalidInput, "inventory state root is not a directory"), - )); +#[cfg(target_os = "linux")] +fn read_bounded(file: &mut fs::File) -> Result, InventoryError> { + let mut bytes = Vec::new(); + file.take((MAX_PERSISTED_BYTES + 1) as u64) + .read_to_end(&mut bytes) + .map_err(|_| InventoryError::StateIo)?; + if bytes.len() > MAX_PERSISTED_BYTES { + return Err(InventoryError::StateOversize); } - match create(directory) { - Ok(()) => {} - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => { - let metadata = fs::symlink_metadata(directory).map_err(|source| state_io(directory, source))?; - if metadata.file_type().is_symlink() || !metadata.is_dir() { - return Err(state_io( - directory, - io::Error::new(io::ErrorKind::InvalidInput, "inventory state path is not a directory"), - )); - } + Ok(bytes) +} + +#[cfg(target_os = "linux")] +// SAFETY: libc path operations use validated directory descriptors and checked C strings; returned descriptors become owned files. +#[allow(unsafe_code)] +fn open_inventory_directory(path: &Path) -> Result<(StateRootAnchor, fs::File), InventoryError> { + use std::os::fd::AsRawFd as _; + use std::path::Component; + + let absolute = if path.is_absolute() { + path.to_path_buf() + } else { + std::env::current_dir().map_err(|_| InventoryError::StateIo)?.join(path) + }; + let root = fs::File::open("/").map_err(|_| InventoryError::StateIo)?; + validate_directory(&root, false)?; + let components = absolute.components().collect::>(); + let names = components + .iter() + .filter_map(|component| match component { + Component::Normal(name) => Some(name.to_os_string()), + Component::RootDir => None, + _ => Some(std::ffi::OsString::new()), + }) + .collect::>(); + if names.iter().any(|name| name.is_empty()) || names.is_empty() { + return Err(InventoryError::StatePath); + } + let component_count = names.len(); + let mut state_root = StateRootAnchor { + root, + components: Vec::with_capacity(component_count), + }; + for (index, name) in names.into_iter().enumerate() { + let parent = state_root + .components + .last() + .map(|(_, directory)| directory) + .unwrap_or(&state_root.root); + let directory = open_directory_component_at(parent, &name)?; + validate_directory(&directory, index + 1 == component_count)?; + state_root.components.push((name, directory)); + } + + let directory = state_root.state_root()?; + let inventory = c_name("inventory")?; + // SAFETY: descriptor and C string are valid; mode is applied only if the directory is created. + if unsafe { libc::mkdirat(directory.as_raw_fd(), inventory.as_ptr(), 0o700) } != 0 { + let error = io::Error::last_os_error(); + if error.kind() != io::ErrorKind::AlreadyExists { + return Err(InventoryError::StateIo); } - Err(source) => return Err(state_io(directory, source)), } - sync(directory).map_err(|source| state_io(directory, source))?; - sync(root).map_err(|source| state_io(root, source)) + let child = open_directory_at(directory, "inventory")?; + validate_directory(&child, true)?; + sync_inventory_anchor(&child, directory)?; + Ok((state_root, child)) } -fn create_inventory_directory(directory: &Path) -> io::Result<()> { - let mut builder = fs::DirBuilder::new(); - #[cfg(unix)] - { - use std::os::unix::fs::DirBuilderExt as _; - builder.mode(0o700); - } - builder.create(directory) +#[cfg(target_os = "linux")] +fn sync_inventory_anchor(inventory: &fs::File, state_root: &fs::File) -> Result<(), InventoryError> { + sync_inventory_anchor_with(|inventory_target| { + if inventory_target { + inventory.sync_all() + } else { + state_root.sync_all() + } + }) } -fn stage(directory: &Path, destination: &Path, bytes: &[u8]) -> Result { - let name = filename(destination)?; +#[cfg(any(target_os = "linux", test))] +fn sync_inventory_anchor_with(mut sync: impl FnMut(bool) -> io::Result<()>) -> Result<(), InventoryError> { + sync(true).map_err(|_| InventoryError::StateIo)?; + sync(false).map_err(|_| InventoryError::StateIo) +} + +#[cfg(target_os = "linux")] +fn open_directory_at(parent: &fs::File, name: &str) -> Result { + open_directory_component_at(parent, std::ffi::OsStr::new(name)) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn open_directory_component_at(parent: &fs::File, name: &std::ffi::OsStr) -> Result { + use std::os::fd::{AsRawFd as _, FromRawFd as _}; + use std::os::unix::ffi::OsStrExt as _; + let name = std::ffi::CString::new(name.as_bytes()).map_err(|_| InventoryError::StatePath)?; + // SAFETY: the parent descriptor and C string are valid; ownership of a successful descriptor is transferred. + let fd = unsafe { + libc::openat( + parent.as_raw_fd(), + name.as_ptr(), + libc::O_RDONLY | libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_DIRECTORY, + ) + }; + if fd < 0 { + return Err(InventoryError::PersistenceSecurity); + } + // SAFETY: openat returned a new owned descriptor. + Ok(unsafe { fs::File::from_raw_fd(fd) }) +} + +#[cfg(target_os = "linux")] +fn validate_directory(directory: &fs::File, dedicated: bool) -> Result<(), InventoryError> { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + let metadata = directory.metadata().map_err(|_| InventoryError::StateIo)?; + let mode = metadata.permissions().mode() & 0o7777; + let uid = process_uid(); + if !metadata.is_dir() || !unix_directory_is_trusted(metadata.uid(), mode, uid, dedicated) { + return Err(InventoryError::PersistenceSecurity); + } + Ok(()) +} + +#[cfg(target_os = "linux")] +fn unix_directory_is_trusted(owner: u32, mode: u32, process: u32, dedicated: bool) -> bool { + let trusted_owner = owner == process || (!dedicated && owner == 0); + let trusted_mode = if dedicated { mode == 0o700 } else { mode & 0o7022 == 0 }; + trusted_owner && trusted_mode +} + +#[cfg(target_os = "linux")] +// SAFETY: openat receives a live directory descriptor and checked C string; a successful descriptor becomes an owned file. +#[allow(unsafe_code)] +fn open_file_at(directory: &fs::File, name: &str, create: bool, write: bool) -> Result { + use std::os::fd::{AsRawFd as _, FromRawFd as _}; + let name = c_name(name)?; + let mut flags = libc::O_CLOEXEC | libc::O_NOFOLLOW | libc::O_NONBLOCK; + flags |= if write { libc::O_RDWR } else { libc::O_RDONLY }; + if create { + flags |= libc::O_CREAT; + } + // SAFETY: the directory descriptor and C string are valid; ownership of a successful descriptor is transferred. + let fd = unsafe { libc::openat(directory.as_raw_fd(), name.as_ptr(), flags, FILE_MODE) }; + if fd < 0 { + let error = io::Error::last_os_error(); + return if error.kind() == io::ErrorKind::NotFound { + Err(InventoryError::StateMissing) + } else { + Err(InventoryError::PersistenceSecurity) + }; + } + // SAFETY: openat returned a new owned descriptor. + Ok(unsafe { fs::File::from_raw_fd(fd) }) +} + +#[cfg(target_os = "linux")] +// SAFETY: openat receives a live directory descriptor and checked C string; a successful descriptor becomes an owned file. +#[allow(unsafe_code)] +fn stage_at(directory: &fs::File, destination: &str) -> Result<(String, fs::File), InventoryError> { + use std::os::fd::{AsRawFd as _, FromRawFd as _}; loop { - let path = directory.join(format!( - ".{name}.{}.{}.tmp", + let name = format!( + ".{destination}.{}.{}.tmp", std::process::id(), STAGING_SEQUENCE.fetch_add(1, Ordering::Relaxed) - )); - let mut options = fs::OpenOptions::new(); - options.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt as _; - options.mode(FILE_MODE); - } - let mut file = match options.open(&path) { - Ok(file) => file, - Err(source) if source.kind() == io::ErrorKind::AlreadyExists => continue, - Err(source) => return Err(state_io(&path, source)), + ); + let c_name = c_name(&name)?; + // SAFETY: the directory descriptor and C string are valid; ownership of a successful descriptor is transferred. + let fd = unsafe { + libc::openat( + directory.as_raw_fd(), + c_name.as_ptr(), + libc::O_WRONLY | libc::O_CREAT | libc::O_EXCL | libc::O_CLOEXEC | libc::O_NOFOLLOW, + FILE_MODE, + ) }; - if let Err(source) = file.write_all(bytes).and_then(|()| file.sync_all()) { - let _ = fs::remove_file(&path); - return Err(state_io(&path, source)); + if fd >= 0 { + // SAFETY: openat returned a new owned descriptor. + let file = unsafe { fs::File::from_raw_fd(fd) }; + if let Err(error) = validate_regular_file(&file) { + let _ = unlink_at(directory, &name); + return Err(error); + } + return Ok((name, file)); + } + if io::Error::last_os_error().kind() != io::ErrorKind::AlreadyExists { + return Err(InventoryError::StateIo); } - return Ok(path); } } -fn state_io(path: &Path, source: io::Error) -> InventoryError { - InventoryError::StateIo { - path: path.to_path_buf(), - source, +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn rename_at(directory: &fs::File, source: &str, destination: &str) -> io::Result<()> { + use std::os::fd::AsRawFd as _; + let source = std::ffi::CString::new(source).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + let destination = std::ffi::CString::new(destination).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + // SAFETY: both names are valid C strings and both directory descriptors remain open. + if unsafe { libc::renameat(directory.as_raw_fd(), source.as_ptr(), directory.as_raw_fd(), destination.as_ptr()) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) } } -#[cfg(unix)] -fn check_mode(path: &Path) -> Result<(), InventoryError> { - use std::os::unix::fs::PermissionsExt as _; +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn unlink_at(directory: &fs::File, name: &str) -> io::Result<()> { + use std::os::fd::AsRawFd as _; + let name = std::ffi::CString::new(name).map_err(|_| io::Error::from(io::ErrorKind::InvalidInput))?; + // SAFETY: the name is a valid C string and the directory descriptor remains open. + if unsafe { libc::unlinkat(directory.as_raw_fd(), name.as_ptr(), 0) } == 0 { + Ok(()) + } else { + Err(io::Error::last_os_error()) + } +} - let mode = fs::metadata(path) - .map_err(|source| state_io(path, source))? - .permissions() - .mode() - & 0o7777; - if mode != FILE_MODE { - return Err(InventoryError::StatePermissions { - path: path.to_path_buf(), - mode, - expected: FILE_MODE, - }); +#[cfg(target_os = "linux")] +fn validate_regular_file(file: &fs::File) -> Result<(), InventoryError> { + use std::os::unix::fs::{MetadataExt as _, PermissionsExt as _}; + let metadata = file.metadata().map_err(|_| InventoryError::StateIo)?; + if !metadata.is_file() + || !unix_regular_file_is_secure(metadata.uid(), metadata.permissions().mode() & 0o7777, metadata.nlink(), process_uid()) + { + return Err(InventoryError::PersistenceSecurity); } Ok(()) } -#[cfg(not(unix))] -fn check_mode(_path: &Path) -> Result<(), InventoryError> { - Ok(()) +#[cfg(target_os = "linux")] +fn unix_regular_file_is_secure(owner: u32, mode: u32, links: u64, process: u32) -> bool { + owner == process && mode == FILE_MODE && links == 1 } -fn fsync_dir(directory: &Path) -> io::Result<()> { - #[cfg(unix)] - fs::File::open(directory)?.sync_all()?; - #[cfg(not(unix))] - let _ = directory; - Ok(()) +#[cfg(target_os = "linux")] +#[allow(dead_code)] // Used by the crate-private stopped-server reader. +fn file_identity(file: &fs::File) -> Result<(u64, u64, u64), InventoryError> { + use std::os::unix::fs::MetadataExt as _; + let metadata = file.metadata().map_err(|_| InventoryError::StateIo)?; + Ok((metadata.dev(), metadata.ino(), metadata.nlink())) +} + +#[cfg(target_os = "linux")] +fn c_name(name: &str) -> Result { + std::ffi::CString::new(name).map_err(|_| InventoryError::StatePath) +} + +#[cfg(target_os = "linux")] +#[allow(unsafe_code)] +fn process_uid() -> u32 { + // SAFETY: geteuid has no pointer arguments or caller preconditions. + unsafe { libc::geteuid() } } #[derive(Debug, thiserror::Error)] pub enum InventoryError { - #[error("the RustFS inventory version is outside protocol bounds")] + #[error("connect_inventory_snapshot_version")] RustfsVersion, - #[error("the RustFS inventory operating-system version is outside protocol bounds")] + #[error("connect_inventory_snapshot_os_version")] OsVersion, - #[error("the RustFS inventory node count is outside protocol bounds")] + #[error("connect_inventory_snapshot_node_count")] NodeCount, - #[error("the RustFS inventory drive count is outside protocol bounds")] + #[error("connect_inventory_snapshot_drive_count")] DriveCount, - #[error("the RustFS inventory capacity is outside protocol bounds")] + #[error("connect_inventory_snapshot_capacity")] Capacity, - #[error("the RustFS inventory coarse flags are not canonical")] + #[error("connect_inventory_snapshot_flags")] CoarseFlags, - #[error("the RustFS inventory snapshot is incomplete: observed {observed} of {expected} configured drives")] + #[error("connect_inventory_snapshot_incomplete")] SnapshotIncomplete { expected: usize, observed: usize }, - #[error("the Connect inventory schedule is invalid")] + #[error("connect_inventory_schedule")] Schedule, - #[error("the Connect inventory sequence is exhausted")] + #[error("connect_inventory_sequence_exhausted")] SequenceExhausted, - #[error("a Connect inventory runtime already owns this state")] + #[error("connect_inventory_already_running")] AlreadyRunning, - #[error("the persisted Connect inventory changed while delivery was in flight")] + #[error("connect_inventory_state_conflict")] StateConflict, - #[error("the Connect inventory state path is invalid")] + #[error("connect_inventory_state_path")] StatePath, - #[error("Connect inventory state I/O failed at {path}: {source}")] - StateIo { - path: PathBuf, - #[source] - source: io::Error, - }, - #[error("Connect inventory state at {path} is invalid: {source}")] - StateInvalid { - path: PathBuf, - #[source] - source: serde_json::Error, - }, - #[error("Connect inventory state at {path} violates the protocol invariants")] - StateCorrupt { path: PathBuf }, - #[cfg(unix)] - #[error("Connect inventory state at {path} has mode {mode:o}, expected {expected:o}")] - StatePermissions { path: PathBuf, mode: u32, expected: u32 }, - #[error("Connect returned an invalid inventory response")] + #[error("connect_inventory_state_missing")] + StateMissing, + #[error("connect_inventory_state_io")] + StateIo, + #[error("connect_inventory_state_invalid")] + StateInvalid, + #[error("connect_inventory_state_corrupt")] + StateCorrupt, + #[error("connect_inventory_state_oversize")] + StateOversize, + #[error("connect_inventory_persistence_security")] + PersistenceSecurity, + #[error("connect_inventory_platform_security")] + PlatformSecurity, + #[error("connect_inventory_cancelled")] + Cancelled, + #[error("connect_inventory_durability_after_commit")] + DurabilityAfterCommit, + #[error("connect_inventory_envelope_invalid")] + EnvelopeInvalid, + #[error("connect_inventory_envelope_version")] + EnvelopeVersion, + #[error("connect_inventory_envelope_timestamp")] + EnvelopeTimestamp, + #[error("connect_inventory_envelope_future")] + EnvelopeFuture, + #[error("connect_inventory_envelope_hash")] + EnvelopeHash, + #[error("connect_inventory_response")] Response, - #[error(transparent)] + #[error("connect_inventory_json")] Json(#[from] serde_json::Error), - #[error("Connect inventory delivery failed: {0}")] - Telemetry(String), + #[error("connect_inventory_telemetry")] + Telemetry, } impl From for InventoryError { - fn from(error: TelemetryError) -> Self { - Self::Telemetry(error.to_string()) + fn from(_error: TelemetryError) -> Self { + Self::Telemetry } } #[cfg(test)] mod tests { - use std::cell::RefCell; - use std::rc::Rc; - use super::*; - fn create_directory(path: &Path) -> io::Result<()> { - fs::create_dir(path) + fn safe_tempdir() -> tempfile::TempDir { + let temp = tempfile::tempdir_in(env!("CARGO_MANIFEST_DIR")).expect("safe temporary directory"); + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(temp.path(), fs::Permissions::from_mode(0o700)).expect("private temporary directory"); + } + temp + } + + #[cfg(not(target_os = "linux"))] + #[test] + fn persistence_fails_closed_before_accessing_state() { + let temp = safe_tempdir(); + let state = temp.path().join("state-must-not-be-created"); + + assert!(matches!( + InventoryStateStore::from_state_root(&state), + Err(InventoryError::PlatformSecurity) + )); + assert!(!state.exists()); + } + + #[cfg(target_os = "linux")] + #[allow(unsafe_code)] + fn make_fifo(path: &Path) { + use std::os::unix::ffi::OsStrExt as _; + + let path = std::ffi::CString::new(path.as_os_str().as_bytes()).expect("FIFO path"); + // SAFETY: the path is a valid C string and mkfifo does not retain it. + assert_eq!(unsafe { libc::mkfifo(path.as_ptr(), FILE_MODE as libc::mode_t) }, 0, "create FIFO"); + } + + fn snapshot() -> InventorySnapshot { + InventorySnapshot::new("1.2.3", None, 2, 4, 1_000, 400, [InventoryFlag::DriveOffline]).expect("snapshot") } #[test] - fn inventory_directory_creation_is_synced_before_state_can_be_committed() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().join("root"); - fs::create_dir(&root).expect("state root"); - let directory = root.join("inventory"); - let events = Rc::new(RefCell::new(Vec::new())); - let create_events = events.clone(); - let sync_events = events.clone(); - - prepare_inventory_directory_with( - &directory, - move |path| { - create_events.borrow_mut().push(format!("mkdir:{}", path.display())); - fs::create_dir(path) - }, - move |path| { - sync_events.borrow_mut().push(format!("sync:{}", path.display())); - Ok(()) - }, - ) - .expect("durable directory"); + fn envelope_has_stable_canonical_bytes_and_hash() { + let bytes = encode_envelope(snapshot(), "2026-08-23T01:02:03Z".to_owned()).expect("envelope"); assert_eq!( - events.borrow().as_slice(), - [ - format!("mkdir:{}", directory.display()), - format!("sync:{}", directory.display()), - format!("sync:{}", root.display()), - ] + String::from_utf8(bytes).expect("JSON"), + r#"{"formatVersion":"v1","capturedAt":"2026-08-23T01:02:03Z","snapshot":{"rustfsVersion":"1.2.3","osVersion":null,"nodeCount":2,"driveCount":4,"capacityTotalBytes":1000,"capacityUsedBytes":400,"coarseFlags":["drive.offline"]},"envelopeHash":"fb927e66c9635e0020b97993c868636bff1f8ddefe01b49345286de6239865e3"}"# ); } #[test] - fn inventory_directory_sync_failure_prevents_state_commit_and_is_retried() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().join("root"); - fs::create_dir(&root).expect("state root"); - let directory = root.join("inventory"); - let state = directory.join("state.json"); - let error = prepare_inventory_directory_with(&directory, create_directory, |path| { - if path == directory { - Err(io::Error::other("injected leaf sync failure")) - } else { - Ok(()) - } - }) - .expect_err("sync failure"); - assert!(matches!(error, InventoryError::StateIo { path, .. } if path == directory)); - assert!(!state.exists()); - - let error = prepare_inventory_directory_with(&directory, create_directory, |path| { - if path == root { - Err(io::Error::other("injected parent sync failure")) - } else { - Ok(()) - } - }) - .expect_err("parent sync failure"); - assert!(matches!(error, InventoryError::StateIo { path, .. } if path == root)); - assert!(!state.exists()); - - let mut synced = Vec::new(); - prepare_inventory_directory_with(&directory, create_directory, |path| { - synced.push(path.to_path_buf()); - Ok(()) - }) - .expect("retry must sync an already-created leaf"); - assert_eq!(synced, [directory, root]); + fn telemetry_failures_are_normalized_before_runtime_status() { + assert_eq!(InventoryError::from(TelemetryError::Endpoint).to_string(), "connect_inventory_telemetry"); } #[test] - fn inventory_directory_requires_its_fixed_root_to_exist() { - let temp = tempfile::tempdir().expect("tempdir"); - let root = temp.path().join("missing-root"); - let directory = root.join("inventory"); + fn inventory_anchor_syncs_child_then_state_root_and_retries_failures() { + let mut calls = Vec::new(); + let error = sync_inventory_anchor_with(|inventory_target| { + calls.push(inventory_target); + if !inventory_target { + Err(io::Error::other("injected state-root sync failure")) + } else { + Ok(()) + } + }) + .expect_err("state-root sync failure"); + assert!(matches!(error, InventoryError::StateIo)); + assert_eq!(calls, [true, false]); + + calls.clear(); + sync_inventory_anchor_with(|inventory_target| { + calls.push(inventory_target); + Ok(()) + }) + .expect("retry syncs the whole anchor"); + assert_eq!(calls, [true, false]); + } + + #[test] + fn reader_rejects_tampering_unknown_members_and_future_time() { + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T01:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + let valid = encode_envelope(snapshot(), "2026-08-23T01:02:03Z".to_owned()).expect("envelope"); + assert_eq!(decode_envelope(&valid, now).expect("valid envelope").age, Duration::ZERO); + + let mut hash_tampered: serde_json::Value = serde_json::from_slice(&valid).expect("JSON"); + hash_tampered["snapshot"]["nodeCount"] = 3.into(); + assert!(matches!( + decode_envelope(&serde_json::to_vec(&hash_tampered).expect("JSON"), now), + Err(InventoryError::EnvelopeHash) + )); + + let mut timestamp_tampered: serde_json::Value = serde_json::from_slice(&valid).expect("JSON"); + timestamp_tampered["capturedAt"] = "2026-08-23T01:02:04Z".into(); + assert!(matches!( + decode_envelope(&serde_json::to_vec(×tamp_tampered).expect("JSON"), now), + Err(InventoryError::EnvelopeHash) + )); + + let mut unknown: serde_json::Value = serde_json::from_slice(&valid).expect("JSON"); + unknown["extra"] = true.into(); + assert!(matches!( + decode_envelope(&serde_json::to_vec(&unknown).expect("JSON"), now), + Err(InventoryError::EnvelopeInvalid) + )); + + let mut unknown_snapshot: serde_json::Value = serde_json::from_slice(&valid).expect("JSON"); + unknown_snapshot["snapshot"]["hostname"] = "secret.invalid".into(); + assert!(matches!( + decode_envelope(&serde_json::to_vec(&unknown_snapshot).expect("JSON"), now), + Err(InventoryError::EnvelopeInvalid) + )); + + let mut version: serde_json::Value = serde_json::from_slice(&valid).expect("JSON"); + version["formatVersion"] = "v2".into(); + assert!(matches!( + decode_envelope(&serde_json::to_vec(&version).expect("JSON"), now), + Err(InventoryError::EnvelopeVersion) + )); + + let mut timestamp: serde_json::Value = serde_json::from_slice(&valid).expect("JSON"); + timestamp["capturedAt"] = "2026-08-23T01:02:03.000Z".into(); + assert!(matches!( + decode_envelope(&serde_json::to_vec(×tamp).expect("JSON"), now), + Err(InventoryError::EnvelopeTimestamp) + )); + + let future = encode_envelope(snapshot(), "2026-08-23T01:07:04Z".to_owned()).expect("envelope"); + assert!(matches!(decode_envelope(&future, now), Err(InventoryError::EnvelopeFuture))); + let skew_boundary = encode_envelope(snapshot(), "2026-08-23T01:07:03Z".to_owned()).expect("envelope"); + assert_eq!(decode_envelope(&skew_boundary, now).expect("five-minute skew").age, Duration::ZERO); + let old = encode_envelope(snapshot(), "2026-08-22T01:02:03Z".to_owned()).expect("envelope"); + assert_eq!(decode_envelope(&old, now).expect("old envelope").age, Duration::from_secs(24 * 60 * 60)); + } + + #[cfg(target_os = "linux")] + #[test] + fn cancelled_publish_keeps_last_good_and_removes_staging_file() { + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + let last_good = encode_envelope(snapshot(), "2026-08-23T01:02:03Z".to_owned()).expect("last good"); + store.replace_file("latest.json", &last_good, || false).expect("seed latest"); + let token = tokio_util::sync::CancellationToken::new(); + token.cancel(); + let replacement = InventorySnapshot::new("1.2.3", None, 2, 4, 1_001, 401, []).expect("replacement"); + let error = store + .publish_latest_sync(replacement, "2026-08-23T02:02:03Z".to_owned(), &token) + .expect_err("cancelled before commit"); + + assert!(matches!(error, InventoryError::Cancelled)); + assert_eq!(fs::read(temp.path().join("inventory/latest.json")).expect("last good"), last_good); + let entries = fs::read_dir(temp.path().join("inventory")) + .expect("inventory directory") + .map(|entry| entry.expect("entry").file_name()) + .collect::>(); + assert_eq!(entries, vec![std::ffi::OsString::from("latest.json")]); + } + + #[cfg(target_os = "linux")] + #[test] + fn reader_bounds_the_opened_latest_file_and_reports_missing_state() { + use std::os::unix::fs::PermissionsExt as _; + + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T01:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + assert!(matches!(store.read_latest(now), Err(InventoryError::StateMissing))); + + let latest = temp.path().join("inventory/latest.json"); + fs::write(&latest, vec![b'x'; MAX_PERSISTED_BYTES]).expect("bounded latest"); + fs::set_permissions(&latest, fs::Permissions::from_mode(0o600)).expect("mode"); + assert!(matches!(store.read_latest(now), Err(InventoryError::EnvelopeInvalid))); + fs::write(&latest, vec![b'x'; MAX_PERSISTED_BYTES + 1]).expect("oversized latest"); + assert!(matches!(store.read_latest(now), Err(InventoryError::StateOversize))); + fs::write(&latest, b"not-json").expect("corrupt latest"); + assert!(matches!(store.read_latest(now), Err(InventoryError::EnvelopeInvalid))); + } + + #[cfg(target_os = "linux")] + #[test] + fn fifo_children_are_rejected_without_blocking_readers_or_writers() { + let latest_temp = safe_tempdir(); + let latest_store = InventoryStateStore::from_state_root(latest_temp.path()).expect("latest store"); + let latest = latest_temp.path().join("inventory/latest.json"); + make_fifo(&latest); + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T01:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + assert!(matches!(latest_store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + assert!(matches!( + latest_store.publish_latest_sync( + snapshot(), + "2026-08-23T01:02:03Z".to_owned(), + &tokio_util::sync::CancellationToken::new() + ), + Err(InventoryError::PersistenceSecurity) + )); + + let state_temp = safe_tempdir(); + let state_store = InventoryStateStore::from_state_root(state_temp.path()).expect("state store"); + let state = state_temp.path().join("inventory/state.json"); + make_fifo(&state); + assert!(matches!(state_store.read(), Err(InventoryError::PersistenceSecurity))); + assert!(matches!( + state_store.write(&InventoryState::default()), + Err(InventoryError::PersistenceSecurity) + )); + } + + #[cfg(target_os = "linux")] + #[test] + fn reader_rejects_insecure_file_modes_and_hardlinks() { + use std::os::unix::fs::{PermissionsExt as _, symlink}; + + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + store + .publish_latest_sync(snapshot(), "2026-08-23T01:02:03Z".to_owned(), &tokio_util::sync::CancellationToken::new()) + .expect("publish"); + let latest = temp.path().join("inventory/latest.json"); + fs::set_permissions(&latest, fs::Permissions::from_mode(0o4600)).expect("special-bit mode"); + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T01:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + assert!(matches!(store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + + fs::set_permissions(&latest, fs::Permissions::from_mode(0o644)).expect("mode"); + assert!(matches!(store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + + fs::set_permissions(&latest, fs::Permissions::from_mode(0o600)).expect("mode"); + fs::hard_link(&latest, temp.path().join("inventory/second-link")).expect("hard link"); + assert!(matches!(store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + + fs::remove_file(&latest).expect("remove latest link"); + symlink(temp.path().join("inventory/second-link"), &latest).expect("latest symlink"); + assert!(matches!(store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + } + + #[cfg(target_os = "linux")] + #[test] + fn store_rejects_unsafe_ancestors_and_symlinked_inventory_directory() { + use std::os::unix::fs::{PermissionsExt as _, symlink}; + + let temp = safe_tempdir(); + let unsafe_parent = temp.path().join("unsafe"); + fs::create_dir(&unsafe_parent).expect("unsafe parent"); + fs::set_permissions(&unsafe_parent, fs::Permissions::from_mode(0o777)).expect("unsafe mode"); + let state = unsafe_parent.join("state"); + fs::create_dir(&state).expect("state root"); + fs::set_permissions(&state, fs::Permissions::from_mode(0o700)).expect("state mode"); + assert!(matches!( + InventoryStateStore::from_state_root(&state), + Err(InventoryError::PersistenceSecurity) + )); + + let safe_state = temp.path().join("safe-state"); + fs::create_dir(&safe_state).expect("safe state root"); + fs::set_permissions(&safe_state, fs::Permissions::from_mode(0o1700)).expect("special-bit state mode"); + assert!(matches!( + InventoryStateStore::from_state_root(&safe_state), + Err(InventoryError::PersistenceSecurity) + )); + fs::set_permissions(&safe_state, fs::Permissions::from_mode(0o700)).expect("state mode"); + symlink(temp.path(), safe_state.join("inventory")).expect("inventory symlink"); + assert!(matches!( + InventoryStateStore::from_state_root(&safe_state), + Err(InventoryError::PersistenceSecurity) + )); + } + + #[cfg(target_os = "linux")] + #[test] + fn unix_persistence_policy_rejects_wrong_owners_modes_and_link_counts() { + let uid = 501; + assert!(unix_directory_is_trusted(uid, 0o700, uid, true)); + assert!(!unix_directory_is_trusted(uid + 1, 0o700, uid, true)); + assert!(!unix_directory_is_trusted(uid, 0o755, uid, true)); + assert!(!unix_directory_is_trusted(uid, 0o1700, uid, true)); + assert!(unix_directory_is_trusted(0, 0o755, uid, false)); + assert!(!unix_directory_is_trusted(0, 0o777, uid, false)); + assert!(!unix_directory_is_trusted(0, 0o1755, uid, false)); + + assert!(unix_regular_file_is_secure(uid, 0o600, 1, uid)); + assert!(!unix_regular_file_is_secure(uid + 1, 0o600, 1, uid)); + assert!(!unix_regular_file_is_secure(uid, 0o644, 1, uid)); + assert!(!unix_regular_file_is_secure(uid, 0o4600, 1, uid)); + assert!(!unix_regular_file_is_secure(uid, 0o600, 2, uid)); + } + + #[cfg(target_os = "linux")] + #[test] + fn reader_rejects_a_real_wrong_owner_when_chown_is_permitted() { + use std::os::unix::fs::chown; + + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + store + .publish_latest_sync(snapshot(), "2026-08-23T01:02:03Z".to_owned(), &tokio_util::sync::CancellationToken::new()) + .expect("publish"); + let latest = temp.path().join("inventory/latest.json"); + let wrong_uid = process_uid().checked_add(1).unwrap_or_else(|| process_uid() - 1); + if let Err(error) = chown(&latest, Some(wrong_uid), None) { + assert_eq!(error.kind(), io::ErrorKind::PermissionDenied, "unexpected chown failure"); + return; + } + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T01:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + assert!(matches!(store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + } + + #[cfg(target_os = "linux")] + #[test] + fn directory_component_exchange_is_rejected_by_the_open_anchor() { + use std::os::unix::fs::PermissionsExt as _; + + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + store + .publish_latest_sync(snapshot(), "2026-08-23T01:02:03Z".to_owned(), &tokio_util::sync::CancellationToken::new()) + .expect("publish"); + fs::rename(temp.path().join("inventory"), temp.path().join("original-inventory")).expect("exchange original"); + fs::create_dir(temp.path().join("inventory")).expect("replacement inventory"); + fs::set_permissions(temp.path().join("inventory"), fs::Permissions::from_mode(0o700)).expect("replacement mode"); + + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T01:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + assert!(matches!(store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + assert!(matches!( + store.publish_latest_sync(snapshot(), "2026-08-23T02:02:03Z".to_owned(), &tokio_util::sync::CancellationToken::new()), + Err(InventoryError::PersistenceSecurity) + )); + assert!(!temp.path().join("inventory/latest.json").exists()); + } + + #[cfg(target_os = "linux")] + #[test] + fn state_root_and_ancestor_exchanges_are_rejected_by_the_path_anchor() { + use std::os::unix::fs::PermissionsExt as _; + + for exchange_ancestor in [false, true] { + let temp = safe_tempdir(); + let ancestor = temp.path().join("anchor"); + let state_root = ancestor.join("state"); + fs::create_dir_all(&state_root).expect("state root"); + fs::set_permissions(&ancestor, fs::Permissions::from_mode(0o700)).expect("ancestor mode"); + fs::set_permissions(&state_root, fs::Permissions::from_mode(0o700)).expect("state mode"); + let store = InventoryStateStore::from_state_root(&state_root).expect("store"); + store + .publish_latest_sync(snapshot(), "2026-08-23T01:02:03Z".to_owned(), &tokio_util::sync::CancellationToken::new()) + .expect("publish"); + + let exchanged = if exchange_ancestor { &ancestor } else { &state_root }; + fs::rename(exchanged, temp.path().join("original")).expect("exchange original directory"); + fs::create_dir_all(&state_root).expect("replacement state root"); + fs::set_permissions(&ancestor, fs::Permissions::from_mode(0o700)).expect("replacement ancestor mode"); + fs::set_permissions(&state_root, fs::Permissions::from_mode(0o700)).expect("replacement state mode"); + + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T01:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + assert!(matches!(store.read_latest(now), Err(InventoryError::PersistenceSecurity))); + assert!(matches!( + store.publish_latest_sync( + snapshot(), + "2026-08-23T02:02:03Z".to_owned(), + &tokio_util::sync::CancellationToken::new() + ), + Err(InventoryError::PersistenceSecurity) + )); + assert!(!state_root.join("inventory/latest.json").exists()); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn fresh_state_concurrency_creates_one_anchor_and_allows_one_runtime_owner() { + let temp = safe_tempdir(); + let state_root = Arc::new(temp.path().to_path_buf()); + let start = Arc::new(std::sync::Barrier::new(3)); + let release = Arc::new(std::sync::Barrier::new(3)); + let (result_tx, result_rx) = std::sync::mpsc::channel(); + let mut threads = Vec::new(); + for _ in 0..2 { + let state_root = state_root.clone(); + let start = start.clone(); + let release = release.clone(); + let result_tx = result_tx.clone(); + threads.push(std::thread::spawn(move || { + start.wait(); + let result = InventoryStateStore::from_state_root(&state_root).and_then(|store| store.try_runtime_lock()); + result_tx + .send(result.as_ref().map(|_| ()).map_err(|error| error.to_string())) + .expect("result"); + release.wait(); + result + })); + } + start.wait(); + let results = [ + result_rx.recv().expect("first result"), + result_rx.recv().expect("second result"), + ]; + assert_eq!(results.iter().filter(|result| result.is_ok()).count(), 1); + assert_eq!( + results + .iter() + .filter(|result| { matches!(result, Err(error) if error.as_str() == "connect_inventory_already_running") }) + .count(), + 1 + ); + release.wait(); + for thread in threads { + let _ = thread.join().expect("thread"); + } + assert!(state_root.join("inventory/.state.json.lock").is_file()); + assert!(!state_root.join("inventory/.state.lock").exists()); + } + + #[cfg(target_os = "linux")] + fn exchange_path_component(temp: &Path, state_root: &Path, component: &str) { + use std::os::unix::fs::PermissionsExt as _; + + let ancestor = state_root.parent().expect("state ancestor"); + let inventory = state_root.join("inventory"); + let exchanged = match component { + "ancestor" => ancestor, + "state-root" => state_root, + "inventory" => &inventory, + _ => unreachable!(), + }; + fs::rename(exchanged, temp.join(format!("original-{component}"))).expect("exchange path component"); + fs::create_dir_all(&inventory).expect("replacement inventory path"); + for directory in [ancestor, state_root, inventory.as_path()] { + fs::set_permissions(directory, fs::Permissions::from_mode(0o700)).expect("replacement directory mode"); + } + } + + #[cfg(target_os = "linux")] + #[test] + fn path_exchanges_during_reads_and_writes_fail_closed() { + for operation in ["read", "write"] { + for component in ["ancestor", "state-root", "inventory"] { + let temp = safe_tempdir(); + let ancestor = temp.path().join("anchor"); + let state_root = ancestor.join("state"); + fs::create_dir_all(&state_root).expect("state root"); + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(&ancestor, fs::Permissions::from_mode(0o700)).expect("ancestor mode"); + fs::set_permissions(&state_root, fs::Permissions::from_mode(0o700)).expect("state mode"); + let store = InventoryStateStore::from_state_root(&state_root).expect("store"); + let captured_at = "2026-08-23T01:02:03Z"; + store + .publish_latest_sync(snapshot(), captured_at.to_owned(), &tokio_util::sync::CancellationToken::new()) + .expect("seed latest"); + + let error = if operation == "read" { + let now = chrono::DateTime::parse_from_rfc3339(captured_at) + .expect("time") + .with_timezone(&chrono::Utc); + store + .read_latest_after_open(now, || exchange_path_component(temp.path(), &state_root, component)) + .expect_err("path exchange during read") + } else { + let replacement = encode_envelope(snapshot(), "2026-08-23T02:02:03Z".to_owned()).expect("replacement"); + store + .replace_file_inner( + "latest.json", + &replacement, + || false, + None, + || exchange_path_component(temp.path(), &state_root, component), + ) + .expect_err("path exchange before commit") + }; + + assert!(matches!(error, InventoryError::PersistenceSecurity)); + assert!(!state_root.join("inventory/latest.json").exists()); + } + } + } + + #[cfg(target_os = "linux")] + #[tokio::test] + async fn legacy_pending_does_not_replace_a_newer_local_snapshot() { + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + let current = snapshot(); + store + .publish_latest_sync( + current.clone(), + "2026-08-23T01:02:03Z".to_owned(), + &tokio_util::sync::CancellationToken::new(), + ) + .expect("current latest"); + let legacy = InventorySnapshot::new("1.2.3", None, 2, 4, 900, 300, []).expect("legacy snapshot"); + + store + .ensure_latest(legacy, "2026-08-23T02:02:03Z".to_owned(), tokio_util::sync::CancellationToken::new()) + .await + .expect("existing latest remains authoritative"); + + let now = chrono::DateTime::parse_from_rfc3339("2026-08-23T02:02:03Z") + .expect("time") + .with_timezone(&chrono::Utc); + let persisted = store.read_latest(now).expect("latest"); + assert_eq!(persisted.snapshot, current); + assert_eq!(persisted.captured_at, "2026-08-23T01:02:03Z"); + assert_eq!(persisted.age, Duration::from_secs(60 * 60)); + } + + #[cfg(target_os = "linux")] + #[test] + fn precommit_failures_preserve_last_good_and_postcommit_sync_failure_keeps_new_file() { + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + let old = encode_envelope(snapshot(), "2026-08-23T01:02:03Z".to_owned()).expect("old envelope"); + store.replace_file("latest.json", &old, || false).expect("seed latest"); + let new_snapshot = InventorySnapshot::new("1.2.3", None, 2, 4, 1_001, 401, []).expect("new snapshot"); + let new = encode_envelope(new_snapshot, "2026-08-23T02:02:03Z".to_owned()).expect("new envelope"); + + for fault in [PersistFault::Write, PersistFault::TempSync, PersistFault::Rename] { + assert!(matches!( + store.replace_file_inner("latest.json", &new, || false, Some(fault), || {}), + Err(InventoryError::StateIo) + )); + assert_eq!(fs::read(temp.path().join("inventory/latest.json")).expect("last good"), old); + assert_eq!( + fs::read_dir(temp.path().join("inventory")) + .expect("inventory directory") + .filter_map(Result::ok) + .count(), + 1, + "failed staging must be removed" + ); + } assert!(matches!( - prepare_inventory_directory_with(&directory, create_directory, |_| Ok(())), - Err(InventoryError::StateIo { path, .. }) if path == root + store.replace_file_inner("latest.json", &new, || false, Some(PersistFault::DirectorySync), || {}), + Err(InventoryError::DurabilityAfterCommit) )); - assert!(!directory.exists()); + assert_eq!(fs::read(temp.path().join("inventory/latest.json")).expect("committed latest"), new); + } + + #[cfg(target_os = "linux")] + #[test] + fn cancellation_during_commit_does_not_interrupt_rename_or_sync() { + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + let replacement = encode_envelope(snapshot(), "2026-08-23T01:02:03Z".to_owned()).expect("envelope"); + let cancellation = tokio_util::sync::CancellationToken::new(); + + store + .replace_file_inner( + "latest.json", + &replacement, + || cancellation.is_cancelled(), + Some(PersistFault::CancelDuringCommit(cancellation.clone())), + || {}, + ) + .expect("commit ignores cancellation after its cancellation gate"); + + assert!(cancellation.is_cancelled()); + assert_eq!( + fs::read(temp.path().join("inventory/latest.json")).expect("committed latest"), + replacement + ); + } + + #[cfg(target_os = "linux")] + #[test] + fn reader_rejects_replacement_of_the_file_it_opened() { + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + let captured_at = "2026-08-23T01:02:03Z"; + let first = encode_envelope(snapshot(), captured_at.to_owned()).expect("first envelope"); + let second_snapshot = InventorySnapshot::new("1.2.3", None, 2, 4, 1_001, 401, []).expect("second snapshot"); + let second = encode_envelope(second_snapshot.clone(), captured_at.to_owned()).expect("second envelope"); + store.replace_file("latest.json", &first, || false).expect("seed latest"); + let now = chrono::DateTime::parse_from_rfc3339(captured_at) + .expect("time") + .with_timezone(&chrono::Utc); + + assert!(matches!( + store.read_latest_after_open(now, || { + store + .replace_file("latest.json", &second, || false) + .expect("replace opened file"); + }), + Err(InventoryError::PersistenceSecurity) + )); + assert_eq!(store.read_latest(now).expect("replacement").snapshot, second_snapshot); + } + + #[cfg(target_os = "linux")] + #[test] + fn concurrent_reader_observes_only_complete_old_or_new_envelopes() { + let temp = safe_tempdir(); + let store = InventoryStateStore::from_state_root(temp.path()).expect("store"); + let first = snapshot(); + let second = InventorySnapshot::new("1.2.3", None, 2, 4, 1_001, 401, []).expect("second snapshot"); + let captured_at = "2026-08-23T01:02:03Z"; + store + .replace_file( + "latest.json", + &encode_envelope(first.clone(), captured_at.to_owned()).expect("first envelope"), + || false, + ) + .expect("seed latest"); + let writer = store.clone(); + let first_writer = first.clone(); + let second_writer = second.clone(); + let start = Arc::new(std::sync::Barrier::new(2)); + let writer_start = start.clone(); + let thread = std::thread::spawn(move || { + writer_start.wait(); + for index in 0..100 { + let snapshot = if index % 2 == 0 { + first_writer.clone() + } else { + second_writer.clone() + }; + let bytes = encode_envelope(snapshot, captured_at.to_owned()).expect("envelope"); + writer.replace_file("latest.json", &bytes, || false).expect("atomic replace"); + } + }); + let now = chrono::DateTime::parse_from_rfc3339(captured_at) + .expect("time") + .with_timezone(&chrono::Utc); + start.wait(); + for _ in 0..100 { + match store.read_latest(now) { + Ok(observed) => assert!(observed.snapshot == first || observed.snapshot == second), + Err(InventoryError::PersistenceSecurity) => {} + Err(error) => panic!("reader observed neither a complete envelope nor a replacement: {error}"), + } + } + thread.join().expect("writer"); + let observed = store.read_latest(now).expect("stable final envelope").snapshot; + assert!(observed == first || observed == second); } } diff --git a/rustfs/src/connect/runtime.rs b/rustfs/src/connect/runtime.rs index eb7652eb1..7f2b60625 100644 --- a/rustfs/src/connect/runtime.rs +++ b/rustfs/src/connect/runtime.rs @@ -32,7 +32,6 @@ pub struct HeartbeatRuntime { shutdown: CancellationToken, status: watch::Receiver, task: Option>, - inventory: Option, } impl HeartbeatRuntime { @@ -40,22 +39,11 @@ impl HeartbeatRuntime { self.status.clone() } - pub(crate) fn with_inventory(mut self, inventory: Option) -> Self { - self.inventory = inventory; - self - } - pub async fn shutdown(mut self) { self.shutdown.cancel(); - if let Some(inventory) = self.inventory.as_ref() { - inventory.shutdown.cancel(); - } if let Some(task) = self.task.take() { let _ = task.await; } - if let Some(inventory) = self.inventory.take() { - inventory.shutdown().await; - } } } @@ -90,6 +78,20 @@ impl Drop for InventoryRuntime { } } +pub(crate) async fn shutdown_connect_runtimes(heartbeat: Option, inventory: Option) { + let heartbeat = async move { + if let Some(runtime) = heartbeat { + runtime.shutdown().await; + } + }; + let inventory = async move { + if let Some(runtime) = inventory { + runtime.shutdown().await; + } + }; + tokio::join!(heartbeat, inventory); +} + pub fn spawn_heartbeat_runtime( config: Option, parent_shutdown: &CancellationToken, @@ -101,6 +103,9 @@ where let Some(config) = config else { return Ok(None); }; + if !config.transport_enabled() { + return Ok(None); + } let sender = HeartbeatSender::new(config.clone())?; let store = HeartbeatStateStore::new(config.state_path.clone()); let lock = store.try_runtime_lock()?; @@ -163,7 +168,6 @@ where shutdown, status: status_rx, task: Some(task), - inventory: None, })) } @@ -184,9 +188,14 @@ where return Err(InventoryError::Schedule); } let retry_schedule = config.schedule; - let store = InventoryStateStore::from_heartbeat_path(&config.state_path)?; + let state_root = config.state_root().ok_or(InventoryError::StatePath)?; + let store = InventoryStateStore::from_state_root(state_root)?; let lock = store.try_runtime_lock()?; - let sender = InventorySender::new(config)?; + let sender = if config.transport_enabled() { + Some(InventorySender::new(config)?) + } else { + None + }; let shutdown = parent_shutdown.child_token(); let task_shutdown = shutdown.clone(); let (status_tx, status_rx) = watch::channel(InventoryStatus::Starting); @@ -197,8 +206,19 @@ where if task_shutdown.is_cancelled() { break; } - let pending = match store.pending().await { - Ok(Some(pending)) => pending, + let pending = match if sender.is_some() { store.pending().await } else { Ok(None) } { + Ok(Some((pending, captured_at))) => { + if let Err(error) = store + .ensure_latest(pending.snapshot().clone(), captured_at, task_shutdown.clone()) + .await + { + if matches!(error, InventoryError::Cancelled) && task_shutdown.is_cancelled() { + break; + } + return failed_inventory(&status_tx, error); + } + pending + } Ok(None) => { let snapshot = match cancellable(&task_shutdown, sample()).await { Some(Ok(snapshot)) => snapshot, @@ -218,6 +238,27 @@ where Ok(content_hash) => content_hash, Err(error) => return failed_inventory(&status_tx, error), }; + let captured_at = Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(); + if let Err(error) = store + .publish_latest(snapshot.clone(), captured_at, task_shutdown.clone()) + .await + { + if matches!(error, InventoryError::Cancelled) && task_shutdown.is_cancelled() { + break; + } + return failed_inventory(&status_tx, error); + } + if task_shutdown.is_cancelled() { + break; + } + if sender.is_none() { + backoff = retry_schedule.initial_backoff; + let _ = status_tx.send(InventoryStatus::Unchanged { content_hash }); + if sleep_or_cancel(&task_shutdown, schedule.cadence.saturating_add(jitter(schedule.jitter))).await { + break; + } + continue; + } match store.prepare(snapshot).await { Ok(Some(pending)) => pending, Ok(None) => { @@ -233,7 +274,15 @@ where } Err(error) => return failed_inventory(&status_tx, error), }; - let delivery = match cancellable(&task_shutdown, sender.send(&pending)).await { + let delivery = match cancellable( + &task_shutdown, + sender + .as_ref() + .expect("sender exists when delivery state is prepared") + .send(&pending), + ) + .await + { Some(Ok(delivery)) => delivery, Some(Err(error)) => return failed_inventory(&status_tx, error), None => break, @@ -261,14 +310,13 @@ where let _ = status_tx.send(InventoryStatus::BackingOff { delay }); delay } - InventoryDelivery::AuthenticationStopped { status, reason } => { - let _ = status_tx.send(InventoryStatus::AuthenticationStopped { status, reason }); + InventoryDelivery::AuthenticationStopped { status } => { + let _ = status_tx.send(InventoryStatus::AuthenticationStopped { status, reason: None }); return; } - InventoryDelivery::Rejected { status, reason } => { - let suffix = reason.map_or_else(String::new, |reason| format!("; reason={reason}")); + InventoryDelivery::Rejected { status } => { let _ = status_tx.send(InventoryStatus::Failed { - reason: format!("Connect rejected inventory with HTTP {status}{suffix}"), + reason: format!("connect_inventory_rejected_http_{status}"), }); return; } @@ -292,6 +340,46 @@ fn failed(status: &watch::Sender, error: HeartbeatError) { }); } +pub(crate) fn heartbeat_failure_reason(error: &HeartbeatError) -> &'static str { + use super::registration::CredentialValidationError; + + match error { + HeartbeatError::Endpoint => "connect_heartbeat_endpoint", + HeartbeatError::RootCertificate => "connect_heartbeat_root_certificate", + HeartbeatError::Schedule => "connect_heartbeat_schedule", + HeartbeatError::NotRegistered => "connect_heartbeat_not_registered", + HeartbeatError::IdentityMissing => "connect_heartbeat_identity_missing", + HeartbeatError::IdentityCertificate => "connect_heartbeat_identity_certificate", + HeartbeatError::CredentialName => "connect_heartbeat_credential_name", + HeartbeatError::CredentialExpired => "connect_heartbeat_credential_expired", + HeartbeatError::NodeSummary => "connect_heartbeat_node_summary", + HeartbeatError::SequenceExhausted => "connect_heartbeat_sequence_exhausted", + HeartbeatError::AlreadyRunning => "connect_heartbeat_already_running", + HeartbeatError::StateConflict => "connect_heartbeat_state_conflict", + HeartbeatError::StateIo { .. } => "connect_heartbeat_state_io", + HeartbeatError::StateInvalid { .. } => "connect_heartbeat_state_invalid", + HeartbeatError::StateCorrupt { .. } => "connect_heartbeat_state_corrupt", + #[cfg(unix)] + HeartbeatError::StatePermissions { .. } => "connect_heartbeat_state_permissions", + HeartbeatError::ResponseTooLarge => "connect_heartbeat_response_too_large", + HeartbeatError::Response => "connect_heartbeat_response", + HeartbeatError::Url(_) => "connect_heartbeat_url", + HeartbeatError::Transport(_) => "connect_heartbeat_transport", + HeartbeatError::Identity(_) => "connect_heartbeat_identity", + HeartbeatError::IdentityStore(_) => "connect_heartbeat_identity_store", + HeartbeatError::CredentialStore(_) => "connect_heartbeat_credential_store", + HeartbeatError::CredentialValidation(error) => match error { + CredentialValidationError::Certificate => "connect_heartbeat_credential_certificate", + CredentialValidationError::Chain => "connect_heartbeat_credential_chain", + CredentialValidationError::Identity => "connect_heartbeat_credential_identity", + CredentialValidationError::Key => "connect_heartbeat_credential_key", + CredentialValidationError::Validity => "connect_heartbeat_credential_validity", + CredentialValidationError::CertificateRequest => "connect_heartbeat_credential_request", + CredentialValidationError::RotationTranscript => "connect_heartbeat_credential_rotation_transcript", + }, + } +} + fn failed_inventory(status: &watch::Sender, error: InventoryError) { let _ = status.send(InventoryStatus::Failed { reason: error.to_string(), @@ -327,7 +415,51 @@ mod tests { use super::*; #[tokio::test] - async fn heartbeat_shutdown_cancels_inventory_before_waiting_for_heartbeat() { + async fn state_only_configuration_does_not_start_heartbeat() { + let temp = tempfile::tempdir_in(env!("CARGO_MANIFEST_DIR")).expect("safe temporary directory"); + let config = HeartbeatConfig::state_only(temp.path().to_path_buf()); + let shutdown = CancellationToken::new(); + + assert!( + spawn_heartbeat_runtime(Some(config), &shutdown, || { CoarseNodeSummary::new(1, 0, 0).expect("summary") }) + .expect("disabled transport") + .is_none() + ); + } + + #[test] + fn runtimes_expose_only_stable_machine_reasons() { + let error = HeartbeatError::StateIo { + path: std::path::PathBuf::from("/private/connect/state.json"), + source: std::io::Error::other("transport.internal"), + }; + assert_eq!(heartbeat_failure_reason(&error), "connect_heartbeat_state_io"); + assert_eq!( + heartbeat_failure_reason(&HeartbeatError::StateCorrupt { + path: std::path::PathBuf::from("/private/connect/state.json"), + }), + "connect_heartbeat_state_corrupt" + ); + assert_eq!( + heartbeat_failure_reason(&HeartbeatError::CredentialExpired), + "connect_heartbeat_credential_expired" + ); + assert_eq!( + heartbeat_failure_reason(&HeartbeatError::CredentialValidation( + super::super::registration::CredentialValidationError::Identity + )), + "connect_heartbeat_credential_identity" + ); + assert_eq!( + heartbeat_failure_reason(&HeartbeatError::CredentialValidation( + super::super::registration::CredentialValidationError::Key + )), + "connect_heartbeat_credential_key" + ); + } + + #[tokio::test] + async fn unified_shutdown_cancels_inventory_before_waiting_for_heartbeat() { let heartbeat_shutdown = CancellationToken::new(); let inventory_shutdown = CancellationToken::new(); let task_inventory_shutdown = inventory_shutdown.clone(); @@ -342,18 +474,18 @@ mod tests { task_inventory_shutdown.cancelled().await; let _ = inventory_stopped.send(()); }); - let runtime = HeartbeatRuntime { + let heartbeat = HeartbeatRuntime { shutdown: heartbeat_shutdown, status: heartbeat_status, task: Some(heartbeat_task), - inventory: Some(InventoryRuntime { - shutdown: inventory_shutdown, - status: inventory_status, - task: Some(inventory_task), - }), + }; + let inventory = InventoryRuntime { + shutdown: inventory_shutdown, + status: inventory_status, + task: Some(inventory_task), }; - let shutdown = tokio::spawn(runtime.shutdown()); + let shutdown = tokio::spawn(shutdown_connect_runtimes(Some(heartbeat), Some(inventory))); tokio::time::timeout(Duration::from_millis(250), stopped) .await .expect("inventory cancellation must not wait for heartbeat") diff --git a/rustfs/src/memory_observability.rs b/rustfs/src/memory_observability.rs index 7b3bfb285..00a657026 100644 --- a/rustfs/src/memory_observability.rs +++ b/rustfs/src/memory_observability.rs @@ -391,8 +391,8 @@ pub fn memory_observability_controller_snapshot(ctx: &CancellationToken) -> Memo /// Record the effective memory total and its basis (host or cgroup). fn record_effective_memory(total_bytes: u64) { - let basis = crate::cgroup_resources::memory_basis(); - metrics::gauge!("rustfs_memory_effective_total_bytes", "basis" => basis.to_string()).set(total_bytes as f64); + let basis = crate::cgroup_resources::container_resources().basis; + metrics::gauge!("rustfs_memory_effective_total_bytes", "basis" => basis).set(total_bytes as f64); } /// Record container resource detection results. diff --git a/rustfs/src/server/runtime.rs b/rustfs/src/server/runtime.rs index f5735b436..57fabb4c9 100644 --- a/rustfs/src/server/runtime.rs +++ b/rustfs/src/server/runtime.rs @@ -58,9 +58,9 @@ fn detect_cores() -> usize { #[inline] fn compute_default_worker_threads() -> usize { - // Physical cores are used by default (closer to CPU compute resources and cache topology) + // Cap at 16 worker threads for optimal small-object PUT performance. // Now cgroup-aware: in containers, uses the container's CPU limit - detect_cores() + detect_cores().min(16) } /// Default max_blocking_threads calculations based on sysinfo: @@ -76,7 +76,7 @@ fn compute_default_max_blocking_threads() -> usize { // Each blocking thread can use up to 1 MiB stack space const SMALL_CONTAINER_MAX_THREADS: usize = 256; - let cores = detect_cores(); + let cores = detect_cores().min(16); let mut threads = BASE_THREADS; let mut threshold = BASE_CORES; diff --git a/rustfs/src/startup_lifecycle.rs b/rustfs/src/startup_lifecycle.rs index ad5681366..5586511c1 100644 --- a/rustfs/src/startup_lifecycle.rs +++ b/rustfs/src/startup_lifecycle.rs @@ -14,6 +14,7 @@ use crate::storage_api::startup::lifecycle::ECStore; use crate::{ + connect::runtime::shutdown_connect_runtimes, server::{ServiceStateManager, ShutdownHandle, start_persisted_event_notifier_reconciler, wait_for_shutdown}, startup_iam::{IamBootstrapDisposition, publish_ready_for_iam_bootstrap}, startup_runtime_sources, @@ -129,6 +130,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec let StartupServiceRuntime { optional_runtimes, heartbeat, + inventory, iam_bootstrap, enable_scanner, } = service_runtime; @@ -163,9 +165,7 @@ pub(crate) async fn run_startup_runtime_lifecycle(lifecycle: StartupRuntimeLifec shutdown_token, ) .await; - if let Some(heartbeat) = heartbeat { - heartbeat.shutdown().await; - } + shutdown_connect_runtimes(heartbeat, inventory).await; if let Err(err) = event_notifier_reconciler.await { tracing::warn!( target: "rustfs::main::run", diff --git a/rustfs/src/startup_services.rs b/rustfs/src/startup_services.rs index f8c509d71..8a0292615 100644 --- a/rustfs/src/startup_services.rs +++ b/rustfs/src/startup_services.rs @@ -17,8 +17,9 @@ use crate::storage_api::startup::services::{ECStore, EndpointServerPools, Server use crate::{ config::Config, connect::{ - CoarseNodeSummary, HeartbeatConfig, HeartbeatRuntime, InventoryError, InventoryFlag, InventoryRuntime, InventorySchedule, - InventorySnapshot, spawn_heartbeat_runtime, spawn_inventory_runtime, + CoarseNodeSummary, HeartbeatConfig, HeartbeatError, HeartbeatRuntime, InventoryError, InventoryFlag, InventoryRuntime, + InventorySchedule, InventorySnapshot, runtime::heartbeat_failure_reason, spawn_heartbeat_runtime, + spawn_inventory_runtime, }, init::{init_buffer_profile_system, init_kms_system}, server::ServiceStateManager, @@ -40,6 +41,7 @@ use tokio_util::sync::CancellationToken; pub(crate) struct StartupServiceRuntime { pub(crate) optional_runtimes: OptionalRuntimeServices, pub(crate) heartbeat: Option, + pub(crate) inventory: Option, pub(crate) iam_bootstrap: IamBootstrapDisposition, pub(crate) enable_scanner: bool, } @@ -104,11 +106,11 @@ pub(crate) async fn init_startup_runtime_services( init_observability_runtime(store.clone(), ctx.clone()).await; let heartbeat = start_heartbeat_runtime(heartbeat_config.clone(), heartbeat_nodes, &ctx)?; let inventory = start_inventory_runtime(heartbeat_config, heartbeat_nodes, inventory_drives, store, &ctx)?; - let heartbeat = heartbeat.map(|heartbeat| heartbeat.with_inventory(inventory)); Ok(StartupServiceRuntime { optional_runtimes, heartbeat, + inventory, iam_bootstrap, enable_scanner, }) @@ -122,11 +124,18 @@ fn start_heartbeat_runtime( let Some(config) = config else { return Ok(None); }; + if !config.transport_enabled() { + return Ok(None); + } let summary = u16::try_from(node_count.unwrap_or_default()) .ok() .and_then(|total| CoarseNodeSummary::new(total, 0, 0).ok()) .ok_or_else(|| std::io::Error::other("Connect heartbeat node count is outside protocol bounds"))?; - spawn_heartbeat_runtime(Some(config), shutdown, move || summary).map_err(std::io::Error::other) + spawn_heartbeat_runtime(Some(config), shutdown, move || summary).map_err(startup_heartbeat_error) +} + +fn startup_heartbeat_error(error: HeartbeatError) -> std::io::Error { + std::io::Error::other(heartbeat_failure_reason(&error)) } fn start_inventory_runtime( @@ -318,6 +327,16 @@ fn aggregate_inventory_capacity( mod tests { use super::*; + #[test] + fn heartbeat_startup_errors_expose_only_stable_codes() { + let error = startup_heartbeat_error(HeartbeatError::StateIo { + path: std::path::PathBuf::from("/private/connect/canary/state.json"), + source: std::io::Error::other("private-source-canary"), + }); + + assert_eq!(error.to_string(), "connect_heartbeat_state_io"); + } + fn disk(state: &str, runtime_state: Option<&str>, disk_index: i32) -> rustfs_madmin::Disk { rustfs_madmin::Disk { state: state.to_owned(), diff --git a/rustfs/tests/connect_inventory.rs b/rustfs/tests/connect_inventory.rs index 3aa28f3ca..0261320fc 100644 --- a/rustfs/tests/connect_inventory.rs +++ b/rustfs/tests/connect_inventory.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#![cfg(target_os = "linux")] + use std::collections::VecDeque; use std::fs; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -46,6 +48,10 @@ const CLUSTER_UID: &str = "0198f4b0-2b00-7d20-9e31-3f4a5b6c7d81"; const DEVICE_UID: &str = "0198f4b0-3c00-7e30-8f41-4a5b6c7d8e92"; const SNAPSHOT_UID: &str = "0198f4b0-4d00-7f40-9051-5b6c7d8e9fa3"; +fn safe_tempdir() -> tempfile::TempDir { + tempfile::tempdir_in(env!("CARGO_MANIFEST_DIR")).expect("safe temporary directory") +} + struct TestPki { root_params: CertificateParams, root_key: KeyPair, @@ -244,6 +250,7 @@ fn config(temp: &tempfile::TempDir, pki: &TestPki, server: &TestServer) -> Heart if let Err(error) = fs::create_dir(temp.path().join("private-config-secret")) { assert_eq!(error.kind(), std::io::ErrorKind::AlreadyExists, "Connect state root"); } + private_directory_mode(&temp.path().join("private-config-secret")); HeartbeatConfig { endpoint: server.endpoint.clone(), root_ca_pem: pki.root_pem.as_bytes().to_vec(), @@ -394,6 +401,38 @@ fn connect_inventory_bounds_fail_instead_of_truncating_or_inventing_values() { )); } +#[cfg(target_os = "linux")] +#[tokio::test] +async fn connect_inventory_state_only_persists_without_constructing_transport() { + let temp = safe_tempdir(); + let state = temp.path().join("state"); + fs::create_dir(&state).expect("state root"); + private_directory_mode(&state); + let config = HeartbeatConfig::new( + "", + Vec::new(), + IdentityStore::new(state.join("identity")), + CredentialStore::new(state.join("credential")), + state.join("heartbeat/state.json"), + ); + let shutdown = CancellationToken::new(); + let runtime = spawn_inventory_runtime(Some(config), schedule(), &shutdown, || std::future::ready(Ok(snapshot()))) + .expect("state-only inventory") + .expect("configured inventory"); + let mut status = runtime.status(); + + assert!(matches!( + wait_for(&mut status, |status| matches!(status, InventoryStatus::Unchanged { .. })).await, + InventoryStatus::Unchanged { .. } + )); + let envelope: Value = serde_json::from_slice(&fs::read(state.join("inventory/latest.json")).expect("latest inventory")) + .expect("latest envelope"); + assert_eq!(envelope["formatVersion"], "v1"); + assert_eq!(envelope["snapshot"], serde_json::to_value(snapshot()).expect("snapshot JSON")); + assert_eq!(envelope.as_object().expect("envelope object").len(), 4); + runtime.shutdown().await; +} + #[tokio::test] async fn connect_inventory_rejects_deserialized_invalid_snapshots_before_state_or_network() { let pki = TestPki::new(); @@ -409,7 +448,7 @@ async fn connect_inventory_rejects_deserialized_invalid_snapshots_before_state_o "coarseFlags": [] })) .expect("serde should not bypass the runtime validation boundary"); - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let shutdown = CancellationToken::new(); let runtime = spawn_inventory_runtime(Some(config(&temp, &pki, &server)), schedule(), &shutdown, move || { std::future::ready(Ok(invalid.clone())) @@ -420,7 +459,7 @@ async fn connect_inventory_rejects_deserialized_invalid_snapshots_before_state_o assert!(matches!( wait_for(&mut status, |status| matches!(status, InventoryStatus::Failed { .. })).await, - InventoryStatus::Failed { reason } if reason.contains("version is outside protocol bounds") + InventoryStatus::Failed { reason } if reason == "connect_inventory_snapshot_version" )); assert!(!temp.path().join("private-config-secret/inventory/state.json").exists()); runtime.shutdown().await; @@ -433,7 +472,7 @@ async fn connect_inventory_restart_replays_the_pending_request_and_then_skips_un let pki = TestPki::new(); let content_hash = snapshot().content_hash().expect("content hash"); let first_server = server(&pki, vec![Reply::error(StatusCode::SERVICE_UNAVAILABLE, "UNAVAILABLE")]).await; - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let shutdown = CancellationToken::new(); let samples = Arc::new(AtomicUsize::new(0)); let sampled = samples.clone(); @@ -451,7 +490,40 @@ async fn connect_inventory_restart_replays_the_pending_request_and_then_skips_un )); assert_eq!(samples.load(Ordering::Relaxed), 1); let original = first_server.seen.lock().expect("seen lock")[0].clone(); + let latest: Value = serde_json::from_slice( + &fs::read(temp.path().join("private-config-secret/inventory/latest.json")).expect("latest inventory"), + ) + .expect("latest envelope"); + assert_eq!(latest["snapshot"], serde_json::to_value(snapshot()).expect("snapshot JSON")); + for field in [ + "rustfsVersion", + "osVersion", + "nodeCount", + "driveCount", + "capacityTotalBytes", + "capacityUsedBytes", + "coarseFlags", + ] { + assert_eq!(latest["snapshot"][field], original[field]); + } runtime.shutdown().await; + let state = temp.path().join("private-config-secret/inventory/state.json"); + let legacy_persisted_at = std::time::SystemTime::now() - Duration::from_secs(60 * 60); + fs::File::options() + .write(true) + .open(&state) + .expect("pending state") + .set_times(std::fs::FileTimes::new().set_modified(legacy_persisted_at)) + .expect("legacy pending timestamp"); + let legacy_persisted_at = chrono::DateTime::::from( + fs::metadata(&state) + .and_then(|metadata| metadata.modified()) + .expect("persisted pending timestamp"), + ) + .format("%Y-%m-%dT%H:%M:%SZ") + .to_string(); + fs::remove_file(temp.path().join("private-config-secret/inventory/latest.json")) + .expect("simulate a pending snapshot created before local persistence"); let mut limited = Reply::error(StatusCode::TOO_MANY_REQUESTS, "RATE_LIMITED"); limited.retry_after = Some("0"); @@ -473,10 +545,17 @@ async fn connect_inventory_restart_replays_the_pending_request_and_then_skips_un if accepted == content_hash && received_at == "2026-08-22T01:02:03Z" )); assert_eq!(restart_samples.load(Ordering::Relaxed), 0); + let restored_latest: Value = serde_json::from_slice( + &fs::read(temp.path().join("private-config-secret/inventory/latest.json")).expect("restored latest inventory"), + ) + .expect("restored latest envelope"); + assert_eq!(restored_latest["snapshot"], serde_json::to_value(snapshot()).expect("snapshot JSON")); + assert_eq!(restored_latest["capturedAt"], legacy_persisted_at); let delivered = restart_server.seen.lock().expect("seen lock").clone(); assert_eq!(delivered, vec![original.clone(), original.clone()]); assert_eq!(original["sequence"], 0); let encoded = serde_json::to_string(&original).expect("request JSON"); + let persisted = serde_json::to_string(&latest).expect("persisted JSON"); for forbidden in [ "private-config-secret", "BEGIN CERTIFICATE", @@ -486,9 +565,17 @@ async fn connect_inventory_restart_replays_the_pending_request_and_then_skips_un "path", ] { assert!(!encoded.contains(forbidden), "request exposed {forbidden}"); + assert!(!persisted.contains(forbidden), "persisted inventory exposed {forbidden}"); } assert_eq!(original.as_object().expect("request object").len(), 10); restart.shutdown().await; + #[cfg(target_os = "linux")] + let latest_before_unchanged = { + use std::os::unix::fs::MetadataExt as _; + fs::metadata(temp.path().join("private-config-secret/inventory/latest.json")) + .expect("latest metadata") + .ino() + }; let unchanged_samples = Arc::new(AtomicUsize::new(0)); let sampled = unchanged_samples.clone(); @@ -505,6 +592,17 @@ async fn connect_inventory_restart_replays_the_pending_request_and_then_skips_un )); assert_eq!(unchanged_samples.load(Ordering::Relaxed), 1); assert_eq!(restart_server.seen.lock().expect("seen lock").len(), 2); + #[cfg(target_os = "linux")] + { + use std::os::unix::fs::MetadataExt as _; + assert_ne!( + fs::metadata(temp.path().join("private-config-secret/inventory/latest.json")) + .expect("refreshed latest metadata") + .ino(), + latest_before_unchanged, + "a complete unchanged sample must refresh the local envelope" + ); + } unchanged.shutdown().await; } @@ -512,7 +610,7 @@ async fn connect_inventory_restart_replays_the_pending_request_and_then_skips_un async fn connect_inventory_disconnect_retries_without_resampling() { let pki = TestPki::new(); let unavailable = server(&pki, Vec::new()).await; - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let config = config(&temp, &pki, &unavailable); drop(unavailable); let shutdown = CancellationToken::new(); @@ -542,7 +640,7 @@ async fn connect_inventory_retries_an_incomplete_sample_before_delivery() { let pki = TestPki::new(); let content_hash = snapshot().content_hash().expect("content hash"); let server = server(&pki, vec![Reply::ok(&content_hash)]).await; - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let shutdown = CancellationToken::new(); let samples = Arc::new(AtomicUsize::new(0)); let sampled = samples.clone(); @@ -561,6 +659,11 @@ async fn connect_inventory_retries_an_incomplete_sample_before_delivery() { .expect("configured inventory"); let mut status = runtime.status(); + assert!(matches!( + wait_for(&mut status, |status| matches!(status, InventoryStatus::BackingOff { .. })).await, + InventoryStatus::BackingOff { .. } + )); + assert!(!temp.path().join("private-config-secret/inventory/latest.json").exists()); assert!(matches!( wait_for(&mut status, |status| matches!(status, InventoryStatus::Online { .. })).await, InventoryStatus::Online { content_hash: accepted, .. } if accepted == content_hash @@ -575,7 +678,7 @@ async fn connect_inventory_unchanged_sample_resets_incomplete_backoff() { let pki = TestPki::new(); let content_hash = snapshot().content_hash().expect("content hash"); let server = server(&pki, vec![Reply::ok(&content_hash)]).await; - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let shutdown = CancellationToken::new(); let config = config(&temp, &pki, &server); let seed = spawn_inventory_runtime(Some(config.clone()), schedule(), &shutdown, || std::future::ready(Ok(snapshot()))) @@ -645,7 +748,7 @@ async fn connect_inventory_unchanged_sample_resets_incomplete_backoff() { async fn connect_inventory_revoked_device_stops_without_retrying() { let pki = TestPki::new(); let server = server(&pki, vec![Reply::error(StatusCode::UNAUTHORIZED, "DEVICE_REVOKED")]).await; - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let shutdown = CancellationToken::new(); let runtime = spawn_inventory_runtime(Some(config(&temp, &pki, &server)), schedule(), &shutdown, || { std::future::ready(Ok(snapshot())) @@ -656,7 +759,10 @@ async fn connect_inventory_revoked_device_stops_without_retrying() { assert!(matches!( wait_for(&mut status, |status| matches!(status, InventoryStatus::AuthenticationStopped { .. })).await, - InventoryStatus::AuthenticationStopped { status: 401, reason: Some(reason) } if reason == "DEVICE_REVOKED" + InventoryStatus::AuthenticationStopped { + status: 401, + reason: None + } )); assert_eq!(server.seen.lock().expect("seen lock").len(), 1); runtime.shutdown().await; @@ -666,10 +772,11 @@ async fn connect_inventory_revoked_device_stops_without_retrying() { async fn connect_inventory_sequence_overflow_fails_before_sampling_or_network_delivery() { let pki = TestPki::new(); let server = server(&pki, Vec::new()).await; - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let config = config(&temp, &pki, &server); let state = temp.path().join("private-config-secret/inventory/state.json"); fs::create_dir_all(state.parent().expect("state directory")).expect("create state directory"); + private_directory_mode(state.parent().expect("state directory")); fs::write( &state, br#"{"nextSequence":9007199254740992,"pending":null,"lastAcceptedContentHash":null}"#, @@ -689,7 +796,7 @@ async fn connect_inventory_sequence_overflow_fails_before_sampling_or_network_de assert!(matches!( wait_for(&mut status, |status| matches!(status, InventoryStatus::Failed { .. })).await, - InventoryStatus::Failed { reason } if reason.contains("sequence is exhausted") + InventoryStatus::Failed { reason } if reason == "connect_inventory_sequence_exhausted" )); assert_eq!(samples.load(Ordering::Relaxed), 0); assert!(server.seen.lock().expect("seen lock").is_empty()); @@ -702,10 +809,11 @@ async fn connect_inventory_rejects_noncanonical_persisted_snapshots_before_deliv let server = server(&pki, Vec::new()).await; for invalid_case in ["flags", "os-version"] { - let temp = tempfile::tempdir().expect("tempdir"); + let temp = safe_tempdir(); let config = config(&temp, &pki, &server); let state = temp.path().join("private-config-secret/inventory/state.json"); fs::create_dir_all(state.parent().expect("state directory")).expect("create state directory"); + private_directory_mode(state.parent().expect("state directory")); let mut pending = json!({ "protocolVersion": "v1", "requestId": "00000000-0000-4000-8000-000000000001", @@ -753,7 +861,7 @@ async fn connect_inventory_rejects_noncanonical_persisted_snapshots_before_deliv matches!(status, InventoryStatus::Failed { .. } | InventoryStatus::BackingOff { .. }) }) .await, - InventoryStatus::Failed { reason } if reason.contains("violates the protocol invariants") + InventoryStatus::Failed { reason } if reason == "connect_inventory_state_corrupt" )); assert_eq!(samples.load(Ordering::Relaxed), 0); runtime.shutdown().await; @@ -762,11 +870,20 @@ async fn connect_inventory_rejects_noncanonical_persisted_snapshots_before_deliv assert!(server.seen.lock().expect("seen lock").is_empty()); } -#[cfg(unix)] +#[cfg(target_os = "linux")] fn private_mode(path: &std::path::Path) { use std::os::unix::fs::PermissionsExt as _; fs::set_permissions(path, fs::Permissions::from_mode(0o600)).expect("private permissions"); } -#[cfg(not(unix))] +#[cfg(target_os = "linux")] +fn private_directory_mode(path: &std::path::Path) { + use std::os::unix::fs::PermissionsExt as _; + fs::set_permissions(path, fs::Permissions::from_mode(0o700)).expect("private directory permissions"); +} + +#[cfg(not(target_os = "linux"))] fn private_mode(_path: &std::path::Path) {} + +#[cfg(not(target_os = "linux"))] +fn private_directory_mode(_path: &std::path::Path) {} diff --git a/scripts/fuzz/run.sh b/scripts/fuzz/run.sh index c02f3c22b..a8ef91c80 100755 --- a/scripts/fuzz/run.sh +++ b/scripts/fuzz/run.sh @@ -25,6 +25,7 @@ # Environment variables: # FUZZ_TARGET — run only this target (default: all smoke targets) # MAX_TOTAL_TIME — seconds to fuzz per target (default: 60) +# FUZZ_SEED — replay one libFuzzer seed (default: generate and record one per target) # ARTIFACT_ROOT — artifact output directory (default: artifacts) # BUILD_ONLY — set to 1 to skip fuzz runs (default: 0) # SKIP_BUILD — set to 1 to skip build phase (default: 0) @@ -37,6 +38,7 @@ SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/../.." && pwd) FUZZ_DIR="$REPO_ROOT/fuzz" MAX_TOTAL_TIME=${MAX_TOTAL_TIME:-60} +FUZZ_SEED=${FUZZ_SEED:-} ARTIFACT_ROOT=${ARTIFACT_ROOT:-artifacts} FUZZ_TARGET=${FUZZ_TARGET:-} BUILD_ONLY=${BUILD_ONLY:-0} @@ -44,6 +46,15 @@ SKIP_BUILD=${SKIP_BUILD:-0} USE_PREBUILT_BINARY=${USE_PREBUILT_BINARY:-0} PREBUILT_BINARY_DIR=${PREBUILT_BINARY_DIR:-} +if [ -n "$FUZZ_SEED" ]; then + case "$FUZZ_SEED" in + 0*|*[!0-9]*) + echo "FUZZ_SEED must be a positive decimal integer without leading zeroes: $FUZZ_SEED" >&2 + exit 1 + ;; + esac +fi + cd "$FUZZ_DIR" mkdir -p "$ARTIFACT_ROOT" @@ -75,6 +86,35 @@ for target in $targets; do mkdir -p "$artifact_dir" mkdir -p "$corpus_dir" + seed="$FUZZ_SEED" + if [ -z "$seed" ]; then + seed=$(printf '%s\n' "${GITHUB_RUN_ID:-local}:${GITHUB_RUN_ATTEMPT:-0}:$target:$(date +%s):$$" | cksum | awk '{print $1}') + if [ "$seed" = "0" ]; then + seed=1 + fi + fi + revision=$(git -C "$REPO_ROOT" rev-parse HEAD 2>/dev/null || printf 'unknown') + if [ "$revision" = "unknown" ]; then + git_dirty="unknown" + elif [ -n "$(git -C "$REPO_ROOT" status --porcelain --untracked-files=normal 2>/dev/null)" ]; then + git_dirty="true" + else + git_dirty="false" + fi + if [ "$USE_PREBUILT_BINARY" = "1" ]; then + runner_mode="prebuilt" + else + runner_mode="cargo-fuzz" + fi + { + printf 'target=%s\n' "$target" + printf 'seed=%s\n' "$seed" + printf 'max_total_time=%s\n' "$MAX_TOTAL_TIME" + printf 'git_revision=%s\n' "$revision" + printf 'git_dirty=%s\n' "$git_dirty" + printf 'runner_mode=%s\n' "$runner_mode" + } > "$artifact_dir/run-manifest.txt" + if [ "$USE_PREBUILT_BINARY" = "1" ]; then binary_dir="$PREBUILT_BINARY_DIR" if [ -z "$binary_dir" ]; then @@ -89,11 +129,11 @@ for target in $targets; do echo "Missing executable prebuilt fuzz binary: $binary_path" >&2 exit 1 fi - echo "==> $binary_path (-max_total_time=$MAX_TOTAL_TIME, -artifact_prefix=$artifact_dir/, corpus=$corpus_dir)" - "$binary_path" -max_total_time="$MAX_TOTAL_TIME" -artifact_prefix="$artifact_dir/" "$corpus_dir" + echo "==> $binary_path (-max_total_time=$MAX_TOTAL_TIME, -seed=$seed, -artifact_prefix=$artifact_dir/, corpus=$corpus_dir)" + "$binary_path" -max_total_time="$MAX_TOTAL_TIME" -seed="$seed" -artifact_prefix="$artifact_dir/" "$corpus_dir" continue fi - echo "==> cargo +nightly fuzz run $target (-max_total_time=$MAX_TOTAL_TIME, -artifact_prefix=$artifact_dir/)" - cargo +nightly fuzz run "$target" -- -max_total_time="$MAX_TOTAL_TIME" -artifact_prefix="$artifact_dir/" + echo "==> cargo +nightly fuzz run $target (-max_total_time=$MAX_TOTAL_TIME, -seed=$seed, -artifact_prefix=$artifact_dir/)" + cargo +nightly fuzz run "$target" -- -max_total_time="$MAX_TOTAL_TIME" -seed="$seed" -artifact_prefix="$artifact_dir/" done diff --git a/scripts/run.sh b/scripts/run.sh index 7954c2be1..dbdb78ed4 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -52,10 +52,6 @@ if [ -z "${RUSTFS_UNSAFE_BYPASS_DISK_CHECK+x}" ] && [ -z "${MINIO_CI+x}" ]; then export RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true fi -if [ -z "${RUSTFS_ALLOCATOR_RECLAIM_ENABLED+x}" ]; then - export RUSTFS_ALLOCATOR_RECLAIM_ENABLED=true -fi - export RUSTFS_VOLUMES="${RUSTFS_VOLUMES:-./target/volume/test{1...4}}" # export RUSTFS_VOLUMES="./target/volume/test" export RUSTFS_ADDRESS="${RUSTFS_ADDRESS:-127.0.0.1:9000}" diff --git a/scripts/security/check_persist_credentials.sh b/scripts/security/check_persist_credentials.sh index b24bf3523..785250f56 100755 --- a/scripts/security/check_persist_credentials.sh +++ b/scripts/security/check_persist_credentials.sh @@ -6,8 +6,8 @@ # where it stays for the rest of the job. On this repository that matters more # than usual: pull_request jobs run on self-hosted runners and execute the PR's # own build.rs, proc-macros and tests, any of which can read that file. The -# Test and Lint job additionally holds actions: write, so its token can cancel -# runs and delete the Actions caches the whole pipeline depends on. +# post-failure cancellation job holds actions: write but never checks out +# repository code, so untrusted build scripts cannot read that token from Git. # # A checkout is exempt only when the token IS the credential the job needs — # helm-package pushes to rustfs/helm with it. Mark those with a comment diff --git a/scripts/test/fixtures/keycloak-rustfs-ci-realm.json b/scripts/test/fixtures/keycloak-rustfs-ci-realm.json new file mode 100644 index 000000000..fd6d567e7 --- /dev/null +++ b/scripts/test/fixtures/keycloak-rustfs-ci-realm.json @@ -0,0 +1,42 @@ +{ + "realm": "rustfs-ci", + "enabled": true, + "sslRequired": "none", + "accessTokenLifespan": 300, + "clients": [ + { + "clientId": "rustfs-ci", + "secret": "rustfs-ci-secret", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true + }, + { + "clientId": "wrong-audience", + "secret": "wrong-audience-secret", + "enabled": true, + "protocol": "openid-connect", + "publicClient": false, + "standardFlowEnabled": false, + "directAccessGrantsEnabled": true + } + ], + "users": [ + { + "id": "00000000-0000-0000-0000-000000000001", + "username": "alice", + "email": "alice@example.test", + "emailVerified": true, + "enabled": true, + "credentials": [ + { + "type": "password", + "value": "alice-password", + "temporary": false + } + ] + } + ] +} diff --git a/scripts/test/oidc_keycloak_live.sh b/scripts/test/oidc_keycloak_live.sh new file mode 100755 index 000000000..db2e51197 --- /dev/null +++ b/scripts/test/oidc_keycloak_live.sh @@ -0,0 +1,241 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +RUSTFS_BINARY="${1:-${ROOT_DIR}/target/debug/rustfs}" +REALM_FIXTURE="${ROOT_DIR}/scripts/test/fixtures/keycloak-rustfs-ci-realm.json" +KEYCLOAK_IMAGE="${KEYCLOAK_IMAGE:-quay.io/keycloak/keycloak@sha256:6a7217a100bd3e5de4063a27a538ef999a3c5a88c4b4ec0ffc0a642aee7b2597}" +WORK_DIR="${RUNNER_TEMP:-/tmp}/rustfs-keycloak-live-${$}" +KEYCLOAK_CONTAINER="rustfs-keycloak-live-${$}" +RUSTFS_PID="" +mkdir -p "${WORK_DIR}" + +cleanup() { + local status=$? + if [[ -n "${RUSTFS_PID}" ]] && kill -0 "${RUSTFS_PID}" 2>/dev/null; then + kill "${RUSTFS_PID}" 2>/dev/null || true + wait "${RUSTFS_PID}" 2>/dev/null || true + fi + if [[ "${status}" -ne 0 ]]; then + docker logs "${KEYCLOAK_CONTAINER}" >"${WORK_DIR}/keycloak.log" 2>&1 || true + echo "live-gate logs retained in ${WORK_DIR}" >&2 + fi + docker rm -f "${KEYCLOAK_CONTAINER}" >/dev/null 2>&1 || true + if [[ "${status}" -eq 0 ]]; then + rm -rf "${WORK_DIR}" + fi +} +trap cleanup EXIT INT TERM + +for command in curl docker python3 awscurl; do + command -v "${command}" >/dev/null || { + echo "missing required command: ${command}" >&2 + exit 1 + } +done +[[ -x "${RUSTFS_BINARY}" ]] || { + echo "RustFS binary is not executable: ${RUSTFS_BINARY}" >&2 + exit 1 +} +[[ -f "${REALM_FIXTURE}" ]] || { + echo "Keycloak realm fixture is missing: ${REALM_FIXTURE}" >&2 + exit 1 +} + +free_port() { + python3 - <<'PY' +import socket + +with socket.socket() as sock: + sock.bind(("127.0.0.1", 0)) + print(sock.getsockname()[1]) +PY +} + +KEYCLOAK_PORT="$(free_port)" +RUSTFS_PORT="$(free_port)" +KEYCLOAK_ORIGIN="http://127.0.0.1:${KEYCLOAK_PORT}" +ISSUER="${KEYCLOAK_ORIGIN}/realms/rustfs-ci" +DISCOVERY_URL="${ISSUER}/.well-known/openid-configuration" +RUSTFS_ORIGIN="http://127.0.0.1:${RUSTFS_PORT}" + +docker run --detach --rm \ + --name "${KEYCLOAK_CONTAINER}" \ + --memory 1g \ + --publish "127.0.0.1:${KEYCLOAK_PORT}:8080" \ + --env KC_BOOTSTRAP_ADMIN_USERNAME=admin \ + --env KC_BOOTSTRAP_ADMIN_PASSWORD=admin \ + --env KC_HEALTH_ENABLED=true \ + --env "KC_HOSTNAME=${KEYCLOAK_ORIGIN}" \ + --volume "${REALM_FIXTURE}:/opt/keycloak/data/import/rustfs-ci-realm.json:ro" \ + "${KEYCLOAK_IMAGE}" start-dev --import-realm >/dev/null + +for _ in $(seq 1 120); do + if curl --noproxy '*' -fsS "${DISCOVERY_URL}" >"${WORK_DIR}/discovery.json" 2>/dev/null; then + break + fi + if [[ "$(docker inspect -f '{{.State.Running}}' "${KEYCLOAK_CONTAINER}" 2>/dev/null || true)" != "true" ]]; then + docker logs "${KEYCLOAK_CONTAINER}" >&2 || true + echo "Keycloak exited before discovery became ready" >&2 + exit 1 + fi + sleep 1 +done +curl --noproxy '*' -fsS "${DISCOVERY_URL}" >"${WORK_DIR}/discovery.json" +python3 - "${WORK_DIR}/discovery.json" "${ISSUER}" <<'PY' +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + discovery = json.load(source) +expected_issuer = sys.argv[2] +assert discovery.get("issuer") == expected_issuer, discovery +for field in ("token_endpoint", "jwks_uri"): + assert discovery.get(field), discovery +PY + +mkdir -p "${WORK_DIR}/rustfs-data" +env \ + NO_PROXY=127.0.0.1,localhost \ + RUSTFS_ACCESS_KEY=rustfsadmin \ + RUSTFS_SECRET_KEY=rustfsadmin \ + RUSTFS_OUTBOUND_ALLOW_ORIGINS="${KEYCLOAK_ORIGIN}" \ + RUSTFS_IDENTITY_OPENID_ENABLE=on \ + RUSTFS_IDENTITY_OPENID_CONFIG_URL="${DISCOVERY_URL}" \ + RUSTFS_IDENTITY_OPENID_ISSUER="${ISSUER}" \ + RUSTFS_IDENTITY_OPENID_CLIENT_ID=rustfs-ci \ + RUSTFS_IDENTITY_OPENID_CLIENT_SECRET=rustfs-ci-secret \ + RUSTFS_IDENTITY_OPENID_SCOPES=openid,profile,email \ + RUSTFS_IDENTITY_OPENID_ROLE_POLICY=consoleAdmin \ + "${RUSTFS_BINARY}" --address "127.0.0.1:${RUSTFS_PORT}" "${WORK_DIR}/rustfs-data" \ + >"${WORK_DIR}/rustfs.log" 2>&1 & +RUSTFS_PID=$! + +for _ in $(seq 1 90); do + if curl --noproxy '*' -fsS "${RUSTFS_ORIGIN}/health/ready" >/dev/null 2>&1; then + break + fi + if ! kill -0 "${RUSTFS_PID}" 2>/dev/null; then + cat "${WORK_DIR}/rustfs.log" >&2 + echo "RustFS exited before becoming ready" >&2 + exit 1 + fi + sleep 1 +done +curl --noproxy '*' -fsS "${RUSTFS_ORIGIN}/health/ready" >/dev/null + +token_for_client() { + local client_id="$1" + local client_secret="$2" + local response_file="${WORK_DIR}/token-${client_id}.json" + curl --noproxy '*' -fsS "${ISSUER}/protocol/openid-connect/token" \ + --data-urlencode grant_type=password \ + --data-urlencode "client_id=${client_id}" \ + --data-urlencode "client_secret=${client_secret}" \ + --data-urlencode username=alice \ + --data-urlencode password=alice-password \ + --data-urlencode scope=openid \ + >"${response_file}" + python3 - "${response_file}" "${ISSUER}" "${client_id}" <<'PY' +import base64 +import json +import sys + +with open(sys.argv[1], encoding="utf-8") as source: + token = json.load(source)["id_token"] +payload = token.split(".")[1] +payload += "=" * (-len(payload) % 4) +claims = json.loads(base64.urlsafe_b64decode(payload)) +assert claims["iss"] == sys.argv[2], claims +audience = claims["aud"] +if isinstance(audience, str): + audience = [audience] +assert sys.argv[3] in audience, claims +print(token) +PY +} + +GOOD_TOKEN="$(token_for_client rustfs-ci rustfs-ci-secret)" +GOOD_STATUS="$(curl --noproxy '*' -sS \ + -D "${WORK_DIR}/sts-good.headers" \ + -o "${WORK_DIR}/sts-good.xml" \ + -w '%{http_code}' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -X POST "${RUSTFS_ORIGIN}/" \ + --data-urlencode Action=AssumeRoleWithWebIdentity \ + --data-urlencode Version=2011-06-15 \ + --data-urlencode DurationSeconds=900 \ + --data-urlencode "WebIdentityToken=${GOOD_TOKEN}")" +[[ "${GOOD_STATUS}" == 200 ]] || { + cat "${WORK_DIR}/sts-good.xml" >&2 + cat "${WORK_DIR}/rustfs.log" >&2 + echo "expected valid Keycloak token to return HTTP 200, got ${GOOD_STATUS}" >&2 + exit 1 +} +grep -Eiq '^content-type: application/xml' "${WORK_DIR}/sts-good.headers" + +IFS=$'\t' read -r STS_ACCESS_KEY STS_SECRET_KEY STS_SESSION_TOKEN < <( + python3 - "${WORK_DIR}/sts-good.xml" <<'PY' +import sys +import xml.etree.ElementTree as ET + +root = ET.parse(sys.argv[1]).getroot() +values = {} +for element in root.iter(): + values[element.tag.rsplit("}", 1)[-1]] = element.text or "" +for field in ("AccessKeyId", "SecretAccessKey", "SessionToken", "Expiration", "SubjectFromWebIdentityToken"): + assert values.get(field), values +print("\t".join(values[field] for field in ("AccessKeyId", "SecretAccessKey", "SessionToken"))) +PY +) + +awscurl --fail-with-body --service s3 --region us-east-1 \ + --access_key "${STS_ACCESS_KEY}" \ + --secret_key "${STS_SECRET_KEY}" \ + --security_token "${STS_SESSION_TOKEN}" \ + "${RUSTFS_ORIGIN}/" >"${WORK_DIR}/list-buckets.xml" +grep -q '&2 + echo "expected tampered token to return HTTP 403, got ${TAMPERED_STATUS}" >&2 + exit 1 +} +grep -q 'AccessDenied' "${WORK_DIR}/sts-tampered.xml" + +BAD_TOKEN="$(token_for_client wrong-audience wrong-audience-secret)" +BAD_STATUS="$(curl --noproxy '*' -sS \ + -o "${WORK_DIR}/sts-bad.xml" \ + -w '%{http_code}' \ + -H 'Content-Type: application/x-www-form-urlencoded' \ + -X POST "${RUSTFS_ORIGIN}/" \ + --data-urlencode Action=AssumeRoleWithWebIdentity \ + --data-urlencode Version=2011-06-15 \ + --data-urlencode DurationSeconds=900 \ + --data-urlencode "WebIdentityToken=${BAD_TOKEN}")" +[[ "${BAD_STATUS}" == 403 ]] || { + cat "${WORK_DIR}/sts-bad.xml" >&2 + echo "expected wrong-audience token to return HTTP 403, got ${BAD_STATUS}" >&2 + exit 1 +} +grep -q 'AccessDenied' "${WORK_DIR}/sts-bad.xml" + +echo "OIDC Keycloak live gate passed" diff --git a/scripts/test_fuzz_runner.sh b/scripts/test_fuzz_runner.sh new file mode 100755 index 000000000..76e368162 --- /dev/null +++ b/scripts/test_fuzz_runner.sh @@ -0,0 +1,90 @@ +#!/bin/sh + +set -eu + +SCRIPT_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +REPO_ROOT=$(CDPATH= cd -- "$SCRIPT_DIR/.." && pwd) +TMP_ROOT=$(mktemp -d) +trap 'rm -rf "$TMP_ROOT"' EXIT HUP INT TERM + +mkdir -p "$TMP_ROOT/bin" +cat > "$TMP_ROOT/bin/path_containment" <<'EOF' +#!/bin/sh +printf '%s\n' "$@" > "$FAKE_ARGS_FILE" +EOF +chmod +x "$TMP_ROOT/bin/path_containment" + +EXPLICIT_ARTIFACTS="$TMP_ROOT/explicit-artifacts" +FAKE_ARGS_FILE="$TMP_ROOT/explicit-args" \ +FUZZ_TARGET=path_containment \ +FUZZ_SEED=123456789 \ +MAX_TOTAL_TIME=7 \ +ARTIFACT_ROOT="$EXPLICIT_ARTIFACTS" \ +SKIP_BUILD=1 \ +USE_PREBUILT_BINARY=1 \ +PREBUILT_BINARY_DIR="$TMP_ROOT/bin" \ + "$REPO_ROOT/scripts/fuzz/run.sh" + +EXPLICIT_MANIFEST="$EXPLICIT_ARTIFACTS/path_containment/run-manifest.txt" +test -f "$EXPLICIT_MANIFEST" +grep -Fx -- '-seed=123456789' "$TMP_ROOT/explicit-args" +grep -Fx 'target=path_containment' "$EXPLICIT_MANIFEST" +grep -Fx 'seed=123456789' "$EXPLICIT_MANIFEST" +grep -Fx 'max_total_time=7' "$EXPLICIT_MANIFEST" +grep -Fx "git_revision=$(git -C "$REPO_ROOT" rev-parse HEAD)" "$EXPLICIT_MANIFEST" +grep -E '^git_dirty=(true|false)$' "$EXPLICIT_MANIFEST" +grep -Fx 'runner_mode=prebuilt' "$EXPLICIT_MANIFEST" + +AUTO_ARTIFACTS="$TMP_ROOT/auto-artifacts" +FAKE_ARGS_FILE="$TMP_ROOT/auto-args" \ +FUZZ_TARGET=path_containment \ +MAX_TOTAL_TIME=1 \ +ARTIFACT_ROOT="$AUTO_ARTIFACTS" \ +SKIP_BUILD=1 \ +USE_PREBUILT_BINARY=1 \ +PREBUILT_BINARY_DIR="$TMP_ROOT/bin" \ + "$REPO_ROOT/scripts/fuzz/run.sh" + +AUTO_MANIFEST="$AUTO_ARTIFACTS/path_containment/run-manifest.txt" +auto_seed=$(sed -n 's/^seed=//p' "$AUTO_MANIFEST") +case "$auto_seed" in + ''|*[!0-9]*) + echo "automatic seed was not recorded as an unsigned decimal integer: $auto_seed" >&2 + exit 1 + ;; +esac +grep -Fx -- "-seed=$auto_seed" "$TMP_ROOT/auto-args" + +cat > "$TMP_ROOT/bin/cargo" <<'EOF' +#!/bin/sh +printf '%s\n' "$@" > "$FAKE_CARGO_ARGS_FILE" +EOF +chmod +x "$TMP_ROOT/bin/cargo" + +CARGO_ARTIFACTS="$TMP_ROOT/cargo-artifacts" +PATH="$TMP_ROOT/bin:$PATH" \ +FAKE_CARGO_ARGS_FILE="$TMP_ROOT/cargo-args" \ +FUZZ_TARGET=path_containment \ +FUZZ_SEED=987654321 \ +MAX_TOTAL_TIME=9 \ +ARTIFACT_ROOT="$CARGO_ARTIFACTS" \ +SKIP_BUILD=1 \ + "$REPO_ROOT/scripts/fuzz/run.sh" + +CARGO_MANIFEST="$CARGO_ARTIFACTS/path_containment/run-manifest.txt" +grep -Fx '+nightly' "$TMP_ROOT/cargo-args" +grep -Fx 'fuzz' "$TMP_ROOT/cargo-args" +grep -Fx 'run' "$TMP_ROOT/cargo-args" +grep -Fx 'path_containment' "$TMP_ROOT/cargo-args" +grep -Fx -- '-seed=987654321' "$TMP_ROOT/cargo-args" +grep -Fx 'seed=987654321' "$CARGO_MANIFEST" +grep -Fx 'runner_mode=cargo-fuzz' "$CARGO_MANIFEST" + +for invalid_seed in 0 0123 not-a-number; do + if FUZZ_TARGET=path_containment FUZZ_SEED="$invalid_seed" SKIP_BUILD=1 "$REPO_ROOT/scripts/fuzz/run.sh" >/dev/null 2>&1; then + echo "invalid FUZZ_SEED was accepted: $invalid_seed" >&2 + exit 1 + fi +done + +echo "fuzz runner seed manifest tests passed"