Compare commits

..

31 Commits

Author SHA1 Message Date
cxymds 38aa318fb2 fix(iam): revalidate STS sessions during password rotation 2026-11-20 18:51:32 +08:00
cxymds e7c5d7c198 fix(iam): verify the parent password for Console STS rotations 2026-11-20 17:28:52 +08:00
overtrue e37aa27e40 fix(test): encode rollback digests with the workspace hex encoder 2026-11-20 14:54:34 +08:00
overtrue 6e417a458e test(upgrade): exercise the official 1.0.5 gateway baseline 2026-11-20 14:50:43 +08:00
overtrue 60ed7f1c40 test(upgrade): select mixed storage contracts by source revision 2026-11-20 14:32:11 +08:00
overtrue 7f2bd6b99b test(protocols): match the released error for a missing directory marker 2026-11-20 14:23:19 +08:00
overtrue a798df7906 refactor(protocols): prepare part copy routing and storage from one upload ID 2026-11-20 14:23:19 +08:00
overtrue 793632c0da docs(operations): document rollback after the S3 stack switch 2026-11-20 14:23:18 +08:00
overtrue 301133c1c8 test(gateway): check the release torrent error for boundary keys 2026-11-20 14:23:18 +08:00
overtrue a642ba2a89 chore(ci): tighten the s3s import limit to 33 files 2026-11-20 13:04:07 +08:00
overtrue d893d70b2e refactor(storage): return native errors from S3 storage helpers 2026-11-20 13:03:42 +08:00
overtrue cf15e4d1b6 refactor(http): verify control body checksums with the native hashers 2026-11-20 12:35:23 +08:00
overtrue 5a3e94bbbb chore(ci): ratchet the remaining s3s imports to 34 files 2026-11-20 12:21:40 +08:00
overtrue 29b389bf80 refactor(errors): keep API error codes native through storage handling 2026-11-20 12:21:15 +08:00
overtrue f94e8cae07 refactor(storage): use native listing and object lock inputs 2026-11-20 12:17:42 +08:00
overtrue 64675785fe refactor(bucket): return native S3 errors from admission and config writes 2026-11-20 12:01:48 +08:00
overtrue 5b703b61f9 test(ci): align storage guard fixtures with the surviving helper rule 2026-11-20 12:00:53 +08:00
overtrue 23420e357d refactor(multipart): use native S3 errors for upload validation and storage failures 2026-11-20 11:43:15 +08:00
overtrue 9da1edd8d6 test(e2e): compare complete persisted version maps 2026-11-20 11:39:25 +08:00
overtrue d87c058a4d fix(protocols): audit multipart uploads exactly once 2026-11-20 11:12:35 +08:00
overtrue 54cda17e58 refactor(storage): publish internal write events without legacy S3 responses 2026-11-20 11:12:35 +08:00
overtrue 49f6e815e7 refactor(multipart): retire legacy request envelopes from production 2026-11-20 11:12:35 +08:00
overtrue 58bbb17300 fix(gateway): preserve the missing torrent response for invalid keys 2026-11-20 11:01:04 +08:00
overtrue 310b977f05 test(e2e): compare release and candidate persistence bytes 2026-11-20 10:36:43 +08:00
overtrue bbe396219f refactor(storage): validate tags and website settings with native S3 types 2026-11-20 10:33:22 +08:00
overtrue e286691371 test(admin): exercise health and heal on the native router 2026-11-20 10:32:58 +08:00
overtrue 9ff399afee test(scanner): freeze legacy cache compatibility fixtures 2026-11-20 10:23:41 +08:00
overtrue 23c8c35710 refactor(storage): use native listing errors and write preconditions 2026-11-20 10:08:44 +08:00
Chris An 9c44c5ea6b refactor(admin): serve Object Lambda through native GetObject (#9529)
* test(storage): restore POST form metadata policy fixture

* test(storage): follow the current PUT path in contract checks

* fix(storage): verify archive transport EOF before parsing the final frame

* test(rustfs): restore main verification after gateway migration

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* refactor(storage): retire the legacy S3 storage edge

* fix(storage): bound the audited multipart copy future and trace LIST quorum causes

* test(storage): pin legacy codec answers and trim test dependencies

* refactor(admin): confine the legacy route adapter to parity tests

* fix(deps): align bindgen and prettyplease on syn 3 for FIPS builds

* refactor(admin): keep Object Lambda authorization and responses on the owned request path

* fix(storage): expose the typed legacy cause on owned errors

* refactor(admin): read and return native GetObject fields for Object Lambda

* ci: ratchet the legacy S3 footprint after the control-plane migration

* refactor(admin): remove the unused envelope-to-extensions adapter

* test(admin): guard the Object Lambda response token checks at their definition

* refactor(admin): serialize AssumeRole credentials without the legacy DTO

* test(admin): assert native range and error types at the test boundary

* test(admin): compare errors in the legacy route parity tests

* refactor(notify): initialize event rules from persisted notification types

* refactor(admin): return the idle rebalance response on the owned HTTP path

* refactor(storage): keep the persisted-to-legacy XML oracle in tests

* test(admin): compare extension refusals using the native error code

* fix(obs): filter verbose native gateway protocol events at every sink

* test(storage): preserve stream lengths and request facts on retired seams

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:08:35 +08:00
Chris An 31e0798637 refactor(storage): retire the legacy S3 storage edge (#9527)
* test(storage): restore POST form metadata policy fixture

* test(storage): follow the current PUT path in contract checks

* fix(storage): verify archive transport EOF before parsing the final frame

* test(rustfs): restore main verification after gateway migration

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* refactor(storage): retire the legacy S3 storage edge

* fix(storage): bound the audited multipart copy future and trace LIST quorum causes

* fix(deps): align bindgen and prettyplease on syn 3 for FIPS builds

* fix(storage): expose the typed legacy cause on owned errors

* test(storage): preserve stream lengths and request facts on retired seams

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:07:52 +08:00
Chris An 868af666e9 fix(storage): preserve archive errors and restore gateway CI checks (#9525)
* test(storage): restore POST form metadata policy fixture

* test(storage): follow the current PUT path in contract checks

* fix(storage): verify archive transport EOF before parsing the final frame

* test(rustfs): restore main verification after gateway migration

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(deps): align bindgen and prettyplease on syn 3 for FIPS builds

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:05:46 +08:00
181 changed files with 4238 additions and 9079 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
+8 -2
View File
@@ -114,8 +114,14 @@
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.5",
"role": "gateway compatibility baseline",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
},
{
"version": "1.0.1",
"role": "N-1",
@@ -130,6 +136,6 @@
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
}
}
+2 -2
View File
@@ -13,6 +13,6 @@
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=49
s3_error_lines=95
files=33
s3_error_lines=89
ecstore_files=0
-2
View File
@@ -22,8 +22,6 @@ Identify the tested commit and any local changes. When testing a prebuilt binary
List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
-->
**auto-testing regression coverage** <!-- For each issue-backed bug fix, give the added or extended case ID or PR link, the existing case ID/path and assertion that catches the bug, or "Skipped" with a brief reason. Omit this line for other changes, including documentation-only fixes. -->
## Impact
<!--
Describe user-facing, compatibility, API, deployment, configuration, or documentation impact. Use N/A when there is no expected impact.
+12 -2
View File
@@ -1091,9 +1091,12 @@ jobs:
- name: Verify E2E archive identity
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
- name: Install network fault-injection tools
- name: Install network fault-injection tools and the persistence test clock
run: |
sudo apt-get install -y iptables
sudo apt-get install -y iptables libfaketime
clock_library="$(dpkg-query -L libfaketime | grep -E '/libfaketime[.]so[.]1$')"
test -f "$clock_library"
echo "RUSTFS_PERSISTENCE_CLOCK_LIBRARY=$clock_library" >> "$GITHUB_ENV"
sudo -n iptables --version
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
# runners can run iptables but not touch the rule set; the test then logs
@@ -1200,8 +1203,14 @@ jobs:
# default-filter in .config/nextest.toml is the single wiring mechanism —
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
# debug binary; each test spawns its own rustfs server on a random port.
- name: Download pinned release for persistence byte parity
id: persistence-reference
uses: ./.github/actions/reference-binary
- name: Run e2e full suite
env:
RUSTFS_PERSISTENCE_REFERENCE_BINARY: ${{ steps.persistence-reference.outputs.path }}
RUSTFS_PERSISTENCE_XML_DIR: ${{ runner.temp }}/rustfs-persistence-xml
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
@@ -1220,6 +1229,7 @@ jobs:
path: |
target/nextest/e2e-full/junit.xml
${{ runner.temp }}/rustfs-e2e-full-list.json
${{ runner.temp }}/rustfs-persistence-xml
retention-days: 7
- name: Upload startup CAS evidence
+39 -7
View File
@@ -90,11 +90,16 @@ jobs:
- version: 1.0.1
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
- version: 1.0.5
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -132,6 +137,13 @@ jobs:
unzip -q "$archive" -d "$SOURCE_DIR"
chmod +x "$SOURCE_DIR/rustfs"
test -x "$SOURCE_DIR/rustfs"
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
fi
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
@@ -151,12 +163,14 @@ jobs:
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
)
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
else
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
fi
case "$UPGRADE_SOURCE_VERSION" in
1.0.0-rc.5)
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
esac
status=0
for entry in "${cases[@]}"; do
artifact="${entry%%:*}"
@@ -178,7 +192,7 @@ jobs:
exit "$status"
- name: Run mixed-version MPU recovery
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
timeout-minutes: 15
env:
NO_PROXY: 127.0.0.1,localhost,::1
@@ -199,6 +213,24 @@ jobs:
"$mpu_test" -- --exact --nocapture --test-threads=1 \
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
- name: Verify gateway baseline G09 evidence
if: ${{ success() && matrix.source.version == '1.0.5' }}
shell: bash
run: |
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
directory=mixed-version-upgrade
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
done
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.current_revision == $current and .previous_revision == $previous
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
- name: Upload scanner/heal G09 evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
-12
View File
@@ -80,18 +80,6 @@ Do not commit one-shot plans, trackers, migration ledgers, benchmark snapshots,
Select checks from the final task-owned diff. Scoped `AGENTS.md` files may add a concrete path-specific check, but must not replace this tiering with a generic full-workspace gate.
### Issue Fixes and External Regression Coverage
After fixing a bug tracked by an issue and running the applicable local checks, ask the developer once per fixed issue whether to cover it in [rustfs/auto-testing](https://github.com/rustfs/auto-testing). Ask before opening or updating the fix PR, or before reporting completion if there is no PR. Reuse a choice already made for that fix. Investigation alone and documentation-only changes do not trigger this prompt.
When presenting the three choices, describe the failing scenario and the assertion that would catch it.
- **Add or extend a case.** Prefer extending an existing scenario when it can exercise the regression. Record the resulting case ID or auto-testing PR link.
- **Already covered.** Cite the existing auto-testing case ID or path and the assertion that would catch this bug.
- **Skip.** Record the developer's one-line reason and continue with the fix.
Record the choice in the PR's Verification section or in the completion report for work without a PR. All three choices retain the applicable RustFS verification requirements below.
### Documentation and Instructions
For prose, comments, agent instructions, and skill metadata that cannot affect runtime/build output:
+1 -1
View File
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deployment
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
Generated
+11 -13
View File
@@ -1648,7 +1648,7 @@ dependencies = [
"cexpr",
"clang-sys",
"log",
"prettyplease 0.2.37",
"prettyplease 0.3.0",
"proc-macro2",
"quote",
"regex",
@@ -4238,6 +4238,7 @@ dependencies = [
"s3s",
"same-file",
"serde",
"serde-value",
"serde_json",
"serde_urlencoded",
"sha1 0.11.0",
@@ -10237,7 +10238,6 @@ dependencies = [
"rustix",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
"serial_test",
@@ -10323,7 +10323,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway"
version = "0.66.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"arc-swap",
"bytes",
@@ -10370,7 +10370,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-core"
version = "0.63.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bytes",
"futures-core",
@@ -10391,7 +10391,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-dialect-rustfs-admin"
version = "0.13.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bytes",
"form_urlencoded",
@@ -10405,7 +10405,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-http"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10423,7 +10423,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-macros"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"proc-macro2",
"quote",
@@ -10433,7 +10433,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-sig"
version = "0.15.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10448,7 +10448,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-stream"
version = "0.2.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bitflags 2.13.2",
"bytes",
@@ -10459,7 +10459,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-types"
version = "0.42.0+aws.2026-10-02"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bytes",
"crc-fast",
@@ -10482,7 +10482,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-xml"
version = "0.4.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"quick-xml 0.41.0",
]
@@ -11236,7 +11236,6 @@ dependencies = [
"rustfs-utils",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
"sha2 0.11.1",
@@ -11376,7 +11375,6 @@ dependencies = [
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-utils",
"s3s",
"serde",
"serde_json",
"serial_test",
+7 -6
View File
@@ -152,16 +152,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.2.0", default-features = false }
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.66.0", default-features = false }
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.66.0", default-features = false }
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.13.0", default-features = false }
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.13.0", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
@@ -222,6 +222,7 @@ quick-xml = "0.42.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.229" }
serde-value = "0.7.0"
serde_ignored = { version = "0.1" }
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
serde_json = { version = "1.0.154" }
+2 -1
View File
@@ -129,7 +129,6 @@ md-5 = { workspace = true }
prost.workspace = true
sha2 = { workspace = true }
rustfs-s3-types = { workspace = true }
s3s = { workspace = true, features = ["minio"] }
zstd.workspace = true
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
rcgen.workspace = true
@@ -139,6 +138,8 @@ rustls = { workspace = true, default-features = false, features = ["aws-lc-rs",
clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
serde-value.workspace = true
s3s = { workspace = true, features = ["minio"] }
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
aws-config = { workspace = true }
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
+2
View File
@@ -211,6 +211,8 @@ the wiring source of truth. Committed test-ID digests under
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
The full suite compares persisted bucket configurations and object records against the release pinned in `.config/reference-binary.toml`. Set `RUSTFS_PERSISTENCE_REFERENCE_BINARY` to the verified executable and `RUSTFS_PERSISTENCE_CLOCK_LIBRARY` to the absolute path of libfaketime (`libfaketime.so.1` on Linux or `libfaketime.1.dylib` on macOS). To retain the two sets of 13 XML documents for the gateway golden corpus, set `RUSTFS_PERSISTENCE_XML_DIR` to an output directory. CI includes them in the full suite's JUnit artifact.
**Reproduce a CI failure locally** — run the exact profile/lane:
```bash
+52 -1
View File
@@ -34,7 +34,10 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use crate::common::{
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
init_logging, local_http_client, parse_assume_role_credentials,
};
use hmac::{Hmac, KeyInit as _, Mac};
use http::header::HOST;
use rustfs_s3_types::Body;
@@ -396,6 +399,54 @@ mod tests {
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
env.stop_server();
drop(env);
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
}
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let user = "console-password-user";
let old_password = uuid::Uuid::new_v4().to_string();
let new_password = uuid::Uuid::new_v4().to_string();
admin_create_user(&env, user, &old_password).await?;
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
.await?;
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
let session_request = |method, path, body| {
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
};
let password_request = |current: &str| {
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
};
let (status, body) =
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
assert_eq!(
status,
reqwest::StatusCode::BAD_REQUEST,
"the session secret must not be accepted as the IAM password"
);
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
parse_assume_role_credentials(&response)?;
env.stop_server();
Ok(())
}
@@ -137,6 +137,40 @@ async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::erro
.await
.map_err(|e| format!("DeleteObject failed: {}", e))?;
for (allowed, may_read) in [("s3:GetBucketPolicy", true), ("s3:PutBucketPolicy", false)] {
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": { "AWS": [user_access] },
"Action": allowed,
"Resource": format!("arn:aws:s3:::{bucket_name}")
}]
})
.to_string();
admin_client
.put_bucket_policy()
.bucket(bucket_name)
.policy(&policy)
.send()
.await?;
let read = user_client.get_bucket_policy().bucket(bucket_name).send().await;
let write = user_client
.put_bucket_policy()
.bucket(bucket_name)
.policy(&policy)
.send()
.await;
if may_read {
read?;
let denied = write.expect_err("GetBucketPolicy must not grant PutBucketPolicy");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
} else {
let denied = read.expect_err("PutBucketPolicy must not grant GetBucketPolicy");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
write?;
}
}
info!("Test Passed!");
Ok(())
}
+26 -14
View File
@@ -31,7 +31,7 @@ use super::harness::{
use crate::common::{
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
};
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use std::path::{Path, PathBuf};
@@ -427,7 +427,12 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
Ok(())
}
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
async fn compare_degraded_legacy_list_parts(
dist: &mut DistCluster,
seed: &UpgradeSeed,
previous: &Path,
source: &UpgradeSourceCapabilities,
) -> TestResult {
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
@@ -489,16 +494,22 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
// Three online disks retain metadata quorum but have no part directory;
// the fourth disk is offline. A legacy empty response is not a proof that
// the unavailable participant has no part.
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
assert!(legacy.parts().is_empty());
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
// ListParts read-quorum failure is reported as ServiceUnavailable.
assert_eq!(
current.as_service_error().and_then(|error| error.meta().code()),
Some("ServiceUnavailable")
);
let mut quorum_errors =
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
if source.requires_list_parts_read_quorum {
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
} else {
assert!(
legacy
.expect("previous coordinator should expose its legacy empty-list behavior")
.parts()
.is_empty()
);
}
for error in quorum_errors {
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
}
dist.cluster.start_node_from_binary(3, previous).await?;
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
for (path, before) in original_files {
@@ -520,7 +531,7 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
}
eprintln!(
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
);
Ok(())
}
@@ -592,6 +603,7 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
@@ -601,7 +613,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
for node_idx in [1, 2] {
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
@@ -27,7 +27,7 @@
//! is process-global, so two stacks in one process would read each other's buckets.
mod compare;
mod reference;
pub(crate) mod reference;
mod scenarios;
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
@@ -78,7 +78,7 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
pub(crate) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
if !output.status.success() {
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
+3
View File
@@ -37,6 +37,9 @@ pub mod fake_s3_target;
#[cfg(test)]
mod dual_stack_differential;
#[cfg(test)]
mod persistence_parity;
// On-demand migration (backlog#2147): shared two-server environment, admin
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
// added by later ODM tasks.
@@ -1543,6 +1543,30 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(),
let uploaded = get_out.body.collect().await?.into_bytes();
assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match");
// An empty redirect is rejected when rendering the response, after storage succeeds.
for (field, value, expected_status) in [
("success_action_status", "200", reqwest::StatusCode::OK),
("success_action_status", "204", reqwest::StatusCode::NO_CONTENT),
("success_action_redirect", "", reqwest::StatusCode::BAD_REQUEST),
] {
let key = format!("{field}-{value}.txt");
let form = reqwest::multipart::Form::new()
.text("key", key.clone())
.text(field, value)
.part("file", reqwest::multipart::Part::bytes(expected_body.clone()).file_name("upload.txt"));
let response = local_http_client()
.post(format!("{}/{bucket}", env.url))
.multipart(form)
.send()
.await?;
assert_eq!(response.status(), expected_status);
if value.is_empty() {
assert!(response.text().await?.contains("<Code>InvalidArgument</Code>"));
}
let stored = admin_client.get_object().bucket(bucket).key(key).send().await?;
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body.as_slice());
}
Ok(())
}
+449
View File
@@ -0,0 +1,449 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Compare bytes produced by S3 writes to independent, single-drive volumes.
use crate::common::{RustFSTestEnvironment, admin_request, rustfs_binary_path, signed_s3_request_with_headers};
use crate::dual_stack_differential::reference::binary_version;
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
use crate::on_demand_migration::common::{ODM_ADMIN_ROUTE, ODM_MODULE_SWITCH_ENV, OdmSourceSpec};
use crate::replication_extension_test::send_set_replication_target_request;
use crate::storage_api::persistence_parity::{BucketMetadata, RUSTFS_META_BUCKET};
use crate::upgrade_compatibility_test::{set_bucket_quota, write_multipart};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
use http::{HeaderMap, Method};
use md5::{Digest, Md5};
use rustfs_filemeta::{FileMeta, MetaObject};
use serde_value::Value;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
type Error = Box<dyn std::error::Error + Send + Sync>;
const CONFIGS: &str = "persisted-configs";
const OBJECTS: &str = "persisted-objects";
const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
const LOGS: &str = "rustfs-access-logs";
fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error> {
let key = BucketMetadata::new(bucket).save_file_path();
let object = volume.join(RUSTFS_META_BUCKET).join(&key);
let meta = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
let info = meta.into_fileinfo(RUSTFS_META_BUCKET, &key, "", true, false, true)?;
info.validate_for_metadata_read()?;
let [part] = info.parts.as_slice() else { return Err("bucket metadata must have one part".into()) };
let size = usize::try_from(info.size)?;
if info.erasure.data_blocks != 1 || size != part.size || size > info.erasure.block_size {
return Err("bucket metadata exceeds the single-drive, single-stripe fixture".into());
}
let framed = if info.inline_data() {
info.data.ok_or("missing inline bucket metadata")?.to_vec()
} else {
let directory = info.data_dir.ok_or("missing bucket metadata data directory")?;
std::fs::read(object.join(directory.to_string()).join(format!("part.{}", part.number)))?
};
let algorithm = info.erasure.get_checksum_info(part.number).algorithm;
let (digest, payload) = framed.split_at_checked(algorithm.size()).ok_or("truncated bucket metadata")?;
if digest.is_empty() || payload.len() != size || algorithm.hash_encode(payload).as_ref() != digest {
return Err("invalid bucket metadata shard checksum or size".into());
}
BucketMetadata::check_header(payload)?;
let fields = &payload[4..];
if bucket == CONFIGS {
// The production decoder defaults a missing field to empty. Require the ACL field on disk
// so dropping it cannot look like the expected zero-byte document.
let keys: BTreeMap<&str, serde::de::IgnoredAny> = rmp_serde::from_slice(fields)?;
if !keys.contains_key("BucketAclConfigJSON") {
return Err("missing BucketAclConfigJSON field".into());
}
}
Ok(BucketMetadata::unmarshal(fields)?)
}
type VersionFields = BTreeMap<String, Value>;
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
let [version] = meta.versions.as_slice() else {
return Err(format!("{key}: expected one version").into());
};
let object = version.parse_version_meta()?.object.ok_or("missing V2 object")?;
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
return Err(format!("{key}: missing modification time or data directory").into());
}
Ok((object, nonvolatile_fields(&version.meta, key)?))
}
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
// The production reader skips unknown keys and supplies defaults for some missing keys.
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
return Err(format!("{key}: missing raw V2Obj map").into());
};
for volatile in ["MTime", "DDir"] {
let value = object_fields
.get_mut(&Value::String(volatile.to_owned()))
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
*value = Value::Unit;
}
Ok(fields)
}
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
let mut headers = HeaderMap::new();
headers.insert("content-md5", base64_simd::STANDARD.encode_to_string(Md5::digest(xml)).parse()?);
let response = signed_s3_request_with_headers(
Method::PUT,
&format!("{}/{CONFIGS}?{query}", env.url),
Some(xml.to_owned()),
Some("application/xml"),
&env.access_key,
&env.secret_key,
&headers,
)
.await?;
if !response.status().is_success() {
return Err(format!("PUT ?{query} failed with {}", response.status()).into());
}
Ok(())
}
struct Snapshot {
configs: BTreeMap<&'static str, Vec<u8>>,
objects: BTreeMap<&'static str, VersionFields>,
multipart_files: BTreeMap<String, Vec<u8>>,
}
async fn capture(binary: &Path, target: &FakeS3Target, clock: &str, clock_library: &str) -> Result<Snapshot, Error> {
let mut env = RustFSTestEnvironment::new().await?;
let queue = tempfile::tempdir()?;
let clock_loader = if cfg!(target_os = "macos") {
"DYLD_INSERT_LIBRARIES"
} else {
"LD_PRELOAD"
};
let options = [
(clock_loader, clock_library),
("DYLD_FORCE_FLAT_NAMESPACE", "1"),
("FAKETIME", clock),
("FAKETIME_DONT_FAKE_MONOTONIC", "1"),
("TZ", "UTC"),
("RUSTFS_NOTIFY_ENABLE", "true"),
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", "on"),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", target.endpoint()),
(
"RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY",
queue.path().to_str().ok_or("non-UTF8 queue path")?,
),
("RUSTFS_OUTBOUND_ALLOW_ORIGINS", target.endpoint()),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
(ODM_MODULE_SWITCH_ENV, "true"),
("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
];
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &options)
.await?;
let client = env.create_s3_client();
write_configs(&env, &client, target).await?;
write_objects(&client).await?;
env.stop_server();
read_snapshot(Path::new(&env.temp_dir))
}
async fn write_configs(env: &RustFSTestEnvironment, client: &Client, target: &FakeS3Target) -> Result<(), Error> {
env.create_test_bucket(LOGS).await?;
env.create_test_bucket(OBJECTS).await?;
env.create_test_bucket("persistence-warehouse").await?;
env.create_test_bucket("persistence-migration").await?;
client
.create_bucket()
.bucket(CONFIGS)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
macro_rules! golden {
($path:literal) => {
include_str!(concat!("../../ecstore/tests/fixtures/bucket-config-goldens/", $path, ".in.xml"))
};
}
let target_request = serde_json::json!({
"endpoint": target.address(), "credentials": {"accessKey": FAKE_ACCESS_KEY, "secretKey": FAKE_SECRET_KEY},
"targetbucket": "interop-dr", "type": "replication", "secure": false,
"deploymentID": "0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c"
});
let response = send_set_replication_target_request(env, CONFIGS, false, target_request).await?;
if !response.status().is_success() {
return Err(format!("set remote target failed with {}", response.status()).into());
}
let target_arn: String = response.json().await?;
let replication = golden!("replication/rustfs").replace("arn:rustfs:replication:", "arn:minio:replication:");
if !replication.contains(&format!("<Bucket>{target_arn}</Bucket>")) {
return Err("registered target does not match the golden replication destination".into());
}
for (query, xml) in [
("versioning", golden!("versioning/minio")),
("notification", golden!("notification/rustfs")),
("replication", replication.as_str()),
("lifecycle", golden!("lifecycle/minio")),
("object-lock", golden!("object_lock/rustfs")),
("encryption", golden!("encryption/minio")),
("tagging", golden!("tagging/rustfs")),
("cors", golden!("cors/rustfs")),
("logging", golden!("logging/rustfs")),
("website", golden!("website/rustfs")),
("accelerate", golden!("accelerate/rustfs")),
("requestPayment", golden!("request_payment/rustfs")),
("publicAccessBlock", golden!("public_access_block/rustfs")),
] {
put_xml(env, query, xml).await?;
}
client
.put_bucket_acl()
.bucket(CONFIGS)
.acl(BucketCannedAcl::Private)
.send()
.await?;
let acl = client.get_bucket_acl().bucket(CONFIGS).send().await?;
let owner = acl.owner().and_then(|owner| owner.id()).ok_or("ACL owner missing")?;
let [grant] = acl.grants() else { return Err("expected exactly one ACL grant".into()) };
let grantee = grant.grantee().ok_or("missing ACL grantee")?;
if owner.is_empty()
|| grant.permission().map(|p| p.as_str()) != Some("FULL_CONTROL")
|| grantee.r#type().as_str() != "CanonicalUser"
|| grantee.id().is_some()
{
return Err("GET ACL changed its canonical owner grant".into());
}
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Deny", "Principal": "*", "Action": "s3:DeleteObject",
"Resource": format!("arn:aws:s3:::{CONFIGS}/protected/*"),
}],
});
client
.put_bucket_policy()
.bucket(CONFIGS)
.policy(policy.to_string())
.send()
.await?;
set_bucket_quota(env, CONFIGS, 16 * 1024 * 1024).await?;
let migration = OdmSourceSpec::for_fake_source(target, "migration-source");
for (path, body) in [
("/iceberg/v1/buckets/persistence-warehouse".to_owned(), None),
(format!("{ODM_ADMIN_ROUTE}/persistence-migration"), Some(migration.to_json().to_string())),
] {
let (status, _) = admin_request(&env.url, Method::PUT, &path, body, &env.access_key, &env.secret_key).await?;
if !status.is_success() {
return Err(format!("configuring {path} failed with {status}").into());
}
}
Ok(())
}
async fn write_objects(client: &Client) -> Result<(), Error> {
for key in ["put", "tagged"] {
client
.put_object()
.bucket(OBJECTS)
.key(key)
.metadata("purpose", "byte-parity")
.body(ByteStream::from_static(b"persisted object contents"))
.send()
.await?;
}
client
.copy_object()
.bucket(OBJECTS)
.key("copy")
.copy_source(format!("{OBJECTS}/put"))
.send()
.await?;
let tags = Tagging::builder()
.tag_set(Tag::builder().key("type").value("persisted").build()?)
.build()?;
client
.put_object_tagging()
.bucket(OBJECTS)
.key("tagged")
.tagging(tags)
.send()
.await?;
write_multipart(client, OBJECTS, "multipart", &[vec![0x31; FIRST_PART_SIZE], vec![0x32; 100]]).await?;
Ok(())
}
fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
let b = bucket_metadata(volume, CONFIGS)?;
let configs = BTreeMap::from([
("acl", b.bucket_acl_config_json),
("accelerate", b.accelerate_config_xml),
("cors", b.cors_config_xml),
("encryption", b.encryption_config_xml),
("lifecycle", b.lifecycle_config_xml),
("logging", b.logging_config_xml),
("notification", b.notification_config_xml),
("object_lock", b.object_lock_config_xml),
("public_access_block", b.public_access_block_config_xml),
("request_payment", b.request_payment_config_xml),
("tagging", b.tagging_config_xml),
("versioning", b.versioning_config_xml),
("website", b.website_config_xml),
("durability", b.durability_config_json),
("policy", b.policy_config_json),
("quota", b.quota_config_json),
("replication", b.replication_config_xml),
("targets", b.bucket_targets_config_json),
("table", bucket_metadata(volume, "persistence-warehouse")?.table_bucket_config_json),
("odm", bucket_metadata(volume, "persistence-migration")?.on_demand_migration_config_json),
]);
let mut objects = BTreeMap::new();
for key in ["put", "copy", "tagged"] {
let (_, fields) = read_object(volume, key)?;
objects.insert(key, fields);
}
let (multipart, _) = read_object(volume, "multipart")?;
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
return Err("xl.meta does not describe both uploaded parts".into());
}
let directory = volume
.join(OBJECTS)
.join("multipart")
.join(multipart.data_dir.ok_or("missing multipart data directory")?.to_string());
let mut multipart_files = BTreeMap::new();
for entry in std::fs::read_dir(directory)? {
let entry = entry?;
let name = entry.file_name().into_string().map_err(|_| "invalid multipart file name")?;
if name.starts_with("part.") {
let bytes = std::fs::read(entry.path())?;
if bytes.is_empty() {
return Err(format!("empty multipart file {name}").into());
}
multipart_files.insert(name, bytes);
}
}
for name in ["part.1", "part.2", "part.1.meta", "part.2.meta"] {
if !multipart_files.contains_key(name) {
return Err(format!("missing multipart file {name}").into());
}
}
Ok(Snapshot {
configs,
objects,
multipart_files,
})
}
fn save_xml_samples(root: &Path, side: &str, snapshot: &Snapshot) -> Result<(), Error> {
let directory = root.join(side);
std::fs::create_dir_all(&directory)?;
let families: [&str; 13] = [
"accelerate",
"cors",
"encryption",
"lifecycle",
"logging",
"notification",
"object_lock",
"public_access_block",
"replication",
"request_payment",
"tagging",
"versioning",
"website",
];
for family in families {
let bytes = snapshot
.configs
.get(family)
.ok_or_else(|| format!("missing XML family {family}"))?;
std::fs::write(directory.join(format!("{family}.xml")), bytes)?;
}
Ok(())
}
#[tokio::test]
async fn persistence_byte_parity() -> Result<(), Error> {
let reference = PathBuf::from(
std::env::var_os("RUSTFS_PERSISTENCE_REFERENCE_BINARY").ok_or("missing pinned RUSTFS_PERSISTENCE_REFERENCE_BINARY")?,
);
let candidate = rustfs_binary_path();
if binary_version(&reference).await? == binary_version(&candidate).await? {
return Err("reference and candidate binaries report identical build identities".into());
}
// The quota writer records the process wall clock inside its JSON document. Keep raw bytes
// comparable while the SDK's signature clock and both servers' monotonic timers advance normally.
let clock_library = std::env::var("RUSTFS_PERSISTENCE_CLOCK_LIBRARY")?;
if !Path::new(&clock_library).is_absolute() || !Path::new(&clock_library).is_file() {
return Err("RUSTFS_PERSISTENCE_CLOCK_LIBRARY must name an absolute libfaketime path".into());
}
let clock = time::OffsetDateTime::now_utc()
.format(time::macros::format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"))?;
let target = FakeS3Target::start().await?;
target.create_bucket("migration-source");
target.create_bucket_with_object_lock("interop-dr");
let reference = capture(&reference, &target, &clock, &clock_library).await?;
let candidate = capture(&candidate, &target, &clock, &clock_library).await?;
if let Some(directory) = std::env::var_os("RUSTFS_PERSISTENCE_XML_DIR") {
save_xml_samples(Path::new(&directory), "reference", &reference)?;
save_xml_samples(Path::new(&directory), "candidate", &candidate)?;
}
if reference.configs.len() != 20 || candidate.configs.len() != 20 {
return Err("expected 13 XML and seven JSON fields on each side".into());
}
for (&field, expected) in &reference.configs {
// Canned ACLs acknowledge the write and return one owner grant without persisting an ACL document.
if expected.is_empty() != (field == "acl") || candidate.configs.get(field) != Some(expected) {
return Err(format!("incorrect or different persisted bucket config: {field}").into());
}
}
for (key, expected) in &reference.objects {
if candidate.objects.get(key) != Some(expected) {
return Err(format!("different persisted xl.meta fields: {key}").into());
}
}
if candidate.multipart_files != reference.multipart_files {
return Err("different multipart part files or part metadata".into());
}
eprintln!(
"Compared {} bucket fields (canned ACL empty), {} full object records and multipart part files",
reference.configs.len(),
reference.objects.len()
);
Ok(())
}
#[test]
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
let original = fields(&sample)?;
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
sample["V2Obj"]["MTime"] = serde_json::json!(2);
assert_eq!(fields(&sample)?, original);
sample["V2Obj"]["Future"] = serde_json::Value::Null;
let with_nil = fields(&sample)?;
assert_ne!(with_nil, original);
sample["V2Obj"]["Future"] = serde_json::json!(false);
let with_value = fields(&sample)?;
assert_ne!(with_value, with_nil);
sample["FutureEnvelope"] = serde_json::json!(1);
assert_ne!(fields(&sample)?, with_value);
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
Ok(())
}
@@ -465,7 +465,7 @@ pub(crate) async fn set_replication_target_with_options(
Ok(arn)
}
async fn send_set_replication_target_request(
pub(crate) async fn send_set_replication_target_request(
source_env: &RustFSTestEnvironment,
source_bucket: &str,
update: bool,
+6
View File
@@ -57,6 +57,12 @@ pub(crate) mod replication_extension {
pub(crate) use super::BucketTargetSys;
}
#[cfg(test)]
pub(crate) mod persistence_parity {
pub(crate) use super::RUSTFS_META_BUCKET;
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
}
#[cfg(test)]
pub(crate) mod upgrade_compatibility {
pub(crate) use super::{DiskOption, Endpoint, RUSTFS_META_MULTIPART_BUCKET, new_disk};
+127 -15
View File
@@ -46,9 +46,12 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -142,6 +145,41 @@ fn current_source_revision() -> Result<String, BoxError> {
Ok(revision.to_string())
}
pub(crate) struct UpgradeSourceCapabilities {
supports_delete_recovery: bool,
supports_multipart_publication: bool,
pub(crate) requires_list_parts_read_quorum: bool,
}
impl UpgradeSourceCapabilities {
fn from_revision(revision: &str) -> Result<Self, BoxError> {
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
match revision {
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
supports_delete_recovery: false,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_4_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: true,
requires_list_parts_read_quorum: true,
}),
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
}
}
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
Self::from_revision(&binary_source_revision(binary).await?)
}
}
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
if !output.status.success() {
@@ -225,6 +263,25 @@ mod g09_evidence_tests {
use super::*;
use tempfile::TempDir;
#[test]
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
(RC2_COMMIT, false, false, false),
(RC5_COMMIT, false, false, false),
(STABLE_1_0_1_COMMIT, false, false, false),
(STABLE_1_0_4_COMMIT, true, false, false),
(STABLE_1_0_5_COMMIT, true, true, true),
(STABLE_1_0_6_COMMIT, true, true, true),
] {
let source = UpgradeSourceCapabilities::from_revision(revision)?;
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
}
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
Ok(())
}
#[test]
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
let binary = Path::new("rustfs");
@@ -365,7 +422,7 @@ async fn complete_pending_multipart(
.await?)
}
async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
pub(crate) async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
let pending = stage_multipart(client, bucket, key, parts).await?;
complete_pending_multipart(client, bucket, key, &pending).await?;
Ok(())
@@ -1011,8 +1068,7 @@ async fn exercise_mixed_cluster(
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
);
// 1.0.4 already implements the delete decision protocol. Use a separate
// old-writer key so the original seed still spans the entire rollout.
// Use a separate old-writer key so the original seed still spans the entire rollout.
if previous_has_delete_recovery {
let key = format!(".upgrade-delete/{phase}/previous-writer");
previous_client
@@ -2293,7 +2349,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let previous_binary = source_binary()?;
// Bind the expected protocol to the verified release source, never to the
// DELETE result itself. The older CI source pins retain their 503 contract.
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
let evidence_context = g09_evidence_context(&previous_binary).await?;
let current_binary = rustfs_binary_path();
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2315,17 +2371,21 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.stop_node(0)?;
cluster.start_node_from_binary(0, &current_binary).await?;
let mut mixed_deleted_keys = Vec::new();
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
// One controlled out-of-policy attempt proves fail-closed behavior below
// native write quorum; positive MPU writers remain paused while mixed.
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
let mixed_completion = if source.supports_multipart_publication {
let clients = cluster.create_all_clients()?;
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
} else {
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
None
};
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
cluster.stop_node(node_idx)?;
cluster.start_node_from_binary(node_idx, &current_binary).await?;
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
}
@@ -2335,6 +2395,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let clients = cluster.create_all_clients()?;
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
if let Some(mixed) = mixed_completion {
assert_eq!(
completed_pending.version_id, mixed.version_id,
"full upgrade changed the completed version"
);
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
}
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
@@ -2451,7 +2518,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"phases": phases,
"objects_per_phase": objects_per_phase,
"mixed_load": "plain-put",
"mixed_delete_protocol_supported": previous_has_delete_recovery,
"mixed_delete_protocol_supported": source.supports_delete_recovery,
"mixed_deleted_keys": mixed_deleted_keys,
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
@@ -2461,7 +2528,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"all nodes list every mixed-version object after homogeneous-current convergence",
"the previous-release seed survives the baseline restart and every upgrade phase",
"previous readers read every current-native MPU body after quiescent rollback",
"same legacy upload resumes with identical body, ETag and version after full upgrade"
"the same previous-release upload remains readable after full upgrade and rollback"
],
}),
)?;
@@ -2483,9 +2550,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"previous-release writer publishes objects readable by current node",
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
"the same legacy upload completes and replays once fully upgraded without adding a version",
if previous_has_delete_recovery {
if source.supports_multipart_publication {
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
} else {
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
},
"fully upgraded readers see the same body, ETag and sole version after replay",
if source.supports_delete_recovery {
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
} else {
"modern DELETE fails closed and preserves every byte while old peers remain"
@@ -2832,7 +2903,7 @@ async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, ser
env.start_rustfs_server_from_binary(binary, vec![], server_env).await
}
async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
pub(crate) async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
let path = format!("/rustfs/admin/v3/quota/{bucket}");
let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
@@ -3386,6 +3457,8 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
#[tokio::test]
#[ignore = "requires a pinned previous RustFS release binary"]
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
use sha2::Digest;
init_logging();
let previous_binary = source_binary()?;
let evidence_context = g09_evidence_context(&previous_binary).await?;
@@ -3423,6 +3496,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
let multipart_bytes = multipart_parts.concat();
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
// This release can finish an upload staged by the current gateway.
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
Some((pending, parts.concat()))
} else {
None
};
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
let old_client = env.create_s3_client();
@@ -3438,6 +3521,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
let rollback_multipart = if let Some((pending, expected)) = inflight {
assert_pending_parts(&old_client, &pending).await?;
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(body, expected);
Some((pending.part_sizes, body))
} else {
None
};
// A PUT on the rolled-back release must still honour the encryption
// configuration it decoded out of the current build's metadata blob.
let post_rollback_key = "rollback/written-after-rollback";
@@ -3478,6 +3571,25 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
post_rollback_bytes
);
if let Some((part_sizes, rollback_read)) = rollback_multipart {
let (_, current_read) = read_object(&current_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(current_read, rollback_read);
if let Some(context) = &evidence_context {
let evidence = serde_json::json!({
"previous_revision": context.previous_revision,
"current_revision": context.current_revision,
"part_sizes": part_sizes,
"bytes": current_read.len(),
"rollback_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(&rollback_read), hex_simd::AsciiCase::Lower),
"roll_forward_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(&current_read), hex_simd::AsciiCase::Lower),
});
std::fs::write(
context.directory.join("inflight-multipart-rollback.json"),
serde_json::to_vec_pretty(&evidence)?,
)?;
}
}
if let Some(context) = evidence_context.as_ref() {
write_g09_evidence(
context,
-1
View File
@@ -258,7 +258,6 @@ windows-sys = { workspace = true, features = [
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
aws-smithy-async.workspace = true
rustfs-s3-ops.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
+16 -36
View File
@@ -25,10 +25,8 @@
//! Not responsible for: validating configurations, touching the bucket
//! metadata store, or choosing a canonical shape.
//!
//! Upstream: the bucket-config `deserialize`/`serialize` pair, reached through the
//! integration-test facade `tests/storage_api.rs`, and the configuration types they are
//! generic over (`BucketConfigXml`: the gateway persistence shapes for the families that
//! have moved, the s3s DTOs for the rest). Downstream: the DTO/codec
//! The tests exercise the production bucket configuration codec through `tests/storage_api.rs`.
//! The lifecycle date cases also pin the results of the legacy codec. The DTO/codec
//! replacement (T1.4) and the persisted-bytes comparison (T3.7), which must
//! keep every golden green without regenerating it. Regeneration is reserved
//! to `scripts/gen_bucket_config_goldens.sh`, which sets
@@ -506,13 +504,12 @@ mod bucket_config_goldens {
mod bucket_config_lifecycle_date_compat {
use super::*;
use rustfs_lifecycle::core::format_lifecycle_date;
use time::OffsetDateTime;
const DATES: [&str; 3] = [
"2023-11-14T22:13:20.123456789Z",
"2023-11-14T22:13:20.123456789+05:30",
"2025-01-02T00:00:00+02:00",
// Captured from the 5761ddfe reader and writer. The reader preserves the offset and
// nanoseconds. The writer emits the local clock fields at millisecond precision with a Z.
const DATES: [(&str, &str); 3] = [
("2023-11-14T22:13:20.123456789Z", "2023-11-14T22:13:20.123Z"),
("2023-11-14T22:13:20.123456789+05:30", "2023-11-14T22:13:20.123Z"),
("2025-01-02T00:00:00+02:00", "2025-01-02T00:00:00.000Z"),
];
fn document(date: &str) -> Vec<u8> {
@@ -522,33 +519,17 @@ mod bucket_config_lifecycle_date_compat {
.into_bytes()
}
fn legacy_read(input: &[u8]) -> s3s::dto::BucketLifecycleConfiguration {
let mut reader = s3s::xml::Deserializer::new(input);
let value = s3s::xml::Deserialize::deserialize(&mut reader).expect("the actual old lifecycle reader accepts the date");
reader.expect_eof().expect("the old reader consumes the entire date fixture");
value
}
#[test]
fn lifecycle_xml_reader_preserves_legacy_date_precision_and_offset() {
for date in DATES {
let input = document(date);
let old = legacy_read(&input);
let new: PersistedLifecycleConfiguration = deserialize(&input).expect("read persisted lifecycle date");
let old_date = |value: &s3s::dto::Timestamp| {
format_lifecycle_date(OffsetDateTime::from(value.clone())).expect("format the original old-reader date")
};
for (date, _) in DATES {
let new: PersistedLifecycleConfiguration = deserialize(&document(date)).expect("read persisted lifecycle date");
assert_eq!(
[
new.expiry_updated_at.clone(),
new.rules[0].expiration.as_ref().and_then(|value| value.date.clone()),
new.rules[0].transitions.as_ref().unwrap()[0].date.clone(),
],
[
Some(old_date(old.expiry_updated_at.as_ref().unwrap())),
Some(old_date(old.rules[0].expiration.as_ref().unwrap().date.as_ref().unwrap())),
Some(old_date(old.rules[0].transitions.as_ref().unwrap()[0].date.as_ref().unwrap())),
],
[Some(date.to_owned()), Some(date.to_owned()), Some(date.to_owned())],
"the lifecycle reader must retain the old date representation for {date}"
);
}
@@ -556,11 +537,10 @@ mod bucket_config_lifecycle_date_compat {
#[test]
fn lifecycle_xml_writer_preserves_legacy_date_bytes() {
for date in DATES {
let old = legacy_read(&document(date));
let mut old_xml = Vec::new();
s3s::xml::Serialize::serialize(&old, &mut s3s::xml::Serializer::new(&mut old_xml))
.expect("the actual old lifecycle writer emits the date");
for (date, written_date) in DATES {
let expected = format!(
"<LifecycleConfiguration><ExpiryUpdatedAt>{written_date}</ExpiryUpdatedAt><Rule><Expiration><Date>{written_date}</Date></Expiration><ID>date-compat</ID><Status>Enabled</Status><Transition><Date>{written_date}</Date><StorageClass>GLACIER</StorageClass></Transition></Rule></LifecycleConfiguration>"
);
let new = PersistedLifecycleConfiguration {
expiry_updated_at: Some(date.to_owned()),
rules: vec![PersistedLifecycleRule {
@@ -585,7 +565,7 @@ mod bucket_config_lifecycle_date_compat {
};
assert_eq!(
serialize(&new).expect("write persisted lifecycle date"),
old_xml,
expected.as_bytes(),
"legacy date bytes for {date}"
);
}
+47 -1
View File
@@ -1786,7 +1786,7 @@ where
}
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
self.update_user_secret_key_with_guard(access_key, secret_key, None)
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
.await
.map(|(updated_at, status, _guard)| (updated_at, status))
}
@@ -1796,6 +1796,7 @@ where
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(
OffsetDateTime,
AccountStatus,
@@ -1824,6 +1825,51 @@ where
}
}
// Lock order is parent, then the presenting STS credential. Deletion and
// session refresh use the same STS lock. Release it before the revocation sweep.
let _session_guard = match session {
Some(session) => {
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
let current = self
.api
.load_user_identity(&session.access_key, UserType::Sts)
.await
.map_err(|error| {
if is_err_no_such_user(&error) {
Error::IAMActionNotAllowed
} else {
error
}
})?;
let parent = if current.credentials.parent_user.is_empty() {
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
} else {
Some(current.credentials.parent_user.as_str())
};
use subtle::ConstantTimeEq as _;
if !session.is_valid()
|| !current.credentials.is_valid()
|| !current.credentials.is_temp()
|| current.credentials.is_service_account()
|| current.credentials.access_key != session.access_key
|| parent != Some(access_key)
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|| !bool::from(
current
.credentials
.session_token
.as_bytes()
.ct_eq(session.session_token.as_bytes()),
)
|| current.sts_generation != u.sts_generation
{
return Err(Error::IAMActionNotAllowed);
}
Some(guard)
}
None => None,
};
let mut cred = u.credentials.clone();
cred.secret_key = secret_key.to_string();
+183 -6
View File
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
}
/// Change an enabled regular caller's secret only if its signing secret is
/// still current under the parent mutation lock. Status and policies are kept.
/// Change an enabled regular user's secret only if the supplied current
/// secret still matches under the parent mutation lock. Status and policies are kept.
/// Persistence and revocation continue after caller cancellation.
pub async fn change_own_secret_key(
&self,
@@ -1433,7 +1433,27 @@ impl<T: Store> IamSys<T> {
expected_secret: &str,
secret_key: &str,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
.await
}
/// Change a local IAM password using an authenticated STS session and its parent password.
///
/// The parent and session mutation locks protect revalidation through the secret write.
/// Persistence and session revocation continue after request cancellation.
/// A revocation failure is returned after the new secret has been persisted.
///
/// # Errors
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
/// incorrect current password. Storage and revocation errors are propagated.
pub async fn change_own_secret_key_from_session(
&self,
access_key: &str,
expected_secret: &str,
secret_key: &str,
session: &Credentials,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
.await
}
@@ -1442,16 +1462,18 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
let iam = self.clone();
let access_key = access_key.to_string();
let secret_key = secret_key.to_string();
let expected_secret = expected_secret.map(str::to_string);
let session = session.cloned();
#[cfg(test)]
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
let operation = async move {
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
.await
};
#[cfg(test)]
@@ -1471,6 +1493,7 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
if !is_access_key_valid(access_key) {
return Err(IamError::InvalidAccessKeyLength);
@@ -1482,7 +1505,7 @@ impl<T: Store> IamSys<T> {
let (updated_at, status, parent_guard) = self
.store
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
.await?;
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
drop(parent_guard);
@@ -4815,6 +4838,160 @@ mod tests {
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
}
#[tokio::test]
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
ensure_test_global_credentials();
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
let store = StsTestMockStore::new(false);
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
let parent = "session-change-parent";
let password = "currentParentSecret123";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: password.to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "password-session");
let owner = iam.get_user(parent).await.expect("parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
let pending = {
let iam = Arc::clone(&iam);
let session = session.clone();
tokio::spawn(async move {
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
.await
})
};
if mutation == "revoke" {
iam.delete_temp_account(&session.access_key, false)
.await
.expect("revoke presenting session");
} else {
let mut saved = store.saved_sts_users.lock().expect("saved identities");
let current = saved.get_mut(&session.access_key).expect("durable session");
match mutation {
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
"disable" => current.credentials.status = "off".to_string(),
"parent" => current.credentials.parent_user = "another-parent".to_string(),
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
"token" => current.credentials.session_token = "replacement-token".to_string(),
"generation" => {
assert!(current.sts_generation.take().is_some());
}
_ => unreachable!("known fixture mutation"),
}
}
drop(guard);
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
.await
.expect("queued rotation finishes")
.expect("rotation task")
.expect_err("stale session must fail");
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
let saved = store.saved_sts_users.lock().expect("saved identities");
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
}
}
#[tokio::test]
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
let iam = test_iam_sys().await;
let parent = "stale-password-parent";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: "cachedParentSecret123".to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "fresh-password-session");
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
iam.store
.api
.saved_sts_users
.lock()
.expect("saved identities")
.get_mut(parent)
.expect("durable parent")
.credentials
.secret_key = "durableParentSecret123".to_string();
assert_eq!(
iam.store
.get_user(parent)
.await
.expect("stale cached parent")
.credentials
.secret_key,
"cachedParentSecret123"
);
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
crate::LOAD_USER_NOTIFICATION_PROBE
.scope(
probe,
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
)
.await
.expect("durable current password authorizes the change");
assert_eq!(
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
"replacementParentSecret123"
);
}
#[tokio::test]
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
let iam = test_iam_sys().await;
let parent = "current-session-parent";
let password = "currentParentSecret123";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: password.to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "current-password-session");
let owner = iam.get_user(parent).await.expect("parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
let (_, _, revoked) = tokio::time::timeout(
std::time::Duration::from_secs(5),
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
probe,
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
),
)
.await
.expect("rotation must release its session lock before revocation")
.expect("rotate current session");
assert_eq!(revoked, 1);
assert!(iam.get_user(&session.access_key).await.is_none());
assert_eq!(
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
"replacementParentSecret123"
);
}
#[tokio::test]
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
ensure_test_global_credentials();
+1 -1
View File
@@ -6,7 +6,7 @@ Applies to `crates/kms/`.
When changing key-management behavior, verify compatibility with:
- `rustfs/src/storage/ecfs.rs`
- `rustfs/src/storage/sse.rs`
- `rustfs/src/admin/handlers/kms.rs`
- `rustfs/src/admin/handlers/kms_dynamic.rs`
- `rustfs/src/admin/handlers/kms_keys.rs`
+54 -55
View File
@@ -28,6 +28,7 @@ use tracing_subscriber::{
use super::http2_ping::HYPER_HTTP2_CLIENT_TARGET;
pub const HTTP_SERVER_LOG_TARGET: &str = "rustfs::server::http";
const S3_DEPENDENCY_TARGETS: [&str; 2] = ["s3s", "rustfs_gateway"];
/// Pyroscope emits raw reqwest errors from its background session manager.
/// Those errors can include the configured endpoint, so they must never reach
@@ -44,12 +45,13 @@ pub(super) fn pyroscope_log_filter() -> FilterFn {
filter_fn(allows_non_pyroscope_dependency_event)
}
/// s3s verbose events include response bodies and expected signatures. Keep
/// diagnostics spans, but never let these events reach logs or OTLP exporters.
pub(super) fn s3s_log_filter() -> FilterFn {
/// Keep verbose protocol events out of logs and OTLP while preserving diagnostic spans.
pub(super) fn s3_dependency_log_filter() -> FilterFn {
filter_fn(|metadata| {
!(metadata.is_event()
&& (metadata.target() == "s3s" || metadata.target().starts_with("s3s::"))
&& S3_DEPENDENCY_TARGETS
.iter()
.any(|target| directive_applies_to_target(target, metadata.target()))
&& *metadata.level() > tracing::Level::INFO)
})
}
@@ -60,7 +62,7 @@ pub(super) fn s3s_log_filter() -> FilterFn {
/// `RUST_LOG` environment variable takes precedence over `logger_level`.
/// For non-verbose levels (`info`, `warn`, `error`), noisy dependency targets
/// are automatically suppressed based on the effective log configuration.
/// Transport internals are disabled, while `s3s` remains visible at WARN. The
/// Transport internals are disabled, while S3 protocol warnings remain visible. The
/// diagnostic HTTP/2 PING feature adds back only Hyper's client target at DEBUG
/// so its exact keepalive timeout event reaches the metrics layer.
///
@@ -179,33 +181,14 @@ fn effective_level_for_target<'a>(rust_log: &'a str, target: &str) -> Option<&'a
best_match.map(|(_, _, level)| level)
}
fn should_demote_http_request_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
fn should_demote_target(target: &str, logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
if let Some(level) = default_level {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
if let Some(rust_log) = rust_log {
if let Some(level) = effective_level_for_target(rust_log, HTTP_SERVER_LOG_TARGET) {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
return false;
}
let level = logger_level.trim().to_ascii_lowercase();
matches!(level.as_str(), "info" | "warn")
}
fn should_demote_s3s_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
if let Some(level) = default_level {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
if let Some(rust_log) = rust_log {
if let Some(level) = effective_level_for_target(rust_log, "s3s") {
if let Some(level) = effective_level_for_target(rust_log, target) {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
@@ -259,17 +242,16 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
("tower", LevelFilter::OFF),
];
if should_demote_http_request_logs(logger_level, default_level, rust_log_env.as_deref()) {
if should_demote_target(HTTP_SERVER_LOG_TARGET, logger_level, default_level, rust_log_env.as_deref()) {
// HTTP request logs are demoted to WARN to reduce volume in production,
// but only when the effective log level is not stricter than WARN.
directives.push((HTTP_SERVER_LOG_TARGET, LevelFilter::WARN));
}
if should_demote_s3s_logs(logger_level, default_level, rust_log_env.as_deref()) {
// s3s instruments each authentication request through multiple
// nested INFO spans. Keep warnings and errors in normal production
// logs without promoting it when the effective base is stricter.
directives.push(("s3s", LevelFilter::WARN));
for target in S3_DEPENDENCY_TARGETS {
if should_demote_target(target, logger_level, default_level, rust_log_env.as_deref()) {
directives.push((target, LevelFilter::WARN));
}
}
for (crate_name, level) in directives {
@@ -363,14 +345,14 @@ mod tests {
}
#[test]
fn s3s_filter_blocks_credentials_despite_explicit_verbose_directives() {
fn s3_dependency_filter_blocks_verbose_events_despite_explicit_directives() {
use tracing_subscriber::prelude::*;
for directive in ["trace", concat!("trace,", "s3s::service=trace,", "s3s::ops::signature=debug")] {
let writer = SharedWriter::default();
let captured = Arc::clone(&writer.0);
let subscriber = tracing_subscriber::registry()
.with(EnvFilter::new(directive))
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(
tracing_subscriber::fmt::layer()
.without_time()
@@ -384,6 +366,10 @@ mod tests {
tracing::debug!(target: "s3s::service", response = "secret-response", "response dump");
tracing::trace!(target: "s3s", response = "secret-trace", "response dump");
tracing::debug!(target: "s3s::ops::signature", expected = "secret-signature", "signature mismatch");
tracing::debug!(target: "rustfs_gateway::service", "secret-gateway-debug");
tracing::trace!(target: "rustfs_gateway", "secret-gateway-trace");
tracing::info!(target: "rustfs_gateway::service", "visible-gateway-info");
tracing::debug!(target: "rustfs_gateway_other", "visible-gateway-neighbor");
tracing::info!(target: "s3s::service", "visible-info");
tracing::warn!(target: "s3s::service", "visible-warning");
tracing::debug!(target: "rustfs", "visible-rustfs-debug");
@@ -393,6 +379,8 @@ mod tests {
assert!(!output.contains("secret"), "{output}");
for expected in [
"safe-operation-span",
"visible-gateway-info",
"visible-gateway-neighbor",
"visible-info",
"visible-warning",
"visible-rustfs-debug",
@@ -445,22 +433,29 @@ mod tests {
#[test]
fn test_should_demote_http_request_logs() {
assert!(should_demote_http_request_logs("info", None, None));
assert!(should_demote_http_request_logs("warn", None, None));
assert!(!should_demote_http_request_logs("error", None, None));
assert!(!should_demote_http_request_logs("off", None, None));
assert!(!should_demote_http_request_logs("info", None, Some("ERROR")));
assert!(should_demote_http_request_logs("error", None, Some("WARN")));
assert!(!should_demote_http_request_logs("info", None, Some("foo=warn")));
assert!(!should_demote_http_request_logs("info", None, Some("rustfs=error")));
assert!(!should_demote_http_request_logs("info", None, Some("rustfs::server=error")));
assert!(!should_demote_http_request_logs("info", None, Some(&http_log_directive("error"))));
assert!(!should_demote_http_request_logs(
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, None));
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "warn", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "off", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("ERROR")));
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, Some("WARN")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("foo=warn")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs=error")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs::server=error")));
assert!(!should_demote_target(
HTTP_SERVER_LOG_TARGET,
"info",
None,
Some(&http_log_directive("error"))
));
assert!(!should_demote_target(
HTTP_SERVER_LOG_TARGET,
"info",
None,
Some(&format!("WARN,{}", http_log_directive("error")))
));
assert!(should_demote_http_request_logs(
assert!(should_demote_target(
HTTP_SERVER_LOG_TARGET,
"error",
None,
Some(&format!("WARN,{}", http_log_directive("warn")))
@@ -469,15 +464,15 @@ mod tests {
#[test]
fn test_should_demote_s3s_logs() {
assert!(should_demote_s3s_logs("info", None, None));
assert!(should_demote_s3s_logs("warn", None, None));
assert!(!should_demote_s3s_logs("error", None, None));
assert!(!should_demote_s3s_logs("off", None, None));
assert!(!should_demote_s3s_logs("info", None, Some("ERROR")));
assert!(should_demote_s3s_logs("error", None, Some("WARN")));
assert!(!should_demote_s3s_logs("info", None, Some("foo=warn")));
assert!(!should_demote_s3s_logs("info", None, Some("s3s=error")));
assert!(should_demote_s3s_logs("error", None, Some("WARN,s3s=warn")));
assert!(should_demote_target("s3s", "info", None, None));
assert!(should_demote_target("s3s", "warn", None, None));
assert!(!should_demote_target("s3s", "error", None, None));
assert!(!should_demote_target("s3s", "off", None, None));
assert!(!should_demote_target("s3s", "info", None, Some("ERROR")));
assert!(should_demote_target("s3s", "error", None, Some("WARN")));
assert!(!should_demote_target("s3s", "info", None, Some("foo=warn")));
assert!(!should_demote_target("s3s", "info", None, Some("s3s=error")));
assert!(should_demote_target("s3s", "error", None, Some("WARN,s3s=warn")));
}
#[test]
@@ -619,10 +614,14 @@ mod tests {
let output = capture_with_filter(build_env_filter("info", None), || {
tracing::info!(target: "s3s::auth", "s3s info must be suppressed");
tracing::warn!(target: "s3s::auth", "s3s warning must remain");
tracing::info!(target: "rustfs_gateway::service", "gateway info must be suppressed");
tracing::warn!(target: "rustfs_gateway::service", "gateway warning must remain");
});
assert!(!output.contains("s3s info must be suppressed"), "{output}");
assert!(output.contains("s3s warning must remain"), "{output}");
assert!(!output.contains("gateway info must be suppressed"), "{output}");
assert!(output.contains("gateway warning must remain"), "{output}");
});
}
+3 -3
View File
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
use crate::config::OtelConfig;
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3_dependency_log_filter};
use crate::telemetry::rolling::{RollingAppender, Rotation};
use metrics::counter;
use rustfs_config::observability::{
@@ -423,7 +423,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
.with(env_filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(ErrorLayer::default())
.with(fmt_layer)
.try_init()
@@ -537,7 +537,7 @@ fn init_file_logging_internal(
.with(env_filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(ErrorLayer::default())
.with(file_layer)
.with(stdout_layer)
+4 -4
View File
@@ -39,7 +39,7 @@
use crate::cleaner::types::FileMatchMode;
use crate::config::OtelConfig;
use crate::global::set_observability_metric_enabled;
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3_dependency_log_filter};
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
use crate::telemetry::local::{build_json_log_layer, open_rolling_appender, spawn_cleanup_task};
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -319,7 +319,7 @@ pub(super) fn init_observability_http(
.with(filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(ErrorLayer::default())
.with(file_layer_opt)
.with(stdout_layer_opt)
@@ -1361,7 +1361,7 @@ mod tests {
let subscriber = tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(
tracing_subscriber::fmt::layer()
.with_ansi(false)
@@ -1385,7 +1385,7 @@ mod tests {
tracing_subscriber::registry()
.with(build_env_filter("info", None))
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
.try_init()
.expect("install isolated LogTracer subscriber");
+1 -1
View File
@@ -13,7 +13,7 @@ Applies to `crates/policy/`.
- Validate compatibility with IAM and request-auth call sites:
- `crates/iam/`
- `rustfs/src/auth.rs`
- `rustfs/src/storage/access.rs`
- `rustfs/src/storage/authorization/`
## Error and Observability
-4
View File
@@ -70,10 +70,6 @@ url.workspace = true
urlencoding = { workspace = true }
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
# The remote error code table is checked against the legacy client's.
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
[lints]
workspace = true
+18 -15
View File
@@ -106,7 +106,7 @@ impl fmt::Display for RemoteErrorCode {
}
}
/// The codes the legacy client (s3s at the pinned revision) recognized, sorted.
/// The sorted wire codes recognized by the legacy client at revision 5761ddfe.
const RECOGNIZED_CODES: &[&str] = &[
"AccessControlListNotSupported",
"AccessDenied",
@@ -352,15 +352,19 @@ const RECOGNIZED_CODES: &[&str] = &[
#[cfg(test)]
mod tests {
use super::*;
use sha2::{Digest, Sha256};
#[test]
fn recognizes_exactly_the_codes_the_legacy_client_named() {
assert_eq!(RECOGNIZED_CODES.len(), 239);
assert!(RECOGNIZED_CODES.windows(2).all(|pair| pair[0] < pair[1]), "the table must stay sorted");
// SHA-256 of the sorted, newline-separated wire names in S3_ERROR_CODE_MAP at 5761ddfe.
let digest = Sha256::digest(RECOGNIZED_CODES.join("\n"));
assert_eq!(
hex_simd::encode_to_string(digest.as_slice(), hex_simd::AsciiCase::Lower),
"6fa6be09aa53ae2e05c5c98c10c43dae79713b07a2bdde492dbb6a776888978d"
);
for code in RECOGNIZED_CODES {
let legacy = s3s::S3ErrorCode::from_bytes(code.as_bytes()).expect("utf-8");
assert!(!matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{code} is not named by the legacy client");
assert_eq!(legacy.as_str(), *code);
let parsed = RemoteErrorCode::from_wire(code);
assert!(!parsed.is_custom(), "{code}");
assert_eq!(parsed.as_str(), *code);
@@ -369,19 +373,18 @@ mod tests {
#[test]
fn parses_like_the_legacy_client() {
for text in [
"NoSuchKey",
"nosuchkey",
"NOSUCHKEY",
"PermanentRedirect",
"SomethingNew",
"",
"No Such Key",
for (text, expected, custom) in [
("NoSuchKey", "NoSuchKey", false),
("nosuchkey", "NoSuchKey", false),
("NOSUCHKEY", "NoSuchKey", false),
("PermanentRedirect", "PermanentRedirect", false),
("SomethingNew", "SomethingNew", true),
("", "", true),
("No Such Key", "No Such Key", true),
] {
let legacy = s3s::S3ErrorCode::from_bytes(text.as_bytes()).expect("utf-8");
let parsed = RemoteErrorCode::from_wire(text);
assert_eq!(parsed.is_custom(), matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{text:?}");
assert_eq!(parsed.as_str(), legacy.as_str(), "{text:?}");
assert_eq!(parsed.is_custom(), custom, "{text:?}");
assert_eq!(parsed.as_str(), expected, "{text:?}");
}
assert_eq!(RemoteErrorCode::from_wire("nosuchkey"), RemoteErrorCode::NO_SUCH_KEY);
}
+4 -2
View File
@@ -164,10 +164,10 @@ impl From<legacy_s3s::S3Error> for S3Error {
/// reference only, so the whole error travels inside and this wrapper presents
/// the cause's own text and chain, as if the cause itself had been moved.
#[derive(Debug)]
struct LegacyCause(legacy_s3s::S3Error);
pub(super) struct LegacyCause(legacy_s3s::S3Error);
impl LegacyCause {
fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
pub(super) fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
self.0.source()
}
}
@@ -378,6 +378,8 @@ mod tests {
legacy.set_source(Box::new(io::Error::other("read quorum")));
let mut ours: S3Error = legacy.into();
assert!(ours.source().is_some_and(|cause| cause.is::<io::Error>()));
assert!(std::error::Error::source(&ours).is_some_and(|cause| cause.is::<io::Error>()));
ours.set_message("retry later".to_owned());
let back: legacy_s3s::S3Error = ours.into();
assert_eq!(back.code(), &legacy_s3s::S3ErrorCode::ServiceUnavailable);
+7 -2
View File
@@ -157,7 +157,12 @@ impl S3Error {
/// The cause, if one was attached.
pub fn source(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
self.0.source.as_deref()
let source = self.0.source.as_deref()?;
#[cfg(any(test, feature = "compat-s3s"))]
if let Some(legacy) = source.downcast_ref::<crate::compat_s3s::LegacyCause>() {
return legacy.cause();
}
Some(source)
}
/// The explicit status if one was set, otherwise the code's own status;
@@ -202,7 +207,7 @@ impl fmt::Display for S3Error {
impl std::error::Error for S3Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.0.source.as_deref().map(|source| source as _)
S3Error::source(self).map(|source| source as _)
}
}
-1
View File
@@ -109,7 +109,6 @@ bytes.workspace = true
hex-simd.workspace = true
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
opentelemetry-proto.workspace = true
opentelemetry_sdk.workspace = true
prost.workspace = true
+90 -337
View File
@@ -1341,32 +1341,9 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() {
#[test]
fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after() {
#[derive(Serialize)]
struct OldDataUsageCacheInfo {
name: String,
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
}
let old_info = OldDataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
last_update: None,
skip_healing: true,
lifecycle: None,
replication: None,
failed_objects: HashMap::from([("bad-object".to_string(), 11)]),
};
let mut buf = Vec::new();
old_info
.serialize(&mut rmp_serde::Serializer::new(&mut buf))
.expect("Failed to serialize old cache info");
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(&buf).expect("Failed to deserialize old cache info");
// Seven-field tuple written with the pinned legacy scanner codec.
let old_info = b"\x97\xa6bucket\x07\xc0\xc3\xc0\xc0\x81\xaabad-object\x0b";
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(old_info).expect("decode the historical cache info tuple");
assert_eq!(decoded.name, "bucket");
assert_eq!(decoded.next_cycle, 7);
@@ -1391,13 +1368,13 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
lifecycle: Option<()>,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
scan_resume_after: Option<String>,
scan_checkpoint: Option<DataUsageScanCheckpoint>,
pending_heals: Vec<PendingScannerHeal>,
object_lock: Option<Arc<s3s::dto::ObjectLockConfiguration>>,
object_lock: Option<()>,
}
#[derive(Deserialize)]
@@ -3560,22 +3537,21 @@ const LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD: &[u8] = &[
0x62, 0x6c, 0x65, 0x64, 0xc0,
];
// Keep the old tuple carrier and payload explicit when the production field changes type.
// Project the seven original fields out of the production map. Keep the nested
// MessagePack arrays and maps opaque so their bytes can be checked against the old writer.
#[derive(serde::Serialize, serde::Deserialize)]
struct LegacyLifecycleCacheInfo {
name: String,
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
lifecycle: Value,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
}
struct LifecycleCacheOracleCase {
label: &'static str,
old_tuple: Vec<u8>,
old_payload: Vec<u8>,
current: DataUsageCacheInfo,
current_map: Vec<u8>,
frozen_tuple: &'static [u8],
@@ -3584,7 +3560,6 @@ struct LifecycleCacheOracleCase {
}
fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
use s3s::dto as legacy;
use time::OffsetDateTime;
let expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_456_789)
.expect("the legacy expiration fixture has an exact timestamp");
@@ -3593,136 +3568,6 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
let expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_789_123)
.expect("the legacy expiry update fixture has an exact timestamp");
let absent = legacy::LifecycleRule {
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
status: legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::ENABLED),
transitions: None,
};
let mut empty = absent.clone();
empty.id = Some(String::new());
empty.prefix = Some(String::new());
empty.status = legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::DISABLED);
empty.abort_incomplete_multipart_upload = Some(legacy::AbortIncompleteMultipartUpload {
days_after_initiation: None,
});
empty.del_marker_expiration = Some(legacy::DelMarkerExpiration { days: None });
empty.expiration = Some(legacy::LifecycleExpiration {
days: Some(0),
..Default::default()
});
empty.filter = Some(legacy::LifecycleRuleFilter {
and: Some(legacy::LifecycleRuleAndOperator {
tags: Some(Vec::new()),
..Default::default()
}),
tag: Some(legacy::Tag { key: None, value: None }),
..Default::default()
});
empty.noncurrent_version_expiration = Some(legacy::NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(-2),
});
empty.noncurrent_version_transitions = Some(Vec::new());
empty.transitions = Some(Vec::new());
let mut absent_tags = absent.clone();
absent_tags.filter = Some(legacy::LifecycleRuleFilter {
and: Some(legacy::LifecycleRuleAndOperator::default()),
..Default::default()
});
// Cached values must survive independently of lifecycle policy validation.
let complete = legacy::BucketLifecycleConfiguration {
expiry_updated_at: Some(expiry_updated_at.into()),
rules: vec![
legacy::LifecycleRule {
abort_incomplete_multipart_upload: Some(legacy::AbortIncompleteMultipartUpload {
days_after_initiation: Some(0),
}),
del_marker_expiration: Some(legacy::DelMarkerExpiration { days: Some(-1) }),
expiration: Some(legacy::LifecycleExpiration {
date: Some(expiration_date.into()),
days: Some(-3),
expired_object_delete_marker: Some(false),
expired_object_all_versions: Some(false),
}),
filter: Some(legacy::LifecycleRuleFilter {
cached_tags: Default::default(),
and: Some(legacy::LifecycleRuleAndOperator {
prefix: Some(String::new()),
tags: Some(vec![
legacy::Tag {
key: Some(String::new()),
value: Some(String::new()),
},
legacy::Tag {
key: None,
value: Some("value-without-key".to_string()),
},
legacy::Tag {
key: Some("key-without-value".to_string()),
value: None,
},
]),
object_size_greater_than: Some(-1),
object_size_less_than: Some(4096),
}),
prefix: Some(String::new()),
tag: Some(legacy::Tag {
key: Some("kind".to_string()),
value: Some("archive".to_string()),
}),
object_size_greater_than: Some(0),
object_size_less_than: Some(8192),
}),
id: Some("complete-legacy-rule".to_string()),
noncurrent_version_expiration: Some(legacy::NoncurrentVersionExpiration {
noncurrent_days: Some(-7),
newer_noncurrent_versions: Some(0),
}),
noncurrent_version_transitions: Some(vec![
legacy::NoncurrentVersionTransition {
noncurrent_days: Some(0),
newer_noncurrent_versions: Some(-1),
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
},
legacy::NoncurrentVersionTransition {
noncurrent_days: None,
newer_noncurrent_versions: None,
storage_class: None,
},
]),
prefix: Some("legacy-prefix/".to_string()),
status: legacy::ExpirationStatus::from_static("LegacyUnknownStatus"),
transitions: Some(vec![
legacy::Transition {
date: Some(transition_date.into()),
days: Some(0),
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
},
legacy::Transition {
date: None,
days: None,
storage_class: Some(legacy::TransitionStorageClass::from_static("")),
},
legacy::Transition {
date: None,
days: None,
storage_class: None,
},
]),
},
empty,
absent_tags,
absent,
],
};
use rustfs_gateway_types::persistence as persisted;
let domain_absent = persisted::PersistedLifecycleRule {
abort_incomplete_multipart_upload: None,
@@ -3869,17 +3714,12 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
(
"none",
None,
None,
LEGACY_LIFECYCLE_NONE_TUPLE,
LEGACY_LIFECYCLE_NONE_MAP,
LEGACY_LIFECYCLE_NONE_PAYLOAD,
),
(
"empty",
Some(Arc::new(legacy::BucketLifecycleConfiguration {
rules: Vec::new(),
expiry_updated_at: None,
})),
Some(Arc::new(persisted::PersistedLifecycleConfiguration {
rules: Vec::new(),
expiry_updated_at: None,
@@ -3890,7 +3730,6 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
),
(
"nonempty",
Some(Arc::new(complete)),
Some(Arc::new(domain_complete)),
LEGACY_LIFECYCLE_NONEMPTY_TUPLE,
LEGACY_LIFECYCLE_NONEMPTY_MAP,
@@ -3899,35 +3738,17 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
];
cases
.into_iter()
.map(|(label, lifecycle, domain, frozen_tuple, frozen_map, frozen_payload)| {
let old = LegacyLifecycleCacheInfo {
.map(|(label, lifecycle, frozen_tuple, frozen_map, frozen_payload)| {
let current = DataUsageCacheInfo {
name: "legacy-lifecycle-bucket".to_string(),
next_cycle: 7,
last_update: None,
skip_healing: true,
lifecycle,
replication: None,
failed_objects: HashMap::new(),
};
let mut old_tuple = Vec::new();
old.serialize(&mut rmp_serde::Serializer::new(&mut old_tuple))
.expect("the real old tuple carrier must encode");
let old_payload = rmp_serde::to_vec(&old.lifecycle).expect("the real old lifecycle payload must encode");
let current = DataUsageCacheInfo {
name: old.name.clone(),
next_cycle: old.next_cycle,
last_update: old.last_update,
skip_healing: old.skip_healing,
lifecycle: domain,
replication: None,
failed_objects: old.failed_objects,
..Default::default()
};
let current_map = rmp_serde::to_vec(&current).expect("the actual production map writer must encode the fixture");
LifecycleCacheOracleCase {
label,
old_tuple,
old_payload,
current,
current_map,
frozen_tuple,
@@ -3940,57 +3761,36 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
#[test]
fn test_lifecycle_cache_legacy_msgpack_oracle() {
use time::OffsetDateTime;
for case in lifecycle_cache_oracle_cases() {
let label = case.label;
let old_tuple = case.old_tuple;
let old_payload = case.old_payload;
let current = case.current;
let current_map = case.current_map;
let frozen_tuple = case.frozen_tuple;
let frozen_map = case.frozen_map;
let frozen_payload = case.frozen_payload;
println!("legacy_lifecycle_cache {label}: tuple={old_tuple:02x?}; map={current_map:02x?}; payload={old_payload:02x?}");
let old_from_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&current_map).expect("the actual old reader must read the production map");
if label == "nonempty" {
let lifecycle = old_from_current
.lifecycle
.as_ref()
.expect("the old reader must retain the nonempty lifecycle");
let expiration = lifecycle.rules[0]
.expiration
.as_ref()
.expect("the complete rule has expiration");
let transition = &lifecycle.rules[0]
.transitions
.as_ref()
.expect("the complete rule has transitions")[0];
let expected_expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000)
.expect("the legacy millisecond expiration date is representable");
let expected_transition_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_001_987_000_000)
.expect("the legacy millisecond transition date is representable");
let expected_expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_000_000)
.expect("the legacy millisecond expiry update is representable");
assert_eq!(
(
expiration.date.clone().map(OffsetDateTime::from),
transition.date.clone().map(OffsetDateTime::from),
lifecycle.expiry_updated_at.clone().map(OffsetDateTime::from),
),
(
Some(expected_expiration_date),
Some(expected_transition_date),
Some(expected_expiry_updated_at),
),
"cache serialization must retain the fixed legacy millisecond precision for Date and ExpiryUpdatedAt"
);
}
let LifecycleCacheOracleCase {
label,
current,
current_map,
frozen_tuple,
frozen_map,
frozen_payload,
} = case;
let frozen_tuple_read: DataUsageCacheInfo =
rmp_serde::from_slice(frozen_tuple).expect("the production reader must accept the fixed legacy tuple bytes");
let frozen_map_read: DataUsageCacheInfo =
rmp_serde::from_slice(frozen_map).expect("the production reader must accept the fixed legacy map bytes");
if label == "nonempty" {
let lifecycle = frozen_map_read.lifecycle.as_ref().expect("nonempty lifecycle");
let rule = &lifecycle.rules[0];
assert_eq!(
[
rule.expiration.as_ref().expect("expiration").date.as_deref(),
rule.transitions.as_ref().expect("transitions")[0].date.as_deref(),
lifecycle.expiry_updated_at.as_deref(),
],
[
Some("2023-11-14T22:13:20.123Z"),
Some("2023-11-14T22:13:21.987Z"),
Some("2023-11-14T22:13:22.456Z"),
],
"legacy timestamps retain millisecond precision",
);
}
assert_eq!(
[
rmp_serde::to_vec(&frozen_tuple_read).expect("encode the fixed legacy tuple after production read"),
@@ -4014,21 +3814,21 @@ fn test_lifecycle_cache_legacy_msgpack_oracle() {
fn test_lifecycle_cache_legacy_carrier_round_trips() {
for case in lifecycle_cache_oracle_cases() {
let label = case.label;
let old_tuple = case.old_tuple;
let old_tuple = case.frozen_tuple;
let current_map = case.current_map;
let from_old: DataUsageCacheInfo =
rmp_serde::from_slice(&old_tuple).expect("the production reader must read the old tuple");
rmp_serde::from_slice(old_tuple).expect("the production reader must read the old tuple");
let from_old_map = rmp_serde::to_vec(&from_old).expect("the production writer must retain the old tuple payload");
let old_via_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&from_old_map).expect("the old reader must read the production map after an old-tuple load");
rmp_serde::from_slice(&from_old_map).expect("project the old fields after an old-tuple load");
let old_from_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&current_map).expect("the actual old reader must read the production map");
rmp_serde::from_slice(&current_map).expect("project the old fields after a production write");
assert_eq!(
[
rmp_serde::to_vec(&old_via_current).expect("encode the old reader's complete tuple after production reload"),
rmp_serde::to_vec(&old_from_current).expect("encode the old reader's complete tuple after production write"),
],
[old_tuple.clone(), old_tuple],
[old_tuple, old_tuple],
"both cache directions must preserve the full old carrier for {label}"
);
}
@@ -4051,32 +3851,12 @@ fn test_lifecycle_cache_current_map_round_trip() {
#[test]
fn test_lifecycle_cache_strict_cached_tags_contract() {
use s3s::dto as legacy;
type LegacyPayload = Option<Arc<legacy::BucketLifecycleConfiguration>>;
let old: LegacyPayload = rmp_serde::from_slice(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD)
.expect("the frozen payload must initialize the real old configuration");
let filter = old.as_ref().expect("nonempty old payload").rules[0]
.filter
.as_ref()
.expect("first old rule has a filter");
let object_tags = [(String::new(), String::new()), ("kind".to_owned(), "archive".to_owned())]
.into_iter()
.collect();
// The real old inherent method initializes its OnceLock even if matching fails.
let _ = filter.test_tags(&object_tags);
// The pinned legacy writer produced the same empty map before and after test_tags initialized its cache.
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD))
.expect("the production reader must accept the fixed payload");
assert_eq!(
[
rmp_serde::to_vec(&old).expect("encode the initialized old cache"),
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the new cache")
],
[
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
],
"an initialized old CachedTags must still write the same empty map as the new carrier"
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the cached tags"),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
);
let slot = LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD
@@ -4089,22 +3869,14 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
bytes.extend_from_slice(&LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD[slot + 1..]);
bytes
};
// A historical String/String map is consumed by both real readers, then
// rewritten empty. The original populated input is deliberately not the oracle.
// The pinned reader discarded populated String/String maps and rejected the other shapes below.
let populated = input_with_cache(&[0x81, 0xa3, b'e', b'n', b'v', 0xa4, b'p', b'r', b'o', b'd']);
let old: LegacyPayload = rmp_serde::from_slice(&populated).expect("old reader accepts String/String cached tags");
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(populated.as_slice()))
.expect("new reader accepts String/String cached tags");
assert_eq!(
[
rmp_serde::to_vec(&old).expect("rewrite the old populated cache"),
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the new populated cache")
],
[
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
],
"both readers discard populated cached tags and both writers emit an empty map"
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the populated cache"),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
"populated cached tags are discarded on rewrite",
);
for (label, replacement) in [
@@ -4115,13 +3887,8 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
("nonstring-value", &[0x81, 0xa1, b'k', 0x01][..]),
] {
let input = input_with_cache(replacement);
let old = rmp_serde::from_slice::<LegacyPayload>(&input);
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(input.as_slice()));
assert_eq!(
[old.is_err(), new.is_err()],
[true, true],
"both real readers must reject {label} cached tags"
);
assert!(new.is_err(), "must reject {label} cached tags");
}
}
@@ -4133,73 +3900,59 @@ fn object_lock_cache_matches_the_legacy_dto_bytes() {
use rustfs_gateway_types::persistence::{
PersistedDefaultRetention, PersistedObjectLockConfiguration, PersistedObjectLockRule,
};
use s3s::dto as legacy;
let legacy_cases = vec![
legacy::ObjectLockConfiguration {
object_lock_enabled: None,
rule: None,
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: None,
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from("Disabled".to_owned())),
rule: Some(legacy::ObjectLockRule { default_retention: None }),
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: Some(legacy::ObjectLockRule {
default_retention: Some(legacy::DefaultRetention {
// Captured from the minio-enabled s3s DTO at 5761ddfe4c509fcac87e3b99fd5bd9e176b442c0.
let cases = [
(b"\x92\xc0\xc0".as_slice(), None, None),
(b"\x92\xa7Enabled\xc0".as_slice(), Some("Enabled"), None),
(
b"\x92\xa8Disabled\x91\xc0".as_slice(),
Some("Disabled"),
Some(PersistedObjectLockRule { default_retention: None }),
),
(
b"\x92\xa7Enabled\x91\x93\x1e\xaaGOVERNANCE\xc0".as_slice(),
Some("Enabled"),
Some(PersistedObjectLockRule {
default_retention: Some(PersistedDefaultRetention {
days: Some(30),
mode: Some(legacy::ObjectLockRetentionMode::from_static(legacy::ObjectLockRetentionMode::GOVERNANCE)),
mode: Some("GOVERNANCE".to_owned()),
years: None,
}),
}),
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: Some(legacy::ObjectLockRule {
default_retention: Some(legacy::DefaultRetention {
),
(
b"\x92\xa7Enabled\x91\x93\xc0\xaecompliance-ish\x02".as_slice(),
Some("Enabled"),
Some(PersistedObjectLockRule {
default_retention: Some(PersistedDefaultRetention {
days: None,
mode: Some(legacy::ObjectLockRetentionMode::from("compliance-ish".to_owned())),
mode: Some("compliance-ish".to_owned()),
years: Some(2),
}),
}),
},
),
];
for legacy_config in legacy_cases {
let persisted = PersistedObjectLockConfiguration {
object_lock_enabled: legacy_config
.object_lock_enabled
.as_ref()
.map(|value| value.as_str().to_owned()),
rule: legacy_config.rule.as_ref().map(|rule| PersistedObjectLockRule {
default_retention: rule.default_retention.as_ref().map(|retention| PersistedDefaultRetention {
mode: retention.mode.as_ref().map(|value| value.as_str().to_owned()),
days: retention.days,
years: retention.years,
}),
}),
};
let old_bytes = rmp_serde::to_vec(&Some(Arc::new(legacy_config))).expect("encode the legacy DTO");
let current = Some(Arc::new(persisted.clone()));
for (old_bytes, enabled, rule) in cases {
let current = Some(Arc::new(PersistedObjectLockConfiguration {
object_lock_enabled: enabled.map(str::to_owned),
rule,
}));
let new_bytes = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&current)).expect("encode the new cache");
assert_eq!(new_bytes, old_bytes, "the new writer must produce the legacy bytes for {persisted:?}");
let decoded = crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes.as_slice()))
.expect("decode the legacy bytes");
assert_eq!(decoded.as_deref(), Some(&persisted));
assert_eq!(new_bytes, old_bytes, "must reproduce the original bytes for {current:?}");
let decoded =
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes)).expect("decode the legacy bytes");
assert_eq!(decoded, current);
}
let absent_old = rmp_serde::to_vec(&Option::<Arc<legacy::ObjectLockConfiguration>>::None).expect("encode absent");
let absent_new = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent");
assert_eq!(absent_new, absent_old);
let decoded =
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(absent_old.as_slice())).expect("decode absent");
assert!(decoded.is_none());
let absent = b"\xc0";
assert_eq!(
rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent"),
absent
);
assert!(
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(&absent[..]))
.expect("decode absent")
.is_none()
);
}
#[test]
@@ -9,7 +9,7 @@ This page is a pointer, not a route table. The machine-checked matrix in `route_
| Prefix | Behavior | Rule |
|---|---|---|
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_route` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_router` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
| `/minio/admin` | Compatibility alias accepted by `S3Router::is_match`; `canonicalize_admin_path` rewrites it to `/rustfs/admin` immediately before route lookup (`rustfs/src/admin/router.rs`) | Never register routes twice; preserve canonicalization |
| `/iceberg/v1` | Table catalog prefix registered by `register_table_catalog_route` (`rustfs/src/admin/handlers/table_catalog/routes.rs`) and accepted by `is_admin_path` | Stays outside `/rustfs/admin`; table actions are authorized per handler |
| `/health`, `/health/ready` | Public health endpoints, registered only when `ENV_HEALTH_ENDPOINT_ENABLE` allows | Preserve the unauthenticated bypass |
@@ -1,7 +1,7 @@
# MinIO ↔ RustFS Router Compatibility (Exceptions Only)
**Use this when:** a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different.
**Source of truth:** S3 plane: the `s3s::S3` trait impl in `rustfs/src/storage/ecfs.rs`. Admin plane: `make_admin_route` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
**Source of truth:** S3 plane: the registered operation families in `rustfs/src/server/gateway_handlers/mod.rs`. Admin plane: `make_admin_router` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
This document lists only exceptions. Anything not listed here is implemented with MinIO-equivalent behavior. For the s3tests-level claim see [s3-compatibility-matrix.md](s3-compatibility-matrix.md).
@@ -18,7 +18,7 @@ Admin paths are relative to the canonical `/rustfs/admin` prefix; `/minio/admin`
## S3 Data Plane
All `s3s::S3` trait methods are implemented in `rustfs/src/storage/ecfs.rs` except the following.
Run `python3 scripts/s3-tests/api_coverage.py` for the registered gateway operations and the missing operations in the pinned S3 reference. The following table records compatibility exceptions.
| S3 operation | Status | Detail |
|---|---|---|
+1 -1
View File
@@ -97,7 +97,7 @@ Standard S3 areas that must not be described as complete:
| Area | Status | Evidence |
|---|---|---|
| Bucket access logging | Handlers exist (`get_bucket_logging`, `put_bucket_logging` in `rustfs/src/storage/ecfs.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
| Bucket access logging | Handlers exist (`GetBucketLogging` and `PutBucketLogging` in `rustfs/src/server/gateway_handlers/bucket_core.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
| POST Object form upload checksum handling | Not yet passing | `unimplemented_tests.txt` |
| Bucket ownership controls | No handler | `unimplemented_tests.txt` |
| Multipart upload listing and part lookup compatibility edge cases | Not part of default gate; ListParts returns HTTP 503 `SlowDownRead` when no directory was listed and any participant is unavailable, instead of treating one missing directory as an empty upload | `excluded_tests.txt`; `collect_list_parts_results_rejects_unavailable_empty_upload_dirs` |
+1 -1
View File
@@ -1,7 +1,7 @@
# Audit Entries for Authorization Denials
**Use this when:** you consume the audit stream and need to know which authorization denials appear in it, how many, and how to tune the limit.
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/access.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/authorization/decision.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
With audit enabled (`RUSTFS_AUDIT_ENABLE=true`), a request refused by the authorization layer produces an audit entry in addition to the `s3_authorization_denied` log event. The S3 response is unchanged: a bare `Access Denied`.
+17
View File
@@ -0,0 +1,17 @@
# S3 stack upgrade and rollback
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
## Roll back to the previous release
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
## Client-visible error change
| Conformance case | What clients see |
| --- | --- |
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
+1 -1
View File
@@ -13,7 +13,7 @@ Every fixed RustFS GitHub Security Advisory maps to at least one named regressio
| [GHSA-r5qv-rc46-hv8q](https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q) | Internode RPC authentication must fail closed | rustfs/rustfs#4402 | `ghsa_r5qv_resolve_shared_secret_rejects_default_fallback`, `ghsa_r5qv_verify_rpc_signature_fails_closed_on_missing_or_invalid_auth` (`crates/ecstore/src/cluster/rpc/http_auth.rs`) | unit |
| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with the shared root secret (intentionally unfixed) | n/a; tests pin the by-design behaviour and must flip red -> green when m77q is fixed | `test_ghsa_m77q_sts_session_token_signed_with_root_secret`, `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) | unit |
| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope; a non-owner with `CreateServiceAccountAdminAction` could parent a service account to root | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope`, `ghsa_5354_scope_guard_preserves_non_root_self_scope`, and the `add_service_account_parent_within_scope` invariant they pin (`rustfs/src/admin/handlers/service_account.rs`) | unit |
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit |
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` in `rustfs/src/storage/authorization/tests/actions.rs` covers `versioned_read_action` in `rustfs/src/storage/authorization_actions.rs` | unit |
| [GHSA-v9cp-qfw9-9pfp](https://github.com/rustfs/rustfs/security/advisories/GHSA-v9cp-qfw9-9pfp) | `ForAllValues:`/`ForAnyValue:` negated string operators applied negation to the aggregate instead of the per-value predicate | fixed, GHSA private-fork merge | `ghsa_v9cp_for_all_values_not_equals_partial_overlap`, `ghsa_v9cp_for_any_value_not_equals_partial_overlap` and the absent-key/positive-quantifier cases beside them (`crates/policy/tests/quantified_negation.rs`); the value set must partially overlap the policy set, since contained or disjoint sets cannot tell the quantifiers apart | crate test |
| [GHSA-6r96-hmgc-726c](https://github.com/rustfs/rustfs/security/advisories/GHSA-6r96-hmgc-726c) | Request headers must not populate server-derived IAM condition keys (`userid`, `groups`, `jwt:`/`ldap:` claims) | fixed, GHSA private-fork merge | `ghsa_6r96_identity_condition_keys_ignore_spoofed_headers`, `ghsa_6r96_claim_condition_keys_ignore_spoofed_headers`, and `test_request_headers_still_reach_conditions`, which keeps the reserved set from growing too broad (`rustfs/src/auth.rs`) | unit |
| [GHSA-x298-9x87-fvjq](https://github.com/rustfs/rustfs/security/advisories/GHSA-x298-9x87-fvjq) | Anonymous ListObjectVersions -> `s3:ListBucket` fallback must reach the same public-access gates as a direct grant | fixed, GHSA private-fork merge | `ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled` (`crates/e2e_test/src/anonymous_access_test.rs`); asserts 200 before the public-access block is applied so it proves the gate, not a broken fallback | e2e (`e2e-smoke`) |
+39 -31
View File
@@ -195,27 +195,14 @@ impl Operation for ChangeOwnPasswordHandler {
let request: ChangePasswordRequest = serde_json::from_slice(&body)
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
let iam_store =
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
// Reached only if the identity was deleted between authentication
// and this lookup.
emit_audit(
&audit,
AccountAuditRecord::failure(
AccountAuditOperation::ChangeOwnPassword,
&caller.access_key,
caller.identity_type,
AccountAuditFailure::Internal,
),
);
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
// An STS session signs with its own secret. A direct IAM request must
// have been signed with the same password the caller is replacing.
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
&caller.credentials.secret_key
} else {
&request.current_secret_key
};
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
{
if !constant_time_eq(expected_signing_secret, &signing_secret) {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_ACCOUNT,
@@ -256,8 +243,10 @@ impl Operation for ChangeOwnPasswordHandler {
}
let access_key = caller.access_key.clone();
let new_secret_key = request.new_secret_key.clone();
let current_secret_key = request.current_secret_key;
let new_secret_key = request.new_secret_key;
let identity_type = caller.identity_type;
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
let session_access_key = caller.session_access_key.clone();
let audit_for_task = audit.clone();
@@ -268,16 +257,35 @@ impl Operation for ChangeOwnPasswordHandler {
let iam_store =
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
let (updated_at, status, revoked) = iam_store
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
.await
.map_err(|error| {
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
} else {
iam_error_to_s3_error(error)
}
})?;
let rotation = match session_credentials.as_ref() {
Some(session) => {
iam_store
.change_own_secret_key_from_session(&access_key, &current_secret_key, &new_secret_key, session)
.await
}
None => {
iam_store
.change_own_secret_key(&access_key, &current_secret_key, &new_secret_key)
.await
}
};
let (updated_at, status, revoked) = rotation.map_err(|error| {
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
emit_audit(
&audit_for_task,
AccountAuditRecord::failure(
AccountAuditOperation::ChangeOwnPassword,
&access_key,
identity_type,
AccountAuditFailure::InvalidCurrentSecret,
)
.with_session_access_key(session_access_key.as_deref()),
);
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
} else {
iam_error_to_s3_error(error)
}
})?;
emit_audit(
&audit_for_task,
+12 -38
View File
@@ -1868,6 +1868,8 @@ mod tests {
peer_topology_complete, query_peer_heal_status, query_peer_replacement_recovery_status, read_cluster_heal_status,
reject_heal_admission, validate_heal_request_mode, validate_heal_target, validate_mrf_legacy_responsibility_action,
};
use crate::admin::operation::test_support::TestRequest;
use crate::admin::router::dispatch;
use crate::storage::rpc::node_service::heal::{
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
};
@@ -2153,33 +2155,19 @@ mod tests {
#[tokio::test]
async fn test_nested_heal_routes_still_require_authentication() {
use s3s::route::S3Route;
let mut router = super::S3Router::new(false);
super::register_heal_route(&mut router).expect("register production Heal routes");
for prefix in ["/rustfs/admin", "/minio/admin"] {
for object in ["dir/object.bin", "dir%2Fobject.bin", "literal%252Fslash"] {
let mut req = s3s::S3Request {
input: s3s::Body::empty(),
method: http::Method::POST,
uri: format!("{prefix}/v3/heal/test-bucket/{object}").parse().expect("Heal URI"),
headers: http::HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let request = TestRequest::new(http::Method::POST, format!("{prefix}/v3/heal/test-bucket/{object}")).build();
let err = router
.check_access(&mut req)
.await
.admit(request.method(), request.uri(), request.headers(), false)
.expect_err("router must require a signature");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
let err = router
.call(req)
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
.await
.expect_err("handler must independently require authentication");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert!(err.to_string().contains("authentication required"));
}
}
@@ -2187,34 +2175,20 @@ mod tests {
#[tokio::test]
async fn replacement_recovery_diagnostics_require_authentication() {
use s3s::route::S3Route;
let mut router = super::S3Router::new(false);
super::register_heal_route(&mut router).expect("register production Heal routes");
for prefix in ["/rustfs/admin", "/minio/admin"] {
let path = format!("{prefix}/v4/heal/replacement-recovery");
assert!(router.contains_compatible_route(http::Method::GET, &path));
let mut req = s3s::S3Request {
input: s3s::Body::empty(),
method: http::Method::GET,
uri: path.parse().expect("replacement diagnostics URI"),
headers: http::HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let request = TestRequest::new(http::Method::GET, path).build();
let err = router
.check_access(&mut req)
.await
.admit(request.method(), request.uri(), request.headers(), false)
.expect_err("diagnostics router must require a signature");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
let err = router
.call(req)
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
.await
.expect_err("diagnostics handler must authenticate before inspecting local records");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert!(err.to_string().contains("authentication required"));
}
}
+19 -51
View File
@@ -90,79 +90,47 @@ impl Operation for HealthCheckHandler {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::operation::test_support::TestRequest;
use crate::admin::router::dispatch;
use http_body_util::BodyExt as _;
use s3s::route::S3Route as _;
use serial_test::serial;
use temp_env::with_var;
fn health_request(method: Method) -> s3s::S3Request<s3s::Body> {
s3s::S3Request {
input: s3s::Body::empty(),
method,
uri: HEALTH_PREFIX.parse().expect("health URI"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
async fn call_real_health_handler(method: Method) -> s3s::S3Response<s3s::Body> {
async fn call_real_health_handler(method: Method) -> (StatusCode, HeaderMap, Body) {
let mut router = S3Router::new(false);
router
.insert(method.clone(), HEALTH_PREFIX, AdminOperation(&HealthCheckHandler {}))
.expect("health route should register");
router
.call(health_request(method))
let request = TestRequest::new(method, HEALTH_PREFIX).build();
dispatch(&router, request.into_envelope(), Body::empty())
.await
.expect("health handler should return a response")
.into_parts()
}
#[tokio::test]
async fn real_health_handler_get_and_head_preserve_wire_branches() {
let get = call_real_health_handler(Method::GET).await;
assert_eq!(get.status, Some(StatusCode::OK));
assert_eq!(
get.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
Some("application/json")
);
let payload: serde_json::Value =
serde_json::from_slice(&get.output.collect().await.expect("GET body should read").to_bytes())
.expect("GET health response should be JSON");
let (status, headers, body) = call_real_health_handler(Method::GET).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
let payload: serde_json::Value = serde_json::from_slice(&body.collect().await.expect("GET body should read").to_bytes())
.expect("GET health response should be JSON");
assert_eq!(payload["status"], "ok");
let head = call_real_health_handler(Method::HEAD).await;
assert_eq!(head.status, Some(StatusCode::OK));
assert_eq!(
head.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
Some("application/json")
);
assert!(
head.output
.collect()
.await
.expect("HEAD body should read")
.to_bytes()
.is_empty()
);
let (status, headers, body) = call_real_health_handler(Method::HEAD).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
assert!(body.collect().await.expect("HEAD body should read").to_bytes().is_empty());
}
#[tokio::test]
async fn real_health_handler_rejects_unsupported_methods_with_allow_header() {
let response = call_real_health_handler(Method::POST).await;
let (status, headers, body) = call_real_health_handler(Method::POST).await;
assert_eq!(response.status, Some(StatusCode::METHOD_NOT_ALLOWED));
assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED);
assert_eq!(headers.get(http::header::ALLOW).and_then(|value| value.to_str().ok()), Some("GET, HEAD"));
assert_eq!(
response
.headers
.get(http::header::ALLOW)
.and_then(|value| value.to_str().ok()),
Some("GET, HEAD")
);
assert_eq!(
response.output.collect().await.expect("error body should read").to_bytes(),
body.collect().await.expect("error body should read").to_bytes(),
bytes::Bytes::from_static(b"Method Not Allowed")
);
}
+9 -9
View File
@@ -332,7 +332,7 @@ fn is_lock_timeout(err: &crate::storage_api::error::StorageError) -> bool {
fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError {
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) {
return ApiError {
code: S3ErrorCode::InternalError.into(),
code: S3ErrorCode::InternalError,
message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(),
source: Some(Box::new(err)),
};
@@ -341,7 +341,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// movement gate has a queued writer: the same transient contention.
if is_lock_timeout(&err) || matches!(err, crate::storage_api::error::StorageError::SlowDown) {
return ApiError {
code: S3ErrorCode::ServiceUnavailable.into(),
code: S3ErrorCode::ServiceUnavailable,
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -350,7 +350,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// should say so, not just "Invalid Request".
if matches!(err, crate::storage_api::error::StorageError::DecommissionAlreadyRunning) {
return ApiError {
code: S3ErrorCode::InvalidRequest.into(),
code: S3ErrorCode::InvalidRequest,
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -1261,14 +1261,14 @@ mod pools_handler_tests {
#[test]
fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
let err = crate::error::ApiError {
code: rustfs_s3_types::S3ErrorCode::InvalidRequest.into(),
code: rustfs_s3_types::S3ErrorCode::InvalidRequest,
message: "decommission already running".to_string(),
source: None,
};
let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]");
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest.into());
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest);
assert_eq!(
err.message,
"admin start decommission failed for pools [1, 3]: decommission already running"
@@ -1281,12 +1281,12 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
assert!(err.source.is_some());
let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed"));
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(unrelated.message, "We encountered an internal error, please try again.");
}
@@ -1299,7 +1299,7 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
assert!(err.message.contains("data movement gate (decommission activation)"), "{}", err.message);
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(
@@ -1313,7 +1313,7 @@ mod pools_handler_tests {
fn test_decommission_start_api_error_reports_a_refused_preflight_heal_as_retryable() {
let err = decommission_start_api_error(crate::storage_api::error::StorageError::SlowDown);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
}
+5 -4
View File
@@ -926,10 +926,11 @@ impl Operation for RebalanceStatus {
"admin rebalance state"
);
// Preserve the idle 404 contract without triggering the custom-route error logger.
let (parts, body) = s3s::S3Error::with_message(s3s::S3ErrorCode::NoSuchResource, REBALANCE_NOT_STARTED_MESSAGE)
.to_http_response()?
.into_parts();
return Ok(AdminResponse::with_headers(parts.status, body.into(), parts.headers));
let body = format!(
r#"<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchResource</Code><Message>{REBALANCE_NOT_STARTED_MESSAGE}</Message></Error>"#
);
let headers = HeaderMap::from_iter([(CONTENT_TYPE, HeaderValue::from_static("application/xml"))]);
return Ok(AdminResponse::with_headers(StatusCode::NOT_FOUND, Body::from(body), headers));
}
error!(
+49 -25
View File
@@ -48,7 +48,6 @@ use rustfs_policy::{
};
use rustfs_s3_types::{Body, S3Error, S3ErrorCode, S3Result, s3_error};
use rustfs_utils::MaskedAccessKey;
use s3s::dto::{AssumeRoleOutput, Credentials, Timestamp};
use serde::Deserialize;
use serde_json::Value;
use serde_urlencoded::from_bytes;
@@ -337,33 +336,35 @@ async fn handle_assume_role(
warn!("site replication STS hook failed, err: {err}");
}
let resp = AssumeRoleOutput {
credentials: Some(Credentials {
access_key_id: new_cred.access_key,
expiration: Timestamp::from(
new_cred
.expiration
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600))),
),
secret_access_key: new_cred.secret_key,
session_token: new_cred.session_token,
}),
..Default::default()
};
// getAssumeRoleCredentials
let output = serialize_assume_role_output(&resp)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize assume role output failed: {e}")))?;
let expiration = new_cred
.expiration
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600)));
let output = assume_role_response_xml(&new_cred, expiration)?;
Ok(AdminResponse::new(StatusCode::OK, Body::from(output)))
}
/// The STS answer is an s3s response body, so the s3s encoder writes it; it
/// is not a bucket configuration and never goes through the bucket-config codec.
fn serialize_assume_role_output(output: &AssumeRoleOutput) -> s3s::xml::SerResult<Vec<u8>> {
let mut buf = Vec::with_capacity(256);
s3s::xml::Serialize::serialize(output, &mut s3s::xml::Serializer::new(&mut buf))?;
Ok(buf)
fn assume_role_response_xml(credentials: &rustfs_credentials::Credentials, expiration: OffsetDateTime) -> S3Result<String> {
// The STS codec writes local clock fields with millisecond precision and a four-digit year.
if !(0..=9999).contains(&expiration.year()) {
return Err(s3_error!(InternalError, "cannot serialize STS expiration year"));
}
let expiration = expiration
.format(time::macros::format_description!(
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
))
.map_err(|_| s3_error!(InternalError, "cannot serialize STS expiration"))?;
Ok(format!(
concat!(
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
"<AccessKeyId>{}</AccessKeyId><Expiration>{}</Expiration>",
"<SecretAccessKey>{}</SecretAccessKey><SessionToken>{}</SessionToken>",
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
),
xml_escape(&credentials.access_key),
expiration,
xml_escape(&credentials.secret_key),
xml_escape(&credentials.session_token),
))
}
/// Session claim marking a session that presented a second factor.
@@ -579,6 +580,29 @@ mod tests {
}
}
#[test]
fn assume_role_xml_matches_the_pinned_sts_response() {
let credentials = rustfs_credentials::Credentials {
access_key: "access<&".to_owned(),
secret_key: "synthetic>'".to_owned(),
session_token: "quoted\"token".to_owned(),
..Default::default()
};
let expiration = time::macros::datetime!(2024-02-29 11:22:33.123456789 +05:30);
let answer = assume_role_response_xml(&credentials, expiration).expect("valid timestamp");
// Captured from the pinned 5761ddfe STS writer, including field order and escaping.
assert_eq!(
answer,
concat!(
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
"<AccessKeyId>access&lt;&amp;</AccessKeyId><Expiration>2024-02-29T11:22:33.123Z</Expiration>",
"<SecretAccessKey>synthetic&gt;&apos;</SecretAccessKey><SessionToken>quoted&quot;token</SessionToken>",
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
)
);
assert!(assume_role_response_xml(&credentials, time::macros::datetime!(-1-01-01 00:00 UTC)).is_err());
}
#[test]
fn test_xml_escape() {
assert_eq!(xml_escape("hello"), "hello");
+1 -17
View File
@@ -45,24 +45,8 @@ use handlers::{
user,
};
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
/// Create admin router
///
/// # Arguments
/// * `console_enabled` - Whether the console is enabled
///
/// # Returns
/// An instance of S3Route for admin operations
pub fn make_admin_route(
console_enabled: bool,
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
) -> std::io::Result<impl S3Route> {
make_admin_router(console_enabled, server_ctx)
}
/// The admin router with every route registered, bound to `server_ctx`: what the legacy stack
/// mounts as its custom route and the gateway stack's control-plane host dispatches through.
/// Builds the control-plane router and binds it to this server.
pub(crate) fn make_admin_router(
console_enabled: bool,
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
+232 -331
View File
File diff suppressed because it is too large Load Diff
@@ -77,7 +77,7 @@ fn uri_for(template: &str, bucket: &str) -> String {
async fn send(router: &S3Router<AdminOperation>, mut req: S3Request<Body>) -> S3Result<S3Response<Body>> {
router.check_access(&mut req).await?;
router.call(req).await
router.call(req).await.map_err(Into::into)
}
fn error_code(result: S3Result<S3Response<Body>>, context: &str) -> S3ErrorCode {
@@ -222,7 +222,7 @@ async fn extension_routes_refuse_anonymous_callers_before_dispatch() {
.check_access(&mut req)
.await
.expect_err("an anonymous extension request is refused");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{method} {uri}");
assert_eq!(S3ErrorCode::from(error.code().clone()), S3ErrorCode::AccessDenied, "{method} {uri}");
}
}
+23 -1
View File
@@ -87,6 +87,8 @@ pub(crate) enum CredentialMutationDenial {
ServiceAccount,
/// A derived credential whose parent could not be determined.
UnresolvedParent,
/// A storage delegation has no account-management authority.
TableStorageSession,
}
impl CredentialMutationDenial {
@@ -97,6 +99,7 @@ impl CredentialMutationDenial {
}
Self::FederatedIdentity => "federated identities are managed by their identity provider",
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
}
}
@@ -187,7 +190,7 @@ impl CallerIdentity {
// root", and an unresolved parent must never fall through to a
// permissive answer.
//
// These three denials apply to both capabilities: a machine credential
// These denials apply to both capabilities: a machine credential
// must not take over its parent, a federated identity is owned by its
// IdP, and an unresolvable parent fails closed.
let shared_denial = if unresolved_parent {
@@ -196,6 +199,11 @@ impl CallerIdentity {
Some(CredentialMutationDenial::ServiceAccount)
} else if federated {
Some(CredentialMutationDenial::FederatedIdentity)
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
.iter()
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
{
Some(CredentialMutationDenial::TableStorageSession)
} else {
None
};
@@ -501,6 +509,20 @@ mod tests {
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
}
#[test]
fn table_storage_sessions_cannot_manage_parent_credentials() {
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
let mut credentials = sts_session("TABLEKEY", "table-owner");
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
let caller = CallerIdentity::from_credentials(credentials, false);
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
assert!(!caller.mutability().password);
assert!(caller.ensure_credential_mutation_allowed().is_err());
assert!(caller.ensure_mfa_management_allowed().is_err());
}
}
#[test]
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
// Fail closed: an unresolvable parent must not be treated as a
+1 -1
View File
@@ -37,7 +37,7 @@ use crate::server::collect_dependency_readiness_report as collect_runtime_depend
use crate::shared_types::DependencyReadiness;
use rustfs_data_usage::DataUsageInfo;
use rustfs_madmin::{InfoMessage, StorageInfo};
use s3s::S3ErrorCode;
use rustfs_s3_types::S3ErrorCode;
use std::collections::HashSet;
use std::sync::Arc;
use tracing::{debug, error, info, warn};
+76
View File
@@ -0,0 +1,76 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Operations without a notification event must emit one use-case audit entry.
use crate::guard_sources::SourceSet;
use rustfs_s3_ops::S3Operation;
fn is_read_only(op: S3Operation) -> bool {
let name = op.as_str().trim_start_matches("s3:");
["Get", "Head", "List", "Select"]
.iter()
.any(|prefix| name.starts_with(prefix))
}
fn audit_sites(use_cases: &SourceSet, op: S3Operation) -> usize {
let helper = format!("OperationHelper::audit_only_from_view(&envelope.audit_view(), S3Operation::{op:?}");
use_cases.count_ignoring_whitespace(&format!("audit_only_operation(envelope, S3Operation::{op:?},"))
+ use_cases.count_ignoring_whitespace(&format!("{helper})"))
+ use_cases.count_ignoring_whitespace(&format!("{helper},)"))
}
#[test]
fn mutating_operations_without_a_notification_event_are_audited_once() {
let mut expected: Vec<S3Operation> = S3Operation::ALL
.into_iter()
.filter(|op| !is_read_only(*op) && op.to_event_name().is_none())
.collect();
// AbortMultipartUpload has an internal event but no handler-built helper.
expected.push(S3Operation::AbortMultipartUpload);
assert_eq!(expected.len(), 26, "an operation was added or removed: {expected:?}");
let use_cases = SourceSet::production("src/app/mod.rs", "src/app");
for op in expected {
assert_eq!(audit_sites(&use_cases, op), 1, "expected one audit site for {op:?}");
}
}
const OWNED_HELPER: &str =
"let helper = OperationHelper::audit_only_from_view(\n &envelope.audit_view(),\n S3Operation::UploadPart,\n);";
const OWNED_WRAPPER: &str =
"RequestEnvelope::audit_only_operation(\n envelope,\n S3Operation::UploadPart,\n StatusCode::OK,\n run,\n)";
fn count_sites(source: &str) -> usize {
let root = tempfile::tempdir().unwrap();
let app = root.path().join("src/app");
std::fs::create_dir_all(&app).unwrap();
std::fs::write(app.join("mod.rs"), source).unwrap();
let sources = SourceSet::production_under(root.path(), "src/app/mod.rs", "src/app");
audit_sites(&sources, S3Operation::UploadPart)
}
#[test]
fn the_guard_accepts_exactly_one_audit_place() {
assert_eq!(count_sites(OWNED_HELPER), 1);
assert_eq!(count_sites(OWNED_WRAPPER), 1);
}
#[test]
fn the_guard_refuses_an_operation_audited_nowhere_or_twice() {
assert_eq!(count_sites(""), 0);
assert_eq!(count_sites(&format!("{OWNED_HELPER}\n{OWNED_WRAPPER}")), 2);
assert_eq!(count_sites(&format!("#[cfg(test)]\nmod tests {{\n{OWNED_HELPER}\n}}")), 0);
assert_eq!(count_sites(&OWNED_WRAPPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
assert_eq!(count_sites(&OWNED_HELPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
}
+31 -20
View File
@@ -472,7 +472,7 @@ mod tests {
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
use crate::app::storage_api::bucket_usecase::s3::{
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3, S3Request, S3Response,
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response,
XmlSerialize, XmlSerializer,
};
use crate::app::storage_api::test::StoragePutObjReader;
@@ -1185,10 +1185,11 @@ mod tests {
service: None,
trailing_headers: None,
};
let get_env = crate::app::RequestEnvelope::from_s3s(&get);
let authorized_generation =
super::super::storage_api::bucket_usecase::access::load_bucket_generation_from_store(
&store,
&get,
&get_env,
&input.bucket,
)
.await
@@ -1214,12 +1215,16 @@ mod tests {
assert_ne!(replacement, old.incarnation_id());
sys.remove(&input.bucket);
let mut without_source = get.clone();
let mut without_source_env = crate::app::RequestEnvelope::from_s3s(&without_source);
super::super::storage_api::bucket_usecase::access::prepare_odm_read_generation(
&store,
&mut without_source,
&mut without_source_env,
&input.bucket,
)
.await;
if let Some(guard) = without_source_env.odm_read_generation().cloned() {
without_source.extensions.insert(guard);
}
for state_incarnation in [old.incarnation_id(), replacement] {
sys.apply_for_incarnation(&input.bucket, state_incarnation, Some(old.config()))
.await;
@@ -1315,28 +1320,34 @@ mod tests {
};
for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
let request_marker = input.marker.clone().unwrap_or_default();
let response = tokio::time::timeout(
let list_req = S3Request {
input: input.clone(),
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let envelope = crate::app::RequestEnvelope::from_s3s(&list_req);
let gateway_input =
rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::input_from_s3s(
list_req.input,
)
.expect("convert ListObjectsInput");
let gateway_output = tokio::time::timeout(
Duration::from_secs(10),
S3::list_objects(
&crate::storage::ecfs::FS::new(),
S3Request {
input: input.clone(),
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
},
),
DefaultBucketUsecase::from_global().execute_list_objects(envelope, gateway_input),
)
.await
.expect("v1 pagination must finish")
.expect("list-through must not change v1 listing");
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
let output = response.output;
let output = rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::output_to_s3s(
gateway_output,
)
.expect("convert ListObjectsOutput");
let contents = output.contents.as_ref().expect("local page contents");
assert_eq!(contents.len(), 1);
assert_eq!(contents[0].key.as_deref(), Some(expected_key));
+6 -4
View File
@@ -98,14 +98,13 @@ use rustfs_policy::policy::{
{BucketPolicy, BucketPolicyArgs, Effect},
};
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::Region;
use rustfs_s3_types::{Region, S3Error, S3ErrorCode, S3Result};
use rustfs_targets::{
EventName,
arn::{ARN, TargetID, TargetIDError},
};
use rustfs_utils::http::get_header;
use rustfs_utils::obj::extract_user_defined_metadata;
use s3s::{S3Error, S3ErrorCode, S3Result};
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
@@ -907,7 +906,9 @@ mod tests {
.expect_err("full queue rejects immediately");
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
let response = error.to_http_response().expect("serialize admission error");
let response = s3s::S3Error::from(error)
.to_http_response()
.expect("serialize admission error");
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
use http_body_util::BodyExt as _;
@@ -1000,7 +1001,8 @@ mod tests {
},
);
assert!(req_info_ref(&req).expect("req info should be present").is_owner);
let envelope = RequestEnvelope::from_s3s(&req);
assert!(envelope.req_info().expect("req info should be present").is_owner);
}
#[tokio::test]
+11
View File
@@ -67,6 +67,17 @@ impl DefaultBucketUsecase {
mod tests {
use super::*;
#[tokio::test]
async fn execute_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultBucketUsecase::without_context();
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
let error = usecase
.execute_put_bucket_acl(envelope, "test-bucket", false)
.await
.unwrap_err();
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
let input = gateway_dto::GetBucketAclInput {
+72 -1
View File
@@ -116,7 +116,7 @@ impl DefaultBucketUsecase {
async fn list_objects_page(&self, envelope: &mut RequestEnvelope, request: &PageRequest) -> S3Result<Page> {
let PageRequest { bucket, params, .. } = request;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
let store = get_validated_store(bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED).as_deref() == Some("true");
let table_list_access = envelope.table_list_access().cloned();
@@ -375,6 +375,77 @@ mod tests {
use crate::app::bucket_usecase::test_support::build_request;
use rustfs_gateway_types::BucketName;
#[test]
#[serial_test::serial]
fn list_apis_report_storage_read_quorum_as_unavailable() {
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore};
run_large_stack_test("list-quorum-surface", || async {
let usecase = DefaultBucketUsecase::with_context(Some(shared_gating_ambient().await));
let store = shared_gating_ecstore().await;
let bucket = BucketName::new(format!("list-quorum-{}", uuid::Uuid::new_v4().simple())).unwrap();
store
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
.await
.unwrap();
let set = &store.pools[0].disk_set[0];
let saved_disks = {
let mut disks = set.disks.write().await;
let saved = disks.clone();
disks.iter_mut().skip(1).for_each(|disk| *disk = None);
saved
};
let envelope = || owned_envelope(http::Method::GET);
let results = [
usecase
.execute_list_objects(
envelope(),
gateway_dto::ListObjectsInput {
bucket: bucket.clone(),
..Default::default()
},
)
.await
.map(drop),
usecase
.execute_list_objects_v2(
envelope(),
gateway_dto::ListObjectsV2Input {
bucket: bucket.clone(),
..Default::default()
},
)
.await
.map(drop),
usecase
.execute_list_object_versions(
envelope(),
gateway_dto::ListObjectVersionsInput {
bucket,
..Default::default()
},
)
.await
.map(drop),
];
*set.disks.write().await = saved_disks;
for result in results {
let error = result.expect_err("cannot list below storage read quorum");
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
let source = error.source().expect("retain storage diagnostics");
assert!(
matches!(
source.downcast_ref::<StorageError>(),
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
),
"{source}"
);
}
});
}
fn object(name: &str) -> ObjectInfo {
ObjectInfo {
name: name.to_owned(),
+1 -1
View File
@@ -626,7 +626,7 @@ impl DefaultBucketUsecase {
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
let store = get_validated_store(&bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
@@ -23,8 +23,7 @@ use crate::app::storage_api::s3::dto::{
FilterRule, LambdaFunctionConfiguration, NotificationConfiguration, NotificationConfigurationFilter,
PutBucketNotificationConfigurationInput, QueueConfiguration, S3KeyFilter, TopicConfiguration,
};
use crate::app::storage_api::s3::{S3, S3Access, S3ErrorCode, S3Request};
use crate::app::storage_api::test::ecfs::FS;
use crate::app::storage_api::s3::{S3ErrorCode, S3Request};
use crate::runtime_sources::{NotifyInterface, ServerContextSlot};
use http::Method;
use rustfs_notify::{EventArgs, NotificationError};
@@ -215,6 +214,76 @@ fn notification_request(
request
}
fn notification_filters(filter: Option<NotificationConfigurationFilter>) -> Vec<(Option<String>, Option<String>)> {
filter
.and_then(|f| f.key)
.and_then(|k| k.filter_rules)
.unwrap_or_default()
.into_iter()
.map(|rule| (rule.name.map(|n| n.as_str().to_owned()), rule.value))
.collect()
}
fn notification_destinations(configuration: NotificationConfiguration) -> Vec<NotificationDestination> {
let mut destinations = Vec::new();
for cfg in configuration.queue_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "QueueConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.queue_arn,
filters: notification_filters(cfg.filter),
});
}
for cfg in configuration.topic_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "TopicConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.topic_arn,
filters: notification_filters(cfg.filter),
});
}
for cfg in configuration.lambda_function_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "LambdaFunctionConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.lambda_function_arn,
filters: notification_filters(cfg.filter),
});
}
destinations
}
async fn authorize_notification_put(
slot: &Arc<ServerContextSlot>,
envelope: &mut RequestEnvelope,
bucket: &str,
) -> rustfs_s3_types::S3Result<()> {
use rustfs_policy::policy::action::{Action, S3Action};
crate::app::storage_api::owned_parity::authorize_bucket_config_mutation(
slot,
envelope,
bucket,
Action::S3Action(S3Action::PutBucketNotificationAction),
)
.await
}
async fn put_notification(
slot: &Arc<ServerContextSlot>,
mut envelope: RequestEnvelope,
input: PutBucketNotificationConfigurationInput,
) -> rustfs_s3_types::S3Result<gateway_dto::PutBucketNotificationConfigurationOutput> {
authorize_notification_put(slot, &mut envelope, &input.bucket).await?;
let prepared = crate::storage::persisted_from_s3s(&input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
let destinations = notification_destinations(input.notification_configuration);
DefaultBucketUsecase::with_context(slot.app_context())
.execute_put_bucket_notification_configuration(envelope, &input.bucket, destinations, prepared)
.await
}
fn check_notification_write_order() {
run_large_stack_test("notification-write-order", || async {
let ambient = shared_gating_ambient().await;
@@ -250,13 +319,10 @@ fn check_notification_write_order() {
.map_or_else(|| "eu-west-2".to_owned(), |region| region.to_string());
let slot = ServerContextSlot::new();
assert!(slot.install(context.clone()));
let fs = FS::with_server_ctx(slot.clone());
let audit = MutationCompletionProbe::install(&bucket);
let mut request = notification_request(&bucket, configuration(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut request)
.await
.unwrap();
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
let request = notification_request(&bucket, configuration(), &slot);
let envelope = RequestEnvelope::from_s3s(&request);
let result = observe_reload(&bucket, calls.clone(), put_notification(&slot, envelope, request.input)).await;
let expected = expected_rules();
let mut timeline = vec![Observation::Rules(
Callback::Validate,
@@ -288,7 +354,7 @@ fn check_notification_write_order() {
};
if let Some(message) = error_message {
let error = result.expect_err("injected callback failure");
assert_eq!(error.code(), &S3ErrorCode::InternalError);
assert_eq!(error.code().as_str(), S3ErrorCode::InternalError.as_str());
assert_eq!(error.message(), Some(message));
} else {
result.expect("configuration written");
@@ -307,15 +373,13 @@ fn check_notification_write_order() {
assert_eq!(*recorder.active_rules.lock().unwrap(), expected);
let mut invalid = configuration();
invalid.lambda_function_configurations.as_mut().unwrap()[0].lambda_function_arn = "invalid-arn".into();
let mut invalid = notification_request(&bucket, invalid, &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut invalid)
.await
.unwrap();
let invalid = notification_request(&bucket, invalid, &slot);
let invalid_env = RequestEnvelope::from_s3s(&invalid);
calls.lock().unwrap().clear();
let err = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, invalid))
let err = observe_reload(&bucket, calls.clone(), put_notification(&slot, invalid_env, invalid.input))
.await
.expect_err("the last destination has an invalid ARN");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str());
assert!(
err.message()
.unwrap()
@@ -337,11 +401,9 @@ fn check_notification_write_order() {
assert_eq!(output.queue_configurations[0].events[1].as_str(), "s3:Future:Unrecognized");
calls.lock().unwrap().clear();
let mut empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut empty)
.await
.unwrap();
observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, empty))
let empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
let empty_env = RequestEnvelope::from_s3s(&empty);
observe_reload(&bucket, calls.clone(), put_notification(&slot, empty_env, empty.input))
.await
.unwrap();
assert_eq!(
@@ -417,16 +479,14 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let fs = FS::with_server_ctx(slot.clone());
let mut request = notification_request(&bucket, configuration(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut request)
.await
.unwrap();
assert_eq!(bucket_config_mutation_incarnation(&request, &bucket).unwrap(), Some(first));
let request = notification_request(&bucket, configuration(), &slot);
let mut envelope = RequestEnvelope::from_s3s(&request);
authorize_notification_put(&slot, &mut envelope, &bucket).await.unwrap();
assert_eq!(bucket_config_mutation_incarnation(&envelope, &bucket).unwrap(), Some(first));
let mismatch = DefaultBucketUsecase::without_context()
.execute_put_bucket_notification_configuration(
RequestEnvelope::from_s3s(&request),
envelope.fork_for_authorization(),
"other-bucket",
vec![NotificationDestination {
kind: "QueueConfiguration",
@@ -449,9 +509,22 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
.await
.unwrap();
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
let prepared =
crate::storage::persisted_from_s3s(&request.input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
let destinations = notification_destinations(request.input.notification_configuration);
let result = observe_reload(
&bucket,
calls.clone(),
DefaultBucketUsecase::with_context(slot.app_context()).execute_put_bucket_notification_configuration(
envelope,
&bucket,
destinations,
prepared,
),
)
.await;
assert_eq!(read_notification_xml(&bucket).await, OLD_XML);
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::NoSuchBucket);
assert_eq!(result.unwrap_err().code().as_str(), S3ErrorCode::NoSuchBucket.as_str());
assert!(
matches!(calls.lock().unwrap().as_slice(), [Observation::Rules(Callback::Validate, _, _, xml)] if xml == OLD_XML)
);
@@ -633,6 +633,24 @@ mod tests {
assert!(err.message().is_some_and(|message| message.contains("peer reload failed")));
}
#[tokio::test]
async fn execute_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultBucketUsecase::without_context();
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
let error = usecase
.execute_put_object_lock_configuration(
envelope,
"test-bucket",
Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".into()),
rule: None,
}),
)
.await
.unwrap_err();
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_get_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
let input = gateway_dto::GetObjectLockConfigurationInput {
+12 -27
View File
@@ -243,27 +243,19 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::{build_request, s3_op_total};
use crate::app::storage_api::s3::S3ErrorCode;
use crate::app::storage_api::s3::dto::{DeleteBucketPolicyInput, PutBucketPolicyInput};
use crate::app::bucket_usecase::test_support::build_request;
use http::Method;
#[tokio::test]
async fn execute_delete_bucket_policy_records_s3_operation_before_store_lookup() {
let input = DeleteBucketPolicyInput::builder()
.bucket("test-bucket".to_string())
.build()
.unwrap();
async fn execute_delete_bucket_policy_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::DELETE);
let usecase = DefaultBucketUsecase::without_context();
let req = build_request(input, Method::DELETE);
let fs = crate::storage::ecfs::FS::new();
let before = s3_op_total(S3Operation::DeleteBucketPolicy);
let err = crate::app::storage_api::s3::S3::delete_bucket_policy(&fs, req)
let err = usecase
.execute_delete_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket")
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::DeleteBucketPolicy), before + 1);
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
fn bucket(name: &str) -> rustfs_gateway_types::BucketName {
@@ -300,20 +292,13 @@ mod tests {
#[tokio::test]
async fn execute_put_bucket_policy_returns_internal_error_when_store_uninitialized() {
let input = PutBucketPolicyInput::builder()
.bucket("test-bucket".to_string())
.policy("{}".to_string())
.build()
.unwrap();
let req = build_request((), Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let req = build_request(input, Method::PUT);
let fs = crate::storage::ecfs::FS::new();
let before = s3_op_total(S3Operation::PutBucketPolicy);
let err = crate::app::storage_api::s3::S3::put_bucket_policy(&fs, req)
let err = usecase
.execute_put_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket", "{}")
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::PutBucketPolicy), before + 1);
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
}
+10 -4
View File
@@ -211,11 +211,17 @@ mod tests {
let mut req = build_request(input, Method::PUT);
req.extensions.insert(ReqInfo::default());
req.extensions.insert(slot.clone());
authorize_bucket_config_mutation(&slot, &mut req, &bucket, Action::S3Action(S3Action::PutBucketTaggingAction))
.await
.expect("permission allows tag updates");
let mut envelope = RequestEnvelope::from_s3s(&req);
authorize_bucket_config_mutation(
&slot,
&mut envelope,
&bucket,
Action::S3Action(S3Action::PutBucketTaggingAction),
)
.await
.expect("permission allows tag updates");
let result = usecase
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, persisted)
.execute_put_bucket_tagging(envelope, &req.input.bucket, persisted)
.await;
if valid {
result.expect("valid tags are persisted");
-29
View File
@@ -461,35 +461,6 @@ impl RequestEnvelope {
pub fn residual_extensions(&self) -> &Extensions {
&self.residual_extensions
}
/// The payloads the connection, request and signature layers attached, back in the extensions
/// they arrive in, for an edge that hands this request to a handler still taking those.
pub(crate) fn transport_extensions(&self) -> Extensions {
let mut extensions = self.residual_extensions.clone();
extensions.insert(self.remote_addr.map(RemoteAddr));
if let Some(client_info) = &self.client_info {
extensions.insert(client_info.clone());
}
if let Some(native_tls) = self.native_tls_transport {
extensions.insert(NativeTlsTransport(native_tls));
}
if let Some(request_context) = &self.request_context {
extensions.insert(request_context.clone());
}
if let Some(server_context) = &self.server_context {
extensions.insert(Arc::clone(server_context));
}
if let Some(body_read_control) = &self.body_read_control {
extensions.insert(body_read_control.clone());
}
if let Some(verified) = self.verified_sigv4 {
extensions.insert(verified);
}
if let Some(verified) = self.verified_presigned {
extensions.insert(verified);
}
extensions
}
}
#[cfg_attr(
@@ -28,7 +28,6 @@ use super::storage_api::test::contract::{
namespace::NamespaceLocking as _,
object::{ObjectIO as _, ObjectOperations as _},
};
use super::storage_api::test::ecfs::FS;
use super::storage_api::test::object_utils::to_s3s_etag;
use super::storage_api::test::runtime::{
MockWarmBackend, MockWarmOp, get_global_transition_state, register_mock_tier as register_mock_tier_util,
@@ -16,7 +16,6 @@
use super::common::*;
use super::*;
use crate::app::storage_api::s3::S3 as _;
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
@@ -162,8 +161,7 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
.unwrap(),
Method::PUT,
);
FS::new()
.put_bucket_lifecycle_configuration(req)
put_lifecycle(req)
.await
.expect("Failed to update lifecycle configuration for delete marker cleanup");
@@ -199,10 +197,7 @@ async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
Method::PUT,
);
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("Failed to update lifecycle configuration");
put_lifecycle(req).await.expect("Failed to update lifecycle configuration");
assert!(
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
@@ -231,8 +226,7 @@ async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_obje
Method::PUT,
);
FS::new()
.put_bucket_lifecycle_configuration(req)
put_lifecycle(req)
.await
.expect("ExpiredObjectDeleteMarker should be accepted on object-lock bucket");
}
@@ -258,11 +252,9 @@ async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_obj
Method::PUT,
);
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
let err = put_lifecycle(req)
.await
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket")
.into();
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -294,11 +286,9 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
Method::PUT,
);
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
let err = put_lifecycle(req)
.await
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket")
.into();
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -351,11 +341,9 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
Method::PUT,
);
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
let err = put_lifecycle(req)
.await
.expect_err("zero-day expiration must be rejected with InvalidArgument")
.into();
.expect_err("zero-day expiration must be rejected with InvalidArgument");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -471,7 +459,6 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
),
];
let fs = FS::new();
let handled_count = || {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
@@ -511,8 +498,8 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
.extensions
.insert(parity_test_support::bucket_config_mutation(&readable));
}
let error = fs.put_bucket_lifecycle_configuration(request).await.unwrap_err();
assert_eq!(error.code(), &code, "{name}: {error}");
let error = put_lifecycle(request).await.unwrap_err();
assert_eq!(error.code().as_str(), code.as_str(), "{name}: {error}");
if let Some(message) = message {
assert_eq!(error.message(), Some(message), "{name}");
}
@@ -533,9 +520,7 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
let before = handled_count();
let probe = MutationCompletionProbe::install(&readable);
fs.put_bucket_lifecycle_configuration(request(&readable, Some(("Disabled", false))))
.await
.unwrap();
put_lifecycle(request(&readable, Some(("Disabled", false)))).await.unwrap();
assert_audit(&probe);
assert_eq!(handled_count() - before, 1, "success counts once");
@@ -547,3 +532,26 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
assert_eq!(rules[0].expiration.as_ref().unwrap().date.as_deref(), Some("2099-01-01T00:00:00.000Z"));
assert_eq!(rules[1].expiration.as_ref().unwrap().days, Some(30));
}
async fn put_lifecycle(req: S3Request<PutBucketLifecycleConfigurationInput>) -> rustfs_s3_types::S3Result<()> {
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::PutBucketLifecycleConfiguration);
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
let prepared = match req.input.lifecycle_configuration {
None => crate::app::bucket_usecase::PreparedLifecycleConfiguration::Missing,
Some(cfg) => {
if crate::app::bucket_usecase::validate_lifecycle_rule_status(cfg.rules.iter().map(|rule| rule.status.as_str()))
.is_err()
{
crate::app::bucket_usecase::PreparedLifecycleConfiguration::InvalidStatus
} else {
crate::app::bucket_usecase::PreparedLifecycleConfiguration::Prepared(
crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s(cfg),
)
}
}
};
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_lifecycle_configuration(envelope, &req.input.bucket, prepared)
.await
.map(|_| ())
}
@@ -28,11 +28,10 @@ use super::*;
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn object_lock_handlers_schedule_replication() {
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
use crate::app::storage_api::s3::S3;
use std::sync::atomic::Ordering;
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let object_meta = crate::app::object_meta::ObjectMetaUsecase::with_context(None);
let bucket = format!("test-lock-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "locked/object.txt";
@@ -52,7 +51,7 @@ async fn object_lock_handlers_schedule_replication() {
// PutObjectRetention must compute a replication decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let retention_input = PutObjectRetentionInput::builder()
let _retention_input = PutObjectRetentionInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.retention(Some(ObjectLockRetention {
@@ -61,7 +60,18 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
fs.put_object_retention(build_request(retention_input, Method::PUT))
let req = build_request((), Method::PUT);
object_meta
.put_object_retention_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLockRetention {
mode: Some(ObjectLockRetentionMode::COMPLIANCE.to_owned()),
retain_until: Some(time::macros::datetime!(2030-01-01 00:00:00 UTC)),
}),
)
.await
.expect("put_object_retention should succeed");
assert_eq!(
@@ -73,7 +83,7 @@ async fn object_lock_handlers_schedule_replication() {
// A bucket without a replication configuration can skip the decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let no_replication_input = PutObjectLegalHoldInput::builder()
let _no_replication_input = PutObjectLegalHoldInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.legal_hold(Some(ObjectLockLegalHold {
@@ -81,7 +91,17 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
fs.put_object_legal_hold(build_request(no_replication_input, Method::PUT))
let req = build_request((), Method::PUT);
object_meta
.put_object_legal_hold_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
ObjectLockLegalHoldStatus::OFF.to_owned(),
))),
)
.await
.expect("legal hold without replication should succeed");
assert_eq!(
@@ -93,7 +113,7 @@ async fn object_lock_handlers_schedule_replication() {
// A configured bucket must still evaluate the metadata replication decision.
install_delete_replication_config(&bucket, true, false, None).await;
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let legal_hold_input = PutObjectLegalHoldInput::builder()
let _legal_hold_input = PutObjectLegalHoldInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.legal_hold(Some(ObjectLockLegalHold {
@@ -101,7 +121,17 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
fs.put_object_legal_hold(build_request(legal_hold_input, Method::PUT))
let req = build_request((), Method::PUT);
object_meta
.put_object_legal_hold_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
ObjectLockLegalHoldStatus::ON.to_owned(),
))),
)
.await
.expect("put_object_legal_hold should succeed");
assert_eq!(
+1 -1
View File
@@ -29,7 +29,7 @@ use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::storage_api::s3::dto::{EncodingType, ListObjectVersionsInput, ListObjectsV2Input};
use crate::app::storage_api::s3::xml;
use crate::storage::access::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::authorization::AuthorizationRequest;
use http::{HeaderMap, Uri};
use rustfs_policy::policy::action::S3Action;
+2
View File
@@ -22,6 +22,8 @@
pub mod admin_usecase;
mod answered;
#[cfg(test)]
mod audit_coverage_test;
pub(crate) mod bucket_list_through;
pub mod bucket_usecase;
pub mod context;
+16 -9
View File
@@ -50,8 +50,7 @@ use super::storage_api::multipart_usecase::io::{
EtagAlgorithm, HashReader, WriteEncryption, WritePlan, compression_metadata_value,
};
#[cfg(test)]
use super::storage_api::multipart_usecase::object_utils::to_s3s_etag;
use super::storage_api::multipart_usecase::object_utils::to_s3s_xml_etag;
use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag};
use super::storage_api::multipart_usecase::options::{
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata,
@@ -104,6 +103,7 @@ use futures::StreamExt;
use http::{HeaderMap, Uri};
use metrics::counter;
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
#[cfg(test)]
@@ -115,7 +115,6 @@ use rustfs_utils::http::{
headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
insert_str,
};
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
@@ -193,12 +192,13 @@ fn multipart_max_total_object_size(metadata: &HashMap<String, String>) -> S3Resu
let value = get_consistent_str(metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
"multipart size capability metadata is missing or inconsistent".to_string(),
"multipart size capability metadata is missing or inconsistent",
)
})?;
value.parse::<u64>().map(Some).map_err(|_| {
S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid".to_string())
})
value
.parse::<u64>()
.map(Some)
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid"))
}
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
@@ -291,7 +291,11 @@ impl DefaultMultipartUsecase {
/// or completed".
fn multipart_upload_api_error(err: StorageError) -> S3Error {
if matches!(err, StorageError::MalformedUploadID(_)) {
return S3Error::new(S3ErrorCode::NoSuchUpload);
let mut error = S3Error::new(S3ErrorCode::NoSuchUpload);
if let Some(message) = error.code().default_message() {
error.set_message(message);
}
return error;
}
ApiError::from(err).into()
}
@@ -545,7 +549,7 @@ mod tests {
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
.await
.expect_err("reserved key must be rejected before accessing storage");
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
let complete = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_owned())
@@ -593,6 +597,9 @@ mod tests {
fn multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload() {
let malformed = multipart_upload_api_error(StorageError::MalformedUploadID("bucket/object-not-an-id".to_string()));
assert_eq!(malformed.code(), &S3ErrorCode::NoSuchUpload);
let legacy = crate::app::storage_api::s3::S3Error::new(crate::app::storage_api::s3::S3ErrorCode::NoSuchUpload);
assert_eq!(malformed.message(), legacy.message());
assert_eq!(malformed.status_code(), legacy.status_code());
let unknown = multipart_upload_api_error(StorageError::InvalidUploadID(
"bucket".to_string(),
@@ -61,7 +61,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
req.uri = format!("/{bucket}/{key}").parse().expect("fixture URI");
req.extensions.insert(slot.clone());
req.extensions.insert(request_context.clone());
req
RequestEnvelope::from_s3s(&req)
};
for name in [&bucket, &other] {
store
@@ -143,7 +143,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
(wrong, S3ErrorCode::InternalError),
] {
let err = usecase
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&req), Ok(target()))
.execute_abort_multipart_upload(req, Ok(target()))
.await
.expect_err("Abort requires current generation");
assert_eq!(err.code(), &code);
@@ -162,7 +162,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
.expect("capture current bucket");
current.install_bucket_generation(captured);
usecase
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&current), Ok(target()))
.execute_abort_multipart_upload(current.fork_for_authorization(), Ok(target()))
.await
.expect("current generation allows Abort");
assert!(matches!(
@@ -182,7 +182,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
.expect("create legacy-path upload");
usecase
.execute_abort_multipart_upload(
RequestEnvelope::from_s3s(&current),
current,
Ok(AbortMultipartUploadTarget::Legacy {
bucket: bucket.clone(),
key: key.to_owned(),
+129 -116
View File
@@ -294,10 +294,8 @@ impl DefaultMultipartUsecase {
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
// The input member and the header are one value on the wire; both are checked, as the
// legacy guard checks both.
if if_none_match.as_deref().is_some_and(|condition| condition.trim() != "*") {
return Err(S3Error::new(S3ErrorCode::NotImplemented));
}
reject_non_wildcard_write_if_none_match(None, headers)?;
let specific_if_none_match = if_none_match.as_deref().is_some_and(|condition| condition.trim() != "*");
reject_non_wildcard_write_if_none_match(specific_if_none_match, headers)?;
validate_table_catalog_object_mutation(&bucket, &key).await?;
if if_match.is_some() || if_none_match.is_some() {
@@ -736,6 +734,7 @@ impl DefaultMultipartUsecase {
}
}
#[cfg(test)]
#[cfg(test)]
impl DefaultMultipartUsecase {
/// CompleteMultipartUpload for an s3s request: its facts in an envelope and its input across
@@ -744,9 +743,41 @@ impl DefaultMultipartUsecase {
&self,
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CompleteMultipartUploadInput>,
) -> S3Result<CompleteMultipartUploadOutput> {
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
let envelope = RequestEnvelope::from_s3s(&req);
let (input, parts) =
crate::storage::ecfs::complete_multipart_upload_input(req.input).map_err(crate::app::dto_values::seam_error)?;
let mut s3s_input = req.input;
let has_upload = s3s_input.multipart_upload.is_some();
s3s_input.multipart_upload.get_or_insert_with(Default::default);
let tags: Vec<Option<String>> = s3s_input
.multipart_upload
.iter_mut()
.flat_map(|upload| upload.parts.iter_mut().flatten())
.map(|part| {
part.part_number.get_or_insert(0);
part.e_tag.take().map(|tag| tag.value().to_owned())
})
.collect();
for checksum in [
&mut s3s_input.checksum_crc32,
&mut s3s_input.checksum_crc32c,
&mut s3s_input.checksum_crc64nvme,
&mut s3s_input.checksum_md5,
&mut s3s_input.checksum_sha1,
&mut s3s_input.checksum_sha256,
&mut s3s_input.checksum_sha512,
&mut s3s_input.checksum_xxhash128,
&mut s3s_input.checksum_xxhash3,
&mut s3s_input.checksum_xxhash64,
] {
*checksum = None;
}
let mut input = complete_multipart_upload::input_from_s3s(s3s_input).map_err(crate::app::dto_values::seam_error)?;
for (part, tag) in input.multipart_upload.parts.iter_mut().zip(tags) {
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
}
let parts = std::mem::take(&mut input.multipart_upload);
let parts = has_upload.then_some(parts);
match parts {
Some(parts) => {
let mut input = input;
@@ -1093,23 +1124,18 @@ mod tests {
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
let (_store, bucket) =
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
let fs = crate::storage::ecfs::FS::new();
crate::app::storage_api::s3::S3::put_bucket_versioning(
&fs,
build_request(
crate::app::storage_api::s3::dto::PutBucketVersioningInput::builder()
.bucket(bucket.clone())
.versioning_configuration(crate::app::storage_api::s3::dto::VersioningConfiguration {
status: Some(crate::app::storage_api::s3::dto::BucketVersioningStatus::from_static("Enabled")),
..Default::default()
})
.build()
.expect("bucket versioning input"),
Method::PUT,
),
)
.await
.expect("enable versioning for encrypted multipart listing");
let versioning_req = build_request((), Method::PUT);
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_versioning(
RequestEnvelope::from_s3s(&versioning_req),
&bucket,
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
status: Some("Enabled".to_owned()),
..Default::default()
}),
)
.await
.expect("enable versioning for encrypted multipart listing");
let usecase = DefaultMultipartUsecase::from_global();
let key = "customer-encrypted";
let customer_key = "KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio=";
@@ -1227,9 +1253,9 @@ mod tests {
let etag = etag_to_s3s(&response.e_tag.expect("completed encrypted object ETag"));
let version_id = response.version_id.expect("completed object version");
assert_ne!(version_id, "null");
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
let head = crate::app::storage_api::s3::S3::head_object(
&fs,
assert_multipart_etag_listings(&bucket, key, &etag, &version_id).await;
let head = crate::app::test_seams::head_object(
&crate::app::object::DefaultObjectUsecase::from_global(),
build_request(
crate::app::storage_api::s3::dto::HeadObjectInput {
bucket: bucket.clone(),
@@ -1259,15 +1285,10 @@ mod tests {
});
}
async fn assert_multipart_etag_listings(
fs: &crate::storage::ecfs::FS,
bucket: &str,
key: &str,
etag: &ETag,
version_id: &str,
) {
async fn assert_multipart_etag_listings(bucket: &str, key: &str, etag: &ETag, version_id: &str) {
use crate::app::storage_api::s3::dto::{ListObjectVersionsInput, ListObjectsInput, ListObjectsV2Input};
use crate::app::storage_api::s3::xml::{Serialize, Serializer};
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::{list_object_versions, list_objects};
fn assert_xml_etag<T: Serialize>(output: &T, etag: &ETag) {
let mut bytes = Vec::new();
@@ -1281,21 +1302,25 @@ mod tests {
);
}
let bucket_usecase = crate::app::bucket_usecase::DefaultBucketUsecase::from_global();
// Listings carry neither the SSE-C key nor a per-object decryption request.
let versions = crate::app::storage_api::s3::S3::list_object_versions(
fs,
build_request(
ListObjectVersionsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
),
let versions_req = build_request(
ListObjectVersionsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
);
let versions_env = RequestEnvelope::from_s3s(&versions_req);
let versions_in = list_object_versions::input_from_s3s(versions_req.input).expect("convert ListObjectVersionsInput");
let versions = list_object_versions::output_to_s3s(
bucket_usecase
.execute_list_object_versions(versions_env, versions_in)
.await
.expect("keyless version listing"),
)
.await
.expect("keyless version listing")
.output;
.expect("convert ListObjectVersionsOutput");
let entries = versions.versions.as_ref().expect("completed version must be listed");
assert_eq!(entries.len(), 1, "completion replay must preserve a single version");
assert_eq!(entries[0].key.as_deref(), Some(key));
@@ -1303,22 +1328,25 @@ mod tests {
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
assert_xml_etag(&versions, etag);
let v1 = crate::app::storage_api::s3::S3::list_objects(
fs,
build_request(
ListObjectsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
),
let v1_req = build_request(
ListObjectsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
);
let v1_env = RequestEnvelope::from_s3s(&v1_req);
let v1_in = list_objects::input_from_s3s(v1_req.input).expect("convert ListObjectsInput");
let v1 = list_objects::output_to_s3s(
bucket_usecase
.execute_list_objects(v1_env, v1_in)
.await
.expect("keyless V1 listing"),
)
.await
.expect("keyless V1 listing")
.output;
let v2 = crate::app::storage_api::s3::S3::list_objects_v2(
fs,
.expect("convert ListObjectsOutput");
let v2 = crate::app::test_seams::list_objects_v2(
&bucket_usecase,
build_request(
ListObjectsV2Input {
bucket: bucket.to_string(),
@@ -1382,52 +1410,41 @@ mod tests {
)
.await
.expect("create versioned Object Lock bucket");
let fs = crate::storage::ecfs::FS::new();
let key = "kms-encrypted";
crate::app::storage_api::s3::S3::put_bucket_encryption(
&fs,
build_request(
crate::app::storage_api::s3::dto::PutBucketEncryptionInput::builder()
.bucket(bucket.clone())
.server_side_encryption_configuration(
crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
apply_server_side_encryption_by_default: Some(
crate::app::storage_api::s3::dto::ServerSideEncryptionByDefault {
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
kms_master_key_id: Some(kms_key.key_id.clone()),
},
),
..Default::default()
}],
},
)
.build()
.expect("bucket encryption input"),
Method::PUT,
),
)
.await
.expect("configure bucket-default SSE-KMS");
let upload_id = crate::app::storage_api::s3::S3::create_multipart_upload(
&fs,
build_request(
let enc_req = build_request((), Method::PUT);
let enc_cfg = crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
apply_server_side_encryption_by_default: Some(
crate::app::storage_api::s3::dto::ServerSideEncryptionByDefault {
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
kms_master_key_id: Some(kms_key.key_id.clone()),
},
),
..Default::default()
}],
};
let prepared = crate::storage::persisted_from_s3s(&enc_cfg).map_err(Into::into);
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&enc_req), &bucket, prepared)
.await
.expect("configure bucket-default SSE-KMS");
let usecase = DefaultMultipartUsecase::from_global();
let upload_id = usecase
.execute_create_multipart_upload_from_s3s(build_request(
CreateMultipartUploadInput {
bucket: bucket.clone(),
key: key.to_string(),
..Default::default()
},
Method::POST,
),
)
.await
.expect("create KMS multipart upload")
.output
.upload_id
.expect("KMS upload id");
let part = crate::app::storage_api::s3::S3::upload_part(
&fs,
build_request(
))
.await
.expect("create KMS multipart upload")
.output
.upload_id
.expect("KMS upload id");
let part = usecase
.execute_upload_part(build_request(
UploadPartInput {
bucket: bucket.clone(),
key: key.to_string(),
@@ -1438,11 +1455,10 @@ mod tests {
..Default::default()
},
Method::PUT,
),
)
.await
.expect("upload encrypted KMS part")
.output;
))
.await
.expect("upload encrypted KMS part")
.output;
let complete = CompleteMultipartUploadInput {
bucket: bucket.clone(),
key: key.to_string(),
@@ -1457,19 +1473,16 @@ mod tests {
..Default::default()
};
for _ in 0..2 {
let response = crate::app::storage_api::s3::S3::complete_multipart_upload(
&fs,
build_request(complete.clone(), Method::POST),
)
.await
.expect("complete and replay KMS multipart upload")
.output;
let etag = response.e_tag.expect("KMS completed ETag");
let response = usecase
.complete_for_s3s_request(build_request(complete.clone(), Method::POST))
.await
.expect("complete and replay KMS multipart upload");
let etag = etag_to_s3s(&response.e_tag.expect("KMS completed ETag"));
let version_id = response.version_id.expect("KMS completed version");
assert_ne!(version_id, "null");
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
let head = crate::app::storage_api::s3::S3::head_object(
&fs,
assert_multipart_etag_listings(&bucket, key, &etag, &version_id).await;
let head = crate::app::test_seams::head_object(
&crate::app::object::DefaultObjectUsecase::from_global(),
build_request(
crate::app::storage_api::s3::dto::HeadObjectInput {
bucket: bucket.clone(),
+82 -20
View File
@@ -455,6 +455,58 @@ impl DefaultMultipartUsecase {
}
}
#[cfg(test)]
impl CreateMultipartUploadFields {
pub(crate) fn from_legacy(input: crate::app::storage_api::s3::dto::CreateMultipartUploadInput) -> Self {
Self {
bucket: input.bucket,
key: input.key,
metadata: input.metadata,
tagging: input.tagging,
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
ssekms_key_id: input.ssekms_key_id,
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
}
}
}
#[cfg(test)]
impl DefaultMultipartUsecase {
pub(crate) async fn execute_create_multipart_upload_from_s3s(
&self,
mut req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CreateMultipartUploadInput>,
) -> crate::app::storage_api::s3::S3Result<
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::CreateMultipartUploadOutput>,
> {
use crate::app::storage_api::s3::{S3Response, dto::CreateMultipartUploadOutput};
let envelope = RequestEnvelope::from_s3s(&req);
let query_version = req.input.version_id.take();
let output = self
.execute_create_multipart_upload(envelope, CreateMultipartUploadFields::from_legacy(req.input), query_version)
.await?;
let mut response = S3Response::new(CreateMultipartUploadOutput {
bucket: Some(output.bucket),
key: Some(output.key),
upload_id: Some(output.upload_id),
server_side_encryption: output.server_side_encryption.map(Into::into),
sse_customer_algorithm: output.sse_customer_algorithm,
sse_customer_key_md5: output.sse_customer_key_md5,
ssekms_key_id: output.ssekms_key_id,
checksum_algorithm: output.checksum_algorithm.map(Into::into),
checksum_type: output.checksum_type.map(Into::into),
..Default::default()
});
response.headers.extend(output.response_headers);
Ok(response)
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
@@ -731,11 +783,19 @@ mod tests {
.expect("create bucket");
}
let mut stale = request(&bucket);
let guard = load_bucket_generation_from_store(&store, &stale, &bucket)
let make_request = |target_bucket: &str| {
let mut req = request(target_bucket);
let envelope = RequestEnvelope::from_s3s(&req);
let query_version = req.input.version_id.take();
let fields = CreateMultipartUploadFields::from_legacy(req.input);
(envelope, fields, query_version)
};
let (mut stale_env, stale_fields, stale_ver) = make_request(&bucket);
let guard = load_bucket_generation_from_store(&store, &stale_env, &bucket)
.await
.expect("capture original incarnation");
stale.install_bucket_generation(guard);
stale_env.install_bucket_generation(guard);
let original = store.bucket_incarnation_id(&bucket).await.expect("old bucket generation");
store
@@ -749,18 +809,18 @@ mod tests {
let current = store.bucket_incarnation_id(&bucket).await.expect("new bucket generation");
assert_ne!(original, current);
let missing = request(&bucket);
let mut wrong = request(&bucket);
let guard = load_bucket_generation_from_store(&store, &wrong, &other)
let (missing_env, missing_fields, missing_ver) = make_request(&bucket);
let (mut wrong_env, wrong_fields, wrong_ver) = make_request(&bucket);
let guard = load_bucket_generation_from_store(&store, &wrong_env, &other)
.await
.expect("capture the other bucket");
wrong.install_bucket_generation(guard);
for (request, code) in [
(stale, S3ErrorCode::NoSuchBucket),
(missing, S3ErrorCode::InternalError),
(wrong, S3ErrorCode::InternalError),
wrong_env.install_bucket_generation(guard);
for (envelope, fields, query_version, code) in [
(stale_env, stale_fields, stale_ver, S3ErrorCode::NoSuchBucket),
(missing_env, missing_fields, missing_ver, S3ErrorCode::InternalError),
(wrong_env, wrong_fields, wrong_ver, S3ErrorCode::InternalError),
] {
let result = usecase.execute_create_multipart_upload_from_s3s(request).await;
let result = usecase.execute_create_multipart_upload(envelope, fields, query_version).await;
assert!(
store
.list_multipart_uploads(&bucket, "", None, None, None, 100)
@@ -769,18 +829,20 @@ mod tests {
.uploads
.is_empty()
);
let error = result.expect_err("creation requires the target incarnation");
assert_eq!(error.code(), &code);
let Err(error) = result else {
panic!("creation requires the target incarnation");
};
assert_eq!(error.code().as_str(), code.as_str());
}
let probe = crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe::install(&bucket);
let mut allowed = request(&bucket);
let guard = load_bucket_generation_from_store(&store, &allowed, &bucket)
let (mut allowed_env, allowed_fields, allowed_ver) = make_request(&bucket);
let guard = load_bucket_generation_from_store(&store, &allowed_env, &bucket)
.await
.expect("capture current incarnation");
allowed.install_bucket_generation(guard);
let response = usecase
.execute_create_multipart_upload_from_s3s(allowed)
allowed_env.install_bucket_generation(guard);
let output = usecase
.execute_create_multipart_upload(allowed_env, allowed_fields, allowed_ver)
.await
.expect("a current guard allows creation");
let uploads = store
@@ -789,7 +851,7 @@ mod tests {
.expect("list durable uploads")
.uploads;
assert_eq!(uploads.len(), 1);
assert_eq!(Some(uploads[0].upload_id.as_str()), response.output.upload_id.as_deref());
assert_eq!(uploads[0].upload_id.as_str(), output.upload_id.as_str());
assert_eq!(probe.audits.lock().unwrap().len(), 1);
assert!(probe.events.lock().unwrap().is_empty());
});
+4 -7
View File
@@ -26,11 +26,8 @@ use rustfs_s3_types::S3Result;
use rustfs_s3_types::{S3Error, S3ErrorCode};
pub(crate) enum ListPartsTarget {
Legacy {
bucket: String,
key: String,
upload_id: String,
},
#[allow(dead_code)]
Legacy { bucket: String, key: String, upload_id: String },
Gateway {
bucket: BucketName,
key: ObjectKey,
@@ -82,7 +79,7 @@ where
Ok::<_, StorageError>(page.recorded_owner.take().map(|owner| RecordedListParts { owner, page }))
})
.await
.map_err(|error| S3Error::from(multipart_upload_api_error(error)))?
.map_err(multipart_upload_api_error)?
.map_err(|refusal| S3Error::with_message(S3ErrorCode::NoSuchUpload, refusal.reason()))?;
Ok(record.page)
}
@@ -201,7 +198,7 @@ impl DefaultMultipartUsecase {
ListPartsTarget::Legacy { upload_id, .. } => store
.list_object_parts(bucket, key, upload_id, params.part_number_marker, params.max_parts, &opts)
.await
.map_err(|error| multipart_upload_api_error(error).into()),
.map_err(multipart_upload_api_error),
ListPartsTarget::Gateway {
bucket: bucket_name,
key: object_key,
@@ -165,7 +165,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
storage_error,
StorageError::InvalidUploadID(..) | StorageError::MalformedUploadID(..)
));
let expected = multipart_upload_api_error(storage_error);
let expected: LegacyS3Error = ApiError::from(storage_error).into();
for target in [
gateway_fields(&bucket, key, id, None, None),
ListPartsFields {
@@ -217,10 +217,11 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
let req = || {
let mut req = build_request((), Method::GET);
req.extensions.insert(slot.clone());
req
RequestEnvelope::from_s3s(&req)
};
let capability = |query: &str| {
let mut request = req();
let mut request = build_request((), Method::GET);
request.extensions.insert(slot.clone());
request.uri = format!("/fixture?{query}").parse().expect("fixture URI");
RequestEnvelope::from_s3s(&request)
};
@@ -269,7 +270,7 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
temp_env::async_with_vars([(rustfs_config::ENV_VEEAM_SOSAPI_ENABLE, Some("true"))], async {
let err = execute_counted(
&usecase,
RequestEnvelope::from_s3s(&veeam_request),
veeam_request,
gateway_fields(
&bucket,
".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml",
@@ -294,13 +295,8 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
if let Some(guard) = guard {
request.install_bucket_generation(guard);
}
let result = execute_counted(
&usecase,
RequestEnvelope::from_s3s(&request),
gateway_fields(&bucket, key, &upload.upload_id, marker, None),
lookups,
)
.await;
let result =
execute_counted(&usecase, request, gateway_fields(&bucket, key, &upload.upload_id, marker, None), lookups).await;
assert_eq!(result.as_ref().err().map(|err| err.code().as_str()), code);
if let Ok(page) = result {
assert!(page.upload_id == upload.upload_id);
@@ -43,7 +43,7 @@ impl RecordedUpload for PartUpload {
async fn lookup_part_upload(store: &ECStore, bucket: &str, key: &str, id: &str, opts: &ObjectOptions) -> S3Result<MultipartInfo> {
let result = store.get_multipart_info(bucket, key, id, opts).await;
result.map_err(|error| multipart_upload_api_error(error).into())
result.map_err(multipart_upload_api_error)
}
pub(super) async fn lookup_recorded_part_upload(
@@ -56,7 +56,7 @@ pub(super) async fn lookup_recorded_part_upload(
let result = store.lookup_multipart_part_binding(bucket, key, id, opts).await;
#[cfg(test)]
let result = tests::observe_lookup(result);
match result.map_err(|error| S3Error::from(multipart_upload_api_error(error))) {
match result.map_err(multipart_upload_api_error) {
Ok((info, owner)) => Ok(owner.map(|owner| PartUpload { owner, info })),
Err(error) if error.code() == &S3ErrorCode::NoSuchUpload => Ok(None),
Err(error) => Err(error),
@@ -148,6 +148,18 @@ fn normalize_presigned_part_checksums(req: &mut RequestEnvelope, input: &mut Upl
}
impl DefaultMultipartUsecase {
/// Records the gateway audit. The protocol client records its own projected response.
pub(crate) async fn execute_upload_part_with_audit(
&self,
envelope: RequestEnvelope,
input: UploadPartWriteInput,
) -> S3Result<UploadPartWriteOutput> {
RequestEnvelope::audit_only_operation(envelope, S3Operation::UploadPart, http::StatusCode::OK, |envelope| {
self.execute_upload_part_owned(envelope, input)
})
.await
}
#[instrument(name = "execute_upload_part", level = "debug", skip(self, req, input))]
#[hotpath::measure(impl_type = "MultipartUsecase")]
pub(crate) async fn execute_upload_part_owned(
@@ -501,6 +513,72 @@ impl DefaultMultipartUsecase {
}
}
#[cfg(test)]
impl UploadPartWriteInput {
pub(crate) fn from_legacy(
input: crate::app::storage_api::s3::dto::UploadPartInput,
trailers: Option<crate::app::storage_api::s3::TrailingHeaders>,
) -> Self {
Self {
target: UploadPartTarget::Legacy {
bucket: input.bucket,
key: input.key,
upload_id: input.upload_id,
},
part_number: input.part_number,
content_length: input.content_length,
body: input.body.map(crate::app::object::legacy_put_body),
trailing_headers: crate::app::trailer_adapter::trailer_source(trailers),
frame_trailers: None,
content_md5: input.content_md5,
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
checksum_crc32: input.checksum_crc32,
checksum_crc32c: input.checksum_crc32c,
checksum_crc64nvme: input.checksum_crc64nvme,
checksum_sha1: input.checksum_sha1,
checksum_sha256: input.checksum_sha256,
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
}
}
}
#[cfg(test)]
impl DefaultMultipartUsecase {
pub async fn execute_upload_part(
&self,
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::UploadPartInput>,
) -> crate::app::storage_api::s3::S3Result<
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::UploadPartOutput>,
> {
use crate::app::storage_api::s3::{
S3Response,
dto::{ServerSideEncryption, UploadPartOutput},
};
let envelope = RequestEnvelope::from_s3s(&req);
let input = UploadPartWriteInput::from_legacy(req.input, req.trailing_headers);
let output = self.execute_upload_part_with_audit(envelope, input).await?;
let mut response = S3Response::new(UploadPartOutput {
e_tag: output
.e_tag
.map(|tag| crate::app::storage_api::test::object_utils::to_s3s_etag(&tag)),
checksum_crc32: output.checksum_crc32,
checksum_crc32c: output.checksum_crc32c,
checksum_crc64nvme: output.checksum_crc64nvme,
checksum_sha1: output.checksum_sha1,
checksum_sha256: output.checksum_sha256,
server_side_encryption: output.server_side_encryption.map(ServerSideEncryption::from),
ssekms_key_id: output.ssekms_key_id,
sse_customer_algorithm: output.sse_customer_algorithm,
sse_customer_key_md5: output.sse_customer_key_md5,
..Default::default()
});
response.headers = output.headers;
Ok(response)
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
@@ -104,7 +104,7 @@ async fn native_claim_checks_order_ownership_and_one_active_lookup() {
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
LOOKUP
.scope((Cell::new(0), mode), async {
let result = usecase.execute_upload_part_owned(envelope, input).await;
let result = usecase.execute_upload_part_with_audit(envelope, input).await;
assert_eq!(
result.as_ref().err().map(|err| err.code().as_str()),
expected,
@@ -214,7 +214,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
LOOKUP
.scope((Cell::new(0), mode), async {
let result = Box::pin(usecase.execute_upload_part_copy_owned(envelope, input)).await;
let result = Box::pin(usecase.execute_upload_part_copy_with_audit(envelope, input)).await;
assert_eq!(result.as_ref().err().map(|error| error.code().as_str()), expected, "{mode:?}");
assert_eq!(LOOKUP.with(|(calls, _)| calls.get()), 1, "{mode:?}");
if expected == Some("NoSuchUpload") {
@@ -233,7 +233,8 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
} else if matches!(mode, LookupResult::Outage) {
let Err(error) = result else { panic!("expected read quorum failure") };
let actual: crate::app::storage_api::s3::S3Error = error.into();
let expected = multipart_upload_api_error(StorageError::ErasureReadQuorum);
let expected: crate::app::storage_api::s3::S3Error =
ApiError::from(StorageError::ErasureReadQuorum).into();
assert_eq!(actual.code(), expected.code());
assert_eq!(actual.status_code(), expected.status_code());
assert_eq!(actual.message(), expected.message());
@@ -13,14 +13,14 @@
// limitations under the License.
//! UploadPartCopy through one owned storage implementation and its legacy adapter.
//! Protocol clients and the legacy FS edge supply request facts; multipart ownership stays in storage.
//! Gateway and protocol clients supply request facts. Storage owns the multipart upload.
use super::*;
#[cfg(test)]
use crate::app::storage_api::s3::dto::CopySource;
use crate::app::storage_api::s3::dto::{
CopyPartResult, ETag, ServerSideEncryption, Timestamp, UploadPartCopyInput, UploadPartCopyOutput,
CopyPartResult, CopySource, ETag, ServerSideEncryption, Timestamp, UploadPartCopyInput, UploadPartCopyOutput,
};
#[cfg(test)]
use crate::app::storage_api::s3::{S3Request, S3Response};
use crate::app::{RequestEnvelope, object_usecase::CopySourceCondition};
use rustfs_gateway_types::CopySourceValue;
@@ -100,6 +100,7 @@ fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_siz
}
impl DefaultMultipartUsecase {
#[cfg(test)]
pub async fn execute_upload_part_copy(
&self,
req: S3Request<UploadPartCopyInput>,
@@ -129,7 +130,7 @@ impl DefaultMultipartUsecase {
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
};
let output = self.execute_upload_part_copy_owned(envelope, input).await?;
let output = self.execute_upload_part_copy_with_audit(envelope, input).await?;
Ok(S3Response::new(UploadPartCopyOutput {
copy_part_result: Some(CopyPartResult {
e_tag: output.e_tag.as_deref().map(to_s3s_xml_etag),
@@ -151,6 +152,18 @@ impl DefaultMultipartUsecase {
}
/// Copies plaintext into an existing upload without changing its recorded owner.
/// Records the gateway audit. The protocol client records its own projected response.
pub(crate) async fn execute_upload_part_copy_with_audit(
&self,
envelope: RequestEnvelope,
input: UploadPartCopyWriteInput,
) -> S3Result<UploadPartCopyWriteOutput> {
RequestEnvelope::audit_only_operation(envelope, S3Operation::UploadPartCopy, http::StatusCode::OK, |envelope| {
Box::pin(self.execute_upload_part_copy_owned(envelope, input))
})
.await
}
#[instrument(level = "debug", skip(self, envelope, input))]
#[hotpath::measure(impl_type = "MultipartUsecase")]
pub(crate) async fn execute_upload_part_copy_owned(
@@ -437,7 +450,7 @@ impl DefaultMultipartUsecase {
.user_defined
.contains_key("x-amz-server-side-encryption-customer-algorithm");
let ssekms_key_id = match server_side_encryption.as_ref() {
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info
Some(sse) if sse.as_str() == rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS => mp_info
.user_defined
.get("x-amz-server-side-encryption-aws-kms-key-id")
.map(|s| s.to_string()),
+3 -3
View File
@@ -162,7 +162,7 @@ impl DefaultObjectUsecase {
envelope.uri().query(),
envelope.verified_presigned().is_some(),
)?;
reject_non_wildcard_write_if_none_match_native(input.specific_if_none_match, envelope.headers())?;
reject_non_wildcard_write_if_none_match(input.specific_if_none_match, envelope.headers())?;
if let Some(context) = &self.context {
let _ = context.object_store();
}
@@ -1051,8 +1051,8 @@ mod tests {
.expect_err("short copy source must fail before destination write succeeds");
let api_error = ApiError::from(err);
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
assert_ne!(api_error.code, S3ErrorCode::IncompleteBody);
assert_eq!(api_error.code, rustfs_s3_types::S3ErrorCode::ServiceUnavailable);
assert_ne!(api_error.code, rustfs_s3_types::S3ErrorCode::IncompleteBody);
}
#[tokio::test]
+51 -1
View File
@@ -99,7 +99,57 @@ impl CopyObjectWriteInput {
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CopyObjectInput>,
) -> (crate::app::RequestEnvelope, Self) {
use crate::app::object_usecase::CopySourceCondition;
use crate::app::storage_api::s3::dto::{CopySource, ETag, ETagCondition};
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
(envelope, crate::storage::ecfs::copy_input_from_legacy(req.input))
let input = req.input;
let condition = |value: ETagCondition| match value {
ETagCondition::Any => CopySourceCondition::Any,
ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
};
let copy_source = match &input.copy_source {
CopySource::AccessPoint { .. } | CopySource::Outpost { .. } => None,
CopySource::Bucket { bucket, key, version_id } => Some(rustfs_gateway_types::CopySourceValue::Bucket {
bucket: bucket.to_string(),
key: key.to_string(),
version_id: version_id.as_ref().map(|v| v.to_string()),
}),
};
let write_input = CopyObjectWriteInput {
copy_source,
bucket: input.bucket,
key: input.key,
version_id: input.version_id,
specific_if_none_match: matches!(input.if_none_match, Some(ETagCondition::ETag(_))),
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
ssekms_key_id: input.ssekms_key_id,
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
metadata_directive: input.metadata_directive.map(|value| value.as_str().to_owned()),
metadata: input.metadata,
tagging: input.tagging,
tagging_directive: input.tagging_directive.map(|value| value.as_str().to_owned()),
copy_source_if_match: input.copy_source_if_match.map(condition),
copy_source_if_none_match: input.copy_source_if_none_match.map(condition),
cache_control: input.cache_control,
content_disposition: input.content_disposition,
content_encoding: input.content_encoding,
content_language: input.content_language,
content_type: input.content_type,
expires: input.expires,
website_redirect_location: input.website_redirect_location,
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
};
(envelope, write_input)
}
}
+68 -5
View File
@@ -503,7 +503,7 @@ fn project_delete_objects_pre_stat_error(
error: ApiError,
) -> S3Result<DeleteBatchError> {
// Bucket loss invalidates the shared request, not just one object.
if error.code == S3ErrorCode::NoSuchBucket {
if error.code == rustfs_s3_types::S3ErrorCode::NoSuchBucket {
return Err(error.into());
}
Ok(DeleteBatchError {
@@ -676,10 +676,9 @@ impl DefaultObjectUsecase {
let key = normalize_delete_object_key(&obj_id.key);
// A rejected wire key has no authorization resource. Its normalized text cannot become a target.
let key_error = if obj_id.key_too_long {
Some((
S3ErrorCode::InvalidArgument,
ApiError::error_code_to_message(&S3ErrorCode::InvalidArgument),
))
let code = rustfs_s3_types::S3ErrorCode::InvalidArgument;
let message = ApiError::error_code_to_message(&code);
Some((code, message))
} else {
validate_object_key(&key, "DeleteObjects")
.err()
@@ -4460,3 +4459,67 @@ mod bulk_denial_audit_wiring_tests {
);
}
}
#[cfg(test)]
impl From<crate::app::storage_api::s3::dto::DeleteObjectsInput> for DeleteBatchInput {
fn from(input: crate::app::storage_api::s3::dto::DeleteObjectsInput) -> Self {
Self {
bucket: input.bucket,
objects: input.delete.objects.into_iter().map(Into::into).collect(),
quiet: input.delete.quiet,
}
}
}
#[cfg(test)]
impl From<crate::app::storage_api::s3::dto::ObjectIdentifier> for DeleteBatchEntry {
fn from(object: crate::app::storage_api::s3::dto::ObjectIdentifier) -> Self {
use crate::app::storage_api::s3::dto::ETag;
Self {
key: object.key,
key_too_long: false,
version_id: object.version_id,
e_tag: object.e_tag.map(|tag| match tag {
ETag::Strong(value) => DeleteBatchETag { value, weak: false },
ETag::Weak(value) => DeleteBatchETag { value, weak: true },
}),
_last_modified_time: object.last_modified_time.map(|value| {
let instant = time::OffsetDateTime::from(value);
(instant.unix_timestamp(), instant.nanosecond())
}),
_size: object.size,
}
}
}
#[cfg(test)]
impl From<DeleteBatchOutput> for crate::app::storage_api::s3::dto::DeleteObjectsOutput {
fn from(output: DeleteBatchOutput) -> Self {
use crate::app::storage_api::s3::dto::{DeletedObject, Error};
Self {
deleted: output.deleted.map(|entries| {
entries
.into_iter()
.map(|entry| DeletedObject {
delete_marker: entry.delete_marker,
delete_marker_version_id: entry.delete_marker_version_id,
key: entry.key,
version_id: entry.version_id,
})
.collect()
}),
errors: output.errors.map(|entries| {
entries
.into_iter()
.map(|entry| Error {
code: entry.code,
key: entry.key,
message: entry.message,
version_id: entry.version_id,
})
.collect()
}),
request_charged: None,
}
}
}
+52 -6
View File
@@ -139,6 +139,30 @@ fn extract_archive_incomplete_body(remaining: u64) -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, rustfs_rio::IncompleteBody { remaining })
}
/// Settle the transport checksum and trailers before giving the final frame to the TAR parser.
/// A parser that rejects that frame may never poll the body again.
fn archive_transport_chunks(
chunks: super::request_body::UploadChunks,
expected_length: u64,
) -> impl Stream<Item = std::io::Result<Bytes>> {
futures::stream::try_unfold((chunks.fuse(), expected_length), |(mut chunks, remaining)| async move {
let Some(chunk) = chunks.next().await else {
return Ok(None);
};
let chunk = chunk.map_err(s3s_body_error_to_io)?;
let overlong = || std::io::Error::new(std::io::ErrorKind::InvalidData, "archive body exceeded expected Content-Length");
let remaining = remaining.checked_sub(chunk.len() as u64).ok_or_else(overlong)?;
if remaining == 0 {
while let Some(trailing) = chunks.next().await {
if !trailing.map_err(s3s_body_error_to_io)?.is_empty() {
return Err(overlong());
}
}
}
Ok(Some((chunk, (chunks, remaining))))
})
}
impl<R: AsyncRead> AsyncRead for ExtractArchiveEtagReader<R> {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
let mut this = self.project();
@@ -1226,7 +1250,7 @@ pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: boo
fn map_extract_archive_error(err: std::io::Error) -> S3Error {
let api_error = ApiError::from(err);
if api_error.code != S3ErrorCode::InternalError {
if api_error.code != rustfs_s3_types::S3ErrorCode::InternalError {
return api_error.into();
}
@@ -1299,7 +1323,7 @@ fn is_empty_extract_entry_path(path: &str) -> bool {
fn validate_extract_member_key(path: &str, limits: ArchiveLimits) -> Result<(), ExtractEntryError> {
validate_put_object_extract_entry_path(path, limits).map_err(ExtractEntryError::Recoverable)?;
validate_new_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable)
validate_new_object_key(path, "PUT").map_err(|err| ExtractEntryError::Recoverable(err.into()))
}
fn record_extract_pax_metadata_bytes(
@@ -2204,8 +2228,9 @@ impl DefaultObjectUsecase {
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
// Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile.
let buffer_size = get_buffer_size_opt_in(size);
let body =
tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))));
let expected_archive_length = u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent.into()))?;
let chunks = Box::pin(archive_transport_chunks(body, expected_archive_length));
let body = tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(chunks));
let md5hex = if let Some(base64_md5) = content_md5 {
let md5 = base64_simd::STANDARD
@@ -2226,7 +2251,6 @@ impl DefaultObjectUsecase {
return Err(ApiError::from(err).into());
}
let expected_archive_length = u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent.into()))?;
let archive_upload_state = Arc::new(Mutex::new(ExtractArchiveUploadState::default()));
let extract_limits = put_object_extract_limits();
let tracked_archive =
@@ -3642,7 +3666,7 @@ mod tests {
] {
let error = map_extract_archive_error(error);
assert_eq!(error.code(), &code);
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code).as_str()));
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code.into()).as_str()));
assert!(error.source().is_some());
}
let error = map_extract_archive_error(io::Error::new(io::ErrorKind::InvalidData, "invalid TAR header"));
@@ -4293,6 +4317,28 @@ mod tests {
.expect("fixture member should parse")
}
#[tokio::test]
async fn archive_transport_checks_eof_before_releasing_the_final_frame() {
let data = || Ok(Bytes::from_static(b"ab"));
let late_error: StdError = Box::new(std::io::Error::other("trailer verification failed"));
for (tail, expected) in [
(Ok(Bytes::new()), None),
(data(), Some("archive body exceeded expected Content-Length")),
(Err(late_error), Some("trailer verification failed")),
] {
let body = Box::new(futures::stream::iter([data(), data(), tail]));
let mut chunks = Box::pin(archive_transport_chunks(body, 4));
assert_eq!(chunks.next().await.unwrap().unwrap(), "ab");
let last = chunks.next().await.unwrap();
if let Some(message) = expected {
assert_eq!(last.unwrap_err().to_string(), message);
} else {
assert_eq!(last.unwrap(), "ab");
}
assert!(chunks.next().await.is_none());
}
}
#[tokio::test]
async fn archive_etag_reader_validates_raw_eof_before_returning_final_byte() {
let payload = b"decoder-consumed-exact-body".to_vec();
+4 -3
View File
@@ -2602,8 +2602,9 @@ impl DefaultObjectUsecase {
};
return Err(with_delete_marker_read_headers(S3Error::new(code), info, input.version_id.as_deref()));
}
let result =
check_preconditions(envelope.headers(), info).and_then(|()| Self::validate_get_object_part_number(part_number, info));
let result = check_preconditions(envelope.headers(), info)
.map_err(S3Error::from)
.and_then(|()| Self::validate_get_object_part_number(part_number, info));
if result.is_err() {
let principal = SseKmsPrincipal::from_request(envelope);
crate::storage::sse::validate_conditional_read(
@@ -12313,7 +12314,7 @@ mod tests {
fn parse_get_object_part_number_rejects_above_s3_max() {
let err = parse_part_number_i32_to_usize(Some(10001), "GET").expect_err("partNumber above S3 max must fail");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("GET: partNumber must be between 1 and 10000"));
}
+36
View File
@@ -36,6 +36,42 @@ pub(crate) struct GetObjectFields {
pub(crate) sse_customer_key_md5: Option<String>,
}
impl GetObjectFields {
/// The conditional-read wire format permits a wildcard, a quoted tag, or a bare S3 tag.
pub(crate) fn valid_etag_condition(value: &[u8]) -> bool {
match value {
b"*" => true,
[b'"', tag @ .., b'"'] | [b'W', b'/', b'"', tag @ .., b'"'] => tag.iter().all(|b| matches!(*b, b'\t' | b' '..=b'~')),
bare => !bare.is_empty() && bare.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'-'),
}
}
pub(crate) fn valid_date_condition(value: &str) -> bool {
time::PrimitiveDateTime::parse(value, super::RFC1123).is_ok()
}
/// Object Lambda and website reads accept the historical single-range wire grammar.
pub(crate) fn parse_range(value: &str) -> Option<ByteRange> {
use rustfs_gateway_types::RangeParse;
let spec = value.strip_prefix("bytes=")?;
if !spec.bytes().all(|b| b.is_ascii_digit() || b == b'-') {
return None;
}
if spec == "-" {
// An omitted suffix length historically selects a zero-length suffix.
return Some(ByteRange::Suffix { length: 0 });
}
let RangeParse::One(range) = RangeParse::parse(value) else {
return None;
};
match range {
ByteRange::From { first } if first > i64::MAX as u64 => None,
ByteRange::FromTo { last, .. } if last > i64::MAX as u64 => None,
_ => Some(range),
}
}
}
/// Body and ETag retain their native representation until the final edge.
#[derive(Debug, Default)]
pub(crate) struct GetObjectOutputFacts {
@@ -20,6 +20,73 @@ use crate::app::storage_api::s3::dto::Range;
use crate::app::storage_api::test::ReqInfo;
use http::Method;
#[test]
fn website_and_object_lambda_ranges_match_the_legacy_wire_reader() {
for value in [
"bytes=0-9",
"bytes=0-",
"bytes=-7",
"bytes=-0",
"bytes=-",
"bytes=-18446744073709551615",
"bytes=9223372036854775807-",
"bytes=9223372036854775808-",
"bytes=0-9223372036854775808",
"bytes=-18446744073709551616",
"bytes=10-5",
"bytes=1-2 ",
"bytes=1 -2",
"bytes=1-2,3-4",
"bytes=-+1",
"Bytes=1-2",
"bytes=",
] {
let expected = Range::parse(value).ok().map(|range| match range {
Range::Int { first, last: Some(last) } => rustfs_gateway_types::ByteRange::FromTo { first, last },
Range::Int { first, last: None } => rustfs_gateway_types::ByteRange::From { first },
Range::Suffix { length } => rustfs_gateway_types::ByteRange::Suffix { length },
});
assert_eq!(GetObjectFields::parse_range(value), expected, "{value}");
}
}
#[test]
fn website_and_object_lambda_conditions_match_the_legacy_wire_reader() {
use s3s::dto::{ETagCondition, Timestamp, TimestampFormat};
for value in [
b"*".as_slice(),
b"abc-2",
b"\"\"",
b"W/\"x\"",
b"\"a \tb\"",
b"\"a\"b\"",
b"",
b"bad etag",
b"w/\"x\"",
b"\"\xff\"",
b"\"\x7f\"",
b"\"x",
] {
assert_eq!(
GetObjectFields::valid_etag_condition(value),
ETagCondition::parse_http_header(value).is_ok(),
"{value:?}"
);
}
for value in [
"Sun, 06 Nov 1994 08:49:37 GMT",
"Sunday, 06-Nov-94 08:49:37 GMT",
"2025-01-01",
"Sun, 32 Nov 1994 08:49:37 GMT",
] {
assert_eq!(
GetObjectFields::valid_date_condition(value),
Timestamp::parse(TimestampFormat::HttpDate, value).is_ok(),
"{value}"
);
}
}
fn get_request(input: GetObjectInput) -> S3Request<GetObjectInput> {
let info = ReqInfo {
bucket: Some(input.bucket.clone()),
+5
View File
@@ -45,6 +45,11 @@ pub(crate) struct ResponseEtag {
}
impl ResponseEtag {
pub(crate) fn to_http_header(&self) -> Result<http::HeaderValue, http::header::InvalidHeaderValue> {
let prefix = if self.weak { "W/" } else { "" };
http::HeaderValue::from_str(&format!("{prefix}\"{}\"", self.opaque))
}
pub(super) fn from_stored(value: &str) -> Self {
let weak = value.starts_with("W/\"");
let etag = stored_etag(value);
+7 -17
View File
@@ -22,7 +22,7 @@
//! multipart handlers' policy steps against the same storage contract.
use super::*;
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Response, S3Result};
use rustfs_s3_types::{S3Error, S3ErrorCode};
use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success;
use crate::app::runtime_sources::current_app_context;
@@ -80,7 +80,8 @@ pub(crate) struct InternalPutContext {
const INTERNAL_PUT_METHOD_NAME: &str = "PUT";
fn api_error_from_s3(err: S3Error) -> ApiError {
fn api_error_from_s3(err: impl Into<S3Error>) -> ApiError {
let err = err.into();
ApiError {
code: err.code().clone(),
message: err.message().unwrap_or_default().to_string(),
@@ -139,7 +140,7 @@ async fn validate_internal_write_target(key: &str, bucket: &str, headers: &Heade
headers.get("content-type").and_then(|value| value.to_str().ok()),
headers.get("content-encoding").and_then(|value| value.to_str().ok()),
)
.map_err(|err| api_error_from_s3(err.into()))
.map_err(api_error_from_s3)
}
fn internal_events_wanted() -> bool {
@@ -213,17 +214,7 @@ impl InternalPutObjectEvent {
}
}
/// Publish the event when `result` is a success; failures publish nothing.
pub(super) fn complete<T>(self, result: &S3Result<S3Response<T>>) {
if let Ok(response) = result {
self.complete_owned(&Ok(()), &response.headers);
}
}
pub(super) fn complete_owned<T>(self, result: &rustfs_s3_types::S3Result<T>, headers: &HeaderMap) {
if result.is_err() {
return;
}
pub(super) fn publish(self, headers: &HeaderMap) {
let Self {
builder,
notify,
@@ -343,7 +334,7 @@ impl DefaultObjectUsecase {
.put_object_core(write, super::native_body::from_io_stream(body), None, start_time)
.await
.map_err(|error| ApiError {
code: error.code().clone().into(),
code: error.code().clone(),
message: error.message().unwrap_or_default().to_owned(),
source: None,
})?;
@@ -767,8 +758,7 @@ impl DefaultObjectUsecase {
if versioned && let Some(version_id) = obj_info.version_id {
event = event.version_id(version_id.to_string());
}
let result: S3Result<S3Response<()>> = Ok(S3Response::new(()));
event.complete(&result);
event.publish(&HeaderMap::new());
}
Ok(obj_info)
}
+3 -2
View File
@@ -79,8 +79,6 @@ use super::storage_api::object_usecase::data_usage::{
record_bucket_object_write_unknown_previous_memory,
};
use super::storage_api::object_usecase::deadlock_detector;
#[cfg(test)]
use super::storage_api::object_usecase::ecfs::FS;
use super::storage_api::object_usecase::error::{
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
@@ -239,11 +237,14 @@ pub(crate) mod post_body;
pub(crate) mod post_headers;
mod put;
mod put_input;
#[cfg(test)]
pub(crate) use native_body::from_legacy as legacy_put_body;
pub(crate) use native_body::{BodyChunks, FrameTrailers};
pub(crate) use put_input::{PutObjectWriteInput, PutObjectWriteOutput};
pub(crate) mod request_body;
mod restore;
#[cfg(test)]
mod s3s_compat;
pub(crate) mod shared;
#[cfg(test)]
mod test_support;
+9 -2
View File
@@ -15,9 +15,13 @@
//! Upload frame/chunk boundary. Protocol edges supply a native body; storage consumes chunks.
//! This adapter preserves trailers and source errors without applying another length check.
use crate::app::storage_api::request_body::{ByteStream, StdError, StreamingBlob};
use crate::app::storage_api::request_body::StdError;
#[cfg(test)]
use crate::app::storage_api::request_body::{ByteStream, StreamingBlob};
use bytes::Bytes;
use futures::{Stream, StreamExt};
use futures::Stream;
#[cfg(test)]
use futures::StreamExt;
use http::HeaderMap;
use http_body::Body as _;
use parking_lot::Mutex;
@@ -30,6 +34,7 @@ use std::sync::Arc;
use std::task::{Context, Poll};
/// The legacy edge preserves the producer's length hint without enforcing it a second time.
#[cfg(test)]
pub(crate) fn from_legacy(body: StreamingBlob) -> Body {
let stream = LegacyPayload { body, ended: false };
Body::from_payload(Payload::Stream(Box::pin(stream)))
@@ -49,11 +54,13 @@ impl std::error::Error for ProducerSource {
}
}
#[cfg(test)]
struct LegacyPayload {
body: StreamingBlob,
ended: bool,
}
#[cfg(test)]
impl PayloadStream for LegacyPayload {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<PayloadRead, StreamError>> {
if self.ended {
@@ -32,7 +32,7 @@
//! `odm-source-etag`.
use super::*;
use crate::app::storage_api::s3::S3ErrorCode;
use rustfs_s3_types::S3ErrorCode;
use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
use crate::on_demand_migration::{
+7 -5
View File
@@ -150,7 +150,7 @@ pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
if u64::try_from(size).is_ok_and(|size| size > rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE) {
return Err(S3Error::with_message(
S3ErrorCode::EntityTooLarge,
ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge),
));
}
Ok(())
@@ -1174,8 +1174,10 @@ impl PutObjectCompletion {
match self {
Self::S3(helper) => Self::S3(helper.complete_owned(result, status, headers)),
Self::Internal(event) => {
if let Some(event) = event {
event.complete_owned(result, headers);
if result.is_ok()
&& let Some(event) = event
{
event.publish(headers);
}
Self::Internal(None)
}
@@ -1322,7 +1324,7 @@ impl DefaultObjectUsecase {
{
return Err(rustfs_s3_types::s3_error!(InvalidStorageClass));
}
reject_non_wildcard_write_if_none_match_native(input.specific_if_none_match, envelope.headers())?;
reject_non_wildcard_write_if_none_match(input.specific_if_none_match, envelope.headers())?;
// An authorized inbound replication PUT must store the replica verbatim.
// Legacy snowball-extracted members may still carry the auto-extract
// metadata, which replication replays as a header. Do not interpret that
@@ -5667,7 +5669,7 @@ mod oversize_single_upload_tests {
assert_eq!(*err.code(), S3ErrorCode::EntityTooLarge);
assert_eq!(
err.message(),
Some(ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge).as_str())
Some(ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge).as_str())
);
}
}
+45
View File
@@ -91,6 +91,51 @@ impl PutObjectWriteInput {
#[cfg(test)]
impl PutObjectWriteInput {
pub(crate) fn from_legacy(
input: crate::app::storage_api::s3::PutObjectInput,
trailing_headers: Option<crate::app::storage_api::s3::TrailingHeaders>,
) -> Self {
use crate::app::storage_api::s3::dto::ETagCondition;
Self {
bucket: input.bucket,
key: input.key,
body: input.body.map(crate::app::object::legacy_put_body),
post_archive: None,
frame_trailers: None,
trailing_headers: crate::app::trailer_adapter::trailer_source(trailing_headers),
content_length: input.content_length,
metadata: input.metadata,
metadata_headers: None,
checksum_headers: None,
specific_if_none_match: matches!(input.if_none_match, Some(ETagCondition::ETag(_))),
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
cache_control: input.cache_control,
content_disposition: input.content_disposition,
content_encoding: input.content_encoding,
content_language: input.content_language,
content_type: input.content_type,
expires: input.expires,
tagging: input.tagging,
version_id: input.version_id,
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
ssekms_key_id: input.ssekms_key_id,
content_md5: input.content_md5,
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
website_redirect_location: input.website_redirect_location,
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
checksum_crc32: input.checksum_crc32,
checksum_crc32c: input.checksum_crc32c,
checksum_sha1: input.checksum_sha1,
checksum_sha256: input.checksum_sha256,
checksum_crc64nvme: input.checksum_crc64nvme,
}
}
/// A test's s3s fixture request as the envelope and input a gateway handler hands the use case.
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::PutObjectInput>,
+2 -1
View File
@@ -13,10 +13,11 @@
// limitations under the License.
use super::*;
use crate::app::storage_api::s3::{Body as S3Body, S3Error, S3ErrorCode, StreamingBlob};
use crate::app::storage_api::s3::{Body as S3Body, S3Error, StreamingBlob};
use crate::error::ApiError;
use futures::{StreamExt, poll};
use http_body_util::StreamBody;
use rustfs_s3_types::S3ErrorCode;
use std::io;
use tokio::io::AsyncReadExt;
use tokio::sync::mpsc;
+36
View File
@@ -539,6 +539,42 @@ fn restore_output() -> RestoreObjectOutput {
}
}
#[cfg(test)]
impl DefaultObjectUsecase {
/// Projects a legacy RestoreObject request onto the shared restore path.
pub async fn execute_restore_object(
&self,
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::RestoreObjectInput>,
) -> crate::app::storage_api::s3::S3Result<
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::RestoreObjectOutput>,
> {
use crate::app::storage_api::object_usecase::bucket::restore_request_from_wire;
use crate::app::storage_api::s3::{
S3Response,
dto::{RequestCharged, RestoreObjectOutput},
};
let envelope = RequestEnvelope::from_s3s(&req);
let input = req.input;
self.execute_restore_object_owned(
envelope,
input.bucket,
input.key,
input.version_id,
input.restore_request.map(restore_request_from_wire),
)
.await
.map(|output| {
S3Response::new(RestoreObjectOutput {
request_charged: output
.request_charged
.map(|value| RequestCharged::from(value.as_str().to_owned())),
restore_output_path: output.restore_output_path,
})
})
.map_err(Into::into)
}
}
#[cfg(test)]
mod tests {
use super::*;
+181
View File
@@ -0,0 +1,181 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Compatibility conversions between owned Get/Head object facts and s3s DTOs.
//!
//! Retained for `admin/router.rs` (object-lambda), `app/website.rs`, and `app/test_seams` until
//! T4.2 removes the remaining `s3s` dependency from `admin/router.rs`.
use crate::app::object_usecase::{GetObjectBody, GetObjectFields, GetObjectOutputFacts};
#[cfg(test)]
use crate::app::object_usecase::{HeadObjectFields, HeadObjectOutputFacts};
use rustfs_gateway_stream::{BoxPayloadStream, PayloadRead, StreamError, StreamErrorKind};
use rustfs_gateway_types::ByteRange;
use rustfs_gateway_types::compat::ConversionError;
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::get_object;
#[cfg(test)]
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::head_object;
use s3s::dto::{ETag, GetObjectInput, GetObjectOutput, Range, StreamingBlob};
#[cfg(test)]
use s3s::dto::{HeadObjectInput, HeadObjectOutput};
use std::pin::Pin;
use std::sync::{Mutex, PoisonError};
use std::task::{Context, Poll};
#[cfg(test)]
impl HeadObjectFields {
pub(crate) fn from_s3s(input: HeadObjectInput) -> Self {
Self {
bucket: input.bucket,
key: input.key,
version_id: input.version_id,
part_number: input.part_number,
range: input.range.map(|range| match range {
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
Range::Int { first, last: None } => ByteRange::From { first },
Range::Suffix { length } => ByteRange::Suffix { length },
}),
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
}
}
}
#[cfg(test)]
impl HeadObjectOutputFacts {
pub(crate) fn into_s3s(self) -> Result<HeadObjectOutput, ConversionError> {
let mut output = head_object::output_to_s3s(self.dto)?;
output.e_tag = self.etag.map(|etag| {
if etag.weak {
ETag::Weak(etag.opaque)
} else {
ETag::Strong(etag.opaque)
}
});
Ok(output)
}
}
impl GetObjectFields {
pub(crate) fn from_s3s(input: GetObjectInput) -> Self {
Self {
bucket: input.bucket,
key: input.key,
version_id: input.version_id,
part_number: input.part_number,
range: input.range.map(|range| match range {
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
Range::Int { first, last: None } => ByteRange::From { first },
Range::Suffix { length } => ByteRange::Suffix { length },
}),
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
}
}
}
impl GetObjectOutputFacts {
pub(crate) fn into_s3s(self) -> Result<GetObjectOutput, ConversionError> {
let mut output = get_object::output_to_s3s(self.dto)?;
output.e_tag = self.etag.map(|etag| {
if etag.weak {
ETag::Weak(etag.opaque)
} else {
ETag::Strong(etag.opaque)
}
});
output.body = self.body.map(GetObjectBody::into_s3s);
Ok(output)
}
}
impl GetObjectBody {
pub(crate) fn into_s3s(self) -> StreamingBlob {
match self {
Self::Bytes(bytes) => StreamingBlob::from_bytes(bytes),
Self::Stream(payload) => StreamingBlob::new(LegacyGetStream {
payload: Mutex::new(payload),
done: false,
}),
}
}
}
// The legacy body requires Sync. Polling uses exclusive access, not a mutex lock.
struct LegacyGetStream {
payload: Mutex<BoxPayloadStream>,
done: bool,
}
impl futures::Stream for LegacyGetStream {
type Item = Result<bytes::Bytes, Box<dyn std::error::Error + Send + Sync>>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
if this.done {
return Poll::Ready(None);
}
let event = std::task::ready!(
this.payload
.get_mut()
.unwrap_or_else(PoisonError::into_inner)
.as_mut()
.poll_read(cx)
);
Poll::Ready(match event {
Ok(PayloadRead::Chunk(bytes)) => Some(Ok(bytes)),
Ok(PayloadRead::Eof { trailers }) => {
this.done = true;
if trailers.is_empty() {
None
} else {
Some(Err(Box::new(ConversionError {
field: "body",
reason: "legacy GetObject cannot encode response trailers",
})))
}
}
Err(error) => {
this.done = true;
Some(Err(legacy_get_body_error(error)))
}
})
}
}
fn legacy_get_body_error(error: StreamError) -> Box<dyn std::error::Error + Send + Sync> {
if matches!(error.kind(), StreamErrorKind::Upstream(_)) {
let StreamErrorKind::Upstream(source) = error.into_kind() else {
unreachable!("the owned error kind was checked above")
};
source
} else {
Box::new(error)
}
}
impl s3s::stream::ByteStream for LegacyGetStream {
fn remaining_length(&self) -> s3s::stream::RemainingLength {
match self
.payload
.lock()
.unwrap_or_else(PoisonError::into_inner)
.len_hint()
.and_then(|length| usize::try_from(length).ok())
{
Some(length) => s3s::stream::RemainingLength::new_exact(length),
None => s3s::stream::RemainingLength::unknown(),
}
}
}
+7 -31
View File
@@ -1030,12 +1030,6 @@ pub(in crate::app) fn object_lock_checks_required_for_state(state: &metadata_sys
pub(crate) const ON_DEMAND_MIGRATION_HEADER: http::HeaderName = http::HeaderName::from_static("x-rustfs-on-demand-migration");
pub(crate) const ON_DEMAND_MIGRATION_SOURCE: HeaderValue = HeaderValue::from_static("source");
/// Response header marking a `ListObjectsV2` that could not consult the source
/// and was answered from local state alone (rustfs/backlog#2164).
pub(crate) const ON_DEMAND_MIGRATION_LIST_HEADER: http::HeaderName =
http::HeaderName::from_static("x-rustfs-on-demand-migration-list");
pub(crate) const ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY: HeaderValue = HeaderValue::from_static("local_only");
/// Custom S3 error code for a source failure surfaced under
/// `policy.source_error = propagate`; carried on HTTP 424.
pub(crate) const ODM_SOURCE_UNAVAILABLE_CODE: &str = "SourceUnavailable";
@@ -1109,12 +1103,6 @@ pub(crate) fn mark_on_demand_migration_response(headers: &mut HeaderMap) {
headers.insert(ON_DEMAND_MIGRATION_HEADER, ON_DEMAND_MIGRATION_SOURCE);
}
/// Marks a merged listing that fell back to local state because the source
/// could not be consulted (rustfs/backlog#2164).
pub(crate) fn mark_on_demand_migration_list_local_only(headers: &mut HeaderMap) {
headers.insert(ON_DEMAND_MIGRATION_LIST_HEADER, ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY);
}
/// Evaluates the request's conditional headers (`If-Match`, `If-None-Match`,
/// `If-Modified-Since`, `If-Unmodified-Since`) against the source's view of
/// the object, with the same S3 semantics the local path applies to
@@ -1148,21 +1136,14 @@ pub(crate) fn odm_check_source_preconditions(headers: &HeaderMap, head: &SourceH
user_defined: metadata.into(),
..Default::default()
};
check_preconditions(headers, &info)
Ok(check_preconditions(headers, &info)?)
}
/// S3 documents write-side `If-None-Match` only as `*` (create-only).
/// Reject unsupported ETag comparisons with 501 NotImplemented.
/// Treating it as an ETag comparison would let a write overwrite an existing key
/// while still satisfying a policy that requires `s3:if-none-match`.
pub(crate) fn reject_non_wildcard_write_if_none_match(
if_none_match: Option<&s3s::dto::ETagCondition>,
headers: &HeaderMap,
) -> S3Result<()> {
reject_non_wildcard_write_if_none_match_native(matches!(if_none_match, Some(s3s::dto::ETagCondition::ETag(_))), headers)
}
pub(crate) fn reject_non_wildcard_write_if_none_match_native(specific_if_none_match: bool, headers: &HeaderMap) -> S3Result<()> {
pub(crate) fn reject_non_wildcard_write_if_none_match(specific_if_none_match: bool, headers: &HeaderMap) -> S3Result<()> {
// POST parses conditional DTO fields from the form, so HTTP headers also need validation.
if specific_if_none_match
|| headers.get_all(http::header::IF_NONE_MATCH).iter().any(|value| {
@@ -1180,7 +1161,6 @@ pub(crate) fn reject_non_wildcard_write_if_none_match_native(specific_if_none_ma
#[cfg(test)]
mod tests {
use super::*;
use crate::app::storage_api::s3::dto::ETag;
use http::{HeaderMap, HeaderValue};
use rustfs_gateway_types::persistence::{
PersistedBucketEncryptionRule as ServerSideEncryptionRule, PersistedEncryptionByDefault as ServerSideEncryptionByDefault,
@@ -1190,12 +1170,8 @@ mod tests {
#[test]
fn reject_non_wildcard_write_if_none_match_allows_only_wildcard() {
let headers = HeaderMap::new();
assert!(reject_non_wildcard_write_if_none_match(None, &headers).is_ok());
assert!(reject_non_wildcard_write_if_none_match(Some(&s3s::dto::ETagCondition::Any), &headers).is_ok());
let etag = s3s::dto::ETagCondition::ETag(ETag::Strong("9b2cf535f27731c974343645a3985328".to_owned()));
let err =
reject_non_wildcard_write_if_none_match(Some(&etag), &headers).expect_err("an ETag If-None-Match must be rejected");
assert!(reject_non_wildcard_write_if_none_match(false, &headers).is_ok());
let err = reject_non_wildcard_write_if_none_match(true, &headers).expect_err("an ETag If-None-Match must be rejected");
assert_eq!(*err.code(), S3ErrorCode::NotImplemented);
assert_eq!(err.status_code(), Some(http::StatusCode::NOT_IMPLEMENTED));
assert_eq!(
@@ -1209,12 +1185,12 @@ mod tests {
for value in ["*", " * ", "", " "] {
let mut headers = HeaderMap::new();
headers.insert(http::header::IF_NONE_MATCH, http::HeaderValue::from_str(value).expect("valid header"));
assert!(reject_non_wildcard_write_if_none_match(None, &headers).is_ok(), "{value:?}");
assert!(reject_non_wildcard_write_if_none_match(false, &headers).is_ok(), "{value:?}");
}
for value in ["\"different-etag\"", "W/\"weak-etag\"", "unquoted-etag"] {
let mut headers = HeaderMap::new();
headers.insert(http::header::IF_NONE_MATCH, http::HeaderValue::from_str(value).expect("valid header"));
let err = reject_non_wildcard_write_if_none_match(None, &headers)
let err = reject_non_wildcard_write_if_none_match(false, &headers)
.expect_err("header-only ETag precondition must be rejected");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -1229,7 +1205,7 @@ mod tests {
let mut headers = HeaderMap::new();
headers.append(http::header::IF_NONE_MATCH, http::HeaderValue::from_static("*"));
headers.append(http::header::IF_NONE_MATCH, value);
let err = reject_non_wildcard_write_if_none_match(Some(&s3s::dto::ETagCondition::Any), &headers)
let err = reject_non_wildcard_write_if_none_match(false, &headers)
.expect_err("a form wildcard must not hide an unsupported raw HTTP precondition");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
+21 -33
View File
@@ -614,7 +614,7 @@ fn veeam_sosapi_http_contract() {
crate::app::gating_test_env::run_large_stack_test("veeam-sosapi-http", || async {
temp_env::async_with_vars([(rustfs_config::ENV_VEEAM_SOSAPI_ENABLE, None::<&str>)], async {
use crate::app::runtime_sources::{AppContext, ServerContextSlot};
use crate::app::storage_api::s3::{Body, S3ServiceBuilder, SimpleAuth};
use std::time::Duration;
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use crate::app::storage_api::test::contract::list::ListOperations as _;
use crate::app::storage_api::test::contract::object::ObjectOperations as _;
@@ -626,17 +626,18 @@ fn veeam_sosapi_http_contract() {
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let fs = super::super::FS::with_server_ctx(slot);
let owner = S3ServiceBuilder::new(fs.clone()).build();
let mut anonymous_builder = S3ServiceBuilder::new(fs.clone());
anonymous_builder.set_auth(SimpleAuth::from_single("test-access", "test-secret"));
anonymous_builder.set_access(fs.clone());
let anonymous = anonymous_builder.build();
let mut bad_auth_builder = S3ServiceBuilder::new(fs.clone());
bad_auth_builder.set_auth(SimpleAuth::from_single("test-access", "test-secret"));
bad_auth_builder.set_access(fs);
let bad_auth = bad_auth_builder.build();
assert!(slot.install(Arc::clone(&context)));
let gateway = crate::server::gateway_stack::TestGateway::over(&context, slot);
struct Owner<'a>(&'a crate::server::gateway_stack::TestGateway);
impl Owner<'_> {
async fn call(
&self,
(method, uri, headers, body): (Method, String, Vec<(&'static str, String)>, Bytes),
) -> Result<http::Response<rustfs_gateway::Body>, std::convert::Infallible> {
self.0.signed(method, &uri, headers, body).await
}
}
let owner = Owner(&gateway);
let bucket = format!("veeam-sosapi-{}", super::super::Uuid::new_v4());
store
@@ -644,19 +645,8 @@ fn veeam_sosapi_http_contract() {
.await
.expect("create sosapi bucket");
let owner_call = |method: Method, uri: String, headers: Vec<(&str, String)>, body: Bytes| {
let mut builder = http::Request::builder().method(method).uri(uri).header("host", "localhost");
for (name, value) in headers {
builder = builder.header(name, value);
}
builder
.extension(crate::storage::req_info::ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
is_owner: true,
..Default::default()
})
.body(Body::from(body))
.expect("HTTP request")
let owner_call = |method: Method, uri: String, headers: Vec<(&'static str, String)>, body: Bytes| {
(method, uri, headers, body)
};
let system_uri = format!("/{bucket}/{SYSTEM_KEY}");
@@ -1143,17 +1133,16 @@ fn veeam_sosapi_http_contract() {
.expect("list versions");
assert!(versions.objects.is_empty());
let anon = anonymous
let anon = gateway
.call(
http::Request::builder()
.method(Method::GET)
.uri(system_uri.clone())
.header("host", "localhost")
.body(Body::empty())
.body(rustfs_gateway::Body::empty())
.expect("anon request"),
)
.await
.expect("anonymous GET");
.await;
let anon_status = anon.status();
let anon_body = BodyExt::collect(anon.into_body()).await.expect("anon body").to_bytes();
let anon_xml = String::from_utf8_lossy(&anon_body);
@@ -1163,7 +1152,7 @@ fn veeam_sosapi_http_contract() {
let now = OffsetDateTime::now_utc();
let scope_date = format!("{:04}{:02}{:02}", now.year(), u8::from(now.month()), now.day());
let amz_date = format!("{scope_date}T{:02}{:02}{:02}Z", now.hour(), now.minute(), now.second());
let bad = bad_auth
let bad = gateway
.call(
http::Request::builder()
.method(Method::GET)
@@ -1177,11 +1166,10 @@ fn veeam_sosapi_http_contract() {
"AWS4-HMAC-SHA256 Credential=bad/{scope_date}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0000000000000000000000000000000000000000000000000000000000000000"
),
)
.body(Body::empty())
.body(rustfs_gateway::Body::empty())
.expect("bad auth request"),
)
.await
.expect("bad auth GET");
.await;
let bad_status = bad.status();
let bad_body = BodyExt::collect(bad.into_body()).await.expect("bad body").to_bytes();
let bad_xml = String::from_utf8_lossy(&bad_body);
+1 -1
View File
@@ -69,7 +69,7 @@ impl ObjectMetaUsecase {
.get_bucket_info(bucket, &BucketOptions::default())
.await
.map_err(ApiError::from)?;
return Err(err.into());
return Err(err);
}
let opts: ObjectOptions = get_opts(bucket, key, version_id.clone(), None, envelope.headers())

Some files were not shown because too many files have changed in this diff Show More