mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 15:07:34 +00:00
Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| faa76508dd | |||
| 95ba225098 |
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
|
||||
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
|
||||
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
|
||||
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
|
||||
|
||||
@@ -114,14 +114,8 @@
|
||||
}
|
||||
],
|
||||
"upgrade_sources": {
|
||||
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
|
||||
"sources": [
|
||||
{
|
||||
"version": "1.0.5",
|
||||
"role": "gateway compatibility baseline",
|
||||
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
|
||||
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
|
||||
},
|
||||
{
|
||||
"version": "1.0.1",
|
||||
"role": "N-1",
|
||||
@@ -136,6 +130,6 @@
|
||||
}
|
||||
],
|
||||
"rollback_supported": false,
|
||||
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
|
||||
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,6 @@
|
||||
# s3_error_lines lines holding s3_error!, same scope, minus those the script
|
||||
# header's rule clears (rustfs_s3_types's macro); fails closed
|
||||
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
|
||||
files=33
|
||||
s3_error_lines=89
|
||||
files=49
|
||||
s3_error_lines=95
|
||||
ecstore_files=0
|
||||
|
||||
@@ -22,6 +22,8 @@ Identify the tested commit and any local changes. When testing a prebuilt binary
|
||||
List relevant checks not run and the remaining risk. Use the validation tier in AGENTS.md; do not run broader checks solely to fill this section. For documentation-only changes, list the applicable documentation checks. Use N/A only when verification is not applicable.
|
||||
-->
|
||||
|
||||
**auto-testing regression coverage** <!-- For each issue-backed bug fix, give the added or extended case ID or PR link, the existing case ID/path and assertion that catches the bug, or "Skipped" with a brief reason. Omit this line for other changes, including documentation-only fixes. -->
|
||||
|
||||
## Impact
|
||||
<!--
|
||||
Describe user-facing, compatibility, API, deployment, configuration, or documentation impact. Use N/A when there is no expected impact.
|
||||
|
||||
@@ -1091,12 +1091,9 @@ jobs:
|
||||
- name: Verify E2E archive identity
|
||||
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
|
||||
|
||||
- name: Install network fault-injection tools and the persistence test clock
|
||||
- name: Install network fault-injection tools
|
||||
run: |
|
||||
sudo apt-get install -y iptables libfaketime
|
||||
clock_library="$(dpkg-query -L libfaketime | grep -E '/libfaketime[.]so[.]1$')"
|
||||
test -f "$clock_library"
|
||||
echo "RUSTFS_PERSISTENCE_CLOCK_LIBRARY=$clock_library" >> "$GITHUB_ENV"
|
||||
sudo apt-get install -y iptables
|
||||
sudo -n iptables --version
|
||||
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
|
||||
# runners can run iptables but not touch the rule set; the test then logs
|
||||
@@ -1203,14 +1200,8 @@ jobs:
|
||||
# default-filter in .config/nextest.toml is the single wiring mechanism —
|
||||
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
|
||||
# debug binary; each test spawns its own rustfs server on a random port.
|
||||
- name: Download pinned release for persistence byte parity
|
||||
id: persistence-reference
|
||||
uses: ./.github/actions/reference-binary
|
||||
|
||||
- name: Run e2e full suite
|
||||
env:
|
||||
RUSTFS_PERSISTENCE_REFERENCE_BINARY: ${{ steps.persistence-reference.outputs.path }}
|
||||
RUSTFS_PERSISTENCE_XML_DIR: ${{ runner.temp }}/rustfs-persistence-xml
|
||||
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
|
||||
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
|
||||
@@ -1229,7 +1220,6 @@ jobs:
|
||||
path: |
|
||||
target/nextest/e2e-full/junit.xml
|
||||
${{ runner.temp }}/rustfs-e2e-full-list.json
|
||||
${{ runner.temp }}/rustfs-persistence-xml
|
||||
retention-days: 7
|
||||
|
||||
- name: Upload startup CAS evidence
|
||||
|
||||
@@ -90,16 +90,11 @@ jobs:
|
||||
- version: 1.0.1
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
|
||||
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
|
||||
- version: 1.0.5
|
||||
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
|
||||
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
|
||||
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
|
||||
env:
|
||||
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
|
||||
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
|
||||
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
|
||||
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
|
||||
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
|
||||
@@ -137,13 +132,6 @@ jobs:
|
||||
unzip -q "$archive" -d "$SOURCE_DIR"
|
||||
chmod +x "$SOURCE_DIR/rustfs"
|
||||
test -x "$SOURCE_DIR/rustfs"
|
||||
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
|
||||
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
|
||||
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
|
||||
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
|
||||
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
|
||||
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
|
||||
fi
|
||||
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
|
||||
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
|
||||
|
||||
@@ -163,14 +151,12 @@ jobs:
|
||||
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
|
||||
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
|
||||
)
|
||||
case "$UPGRADE_SOURCE_VERSION" in
|
||||
1.0.0-rc.5)
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
|
||||
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
|
||||
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
|
||||
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
|
||||
esac
|
||||
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
|
||||
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
|
||||
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
|
||||
else
|
||||
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
|
||||
fi
|
||||
status=0
|
||||
for entry in "${cases[@]}"; do
|
||||
artifact="${entry%%:*}"
|
||||
@@ -192,7 +178,7 @@ jobs:
|
||||
exit "$status"
|
||||
|
||||
- name: Run mixed-version MPU recovery
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
|
||||
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
|
||||
timeout-minutes: 15
|
||||
env:
|
||||
NO_PROXY: 127.0.0.1,localhost,::1
|
||||
@@ -213,24 +199,6 @@ jobs:
|
||||
"$mpu_test" -- --exact --nocapture --test-threads=1 \
|
||||
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
|
||||
|
||||
- name: Verify gateway baseline G09 evidence
|
||||
if: ${{ success() && matrix.source.version == '1.0.5' }}
|
||||
shell: bash
|
||||
run: |
|
||||
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
|
||||
directory=mixed-version-upgrade
|
||||
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
|
||||
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
|
||||
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
|
||||
done
|
||||
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
|
||||
'.current_revision == $current and .previous_revision == $previous
|
||||
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
|
||||
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
|
||||
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
|
||||
|
||||
- name: Upload scanner/heal G09 evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
|
||||
|
||||
@@ -80,6 +80,18 @@ Do not commit one-shot plans, trackers, migration ledgers, benchmark snapshots,
|
||||
|
||||
Select checks from the final task-owned diff. Scoped `AGENTS.md` files may add a concrete path-specific check, but must not replace this tiering with a generic full-workspace gate.
|
||||
|
||||
### Issue Fixes and External Regression Coverage
|
||||
|
||||
After fixing a bug tracked by an issue and running the applicable local checks, ask the developer once per fixed issue whether to cover it in [rustfs/auto-testing](https://github.com/rustfs/auto-testing). Ask before opening or updating the fix PR, or before reporting completion if there is no PR. Reuse a choice already made for that fix. Investigation alone and documentation-only changes do not trigger this prompt.
|
||||
|
||||
When presenting the three choices, describe the failing scenario and the assertion that would catch it.
|
||||
|
||||
- **Add or extend a case.** Prefer extending an existing scenario when it can exercise the regression. Record the resulting case ID or auto-testing PR link.
|
||||
- **Already covered.** Cite the existing auto-testing case ID or path and the assertion that would catch this bug.
|
||||
- **Skip.** Record the developer's one-line reason and continue with the fix.
|
||||
|
||||
Record the choice in the PR's Verification section or in the completion report for work without a PR. All three choices retain the applicable RustFS verification requirements below.
|
||||
|
||||
### Documentation and Instructions
|
||||
|
||||
For prose, comments, agent instructions, and skill metadata that cannot affect runtime/build output:
|
||||
|
||||
+1
-1
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
|
||||
|
||||
### Deployment
|
||||
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
|
||||
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
|
||||
|
||||
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
|
||||
|
||||
|
||||
Generated
+13
-11
@@ -1648,7 +1648,7 @@ dependencies = [
|
||||
"cexpr",
|
||||
"clang-sys",
|
||||
"log",
|
||||
"prettyplease 0.3.0",
|
||||
"prettyplease 0.2.37",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"regex",
|
||||
@@ -4238,7 +4238,6 @@ dependencies = [
|
||||
"s3s",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde-value",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sha1 0.11.0",
|
||||
@@ -10238,6 +10237,7 @@ dependencies = [
|
||||
"rustix",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
@@ -10323,7 +10323,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway"
|
||||
version = "0.66.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"bytes",
|
||||
@@ -10370,7 +10370,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-core"
|
||||
version = "0.63.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures-core",
|
||||
@@ -10391,7 +10391,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-dialect-rustfs-admin"
|
||||
version = "0.13.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"form_urlencoded",
|
||||
@@ -10405,7 +10405,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-http"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
@@ -10423,7 +10423,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-macros"
|
||||
version = "0.1.4"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -10433,7 +10433,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-sig"
|
||||
version = "0.15.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"hmac 0.13.0",
|
||||
"http 1.5.0",
|
||||
@@ -10448,7 +10448,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-stream"
|
||||
version = "0.2.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"bitflags 2.13.2",
|
||||
"bytes",
|
||||
@@ -10459,7 +10459,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-types"
|
||||
version = "0.42.0+aws.2026-10-02"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"crc-fast",
|
||||
@@ -10482,7 +10482,7 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "rustfs-gateway-xml"
|
||||
version = "0.4.0"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
|
||||
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
|
||||
dependencies = [
|
||||
"quick-xml 0.41.0",
|
||||
]
|
||||
@@ -11236,6 +11236,7 @@ dependencies = [
|
||||
"rustfs-utils",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.11.1",
|
||||
@@ -11375,6 +11376,7 @@ dependencies = [
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-utils",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
|
||||
+6
-7
@@ -152,16 +152,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
|
||||
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
|
||||
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
|
||||
# the s3s-facing edge of the rustfs binary.
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.2.0", default-features = false }
|
||||
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.42.0", default-features = false }
|
||||
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.2.0", default-features = false }
|
||||
# The gateway framework itself, consumed for its extension traits by the ring-2
|
||||
# adapter. `default-features = false` keeps the `server` feature, and with it the
|
||||
# server crate, out of the build.
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.66.0", default-features = false }
|
||||
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.66.0", default-features = false }
|
||||
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.13.0", default-features = false }
|
||||
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", default-features = false }
|
||||
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.63.0", default-features = false }
|
||||
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.13.0", default-features = false }
|
||||
|
||||
# Async Runtime and Networking
|
||||
async-channel = "2.5.0"
|
||||
@@ -222,7 +222,6 @@ quick-xml = "0.42.0"
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.229" }
|
||||
serde-value = "0.7.0"
|
||||
serde_ignored = { version = "0.1" }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
|
||||
serde_json = { version = "1.0.154" }
|
||||
|
||||
@@ -129,6 +129,7 @@ md-5 = { workspace = true }
|
||||
prost.workspace = true
|
||||
sha2 = { workspace = true }
|
||||
rustfs-s3-types = { workspace = true }
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
zstd.workspace = true
|
||||
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
|
||||
rcgen.workspace = true
|
||||
@@ -138,8 +139,6 @@ rustls = { workspace = true, default-features = false, features = ["aws-lc-rs",
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
serde-value.workspace = true
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
|
||||
aws-config = { workspace = true }
|
||||
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
|
||||
|
||||
@@ -211,8 +211,6 @@ the wiring source of truth. Committed test-ID digests under
|
||||
|
||||
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
|
||||
|
||||
The full suite compares persisted bucket configurations and object records against the release pinned in `.config/reference-binary.toml`. Set `RUSTFS_PERSISTENCE_REFERENCE_BINARY` to the verified executable and `RUSTFS_PERSISTENCE_CLOCK_LIBRARY` to the absolute path of libfaketime (`libfaketime.so.1` on Linux or `libfaketime.1.dylib` on macOS). To retain the two sets of 13 XML documents for the gateway golden corpus, set `RUSTFS_PERSISTENCE_XML_DIR` to an output directory. CI includes them in the full suite's JUnit artifact.
|
||||
|
||||
**Reproduce a CI failure locally** — run the exact profile/lane:
|
||||
|
||||
```bash
|
||||
|
||||
@@ -34,10 +34,7 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
@@ -399,54 +396,6 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -137,40 +137,6 @@ async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::erro
|
||||
.await
|
||||
.map_err(|e| format!("DeleteObject failed: {}", e))?;
|
||||
|
||||
for (allowed, may_read) in [("s3:GetBucketPolicy", true), ("s3:PutBucketPolicy", false)] {
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Allow",
|
||||
"Principal": { "AWS": [user_access] },
|
||||
"Action": allowed,
|
||||
"Resource": format!("arn:aws:s3:::{bucket_name}")
|
||||
}]
|
||||
})
|
||||
.to_string();
|
||||
admin_client
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket_name)
|
||||
.policy(&policy)
|
||||
.send()
|
||||
.await?;
|
||||
let read = user_client.get_bucket_policy().bucket(bucket_name).send().await;
|
||||
let write = user_client
|
||||
.put_bucket_policy()
|
||||
.bucket(bucket_name)
|
||||
.policy(&policy)
|
||||
.send()
|
||||
.await;
|
||||
if may_read {
|
||||
read?;
|
||||
let denied = write.expect_err("GetBucketPolicy must not grant PutBucketPolicy");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
} else {
|
||||
let denied = read.expect_err("PutBucketPolicy must not grant GetBucketPolicy");
|
||||
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
|
||||
write?;
|
||||
}
|
||||
}
|
||||
info!("Test Passed!");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -31,7 +31,7 @@ use super::harness::{
|
||||
use crate::common::{
|
||||
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
|
||||
};
|
||||
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
|
||||
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -427,12 +427,7 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn compare_degraded_legacy_list_parts(
|
||||
dist: &mut DistCluster,
|
||||
seed: &UpgradeSeed,
|
||||
previous: &Path,
|
||||
source: &UpgradeSourceCapabilities,
|
||||
) -> TestResult {
|
||||
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
|
||||
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
|
||||
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
|
||||
@@ -494,22 +489,16 @@ async fn compare_degraded_legacy_list_parts(
|
||||
// Three online disks retain metadata quorum but have no part directory;
|
||||
// the fourth disk is offline. A legacy empty response is not a proof that
|
||||
// the unavailable participant has no part.
|
||||
let mut quorum_errors =
|
||||
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
|
||||
if source.requires_list_parts_read_quorum {
|
||||
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
|
||||
} else {
|
||||
assert!(
|
||||
legacy
|
||||
.expect("previous coordinator should expose its legacy empty-list behavior")
|
||||
.parts()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
for error in quorum_errors {
|
||||
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
|
||||
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
|
||||
}
|
||||
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
|
||||
assert!(legacy.parts().is_empty());
|
||||
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
|
||||
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
|
||||
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
|
||||
// ListParts read-quorum failure is reported as ServiceUnavailable.
|
||||
assert_eq!(
|
||||
current.as_service_error().and_then(|error| error.meta().code()),
|
||||
Some("ServiceUnavailable")
|
||||
);
|
||||
dist.cluster.start_node_from_binary(3, previous).await?;
|
||||
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
|
||||
for (path, before) in original_files {
|
||||
@@ -531,7 +520,7 @@ async fn compare_degraded_legacy_list_parts(
|
||||
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
|
||||
}
|
||||
eprintln!(
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
|
||||
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
@@ -603,7 +592,6 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
|
||||
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
|
||||
init_logging();
|
||||
let previous = source_binary()?;
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
|
||||
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
|
||||
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
|
||||
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
|
||||
@@ -613,7 +601,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
|
||||
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
|
||||
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
|
||||
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
|
||||
|
||||
for node_idx in [1, 2] {
|
||||
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
|
||||
|
||||
@@ -27,7 +27,7 @@
|
||||
//! is process-global, so two stacks in one process would read each other's buckets.
|
||||
|
||||
mod compare;
|
||||
pub(crate) mod reference;
|
||||
mod reference;
|
||||
mod scenarios;
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
|
||||
|
||||
@@ -78,7 +78,7 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
|
||||
|
||||
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
|
||||
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
|
||||
pub(crate) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
|
||||
|
||||
@@ -37,9 +37,6 @@ pub mod fake_s3_target;
|
||||
#[cfg(test)]
|
||||
mod dual_stack_differential;
|
||||
|
||||
#[cfg(test)]
|
||||
mod persistence_parity;
|
||||
|
||||
// On-demand migration (backlog#2147): shared two-server environment, admin
|
||||
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
|
||||
// added by later ODM tasks.
|
||||
|
||||
@@ -1543,30 +1543,6 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(),
|
||||
let uploaded = get_out.body.collect().await?.into_bytes();
|
||||
assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match");
|
||||
|
||||
// An empty redirect is rejected when rendering the response, after storage succeeds.
|
||||
for (field, value, expected_status) in [
|
||||
("success_action_status", "200", reqwest::StatusCode::OK),
|
||||
("success_action_status", "204", reqwest::StatusCode::NO_CONTENT),
|
||||
("success_action_redirect", "", reqwest::StatusCode::BAD_REQUEST),
|
||||
] {
|
||||
let key = format!("{field}-{value}.txt");
|
||||
let form = reqwest::multipart::Form::new()
|
||||
.text("key", key.clone())
|
||||
.text(field, value)
|
||||
.part("file", reqwest::multipart::Part::bytes(expected_body.clone()).file_name("upload.txt"));
|
||||
let response = local_http_client()
|
||||
.post(format!("{}/{bucket}", env.url))
|
||||
.multipart(form)
|
||||
.send()
|
||||
.await?;
|
||||
assert_eq!(response.status(), expected_status);
|
||||
if value.is_empty() {
|
||||
assert!(response.text().await?.contains("<Code>InvalidArgument</Code>"));
|
||||
}
|
||||
let stored = admin_client.get_object().bucket(bucket).key(key).send().await?;
|
||||
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body.as_slice());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1,449 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Compare bytes produced by S3 writes to independent, single-drive volumes.
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, admin_request, rustfs_binary_path, signed_s3_request_with_headers};
|
||||
use crate::dual_stack_differential::reference::binary_version;
|
||||
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
|
||||
use crate::on_demand_migration::common::{ODM_ADMIN_ROUTE, ODM_MODULE_SWITCH_ENV, OdmSourceSpec};
|
||||
use crate::replication_extension_test::send_set_replication_target_request;
|
||||
use crate::storage_api::persistence_parity::{BucketMetadata, RUSTFS_META_BUCKET};
|
||||
use crate::upgrade_compatibility_test::{set_bucket_quota, write_multipart};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
|
||||
use http::{HeaderMap, Method};
|
||||
use md5::{Digest, Md5};
|
||||
use rustfs_filemeta::{FileMeta, MetaObject};
|
||||
use serde_value::Value;
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
type Error = Box<dyn std::error::Error + Send + Sync>;
|
||||
const CONFIGS: &str = "persisted-configs";
|
||||
const OBJECTS: &str = "persisted-objects";
|
||||
const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
const LOGS: &str = "rustfs-access-logs";
|
||||
|
||||
fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error> {
|
||||
let key = BucketMetadata::new(bucket).save_file_path();
|
||||
let object = volume.join(RUSTFS_META_BUCKET).join(&key);
|
||||
let meta = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
|
||||
let info = meta.into_fileinfo(RUSTFS_META_BUCKET, &key, "", true, false, true)?;
|
||||
info.validate_for_metadata_read()?;
|
||||
let [part] = info.parts.as_slice() else { return Err("bucket metadata must have one part".into()) };
|
||||
let size = usize::try_from(info.size)?;
|
||||
if info.erasure.data_blocks != 1 || size != part.size || size > info.erasure.block_size {
|
||||
return Err("bucket metadata exceeds the single-drive, single-stripe fixture".into());
|
||||
}
|
||||
let framed = if info.inline_data() {
|
||||
info.data.ok_or("missing inline bucket metadata")?.to_vec()
|
||||
} else {
|
||||
let directory = info.data_dir.ok_or("missing bucket metadata data directory")?;
|
||||
std::fs::read(object.join(directory.to_string()).join(format!("part.{}", part.number)))?
|
||||
};
|
||||
let algorithm = info.erasure.get_checksum_info(part.number).algorithm;
|
||||
let (digest, payload) = framed.split_at_checked(algorithm.size()).ok_or("truncated bucket metadata")?;
|
||||
if digest.is_empty() || payload.len() != size || algorithm.hash_encode(payload).as_ref() != digest {
|
||||
return Err("invalid bucket metadata shard checksum or size".into());
|
||||
}
|
||||
BucketMetadata::check_header(payload)?;
|
||||
let fields = &payload[4..];
|
||||
if bucket == CONFIGS {
|
||||
// The production decoder defaults a missing field to empty. Require the ACL field on disk
|
||||
// so dropping it cannot look like the expected zero-byte document.
|
||||
let keys: BTreeMap<&str, serde::de::IgnoredAny> = rmp_serde::from_slice(fields)?;
|
||||
if !keys.contains_key("BucketAclConfigJSON") {
|
||||
return Err("missing BucketAclConfigJSON field".into());
|
||||
}
|
||||
}
|
||||
Ok(BucketMetadata::unmarshal(fields)?)
|
||||
}
|
||||
|
||||
type VersionFields = BTreeMap<String, Value>;
|
||||
|
||||
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
|
||||
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
|
||||
let [version] = meta.versions.as_slice() else {
|
||||
return Err(format!("{key}: expected one version").into());
|
||||
};
|
||||
let object = version.parse_version_meta()?.object.ok_or("missing V2 object")?;
|
||||
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
|
||||
return Err(format!("{key}: missing modification time or data directory").into());
|
||||
}
|
||||
Ok((object, nonvolatile_fields(&version.meta, key)?))
|
||||
}
|
||||
|
||||
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
|
||||
// The production reader skips unknown keys and supplies defaults for some missing keys.
|
||||
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
|
||||
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
|
||||
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
|
||||
return Err(format!("{key}: missing raw V2Obj map").into());
|
||||
};
|
||||
for volatile in ["MTime", "DDir"] {
|
||||
let value = object_fields
|
||||
.get_mut(&Value::String(volatile.to_owned()))
|
||||
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
|
||||
*value = Value::Unit;
|
||||
}
|
||||
Ok(fields)
|
||||
}
|
||||
|
||||
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert("content-md5", base64_simd::STANDARD.encode_to_string(Md5::digest(xml)).parse()?);
|
||||
let response = signed_s3_request_with_headers(
|
||||
Method::PUT,
|
||||
&format!("{}/{CONFIGS}?{query}", env.url),
|
||||
Some(xml.to_owned()),
|
||||
Some("application/xml"),
|
||||
&env.access_key,
|
||||
&env.secret_key,
|
||||
&headers,
|
||||
)
|
||||
.await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("PUT ?{query} failed with {}", response.status()).into());
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
struct Snapshot {
|
||||
configs: BTreeMap<&'static str, Vec<u8>>,
|
||||
objects: BTreeMap<&'static str, VersionFields>,
|
||||
multipart_files: BTreeMap<String, Vec<u8>>,
|
||||
}
|
||||
|
||||
async fn capture(binary: &Path, target: &FakeS3Target, clock: &str, clock_library: &str) -> Result<Snapshot, Error> {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let queue = tempfile::tempdir()?;
|
||||
let clock_loader = if cfg!(target_os = "macos") {
|
||||
"DYLD_INSERT_LIBRARIES"
|
||||
} else {
|
||||
"LD_PRELOAD"
|
||||
};
|
||||
let options = [
|
||||
(clock_loader, clock_library),
|
||||
("DYLD_FORCE_FLAT_NAMESPACE", "1"),
|
||||
("FAKETIME", clock),
|
||||
("FAKETIME_DONT_FAKE_MONOTONIC", "1"),
|
||||
("TZ", "UTC"),
|
||||
("RUSTFS_NOTIFY_ENABLE", "true"),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", "on"),
|
||||
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", target.endpoint()),
|
||||
(
|
||||
"RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY",
|
||||
queue.path().to_str().ok_or("non-UTF8 queue path")?,
|
||||
),
|
||||
("RUSTFS_OUTBOUND_ALLOW_ORIGINS", target.endpoint()),
|
||||
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
|
||||
(ODM_MODULE_SWITCH_ENV, "true"),
|
||||
("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
];
|
||||
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &options)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
write_configs(&env, &client, target).await?;
|
||||
write_objects(&client).await?;
|
||||
env.stop_server();
|
||||
read_snapshot(Path::new(&env.temp_dir))
|
||||
}
|
||||
|
||||
async fn write_configs(env: &RustFSTestEnvironment, client: &Client, target: &FakeS3Target) -> Result<(), Error> {
|
||||
env.create_test_bucket(LOGS).await?;
|
||||
env.create_test_bucket(OBJECTS).await?;
|
||||
env.create_test_bucket("persistence-warehouse").await?;
|
||||
env.create_test_bucket("persistence-migration").await?;
|
||||
client
|
||||
.create_bucket()
|
||||
.bucket(CONFIGS)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
macro_rules! golden {
|
||||
($path:literal) => {
|
||||
include_str!(concat!("../../ecstore/tests/fixtures/bucket-config-goldens/", $path, ".in.xml"))
|
||||
};
|
||||
}
|
||||
let target_request = serde_json::json!({
|
||||
"endpoint": target.address(), "credentials": {"accessKey": FAKE_ACCESS_KEY, "secretKey": FAKE_SECRET_KEY},
|
||||
"targetbucket": "interop-dr", "type": "replication", "secure": false,
|
||||
"deploymentID": "0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c"
|
||||
});
|
||||
let response = send_set_replication_target_request(env, CONFIGS, false, target_request).await?;
|
||||
if !response.status().is_success() {
|
||||
return Err(format!("set remote target failed with {}", response.status()).into());
|
||||
}
|
||||
let target_arn: String = response.json().await?;
|
||||
let replication = golden!("replication/rustfs").replace("arn:rustfs:replication:", "arn:minio:replication:");
|
||||
if !replication.contains(&format!("<Bucket>{target_arn}</Bucket>")) {
|
||||
return Err("registered target does not match the golden replication destination".into());
|
||||
}
|
||||
for (query, xml) in [
|
||||
("versioning", golden!("versioning/minio")),
|
||||
("notification", golden!("notification/rustfs")),
|
||||
("replication", replication.as_str()),
|
||||
("lifecycle", golden!("lifecycle/minio")),
|
||||
("object-lock", golden!("object_lock/rustfs")),
|
||||
("encryption", golden!("encryption/minio")),
|
||||
("tagging", golden!("tagging/rustfs")),
|
||||
("cors", golden!("cors/rustfs")),
|
||||
("logging", golden!("logging/rustfs")),
|
||||
("website", golden!("website/rustfs")),
|
||||
("accelerate", golden!("accelerate/rustfs")),
|
||||
("requestPayment", golden!("request_payment/rustfs")),
|
||||
("publicAccessBlock", golden!("public_access_block/rustfs")),
|
||||
] {
|
||||
put_xml(env, query, xml).await?;
|
||||
}
|
||||
client
|
||||
.put_bucket_acl()
|
||||
.bucket(CONFIGS)
|
||||
.acl(BucketCannedAcl::Private)
|
||||
.send()
|
||||
.await?;
|
||||
let acl = client.get_bucket_acl().bucket(CONFIGS).send().await?;
|
||||
let owner = acl.owner().and_then(|owner| owner.id()).ok_or("ACL owner missing")?;
|
||||
let [grant] = acl.grants() else { return Err("expected exactly one ACL grant".into()) };
|
||||
let grantee = grant.grantee().ok_or("missing ACL grantee")?;
|
||||
if owner.is_empty()
|
||||
|| grant.permission().map(|p| p.as_str()) != Some("FULL_CONTROL")
|
||||
|| grantee.r#type().as_str() != "CanonicalUser"
|
||||
|| grantee.id().is_some()
|
||||
{
|
||||
return Err("GET ACL changed its canonical owner grant".into());
|
||||
}
|
||||
let policy = serde_json::json!({
|
||||
"Version": "2012-10-17",
|
||||
"Statement": [{
|
||||
"Effect": "Deny", "Principal": "*", "Action": "s3:DeleteObject",
|
||||
"Resource": format!("arn:aws:s3:::{CONFIGS}/protected/*"),
|
||||
}],
|
||||
});
|
||||
client
|
||||
.put_bucket_policy()
|
||||
.bucket(CONFIGS)
|
||||
.policy(policy.to_string())
|
||||
.send()
|
||||
.await?;
|
||||
set_bucket_quota(env, CONFIGS, 16 * 1024 * 1024).await?;
|
||||
let migration = OdmSourceSpec::for_fake_source(target, "migration-source");
|
||||
for (path, body) in [
|
||||
("/iceberg/v1/buckets/persistence-warehouse".to_owned(), None),
|
||||
(format!("{ODM_ADMIN_ROUTE}/persistence-migration"), Some(migration.to_json().to_string())),
|
||||
] {
|
||||
let (status, _) = admin_request(&env.url, Method::PUT, &path, body, &env.access_key, &env.secret_key).await?;
|
||||
if !status.is_success() {
|
||||
return Err(format!("configuring {path} failed with {status}").into());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn write_objects(client: &Client) -> Result<(), Error> {
|
||||
for key in ["put", "tagged"] {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(OBJECTS)
|
||||
.key(key)
|
||||
.metadata("purpose", "byte-parity")
|
||||
.body(ByteStream::from_static(b"persisted object contents"))
|
||||
.send()
|
||||
.await?;
|
||||
}
|
||||
client
|
||||
.copy_object()
|
||||
.bucket(OBJECTS)
|
||||
.key("copy")
|
||||
.copy_source(format!("{OBJECTS}/put"))
|
||||
.send()
|
||||
.await?;
|
||||
let tags = Tagging::builder()
|
||||
.tag_set(Tag::builder().key("type").value("persisted").build()?)
|
||||
.build()?;
|
||||
client
|
||||
.put_object_tagging()
|
||||
.bucket(OBJECTS)
|
||||
.key("tagged")
|
||||
.tagging(tags)
|
||||
.send()
|
||||
.await?;
|
||||
write_multipart(client, OBJECTS, "multipart", &[vec![0x31; FIRST_PART_SIZE], vec![0x32; 100]]).await?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
|
||||
let b = bucket_metadata(volume, CONFIGS)?;
|
||||
let configs = BTreeMap::from([
|
||||
("acl", b.bucket_acl_config_json),
|
||||
("accelerate", b.accelerate_config_xml),
|
||||
("cors", b.cors_config_xml),
|
||||
("encryption", b.encryption_config_xml),
|
||||
("lifecycle", b.lifecycle_config_xml),
|
||||
("logging", b.logging_config_xml),
|
||||
("notification", b.notification_config_xml),
|
||||
("object_lock", b.object_lock_config_xml),
|
||||
("public_access_block", b.public_access_block_config_xml),
|
||||
("request_payment", b.request_payment_config_xml),
|
||||
("tagging", b.tagging_config_xml),
|
||||
("versioning", b.versioning_config_xml),
|
||||
("website", b.website_config_xml),
|
||||
("durability", b.durability_config_json),
|
||||
("policy", b.policy_config_json),
|
||||
("quota", b.quota_config_json),
|
||||
("replication", b.replication_config_xml),
|
||||
("targets", b.bucket_targets_config_json),
|
||||
("table", bucket_metadata(volume, "persistence-warehouse")?.table_bucket_config_json),
|
||||
("odm", bucket_metadata(volume, "persistence-migration")?.on_demand_migration_config_json),
|
||||
]);
|
||||
let mut objects = BTreeMap::new();
|
||||
for key in ["put", "copy", "tagged"] {
|
||||
let (_, fields) = read_object(volume, key)?;
|
||||
objects.insert(key, fields);
|
||||
}
|
||||
let (multipart, _) = read_object(volume, "multipart")?;
|
||||
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
|
||||
return Err("xl.meta does not describe both uploaded parts".into());
|
||||
}
|
||||
let directory = volume
|
||||
.join(OBJECTS)
|
||||
.join("multipart")
|
||||
.join(multipart.data_dir.ok_or("missing multipart data directory")?.to_string());
|
||||
let mut multipart_files = BTreeMap::new();
|
||||
for entry in std::fs::read_dir(directory)? {
|
||||
let entry = entry?;
|
||||
let name = entry.file_name().into_string().map_err(|_| "invalid multipart file name")?;
|
||||
if name.starts_with("part.") {
|
||||
let bytes = std::fs::read(entry.path())?;
|
||||
if bytes.is_empty() {
|
||||
return Err(format!("empty multipart file {name}").into());
|
||||
}
|
||||
multipart_files.insert(name, bytes);
|
||||
}
|
||||
}
|
||||
for name in ["part.1", "part.2", "part.1.meta", "part.2.meta"] {
|
||||
if !multipart_files.contains_key(name) {
|
||||
return Err(format!("missing multipart file {name}").into());
|
||||
}
|
||||
}
|
||||
Ok(Snapshot {
|
||||
configs,
|
||||
objects,
|
||||
multipart_files,
|
||||
})
|
||||
}
|
||||
|
||||
fn save_xml_samples(root: &Path, side: &str, snapshot: &Snapshot) -> Result<(), Error> {
|
||||
let directory = root.join(side);
|
||||
std::fs::create_dir_all(&directory)?;
|
||||
let families: [&str; 13] = [
|
||||
"accelerate",
|
||||
"cors",
|
||||
"encryption",
|
||||
"lifecycle",
|
||||
"logging",
|
||||
"notification",
|
||||
"object_lock",
|
||||
"public_access_block",
|
||||
"replication",
|
||||
"request_payment",
|
||||
"tagging",
|
||||
"versioning",
|
||||
"website",
|
||||
];
|
||||
for family in families {
|
||||
let bytes = snapshot
|
||||
.configs
|
||||
.get(family)
|
||||
.ok_or_else(|| format!("missing XML family {family}"))?;
|
||||
std::fs::write(directory.join(format!("{family}.xml")), bytes)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persistence_byte_parity() -> Result<(), Error> {
|
||||
let reference = PathBuf::from(
|
||||
std::env::var_os("RUSTFS_PERSISTENCE_REFERENCE_BINARY").ok_or("missing pinned RUSTFS_PERSISTENCE_REFERENCE_BINARY")?,
|
||||
);
|
||||
let candidate = rustfs_binary_path();
|
||||
if binary_version(&reference).await? == binary_version(&candidate).await? {
|
||||
return Err("reference and candidate binaries report identical build identities".into());
|
||||
}
|
||||
// The quota writer records the process wall clock inside its JSON document. Keep raw bytes
|
||||
// comparable while the SDK's signature clock and both servers' monotonic timers advance normally.
|
||||
let clock_library = std::env::var("RUSTFS_PERSISTENCE_CLOCK_LIBRARY")?;
|
||||
if !Path::new(&clock_library).is_absolute() || !Path::new(&clock_library).is_file() {
|
||||
return Err("RUSTFS_PERSISTENCE_CLOCK_LIBRARY must name an absolute libfaketime path".into());
|
||||
}
|
||||
let clock = time::OffsetDateTime::now_utc()
|
||||
.format(time::macros::format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"))?;
|
||||
let target = FakeS3Target::start().await?;
|
||||
target.create_bucket("migration-source");
|
||||
target.create_bucket_with_object_lock("interop-dr");
|
||||
let reference = capture(&reference, &target, &clock, &clock_library).await?;
|
||||
let candidate = capture(&candidate, &target, &clock, &clock_library).await?;
|
||||
if let Some(directory) = std::env::var_os("RUSTFS_PERSISTENCE_XML_DIR") {
|
||||
save_xml_samples(Path::new(&directory), "reference", &reference)?;
|
||||
save_xml_samples(Path::new(&directory), "candidate", &candidate)?;
|
||||
}
|
||||
if reference.configs.len() != 20 || candidate.configs.len() != 20 {
|
||||
return Err("expected 13 XML and seven JSON fields on each side".into());
|
||||
}
|
||||
for (&field, expected) in &reference.configs {
|
||||
// Canned ACLs acknowledge the write and return one owner grant without persisting an ACL document.
|
||||
if expected.is_empty() != (field == "acl") || candidate.configs.get(field) != Some(expected) {
|
||||
return Err(format!("incorrect or different persisted bucket config: {field}").into());
|
||||
}
|
||||
}
|
||||
for (key, expected) in &reference.objects {
|
||||
if candidate.objects.get(key) != Some(expected) {
|
||||
return Err(format!("different persisted xl.meta fields: {key}").into());
|
||||
}
|
||||
}
|
||||
if candidate.multipart_files != reference.multipart_files {
|
||||
return Err("different multipart part files or part metadata".into());
|
||||
}
|
||||
eprintln!(
|
||||
"Compared {} bucket fields (canned ACL empty), {} full object records and multipart part files",
|
||||
reference.configs.len(),
|
||||
reference.objects.len()
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
|
||||
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
|
||||
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
|
||||
let original = fields(&sample)?;
|
||||
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
|
||||
sample["V2Obj"]["MTime"] = serde_json::json!(2);
|
||||
assert_eq!(fields(&sample)?, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::Value::Null;
|
||||
let with_nil = fields(&sample)?;
|
||||
assert_ne!(with_nil, original);
|
||||
sample["V2Obj"]["Future"] = serde_json::json!(false);
|
||||
let with_value = fields(&sample)?;
|
||||
assert_ne!(with_value, with_nil);
|
||||
sample["FutureEnvelope"] = serde_json::json!(1);
|
||||
assert_ne!(fields(&sample)?, with_value);
|
||||
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
|
||||
Ok(())
|
||||
}
|
||||
@@ -465,7 +465,7 @@ pub(crate) async fn set_replication_target_with_options(
|
||||
Ok(arn)
|
||||
}
|
||||
|
||||
pub(crate) async fn send_set_replication_target_request(
|
||||
async fn send_set_replication_target_request(
|
||||
source_env: &RustFSTestEnvironment,
|
||||
source_bucket: &str,
|
||||
update: bool,
|
||||
|
||||
@@ -57,12 +57,6 @@ pub(crate) mod replication_extension {
|
||||
pub(crate) use super::BucketTargetSys;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod persistence_parity {
|
||||
pub(crate) use super::RUSTFS_META_BUCKET;
|
||||
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) mod upgrade_compatibility {
|
||||
pub(crate) use super::{DiskOption, Endpoint, RUSTFS_META_MULTIPART_BUCKET, new_disk};
|
||||
|
||||
@@ -46,12 +46,9 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
|
||||
|
||||
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
|
||||
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
|
||||
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
|
||||
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
|
||||
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
|
||||
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
|
||||
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
|
||||
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
|
||||
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
|
||||
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
|
||||
const PLAIN_BUCKET: &str = "upgrade-plain-data";
|
||||
@@ -145,41 +142,6 @@ fn current_source_revision() -> Result<String, BoxError> {
|
||||
Ok(revision.to_string())
|
||||
}
|
||||
|
||||
pub(crate) struct UpgradeSourceCapabilities {
|
||||
supports_delete_recovery: bool,
|
||||
supports_multipart_publication: bool,
|
||||
pub(crate) requires_list_parts_read_quorum: bool,
|
||||
}
|
||||
|
||||
impl UpgradeSourceCapabilities {
|
||||
fn from_revision(revision: &str) -> Result<Self, BoxError> {
|
||||
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
|
||||
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
|
||||
match revision {
|
||||
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: false,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_4_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: false,
|
||||
requires_list_parts_read_quorum: false,
|
||||
}),
|
||||
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
|
||||
supports_delete_recovery: true,
|
||||
supports_multipart_publication: true,
|
||||
requires_list_parts_read_quorum: true,
|
||||
}),
|
||||
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
|
||||
Self::from_revision(&binary_source_revision(binary).await?)
|
||||
}
|
||||
}
|
||||
|
||||
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
|
||||
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
|
||||
if !output.status.success() {
|
||||
@@ -263,25 +225,6 @@ mod g09_evidence_tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[test]
|
||||
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
|
||||
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
|
||||
(RC2_COMMIT, false, false, false),
|
||||
(RC5_COMMIT, false, false, false),
|
||||
(STABLE_1_0_1_COMMIT, false, false, false),
|
||||
(STABLE_1_0_4_COMMIT, true, false, false),
|
||||
(STABLE_1_0_5_COMMIT, true, true, true),
|
||||
(STABLE_1_0_6_COMMIT, true, true, true),
|
||||
] {
|
||||
let source = UpgradeSourceCapabilities::from_revision(revision)?;
|
||||
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
|
||||
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
|
||||
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
|
||||
}
|
||||
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
|
||||
let binary = Path::new("rustfs");
|
||||
@@ -422,7 +365,7 @@ async fn complete_pending_multipart(
|
||||
.await?)
|
||||
}
|
||||
|
||||
pub(crate) async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
|
||||
async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
|
||||
let pending = stage_multipart(client, bucket, key, parts).await?;
|
||||
complete_pending_multipart(client, bucket, key, &pending).await?;
|
||||
Ok(())
|
||||
@@ -1068,7 +1011,8 @@ async fn exercise_mixed_cluster(
|
||||
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
|
||||
);
|
||||
|
||||
// Use a separate old-writer key so the original seed still spans the entire rollout.
|
||||
// 1.0.4 already implements the delete decision protocol. Use a separate
|
||||
// old-writer key so the original seed still spans the entire rollout.
|
||||
if previous_has_delete_recovery {
|
||||
let key = format!(".upgrade-delete/{phase}/previous-writer");
|
||||
previous_client
|
||||
@@ -2349,7 +2293,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let previous_binary = source_binary()?;
|
||||
// Bind the expected protocol to the verified release source, never to the
|
||||
// DELETE result itself. The older CI source pins retain their 503 contract.
|
||||
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
|
||||
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
let current_binary = rustfs_binary_path();
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
|
||||
@@ -2371,21 +2315,17 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
cluster.stop_node(0)?;
|
||||
cluster.start_node_from_binary(0, ¤t_binary).await?;
|
||||
let mut mixed_deleted_keys = Vec::new();
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
let mixed_completion = if source.supports_multipart_publication {
|
||||
let clients = cluster.create_all_clients()?;
|
||||
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
|
||||
} else {
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
None
|
||||
};
|
||||
// One controlled out-of-policy attempt proves fail-closed behavior below
|
||||
// native write quorum; positive MPU writers remain paused while mixed.
|
||||
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
|
||||
|
||||
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
|
||||
cluster.stop_node(node_idx)?;
|
||||
cluster.start_node_from_binary(node_idx, ¤t_binary).await?;
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
|
||||
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
|
||||
mixed_deleted_keys.push(key);
|
||||
}
|
||||
}
|
||||
@@ -2395,13 +2335,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
let clients = cluster.create_all_clients()?;
|
||||
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
|
||||
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
|
||||
if let Some(mixed) = mixed_completion {
|
||||
assert_eq!(
|
||||
completed_pending.version_id, mixed.version_id,
|
||||
"full upgrade changed the completed version"
|
||||
);
|
||||
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
|
||||
}
|
||||
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
|
||||
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
|
||||
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
|
||||
@@ -2518,7 +2451,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"phases": phases,
|
||||
"objects_per_phase": objects_per_phase,
|
||||
"mixed_load": "plain-put",
|
||||
"mixed_delete_protocol_supported": source.supports_delete_recovery,
|
||||
"mixed_delete_protocol_supported": previous_has_delete_recovery,
|
||||
"mixed_deleted_keys": mixed_deleted_keys,
|
||||
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
|
||||
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
|
||||
@@ -2528,7 +2461,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"all nodes list every mixed-version object after homogeneous-current convergence",
|
||||
"the previous-release seed survives the baseline restart and every upgrade phase",
|
||||
"previous readers read every current-native MPU body after quiescent rollback",
|
||||
"the same previous-release upload remains readable after full upgrade and rollback"
|
||||
"same legacy upload resumes with identical body, ETag and version after full upgrade"
|
||||
],
|
||||
}),
|
||||
)?;
|
||||
@@ -2550,13 +2483,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
|
||||
"previous-release writer publishes objects readable by current node",
|
||||
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
|
||||
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
|
||||
if source.supports_multipart_publication {
|
||||
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
|
||||
} else {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
|
||||
},
|
||||
"fully upgraded readers see the same body, ETag and sole version after replay",
|
||||
if source.supports_delete_recovery {
|
||||
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
|
||||
"the same legacy upload completes and replays once fully upgraded without adding a version",
|
||||
if previous_has_delete_recovery {
|
||||
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
|
||||
} else {
|
||||
"modern DELETE fails closed and preserves every byte while old peers remain"
|
||||
@@ -2903,7 +2832,7 @@ async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, ser
|
||||
env.start_rustfs_server_from_binary(binary, vec![], server_env).await
|
||||
}
|
||||
|
||||
pub(crate) async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
|
||||
async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
|
||||
let path = format!("/rustfs/admin/v3/quota/{bucket}");
|
||||
let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
|
||||
let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
|
||||
@@ -3457,8 +3386,6 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
|
||||
#[tokio::test]
|
||||
#[ignore = "requires a pinned previous RustFS release binary"]
|
||||
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
|
||||
use sha2::Digest;
|
||||
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
@@ -3496,16 +3423,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
let multipart_bytes = multipart_parts.concat();
|
||||
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
|
||||
|
||||
// This release can finish an upload staged by the current gateway.
|
||||
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
|
||||
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
|
||||
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
|
||||
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
|
||||
Some((pending, parts.concat()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
|
||||
let old_client = env.create_s3_client();
|
||||
|
||||
@@ -3521,16 +3438,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
|
||||
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
|
||||
|
||||
let rollback_multipart = if let Some((pending, expected)) = inflight {
|
||||
assert_pending_parts(&old_client, &pending).await?;
|
||||
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
|
||||
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(body, expected);
|
||||
Some((pending.part_sizes, body))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// A PUT on the rolled-back release must still honour the encryption
|
||||
// configuration it decoded out of the current build's metadata blob.
|
||||
let post_rollback_key = "rollback/written-after-rollback";
|
||||
@@ -3571,25 +3478,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
post_rollback_bytes
|
||||
);
|
||||
|
||||
if let Some((part_sizes, rollback_read)) = rollback_multipart {
|
||||
let (_, current_read) = read_object(¤t_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
|
||||
assert_eq!(current_read, rollback_read);
|
||||
if let Some(context) = &evidence_context {
|
||||
let evidence = serde_json::json!({
|
||||
"previous_revision": context.previous_revision,
|
||||
"current_revision": context.current_revision,
|
||||
"part_sizes": part_sizes,
|
||||
"bytes": current_read.len(),
|
||||
"rollback_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(&rollback_read), hex_simd::AsciiCase::Lower),
|
||||
"roll_forward_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(¤t_read), hex_simd::AsciiCase::Lower),
|
||||
});
|
||||
std::fs::write(
|
||||
context.directory.join("inflight-multipart-rollback.json"),
|
||||
serde_json::to_vec_pretty(&evidence)?,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(context) = evidence_context.as_ref() {
|
||||
write_g09_evidence(
|
||||
context,
|
||||
|
||||
@@ -258,6 +258,7 @@ windows-sys = { workspace = true, features = [
|
||||
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
|
||||
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
aws-smithy-async.workspace = true
|
||||
rustfs-s3-ops.workspace = true
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
|
||||
|
||||
@@ -25,8 +25,10 @@
|
||||
//! Not responsible for: validating configurations, touching the bucket
|
||||
//! metadata store, or choosing a canonical shape.
|
||||
//!
|
||||
//! The tests exercise the production bucket configuration codec through `tests/storage_api.rs`.
|
||||
//! The lifecycle date cases also pin the results of the legacy codec. The DTO/codec
|
||||
//! Upstream: the bucket-config `deserialize`/`serialize` pair, reached through the
|
||||
//! integration-test facade `tests/storage_api.rs`, and the configuration types they are
|
||||
//! generic over (`BucketConfigXml`: the gateway persistence shapes for the families that
|
||||
//! have moved, the s3s DTOs for the rest). Downstream: the DTO/codec
|
||||
//! replacement (T1.4) and the persisted-bytes comparison (T3.7), which must
|
||||
//! keep every golden green without regenerating it. Regeneration is reserved
|
||||
//! to `scripts/gen_bucket_config_goldens.sh`, which sets
|
||||
@@ -504,12 +506,13 @@ mod bucket_config_goldens {
|
||||
|
||||
mod bucket_config_lifecycle_date_compat {
|
||||
use super::*;
|
||||
// Captured from the 5761ddfe reader and writer. The reader preserves the offset and
|
||||
// nanoseconds. The writer emits the local clock fields at millisecond precision with a Z.
|
||||
const DATES: [(&str, &str); 3] = [
|
||||
("2023-11-14T22:13:20.123456789Z", "2023-11-14T22:13:20.123Z"),
|
||||
("2023-11-14T22:13:20.123456789+05:30", "2023-11-14T22:13:20.123Z"),
|
||||
("2025-01-02T00:00:00+02:00", "2025-01-02T00:00:00.000Z"),
|
||||
use rustfs_lifecycle::core::format_lifecycle_date;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
const DATES: [&str; 3] = [
|
||||
"2023-11-14T22:13:20.123456789Z",
|
||||
"2023-11-14T22:13:20.123456789+05:30",
|
||||
"2025-01-02T00:00:00+02:00",
|
||||
];
|
||||
|
||||
fn document(date: &str) -> Vec<u8> {
|
||||
@@ -519,17 +522,33 @@ mod bucket_config_lifecycle_date_compat {
|
||||
.into_bytes()
|
||||
}
|
||||
|
||||
fn legacy_read(input: &[u8]) -> s3s::dto::BucketLifecycleConfiguration {
|
||||
let mut reader = s3s::xml::Deserializer::new(input);
|
||||
let value = s3s::xml::Deserialize::deserialize(&mut reader).expect("the actual old lifecycle reader accepts the date");
|
||||
reader.expect_eof().expect("the old reader consumes the entire date fixture");
|
||||
value
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn lifecycle_xml_reader_preserves_legacy_date_precision_and_offset() {
|
||||
for (date, _) in DATES {
|
||||
let new: PersistedLifecycleConfiguration = deserialize(&document(date)).expect("read persisted lifecycle date");
|
||||
for date in DATES {
|
||||
let input = document(date);
|
||||
let old = legacy_read(&input);
|
||||
let new: PersistedLifecycleConfiguration = deserialize(&input).expect("read persisted lifecycle date");
|
||||
let old_date = |value: &s3s::dto::Timestamp| {
|
||||
format_lifecycle_date(OffsetDateTime::from(value.clone())).expect("format the original old-reader date")
|
||||
};
|
||||
assert_eq!(
|
||||
[
|
||||
new.expiry_updated_at.clone(),
|
||||
new.rules[0].expiration.as_ref().and_then(|value| value.date.clone()),
|
||||
new.rules[0].transitions.as_ref().unwrap()[0].date.clone(),
|
||||
],
|
||||
[Some(date.to_owned()), Some(date.to_owned()), Some(date.to_owned())],
|
||||
[
|
||||
Some(old_date(old.expiry_updated_at.as_ref().unwrap())),
|
||||
Some(old_date(old.rules[0].expiration.as_ref().unwrap().date.as_ref().unwrap())),
|
||||
Some(old_date(old.rules[0].transitions.as_ref().unwrap()[0].date.as_ref().unwrap())),
|
||||
],
|
||||
"the lifecycle reader must retain the old date representation for {date}"
|
||||
);
|
||||
}
|
||||
@@ -537,10 +556,11 @@ mod bucket_config_lifecycle_date_compat {
|
||||
|
||||
#[test]
|
||||
fn lifecycle_xml_writer_preserves_legacy_date_bytes() {
|
||||
for (date, written_date) in DATES {
|
||||
let expected = format!(
|
||||
"<LifecycleConfiguration><ExpiryUpdatedAt>{written_date}</ExpiryUpdatedAt><Rule><Expiration><Date>{written_date}</Date></Expiration><ID>date-compat</ID><Status>Enabled</Status><Transition><Date>{written_date}</Date><StorageClass>GLACIER</StorageClass></Transition></Rule></LifecycleConfiguration>"
|
||||
);
|
||||
for date in DATES {
|
||||
let old = legacy_read(&document(date));
|
||||
let mut old_xml = Vec::new();
|
||||
s3s::xml::Serialize::serialize(&old, &mut s3s::xml::Serializer::new(&mut old_xml))
|
||||
.expect("the actual old lifecycle writer emits the date");
|
||||
let new = PersistedLifecycleConfiguration {
|
||||
expiry_updated_at: Some(date.to_owned()),
|
||||
rules: vec![PersistedLifecycleRule {
|
||||
@@ -565,7 +585,7 @@ mod bucket_config_lifecycle_date_compat {
|
||||
};
|
||||
assert_eq!(
|
||||
serialize(&new).expect("write persisted lifecycle date"),
|
||||
expected.as_bytes(),
|
||||
old_xml,
|
||||
"legacy date bytes for {date}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1786,7 +1786,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None)
|
||||
.await
|
||||
.map(|(updated_at, status, _guard)| (updated_at, status))
|
||||
}
|
||||
@@ -1796,7 +1796,6 @@ where
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(
|
||||
OffsetDateTime,
|
||||
AccountStatus,
|
||||
@@ -1825,51 +1824,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order is parent, then the presenting STS credential. Deletion and
|
||||
// session refresh use the same STS lock. Release it before the revocation sweep.
|
||||
let _session_guard = match session {
|
||||
Some(session) => {
|
||||
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
|
||||
let current = self
|
||||
.api
|
||||
.load_user_identity(&session.access_key, UserType::Sts)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if is_err_no_such_user(&error) {
|
||||
Error::IAMActionNotAllowed
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let parent = if current.credentials.parent_user.is_empty() {
|
||||
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
|
||||
} else {
|
||||
Some(current.credentials.parent_user.as_str())
|
||||
};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
if !session.is_valid()
|
||||
|| !current.credentials.is_valid()
|
||||
|| !current.credentials.is_temp()
|
||||
|| current.credentials.is_service_account()
|
||||
|| current.credentials.access_key != session.access_key
|
||||
|| parent != Some(access_key)
|
||||
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|
||||
|| !bool::from(
|
||||
current
|
||||
.credentials
|
||||
.session_token
|
||||
.as_bytes()
|
||||
.ct_eq(session.session_token.as_bytes()),
|
||||
)
|
||||
|| current.sts_generation != u.sts_generation
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Some(guard)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
|
||||
+6
-183
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
|
||||
}
|
||||
|
||||
/// Change an enabled regular user's secret only if the supplied current
|
||||
/// secret still matches under the parent mutation lock. Status and policies are kept.
|
||||
/// Change an enabled regular caller's secret only if its signing secret is
|
||||
/// still current under the parent mutation lock. Status and policies are kept.
|
||||
/// Persistence and revocation continue after caller cancellation.
|
||||
pub async fn change_own_secret_key(
|
||||
&self,
|
||||
@@ -1433,27 +1433,7 @@ impl<T: Store> IamSys<T> {
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Change a local IAM password using an authenticated STS session and its parent password.
|
||||
///
|
||||
/// The parent and session mutation locks protect revalidation through the secret write.
|
||||
/// Persistence and session revocation continue after request cancellation.
|
||||
/// A revocation failure is returned after the new secret has been persisted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
|
||||
/// incorrect current password. Storage and revocation errors are propagated.
|
||||
pub async fn change_own_secret_key_from_session(
|
||||
&self,
|
||||
access_key: &str,
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
session: &Credentials,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1462,18 +1442,16 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
let access_key = access_key.to_string();
|
||||
let secret_key = secret_key.to_string();
|
||||
let expected_secret = expected_secret.map(str::to_string);
|
||||
let session = session.cloned();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
|
||||
.await
|
||||
};
|
||||
#[cfg(test)]
|
||||
@@ -1493,7 +1471,6 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1505,7 +1482,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, status, parent_guard) = self
|
||||
.store
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
|
||||
.await?;
|
||||
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
drop(parent_guard);
|
||||
@@ -4838,160 +4815,6 @@ mod tests {
|
||||
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
|
||||
ensure_test_global_credentials();
|
||||
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
|
||||
let parent = "session-change-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
|
||||
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
|
||||
let pending = {
|
||||
let iam = Arc::clone(&iam);
|
||||
let session = session.clone();
|
||||
tokio::spawn(async move {
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
|
||||
.await
|
||||
})
|
||||
};
|
||||
if mutation == "revoke" {
|
||||
iam.delete_temp_account(&session.access_key, false)
|
||||
.await
|
||||
.expect("revoke presenting session");
|
||||
} else {
|
||||
let mut saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
let current = saved.get_mut(&session.access_key).expect("durable session");
|
||||
match mutation {
|
||||
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
|
||||
"disable" => current.credentials.status = "off".to_string(),
|
||||
"parent" => current.credentials.parent_user = "another-parent".to_string(),
|
||||
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
|
||||
"token" => current.credentials.session_token = "replacement-token".to_string(),
|
||||
"generation" => {
|
||||
assert!(current.sts_generation.take().is_some());
|
||||
}
|
||||
_ => unreachable!("known fixture mutation"),
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
|
||||
.await
|
||||
.expect("queued rotation finishes")
|
||||
.expect("rotation task")
|
||||
.expect_err("stale session must fail");
|
||||
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
|
||||
let saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "stale-password-parent";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "cachedParentSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "fresh-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
iam.store
|
||||
.api
|
||||
.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved identities")
|
||||
.get_mut(parent)
|
||||
.expect("durable parent")
|
||||
.credentials
|
||||
.secret_key = "durableParentSecret123".to_string();
|
||||
assert_eq!(
|
||||
iam.store
|
||||
.get_user(parent)
|
||||
.await
|
||||
.expect("stale cached parent")
|
||||
.credentials
|
||||
.secret_key,
|
||||
"cachedParentSecret123"
|
||||
);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
|
||||
)
|
||||
.await
|
||||
.expect("durable current password authorizes the change");
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "current-session-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "current-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let (_, _, revoked) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("rotation must release its session lock before revocation")
|
||||
.expect("rotate current session");
|
||||
assert_eq!(revoked, 1);
|
||||
assert!(iam.get_user(&session.access_key).await.is_none());
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -6,7 +6,7 @@ Applies to `crates/kms/`.
|
||||
|
||||
When changing key-management behavior, verify compatibility with:
|
||||
|
||||
- `rustfs/src/storage/sse.rs`
|
||||
- `rustfs/src/storage/ecfs.rs`
|
||||
- `rustfs/src/admin/handlers/kms.rs`
|
||||
- `rustfs/src/admin/handlers/kms_dynamic.rs`
|
||||
- `rustfs/src/admin/handlers/kms_keys.rs`
|
||||
|
||||
@@ -28,7 +28,6 @@ use tracing_subscriber::{
|
||||
use super::http2_ping::HYPER_HTTP2_CLIENT_TARGET;
|
||||
|
||||
pub const HTTP_SERVER_LOG_TARGET: &str = "rustfs::server::http";
|
||||
const S3_DEPENDENCY_TARGETS: [&str; 2] = ["s3s", "rustfs_gateway"];
|
||||
|
||||
/// Pyroscope emits raw reqwest errors from its background session manager.
|
||||
/// Those errors can include the configured endpoint, so they must never reach
|
||||
@@ -45,13 +44,12 @@ pub(super) fn pyroscope_log_filter() -> FilterFn {
|
||||
filter_fn(allows_non_pyroscope_dependency_event)
|
||||
}
|
||||
|
||||
/// Keep verbose protocol events out of logs and OTLP while preserving diagnostic spans.
|
||||
pub(super) fn s3_dependency_log_filter() -> FilterFn {
|
||||
/// s3s verbose events include response bodies and expected signatures. Keep
|
||||
/// diagnostics spans, but never let these events reach logs or OTLP exporters.
|
||||
pub(super) fn s3s_log_filter() -> FilterFn {
|
||||
filter_fn(|metadata| {
|
||||
!(metadata.is_event()
|
||||
&& S3_DEPENDENCY_TARGETS
|
||||
.iter()
|
||||
.any(|target| directive_applies_to_target(target, metadata.target()))
|
||||
&& (metadata.target() == "s3s" || metadata.target().starts_with("s3s::"))
|
||||
&& *metadata.level() > tracing::Level::INFO)
|
||||
})
|
||||
}
|
||||
@@ -62,7 +60,7 @@ pub(super) fn s3_dependency_log_filter() -> FilterFn {
|
||||
/// `RUST_LOG` environment variable takes precedence over `logger_level`.
|
||||
/// For non-verbose levels (`info`, `warn`, `error`), noisy dependency targets
|
||||
/// are automatically suppressed based on the effective log configuration.
|
||||
/// Transport internals are disabled, while S3 protocol warnings remain visible. The
|
||||
/// Transport internals are disabled, while `s3s` remains visible at WARN. The
|
||||
/// diagnostic HTTP/2 PING feature adds back only Hyper's client target at DEBUG
|
||||
/// so its exact keepalive timeout event reaches the metrics layer.
|
||||
///
|
||||
@@ -181,14 +179,33 @@ fn effective_level_for_target<'a>(rust_log: &'a str, target: &str) -> Option<&'a
|
||||
best_match.map(|(_, _, level)| level)
|
||||
}
|
||||
|
||||
fn should_demote_target(target: &str, logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
|
||||
fn should_demote_http_request_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
|
||||
if let Some(level) = default_level {
|
||||
let level = level.trim().to_ascii_lowercase();
|
||||
return matches!(level.as_str(), "info" | "warn");
|
||||
}
|
||||
|
||||
if let Some(rust_log) = rust_log {
|
||||
if let Some(level) = effective_level_for_target(rust_log, target) {
|
||||
if let Some(level) = effective_level_for_target(rust_log, HTTP_SERVER_LOG_TARGET) {
|
||||
let level = level.trim().to_ascii_lowercase();
|
||||
return matches!(level.as_str(), "info" | "warn");
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
let level = logger_level.trim().to_ascii_lowercase();
|
||||
matches!(level.as_str(), "info" | "warn")
|
||||
}
|
||||
|
||||
fn should_demote_s3s_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
|
||||
if let Some(level) = default_level {
|
||||
let level = level.trim().to_ascii_lowercase();
|
||||
return matches!(level.as_str(), "info" | "warn");
|
||||
}
|
||||
|
||||
if let Some(rust_log) = rust_log {
|
||||
if let Some(level) = effective_level_for_target(rust_log, "s3s") {
|
||||
let level = level.trim().to_ascii_lowercase();
|
||||
return matches!(level.as_str(), "info" | "warn");
|
||||
}
|
||||
@@ -242,16 +259,17 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
|
||||
("tower", LevelFilter::OFF),
|
||||
];
|
||||
|
||||
if should_demote_target(HTTP_SERVER_LOG_TARGET, logger_level, default_level, rust_log_env.as_deref()) {
|
||||
if should_demote_http_request_logs(logger_level, default_level, rust_log_env.as_deref()) {
|
||||
// HTTP request logs are demoted to WARN to reduce volume in production,
|
||||
// but only when the effective log level is not stricter than WARN.
|
||||
directives.push((HTTP_SERVER_LOG_TARGET, LevelFilter::WARN));
|
||||
}
|
||||
|
||||
for target in S3_DEPENDENCY_TARGETS {
|
||||
if should_demote_target(target, logger_level, default_level, rust_log_env.as_deref()) {
|
||||
directives.push((target, LevelFilter::WARN));
|
||||
}
|
||||
if should_demote_s3s_logs(logger_level, default_level, rust_log_env.as_deref()) {
|
||||
// s3s instruments each authentication request through multiple
|
||||
// nested INFO spans. Keep warnings and errors in normal production
|
||||
// logs without promoting it when the effective base is stricter.
|
||||
directives.push(("s3s", LevelFilter::WARN));
|
||||
}
|
||||
|
||||
for (crate_name, level) in directives {
|
||||
@@ -345,14 +363,14 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3_dependency_filter_blocks_verbose_events_despite_explicit_directives() {
|
||||
fn s3s_filter_blocks_credentials_despite_explicit_verbose_directives() {
|
||||
use tracing_subscriber::prelude::*;
|
||||
for directive in ["trace", concat!("trace,", "s3s::service=trace,", "s3s::ops::signature=debug")] {
|
||||
let writer = SharedWriter::default();
|
||||
let captured = Arc::clone(&writer.0);
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(EnvFilter::new(directive))
|
||||
.with(s3_dependency_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.without_time()
|
||||
@@ -366,10 +384,6 @@ mod tests {
|
||||
tracing::debug!(target: "s3s::service", response = "secret-response", "response dump");
|
||||
tracing::trace!(target: "s3s", response = "secret-trace", "response dump");
|
||||
tracing::debug!(target: "s3s::ops::signature", expected = "secret-signature", "signature mismatch");
|
||||
tracing::debug!(target: "rustfs_gateway::service", "secret-gateway-debug");
|
||||
tracing::trace!(target: "rustfs_gateway", "secret-gateway-trace");
|
||||
tracing::info!(target: "rustfs_gateway::service", "visible-gateway-info");
|
||||
tracing::debug!(target: "rustfs_gateway_other", "visible-gateway-neighbor");
|
||||
tracing::info!(target: "s3s::service", "visible-info");
|
||||
tracing::warn!(target: "s3s::service", "visible-warning");
|
||||
tracing::debug!(target: "rustfs", "visible-rustfs-debug");
|
||||
@@ -379,8 +393,6 @@ mod tests {
|
||||
assert!(!output.contains("secret"), "{output}");
|
||||
for expected in [
|
||||
"safe-operation-span",
|
||||
"visible-gateway-info",
|
||||
"visible-gateway-neighbor",
|
||||
"visible-info",
|
||||
"visible-warning",
|
||||
"visible-rustfs-debug",
|
||||
@@ -433,29 +445,22 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_should_demote_http_request_logs() {
|
||||
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, None));
|
||||
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "warn", None, None));
|
||||
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, None));
|
||||
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "off", None, None));
|
||||
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("ERROR")));
|
||||
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, Some("WARN")));
|
||||
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("foo=warn")));
|
||||
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs=error")));
|
||||
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs::server=error")));
|
||||
assert!(!should_demote_target(
|
||||
HTTP_SERVER_LOG_TARGET,
|
||||
"info",
|
||||
None,
|
||||
Some(&http_log_directive("error"))
|
||||
));
|
||||
assert!(!should_demote_target(
|
||||
HTTP_SERVER_LOG_TARGET,
|
||||
assert!(should_demote_http_request_logs("info", None, None));
|
||||
assert!(should_demote_http_request_logs("warn", None, None));
|
||||
assert!(!should_demote_http_request_logs("error", None, None));
|
||||
assert!(!should_demote_http_request_logs("off", None, None));
|
||||
assert!(!should_demote_http_request_logs("info", None, Some("ERROR")));
|
||||
assert!(should_demote_http_request_logs("error", None, Some("WARN")));
|
||||
assert!(!should_demote_http_request_logs("info", None, Some("foo=warn")));
|
||||
assert!(!should_demote_http_request_logs("info", None, Some("rustfs=error")));
|
||||
assert!(!should_demote_http_request_logs("info", None, Some("rustfs::server=error")));
|
||||
assert!(!should_demote_http_request_logs("info", None, Some(&http_log_directive("error"))));
|
||||
assert!(!should_demote_http_request_logs(
|
||||
"info",
|
||||
None,
|
||||
Some(&format!("WARN,{}", http_log_directive("error")))
|
||||
));
|
||||
assert!(should_demote_target(
|
||||
HTTP_SERVER_LOG_TARGET,
|
||||
assert!(should_demote_http_request_logs(
|
||||
"error",
|
||||
None,
|
||||
Some(&format!("WARN,{}", http_log_directive("warn")))
|
||||
@@ -464,15 +469,15 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_should_demote_s3s_logs() {
|
||||
assert!(should_demote_target("s3s", "info", None, None));
|
||||
assert!(should_demote_target("s3s", "warn", None, None));
|
||||
assert!(!should_demote_target("s3s", "error", None, None));
|
||||
assert!(!should_demote_target("s3s", "off", None, None));
|
||||
assert!(!should_demote_target("s3s", "info", None, Some("ERROR")));
|
||||
assert!(should_demote_target("s3s", "error", None, Some("WARN")));
|
||||
assert!(!should_demote_target("s3s", "info", None, Some("foo=warn")));
|
||||
assert!(!should_demote_target("s3s", "info", None, Some("s3s=error")));
|
||||
assert!(should_demote_target("s3s", "error", None, Some("WARN,s3s=warn")));
|
||||
assert!(should_demote_s3s_logs("info", None, None));
|
||||
assert!(should_demote_s3s_logs("warn", None, None));
|
||||
assert!(!should_demote_s3s_logs("error", None, None));
|
||||
assert!(!should_demote_s3s_logs("off", None, None));
|
||||
assert!(!should_demote_s3s_logs("info", None, Some("ERROR")));
|
||||
assert!(should_demote_s3s_logs("error", None, Some("WARN")));
|
||||
assert!(!should_demote_s3s_logs("info", None, Some("foo=warn")));
|
||||
assert!(!should_demote_s3s_logs("info", None, Some("s3s=error")));
|
||||
assert!(should_demote_s3s_logs("error", None, Some("WARN,s3s=warn")));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -614,14 +619,10 @@ mod tests {
|
||||
let output = capture_with_filter(build_env_filter("info", None), || {
|
||||
tracing::info!(target: "s3s::auth", "s3s info must be suppressed");
|
||||
tracing::warn!(target: "s3s::auth", "s3s warning must remain");
|
||||
tracing::info!(target: "rustfs_gateway::service", "gateway info must be suppressed");
|
||||
tracing::warn!(target: "rustfs_gateway::service", "gateway warning must remain");
|
||||
});
|
||||
|
||||
assert!(!output.contains("s3s info must be suppressed"), "{output}");
|
||||
assert!(output.contains("s3s warning must remain"), "{output}");
|
||||
assert!(!output.contains("gateway info must be suppressed"), "{output}");
|
||||
assert!(output.contains("gateway warning must remain"), "{output}");
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
|
||||
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
|
||||
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3_dependency_log_filter};
|
||||
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3s_log_filter};
|
||||
use crate::telemetry::rolling::{RollingAppender, Rotation};
|
||||
use metrics::counter;
|
||||
use rustfs_config::observability::{
|
||||
@@ -423,7 +423,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
.with(env_filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3_dependency_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(fmt_layer)
|
||||
.try_init()
|
||||
@@ -537,7 +537,7 @@ fn init_file_logging_internal(
|
||||
.with(env_filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3_dependency_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer)
|
||||
.with(stdout_layer)
|
||||
|
||||
@@ -39,7 +39,7 @@
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::set_observability_metric_enabled;
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3_dependency_log_filter};
|
||||
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3s_log_filter};
|
||||
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
|
||||
use crate::telemetry::local::{build_json_log_layer, open_rolling_appender, spawn_cleanup_task};
|
||||
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
|
||||
@@ -319,7 +319,7 @@ pub(super) fn init_observability_http(
|
||||
.with(filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3_dependency_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
.with(file_layer_opt)
|
||||
.with(stdout_layer_opt)
|
||||
@@ -1361,7 +1361,7 @@ mod tests {
|
||||
let subscriber = tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3_dependency_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(
|
||||
tracing_subscriber::fmt::layer()
|
||||
.with_ansi(false)
|
||||
@@ -1385,7 +1385,7 @@ mod tests {
|
||||
tracing_subscriber::registry()
|
||||
.with(build_env_filter("info", None))
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3_dependency_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
|
||||
.try_init()
|
||||
.expect("install isolated LogTracer subscriber");
|
||||
|
||||
@@ -13,7 +13,7 @@ Applies to `crates/policy/`.
|
||||
- Validate compatibility with IAM and request-auth call sites:
|
||||
- `crates/iam/`
|
||||
- `rustfs/src/auth.rs`
|
||||
- `rustfs/src/storage/authorization/`
|
||||
- `rustfs/src/storage/access.rs`
|
||||
|
||||
## Error and Observability
|
||||
|
||||
|
||||
@@ -70,6 +70,10 @@ url.workspace = true
|
||||
urlencoding = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
|
||||
|
||||
# The remote error code table is checked against the legacy client's.
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
|
||||
@@ -106,7 +106,7 @@ impl fmt::Display for RemoteErrorCode {
|
||||
}
|
||||
}
|
||||
|
||||
/// The sorted wire codes recognized by the legacy client at revision 5761ddfe.
|
||||
/// The codes the legacy client (s3s at the pinned revision) recognized, sorted.
|
||||
const RECOGNIZED_CODES: &[&str] = &[
|
||||
"AccessControlListNotSupported",
|
||||
"AccessDenied",
|
||||
@@ -352,19 +352,15 @@ const RECOGNIZED_CODES: &[&str] = &[
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[test]
|
||||
fn recognizes_exactly_the_codes_the_legacy_client_named() {
|
||||
assert_eq!(RECOGNIZED_CODES.len(), 239);
|
||||
assert!(RECOGNIZED_CODES.windows(2).all(|pair| pair[0] < pair[1]), "the table must stay sorted");
|
||||
// SHA-256 of the sorted, newline-separated wire names in S3_ERROR_CODE_MAP at 5761ddfe.
|
||||
let digest = Sha256::digest(RECOGNIZED_CODES.join("\n"));
|
||||
assert_eq!(
|
||||
hex_simd::encode_to_string(digest.as_slice(), hex_simd::AsciiCase::Lower),
|
||||
"6fa6be09aa53ae2e05c5c98c10c43dae79713b07a2bdde492dbb6a776888978d"
|
||||
);
|
||||
for code in RECOGNIZED_CODES {
|
||||
let legacy = s3s::S3ErrorCode::from_bytes(code.as_bytes()).expect("utf-8");
|
||||
assert!(!matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{code} is not named by the legacy client");
|
||||
assert_eq!(legacy.as_str(), *code);
|
||||
let parsed = RemoteErrorCode::from_wire(code);
|
||||
assert!(!parsed.is_custom(), "{code}");
|
||||
assert_eq!(parsed.as_str(), *code);
|
||||
@@ -373,18 +369,19 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn parses_like_the_legacy_client() {
|
||||
for (text, expected, custom) in [
|
||||
("NoSuchKey", "NoSuchKey", false),
|
||||
("nosuchkey", "NoSuchKey", false),
|
||||
("NOSUCHKEY", "NoSuchKey", false),
|
||||
("PermanentRedirect", "PermanentRedirect", false),
|
||||
("SomethingNew", "SomethingNew", true),
|
||||
("", "", true),
|
||||
("No Such Key", "No Such Key", true),
|
||||
for text in [
|
||||
"NoSuchKey",
|
||||
"nosuchkey",
|
||||
"NOSUCHKEY",
|
||||
"PermanentRedirect",
|
||||
"SomethingNew",
|
||||
"",
|
||||
"No Such Key",
|
||||
] {
|
||||
let legacy = s3s::S3ErrorCode::from_bytes(text.as_bytes()).expect("utf-8");
|
||||
let parsed = RemoteErrorCode::from_wire(text);
|
||||
assert_eq!(parsed.is_custom(), custom, "{text:?}");
|
||||
assert_eq!(parsed.as_str(), expected, "{text:?}");
|
||||
assert_eq!(parsed.is_custom(), matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{text:?}");
|
||||
assert_eq!(parsed.as_str(), legacy.as_str(), "{text:?}");
|
||||
}
|
||||
assert_eq!(RemoteErrorCode::from_wire("nosuchkey"), RemoteErrorCode::NO_SUCH_KEY);
|
||||
}
|
||||
|
||||
@@ -164,10 +164,10 @@ impl From<legacy_s3s::S3Error> for S3Error {
|
||||
/// reference only, so the whole error travels inside and this wrapper presents
|
||||
/// the cause's own text and chain, as if the cause itself had been moved.
|
||||
#[derive(Debug)]
|
||||
pub(super) struct LegacyCause(legacy_s3s::S3Error);
|
||||
struct LegacyCause(legacy_s3s::S3Error);
|
||||
|
||||
impl LegacyCause {
|
||||
pub(super) fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
|
||||
fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
|
||||
self.0.source()
|
||||
}
|
||||
}
|
||||
@@ -378,8 +378,6 @@ mod tests {
|
||||
legacy.set_source(Box::new(io::Error::other("read quorum")));
|
||||
|
||||
let mut ours: S3Error = legacy.into();
|
||||
assert!(ours.source().is_some_and(|cause| cause.is::<io::Error>()));
|
||||
assert!(std::error::Error::source(&ours).is_some_and(|cause| cause.is::<io::Error>()));
|
||||
ours.set_message("retry later".to_owned());
|
||||
let back: legacy_s3s::S3Error = ours.into();
|
||||
assert_eq!(back.code(), &legacy_s3s::S3ErrorCode::ServiceUnavailable);
|
||||
|
||||
@@ -157,12 +157,7 @@ impl S3Error {
|
||||
|
||||
/// The cause, if one was attached.
|
||||
pub fn source(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
|
||||
let source = self.0.source.as_deref()?;
|
||||
#[cfg(any(test, feature = "compat-s3s"))]
|
||||
if let Some(legacy) = source.downcast_ref::<crate::compat_s3s::LegacyCause>() {
|
||||
return legacy.cause();
|
||||
}
|
||||
Some(source)
|
||||
self.0.source.as_deref()
|
||||
}
|
||||
|
||||
/// The explicit status if one was set, otherwise the code's own status;
|
||||
@@ -207,7 +202,7 @@ impl fmt::Display for S3Error {
|
||||
|
||||
impl std::error::Error for S3Error {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
S3Error::source(self).map(|source| source as _)
|
||||
self.0.source.as_deref().map(|source| source as _)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -109,6 +109,7 @@ bytes.workspace = true
|
||||
hex-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
s3s = { workspace = true, features = ["minio"] }
|
||||
opentelemetry-proto.workspace = true
|
||||
opentelemetry_sdk.workspace = true
|
||||
prost.workspace = true
|
||||
|
||||
@@ -1341,9 +1341,32 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() {
|
||||
|
||||
#[test]
|
||||
fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after() {
|
||||
// Seven-field tuple written with the pinned legacy scanner codec.
|
||||
let old_info = b"\x97\xa6bucket\x07\xc0\xc3\xc0\xc0\x81\xaabad-object\x0b";
|
||||
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(old_info).expect("decode the historical cache info tuple");
|
||||
#[derive(Serialize)]
|
||||
struct OldDataUsageCacheInfo {
|
||||
name: String,
|
||||
next_cycle: u64,
|
||||
last_update: Option<SystemTime>,
|
||||
skip_healing: bool,
|
||||
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
|
||||
replication: Option<Arc<ReplicationConfig>>,
|
||||
failed_objects: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
let old_info = OldDataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
last_update: None,
|
||||
skip_healing: true,
|
||||
lifecycle: None,
|
||||
replication: None,
|
||||
failed_objects: HashMap::from([("bad-object".to_string(), 11)]),
|
||||
};
|
||||
let mut buf = Vec::new();
|
||||
old_info
|
||||
.serialize(&mut rmp_serde::Serializer::new(&mut buf))
|
||||
.expect("Failed to serialize old cache info");
|
||||
|
||||
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(&buf).expect("Failed to deserialize old cache info");
|
||||
|
||||
assert_eq!(decoded.name, "bucket");
|
||||
assert_eq!(decoded.next_cycle, 7);
|
||||
@@ -1368,13 +1391,13 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
|
||||
next_cycle: u64,
|
||||
last_update: Option<SystemTime>,
|
||||
skip_healing: bool,
|
||||
lifecycle: Option<()>,
|
||||
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
|
||||
replication: Option<Arc<ReplicationConfig>>,
|
||||
failed_objects: HashMap<String, u64>,
|
||||
scan_resume_after: Option<String>,
|
||||
scan_checkpoint: Option<DataUsageScanCheckpoint>,
|
||||
pending_heals: Vec<PendingScannerHeal>,
|
||||
object_lock: Option<()>,
|
||||
object_lock: Option<Arc<s3s::dto::ObjectLockConfiguration>>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -3537,21 +3560,22 @@ const LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD: &[u8] = &[
|
||||
0x62, 0x6c, 0x65, 0x64, 0xc0,
|
||||
];
|
||||
|
||||
// Project the seven original fields out of the production map. Keep the nested
|
||||
// MessagePack arrays and maps opaque so their bytes can be checked against the old writer.
|
||||
// Keep the old tuple carrier and payload explicit when the production field changes type.
|
||||
#[derive(serde::Serialize, serde::Deserialize)]
|
||||
struct LegacyLifecycleCacheInfo {
|
||||
name: String,
|
||||
next_cycle: u64,
|
||||
last_update: Option<SystemTime>,
|
||||
skip_healing: bool,
|
||||
lifecycle: Value,
|
||||
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
|
||||
replication: Option<Arc<ReplicationConfig>>,
|
||||
failed_objects: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
struct LifecycleCacheOracleCase {
|
||||
label: &'static str,
|
||||
old_tuple: Vec<u8>,
|
||||
old_payload: Vec<u8>,
|
||||
current: DataUsageCacheInfo,
|
||||
current_map: Vec<u8>,
|
||||
frozen_tuple: &'static [u8],
|
||||
@@ -3560,6 +3584,7 @@ struct LifecycleCacheOracleCase {
|
||||
}
|
||||
|
||||
fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
|
||||
use s3s::dto as legacy;
|
||||
use time::OffsetDateTime;
|
||||
let expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_456_789)
|
||||
.expect("the legacy expiration fixture has an exact timestamp");
|
||||
@@ -3568,6 +3593,136 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
|
||||
let expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_789_123)
|
||||
.expect("the legacy expiry update fixture has an exact timestamp");
|
||||
|
||||
let absent = legacy::LifecycleRule {
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
expiration: None,
|
||||
filter: None,
|
||||
id: None,
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
status: legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::ENABLED),
|
||||
transitions: None,
|
||||
};
|
||||
let mut empty = absent.clone();
|
||||
empty.id = Some(String::new());
|
||||
empty.prefix = Some(String::new());
|
||||
empty.status = legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::DISABLED);
|
||||
empty.abort_incomplete_multipart_upload = Some(legacy::AbortIncompleteMultipartUpload {
|
||||
days_after_initiation: None,
|
||||
});
|
||||
empty.del_marker_expiration = Some(legacy::DelMarkerExpiration { days: None });
|
||||
empty.expiration = Some(legacy::LifecycleExpiration {
|
||||
days: Some(0),
|
||||
..Default::default()
|
||||
});
|
||||
empty.filter = Some(legacy::LifecycleRuleFilter {
|
||||
and: Some(legacy::LifecycleRuleAndOperator {
|
||||
tags: Some(Vec::new()),
|
||||
..Default::default()
|
||||
}),
|
||||
tag: Some(legacy::Tag { key: None, value: None }),
|
||||
..Default::default()
|
||||
});
|
||||
empty.noncurrent_version_expiration = Some(legacy::NoncurrentVersionExpiration {
|
||||
noncurrent_days: None,
|
||||
newer_noncurrent_versions: Some(-2),
|
||||
});
|
||||
empty.noncurrent_version_transitions = Some(Vec::new());
|
||||
empty.transitions = Some(Vec::new());
|
||||
let mut absent_tags = absent.clone();
|
||||
absent_tags.filter = Some(legacy::LifecycleRuleFilter {
|
||||
and: Some(legacy::LifecycleRuleAndOperator::default()),
|
||||
..Default::default()
|
||||
});
|
||||
// Cached values must survive independently of lifecycle policy validation.
|
||||
let complete = legacy::BucketLifecycleConfiguration {
|
||||
expiry_updated_at: Some(expiry_updated_at.into()),
|
||||
rules: vec![
|
||||
legacy::LifecycleRule {
|
||||
abort_incomplete_multipart_upload: Some(legacy::AbortIncompleteMultipartUpload {
|
||||
days_after_initiation: Some(0),
|
||||
}),
|
||||
del_marker_expiration: Some(legacy::DelMarkerExpiration { days: Some(-1) }),
|
||||
expiration: Some(legacy::LifecycleExpiration {
|
||||
date: Some(expiration_date.into()),
|
||||
days: Some(-3),
|
||||
expired_object_delete_marker: Some(false),
|
||||
expired_object_all_versions: Some(false),
|
||||
}),
|
||||
filter: Some(legacy::LifecycleRuleFilter {
|
||||
cached_tags: Default::default(),
|
||||
and: Some(legacy::LifecycleRuleAndOperator {
|
||||
prefix: Some(String::new()),
|
||||
tags: Some(vec![
|
||||
legacy::Tag {
|
||||
key: Some(String::new()),
|
||||
value: Some(String::new()),
|
||||
},
|
||||
legacy::Tag {
|
||||
key: None,
|
||||
value: Some("value-without-key".to_string()),
|
||||
},
|
||||
legacy::Tag {
|
||||
key: Some("key-without-value".to_string()),
|
||||
value: None,
|
||||
},
|
||||
]),
|
||||
object_size_greater_than: Some(-1),
|
||||
object_size_less_than: Some(4096),
|
||||
}),
|
||||
prefix: Some(String::new()),
|
||||
tag: Some(legacy::Tag {
|
||||
key: Some("kind".to_string()),
|
||||
value: Some("archive".to_string()),
|
||||
}),
|
||||
object_size_greater_than: Some(0),
|
||||
object_size_less_than: Some(8192),
|
||||
}),
|
||||
id: Some("complete-legacy-rule".to_string()),
|
||||
noncurrent_version_expiration: Some(legacy::NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(-7),
|
||||
newer_noncurrent_versions: Some(0),
|
||||
}),
|
||||
noncurrent_version_transitions: Some(vec![
|
||||
legacy::NoncurrentVersionTransition {
|
||||
noncurrent_days: Some(0),
|
||||
newer_noncurrent_versions: Some(-1),
|
||||
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
|
||||
},
|
||||
legacy::NoncurrentVersionTransition {
|
||||
noncurrent_days: None,
|
||||
newer_noncurrent_versions: None,
|
||||
storage_class: None,
|
||||
},
|
||||
]),
|
||||
prefix: Some("legacy-prefix/".to_string()),
|
||||
status: legacy::ExpirationStatus::from_static("LegacyUnknownStatus"),
|
||||
transitions: Some(vec![
|
||||
legacy::Transition {
|
||||
date: Some(transition_date.into()),
|
||||
days: Some(0),
|
||||
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
|
||||
},
|
||||
legacy::Transition {
|
||||
date: None,
|
||||
days: None,
|
||||
storage_class: Some(legacy::TransitionStorageClass::from_static("")),
|
||||
},
|
||||
legacy::Transition {
|
||||
date: None,
|
||||
days: None,
|
||||
storage_class: None,
|
||||
},
|
||||
]),
|
||||
},
|
||||
empty,
|
||||
absent_tags,
|
||||
absent,
|
||||
],
|
||||
};
|
||||
|
||||
use rustfs_gateway_types::persistence as persisted;
|
||||
let domain_absent = persisted::PersistedLifecycleRule {
|
||||
abort_incomplete_multipart_upload: None,
|
||||
@@ -3714,12 +3869,17 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
|
||||
(
|
||||
"none",
|
||||
None,
|
||||
None,
|
||||
LEGACY_LIFECYCLE_NONE_TUPLE,
|
||||
LEGACY_LIFECYCLE_NONE_MAP,
|
||||
LEGACY_LIFECYCLE_NONE_PAYLOAD,
|
||||
),
|
||||
(
|
||||
"empty",
|
||||
Some(Arc::new(legacy::BucketLifecycleConfiguration {
|
||||
rules: Vec::new(),
|
||||
expiry_updated_at: None,
|
||||
})),
|
||||
Some(Arc::new(persisted::PersistedLifecycleConfiguration {
|
||||
rules: Vec::new(),
|
||||
expiry_updated_at: None,
|
||||
@@ -3730,6 +3890,7 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
|
||||
),
|
||||
(
|
||||
"nonempty",
|
||||
Some(Arc::new(complete)),
|
||||
Some(Arc::new(domain_complete)),
|
||||
LEGACY_LIFECYCLE_NONEMPTY_TUPLE,
|
||||
LEGACY_LIFECYCLE_NONEMPTY_MAP,
|
||||
@@ -3738,17 +3899,35 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
|
||||
];
|
||||
cases
|
||||
.into_iter()
|
||||
.map(|(label, lifecycle, frozen_tuple, frozen_map, frozen_payload)| {
|
||||
let current = DataUsageCacheInfo {
|
||||
.map(|(label, lifecycle, domain, frozen_tuple, frozen_map, frozen_payload)| {
|
||||
let old = LegacyLifecycleCacheInfo {
|
||||
name: "legacy-lifecycle-bucket".to_string(),
|
||||
next_cycle: 7,
|
||||
last_update: None,
|
||||
skip_healing: true,
|
||||
lifecycle,
|
||||
replication: None,
|
||||
failed_objects: HashMap::new(),
|
||||
};
|
||||
let mut old_tuple = Vec::new();
|
||||
old.serialize(&mut rmp_serde::Serializer::new(&mut old_tuple))
|
||||
.expect("the real old tuple carrier must encode");
|
||||
let old_payload = rmp_serde::to_vec(&old.lifecycle).expect("the real old lifecycle payload must encode");
|
||||
let current = DataUsageCacheInfo {
|
||||
name: old.name.clone(),
|
||||
next_cycle: old.next_cycle,
|
||||
last_update: old.last_update,
|
||||
skip_healing: old.skip_healing,
|
||||
lifecycle: domain,
|
||||
replication: None,
|
||||
failed_objects: old.failed_objects,
|
||||
..Default::default()
|
||||
};
|
||||
let current_map = rmp_serde::to_vec(¤t).expect("the actual production map writer must encode the fixture");
|
||||
LifecycleCacheOracleCase {
|
||||
label,
|
||||
old_tuple,
|
||||
old_payload,
|
||||
current,
|
||||
current_map,
|
||||
frozen_tuple,
|
||||
@@ -3761,36 +3940,57 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
|
||||
|
||||
#[test]
|
||||
fn test_lifecycle_cache_legacy_msgpack_oracle() {
|
||||
use time::OffsetDateTime;
|
||||
|
||||
for case in lifecycle_cache_oracle_cases() {
|
||||
let LifecycleCacheOracleCase {
|
||||
label,
|
||||
current,
|
||||
current_map,
|
||||
frozen_tuple,
|
||||
frozen_map,
|
||||
frozen_payload,
|
||||
} = case;
|
||||
let label = case.label;
|
||||
let old_tuple = case.old_tuple;
|
||||
let old_payload = case.old_payload;
|
||||
let current = case.current;
|
||||
let current_map = case.current_map;
|
||||
let frozen_tuple = case.frozen_tuple;
|
||||
let frozen_map = case.frozen_map;
|
||||
let frozen_payload = case.frozen_payload;
|
||||
println!("legacy_lifecycle_cache {label}: tuple={old_tuple:02x?}; map={current_map:02x?}; payload={old_payload:02x?}");
|
||||
let old_from_current: LegacyLifecycleCacheInfo =
|
||||
rmp_serde::from_slice(¤t_map).expect("the actual old reader must read the production map");
|
||||
if label == "nonempty" {
|
||||
let lifecycle = old_from_current
|
||||
.lifecycle
|
||||
.as_ref()
|
||||
.expect("the old reader must retain the nonempty lifecycle");
|
||||
let expiration = lifecycle.rules[0]
|
||||
.expiration
|
||||
.as_ref()
|
||||
.expect("the complete rule has expiration");
|
||||
let transition = &lifecycle.rules[0]
|
||||
.transitions
|
||||
.as_ref()
|
||||
.expect("the complete rule has transitions")[0];
|
||||
let expected_expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000)
|
||||
.expect("the legacy millisecond expiration date is representable");
|
||||
let expected_transition_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_001_987_000_000)
|
||||
.expect("the legacy millisecond transition date is representable");
|
||||
let expected_expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_000_000)
|
||||
.expect("the legacy millisecond expiry update is representable");
|
||||
assert_eq!(
|
||||
(
|
||||
expiration.date.clone().map(OffsetDateTime::from),
|
||||
transition.date.clone().map(OffsetDateTime::from),
|
||||
lifecycle.expiry_updated_at.clone().map(OffsetDateTime::from),
|
||||
),
|
||||
(
|
||||
Some(expected_expiration_date),
|
||||
Some(expected_transition_date),
|
||||
Some(expected_expiry_updated_at),
|
||||
),
|
||||
"cache serialization must retain the fixed legacy millisecond precision for Date and ExpiryUpdatedAt"
|
||||
);
|
||||
}
|
||||
let frozen_tuple_read: DataUsageCacheInfo =
|
||||
rmp_serde::from_slice(frozen_tuple).expect("the production reader must accept the fixed legacy tuple bytes");
|
||||
let frozen_map_read: DataUsageCacheInfo =
|
||||
rmp_serde::from_slice(frozen_map).expect("the production reader must accept the fixed legacy map bytes");
|
||||
if label == "nonempty" {
|
||||
let lifecycle = frozen_map_read.lifecycle.as_ref().expect("nonempty lifecycle");
|
||||
let rule = &lifecycle.rules[0];
|
||||
assert_eq!(
|
||||
[
|
||||
rule.expiration.as_ref().expect("expiration").date.as_deref(),
|
||||
rule.transitions.as_ref().expect("transitions")[0].date.as_deref(),
|
||||
lifecycle.expiry_updated_at.as_deref(),
|
||||
],
|
||||
[
|
||||
Some("2023-11-14T22:13:20.123Z"),
|
||||
Some("2023-11-14T22:13:21.987Z"),
|
||||
Some("2023-11-14T22:13:22.456Z"),
|
||||
],
|
||||
"legacy timestamps retain millisecond precision",
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
[
|
||||
rmp_serde::to_vec(&frozen_tuple_read).expect("encode the fixed legacy tuple after production read"),
|
||||
@@ -3814,21 +4014,21 @@ fn test_lifecycle_cache_legacy_msgpack_oracle() {
|
||||
fn test_lifecycle_cache_legacy_carrier_round_trips() {
|
||||
for case in lifecycle_cache_oracle_cases() {
|
||||
let label = case.label;
|
||||
let old_tuple = case.frozen_tuple;
|
||||
let old_tuple = case.old_tuple;
|
||||
let current_map = case.current_map;
|
||||
let from_old: DataUsageCacheInfo =
|
||||
rmp_serde::from_slice(old_tuple).expect("the production reader must read the old tuple");
|
||||
rmp_serde::from_slice(&old_tuple).expect("the production reader must read the old tuple");
|
||||
let from_old_map = rmp_serde::to_vec(&from_old).expect("the production writer must retain the old tuple payload");
|
||||
let old_via_current: LegacyLifecycleCacheInfo =
|
||||
rmp_serde::from_slice(&from_old_map).expect("project the old fields after an old-tuple load");
|
||||
rmp_serde::from_slice(&from_old_map).expect("the old reader must read the production map after an old-tuple load");
|
||||
let old_from_current: LegacyLifecycleCacheInfo =
|
||||
rmp_serde::from_slice(¤t_map).expect("project the old fields after a production write");
|
||||
rmp_serde::from_slice(¤t_map).expect("the actual old reader must read the production map");
|
||||
assert_eq!(
|
||||
[
|
||||
rmp_serde::to_vec(&old_via_current).expect("encode the old reader's complete tuple after production reload"),
|
||||
rmp_serde::to_vec(&old_from_current).expect("encode the old reader's complete tuple after production write"),
|
||||
],
|
||||
[old_tuple, old_tuple],
|
||||
[old_tuple.clone(), old_tuple],
|
||||
"both cache directions must preserve the full old carrier for {label}"
|
||||
);
|
||||
}
|
||||
@@ -3851,12 +4051,32 @@ fn test_lifecycle_cache_current_map_round_trip() {
|
||||
|
||||
#[test]
|
||||
fn test_lifecycle_cache_strict_cached_tags_contract() {
|
||||
// The pinned legacy writer produced the same empty map before and after test_tags initialized its cache.
|
||||
use s3s::dto as legacy;
|
||||
type LegacyPayload = Option<Arc<legacy::BucketLifecycleConfiguration>>;
|
||||
|
||||
let old: LegacyPayload = rmp_serde::from_slice(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD)
|
||||
.expect("the frozen payload must initialize the real old configuration");
|
||||
let filter = old.as_ref().expect("nonempty old payload").rules[0]
|
||||
.filter
|
||||
.as_ref()
|
||||
.expect("first old rule has a filter");
|
||||
let object_tags = [(String::new(), String::new()), ("kind".to_owned(), "archive".to_owned())]
|
||||
.into_iter()
|
||||
.collect();
|
||||
// The real old inherent method initializes its OnceLock even if matching fails.
|
||||
let _ = filter.test_tags(&object_tags);
|
||||
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD))
|
||||
.expect("the production reader must accept the fixed payload");
|
||||
assert_eq!(
|
||||
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the cached tags"),
|
||||
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
|
||||
[
|
||||
rmp_serde::to_vec(&old).expect("encode the initialized old cache"),
|
||||
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the new cache")
|
||||
],
|
||||
[
|
||||
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
|
||||
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
|
||||
],
|
||||
"an initialized old CachedTags must still write the same empty map as the new carrier"
|
||||
);
|
||||
|
||||
let slot = LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD
|
||||
@@ -3869,14 +4089,22 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
|
||||
bytes.extend_from_slice(&LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD[slot + 1..]);
|
||||
bytes
|
||||
};
|
||||
// The pinned reader discarded populated String/String maps and rejected the other shapes below.
|
||||
// A historical String/String map is consumed by both real readers, then
|
||||
// rewritten empty. The original populated input is deliberately not the oracle.
|
||||
let populated = input_with_cache(&[0x81, 0xa3, b'e', b'n', b'v', 0xa4, b'p', b'r', b'o', b'd']);
|
||||
let old: LegacyPayload = rmp_serde::from_slice(&populated).expect("old reader accepts String/String cached tags");
|
||||
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(populated.as_slice()))
|
||||
.expect("new reader accepts String/String cached tags");
|
||||
assert_eq!(
|
||||
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the populated cache"),
|
||||
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
|
||||
"populated cached tags are discarded on rewrite",
|
||||
[
|
||||
rmp_serde::to_vec(&old).expect("rewrite the old populated cache"),
|
||||
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the new populated cache")
|
||||
],
|
||||
[
|
||||
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
|
||||
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
|
||||
],
|
||||
"both readers discard populated cached tags and both writers emit an empty map"
|
||||
);
|
||||
|
||||
for (label, replacement) in [
|
||||
@@ -3887,8 +4115,13 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
|
||||
("nonstring-value", &[0x81, 0xa1, b'k', 0x01][..]),
|
||||
] {
|
||||
let input = input_with_cache(replacement);
|
||||
let old = rmp_serde::from_slice::<LegacyPayload>(&input);
|
||||
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(input.as_slice()));
|
||||
assert!(new.is_err(), "must reject {label} cached tags");
|
||||
assert_eq!(
|
||||
[old.is_err(), new.is_err()],
|
||||
[true, true],
|
||||
"both real readers must reject {label} cached tags"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3900,59 +4133,73 @@ fn object_lock_cache_matches_the_legacy_dto_bytes() {
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedDefaultRetention, PersistedObjectLockConfiguration, PersistedObjectLockRule,
|
||||
};
|
||||
// Captured from the minio-enabled s3s DTO at 5761ddfe4c509fcac87e3b99fd5bd9e176b442c0.
|
||||
let cases = [
|
||||
(b"\x92\xc0\xc0".as_slice(), None, None),
|
||||
(b"\x92\xa7Enabled\xc0".as_slice(), Some("Enabled"), None),
|
||||
(
|
||||
b"\x92\xa8Disabled\x91\xc0".as_slice(),
|
||||
Some("Disabled"),
|
||||
Some(PersistedObjectLockRule { default_retention: None }),
|
||||
),
|
||||
(
|
||||
b"\x92\xa7Enabled\x91\x93\x1e\xaaGOVERNANCE\xc0".as_slice(),
|
||||
Some("Enabled"),
|
||||
Some(PersistedObjectLockRule {
|
||||
default_retention: Some(PersistedDefaultRetention {
|
||||
use s3s::dto as legacy;
|
||||
|
||||
let legacy_cases = vec![
|
||||
legacy::ObjectLockConfiguration {
|
||||
object_lock_enabled: None,
|
||||
rule: None,
|
||||
},
|
||||
legacy::ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
|
||||
rule: None,
|
||||
},
|
||||
legacy::ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(legacy::ObjectLockEnabled::from("Disabled".to_owned())),
|
||||
rule: Some(legacy::ObjectLockRule { default_retention: None }),
|
||||
},
|
||||
legacy::ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(legacy::ObjectLockRule {
|
||||
default_retention: Some(legacy::DefaultRetention {
|
||||
days: Some(30),
|
||||
mode: Some("GOVERNANCE".to_owned()),
|
||||
mode: Some(legacy::ObjectLockRetentionMode::from_static(legacy::ObjectLockRetentionMode::GOVERNANCE)),
|
||||
years: None,
|
||||
}),
|
||||
}),
|
||||
),
|
||||
(
|
||||
b"\x92\xa7Enabled\x91\x93\xc0\xaecompliance-ish\x02".as_slice(),
|
||||
Some("Enabled"),
|
||||
Some(PersistedObjectLockRule {
|
||||
default_retention: Some(PersistedDefaultRetention {
|
||||
},
|
||||
legacy::ObjectLockConfiguration {
|
||||
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
|
||||
rule: Some(legacy::ObjectLockRule {
|
||||
default_retention: Some(legacy::DefaultRetention {
|
||||
days: None,
|
||||
mode: Some("compliance-ish".to_owned()),
|
||||
mode: Some(legacy::ObjectLockRetentionMode::from("compliance-ish".to_owned())),
|
||||
years: Some(2),
|
||||
}),
|
||||
}),
|
||||
),
|
||||
},
|
||||
];
|
||||
for (old_bytes, enabled, rule) in cases {
|
||||
let current = Some(Arc::new(PersistedObjectLockConfiguration {
|
||||
object_lock_enabled: enabled.map(str::to_owned),
|
||||
rule,
|
||||
}));
|
||||
|
||||
for legacy_config in legacy_cases {
|
||||
let persisted = PersistedObjectLockConfiguration {
|
||||
object_lock_enabled: legacy_config
|
||||
.object_lock_enabled
|
||||
.as_ref()
|
||||
.map(|value| value.as_str().to_owned()),
|
||||
rule: legacy_config.rule.as_ref().map(|rule| PersistedObjectLockRule {
|
||||
default_retention: rule.default_retention.as_ref().map(|retention| PersistedDefaultRetention {
|
||||
mode: retention.mode.as_ref().map(|value| value.as_str().to_owned()),
|
||||
days: retention.days,
|
||||
years: retention.years,
|
||||
}),
|
||||
}),
|
||||
};
|
||||
let old_bytes = rmp_serde::to_vec(&Some(Arc::new(legacy_config))).expect("encode the legacy DTO");
|
||||
let current = Some(Arc::new(persisted.clone()));
|
||||
let new_bytes = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(¤t)).expect("encode the new cache");
|
||||
assert_eq!(new_bytes, old_bytes, "must reproduce the original bytes for {current:?}");
|
||||
let decoded =
|
||||
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes)).expect("decode the legacy bytes");
|
||||
assert_eq!(decoded, current);
|
||||
assert_eq!(new_bytes, old_bytes, "the new writer must produce the legacy bytes for {persisted:?}");
|
||||
|
||||
let decoded = crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes.as_slice()))
|
||||
.expect("decode the legacy bytes");
|
||||
assert_eq!(decoded.as_deref(), Some(&persisted));
|
||||
}
|
||||
let absent = b"\xc0";
|
||||
assert_eq!(
|
||||
rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent"),
|
||||
absent
|
||||
);
|
||||
assert!(
|
||||
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(&absent[..]))
|
||||
.expect("decode absent")
|
||||
.is_none()
|
||||
);
|
||||
|
||||
let absent_old = rmp_serde::to_vec(&Option::<Arc<legacy::ObjectLockConfiguration>>::None).expect("encode absent");
|
||||
let absent_new = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent");
|
||||
assert_eq!(absent_new, absent_old);
|
||||
let decoded =
|
||||
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(absent_old.as_slice())).expect("decode absent");
|
||||
assert!(decoded.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -9,7 +9,7 @@ This page is a pointer, not a route table. The machine-checked matrix in `route_
|
||||
|
||||
| Prefix | Behavior | Rule |
|
||||
|---|---|---|
|
||||
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_router` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
|
||||
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_route` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
|
||||
| `/minio/admin` | Compatibility alias accepted by `S3Router::is_match`; `canonicalize_admin_path` rewrites it to `/rustfs/admin` immediately before route lookup (`rustfs/src/admin/router.rs`) | Never register routes twice; preserve canonicalization |
|
||||
| `/iceberg/v1` | Table catalog prefix registered by `register_table_catalog_route` (`rustfs/src/admin/handlers/table_catalog/routes.rs`) and accepted by `is_admin_path` | Stays outside `/rustfs/admin`; table actions are authorized per handler |
|
||||
| `/health`, `/health/ready` | Public health endpoints, registered only when `ENV_HEALTH_ENDPOINT_ENABLE` allows | Preserve the unauthenticated bypass |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# MinIO ↔ RustFS Router Compatibility (Exceptions Only)
|
||||
|
||||
**Use this when:** a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different.
|
||||
**Source of truth:** S3 plane: the registered operation families in `rustfs/src/server/gateway_handlers/mod.rs`. Admin plane: `make_admin_router` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
|
||||
**Source of truth:** S3 plane: the `s3s::S3` trait impl in `rustfs/src/storage/ecfs.rs`. Admin plane: `make_admin_route` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
|
||||
|
||||
This document lists only exceptions. Anything not listed here is implemented with MinIO-equivalent behavior. For the s3tests-level claim see [s3-compatibility-matrix.md](s3-compatibility-matrix.md).
|
||||
|
||||
@@ -18,7 +18,7 @@ Admin paths are relative to the canonical `/rustfs/admin` prefix; `/minio/admin`
|
||||
|
||||
## S3 Data Plane
|
||||
|
||||
Run `python3 scripts/s3-tests/api_coverage.py` for the registered gateway operations and the missing operations in the pinned S3 reference. The following table records compatibility exceptions.
|
||||
All `s3s::S3` trait methods are implemented in `rustfs/src/storage/ecfs.rs` except the following.
|
||||
|
||||
| S3 operation | Status | Detail |
|
||||
|---|---|---|
|
||||
|
||||
@@ -97,7 +97,7 @@ Standard S3 areas that must not be described as complete:
|
||||
|
||||
| Area | Status | Evidence |
|
||||
|---|---|---|
|
||||
| Bucket access logging | Handlers exist (`GetBucketLogging` and `PutBucketLogging` in `rustfs/src/server/gateway_handlers/bucket_core.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
|
||||
| Bucket access logging | Handlers exist (`get_bucket_logging`, `put_bucket_logging` in `rustfs/src/storage/ecfs.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
|
||||
| POST Object form upload checksum handling | Not yet passing | `unimplemented_tests.txt` |
|
||||
| Bucket ownership controls | No handler | `unimplemented_tests.txt` |
|
||||
| Multipart upload listing and part lookup compatibility edge cases | Not part of default gate; ListParts returns HTTP 503 `SlowDownRead` when no directory was listed and any participant is unavailable, instead of treating one missing directory as an empty upload | `excluded_tests.txt`; `collect_list_parts_results_rejects_unavailable_empty_upload_dirs` |
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Audit Entries for Authorization Denials
|
||||
|
||||
**Use this when:** you consume the audit stream and need to know which authorization denials appear in it, how many, and how to tune the limit.
|
||||
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/authorization/decision.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
|
||||
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/access.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
|
||||
|
||||
With audit enabled (`RUSTFS_AUDIT_ENABLE=true`), a request refused by the authorization layer produces an audit entry in addition to the `s3_authorization_denied` log event. The S3 response is unchanged: a bare `Access Denied`.
|
||||
|
||||
|
||||
@@ -1,17 +0,0 @@
|
||||
# S3 stack upgrade and rollback
|
||||
|
||||
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
|
||||
|
||||
## Roll back to the previous release
|
||||
|
||||
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
|
||||
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
|
||||
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
|
||||
|
||||
## Client-visible error change
|
||||
|
||||
| Conformance case | What clients see |
|
||||
| --- | --- |
|
||||
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
|
||||
|
||||
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
|
||||
@@ -13,7 +13,7 @@ Every fixed RustFS GitHub Security Advisory maps to at least one named regressio
|
||||
| [GHSA-r5qv-rc46-hv8q](https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q) | Internode RPC authentication must fail closed | rustfs/rustfs#4402 | `ghsa_r5qv_resolve_shared_secret_rejects_default_fallback`, `ghsa_r5qv_verify_rpc_signature_fails_closed_on_missing_or_invalid_auth` (`crates/ecstore/src/cluster/rpc/http_auth.rs`) | unit |
|
||||
| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with the shared root secret (intentionally unfixed) | n/a; tests pin the by-design behaviour and must flip red -> green when m77q is fixed | `test_ghsa_m77q_sts_session_token_signed_with_root_secret`, `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) | unit |
|
||||
| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope; a non-owner with `CreateServiceAccountAdminAction` could parent a service account to root | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope`, `ghsa_5354_scope_guard_preserves_non_root_self_scope`, and the `add_service_account_parent_within_scope` invariant they pin (`rustfs/src/admin/handlers/service_account.rs`) | unit |
|
||||
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` in `rustfs/src/storage/authorization/tests/actions.rs` covers `versioned_read_action` in `rustfs/src/storage/authorization_actions.rs` | unit |
|
||||
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit |
|
||||
| [GHSA-v9cp-qfw9-9pfp](https://github.com/rustfs/rustfs/security/advisories/GHSA-v9cp-qfw9-9pfp) | `ForAllValues:`/`ForAnyValue:` negated string operators applied negation to the aggregate instead of the per-value predicate | fixed, GHSA private-fork merge | `ghsa_v9cp_for_all_values_not_equals_partial_overlap`, `ghsa_v9cp_for_any_value_not_equals_partial_overlap` and the absent-key/positive-quantifier cases beside them (`crates/policy/tests/quantified_negation.rs`); the value set must partially overlap the policy set, since contained or disjoint sets cannot tell the quantifiers apart | crate test |
|
||||
| [GHSA-6r96-hmgc-726c](https://github.com/rustfs/rustfs/security/advisories/GHSA-6r96-hmgc-726c) | Request headers must not populate server-derived IAM condition keys (`userid`, `groups`, `jwt:`/`ldap:` claims) | fixed, GHSA private-fork merge | `ghsa_6r96_identity_condition_keys_ignore_spoofed_headers`, `ghsa_6r96_claim_condition_keys_ignore_spoofed_headers`, and `test_request_headers_still_reach_conditions`, which keeps the reserved set from growing too broad (`rustfs/src/auth.rs`) | unit |
|
||||
| [GHSA-x298-9x87-fvjq](https://github.com/rustfs/rustfs/security/advisories/GHSA-x298-9x87-fvjq) | Anonymous ListObjectVersions -> `s3:ListBucket` fallback must reach the same public-access gates as a direct grant | fixed, GHSA private-fork merge | `ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled` (`crates/e2e_test/src/anonymous_access_test.rs`); asserts 200 before the public-access block is applied so it proves the gate, not a broken fallback | e2e (`e2e-smoke`) |
|
||||
|
||||
@@ -195,14 +195,27 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let request: ChangePasswordRequest = serde_json::from_slice(&body)
|
||||
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
|
||||
|
||||
// An STS session signs with its own secret. A direct IAM request must
|
||||
// have been signed with the same password the caller is replacing.
|
||||
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
|
||||
&caller.credentials.secret_key
|
||||
} else {
|
||||
&request.current_secret_key
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
|
||||
// Reached only if the identity was deleted between authentication
|
||||
// and this lookup.
|
||||
emit_audit(
|
||||
&audit,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&caller.access_key,
|
||||
caller.identity_type,
|
||||
AccountAuditFailure::Internal,
|
||||
),
|
||||
);
|
||||
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
|
||||
};
|
||||
if !constant_time_eq(expected_signing_secret, &signing_secret) {
|
||||
|
||||
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|
||||
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
|
||||
{
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ACCOUNT,
|
||||
@@ -243,10 +256,8 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
}
|
||||
|
||||
let access_key = caller.access_key.clone();
|
||||
let current_secret_key = request.current_secret_key;
|
||||
let new_secret_key = request.new_secret_key;
|
||||
let new_secret_key = request.new_secret_key.clone();
|
||||
let identity_type = caller.identity_type;
|
||||
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
|
||||
let session_access_key = caller.session_access_key.clone();
|
||||
let audit_for_task = audit.clone();
|
||||
|
||||
@@ -257,35 +268,16 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let rotation = match session_credentials.as_ref() {
|
||||
Some(session) => {
|
||||
iam_store
|
||||
.change_own_secret_key_from_session(&access_key, ¤t_secret_key, &new_secret_key, session)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
iam_store
|
||||
.change_own_secret_key(&access_key, ¤t_secret_key, &new_secret_key)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let (updated_at, status, revoked) = rotation.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&access_key,
|
||||
identity_type,
|
||||
AccountAuditFailure::InvalidCurrentSecret,
|
||||
)
|
||||
.with_session_access_key(session_access_key.as_deref()),
|
||||
);
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
let (updated_at, status, revoked) = iam_store
|
||||
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
|
||||
@@ -1868,8 +1868,6 @@ mod tests {
|
||||
peer_topology_complete, query_peer_heal_status, query_peer_replacement_recovery_status, read_cluster_heal_status,
|
||||
reject_heal_admission, validate_heal_request_mode, validate_heal_target, validate_mrf_legacy_responsibility_action,
|
||||
};
|
||||
use crate::admin::operation::test_support::TestRequest;
|
||||
use crate::admin::router::dispatch;
|
||||
use crate::storage::rpc::node_service::heal::{
|
||||
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
|
||||
};
|
||||
@@ -2155,19 +2153,33 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_nested_heal_routes_still_require_authentication() {
|
||||
use s3s::route::S3Route;
|
||||
|
||||
let mut router = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut router).expect("register production Heal routes");
|
||||
for prefix in ["/rustfs/admin", "/minio/admin"] {
|
||||
for object in ["dir/object.bin", "dir%2Fobject.bin", "literal%252Fslash"] {
|
||||
let request = TestRequest::new(http::Method::POST, format!("{prefix}/v3/heal/test-bucket/{object}")).build();
|
||||
let mut req = s3s::S3Request {
|
||||
input: s3s::Body::empty(),
|
||||
method: http::Method::POST,
|
||||
uri: format!("{prefix}/v3/heal/test-bucket/{object}").parse().expect("Heal URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let err = router
|
||||
.admit(request.method(), request.uri(), request.headers(), false)
|
||||
.check_access(&mut req)
|
||||
.await
|
||||
.expect_err("router must require a signature");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
|
||||
let err = router
|
||||
.call(req)
|
||||
.await
|
||||
.expect_err("handler must independently require authentication");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
|
||||
assert!(err.to_string().contains("authentication required"));
|
||||
}
|
||||
}
|
||||
@@ -2175,20 +2187,34 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn replacement_recovery_diagnostics_require_authentication() {
|
||||
use s3s::route::S3Route;
|
||||
|
||||
let mut router = super::S3Router::new(false);
|
||||
super::register_heal_route(&mut router).expect("register production Heal routes");
|
||||
for prefix in ["/rustfs/admin", "/minio/admin"] {
|
||||
let path = format!("{prefix}/v4/heal/replacement-recovery");
|
||||
assert!(router.contains_compatible_route(http::Method::GET, &path));
|
||||
let request = TestRequest::new(http::Method::GET, path).build();
|
||||
let mut req = s3s::S3Request {
|
||||
input: s3s::Body::empty(),
|
||||
method: http::Method::GET,
|
||||
uri: path.parse().expect("replacement diagnostics URI"),
|
||||
headers: http::HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let err = router
|
||||
.admit(request.method(), request.uri(), request.headers(), false)
|
||||
.check_access(&mut req)
|
||||
.await
|
||||
.expect_err("diagnostics router must require a signature");
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
|
||||
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
|
||||
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
|
||||
let err = router
|
||||
.call(req)
|
||||
.await
|
||||
.expect_err("diagnostics handler must authenticate before inspecting local records");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
|
||||
assert!(err.to_string().contains("authentication required"));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,47 +90,79 @@ impl Operation for HealthCheckHandler {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::admin::operation::test_support::TestRequest;
|
||||
use crate::admin::router::dispatch;
|
||||
use http_body_util::BodyExt as _;
|
||||
use s3s::route::S3Route as _;
|
||||
use serial_test::serial;
|
||||
use temp_env::with_var;
|
||||
|
||||
async fn call_real_health_handler(method: Method) -> (StatusCode, HeaderMap, Body) {
|
||||
fn health_request(method: Method) -> s3s::S3Request<s3s::Body> {
|
||||
s3s::S3Request {
|
||||
input: s3s::Body::empty(),
|
||||
method,
|
||||
uri: HEALTH_PREFIX.parse().expect("health URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
async fn call_real_health_handler(method: Method) -> s3s::S3Response<s3s::Body> {
|
||||
let mut router = S3Router::new(false);
|
||||
router
|
||||
.insert(method.clone(), HEALTH_PREFIX, AdminOperation(&HealthCheckHandler {}))
|
||||
.expect("health route should register");
|
||||
let request = TestRequest::new(method, HEALTH_PREFIX).build();
|
||||
dispatch(&router, request.into_envelope(), Body::empty())
|
||||
router
|
||||
.call(health_request(method))
|
||||
.await
|
||||
.expect("health handler should return a response")
|
||||
.into_parts()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_health_handler_get_and_head_preserve_wire_branches() {
|
||||
let (status, headers, body) = call_real_health_handler(Method::GET).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
|
||||
let payload: serde_json::Value = serde_json::from_slice(&body.collect().await.expect("GET body should read").to_bytes())
|
||||
.expect("GET health response should be JSON");
|
||||
let get = call_real_health_handler(Method::GET).await;
|
||||
assert_eq!(get.status, Some(StatusCode::OK));
|
||||
assert_eq!(
|
||||
get.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
|
||||
Some("application/json")
|
||||
);
|
||||
let payload: serde_json::Value =
|
||||
serde_json::from_slice(&get.output.collect().await.expect("GET body should read").to_bytes())
|
||||
.expect("GET health response should be JSON");
|
||||
assert_eq!(payload["status"], "ok");
|
||||
|
||||
let (status, headers, body) = call_real_health_handler(Method::HEAD).await;
|
||||
assert_eq!(status, StatusCode::OK);
|
||||
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
|
||||
assert!(body.collect().await.expect("HEAD body should read").to_bytes().is_empty());
|
||||
let head = call_real_health_handler(Method::HEAD).await;
|
||||
assert_eq!(head.status, Some(StatusCode::OK));
|
||||
assert_eq!(
|
||||
head.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
|
||||
Some("application/json")
|
||||
);
|
||||
assert!(
|
||||
head.output
|
||||
.collect()
|
||||
.await
|
||||
.expect("HEAD body should read")
|
||||
.to_bytes()
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_health_handler_rejects_unsupported_methods_with_allow_header() {
|
||||
let (status, headers, body) = call_real_health_handler(Method::POST).await;
|
||||
let response = call_real_health_handler(Method::POST).await;
|
||||
|
||||
assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED);
|
||||
assert_eq!(headers.get(http::header::ALLOW).and_then(|value| value.to_str().ok()), Some("GET, HEAD"));
|
||||
assert_eq!(response.status, Some(StatusCode::METHOD_NOT_ALLOWED));
|
||||
assert_eq!(
|
||||
body.collect().await.expect("error body should read").to_bytes(),
|
||||
response
|
||||
.headers
|
||||
.get(http::header::ALLOW)
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("GET, HEAD")
|
||||
);
|
||||
assert_eq!(
|
||||
response.output.collect().await.expect("error body should read").to_bytes(),
|
||||
bytes::Bytes::from_static(b"Method Not Allowed")
|
||||
);
|
||||
}
|
||||
|
||||
@@ -332,7 +332,7 @@ fn is_lock_timeout(err: &crate::storage_api::error::StorageError) -> bool {
|
||||
fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError {
|
||||
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::InternalError,
|
||||
code: S3ErrorCode::InternalError.into(),
|
||||
message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
@@ -341,7 +341,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
|
||||
// movement gate has a queued writer: the same transient contention.
|
||||
if is_lock_timeout(&err) || matches!(err, crate::storage_api::error::StorageError::SlowDown) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::ServiceUnavailable,
|
||||
code: S3ErrorCode::ServiceUnavailable.into(),
|
||||
message: err.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
@@ -350,7 +350,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
|
||||
// should say so, not just "Invalid Request".
|
||||
if matches!(err, crate::storage_api::error::StorageError::DecommissionAlreadyRunning) {
|
||||
return ApiError {
|
||||
code: S3ErrorCode::InvalidRequest,
|
||||
code: S3ErrorCode::InvalidRequest.into(),
|
||||
message: err.to_string(),
|
||||
source: Some(Box::new(err)),
|
||||
};
|
||||
@@ -1261,14 +1261,14 @@ mod pools_handler_tests {
|
||||
#[test]
|
||||
fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
|
||||
let err = crate::error::ApiError {
|
||||
code: rustfs_s3_types::S3ErrorCode::InvalidRequest,
|
||||
code: rustfs_s3_types::S3ErrorCode::InvalidRequest.into(),
|
||||
message: "decommission already running".to_string(),
|
||||
source: None,
|
||||
};
|
||||
|
||||
let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]");
|
||||
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest.into());
|
||||
assert_eq!(
|
||||
err.message,
|
||||
"admin start decommission failed for pools [1, 3]: decommission already running"
|
||||
@@ -1281,12 +1281,12 @@ mod pools_handler_tests {
|
||||
|
||||
let err = decommission_start_api_error(err);
|
||||
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
|
||||
assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
|
||||
assert!(err.source.is_some());
|
||||
|
||||
let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed"));
|
||||
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
|
||||
assert_eq!(unrelated.message, "We encountered an internal error, please try again.");
|
||||
}
|
||||
|
||||
@@ -1299,7 +1299,7 @@ mod pools_handler_tests {
|
||||
|
||||
let err = decommission_start_api_error(err);
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
|
||||
assert!(err.message.contains("data movement gate (decommission activation)"), "{}", err.message);
|
||||
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
|
||||
assert_eq!(
|
||||
@@ -1313,7 +1313,7 @@ mod pools_handler_tests {
|
||||
fn test_decommission_start_api_error_reports_a_refused_preflight_heal_as_retryable() {
|
||||
let err = decommission_start_api_error(crate::storage_api::error::StorageError::SlowDown);
|
||||
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
|
||||
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
|
||||
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
|
||||
}
|
||||
|
||||
@@ -926,11 +926,10 @@ impl Operation for RebalanceStatus {
|
||||
"admin rebalance state"
|
||||
);
|
||||
// Preserve the idle 404 contract without triggering the custom-route error logger.
|
||||
let body = format!(
|
||||
r#"<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchResource</Code><Message>{REBALANCE_NOT_STARTED_MESSAGE}</Message></Error>"#
|
||||
);
|
||||
let headers = HeaderMap::from_iter([(CONTENT_TYPE, HeaderValue::from_static("application/xml"))]);
|
||||
return Ok(AdminResponse::with_headers(StatusCode::NOT_FOUND, Body::from(body), headers));
|
||||
let (parts, body) = s3s::S3Error::with_message(s3s::S3ErrorCode::NoSuchResource, REBALANCE_NOT_STARTED_MESSAGE)
|
||||
.to_http_response()?
|
||||
.into_parts();
|
||||
return Ok(AdminResponse::with_headers(parts.status, body.into(), parts.headers));
|
||||
}
|
||||
|
||||
error!(
|
||||
|
||||
@@ -48,6 +48,7 @@ use rustfs_policy::{
|
||||
};
|
||||
use rustfs_s3_types::{Body, S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use rustfs_utils::MaskedAccessKey;
|
||||
use s3s::dto::{AssumeRoleOutput, Credentials, Timestamp};
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use serde_urlencoded::from_bytes;
|
||||
@@ -336,35 +337,33 @@ async fn handle_assume_role(
|
||||
warn!("site replication STS hook failed, err: {err}");
|
||||
}
|
||||
|
||||
let expiration = new_cred
|
||||
.expiration
|
||||
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600)));
|
||||
let output = assume_role_response_xml(&new_cred, expiration)?;
|
||||
let resp = AssumeRoleOutput {
|
||||
credentials: Some(Credentials {
|
||||
access_key_id: new_cred.access_key,
|
||||
expiration: Timestamp::from(
|
||||
new_cred
|
||||
.expiration
|
||||
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600))),
|
||||
),
|
||||
secret_access_key: new_cred.secret_key,
|
||||
session_token: new_cred.session_token,
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// getAssumeRoleCredentials
|
||||
let output = serialize_assume_role_output(&resp)
|
||||
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize assume role output failed: {e}")))?;
|
||||
|
||||
Ok(AdminResponse::new(StatusCode::OK, Body::from(output)))
|
||||
}
|
||||
|
||||
fn assume_role_response_xml(credentials: &rustfs_credentials::Credentials, expiration: OffsetDateTime) -> S3Result<String> {
|
||||
// The STS codec writes local clock fields with millisecond precision and a four-digit year.
|
||||
if !(0..=9999).contains(&expiration.year()) {
|
||||
return Err(s3_error!(InternalError, "cannot serialize STS expiration year"));
|
||||
}
|
||||
let expiration = expiration
|
||||
.format(time::macros::format_description!(
|
||||
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
|
||||
))
|
||||
.map_err(|_| s3_error!(InternalError, "cannot serialize STS expiration"))?;
|
||||
Ok(format!(
|
||||
concat!(
|
||||
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
|
||||
"<AccessKeyId>{}</AccessKeyId><Expiration>{}</Expiration>",
|
||||
"<SecretAccessKey>{}</SecretAccessKey><SessionToken>{}</SessionToken>",
|
||||
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
|
||||
),
|
||||
xml_escape(&credentials.access_key),
|
||||
expiration,
|
||||
xml_escape(&credentials.secret_key),
|
||||
xml_escape(&credentials.session_token),
|
||||
))
|
||||
/// The STS answer is an s3s response body, so the s3s encoder writes it; it
|
||||
/// is not a bucket configuration and never goes through the bucket-config codec.
|
||||
fn serialize_assume_role_output(output: &AssumeRoleOutput) -> s3s::xml::SerResult<Vec<u8>> {
|
||||
let mut buf = Vec::with_capacity(256);
|
||||
s3s::xml::Serialize::serialize(output, &mut s3s::xml::Serializer::new(&mut buf))?;
|
||||
Ok(buf)
|
||||
}
|
||||
|
||||
/// Session claim marking a session that presented a second factor.
|
||||
@@ -580,29 +579,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn assume_role_xml_matches_the_pinned_sts_response() {
|
||||
let credentials = rustfs_credentials::Credentials {
|
||||
access_key: "access<&".to_owned(),
|
||||
secret_key: "synthetic>'".to_owned(),
|
||||
session_token: "quoted\"token".to_owned(),
|
||||
..Default::default()
|
||||
};
|
||||
let expiration = time::macros::datetime!(2024-02-29 11:22:33.123456789 +05:30);
|
||||
let answer = assume_role_response_xml(&credentials, expiration).expect("valid timestamp");
|
||||
// Captured from the pinned 5761ddfe STS writer, including field order and escaping.
|
||||
assert_eq!(
|
||||
answer,
|
||||
concat!(
|
||||
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
|
||||
"<AccessKeyId>access<&</AccessKeyId><Expiration>2024-02-29T11:22:33.123Z</Expiration>",
|
||||
"<SecretAccessKey>synthetic>'</SecretAccessKey><SessionToken>quoted"token</SessionToken>",
|
||||
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
|
||||
)
|
||||
);
|
||||
assert!(assume_role_response_xml(&credentials, time::macros::datetime!(-1-01-01 00:00 UTC)).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_xml_escape() {
|
||||
assert_eq!(xml_escape("hello"), "hello");
|
||||
|
||||
+17
-1
@@ -45,8 +45,24 @@ use handlers::{
|
||||
user,
|
||||
};
|
||||
use router::{AdminOperation, S3Router};
|
||||
use s3s::route::S3Route;
|
||||
|
||||
/// Builds the control-plane router and binds it to this server.
|
||||
/// Create admin router
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `console_enabled` - Whether the console is enabled
|
||||
///
|
||||
/// # Returns
|
||||
/// An instance of S3Route for admin operations
|
||||
pub fn make_admin_route(
|
||||
console_enabled: bool,
|
||||
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
|
||||
) -> std::io::Result<impl S3Route> {
|
||||
make_admin_router(console_enabled, server_ctx)
|
||||
}
|
||||
|
||||
/// The admin router with every route registered, bound to `server_ctx`: what the legacy stack
|
||||
/// mounts as its custom route and the gateway stack's control-plane host dispatches through.
|
||||
pub(crate) fn make_admin_router(
|
||||
console_enabled: bool,
|
||||
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
|
||||
|
||||
+331
-232
File diff suppressed because it is too large
Load Diff
@@ -77,7 +77,7 @@ fn uri_for(template: &str, bucket: &str) -> String {
|
||||
|
||||
async fn send(router: &S3Router<AdminOperation>, mut req: S3Request<Body>) -> S3Result<S3Response<Body>> {
|
||||
router.check_access(&mut req).await?;
|
||||
router.call(req).await.map_err(Into::into)
|
||||
router.call(req).await
|
||||
}
|
||||
|
||||
fn error_code(result: S3Result<S3Response<Body>>, context: &str) -> S3ErrorCode {
|
||||
@@ -222,7 +222,7 @@ async fn extension_routes_refuse_anonymous_callers_before_dispatch() {
|
||||
.check_access(&mut req)
|
||||
.await
|
||||
.expect_err("an anonymous extension request is refused");
|
||||
assert_eq!(S3ErrorCode::from(error.code().clone()), S3ErrorCode::AccessDenied, "{method} {uri}");
|
||||
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{method} {uri}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -87,8 +87,6 @@ pub(crate) enum CredentialMutationDenial {
|
||||
ServiceAccount,
|
||||
/// A derived credential whose parent could not be determined.
|
||||
UnresolvedParent,
|
||||
/// A storage delegation has no account-management authority.
|
||||
TableStorageSession,
|
||||
}
|
||||
|
||||
impl CredentialMutationDenial {
|
||||
@@ -99,7 +97,6 @@ impl CredentialMutationDenial {
|
||||
}
|
||||
Self::FederatedIdentity => "federated identities are managed by their identity provider",
|
||||
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
|
||||
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
|
||||
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
|
||||
}
|
||||
}
|
||||
@@ -190,7 +187,7 @@ impl CallerIdentity {
|
||||
// root", and an unresolved parent must never fall through to a
|
||||
// permissive answer.
|
||||
//
|
||||
// These denials apply to both capabilities: a machine credential
|
||||
// These three denials apply to both capabilities: a machine credential
|
||||
// must not take over its parent, a federated identity is owned by its
|
||||
// IdP, and an unresolvable parent fails closed.
|
||||
let shared_denial = if unresolved_parent {
|
||||
@@ -199,11 +196,6 @@ impl CallerIdentity {
|
||||
Some(CredentialMutationDenial::ServiceAccount)
|
||||
} else if federated {
|
||||
Some(CredentialMutationDenial::FederatedIdentity)
|
||||
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
|
||||
.iter()
|
||||
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
|
||||
{
|
||||
Some(CredentialMutationDenial::TableStorageSession)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -509,20 +501,6 @@ mod tests {
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_storage_sessions_cannot_manage_parent_credentials() {
|
||||
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
|
||||
let mut credentials = sts_session("TABLEKEY", "table-owner");
|
||||
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
|
||||
let caller = CallerIdentity::from_credentials(credentials, false);
|
||||
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
|
||||
assert!(!caller.mutability().password);
|
||||
assert!(caller.ensure_credential_mutation_allowed().is_err());
|
||||
assert!(caller.ensure_mfa_management_allowed().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
|
||||
// Fail closed: an unresolvable parent must not be treated as a
|
||||
|
||||
@@ -37,7 +37,7 @@ use crate::server::collect_dependency_readiness_report as collect_runtime_depend
|
||||
use crate::shared_types::DependencyReadiness;
|
||||
use rustfs_data_usage::DataUsageInfo;
|
||||
use rustfs_madmin::{InfoMessage, StorageInfo};
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use s3s::S3ErrorCode;
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
@@ -1,76 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Operations without a notification event must emit one use-case audit entry.
|
||||
use crate::guard_sources::SourceSet;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
|
||||
fn is_read_only(op: S3Operation) -> bool {
|
||||
let name = op.as_str().trim_start_matches("s3:");
|
||||
["Get", "Head", "List", "Select"]
|
||||
.iter()
|
||||
.any(|prefix| name.starts_with(prefix))
|
||||
}
|
||||
|
||||
fn audit_sites(use_cases: &SourceSet, op: S3Operation) -> usize {
|
||||
let helper = format!("OperationHelper::audit_only_from_view(&envelope.audit_view(), S3Operation::{op:?}");
|
||||
use_cases.count_ignoring_whitespace(&format!("audit_only_operation(envelope, S3Operation::{op:?},"))
|
||||
+ use_cases.count_ignoring_whitespace(&format!("{helper})"))
|
||||
+ use_cases.count_ignoring_whitespace(&format!("{helper},)"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mutating_operations_without_a_notification_event_are_audited_once() {
|
||||
let mut expected: Vec<S3Operation> = S3Operation::ALL
|
||||
.into_iter()
|
||||
.filter(|op| !is_read_only(*op) && op.to_event_name().is_none())
|
||||
.collect();
|
||||
// AbortMultipartUpload has an internal event but no handler-built helper.
|
||||
expected.push(S3Operation::AbortMultipartUpload);
|
||||
assert_eq!(expected.len(), 26, "an operation was added or removed: {expected:?}");
|
||||
|
||||
let use_cases = SourceSet::production("src/app/mod.rs", "src/app");
|
||||
for op in expected {
|
||||
assert_eq!(audit_sites(&use_cases, op), 1, "expected one audit site for {op:?}");
|
||||
}
|
||||
}
|
||||
|
||||
const OWNED_HELPER: &str =
|
||||
"let helper = OperationHelper::audit_only_from_view(\n &envelope.audit_view(),\n S3Operation::UploadPart,\n);";
|
||||
const OWNED_WRAPPER: &str =
|
||||
"RequestEnvelope::audit_only_operation(\n envelope,\n S3Operation::UploadPart,\n StatusCode::OK,\n run,\n)";
|
||||
|
||||
fn count_sites(source: &str) -> usize {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let app = root.path().join("src/app");
|
||||
std::fs::create_dir_all(&app).unwrap();
|
||||
std::fs::write(app.join("mod.rs"), source).unwrap();
|
||||
let sources = SourceSet::production_under(root.path(), "src/app/mod.rs", "src/app");
|
||||
audit_sites(&sources, S3Operation::UploadPart)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_guard_accepts_exactly_one_audit_place() {
|
||||
assert_eq!(count_sites(OWNED_HELPER), 1);
|
||||
assert_eq!(count_sites(OWNED_WRAPPER), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_guard_refuses_an_operation_audited_nowhere_or_twice() {
|
||||
assert_eq!(count_sites(""), 0);
|
||||
assert_eq!(count_sites(&format!("{OWNED_HELPER}\n{OWNED_WRAPPER}")), 2);
|
||||
assert_eq!(count_sites(&format!("#[cfg(test)]\nmod tests {{\n{OWNED_HELPER}\n}}")), 0);
|
||||
assert_eq!(count_sites(&OWNED_WRAPPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
|
||||
assert_eq!(count_sites(&OWNED_HELPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
|
||||
}
|
||||
@@ -472,7 +472,7 @@ mod tests {
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
|
||||
use crate::app::storage_api::bucket_usecase::s3::{
|
||||
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response,
|
||||
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3, S3Request, S3Response,
|
||||
XmlSerialize, XmlSerializer,
|
||||
};
|
||||
use crate::app::storage_api::test::StoragePutObjReader;
|
||||
@@ -1185,11 +1185,10 @@ mod tests {
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let get_env = crate::app::RequestEnvelope::from_s3s(&get);
|
||||
let authorized_generation =
|
||||
super::super::storage_api::bucket_usecase::access::load_bucket_generation_from_store(
|
||||
&store,
|
||||
&get_env,
|
||||
&get,
|
||||
&input.bucket,
|
||||
)
|
||||
.await
|
||||
@@ -1215,16 +1214,12 @@ mod tests {
|
||||
assert_ne!(replacement, old.incarnation_id());
|
||||
sys.remove(&input.bucket);
|
||||
let mut without_source = get.clone();
|
||||
let mut without_source_env = crate::app::RequestEnvelope::from_s3s(&without_source);
|
||||
super::super::storage_api::bucket_usecase::access::prepare_odm_read_generation(
|
||||
&store,
|
||||
&mut without_source_env,
|
||||
&mut without_source,
|
||||
&input.bucket,
|
||||
)
|
||||
.await;
|
||||
if let Some(guard) = without_source_env.odm_read_generation().cloned() {
|
||||
without_source.extensions.insert(guard);
|
||||
}
|
||||
for state_incarnation in [old.incarnation_id(), replacement] {
|
||||
sys.apply_for_incarnation(&input.bucket, state_incarnation, Some(old.config()))
|
||||
.await;
|
||||
@@ -1320,34 +1315,28 @@ mod tests {
|
||||
};
|
||||
for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
|
||||
let request_marker = input.marker.clone().unwrap_or_default();
|
||||
let list_req = S3Request {
|
||||
input: input.clone(),
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
};
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&list_req);
|
||||
let gateway_input =
|
||||
rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::input_from_s3s(
|
||||
list_req.input,
|
||||
)
|
||||
.expect("convert ListObjectsInput");
|
||||
let gateway_output = tokio::time::timeout(
|
||||
let response = tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
DefaultBucketUsecase::from_global().execute_list_objects(envelope, gateway_input),
|
||||
S3::list_objects(
|
||||
&crate::storage::ecfs::FS::new(),
|
||||
S3Request {
|
||||
input: input.clone(),
|
||||
method: http::Method::GET,
|
||||
uri: http::Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions: http::Extensions::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
},
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("v1 pagination must finish")
|
||||
.expect("list-through must not change v1 listing");
|
||||
let output = rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::output_to_s3s(
|
||||
gateway_output,
|
||||
)
|
||||
.expect("convert ListObjectsOutput");
|
||||
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
|
||||
let output = response.output;
|
||||
let contents = output.contents.as_ref().expect("local page contents");
|
||||
assert_eq!(contents.len(), 1);
|
||||
assert_eq!(contents[0].key.as_deref(), Some(expected_key));
|
||||
|
||||
@@ -98,13 +98,14 @@ use rustfs_policy::policy::{
|
||||
{BucketPolicy, BucketPolicyArgs, Effect},
|
||||
};
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{Region, S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_s3_types::Region;
|
||||
use rustfs_targets::{
|
||||
EventName,
|
||||
arn::{ARN, TargetID, TargetIDError},
|
||||
};
|
||||
use rustfs_utils::http::get_header;
|
||||
use rustfs_utils::obj::extract_user_defined_metadata;
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result};
|
||||
use std::{
|
||||
borrow::Cow,
|
||||
collections::{HashMap, HashSet},
|
||||
@@ -906,9 +907,7 @@ mod tests {
|
||||
.expect_err("full queue rejects immediately");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
|
||||
let response = s3s::S3Error::from(error)
|
||||
.to_http_response()
|
||||
.expect("serialize admission error");
|
||||
let response = error.to_http_response().expect("serialize admission error");
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
|
||||
use http_body_util::BodyExt as _;
|
||||
@@ -1001,8 +1000,7 @@ mod tests {
|
||||
},
|
||||
);
|
||||
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
assert!(envelope.req_info().expect("req info should be present").is_owner);
|
||||
assert!(req_info_ref(&req).expect("req info should be present").is_owner);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -67,17 +67,6 @@ impl DefaultBucketUsecase {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
|
||||
let error = usecase
|
||||
.execute_put_bucket_acl(envelope, "test-bucket", false)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
|
||||
let input = gateway_dto::GetBucketAclInput {
|
||||
|
||||
@@ -116,7 +116,7 @@ impl DefaultBucketUsecase {
|
||||
|
||||
async fn list_objects_page(&self, envelope: &mut RequestEnvelope, request: &PageRequest) -> S3Result<Page> {
|
||||
let PageRequest { bucket, params, .. } = request;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
|
||||
let store = get_validated_store(bucket).await?;
|
||||
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED).as_deref() == Some("true");
|
||||
let table_list_access = envelope.table_list_access().cloned();
|
||||
@@ -375,77 +375,6 @@ mod tests {
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use rustfs_gateway_types::BucketName;
|
||||
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
fn list_apis_report_storage_read_quorum_as_unavailable() {
|
||||
use crate::app::bucket_usecase::test_support::owned_envelope;
|
||||
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore};
|
||||
run_large_stack_test("list-quorum-surface", || async {
|
||||
let usecase = DefaultBucketUsecase::with_context(Some(shared_gating_ambient().await));
|
||||
let store = shared_gating_ecstore().await;
|
||||
let bucket = BucketName::new(format!("list-quorum-{}", uuid::Uuid::new_v4().simple())).unwrap();
|
||||
store
|
||||
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
|
||||
.await
|
||||
.unwrap();
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let saved_disks = {
|
||||
let mut disks = set.disks.write().await;
|
||||
let saved = disks.clone();
|
||||
disks.iter_mut().skip(1).for_each(|disk| *disk = None);
|
||||
saved
|
||||
};
|
||||
let envelope = || owned_envelope(http::Method::GET);
|
||||
let results = [
|
||||
usecase
|
||||
.execute_list_objects(
|
||||
envelope(),
|
||||
gateway_dto::ListObjectsInput {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(drop),
|
||||
usecase
|
||||
.execute_list_objects_v2(
|
||||
envelope(),
|
||||
gateway_dto::ListObjectsV2Input {
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(drop),
|
||||
usecase
|
||||
.execute_list_object_versions(
|
||||
envelope(),
|
||||
gateway_dto::ListObjectVersionsInput {
|
||||
bucket,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map(drop),
|
||||
];
|
||||
*set.disks.write().await = saved_disks;
|
||||
for result in results {
|
||||
let error = result.expect_err("cannot list below storage read quorum");
|
||||
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
|
||||
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
|
||||
let source = error.source().expect("retain storage diagnostics");
|
||||
assert!(
|
||||
matches!(
|
||||
source.downcast_ref::<StorageError>(),
|
||||
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
|
||||
),
|
||||
"{source}"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
fn object(name: &str) -> ObjectInfo {
|
||||
ObjectInfo {
|
||||
name: name.to_owned(),
|
||||
|
||||
@@ -626,7 +626,7 @@ impl DefaultBucketUsecase {
|
||||
|
||||
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
|
||||
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
|
||||
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
|
||||
|
||||
let store = get_validated_store(&bucket).await?;
|
||||
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
|
||||
|
||||
@@ -23,7 +23,8 @@ use crate::app::storage_api::s3::dto::{
|
||||
FilterRule, LambdaFunctionConfiguration, NotificationConfiguration, NotificationConfigurationFilter,
|
||||
PutBucketNotificationConfigurationInput, QueueConfiguration, S3KeyFilter, TopicConfiguration,
|
||||
};
|
||||
use crate::app::storage_api::s3::{S3ErrorCode, S3Request};
|
||||
use crate::app::storage_api::s3::{S3, S3Access, S3ErrorCode, S3Request};
|
||||
use crate::app::storage_api::test::ecfs::FS;
|
||||
use crate::runtime_sources::{NotifyInterface, ServerContextSlot};
|
||||
use http::Method;
|
||||
use rustfs_notify::{EventArgs, NotificationError};
|
||||
@@ -214,76 +215,6 @@ fn notification_request(
|
||||
request
|
||||
}
|
||||
|
||||
fn notification_filters(filter: Option<NotificationConfigurationFilter>) -> Vec<(Option<String>, Option<String>)> {
|
||||
filter
|
||||
.and_then(|f| f.key)
|
||||
.and_then(|k| k.filter_rules)
|
||||
.unwrap_or_default()
|
||||
.into_iter()
|
||||
.map(|rule| (rule.name.map(|n| n.as_str().to_owned()), rule.value))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn notification_destinations(configuration: NotificationConfiguration) -> Vec<NotificationDestination> {
|
||||
let mut destinations = Vec::new();
|
||||
for cfg in configuration.queue_configurations.into_iter().flatten() {
|
||||
destinations.push(NotificationDestination {
|
||||
kind: "QueueConfiguration",
|
||||
id: cfg.id,
|
||||
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
|
||||
target_arn: cfg.queue_arn,
|
||||
filters: notification_filters(cfg.filter),
|
||||
});
|
||||
}
|
||||
for cfg in configuration.topic_configurations.into_iter().flatten() {
|
||||
destinations.push(NotificationDestination {
|
||||
kind: "TopicConfiguration",
|
||||
id: cfg.id,
|
||||
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
|
||||
target_arn: cfg.topic_arn,
|
||||
filters: notification_filters(cfg.filter),
|
||||
});
|
||||
}
|
||||
for cfg in configuration.lambda_function_configurations.into_iter().flatten() {
|
||||
destinations.push(NotificationDestination {
|
||||
kind: "LambdaFunctionConfiguration",
|
||||
id: cfg.id,
|
||||
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
|
||||
target_arn: cfg.lambda_function_arn,
|
||||
filters: notification_filters(cfg.filter),
|
||||
});
|
||||
}
|
||||
destinations
|
||||
}
|
||||
|
||||
async fn authorize_notification_put(
|
||||
slot: &Arc<ServerContextSlot>,
|
||||
envelope: &mut RequestEnvelope,
|
||||
bucket: &str,
|
||||
) -> rustfs_s3_types::S3Result<()> {
|
||||
use rustfs_policy::policy::action::{Action, S3Action};
|
||||
crate::app::storage_api::owned_parity::authorize_bucket_config_mutation(
|
||||
slot,
|
||||
envelope,
|
||||
bucket,
|
||||
Action::S3Action(S3Action::PutBucketNotificationAction),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn put_notification(
|
||||
slot: &Arc<ServerContextSlot>,
|
||||
mut envelope: RequestEnvelope,
|
||||
input: PutBucketNotificationConfigurationInput,
|
||||
) -> rustfs_s3_types::S3Result<gateway_dto::PutBucketNotificationConfigurationOutput> {
|
||||
authorize_notification_put(slot, &mut envelope, &input.bucket).await?;
|
||||
let prepared = crate::storage::persisted_from_s3s(&input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
|
||||
let destinations = notification_destinations(input.notification_configuration);
|
||||
DefaultBucketUsecase::with_context(slot.app_context())
|
||||
.execute_put_bucket_notification_configuration(envelope, &input.bucket, destinations, prepared)
|
||||
.await
|
||||
}
|
||||
|
||||
fn check_notification_write_order() {
|
||||
run_large_stack_test("notification-write-order", || async {
|
||||
let ambient = shared_gating_ambient().await;
|
||||
@@ -319,10 +250,13 @@ fn check_notification_write_order() {
|
||||
.map_or_else(|| "eu-west-2".to_owned(), |region| region.to_string());
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context.clone()));
|
||||
let fs = FS::with_server_ctx(slot.clone());
|
||||
let audit = MutationCompletionProbe::install(&bucket);
|
||||
let request = notification_request(&bucket, configuration(), &slot);
|
||||
let envelope = RequestEnvelope::from_s3s(&request);
|
||||
let result = observe_reload(&bucket, calls.clone(), put_notification(&slot, envelope, request.input)).await;
|
||||
let mut request = notification_request(&bucket, configuration(), &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut request)
|
||||
.await
|
||||
.unwrap();
|
||||
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
|
||||
let expected = expected_rules();
|
||||
let mut timeline = vec![Observation::Rules(
|
||||
Callback::Validate,
|
||||
@@ -354,7 +288,7 @@ fn check_notification_write_order() {
|
||||
};
|
||||
if let Some(message) = error_message {
|
||||
let error = result.expect_err("injected callback failure");
|
||||
assert_eq!(error.code().as_str(), S3ErrorCode::InternalError.as_str());
|
||||
assert_eq!(error.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(error.message(), Some(message));
|
||||
} else {
|
||||
result.expect("configuration written");
|
||||
@@ -373,13 +307,15 @@ fn check_notification_write_order() {
|
||||
assert_eq!(*recorder.active_rules.lock().unwrap(), expected);
|
||||
let mut invalid = configuration();
|
||||
invalid.lambda_function_configurations.as_mut().unwrap()[0].lambda_function_arn = "invalid-arn".into();
|
||||
let invalid = notification_request(&bucket, invalid, &slot);
|
||||
let invalid_env = RequestEnvelope::from_s3s(&invalid);
|
||||
let mut invalid = notification_request(&bucket, invalid, &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut invalid)
|
||||
.await
|
||||
.unwrap();
|
||||
calls.lock().unwrap().clear();
|
||||
let err = observe_reload(&bucket, calls.clone(), put_notification(&slot, invalid_env, invalid.input))
|
||||
let err = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, invalid))
|
||||
.await
|
||||
.expect_err("the last destination has an invalid ARN");
|
||||
assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str());
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert!(
|
||||
err.message()
|
||||
.unwrap()
|
||||
@@ -401,9 +337,11 @@ fn check_notification_write_order() {
|
||||
assert_eq!(output.queue_configurations[0].events[1].as_str(), "s3:Future:Unrecognized");
|
||||
|
||||
calls.lock().unwrap().clear();
|
||||
let empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
|
||||
let empty_env = RequestEnvelope::from_s3s(&empty);
|
||||
observe_reload(&bucket, calls.clone(), put_notification(&slot, empty_env, empty.input))
|
||||
let mut empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut empty)
|
||||
.await
|
||||
.unwrap();
|
||||
observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, empty))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
@@ -479,14 +417,16 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
|
||||
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let request = notification_request(&bucket, configuration(), &slot);
|
||||
let mut envelope = RequestEnvelope::from_s3s(&request);
|
||||
authorize_notification_put(&slot, &mut envelope, &bucket).await.unwrap();
|
||||
assert_eq!(bucket_config_mutation_incarnation(&envelope, &bucket).unwrap(), Some(first));
|
||||
let fs = FS::with_server_ctx(slot.clone());
|
||||
let mut request = notification_request(&bucket, configuration(), &slot);
|
||||
S3Access::put_bucket_notification_configuration(&fs, &mut request)
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(bucket_config_mutation_incarnation(&request, &bucket).unwrap(), Some(first));
|
||||
|
||||
let mismatch = DefaultBucketUsecase::without_context()
|
||||
.execute_put_bucket_notification_configuration(
|
||||
envelope.fork_for_authorization(),
|
||||
RequestEnvelope::from_s3s(&request),
|
||||
"other-bucket",
|
||||
vec![NotificationDestination {
|
||||
kind: "QueueConfiguration",
|
||||
@@ -509,22 +449,9 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let prepared =
|
||||
crate::storage::persisted_from_s3s(&request.input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
|
||||
let destinations = notification_destinations(request.input.notification_configuration);
|
||||
let result = observe_reload(
|
||||
&bucket,
|
||||
calls.clone(),
|
||||
DefaultBucketUsecase::with_context(slot.app_context()).execute_put_bucket_notification_configuration(
|
||||
envelope,
|
||||
&bucket,
|
||||
destinations,
|
||||
prepared,
|
||||
),
|
||||
)
|
||||
.await;
|
||||
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
|
||||
assert_eq!(read_notification_xml(&bucket).await, OLD_XML);
|
||||
assert_eq!(result.unwrap_err().code().as_str(), S3ErrorCode::NoSuchBucket.as_str());
|
||||
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::NoSuchBucket);
|
||||
assert!(
|
||||
matches!(calls.lock().unwrap().as_slice(), [Observation::Rules(Callback::Validate, _, _, xml)] if xml == OLD_XML)
|
||||
);
|
||||
|
||||
@@ -633,24 +633,6 @@ mod tests {
|
||||
assert!(err.message().is_some_and(|message| message.contains("peer reload failed")));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
|
||||
let error = usecase
|
||||
.execute_put_object_lock_configuration(
|
||||
envelope,
|
||||
"test-bucket",
|
||||
Some(ObjectLockConfiguration {
|
||||
object_lock_enabled: Some("Enabled".into()),
|
||||
rule: None,
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_get_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
|
||||
let input = gateway_dto::GetObjectLockConfigurationInput {
|
||||
|
||||
@@ -243,19 +243,27 @@ impl DefaultBucketUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::bucket_usecase::test_support::build_request;
|
||||
use crate::app::bucket_usecase::test_support::{build_request, s3_op_total};
|
||||
use crate::app::storage_api::s3::S3ErrorCode;
|
||||
use crate::app::storage_api::s3::dto::{DeleteBucketPolicyInput, PutBucketPolicyInput};
|
||||
use http::Method;
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_delete_bucket_policy_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::DELETE);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
async fn execute_delete_bucket_policy_records_s3_operation_before_store_lookup() {
|
||||
let input = DeleteBucketPolicyInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let err = usecase
|
||||
.execute_delete_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket")
|
||||
let req = build_request(input, Method::DELETE);
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
let before = s3_op_total(S3Operation::DeleteBucketPolicy);
|
||||
|
||||
let err = crate::app::storage_api::s3::S3::delete_bucket_policy(&fs, req)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(s3_op_total(S3Operation::DeleteBucketPolicy), before + 1);
|
||||
}
|
||||
|
||||
fn bucket(name: &str) -> rustfs_gateway_types::BucketName {
|
||||
@@ -292,13 +300,20 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_put_bucket_policy_returns_internal_error_when_store_uninitialized() {
|
||||
let req = build_request((), Method::PUT);
|
||||
let usecase = DefaultBucketUsecase::without_context();
|
||||
let input = PutBucketPolicyInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.policy("{}".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket", "{}")
|
||||
let req = build_request(input, Method::PUT);
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
let before = s3_op_total(S3Operation::PutBucketPolicy);
|
||||
|
||||
let err = crate::app::storage_api::s3::S3::put_bucket_policy(&fs, req)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
assert_eq!(s3_op_total(S3Operation::PutBucketPolicy), before + 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -211,17 +211,11 @@ mod tests {
|
||||
let mut req = build_request(input, Method::PUT);
|
||||
req.extensions.insert(ReqInfo::default());
|
||||
req.extensions.insert(slot.clone());
|
||||
let mut envelope = RequestEnvelope::from_s3s(&req);
|
||||
authorize_bucket_config_mutation(
|
||||
&slot,
|
||||
&mut envelope,
|
||||
&bucket,
|
||||
Action::S3Action(S3Action::PutBucketTaggingAction),
|
||||
)
|
||||
.await
|
||||
.expect("permission allows tag updates");
|
||||
authorize_bucket_config_mutation(&slot, &mut req, &bucket, Action::S3Action(S3Action::PutBucketTaggingAction))
|
||||
.await
|
||||
.expect("permission allows tag updates");
|
||||
let result = usecase
|
||||
.execute_put_bucket_tagging(envelope, &req.input.bucket, persisted)
|
||||
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, persisted)
|
||||
.await;
|
||||
if valid {
|
||||
result.expect("valid tags are persisted");
|
||||
|
||||
@@ -461,6 +461,35 @@ impl RequestEnvelope {
|
||||
pub fn residual_extensions(&self) -> &Extensions {
|
||||
&self.residual_extensions
|
||||
}
|
||||
|
||||
/// The payloads the connection, request and signature layers attached, back in the extensions
|
||||
/// they arrive in, for an edge that hands this request to a handler still taking those.
|
||||
pub(crate) fn transport_extensions(&self) -> Extensions {
|
||||
let mut extensions = self.residual_extensions.clone();
|
||||
extensions.insert(self.remote_addr.map(RemoteAddr));
|
||||
if let Some(client_info) = &self.client_info {
|
||||
extensions.insert(client_info.clone());
|
||||
}
|
||||
if let Some(native_tls) = self.native_tls_transport {
|
||||
extensions.insert(NativeTlsTransport(native_tls));
|
||||
}
|
||||
if let Some(request_context) = &self.request_context {
|
||||
extensions.insert(request_context.clone());
|
||||
}
|
||||
if let Some(server_context) = &self.server_context {
|
||||
extensions.insert(Arc::clone(server_context));
|
||||
}
|
||||
if let Some(body_read_control) = &self.body_read_control {
|
||||
extensions.insert(body_read_control.clone());
|
||||
}
|
||||
if let Some(verified) = self.verified_sigv4 {
|
||||
extensions.insert(verified);
|
||||
}
|
||||
if let Some(verified) = self.verified_presigned {
|
||||
extensions.insert(verified);
|
||||
}
|
||||
extensions
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg_attr(
|
||||
|
||||
@@ -28,6 +28,7 @@ use super::storage_api::test::contract::{
|
||||
namespace::NamespaceLocking as _,
|
||||
object::{ObjectIO as _, ObjectOperations as _},
|
||||
};
|
||||
use super::storage_api::test::ecfs::FS;
|
||||
use super::storage_api::test::object_utils::to_s3s_etag;
|
||||
use super::storage_api::test::runtime::{
|
||||
MockWarmBackend, MockWarmOp, get_global_transition_state, register_mock_tier as register_mock_tier_util,
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
|
||||
use super::common::*;
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::S3 as _;
|
||||
|
||||
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
|
||||
BucketLifecycleConfiguration {
|
||||
@@ -161,7 +162,8 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
put_lifecycle(req)
|
||||
FS::new()
|
||||
.put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect("Failed to update lifecycle configuration for delete marker cleanup");
|
||||
|
||||
@@ -197,7 +199,10 @@ async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
put_lifecycle(req).await.expect("Failed to update lifecycle configuration");
|
||||
FS::new()
|
||||
.put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect("Failed to update lifecycle configuration");
|
||||
|
||||
assert!(
|
||||
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
|
||||
@@ -226,7 +231,8 @@ async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_obje
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
put_lifecycle(req)
|
||||
FS::new()
|
||||
.put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect("ExpiredObjectDeleteMarker should be accepted on object-lock bucket");
|
||||
}
|
||||
@@ -252,9 +258,11 @@ async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_obj
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let err = put_lifecycle(req)
|
||||
let err: rustfs_s3_types::S3Error = FS::new()
|
||||
.put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket");
|
||||
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket")
|
||||
.into();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
let message = err.message().unwrap_or_default();
|
||||
assert!(
|
||||
@@ -286,9 +294,11 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let err = put_lifecycle(req)
|
||||
let err: rustfs_s3_types::S3Error = FS::new()
|
||||
.put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket");
|
||||
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket")
|
||||
.into();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
let message = err.message().unwrap_or_default();
|
||||
assert!(
|
||||
@@ -341,9 +351,11 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let err = put_lifecycle(req)
|
||||
let err: rustfs_s3_types::S3Error = FS::new()
|
||||
.put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect_err("zero-day expiration must be rejected with InvalidArgument");
|
||||
.expect_err("zero-day expiration must be rejected with InvalidArgument")
|
||||
.into();
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
let message = err.message().unwrap_or_default();
|
||||
assert!(
|
||||
@@ -459,6 +471,7 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
|
||||
),
|
||||
];
|
||||
|
||||
let fs = FS::new();
|
||||
let handled_count = || {
|
||||
rustfs_io_metrics::s3_op_metrics_snapshot()
|
||||
.into_iter()
|
||||
@@ -498,8 +511,8 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
|
||||
.extensions
|
||||
.insert(parity_test_support::bucket_config_mutation(&readable));
|
||||
}
|
||||
let error = put_lifecycle(request).await.unwrap_err();
|
||||
assert_eq!(error.code().as_str(), code.as_str(), "{name}: {error}");
|
||||
let error = fs.put_bucket_lifecycle_configuration(request).await.unwrap_err();
|
||||
assert_eq!(error.code(), &code, "{name}: {error}");
|
||||
if let Some(message) = message {
|
||||
assert_eq!(error.message(), Some(message), "{name}");
|
||||
}
|
||||
@@ -520,7 +533,9 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
|
||||
|
||||
let before = handled_count();
|
||||
let probe = MutationCompletionProbe::install(&readable);
|
||||
put_lifecycle(request(&readable, Some(("Disabled", false)))).await.unwrap();
|
||||
fs.put_bucket_lifecycle_configuration(request(&readable, Some(("Disabled", false))))
|
||||
.await
|
||||
.unwrap();
|
||||
assert_audit(&probe);
|
||||
assert_eq!(handled_count() - before, 1, "success counts once");
|
||||
|
||||
@@ -532,26 +547,3 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
|
||||
assert_eq!(rules[0].expiration.as_ref().unwrap().date.as_deref(), Some("2099-01-01T00:00:00.000Z"));
|
||||
assert_eq!(rules[1].expiration.as_ref().unwrap().days, Some(30));
|
||||
}
|
||||
|
||||
async fn put_lifecycle(req: S3Request<PutBucketLifecycleConfigurationInput>) -> rustfs_s3_types::S3Result<()> {
|
||||
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::PutBucketLifecycleConfiguration);
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
|
||||
let prepared = match req.input.lifecycle_configuration {
|
||||
None => crate::app::bucket_usecase::PreparedLifecycleConfiguration::Missing,
|
||||
Some(cfg) => {
|
||||
if crate::app::bucket_usecase::validate_lifecycle_rule_status(cfg.rules.iter().map(|rule| rule.status.as_str()))
|
||||
.is_err()
|
||||
{
|
||||
crate::app::bucket_usecase::PreparedLifecycleConfiguration::InvalidStatus
|
||||
} else {
|
||||
crate::app::bucket_usecase::PreparedLifecycleConfiguration::Prepared(
|
||||
crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s(cfg),
|
||||
)
|
||||
}
|
||||
}
|
||||
};
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_lifecycle_configuration(envelope, &req.input.bucket, prepared)
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
|
||||
@@ -28,10 +28,11 @@ use super::*;
|
||||
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn object_lock_handlers_schedule_replication() {
|
||||
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
|
||||
use crate::app::storage_api::s3::S3;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let object_meta = crate::app::object_meta::ObjectMetaUsecase::with_context(None);
|
||||
let fs = FS::new();
|
||||
|
||||
let bucket = format!("test-lock-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
let object = "locked/object.txt";
|
||||
@@ -51,7 +52,7 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
|
||||
// PutObjectRetention must compute a replication decision.
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
let _retention_input = PutObjectRetentionInput::builder()
|
||||
let retention_input = PutObjectRetentionInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.retention(Some(ObjectLockRetention {
|
||||
@@ -60,18 +61,7 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request((), Method::PUT);
|
||||
object_meta
|
||||
.put_object_retention_fields(
|
||||
crate::app::RequestEnvelope::from_s3s(&req),
|
||||
&bucket,
|
||||
object,
|
||||
None,
|
||||
Some(crate::storage::request_markers::RequestedObjectLockRetention {
|
||||
mode: Some(ObjectLockRetentionMode::COMPLIANCE.to_owned()),
|
||||
retain_until: Some(time::macros::datetime!(2030-01-01 00:00:00 UTC)),
|
||||
}),
|
||||
)
|
||||
fs.put_object_retention(build_request(retention_input, Method::PUT))
|
||||
.await
|
||||
.expect("put_object_retention should succeed");
|
||||
assert_eq!(
|
||||
@@ -83,7 +73,7 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
|
||||
// A bucket without a replication configuration can skip the decision.
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
let _no_replication_input = PutObjectLegalHoldInput::builder()
|
||||
let no_replication_input = PutObjectLegalHoldInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.legal_hold(Some(ObjectLockLegalHold {
|
||||
@@ -91,17 +81,7 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request((), Method::PUT);
|
||||
object_meta
|
||||
.put_object_legal_hold_fields(
|
||||
crate::app::RequestEnvelope::from_s3s(&req),
|
||||
&bucket,
|
||||
object,
|
||||
None,
|
||||
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
|
||||
ObjectLockLegalHoldStatus::OFF.to_owned(),
|
||||
))),
|
||||
)
|
||||
fs.put_object_legal_hold(build_request(no_replication_input, Method::PUT))
|
||||
.await
|
||||
.expect("legal hold without replication should succeed");
|
||||
assert_eq!(
|
||||
@@ -113,7 +93,7 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
// A configured bucket must still evaluate the metadata replication decision.
|
||||
install_delete_replication_config(&bucket, true, false, None).await;
|
||||
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
|
||||
let _legal_hold_input = PutObjectLegalHoldInput::builder()
|
||||
let legal_hold_input = PutObjectLegalHoldInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.key(object.to_string())
|
||||
.legal_hold(Some(ObjectLockLegalHold {
|
||||
@@ -121,17 +101,7 @@ async fn object_lock_handlers_schedule_replication() {
|
||||
}))
|
||||
.build()
|
||||
.unwrap();
|
||||
let req = build_request((), Method::PUT);
|
||||
object_meta
|
||||
.put_object_legal_hold_fields(
|
||||
crate::app::RequestEnvelope::from_s3s(&req),
|
||||
&bucket,
|
||||
object,
|
||||
None,
|
||||
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
|
||||
ObjectLockLegalHoldStatus::ON.to_owned(),
|
||||
))),
|
||||
)
|
||||
fs.put_object_legal_hold(build_request(legal_hold_input, Method::PUT))
|
||||
.await
|
||||
.expect("put_object_legal_hold should succeed");
|
||||
assert_eq!(
|
||||
|
||||
@@ -29,7 +29,7 @@ use crate::app::RequestEnvelope;
|
||||
use crate::app::bucket_usecase::DefaultBucketUsecase;
|
||||
use crate::app::storage_api::s3::dto::{EncodingType, ListObjectVersionsInput, ListObjectsV2Input};
|
||||
use crate::app::storage_api::s3::xml;
|
||||
use crate::storage::access_owned::listing_multipart::authorize_bucket_listing;
|
||||
use crate::storage::access::access_owned::listing_multipart::authorize_bucket_listing;
|
||||
use crate::storage::authorization::AuthorizationRequest;
|
||||
use http::{HeaderMap, Uri};
|
||||
use rustfs_policy::policy::action::S3Action;
|
||||
|
||||
@@ -22,8 +22,6 @@
|
||||
|
||||
pub mod admin_usecase;
|
||||
mod answered;
|
||||
#[cfg(test)]
|
||||
mod audit_coverage_test;
|
||||
pub(crate) mod bucket_list_through;
|
||||
pub mod bucket_usecase;
|
||||
pub mod context;
|
||||
|
||||
@@ -50,7 +50,8 @@ use super::storage_api::multipart_usecase::io::{
|
||||
EtagAlgorithm, HashReader, WriteEncryption, WritePlan, compression_metadata_value,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag};
|
||||
use super::storage_api::multipart_usecase::object_utils::to_s3s_etag;
|
||||
use super::storage_api::multipart_usecase::object_utils::to_s3s_xml_etag;
|
||||
use super::storage_api::multipart_usecase::options::{
|
||||
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
|
||||
get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata,
|
||||
@@ -103,7 +104,6 @@ use futures::StreamExt;
|
||||
use http::{HeaderMap, Uri};
|
||||
use metrics::counter;
|
||||
use rustfs_s3_ops::S3Operation;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use rustfs_targets::EventName;
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
#[cfg(test)]
|
||||
@@ -115,6 +115,7 @@ use rustfs_utils::http::{
|
||||
headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
|
||||
insert_str,
|
||||
};
|
||||
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -192,13 +193,12 @@ fn multipart_max_total_object_size(metadata: &HashMap<String, String>) -> S3Resu
|
||||
let value = get_consistent_str(metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE).ok_or_else(|| {
|
||||
S3Error::with_message(
|
||||
S3ErrorCode::InvalidRequest,
|
||||
"multipart size capability metadata is missing or inconsistent",
|
||||
"multipart size capability metadata is missing or inconsistent".to_string(),
|
||||
)
|
||||
})?;
|
||||
value
|
||||
.parse::<u64>()
|
||||
.map(Some)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid"))
|
||||
value.parse::<u64>().map(Some).map_err(|_| {
|
||||
S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid".to_string())
|
||||
})
|
||||
}
|
||||
|
||||
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
|
||||
@@ -291,11 +291,7 @@ impl DefaultMultipartUsecase {
|
||||
/// or completed".
|
||||
fn multipart_upload_api_error(err: StorageError) -> S3Error {
|
||||
if matches!(err, StorageError::MalformedUploadID(_)) {
|
||||
let mut error = S3Error::new(S3ErrorCode::NoSuchUpload);
|
||||
if let Some(message) = error.code().default_message() {
|
||||
error.set_message(message);
|
||||
}
|
||||
return error;
|
||||
return S3Error::new(S3ErrorCode::NoSuchUpload);
|
||||
}
|
||||
ApiError::from(err).into()
|
||||
}
|
||||
@@ -549,7 +545,7 @@ mod tests {
|
||||
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
|
||||
.await
|
||||
.expect_err("reserved key must be rejected before accessing storage");
|
||||
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
let complete = CompleteMultipartUploadInput::builder()
|
||||
.bucket("bucket".to_owned())
|
||||
@@ -597,9 +593,6 @@ mod tests {
|
||||
fn multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload() {
|
||||
let malformed = multipart_upload_api_error(StorageError::MalformedUploadID("bucket/object-not-an-id".to_string()));
|
||||
assert_eq!(malformed.code(), &S3ErrorCode::NoSuchUpload);
|
||||
let legacy = crate::app::storage_api::s3::S3Error::new(crate::app::storage_api::s3::S3ErrorCode::NoSuchUpload);
|
||||
assert_eq!(malformed.message(), legacy.message());
|
||||
assert_eq!(malformed.status_code(), legacy.status_code());
|
||||
|
||||
let unknown = multipart_upload_api_error(StorageError::InvalidUploadID(
|
||||
"bucket".to_string(),
|
||||
|
||||
@@ -61,7 +61,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
|
||||
req.uri = format!("/{bucket}/{key}").parse().expect("fixture URI");
|
||||
req.extensions.insert(slot.clone());
|
||||
req.extensions.insert(request_context.clone());
|
||||
RequestEnvelope::from_s3s(&req)
|
||||
req
|
||||
};
|
||||
for name in [&bucket, &other] {
|
||||
store
|
||||
@@ -143,7 +143,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
|
||||
(wrong, S3ErrorCode::InternalError),
|
||||
] {
|
||||
let err = usecase
|
||||
.execute_abort_multipart_upload(req, Ok(target()))
|
||||
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&req), Ok(target()))
|
||||
.await
|
||||
.expect_err("Abort requires current generation");
|
||||
assert_eq!(err.code(), &code);
|
||||
@@ -162,7 +162,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
|
||||
.expect("capture current bucket");
|
||||
current.install_bucket_generation(captured);
|
||||
usecase
|
||||
.execute_abort_multipart_upload(current.fork_for_authorization(), Ok(target()))
|
||||
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(¤t), Ok(target()))
|
||||
.await
|
||||
.expect("current generation allows Abort");
|
||||
assert!(matches!(
|
||||
@@ -182,7 +182,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
|
||||
.expect("create legacy-path upload");
|
||||
usecase
|
||||
.execute_abort_multipart_upload(
|
||||
current,
|
||||
RequestEnvelope::from_s3s(¤t),
|
||||
Ok(AbortMultipartUploadTarget::Legacy {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_owned(),
|
||||
|
||||
@@ -294,8 +294,10 @@ impl DefaultMultipartUsecase {
|
||||
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
|
||||
// The input member and the header are one value on the wire; both are checked, as the
|
||||
// legacy guard checks both.
|
||||
let specific_if_none_match = if_none_match.as_deref().is_some_and(|condition| condition.trim() != "*");
|
||||
reject_non_wildcard_write_if_none_match(specific_if_none_match, headers)?;
|
||||
if if_none_match.as_deref().is_some_and(|condition| condition.trim() != "*") {
|
||||
return Err(S3Error::new(S3ErrorCode::NotImplemented));
|
||||
}
|
||||
reject_non_wildcard_write_if_none_match(None, headers)?;
|
||||
validate_table_catalog_object_mutation(&bucket, &key).await?;
|
||||
|
||||
if if_match.is_some() || if_none_match.is_some() {
|
||||
@@ -734,7 +736,6 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[cfg(test)]
|
||||
impl DefaultMultipartUsecase {
|
||||
/// CompleteMultipartUpload for an s3s request: its facts in an envelope and its input across
|
||||
@@ -743,41 +744,9 @@ impl DefaultMultipartUsecase {
|
||||
&self,
|
||||
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CompleteMultipartUploadInput>,
|
||||
) -> S3Result<CompleteMultipartUploadOutput> {
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
|
||||
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let mut s3s_input = req.input;
|
||||
let has_upload = s3s_input.multipart_upload.is_some();
|
||||
s3s_input.multipart_upload.get_or_insert_with(Default::default);
|
||||
let tags: Vec<Option<String>> = s3s_input
|
||||
.multipart_upload
|
||||
.iter_mut()
|
||||
.flat_map(|upload| upload.parts.iter_mut().flatten())
|
||||
.map(|part| {
|
||||
part.part_number.get_or_insert(0);
|
||||
part.e_tag.take().map(|tag| tag.value().to_owned())
|
||||
})
|
||||
.collect();
|
||||
for checksum in [
|
||||
&mut s3s_input.checksum_crc32,
|
||||
&mut s3s_input.checksum_crc32c,
|
||||
&mut s3s_input.checksum_crc64nvme,
|
||||
&mut s3s_input.checksum_md5,
|
||||
&mut s3s_input.checksum_sha1,
|
||||
&mut s3s_input.checksum_sha256,
|
||||
&mut s3s_input.checksum_sha512,
|
||||
&mut s3s_input.checksum_xxhash128,
|
||||
&mut s3s_input.checksum_xxhash3,
|
||||
&mut s3s_input.checksum_xxhash64,
|
||||
] {
|
||||
*checksum = None;
|
||||
}
|
||||
let mut input = complete_multipart_upload::input_from_s3s(s3s_input).map_err(crate::app::dto_values::seam_error)?;
|
||||
for (part, tag) in input.multipart_upload.parts.iter_mut().zip(tags) {
|
||||
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
|
||||
}
|
||||
let parts = std::mem::take(&mut input.multipart_upload);
|
||||
let parts = has_upload.then_some(parts);
|
||||
let (input, parts) =
|
||||
crate::storage::ecfs::complete_multipart_upload_input(req.input).map_err(crate::app::dto_values::seam_error)?;
|
||||
match parts {
|
||||
Some(parts) => {
|
||||
let mut input = input;
|
||||
@@ -1124,18 +1093,23 @@ mod tests {
|
||||
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
|
||||
let (_store, bucket) =
|
||||
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
|
||||
let versioning_req = build_request((), Method::PUT);
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_versioning(
|
||||
RequestEnvelope::from_s3s(&versioning_req),
|
||||
&bucket,
|
||||
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
|
||||
status: Some("Enabled".to_owned()),
|
||||
..Default::default()
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning for encrypted multipart listing");
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
crate::app::storage_api::s3::S3::put_bucket_versioning(
|
||||
&fs,
|
||||
build_request(
|
||||
crate::app::storage_api::s3::dto::PutBucketVersioningInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.versioning_configuration(crate::app::storage_api::s3::dto::VersioningConfiguration {
|
||||
status: Some(crate::app::storage_api::s3::dto::BucketVersioningStatus::from_static("Enabled")),
|
||||
..Default::default()
|
||||
})
|
||||
.build()
|
||||
.expect("bucket versioning input"),
|
||||
Method::PUT,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("enable versioning for encrypted multipart listing");
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let key = "customer-encrypted";
|
||||
let customer_key = "KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio=";
|
||||
@@ -1253,9 +1227,9 @@ mod tests {
|
||||
let etag = etag_to_s3s(&response.e_tag.expect("completed encrypted object ETag"));
|
||||
let version_id = response.version_id.expect("completed object version");
|
||||
assert_ne!(version_id, "null");
|
||||
assert_multipart_etag_listings(&bucket, key, &etag, &version_id).await;
|
||||
let head = crate::app::test_seams::head_object(
|
||||
&crate::app::object::DefaultObjectUsecase::from_global(),
|
||||
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
|
||||
let head = crate::app::storage_api::s3::S3::head_object(
|
||||
&fs,
|
||||
build_request(
|
||||
crate::app::storage_api::s3::dto::HeadObjectInput {
|
||||
bucket: bucket.clone(),
|
||||
@@ -1285,10 +1259,15 @@ mod tests {
|
||||
});
|
||||
}
|
||||
|
||||
async fn assert_multipart_etag_listings(bucket: &str, key: &str, etag: &ETag, version_id: &str) {
|
||||
async fn assert_multipart_etag_listings(
|
||||
fs: &crate::storage::ecfs::FS,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
etag: &ETag,
|
||||
version_id: &str,
|
||||
) {
|
||||
use crate::app::storage_api::s3::dto::{ListObjectVersionsInput, ListObjectsInput, ListObjectsV2Input};
|
||||
use crate::app::storage_api::s3::xml::{Serialize, Serializer};
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::{list_object_versions, list_objects};
|
||||
|
||||
fn assert_xml_etag<T: Serialize>(output: &T, etag: &ETag) {
|
||||
let mut bytes = Vec::new();
|
||||
@@ -1302,25 +1281,21 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
let bucket_usecase = crate::app::bucket_usecase::DefaultBucketUsecase::from_global();
|
||||
// Listings carry neither the SSE-C key nor a per-object decryption request.
|
||||
let versions_req = build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
);
|
||||
let versions_env = RequestEnvelope::from_s3s(&versions_req);
|
||||
let versions_in = list_object_versions::input_from_s3s(versions_req.input).expect("convert ListObjectVersionsInput");
|
||||
let versions = list_object_versions::output_to_s3s(
|
||||
bucket_usecase
|
||||
.execute_list_object_versions(versions_env, versions_in)
|
||||
.await
|
||||
.expect("keyless version listing"),
|
||||
let versions = crate::app::storage_api::s3::S3::list_object_versions(
|
||||
fs,
|
||||
build_request(
|
||||
ListObjectVersionsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.expect("convert ListObjectVersionsOutput");
|
||||
.await
|
||||
.expect("keyless version listing")
|
||||
.output;
|
||||
let entries = versions.versions.as_ref().expect("completed version must be listed");
|
||||
assert_eq!(entries.len(), 1, "completion replay must preserve a single version");
|
||||
assert_eq!(entries[0].key.as_deref(), Some(key));
|
||||
@@ -1328,25 +1303,22 @@ mod tests {
|
||||
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
|
||||
assert_xml_etag(&versions, etag);
|
||||
|
||||
let v1_req = build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
);
|
||||
let v1_env = RequestEnvelope::from_s3s(&v1_req);
|
||||
let v1_in = list_objects::input_from_s3s(v1_req.input).expect("convert ListObjectsInput");
|
||||
let v1 = list_objects::output_to_s3s(
|
||||
bucket_usecase
|
||||
.execute_list_objects(v1_env, v1_in)
|
||||
.await
|
||||
.expect("keyless V1 listing"),
|
||||
let v1 = crate::app::storage_api::s3::S3::list_objects(
|
||||
fs,
|
||||
build_request(
|
||||
ListObjectsInput {
|
||||
bucket: bucket.to_string(),
|
||||
prefix: Some(key.to_string()),
|
||||
..Default::default()
|
||||
},
|
||||
Method::GET,
|
||||
),
|
||||
)
|
||||
.expect("convert ListObjectsOutput");
|
||||
let v2 = crate::app::test_seams::list_objects_v2(
|
||||
&bucket_usecase,
|
||||
.await
|
||||
.expect("keyless V1 listing")
|
||||
.output;
|
||||
let v2 = crate::app::storage_api::s3::S3::list_objects_v2(
|
||||
fs,
|
||||
build_request(
|
||||
ListObjectsV2Input {
|
||||
bucket: bucket.to_string(),
|
||||
@@ -1410,41 +1382,52 @@ mod tests {
|
||||
)
|
||||
.await
|
||||
.expect("create versioned Object Lock bucket");
|
||||
let fs = crate::storage::ecfs::FS::new();
|
||||
let key = "kms-encrypted";
|
||||
let enc_req = build_request((), Method::PUT);
|
||||
let enc_cfg = crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
|
||||
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
|
||||
apply_server_side_encryption_by_default: Some(
|
||||
crate::app::storage_api::s3::dto::ServerSideEncryptionByDefault {
|
||||
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
kms_master_key_id: Some(kms_key.key_id.clone()),
|
||||
},
|
||||
),
|
||||
..Default::default()
|
||||
}],
|
||||
};
|
||||
let prepared = crate::storage::persisted_from_s3s(&enc_cfg).map_err(Into::into);
|
||||
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
|
||||
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&enc_req), &bucket, prepared)
|
||||
.await
|
||||
.expect("configure bucket-default SSE-KMS");
|
||||
let usecase = DefaultMultipartUsecase::from_global();
|
||||
let upload_id = usecase
|
||||
.execute_create_multipart_upload_from_s3s(build_request(
|
||||
crate::app::storage_api::s3::S3::put_bucket_encryption(
|
||||
&fs,
|
||||
build_request(
|
||||
crate::app::storage_api::s3::dto::PutBucketEncryptionInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.server_side_encryption_configuration(
|
||||
crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
|
||||
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
|
||||
apply_server_side_encryption_by_default: Some(
|
||||
crate::app::storage_api::s3::dto::ServerSideEncryptionByDefault {
|
||||
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
kms_master_key_id: Some(kms_key.key_id.clone()),
|
||||
},
|
||||
),
|
||||
..Default::default()
|
||||
}],
|
||||
},
|
||||
)
|
||||
.build()
|
||||
.expect("bucket encryption input"),
|
||||
Method::PUT,
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("configure bucket-default SSE-KMS");
|
||||
let upload_id = crate::app::storage_api::s3::S3::create_multipart_upload(
|
||||
&fs,
|
||||
build_request(
|
||||
CreateMultipartUploadInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
Method::POST,
|
||||
))
|
||||
.await
|
||||
.expect("create KMS multipart upload")
|
||||
.output
|
||||
.upload_id
|
||||
.expect("KMS upload id");
|
||||
let part = usecase
|
||||
.execute_upload_part(build_request(
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("create KMS multipart upload")
|
||||
.output
|
||||
.upload_id
|
||||
.expect("KMS upload id");
|
||||
let part = crate::app::storage_api::s3::S3::upload_part(
|
||||
&fs,
|
||||
build_request(
|
||||
UploadPartInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
@@ -1455,10 +1438,11 @@ mod tests {
|
||||
..Default::default()
|
||||
},
|
||||
Method::PUT,
|
||||
))
|
||||
.await
|
||||
.expect("upload encrypted KMS part")
|
||||
.output;
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("upload encrypted KMS part")
|
||||
.output;
|
||||
let complete = CompleteMultipartUploadInput {
|
||||
bucket: bucket.clone(),
|
||||
key: key.to_string(),
|
||||
@@ -1473,16 +1457,19 @@ mod tests {
|
||||
..Default::default()
|
||||
};
|
||||
for _ in 0..2 {
|
||||
let response = usecase
|
||||
.complete_for_s3s_request(build_request(complete.clone(), Method::POST))
|
||||
.await
|
||||
.expect("complete and replay KMS multipart upload");
|
||||
let etag = etag_to_s3s(&response.e_tag.expect("KMS completed ETag"));
|
||||
let response = crate::app::storage_api::s3::S3::complete_multipart_upload(
|
||||
&fs,
|
||||
build_request(complete.clone(), Method::POST),
|
||||
)
|
||||
.await
|
||||
.expect("complete and replay KMS multipart upload")
|
||||
.output;
|
||||
let etag = response.e_tag.expect("KMS completed ETag");
|
||||
let version_id = response.version_id.expect("KMS completed version");
|
||||
assert_ne!(version_id, "null");
|
||||
assert_multipart_etag_listings(&bucket, key, &etag, &version_id).await;
|
||||
let head = crate::app::test_seams::head_object(
|
||||
&crate::app::object::DefaultObjectUsecase::from_global(),
|
||||
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
|
||||
let head = crate::app::storage_api::s3::S3::head_object(
|
||||
&fs,
|
||||
build_request(
|
||||
crate::app::storage_api::s3::dto::HeadObjectInput {
|
||||
bucket: bucket.clone(),
|
||||
|
||||
@@ -455,58 +455,6 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl CreateMultipartUploadFields {
|
||||
pub(crate) fn from_legacy(input: crate::app::storage_api::s3::dto::CreateMultipartUploadInput) -> Self {
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
metadata: input.metadata,
|
||||
tagging: input.tagging,
|
||||
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
|
||||
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
ssekms_key_id: input.ssekms_key_id,
|
||||
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
|
||||
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
|
||||
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DefaultMultipartUsecase {
|
||||
pub(crate) async fn execute_create_multipart_upload_from_s3s(
|
||||
&self,
|
||||
mut req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CreateMultipartUploadInput>,
|
||||
) -> crate::app::storage_api::s3::S3Result<
|
||||
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::CreateMultipartUploadOutput>,
|
||||
> {
|
||||
use crate::app::storage_api::s3::{S3Response, dto::CreateMultipartUploadOutput};
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let query_version = req.input.version_id.take();
|
||||
let output = self
|
||||
.execute_create_multipart_upload(envelope, CreateMultipartUploadFields::from_legacy(req.input), query_version)
|
||||
.await?;
|
||||
let mut response = S3Response::new(CreateMultipartUploadOutput {
|
||||
bucket: Some(output.bucket),
|
||||
key: Some(output.key),
|
||||
upload_id: Some(output.upload_id),
|
||||
server_side_encryption: output.server_side_encryption.map(Into::into),
|
||||
sse_customer_algorithm: output.sse_customer_algorithm,
|
||||
sse_customer_key_md5: output.sse_customer_key_md5,
|
||||
ssekms_key_id: output.ssekms_key_id,
|
||||
checksum_algorithm: output.checksum_algorithm.map(Into::into),
|
||||
checksum_type: output.checksum_type.map(Into::into),
|
||||
..Default::default()
|
||||
});
|
||||
response.headers.extend(output.response_headers);
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
@@ -783,19 +731,11 @@ mod tests {
|
||||
.expect("create bucket");
|
||||
}
|
||||
|
||||
let make_request = |target_bucket: &str| {
|
||||
let mut req = request(target_bucket);
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let query_version = req.input.version_id.take();
|
||||
let fields = CreateMultipartUploadFields::from_legacy(req.input);
|
||||
(envelope, fields, query_version)
|
||||
};
|
||||
|
||||
let (mut stale_env, stale_fields, stale_ver) = make_request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &stale_env, &bucket)
|
||||
let mut stale = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &stale, &bucket)
|
||||
.await
|
||||
.expect("capture original incarnation");
|
||||
stale_env.install_bucket_generation(guard);
|
||||
stale.install_bucket_generation(guard);
|
||||
let original = store.bucket_incarnation_id(&bucket).await.expect("old bucket generation");
|
||||
|
||||
store
|
||||
@@ -809,18 +749,18 @@ mod tests {
|
||||
let current = store.bucket_incarnation_id(&bucket).await.expect("new bucket generation");
|
||||
assert_ne!(original, current);
|
||||
|
||||
let (missing_env, missing_fields, missing_ver) = make_request(&bucket);
|
||||
let (mut wrong_env, wrong_fields, wrong_ver) = make_request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &wrong_env, &other)
|
||||
let missing = request(&bucket);
|
||||
let mut wrong = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &wrong, &other)
|
||||
.await
|
||||
.expect("capture the other bucket");
|
||||
wrong_env.install_bucket_generation(guard);
|
||||
for (envelope, fields, query_version, code) in [
|
||||
(stale_env, stale_fields, stale_ver, S3ErrorCode::NoSuchBucket),
|
||||
(missing_env, missing_fields, missing_ver, S3ErrorCode::InternalError),
|
||||
(wrong_env, wrong_fields, wrong_ver, S3ErrorCode::InternalError),
|
||||
wrong.install_bucket_generation(guard);
|
||||
for (request, code) in [
|
||||
(stale, S3ErrorCode::NoSuchBucket),
|
||||
(missing, S3ErrorCode::InternalError),
|
||||
(wrong, S3ErrorCode::InternalError),
|
||||
] {
|
||||
let result = usecase.execute_create_multipart_upload(envelope, fields, query_version).await;
|
||||
let result = usecase.execute_create_multipart_upload_from_s3s(request).await;
|
||||
assert!(
|
||||
store
|
||||
.list_multipart_uploads(&bucket, "", None, None, None, 100)
|
||||
@@ -829,20 +769,18 @@ mod tests {
|
||||
.uploads
|
||||
.is_empty()
|
||||
);
|
||||
let Err(error) = result else {
|
||||
panic!("creation requires the target incarnation");
|
||||
};
|
||||
assert_eq!(error.code().as_str(), code.as_str());
|
||||
let error = result.expect_err("creation requires the target incarnation");
|
||||
assert_eq!(error.code(), &code);
|
||||
}
|
||||
|
||||
let probe = crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe::install(&bucket);
|
||||
let (mut allowed_env, allowed_fields, allowed_ver) = make_request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &allowed_env, &bucket)
|
||||
let mut allowed = request(&bucket);
|
||||
let guard = load_bucket_generation_from_store(&store, &allowed, &bucket)
|
||||
.await
|
||||
.expect("capture current incarnation");
|
||||
allowed_env.install_bucket_generation(guard);
|
||||
let output = usecase
|
||||
.execute_create_multipart_upload(allowed_env, allowed_fields, allowed_ver)
|
||||
allowed.install_bucket_generation(guard);
|
||||
let response = usecase
|
||||
.execute_create_multipart_upload_from_s3s(allowed)
|
||||
.await
|
||||
.expect("a current guard allows creation");
|
||||
let uploads = store
|
||||
@@ -851,7 +789,7 @@ mod tests {
|
||||
.expect("list durable uploads")
|
||||
.uploads;
|
||||
assert_eq!(uploads.len(), 1);
|
||||
assert_eq!(uploads[0].upload_id.as_str(), output.upload_id.as_str());
|
||||
assert_eq!(Some(uploads[0].upload_id.as_str()), response.output.upload_id.as_deref());
|
||||
assert_eq!(probe.audits.lock().unwrap().len(), 1);
|
||||
assert!(probe.events.lock().unwrap().is_empty());
|
||||
});
|
||||
|
||||
@@ -26,8 +26,11 @@ use rustfs_s3_types::S3Result;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode};
|
||||
|
||||
pub(crate) enum ListPartsTarget {
|
||||
#[allow(dead_code)]
|
||||
Legacy { bucket: String, key: String, upload_id: String },
|
||||
Legacy {
|
||||
bucket: String,
|
||||
key: String,
|
||||
upload_id: String,
|
||||
},
|
||||
Gateway {
|
||||
bucket: BucketName,
|
||||
key: ObjectKey,
|
||||
@@ -79,7 +82,7 @@ where
|
||||
Ok::<_, StorageError>(page.recorded_owner.take().map(|owner| RecordedListParts { owner, page }))
|
||||
})
|
||||
.await
|
||||
.map_err(multipart_upload_api_error)?
|
||||
.map_err(|error| S3Error::from(multipart_upload_api_error(error)))?
|
||||
.map_err(|refusal| S3Error::with_message(S3ErrorCode::NoSuchUpload, refusal.reason()))?;
|
||||
Ok(record.page)
|
||||
}
|
||||
@@ -198,7 +201,7 @@ impl DefaultMultipartUsecase {
|
||||
ListPartsTarget::Legacy { upload_id, .. } => store
|
||||
.list_object_parts(bucket, key, upload_id, params.part_number_marker, params.max_parts, &opts)
|
||||
.await
|
||||
.map_err(multipart_upload_api_error),
|
||||
.map_err(|error| multipart_upload_api_error(error).into()),
|
||||
ListPartsTarget::Gateway {
|
||||
bucket: bucket_name,
|
||||
key: object_key,
|
||||
|
||||
@@ -165,7 +165,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
|
||||
storage_error,
|
||||
StorageError::InvalidUploadID(..) | StorageError::MalformedUploadID(..)
|
||||
));
|
||||
let expected: LegacyS3Error = ApiError::from(storage_error).into();
|
||||
let expected = multipart_upload_api_error(storage_error);
|
||||
for target in [
|
||||
gateway_fields(&bucket, key, id, None, None),
|
||||
ListPartsFields {
|
||||
@@ -217,11 +217,10 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
|
||||
let req = || {
|
||||
let mut req = build_request((), Method::GET);
|
||||
req.extensions.insert(slot.clone());
|
||||
RequestEnvelope::from_s3s(&req)
|
||||
req
|
||||
};
|
||||
let capability = |query: &str| {
|
||||
let mut request = build_request((), Method::GET);
|
||||
request.extensions.insert(slot.clone());
|
||||
let mut request = req();
|
||||
request.uri = format!("/fixture?{query}").parse().expect("fixture URI");
|
||||
RequestEnvelope::from_s3s(&request)
|
||||
};
|
||||
@@ -270,7 +269,7 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_VEEAM_SOSAPI_ENABLE, Some("true"))], async {
|
||||
let err = execute_counted(
|
||||
&usecase,
|
||||
veeam_request,
|
||||
RequestEnvelope::from_s3s(&veeam_request),
|
||||
gateway_fields(
|
||||
&bucket,
|
||||
".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml",
|
||||
@@ -295,8 +294,13 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
|
||||
if let Some(guard) = guard {
|
||||
request.install_bucket_generation(guard);
|
||||
}
|
||||
let result =
|
||||
execute_counted(&usecase, request, gateway_fields(&bucket, key, &upload.upload_id, marker, None), lookups).await;
|
||||
let result = execute_counted(
|
||||
&usecase,
|
||||
RequestEnvelope::from_s3s(&request),
|
||||
gateway_fields(&bucket, key, &upload.upload_id, marker, None),
|
||||
lookups,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(result.as_ref().err().map(|err| err.code().as_str()), code);
|
||||
if let Ok(page) = result {
|
||||
assert!(page.upload_id == upload.upload_id);
|
||||
|
||||
@@ -43,7 +43,7 @@ impl RecordedUpload for PartUpload {
|
||||
|
||||
async fn lookup_part_upload(store: &ECStore, bucket: &str, key: &str, id: &str, opts: &ObjectOptions) -> S3Result<MultipartInfo> {
|
||||
let result = store.get_multipart_info(bucket, key, id, opts).await;
|
||||
result.map_err(multipart_upload_api_error)
|
||||
result.map_err(|error| multipart_upload_api_error(error).into())
|
||||
}
|
||||
|
||||
pub(super) async fn lookup_recorded_part_upload(
|
||||
@@ -56,7 +56,7 @@ pub(super) async fn lookup_recorded_part_upload(
|
||||
let result = store.lookup_multipart_part_binding(bucket, key, id, opts).await;
|
||||
#[cfg(test)]
|
||||
let result = tests::observe_lookup(result);
|
||||
match result.map_err(multipart_upload_api_error) {
|
||||
match result.map_err(|error| S3Error::from(multipart_upload_api_error(error))) {
|
||||
Ok((info, owner)) => Ok(owner.map(|owner| PartUpload { owner, info })),
|
||||
Err(error) if error.code() == &S3ErrorCode::NoSuchUpload => Ok(None),
|
||||
Err(error) => Err(error),
|
||||
@@ -148,18 +148,6 @@ fn normalize_presigned_part_checksums(req: &mut RequestEnvelope, input: &mut Upl
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
/// Records the gateway audit. The protocol client records its own projected response.
|
||||
pub(crate) async fn execute_upload_part_with_audit(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
input: UploadPartWriteInput,
|
||||
) -> S3Result<UploadPartWriteOutput> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::UploadPart, http::StatusCode::OK, |envelope| {
|
||||
self.execute_upload_part_owned(envelope, input)
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(name = "execute_upload_part", level = "debug", skip(self, req, input))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub(crate) async fn execute_upload_part_owned(
|
||||
@@ -513,72 +501,6 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl UploadPartWriteInput {
|
||||
pub(crate) fn from_legacy(
|
||||
input: crate::app::storage_api::s3::dto::UploadPartInput,
|
||||
trailers: Option<crate::app::storage_api::s3::TrailingHeaders>,
|
||||
) -> Self {
|
||||
Self {
|
||||
target: UploadPartTarget::Legacy {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
upload_id: input.upload_id,
|
||||
},
|
||||
part_number: input.part_number,
|
||||
content_length: input.content_length,
|
||||
body: input.body.map(crate::app::object::legacy_put_body),
|
||||
trailing_headers: crate::app::trailer_adapter::trailer_source(trailers),
|
||||
frame_trailers: None,
|
||||
content_md5: input.content_md5,
|
||||
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
|
||||
checksum_crc32: input.checksum_crc32,
|
||||
checksum_crc32c: input.checksum_crc32c,
|
||||
checksum_crc64nvme: input.checksum_crc64nvme,
|
||||
checksum_sha1: input.checksum_sha1,
|
||||
checksum_sha256: input.checksum_sha256,
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DefaultMultipartUsecase {
|
||||
pub async fn execute_upload_part(
|
||||
&self,
|
||||
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::UploadPartInput>,
|
||||
) -> crate::app::storage_api::s3::S3Result<
|
||||
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::UploadPartOutput>,
|
||||
> {
|
||||
use crate::app::storage_api::s3::{
|
||||
S3Response,
|
||||
dto::{ServerSideEncryption, UploadPartOutput},
|
||||
};
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let input = UploadPartWriteInput::from_legacy(req.input, req.trailing_headers);
|
||||
let output = self.execute_upload_part_with_audit(envelope, input).await?;
|
||||
let mut response = S3Response::new(UploadPartOutput {
|
||||
e_tag: output
|
||||
.e_tag
|
||||
.map(|tag| crate::app::storage_api::test::object_utils::to_s3s_etag(&tag)),
|
||||
checksum_crc32: output.checksum_crc32,
|
||||
checksum_crc32c: output.checksum_crc32c,
|
||||
checksum_crc64nvme: output.checksum_crc64nvme,
|
||||
checksum_sha1: output.checksum_sha1,
|
||||
checksum_sha256: output.checksum_sha256,
|
||||
server_side_encryption: output.server_side_encryption.map(ServerSideEncryption::from),
|
||||
ssekms_key_id: output.ssekms_key_id,
|
||||
sse_customer_algorithm: output.sse_customer_algorithm,
|
||||
sse_customer_key_md5: output.sse_customer_key_md5,
|
||||
..Default::default()
|
||||
});
|
||||
response.headers = output.headers;
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::super::test_support::*;
|
||||
|
||||
@@ -104,7 +104,7 @@ async fn native_claim_checks_order_ownership_and_one_active_lookup() {
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
LOOKUP
|
||||
.scope((Cell::new(0), mode), async {
|
||||
let result = usecase.execute_upload_part_with_audit(envelope, input).await;
|
||||
let result = usecase.execute_upload_part_owned(envelope, input).await;
|
||||
assert_eq!(
|
||||
result.as_ref().err().map(|err| err.code().as_str()),
|
||||
expected,
|
||||
@@ -214,7 +214,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
|
||||
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
|
||||
LOOKUP
|
||||
.scope((Cell::new(0), mode), async {
|
||||
let result = Box::pin(usecase.execute_upload_part_copy_with_audit(envelope, input)).await;
|
||||
let result = Box::pin(usecase.execute_upload_part_copy_owned(envelope, input)).await;
|
||||
assert_eq!(result.as_ref().err().map(|error| error.code().as_str()), expected, "{mode:?}");
|
||||
assert_eq!(LOOKUP.with(|(calls, _)| calls.get()), 1, "{mode:?}");
|
||||
if expected == Some("NoSuchUpload") {
|
||||
@@ -233,8 +233,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
|
||||
} else if matches!(mode, LookupResult::Outage) {
|
||||
let Err(error) = result else { panic!("expected read quorum failure") };
|
||||
let actual: crate::app::storage_api::s3::S3Error = error.into();
|
||||
let expected: crate::app::storage_api::s3::S3Error =
|
||||
ApiError::from(StorageError::ErasureReadQuorum).into();
|
||||
let expected = multipart_upload_api_error(StorageError::ErasureReadQuorum);
|
||||
assert_eq!(actual.code(), expected.code());
|
||||
assert_eq!(actual.status_code(), expected.status_code());
|
||||
assert_eq!(actual.message(), expected.message());
|
||||
|
||||
@@ -13,14 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
//! UploadPartCopy through one owned storage implementation and its legacy adapter.
|
||||
//! Gateway and protocol clients supply request facts. Storage owns the multipart upload.
|
||||
//! Protocol clients and the legacy FS edge supply request facts; multipart ownership stays in storage.
|
||||
|
||||
use super::*;
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::dto::CopySource;
|
||||
use crate::app::storage_api::s3::dto::{
|
||||
CopyPartResult, CopySource, ETag, ServerSideEncryption, Timestamp, UploadPartCopyInput, UploadPartCopyOutput,
|
||||
CopyPartResult, ETag, ServerSideEncryption, Timestamp, UploadPartCopyInput, UploadPartCopyOutput,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::s3::{S3Request, S3Response};
|
||||
use crate::app::{RequestEnvelope, object_usecase::CopySourceCondition};
|
||||
use rustfs_gateway_types::CopySourceValue;
|
||||
@@ -100,7 +100,6 @@ fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_siz
|
||||
}
|
||||
|
||||
impl DefaultMultipartUsecase {
|
||||
#[cfg(test)]
|
||||
pub async fn execute_upload_part_copy(
|
||||
&self,
|
||||
req: S3Request<UploadPartCopyInput>,
|
||||
@@ -130,7 +129,7 @@ impl DefaultMultipartUsecase {
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
};
|
||||
let output = self.execute_upload_part_copy_with_audit(envelope, input).await?;
|
||||
let output = self.execute_upload_part_copy_owned(envelope, input).await?;
|
||||
Ok(S3Response::new(UploadPartCopyOutput {
|
||||
copy_part_result: Some(CopyPartResult {
|
||||
e_tag: output.e_tag.as_deref().map(to_s3s_xml_etag),
|
||||
@@ -152,18 +151,6 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
|
||||
/// Copies plaintext into an existing upload without changing its recorded owner.
|
||||
/// Records the gateway audit. The protocol client records its own projected response.
|
||||
pub(crate) async fn execute_upload_part_copy_with_audit(
|
||||
&self,
|
||||
envelope: RequestEnvelope,
|
||||
input: UploadPartCopyWriteInput,
|
||||
) -> S3Result<UploadPartCopyWriteOutput> {
|
||||
RequestEnvelope::audit_only_operation(envelope, S3Operation::UploadPartCopy, http::StatusCode::OK, |envelope| {
|
||||
Box::pin(self.execute_upload_part_copy_owned(envelope, input))
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", skip(self, envelope, input))]
|
||||
#[hotpath::measure(impl_type = "MultipartUsecase")]
|
||||
pub(crate) async fn execute_upload_part_copy_owned(
|
||||
@@ -450,7 +437,7 @@ impl DefaultMultipartUsecase {
|
||||
.user_defined
|
||||
.contains_key("x-amz-server-side-encryption-customer-algorithm");
|
||||
let ssekms_key_id = match server_side_encryption.as_ref() {
|
||||
Some(sse) if sse.as_str() == rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS => mp_info
|
||||
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.map(|s| s.to_string()),
|
||||
|
||||
@@ -162,7 +162,7 @@ impl DefaultObjectUsecase {
|
||||
envelope.uri().query(),
|
||||
envelope.verified_presigned().is_some(),
|
||||
)?;
|
||||
reject_non_wildcard_write_if_none_match(input.specific_if_none_match, envelope.headers())?;
|
||||
reject_non_wildcard_write_if_none_match_native(input.specific_if_none_match, envelope.headers())?;
|
||||
if let Some(context) = &self.context {
|
||||
let _ = context.object_store();
|
||||
}
|
||||
@@ -1051,8 +1051,8 @@ mod tests {
|
||||
.expect_err("short copy source must fail before destination write succeeds");
|
||||
let api_error = ApiError::from(err);
|
||||
|
||||
assert_eq!(api_error.code, rustfs_s3_types::S3ErrorCode::ServiceUnavailable);
|
||||
assert_ne!(api_error.code, rustfs_s3_types::S3ErrorCode::IncompleteBody);
|
||||
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
|
||||
assert_ne!(api_error.code, S3ErrorCode::IncompleteBody);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -99,57 +99,7 @@ impl CopyObjectWriteInput {
|
||||
pub(crate) fn from_test_request(
|
||||
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CopyObjectInput>,
|
||||
) -> (crate::app::RequestEnvelope, Self) {
|
||||
use crate::app::object_usecase::CopySourceCondition;
|
||||
use crate::app::storage_api::s3::dto::{CopySource, ETag, ETagCondition};
|
||||
|
||||
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
|
||||
let input = req.input;
|
||||
let condition = |value: ETagCondition| match value {
|
||||
ETagCondition::Any => CopySourceCondition::Any,
|
||||
ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
|
||||
ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
|
||||
};
|
||||
let copy_source = match &input.copy_source {
|
||||
CopySource::AccessPoint { .. } | CopySource::Outpost { .. } => None,
|
||||
CopySource::Bucket { bucket, key, version_id } => Some(rustfs_gateway_types::CopySourceValue::Bucket {
|
||||
bucket: bucket.to_string(),
|
||||
key: key.to_string(),
|
||||
version_id: version_id.as_ref().map(|v| v.to_string()),
|
||||
}),
|
||||
};
|
||||
let write_input = CopyObjectWriteInput {
|
||||
copy_source,
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
version_id: input.version_id,
|
||||
specific_if_none_match: matches!(input.if_none_match, Some(ETagCondition::ETag(_))),
|
||||
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
|
||||
ssekms_key_id: input.ssekms_key_id,
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
|
||||
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
|
||||
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
|
||||
metadata_directive: input.metadata_directive.map(|value| value.as_str().to_owned()),
|
||||
metadata: input.metadata,
|
||||
tagging: input.tagging,
|
||||
tagging_directive: input.tagging_directive.map(|value| value.as_str().to_owned()),
|
||||
copy_source_if_match: input.copy_source_if_match.map(condition),
|
||||
copy_source_if_none_match: input.copy_source_if_none_match.map(condition),
|
||||
cache_control: input.cache_control,
|
||||
content_disposition: input.content_disposition,
|
||||
content_encoding: input.content_encoding,
|
||||
content_language: input.content_language,
|
||||
content_type: input.content_type,
|
||||
expires: input.expires,
|
||||
website_redirect_location: input.website_redirect_location,
|
||||
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
|
||||
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
|
||||
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
|
||||
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
|
||||
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
|
||||
};
|
||||
(envelope, write_input)
|
||||
(envelope, crate::storage::ecfs::copy_input_from_legacy(req.input))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -503,7 +503,7 @@ fn project_delete_objects_pre_stat_error(
|
||||
error: ApiError,
|
||||
) -> S3Result<DeleteBatchError> {
|
||||
// Bucket loss invalidates the shared request, not just one object.
|
||||
if error.code == rustfs_s3_types::S3ErrorCode::NoSuchBucket {
|
||||
if error.code == S3ErrorCode::NoSuchBucket {
|
||||
return Err(error.into());
|
||||
}
|
||||
Ok(DeleteBatchError {
|
||||
@@ -676,9 +676,10 @@ impl DefaultObjectUsecase {
|
||||
let key = normalize_delete_object_key(&obj_id.key);
|
||||
// A rejected wire key has no authorization resource. Its normalized text cannot become a target.
|
||||
let key_error = if obj_id.key_too_long {
|
||||
let code = rustfs_s3_types::S3ErrorCode::InvalidArgument;
|
||||
let message = ApiError::error_code_to_message(&code);
|
||||
Some((code, message))
|
||||
Some((
|
||||
S3ErrorCode::InvalidArgument,
|
||||
ApiError::error_code_to_message(&S3ErrorCode::InvalidArgument),
|
||||
))
|
||||
} else {
|
||||
validate_object_key(&key, "DeleteObjects")
|
||||
.err()
|
||||
@@ -4459,67 +4460,3 @@ mod bulk_denial_audit_wiring_tests {
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl From<crate::app::storage_api::s3::dto::DeleteObjectsInput> for DeleteBatchInput {
|
||||
fn from(input: crate::app::storage_api::s3::dto::DeleteObjectsInput) -> Self {
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
objects: input.delete.objects.into_iter().map(Into::into).collect(),
|
||||
quiet: input.delete.quiet,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl From<crate::app::storage_api::s3::dto::ObjectIdentifier> for DeleteBatchEntry {
|
||||
fn from(object: crate::app::storage_api::s3::dto::ObjectIdentifier) -> Self {
|
||||
use crate::app::storage_api::s3::dto::ETag;
|
||||
Self {
|
||||
key: object.key,
|
||||
key_too_long: false,
|
||||
version_id: object.version_id,
|
||||
e_tag: object.e_tag.map(|tag| match tag {
|
||||
ETag::Strong(value) => DeleteBatchETag { value, weak: false },
|
||||
ETag::Weak(value) => DeleteBatchETag { value, weak: true },
|
||||
}),
|
||||
_last_modified_time: object.last_modified_time.map(|value| {
|
||||
let instant = time::OffsetDateTime::from(value);
|
||||
(instant.unix_timestamp(), instant.nanosecond())
|
||||
}),
|
||||
_size: object.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl From<DeleteBatchOutput> for crate::app::storage_api::s3::dto::DeleteObjectsOutput {
|
||||
fn from(output: DeleteBatchOutput) -> Self {
|
||||
use crate::app::storage_api::s3::dto::{DeletedObject, Error};
|
||||
Self {
|
||||
deleted: output.deleted.map(|entries| {
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|entry| DeletedObject {
|
||||
delete_marker: entry.delete_marker,
|
||||
delete_marker_version_id: entry.delete_marker_version_id,
|
||||
key: entry.key,
|
||||
version_id: entry.version_id,
|
||||
})
|
||||
.collect()
|
||||
}),
|
||||
errors: output.errors.map(|entries| {
|
||||
entries
|
||||
.into_iter()
|
||||
.map(|entry| Error {
|
||||
code: entry.code,
|
||||
key: entry.key,
|
||||
message: entry.message,
|
||||
version_id: entry.version_id,
|
||||
})
|
||||
.collect()
|
||||
}),
|
||||
request_charged: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -139,30 +139,6 @@ fn extract_archive_incomplete_body(remaining: u64) -> std::io::Error {
|
||||
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, rustfs_rio::IncompleteBody { remaining })
|
||||
}
|
||||
|
||||
/// Settle the transport checksum and trailers before giving the final frame to the TAR parser.
|
||||
/// A parser that rejects that frame may never poll the body again.
|
||||
fn archive_transport_chunks(
|
||||
chunks: super::request_body::UploadChunks,
|
||||
expected_length: u64,
|
||||
) -> impl Stream<Item = std::io::Result<Bytes>> {
|
||||
futures::stream::try_unfold((chunks.fuse(), expected_length), |(mut chunks, remaining)| async move {
|
||||
let Some(chunk) = chunks.next().await else {
|
||||
return Ok(None);
|
||||
};
|
||||
let chunk = chunk.map_err(s3s_body_error_to_io)?;
|
||||
let overlong = || std::io::Error::new(std::io::ErrorKind::InvalidData, "archive body exceeded expected Content-Length");
|
||||
let remaining = remaining.checked_sub(chunk.len() as u64).ok_or_else(overlong)?;
|
||||
if remaining == 0 {
|
||||
while let Some(trailing) = chunks.next().await {
|
||||
if !trailing.map_err(s3s_body_error_to_io)?.is_empty() {
|
||||
return Err(overlong());
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some((chunk, (chunks, remaining))))
|
||||
})
|
||||
}
|
||||
|
||||
impl<R: AsyncRead> AsyncRead for ExtractArchiveEtagReader<R> {
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
let mut this = self.project();
|
||||
@@ -1250,7 +1226,7 @@ pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: boo
|
||||
|
||||
fn map_extract_archive_error(err: std::io::Error) -> S3Error {
|
||||
let api_error = ApiError::from(err);
|
||||
if api_error.code != rustfs_s3_types::S3ErrorCode::InternalError {
|
||||
if api_error.code != S3ErrorCode::InternalError {
|
||||
return api_error.into();
|
||||
}
|
||||
|
||||
@@ -1323,7 +1299,7 @@ fn is_empty_extract_entry_path(path: &str) -> bool {
|
||||
|
||||
fn validate_extract_member_key(path: &str, limits: ArchiveLimits) -> Result<(), ExtractEntryError> {
|
||||
validate_put_object_extract_entry_path(path, limits).map_err(ExtractEntryError::Recoverable)?;
|
||||
validate_new_object_key(path, "PUT").map_err(|err| ExtractEntryError::Recoverable(err.into()))
|
||||
validate_new_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable)
|
||||
}
|
||||
|
||||
fn record_extract_pax_metadata_bytes(
|
||||
@@ -2228,9 +2204,8 @@ impl DefaultObjectUsecase {
|
||||
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
|
||||
// Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile.
|
||||
let buffer_size = get_buffer_size_opt_in(size);
|
||||
let expected_archive_length = u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent.into()))?;
|
||||
let chunks = Box::pin(archive_transport_chunks(body, expected_archive_length));
|
||||
let body = tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(chunks));
|
||||
let body =
|
||||
tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))));
|
||||
|
||||
let md5hex = if let Some(base64_md5) = content_md5 {
|
||||
let md5 = base64_simd::STANDARD
|
||||
@@ -2251,6 +2226,7 @@ impl DefaultObjectUsecase {
|
||||
return Err(ApiError::from(err).into());
|
||||
}
|
||||
|
||||
let expected_archive_length = u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent.into()))?;
|
||||
let archive_upload_state = Arc::new(Mutex::new(ExtractArchiveUploadState::default()));
|
||||
let extract_limits = put_object_extract_limits();
|
||||
let tracked_archive =
|
||||
@@ -3666,7 +3642,7 @@ mod tests {
|
||||
] {
|
||||
let error = map_extract_archive_error(error);
|
||||
assert_eq!(error.code(), &code);
|
||||
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code.into()).as_str()));
|
||||
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code).as_str()));
|
||||
assert!(error.source().is_some());
|
||||
}
|
||||
let error = map_extract_archive_error(io::Error::new(io::ErrorKind::InvalidData, "invalid TAR header"));
|
||||
@@ -4317,28 +4293,6 @@ mod tests {
|
||||
.expect("fixture member should parse")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn archive_transport_checks_eof_before_releasing_the_final_frame() {
|
||||
let data = || Ok(Bytes::from_static(b"ab"));
|
||||
let late_error: StdError = Box::new(std::io::Error::other("trailer verification failed"));
|
||||
for (tail, expected) in [
|
||||
(Ok(Bytes::new()), None),
|
||||
(data(), Some("archive body exceeded expected Content-Length")),
|
||||
(Err(late_error), Some("trailer verification failed")),
|
||||
] {
|
||||
let body = Box::new(futures::stream::iter([data(), data(), tail]));
|
||||
let mut chunks = Box::pin(archive_transport_chunks(body, 4));
|
||||
assert_eq!(chunks.next().await.unwrap().unwrap(), "ab");
|
||||
let last = chunks.next().await.unwrap();
|
||||
if let Some(message) = expected {
|
||||
assert_eq!(last.unwrap_err().to_string(), message);
|
||||
} else {
|
||||
assert_eq!(last.unwrap(), "ab");
|
||||
}
|
||||
assert!(chunks.next().await.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn archive_etag_reader_validates_raw_eof_before_returning_final_byte() {
|
||||
let payload = b"decoder-consumed-exact-body".to_vec();
|
||||
|
||||
@@ -2602,9 +2602,8 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
return Err(with_delete_marker_read_headers(S3Error::new(code), info, input.version_id.as_deref()));
|
||||
}
|
||||
let result = check_preconditions(envelope.headers(), info)
|
||||
.map_err(S3Error::from)
|
||||
.and_then(|()| Self::validate_get_object_part_number(part_number, info));
|
||||
let result =
|
||||
check_preconditions(envelope.headers(), info).and_then(|()| Self::validate_get_object_part_number(part_number, info));
|
||||
if result.is_err() {
|
||||
let principal = SseKmsPrincipal::from_request(envelope);
|
||||
crate::storage::sse::validate_conditional_read(
|
||||
@@ -12314,7 +12313,7 @@ mod tests {
|
||||
fn parse_get_object_part_number_rejects_above_s3_max() {
|
||||
let err = parse_part_number_i32_to_usize(Some(10001), "GET").expect_err("partNumber above S3 max must fail");
|
||||
|
||||
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
assert_eq!(err.message(), Some("GET: partNumber must be between 1 and 10000"));
|
||||
}
|
||||
|
||||
|
||||
@@ -36,42 +36,6 @@ pub(crate) struct GetObjectFields {
|
||||
pub(crate) sse_customer_key_md5: Option<String>,
|
||||
}
|
||||
|
||||
impl GetObjectFields {
|
||||
/// The conditional-read wire format permits a wildcard, a quoted tag, or a bare S3 tag.
|
||||
pub(crate) fn valid_etag_condition(value: &[u8]) -> bool {
|
||||
match value {
|
||||
b"*" => true,
|
||||
[b'"', tag @ .., b'"'] | [b'W', b'/', b'"', tag @ .., b'"'] => tag.iter().all(|b| matches!(*b, b'\t' | b' '..=b'~')),
|
||||
bare => !bare.is_empty() && bare.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'-'),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn valid_date_condition(value: &str) -> bool {
|
||||
time::PrimitiveDateTime::parse(value, super::RFC1123).is_ok()
|
||||
}
|
||||
|
||||
/// Object Lambda and website reads accept the historical single-range wire grammar.
|
||||
pub(crate) fn parse_range(value: &str) -> Option<ByteRange> {
|
||||
use rustfs_gateway_types::RangeParse;
|
||||
let spec = value.strip_prefix("bytes=")?;
|
||||
if !spec.bytes().all(|b| b.is_ascii_digit() || b == b'-') {
|
||||
return None;
|
||||
}
|
||||
if spec == "-" {
|
||||
// An omitted suffix length historically selects a zero-length suffix.
|
||||
return Some(ByteRange::Suffix { length: 0 });
|
||||
}
|
||||
let RangeParse::One(range) = RangeParse::parse(value) else {
|
||||
return None;
|
||||
};
|
||||
match range {
|
||||
ByteRange::From { first } if first > i64::MAX as u64 => None,
|
||||
ByteRange::FromTo { last, .. } if last > i64::MAX as u64 => None,
|
||||
_ => Some(range),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Body and ETag retain their native representation until the final edge.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct GetObjectOutputFacts {
|
||||
|
||||
@@ -20,73 +20,6 @@ use crate::app::storage_api::s3::dto::Range;
|
||||
use crate::app::storage_api::test::ReqInfo;
|
||||
use http::Method;
|
||||
|
||||
#[test]
|
||||
fn website_and_object_lambda_ranges_match_the_legacy_wire_reader() {
|
||||
for value in [
|
||||
"bytes=0-9",
|
||||
"bytes=0-",
|
||||
"bytes=-7",
|
||||
"bytes=-0",
|
||||
"bytes=-",
|
||||
"bytes=-18446744073709551615",
|
||||
"bytes=9223372036854775807-",
|
||||
"bytes=9223372036854775808-",
|
||||
"bytes=0-9223372036854775808",
|
||||
"bytes=-18446744073709551616",
|
||||
"bytes=10-5",
|
||||
"bytes=1-2 ",
|
||||
"bytes=1 -2",
|
||||
"bytes=1-2,3-4",
|
||||
"bytes=-+1",
|
||||
"Bytes=1-2",
|
||||
"bytes=",
|
||||
] {
|
||||
let expected = Range::parse(value).ok().map(|range| match range {
|
||||
Range::Int { first, last: Some(last) } => rustfs_gateway_types::ByteRange::FromTo { first, last },
|
||||
Range::Int { first, last: None } => rustfs_gateway_types::ByteRange::From { first },
|
||||
Range::Suffix { length } => rustfs_gateway_types::ByteRange::Suffix { length },
|
||||
});
|
||||
assert_eq!(GetObjectFields::parse_range(value), expected, "{value}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn website_and_object_lambda_conditions_match_the_legacy_wire_reader() {
|
||||
use s3s::dto::{ETagCondition, Timestamp, TimestampFormat};
|
||||
for value in [
|
||||
b"*".as_slice(),
|
||||
b"abc-2",
|
||||
b"\"\"",
|
||||
b"W/\"x\"",
|
||||
b"\"a \tb\"",
|
||||
b"\"a\"b\"",
|
||||
b"",
|
||||
b"bad etag",
|
||||
b"w/\"x\"",
|
||||
b"\"\xff\"",
|
||||
b"\"\x7f\"",
|
||||
b"\"x",
|
||||
] {
|
||||
assert_eq!(
|
||||
GetObjectFields::valid_etag_condition(value),
|
||||
ETagCondition::parse_http_header(value).is_ok(),
|
||||
"{value:?}"
|
||||
);
|
||||
}
|
||||
for value in [
|
||||
"Sun, 06 Nov 1994 08:49:37 GMT",
|
||||
"Sunday, 06-Nov-94 08:49:37 GMT",
|
||||
"2025-01-01",
|
||||
"Sun, 32 Nov 1994 08:49:37 GMT",
|
||||
] {
|
||||
assert_eq!(
|
||||
GetObjectFields::valid_date_condition(value),
|
||||
Timestamp::parse(TimestampFormat::HttpDate, value).is_ok(),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn get_request(input: GetObjectInput) -> S3Request<GetObjectInput> {
|
||||
let info = ReqInfo {
|
||||
bucket: Some(input.bucket.clone()),
|
||||
|
||||
@@ -45,11 +45,6 @@ pub(crate) struct ResponseEtag {
|
||||
}
|
||||
|
||||
impl ResponseEtag {
|
||||
pub(crate) fn to_http_header(&self) -> Result<http::HeaderValue, http::header::InvalidHeaderValue> {
|
||||
let prefix = if self.weak { "W/" } else { "" };
|
||||
http::HeaderValue::from_str(&format!("{prefix}\"{}\"", self.opaque))
|
||||
}
|
||||
|
||||
pub(super) fn from_stored(value: &str) -> Self {
|
||||
let weak = value.starts_with("W/\"");
|
||||
let etag = stored_etag(value);
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
//! multipart handlers' policy steps against the same storage contract.
|
||||
|
||||
use super::*;
|
||||
use rustfs_s3_types::{S3Error, S3ErrorCode};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Response, S3Result};
|
||||
|
||||
use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success;
|
||||
use crate::app::runtime_sources::current_app_context;
|
||||
@@ -80,8 +80,7 @@ pub(crate) struct InternalPutContext {
|
||||
|
||||
const INTERNAL_PUT_METHOD_NAME: &str = "PUT";
|
||||
|
||||
fn api_error_from_s3(err: impl Into<S3Error>) -> ApiError {
|
||||
let err = err.into();
|
||||
fn api_error_from_s3(err: S3Error) -> ApiError {
|
||||
ApiError {
|
||||
code: err.code().clone(),
|
||||
message: err.message().unwrap_or_default().to_string(),
|
||||
@@ -140,7 +139,7 @@ async fn validate_internal_write_target(key: &str, bucket: &str, headers: &Heade
|
||||
headers.get("content-type").and_then(|value| value.to_str().ok()),
|
||||
headers.get("content-encoding").and_then(|value| value.to_str().ok()),
|
||||
)
|
||||
.map_err(api_error_from_s3)
|
||||
.map_err(|err| api_error_from_s3(err.into()))
|
||||
}
|
||||
|
||||
fn internal_events_wanted() -> bool {
|
||||
@@ -214,7 +213,17 @@ impl InternalPutObjectEvent {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn publish(self, headers: &HeaderMap) {
|
||||
/// Publish the event when `result` is a success; failures publish nothing.
|
||||
pub(super) fn complete<T>(self, result: &S3Result<S3Response<T>>) {
|
||||
if let Ok(response) = result {
|
||||
self.complete_owned(&Ok(()), &response.headers);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn complete_owned<T>(self, result: &rustfs_s3_types::S3Result<T>, headers: &HeaderMap) {
|
||||
if result.is_err() {
|
||||
return;
|
||||
}
|
||||
let Self {
|
||||
builder,
|
||||
notify,
|
||||
@@ -334,7 +343,7 @@ impl DefaultObjectUsecase {
|
||||
.put_object_core(write, super::native_body::from_io_stream(body), None, start_time)
|
||||
.await
|
||||
.map_err(|error| ApiError {
|
||||
code: error.code().clone(),
|
||||
code: error.code().clone().into(),
|
||||
message: error.message().unwrap_or_default().to_owned(),
|
||||
source: None,
|
||||
})?;
|
||||
@@ -758,7 +767,8 @@ impl DefaultObjectUsecase {
|
||||
if versioned && let Some(version_id) = obj_info.version_id {
|
||||
event = event.version_id(version_id.to_string());
|
||||
}
|
||||
event.publish(&HeaderMap::new());
|
||||
let result: S3Result<S3Response<()>> = Ok(S3Response::new(()));
|
||||
event.complete(&result);
|
||||
}
|
||||
Ok(obj_info)
|
||||
}
|
||||
|
||||
@@ -79,6 +79,8 @@ use super::storage_api::object_usecase::data_usage::{
|
||||
record_bucket_object_write_unknown_previous_memory,
|
||||
};
|
||||
use super::storage_api::object_usecase::deadlock_detector;
|
||||
#[cfg(test)]
|
||||
use super::storage_api::object_usecase::ecfs::FS;
|
||||
use super::storage_api::object_usecase::error::{
|
||||
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
|
||||
};
|
||||
@@ -237,14 +239,11 @@ pub(crate) mod post_body;
|
||||
pub(crate) mod post_headers;
|
||||
mod put;
|
||||
mod put_input;
|
||||
#[cfg(test)]
|
||||
pub(crate) use native_body::from_legacy as legacy_put_body;
|
||||
pub(crate) use native_body::{BodyChunks, FrameTrailers};
|
||||
pub(crate) use put_input::{PutObjectWriteInput, PutObjectWriteOutput};
|
||||
pub(crate) mod request_body;
|
||||
mod restore;
|
||||
#[cfg(test)]
|
||||
mod s3s_compat;
|
||||
pub(crate) mod shared;
|
||||
#[cfg(test)]
|
||||
mod test_support;
|
||||
|
||||
@@ -15,13 +15,9 @@
|
||||
//! Upload frame/chunk boundary. Protocol edges supply a native body; storage consumes chunks.
|
||||
//! This adapter preserves trailers and source errors without applying another length check.
|
||||
|
||||
use crate::app::storage_api::request_body::StdError;
|
||||
#[cfg(test)]
|
||||
use crate::app::storage_api::request_body::{ByteStream, StreamingBlob};
|
||||
use crate::app::storage_api::request_body::{ByteStream, StdError, StreamingBlob};
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
#[cfg(test)]
|
||||
use futures::StreamExt;
|
||||
use futures::{Stream, StreamExt};
|
||||
use http::HeaderMap;
|
||||
use http_body::Body as _;
|
||||
use parking_lot::Mutex;
|
||||
@@ -34,7 +30,6 @@ use std::sync::Arc;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
/// The legacy edge preserves the producer's length hint without enforcing it a second time.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn from_legacy(body: StreamingBlob) -> Body {
|
||||
let stream = LegacyPayload { body, ended: false };
|
||||
Body::from_payload(Payload::Stream(Box::pin(stream)))
|
||||
@@ -54,13 +49,11 @@ impl std::error::Error for ProducerSource {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
struct LegacyPayload {
|
||||
body: StreamingBlob,
|
||||
ended: bool,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl PayloadStream for LegacyPayload {
|
||||
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<PayloadRead, StreamError>> {
|
||||
if self.ended {
|
||||
|
||||
@@ -32,7 +32,7 @@
|
||||
//! `odm-source-etag`.
|
||||
|
||||
use super::*;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use crate::app::storage_api::s3::S3ErrorCode;
|
||||
|
||||
use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
|
||||
use crate::on_demand_migration::{
|
||||
|
||||
@@ -150,7 +150,7 @@ pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
|
||||
if u64::try_from(size).is_ok_and(|size| size > rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE) {
|
||||
return Err(S3Error::with_message(
|
||||
S3ErrorCode::EntityTooLarge,
|
||||
ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge),
|
||||
ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
@@ -1174,10 +1174,8 @@ impl PutObjectCompletion {
|
||||
match self {
|
||||
Self::S3(helper) => Self::S3(helper.complete_owned(result, status, headers)),
|
||||
Self::Internal(event) => {
|
||||
if result.is_ok()
|
||||
&& let Some(event) = event
|
||||
{
|
||||
event.publish(headers);
|
||||
if let Some(event) = event {
|
||||
event.complete_owned(result, headers);
|
||||
}
|
||||
Self::Internal(None)
|
||||
}
|
||||
@@ -1324,7 +1322,7 @@ impl DefaultObjectUsecase {
|
||||
{
|
||||
return Err(rustfs_s3_types::s3_error!(InvalidStorageClass));
|
||||
}
|
||||
reject_non_wildcard_write_if_none_match(input.specific_if_none_match, envelope.headers())?;
|
||||
reject_non_wildcard_write_if_none_match_native(input.specific_if_none_match, envelope.headers())?;
|
||||
// An authorized inbound replication PUT must store the replica verbatim.
|
||||
// Legacy snowball-extracted members may still carry the auto-extract
|
||||
// metadata, which replication replays as a header. Do not interpret that
|
||||
@@ -5669,7 +5667,7 @@ mod oversize_single_upload_tests {
|
||||
assert_eq!(*err.code(), S3ErrorCode::EntityTooLarge);
|
||||
assert_eq!(
|
||||
err.message(),
|
||||
Some(ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge).as_str())
|
||||
Some(ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge).as_str())
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -91,51 +91,6 @@ impl PutObjectWriteInput {
|
||||
|
||||
#[cfg(test)]
|
||||
impl PutObjectWriteInput {
|
||||
pub(crate) fn from_legacy(
|
||||
input: crate::app::storage_api::s3::PutObjectInput,
|
||||
trailing_headers: Option<crate::app::storage_api::s3::TrailingHeaders>,
|
||||
) -> Self {
|
||||
use crate::app::storage_api::s3::dto::ETagCondition;
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
body: input.body.map(crate::app::object::legacy_put_body),
|
||||
post_archive: None,
|
||||
frame_trailers: None,
|
||||
trailing_headers: crate::app::trailer_adapter::trailer_source(trailing_headers),
|
||||
content_length: input.content_length,
|
||||
metadata: input.metadata,
|
||||
metadata_headers: None,
|
||||
checksum_headers: None,
|
||||
specific_if_none_match: matches!(input.if_none_match, Some(ETagCondition::ETag(_))),
|
||||
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
|
||||
cache_control: input.cache_control,
|
||||
content_disposition: input.content_disposition,
|
||||
content_encoding: input.content_encoding,
|
||||
content_language: input.content_language,
|
||||
content_type: input.content_type,
|
||||
expires: input.expires,
|
||||
tagging: input.tagging,
|
||||
version_id: input.version_id,
|
||||
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
|
||||
sse_customer_algorithm: input.sse_customer_algorithm,
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
ssekms_key_id: input.ssekms_key_id,
|
||||
content_md5: input.content_md5,
|
||||
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
|
||||
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
|
||||
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
|
||||
website_redirect_location: input.website_redirect_location,
|
||||
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
|
||||
checksum_crc32: input.checksum_crc32,
|
||||
checksum_crc32c: input.checksum_crc32c,
|
||||
checksum_sha1: input.checksum_sha1,
|
||||
checksum_sha256: input.checksum_sha256,
|
||||
checksum_crc64nvme: input.checksum_crc64nvme,
|
||||
}
|
||||
}
|
||||
|
||||
/// A test's s3s fixture request as the envelope and input a gateway handler hands the use case.
|
||||
pub(crate) fn from_test_request(
|
||||
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::PutObjectInput>,
|
||||
|
||||
@@ -13,11 +13,10 @@
|
||||
// limitations under the License.
|
||||
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::{Body as S3Body, S3Error, StreamingBlob};
|
||||
use crate::app::storage_api::s3::{Body as S3Body, S3Error, S3ErrorCode, StreamingBlob};
|
||||
use crate::error::ApiError;
|
||||
use futures::{StreamExt, poll};
|
||||
use http_body_util::StreamBody;
|
||||
use rustfs_s3_types::S3ErrorCode;
|
||||
use std::io;
|
||||
use tokio::io::AsyncReadExt;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
@@ -539,42 +539,6 @@ fn restore_output() -> RestoreObjectOutput {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl DefaultObjectUsecase {
|
||||
/// Projects a legacy RestoreObject request onto the shared restore path.
|
||||
pub async fn execute_restore_object(
|
||||
&self,
|
||||
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::RestoreObjectInput>,
|
||||
) -> crate::app::storage_api::s3::S3Result<
|
||||
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::RestoreObjectOutput>,
|
||||
> {
|
||||
use crate::app::storage_api::object_usecase::bucket::restore_request_from_wire;
|
||||
use crate::app::storage_api::s3::{
|
||||
S3Response,
|
||||
dto::{RequestCharged, RestoreObjectOutput},
|
||||
};
|
||||
let envelope = RequestEnvelope::from_s3s(&req);
|
||||
let input = req.input;
|
||||
self.execute_restore_object_owned(
|
||||
envelope,
|
||||
input.bucket,
|
||||
input.key,
|
||||
input.version_id,
|
||||
input.restore_request.map(restore_request_from_wire),
|
||||
)
|
||||
.await
|
||||
.map(|output| {
|
||||
S3Response::new(RestoreObjectOutput {
|
||||
request_charged: output
|
||||
.request_charged
|
||||
.map(|value| RequestCharged::from(value.as_str().to_owned())),
|
||||
restore_output_path: output.restore_output_path,
|
||||
})
|
||||
})
|
||||
.map_err(Into::into)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,181 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Compatibility conversions between owned Get/Head object facts and s3s DTOs.
|
||||
//!
|
||||
//! Retained for `admin/router.rs` (object-lambda), `app/website.rs`, and `app/test_seams` until
|
||||
//! T4.2 removes the remaining `s3s` dependency from `admin/router.rs`.
|
||||
|
||||
use crate::app::object_usecase::{GetObjectBody, GetObjectFields, GetObjectOutputFacts};
|
||||
#[cfg(test)]
|
||||
use crate::app::object_usecase::{HeadObjectFields, HeadObjectOutputFacts};
|
||||
use rustfs_gateway_stream::{BoxPayloadStream, PayloadRead, StreamError, StreamErrorKind};
|
||||
use rustfs_gateway_types::ByteRange;
|
||||
use rustfs_gateway_types::compat::ConversionError;
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::get_object;
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::head_object;
|
||||
use s3s::dto::{ETag, GetObjectInput, GetObjectOutput, Range, StreamingBlob};
|
||||
#[cfg(test)]
|
||||
use s3s::dto::{HeadObjectInput, HeadObjectOutput};
|
||||
use std::pin::Pin;
|
||||
use std::sync::{Mutex, PoisonError};
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
#[cfg(test)]
|
||||
impl HeadObjectFields {
|
||||
pub(crate) fn from_s3s(input: HeadObjectInput) -> Self {
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
version_id: input.version_id,
|
||||
part_number: input.part_number,
|
||||
range: input.range.map(|range| match range {
|
||||
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
|
||||
Range::Int { first, last: None } => ByteRange::From { first },
|
||||
Range::Suffix { length } => ByteRange::Suffix { length },
|
||||
}),
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
impl HeadObjectOutputFacts {
|
||||
pub(crate) fn into_s3s(self) -> Result<HeadObjectOutput, ConversionError> {
|
||||
let mut output = head_object::output_to_s3s(self.dto)?;
|
||||
output.e_tag = self.etag.map(|etag| {
|
||||
if etag.weak {
|
||||
ETag::Weak(etag.opaque)
|
||||
} else {
|
||||
ETag::Strong(etag.opaque)
|
||||
}
|
||||
});
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
impl GetObjectFields {
|
||||
pub(crate) fn from_s3s(input: GetObjectInput) -> Self {
|
||||
Self {
|
||||
bucket: input.bucket,
|
||||
key: input.key,
|
||||
version_id: input.version_id,
|
||||
part_number: input.part_number,
|
||||
range: input.range.map(|range| match range {
|
||||
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
|
||||
Range::Int { first, last: None } => ByteRange::From { first },
|
||||
Range::Suffix { length } => ByteRange::Suffix { length },
|
||||
}),
|
||||
sse_customer_key: input.sse_customer_key,
|
||||
sse_customer_key_md5: input.sse_customer_key_md5,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl GetObjectOutputFacts {
|
||||
pub(crate) fn into_s3s(self) -> Result<GetObjectOutput, ConversionError> {
|
||||
let mut output = get_object::output_to_s3s(self.dto)?;
|
||||
output.e_tag = self.etag.map(|etag| {
|
||||
if etag.weak {
|
||||
ETag::Weak(etag.opaque)
|
||||
} else {
|
||||
ETag::Strong(etag.opaque)
|
||||
}
|
||||
});
|
||||
output.body = self.body.map(GetObjectBody::into_s3s);
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
impl GetObjectBody {
|
||||
pub(crate) fn into_s3s(self) -> StreamingBlob {
|
||||
match self {
|
||||
Self::Bytes(bytes) => StreamingBlob::from_bytes(bytes),
|
||||
Self::Stream(payload) => StreamingBlob::new(LegacyGetStream {
|
||||
payload: Mutex::new(payload),
|
||||
done: false,
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// The legacy body requires Sync. Polling uses exclusive access, not a mutex lock.
|
||||
struct LegacyGetStream {
|
||||
payload: Mutex<BoxPayloadStream>,
|
||||
done: bool,
|
||||
}
|
||||
|
||||
impl futures::Stream for LegacyGetStream {
|
||||
type Item = Result<bytes::Bytes, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let this = self.get_mut();
|
||||
if this.done {
|
||||
return Poll::Ready(None);
|
||||
}
|
||||
let event = std::task::ready!(
|
||||
this.payload
|
||||
.get_mut()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.as_mut()
|
||||
.poll_read(cx)
|
||||
);
|
||||
Poll::Ready(match event {
|
||||
Ok(PayloadRead::Chunk(bytes)) => Some(Ok(bytes)),
|
||||
Ok(PayloadRead::Eof { trailers }) => {
|
||||
this.done = true;
|
||||
if trailers.is_empty() {
|
||||
None
|
||||
} else {
|
||||
Some(Err(Box::new(ConversionError {
|
||||
field: "body",
|
||||
reason: "legacy GetObject cannot encode response trailers",
|
||||
})))
|
||||
}
|
||||
}
|
||||
Err(error) => {
|
||||
this.done = true;
|
||||
Some(Err(legacy_get_body_error(error)))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn legacy_get_body_error(error: StreamError) -> Box<dyn std::error::Error + Send + Sync> {
|
||||
if matches!(error.kind(), StreamErrorKind::Upstream(_)) {
|
||||
let StreamErrorKind::Upstream(source) = error.into_kind() else {
|
||||
unreachable!("the owned error kind was checked above")
|
||||
};
|
||||
source
|
||||
} else {
|
||||
Box::new(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl s3s::stream::ByteStream for LegacyGetStream {
|
||||
fn remaining_length(&self) -> s3s::stream::RemainingLength {
|
||||
match self
|
||||
.payload
|
||||
.lock()
|
||||
.unwrap_or_else(PoisonError::into_inner)
|
||||
.len_hint()
|
||||
.and_then(|length| usize::try_from(length).ok())
|
||||
{
|
||||
Some(length) => s3s::stream::RemainingLength::new_exact(length),
|
||||
None => s3s::stream::RemainingLength::unknown(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1030,6 +1030,12 @@ pub(in crate::app) fn object_lock_checks_required_for_state(state: &metadata_sys
|
||||
pub(crate) const ON_DEMAND_MIGRATION_HEADER: http::HeaderName = http::HeaderName::from_static("x-rustfs-on-demand-migration");
|
||||
pub(crate) const ON_DEMAND_MIGRATION_SOURCE: HeaderValue = HeaderValue::from_static("source");
|
||||
|
||||
/// Response header marking a `ListObjectsV2` that could not consult the source
|
||||
/// and was answered from local state alone (rustfs/backlog#2164).
|
||||
pub(crate) const ON_DEMAND_MIGRATION_LIST_HEADER: http::HeaderName =
|
||||
http::HeaderName::from_static("x-rustfs-on-demand-migration-list");
|
||||
pub(crate) const ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY: HeaderValue = HeaderValue::from_static("local_only");
|
||||
|
||||
/// Custom S3 error code for a source failure surfaced under
|
||||
/// `policy.source_error = propagate`; carried on HTTP 424.
|
||||
pub(crate) const ODM_SOURCE_UNAVAILABLE_CODE: &str = "SourceUnavailable";
|
||||
@@ -1103,6 +1109,12 @@ pub(crate) fn mark_on_demand_migration_response(headers: &mut HeaderMap) {
|
||||
headers.insert(ON_DEMAND_MIGRATION_HEADER, ON_DEMAND_MIGRATION_SOURCE);
|
||||
}
|
||||
|
||||
/// Marks a merged listing that fell back to local state because the source
|
||||
/// could not be consulted (rustfs/backlog#2164).
|
||||
pub(crate) fn mark_on_demand_migration_list_local_only(headers: &mut HeaderMap) {
|
||||
headers.insert(ON_DEMAND_MIGRATION_LIST_HEADER, ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY);
|
||||
}
|
||||
|
||||
/// Evaluates the request's conditional headers (`If-Match`, `If-None-Match`,
|
||||
/// `If-Modified-Since`, `If-Unmodified-Since`) against the source's view of
|
||||
/// the object, with the same S3 semantics the local path applies to
|
||||
@@ -1136,14 +1148,21 @@ pub(crate) fn odm_check_source_preconditions(headers: &HeaderMap, head: &SourceH
|
||||
user_defined: metadata.into(),
|
||||
..Default::default()
|
||||
};
|
||||
Ok(check_preconditions(headers, &info)?)
|
||||
check_preconditions(headers, &info)
|
||||
}
|
||||
|
||||
/// S3 documents write-side `If-None-Match` only as `*` (create-only).
|
||||
/// Reject unsupported ETag comparisons with 501 NotImplemented.
|
||||
/// Treating it as an ETag comparison would let a write overwrite an existing key
|
||||
/// while still satisfying a policy that requires `s3:if-none-match`.
|
||||
pub(crate) fn reject_non_wildcard_write_if_none_match(specific_if_none_match: bool, headers: &HeaderMap) -> S3Result<()> {
|
||||
pub(crate) fn reject_non_wildcard_write_if_none_match(
|
||||
if_none_match: Option<&s3s::dto::ETagCondition>,
|
||||
headers: &HeaderMap,
|
||||
) -> S3Result<()> {
|
||||
reject_non_wildcard_write_if_none_match_native(matches!(if_none_match, Some(s3s::dto::ETagCondition::ETag(_))), headers)
|
||||
}
|
||||
|
||||
pub(crate) fn reject_non_wildcard_write_if_none_match_native(specific_if_none_match: bool, headers: &HeaderMap) -> S3Result<()> {
|
||||
// POST parses conditional DTO fields from the form, so HTTP headers also need validation.
|
||||
if specific_if_none_match
|
||||
|| headers.get_all(http::header::IF_NONE_MATCH).iter().any(|value| {
|
||||
@@ -1161,6 +1180,7 @@ pub(crate) fn reject_non_wildcard_write_if_none_match(specific_if_none_match: bo
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::storage_api::s3::dto::ETag;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_gateway_types::persistence::{
|
||||
PersistedBucketEncryptionRule as ServerSideEncryptionRule, PersistedEncryptionByDefault as ServerSideEncryptionByDefault,
|
||||
@@ -1170,8 +1190,12 @@ mod tests {
|
||||
#[test]
|
||||
fn reject_non_wildcard_write_if_none_match_allows_only_wildcard() {
|
||||
let headers = HeaderMap::new();
|
||||
assert!(reject_non_wildcard_write_if_none_match(false, &headers).is_ok());
|
||||
let err = reject_non_wildcard_write_if_none_match(true, &headers).expect_err("an ETag If-None-Match must be rejected");
|
||||
assert!(reject_non_wildcard_write_if_none_match(None, &headers).is_ok());
|
||||
assert!(reject_non_wildcard_write_if_none_match(Some(&s3s::dto::ETagCondition::Any), &headers).is_ok());
|
||||
|
||||
let etag = s3s::dto::ETagCondition::ETag(ETag::Strong("9b2cf535f27731c974343645a3985328".to_owned()));
|
||||
let err =
|
||||
reject_non_wildcard_write_if_none_match(Some(&etag), &headers).expect_err("an ETag If-None-Match must be rejected");
|
||||
assert_eq!(*err.code(), S3ErrorCode::NotImplemented);
|
||||
assert_eq!(err.status_code(), Some(http::StatusCode::NOT_IMPLEMENTED));
|
||||
assert_eq!(
|
||||
@@ -1185,12 +1209,12 @@ mod tests {
|
||||
for value in ["*", " * ", "", " "] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::IF_NONE_MATCH, http::HeaderValue::from_str(value).expect("valid header"));
|
||||
assert!(reject_non_wildcard_write_if_none_match(false, &headers).is_ok(), "{value:?}");
|
||||
assert!(reject_non_wildcard_write_if_none_match(None, &headers).is_ok(), "{value:?}");
|
||||
}
|
||||
for value in ["\"different-etag\"", "W/\"weak-etag\"", "unquoted-etag"] {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.insert(http::header::IF_NONE_MATCH, http::HeaderValue::from_str(value).expect("valid header"));
|
||||
let err = reject_non_wildcard_write_if_none_match(false, &headers)
|
||||
let err = reject_non_wildcard_write_if_none_match(None, &headers)
|
||||
.expect_err("header-only ETag precondition must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
|
||||
}
|
||||
@@ -1205,7 +1229,7 @@ mod tests {
|
||||
let mut headers = HeaderMap::new();
|
||||
headers.append(http::header::IF_NONE_MATCH, http::HeaderValue::from_static("*"));
|
||||
headers.append(http::header::IF_NONE_MATCH, value);
|
||||
let err = reject_non_wildcard_write_if_none_match(false, &headers)
|
||||
let err = reject_non_wildcard_write_if_none_match(Some(&s3s::dto::ETagCondition::Any), &headers)
|
||||
.expect_err("a form wildcard must not hide an unsupported raw HTTP precondition");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
|
||||
}
|
||||
|
||||
@@ -614,7 +614,7 @@ fn veeam_sosapi_http_contract() {
|
||||
crate::app::gating_test_env::run_large_stack_test("veeam-sosapi-http", || async {
|
||||
temp_env::async_with_vars([(rustfs_config::ENV_VEEAM_SOSAPI_ENABLE, None::<&str>)], async {
|
||||
use crate::app::runtime_sources::{AppContext, ServerContextSlot};
|
||||
use std::time::Duration;
|
||||
use crate::app::storage_api::s3::{Body, S3ServiceBuilder, SimpleAuth};
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
|
||||
use crate::app::storage_api::test::contract::list::ListOperations as _;
|
||||
use crate::app::storage_api::test::contract::object::ObjectOperations as _;
|
||||
@@ -626,18 +626,17 @@ fn veeam_sosapi_http_contract() {
|
||||
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
|
||||
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(Arc::clone(&context)));
|
||||
let gateway = crate::server::gateway_stack::TestGateway::over(&context, slot);
|
||||
struct Owner<'a>(&'a crate::server::gateway_stack::TestGateway);
|
||||
impl Owner<'_> {
|
||||
async fn call(
|
||||
&self,
|
||||
(method, uri, headers, body): (Method, String, Vec<(&'static str, String)>, Bytes),
|
||||
) -> Result<http::Response<rustfs_gateway::Body>, std::convert::Infallible> {
|
||||
self.0.signed(method, &uri, headers, body).await
|
||||
}
|
||||
}
|
||||
let owner = Owner(&gateway);
|
||||
assert!(slot.install(context));
|
||||
let fs = super::super::FS::with_server_ctx(slot);
|
||||
let owner = S3ServiceBuilder::new(fs.clone()).build();
|
||||
let mut anonymous_builder = S3ServiceBuilder::new(fs.clone());
|
||||
anonymous_builder.set_auth(SimpleAuth::from_single("test-access", "test-secret"));
|
||||
anonymous_builder.set_access(fs.clone());
|
||||
let anonymous = anonymous_builder.build();
|
||||
let mut bad_auth_builder = S3ServiceBuilder::new(fs.clone());
|
||||
bad_auth_builder.set_auth(SimpleAuth::from_single("test-access", "test-secret"));
|
||||
bad_auth_builder.set_access(fs);
|
||||
let bad_auth = bad_auth_builder.build();
|
||||
|
||||
let bucket = format!("veeam-sosapi-{}", super::super::Uuid::new_v4());
|
||||
store
|
||||
@@ -645,8 +644,19 @@ fn veeam_sosapi_http_contract() {
|
||||
.await
|
||||
.expect("create sosapi bucket");
|
||||
|
||||
let owner_call = |method: Method, uri: String, headers: Vec<(&'static str, String)>, body: Bytes| {
|
||||
(method, uri, headers, body)
|
||||
let owner_call = |method: Method, uri: String, headers: Vec<(&str, String)>, body: Bytes| {
|
||||
let mut builder = http::Request::builder().method(method).uri(uri).header("host", "localhost");
|
||||
for (name, value) in headers {
|
||||
builder = builder.header(name, value);
|
||||
}
|
||||
builder
|
||||
.extension(crate::storage::req_info::ReqInfo {
|
||||
cred: Some(rustfs_credentials::Credentials::default()),
|
||||
is_owner: true,
|
||||
..Default::default()
|
||||
})
|
||||
.body(Body::from(body))
|
||||
.expect("HTTP request")
|
||||
};
|
||||
|
||||
let system_uri = format!("/{bucket}/{SYSTEM_KEY}");
|
||||
@@ -1133,16 +1143,17 @@ fn veeam_sosapi_http_contract() {
|
||||
.expect("list versions");
|
||||
assert!(versions.objects.is_empty());
|
||||
|
||||
let anon = gateway
|
||||
let anon = anonymous
|
||||
.call(
|
||||
http::Request::builder()
|
||||
.method(Method::GET)
|
||||
.uri(system_uri.clone())
|
||||
.header("host", "localhost")
|
||||
.body(rustfs_gateway::Body::empty())
|
||||
.body(Body::empty())
|
||||
.expect("anon request"),
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.expect("anonymous GET");
|
||||
let anon_status = anon.status();
|
||||
let anon_body = BodyExt::collect(anon.into_body()).await.expect("anon body").to_bytes();
|
||||
let anon_xml = String::from_utf8_lossy(&anon_body);
|
||||
@@ -1152,7 +1163,7 @@ fn veeam_sosapi_http_contract() {
|
||||
let now = OffsetDateTime::now_utc();
|
||||
let scope_date = format!("{:04}{:02}{:02}", now.year(), u8::from(now.month()), now.day());
|
||||
let amz_date = format!("{scope_date}T{:02}{:02}{:02}Z", now.hour(), now.minute(), now.second());
|
||||
let bad = gateway
|
||||
let bad = bad_auth
|
||||
.call(
|
||||
http::Request::builder()
|
||||
.method(Method::GET)
|
||||
@@ -1166,10 +1177,11 @@ fn veeam_sosapi_http_contract() {
|
||||
"AWS4-HMAC-SHA256 Credential=bad/{scope_date}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0000000000000000000000000000000000000000000000000000000000000000"
|
||||
),
|
||||
)
|
||||
.body(rustfs_gateway::Body::empty())
|
||||
.body(Body::empty())
|
||||
.expect("bad auth request"),
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.expect("bad auth GET");
|
||||
let bad_status = bad.status();
|
||||
let bad_body = BodyExt::collect(bad.into_body()).await.expect("bad body").to_bytes();
|
||||
let bad_xml = String::from_utf8_lossy(&bad_body);
|
||||
|
||||
@@ -69,7 +69,7 @@ impl ObjectMetaUsecase {
|
||||
.get_bucket_info(bucket, &BucketOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
return Err(err);
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
let opts: ObjectOptions = get_opts(bucket, key, version_id.clone(), None, envelope.headers())
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user