Compare commits

..

3 Commits

Author SHA1 Message Date
Hauser Zhi c374fc1e42 Merge branch 'main' into houseme/fix-get-read-quorum-diagnostics 2026-11-20 08:56:52 +08:00
Hauser Zhi 65933bbdc6 perf(s3): size the primed GET chunk to one byte
Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 23:19:44 +08:00
Hauser Zhi 191929a294 fix(s3): clarify the SlowDownRead message
Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>
2026-11-19 22:45:32 +08:00
174 changed files with 9064 additions and 3911 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
+2 -8
View File
@@ -114,14 +114,8 @@
}
],
"upgrade_sources": {
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.5",
"role": "gateway compatibility baseline",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
},
{
"version": "1.0.1",
"role": "N-1",
@@ -136,6 +130,6 @@
}
],
"rollback_supported": false,
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
}
}
+2 -2
View File
@@ -13,6 +13,6 @@
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=33
s3_error_lines=89
files=49
s3_error_lines=95
ecstore_files=0
+2 -12
View File
@@ -1091,12 +1091,9 @@ jobs:
- name: Verify E2E archive identity
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
- name: Install network fault-injection tools and the persistence test clock
- name: Install network fault-injection tools
run: |
sudo apt-get install -y iptables libfaketime
clock_library="$(dpkg-query -L libfaketime | grep -E '/libfaketime[.]so[.]1$')"
test -f "$clock_library"
echo "RUSTFS_PERSISTENCE_CLOCK_LIBRARY=$clock_library" >> "$GITHUB_ENV"
sudo apt-get install -y iptables
sudo -n iptables --version
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
# runners can run iptables but not touch the rule set; the test then logs
@@ -1203,14 +1200,8 @@ jobs:
# default-filter in .config/nextest.toml is the single wiring mechanism —
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
# debug binary; each test spawns its own rustfs server on a random port.
- name: Download pinned release for persistence byte parity
id: persistence-reference
uses: ./.github/actions/reference-binary
- name: Run e2e full suite
env:
RUSTFS_PERSISTENCE_REFERENCE_BINARY: ${{ steps.persistence-reference.outputs.path }}
RUSTFS_PERSISTENCE_XML_DIR: ${{ runner.temp }}/rustfs-persistence-xml
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
@@ -1229,7 +1220,6 @@ jobs:
path: |
target/nextest/e2e-full/junit.xml
${{ runner.temp }}/rustfs-e2e-full-list.json
${{ runner.temp }}/rustfs-persistence-xml
retention-days: 7
- name: Upload startup CAS evidence
+7 -39
View File
@@ -90,16 +90,11 @@ jobs:
- version: 1.0.1
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
- version: 1.0.5
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -137,13 +132,6 @@ jobs:
unzip -q "$archive" -d "$SOURCE_DIR"
chmod +x "$SOURCE_DIR/rustfs"
test -x "$SOURCE_DIR/rustfs"
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
fi
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
@@ -163,14 +151,12 @@ jobs:
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
)
case "$UPGRADE_SOURCE_VERSION" in
1.0.0-rc.5)
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
esac
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
else
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
fi
status=0
for entry in "${cases[@]}"; do
artifact="${entry%%:*}"
@@ -192,7 +178,7 @@ jobs:
exit "$status"
- name: Run mixed-version MPU recovery
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
timeout-minutes: 15
env:
NO_PROXY: 127.0.0.1,localhost,::1
@@ -213,24 +199,6 @@ jobs:
"$mpu_test" -- --exact --nocapture --test-threads=1 \
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
- name: Verify gateway baseline G09 evidence
if: ${{ success() && matrix.source.version == '1.0.5' }}
shell: bash
run: |
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
directory=mixed-version-upgrade
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
done
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.current_revision == $current and .previous_revision == $previous
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
- name: Upload scanner/heal G09 evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+1 -1
View File
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deployment
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
Generated
+13 -11
View File
@@ -1648,7 +1648,7 @@ dependencies = [
"cexpr",
"clang-sys",
"log",
"prettyplease 0.3.0",
"prettyplease 0.2.37",
"proc-macro2",
"quote",
"regex",
@@ -4238,7 +4238,6 @@ dependencies = [
"s3s",
"same-file",
"serde",
"serde-value",
"serde_json",
"serde_urlencoded",
"sha1 0.11.0",
@@ -10238,6 +10237,7 @@ dependencies = [
"rustix",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
"serial_test",
@@ -10323,7 +10323,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway"
version = "0.66.0"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"arc-swap",
"bytes",
@@ -10370,7 +10370,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-core"
version = "0.63.0"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"bytes",
"futures-core",
@@ -10391,7 +10391,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-dialect-rustfs-admin"
version = "0.13.0"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"bytes",
"form_urlencoded",
@@ -10405,7 +10405,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-http"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10423,7 +10423,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-macros"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"proc-macro2",
"quote",
@@ -10433,7 +10433,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-sig"
version = "0.15.0"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10448,7 +10448,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-stream"
version = "0.2.0"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"bitflags 2.13.2",
"bytes",
@@ -10459,7 +10459,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-types"
version = "0.42.0+aws.2026-10-02"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"bytes",
"crc-fast",
@@ -10482,7 +10482,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-xml"
version = "0.4.0"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
dependencies = [
"quick-xml 0.41.0",
]
@@ -11236,6 +11236,7 @@ dependencies = [
"rustfs-utils",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
"sha2 0.11.1",
@@ -11375,6 +11376,7 @@ dependencies = [
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-utils",
"s3s",
"serde",
"serde_json",
"serial_test",
+6 -7
View File
@@ -152,16 +152,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.2.0", default-features = false }
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.66.0", default-features = false }
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.66.0", default-features = false }
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.13.0", default-features = false }
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.13.0", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
@@ -222,7 +222,6 @@ quick-xml = "0.42.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.229" }
serde-value = "0.7.0"
serde_ignored = { version = "0.1" }
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
serde_json = { version = "1.0.154" }
+1 -2
View File
@@ -129,6 +129,7 @@ md-5 = { workspace = true }
prost.workspace = true
sha2 = { workspace = true }
rustfs-s3-types = { workspace = true }
s3s = { workspace = true, features = ["minio"] }
zstd.workspace = true
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
rcgen.workspace = true
@@ -138,8 +139,6 @@ rustls = { workspace = true, default-features = false, features = ["aws-lc-rs",
clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
serde-value.workspace = true
s3s = { workspace = true, features = ["minio"] }
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
aws-config = { workspace = true }
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
-2
View File
@@ -211,8 +211,6 @@ the wiring source of truth. Committed test-ID digests under
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
The full suite compares persisted bucket configurations and object records against the release pinned in `.config/reference-binary.toml`. Set `RUSTFS_PERSISTENCE_REFERENCE_BINARY` to the verified executable and `RUSTFS_PERSISTENCE_CLOCK_LIBRARY` to the absolute path of libfaketime (`libfaketime.so.1` on Linux or `libfaketime.1.dylib` on macOS). To retain the two sets of 13 XML documents for the gateway golden corpus, set `RUSTFS_PERSISTENCE_XML_DIR` to an output directory. CI includes them in the full suite's JUnit artifact.
**Reproduce a CI failure locally** — run the exact profile/lane:
```bash
@@ -137,40 +137,6 @@ async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::erro
.await
.map_err(|e| format!("DeleteObject failed: {}", e))?;
for (allowed, may_read) in [("s3:GetBucketPolicy", true), ("s3:PutBucketPolicy", false)] {
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": { "AWS": [user_access] },
"Action": allowed,
"Resource": format!("arn:aws:s3:::{bucket_name}")
}]
})
.to_string();
admin_client
.put_bucket_policy()
.bucket(bucket_name)
.policy(&policy)
.send()
.await?;
let read = user_client.get_bucket_policy().bucket(bucket_name).send().await;
let write = user_client
.put_bucket_policy()
.bucket(bucket_name)
.policy(&policy)
.send()
.await;
if may_read {
read?;
let denied = write.expect_err("GetBucketPolicy must not grant PutBucketPolicy");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
} else {
let denied = read.expect_err("PutBucketPolicy must not grant GetBucketPolicy");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
write?;
}
}
info!("Test Passed!");
Ok(())
}
+14 -26
View File
@@ -31,7 +31,7 @@ use super::harness::{
use crate::common::{
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
};
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use std::path::{Path, PathBuf};
@@ -427,12 +427,7 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
Ok(())
}
async fn compare_degraded_legacy_list_parts(
dist: &mut DistCluster,
seed: &UpgradeSeed,
previous: &Path,
source: &UpgradeSourceCapabilities,
) -> TestResult {
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
@@ -494,22 +489,16 @@ async fn compare_degraded_legacy_list_parts(
// Three online disks retain metadata quorum but have no part directory;
// the fourth disk is offline. A legacy empty response is not a proof that
// the unavailable participant has no part.
let mut quorum_errors =
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
if source.requires_list_parts_read_quorum {
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
} else {
assert!(
legacy
.expect("previous coordinator should expose its legacy empty-list behavior")
.parts()
.is_empty()
);
}
for error in quorum_errors {
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
}
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
assert!(legacy.parts().is_empty());
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
// ListParts read-quorum failure is reported as ServiceUnavailable.
assert_eq!(
current.as_service_error().and_then(|error| error.meta().code()),
Some("ServiceUnavailable")
);
dist.cluster.start_node_from_binary(3, previous).await?;
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
for (path, before) in original_files {
@@ -531,7 +520,7 @@ async fn compare_degraded_legacy_list_parts(
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
}
eprintln!(
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
);
Ok(())
}
@@ -603,7 +592,6 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
@@ -613,7 +601,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
for node_idx in [1, 2] {
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
@@ -27,7 +27,7 @@
//! is process-global, so two stacks in one process would read each other's buckets.
mod compare;
pub(crate) mod reference;
mod reference;
mod scenarios;
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
@@ -78,7 +78,7 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
pub(crate) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
if !output.status.success() {
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
-3
View File
@@ -37,9 +37,6 @@ pub mod fake_s3_target;
#[cfg(test)]
mod dual_stack_differential;
#[cfg(test)]
mod persistence_parity;
// On-demand migration (backlog#2147): shared two-server environment, admin
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
// added by later ODM tasks.
@@ -1543,30 +1543,6 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(),
let uploaded = get_out.body.collect().await?.into_bytes();
assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match");
// An empty redirect is rejected when rendering the response, after storage succeeds.
for (field, value, expected_status) in [
("success_action_status", "200", reqwest::StatusCode::OK),
("success_action_status", "204", reqwest::StatusCode::NO_CONTENT),
("success_action_redirect", "", reqwest::StatusCode::BAD_REQUEST),
] {
let key = format!("{field}-{value}.txt");
let form = reqwest::multipart::Form::new()
.text("key", key.clone())
.text(field, value)
.part("file", reqwest::multipart::Part::bytes(expected_body.clone()).file_name("upload.txt"));
let response = local_http_client()
.post(format!("{}/{bucket}", env.url))
.multipart(form)
.send()
.await?;
assert_eq!(response.status(), expected_status);
if value.is_empty() {
assert!(response.text().await?.contains("<Code>InvalidArgument</Code>"));
}
let stored = admin_client.get_object().bucket(bucket).key(key).send().await?;
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body.as_slice());
}
Ok(())
}
-449
View File
@@ -1,449 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Compare bytes produced by S3 writes to independent, single-drive volumes.
use crate::common::{RustFSTestEnvironment, admin_request, rustfs_binary_path, signed_s3_request_with_headers};
use crate::dual_stack_differential::reference::binary_version;
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
use crate::on_demand_migration::common::{ODM_ADMIN_ROUTE, ODM_MODULE_SWITCH_ENV, OdmSourceSpec};
use crate::replication_extension_test::send_set_replication_target_request;
use crate::storage_api::persistence_parity::{BucketMetadata, RUSTFS_META_BUCKET};
use crate::upgrade_compatibility_test::{set_bucket_quota, write_multipart};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
use http::{HeaderMap, Method};
use md5::{Digest, Md5};
use rustfs_filemeta::{FileMeta, MetaObject};
use serde_value::Value;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
type Error = Box<dyn std::error::Error + Send + Sync>;
const CONFIGS: &str = "persisted-configs";
const OBJECTS: &str = "persisted-objects";
const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
const LOGS: &str = "rustfs-access-logs";
fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error> {
let key = BucketMetadata::new(bucket).save_file_path();
let object = volume.join(RUSTFS_META_BUCKET).join(&key);
let meta = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
let info = meta.into_fileinfo(RUSTFS_META_BUCKET, &key, "", true, false, true)?;
info.validate_for_metadata_read()?;
let [part] = info.parts.as_slice() else { return Err("bucket metadata must have one part".into()) };
let size = usize::try_from(info.size)?;
if info.erasure.data_blocks != 1 || size != part.size || size > info.erasure.block_size {
return Err("bucket metadata exceeds the single-drive, single-stripe fixture".into());
}
let framed = if info.inline_data() {
info.data.ok_or("missing inline bucket metadata")?.to_vec()
} else {
let directory = info.data_dir.ok_or("missing bucket metadata data directory")?;
std::fs::read(object.join(directory.to_string()).join(format!("part.{}", part.number)))?
};
let algorithm = info.erasure.get_checksum_info(part.number).algorithm;
let (digest, payload) = framed.split_at_checked(algorithm.size()).ok_or("truncated bucket metadata")?;
if digest.is_empty() || payload.len() != size || algorithm.hash_encode(payload).as_ref() != digest {
return Err("invalid bucket metadata shard checksum or size".into());
}
BucketMetadata::check_header(payload)?;
let fields = &payload[4..];
if bucket == CONFIGS {
// The production decoder defaults a missing field to empty. Require the ACL field on disk
// so dropping it cannot look like the expected zero-byte document.
let keys: BTreeMap<&str, serde::de::IgnoredAny> = rmp_serde::from_slice(fields)?;
if !keys.contains_key("BucketAclConfigJSON") {
return Err("missing BucketAclConfigJSON field".into());
}
}
Ok(BucketMetadata::unmarshal(fields)?)
}
type VersionFields = BTreeMap<String, Value>;
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
let [version] = meta.versions.as_slice() else {
return Err(format!("{key}: expected one version").into());
};
let object = version.parse_version_meta()?.object.ok_or("missing V2 object")?;
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
return Err(format!("{key}: missing modification time or data directory").into());
}
Ok((object, nonvolatile_fields(&version.meta, key)?))
}
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
// The production reader skips unknown keys and supplies defaults for some missing keys.
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
return Err(format!("{key}: missing raw V2Obj map").into());
};
for volatile in ["MTime", "DDir"] {
let value = object_fields
.get_mut(&Value::String(volatile.to_owned()))
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
*value = Value::Unit;
}
Ok(fields)
}
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
let mut headers = HeaderMap::new();
headers.insert("content-md5", base64_simd::STANDARD.encode_to_string(Md5::digest(xml)).parse()?);
let response = signed_s3_request_with_headers(
Method::PUT,
&format!("{}/{CONFIGS}?{query}", env.url),
Some(xml.to_owned()),
Some("application/xml"),
&env.access_key,
&env.secret_key,
&headers,
)
.await?;
if !response.status().is_success() {
return Err(format!("PUT ?{query} failed with {}", response.status()).into());
}
Ok(())
}
struct Snapshot {
configs: BTreeMap<&'static str, Vec<u8>>,
objects: BTreeMap<&'static str, VersionFields>,
multipart_files: BTreeMap<String, Vec<u8>>,
}
async fn capture(binary: &Path, target: &FakeS3Target, clock: &str, clock_library: &str) -> Result<Snapshot, Error> {
let mut env = RustFSTestEnvironment::new().await?;
let queue = tempfile::tempdir()?;
let clock_loader = if cfg!(target_os = "macos") {
"DYLD_INSERT_LIBRARIES"
} else {
"LD_PRELOAD"
};
let options = [
(clock_loader, clock_library),
("DYLD_FORCE_FLAT_NAMESPACE", "1"),
("FAKETIME", clock),
("FAKETIME_DONT_FAKE_MONOTONIC", "1"),
("TZ", "UTC"),
("RUSTFS_NOTIFY_ENABLE", "true"),
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", "on"),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", target.endpoint()),
(
"RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY",
queue.path().to_str().ok_or("non-UTF8 queue path")?,
),
("RUSTFS_OUTBOUND_ALLOW_ORIGINS", target.endpoint()),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
(ODM_MODULE_SWITCH_ENV, "true"),
("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
];
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &options)
.await?;
let client = env.create_s3_client();
write_configs(&env, &client, target).await?;
write_objects(&client).await?;
env.stop_server();
read_snapshot(Path::new(&env.temp_dir))
}
async fn write_configs(env: &RustFSTestEnvironment, client: &Client, target: &FakeS3Target) -> Result<(), Error> {
env.create_test_bucket(LOGS).await?;
env.create_test_bucket(OBJECTS).await?;
env.create_test_bucket("persistence-warehouse").await?;
env.create_test_bucket("persistence-migration").await?;
client
.create_bucket()
.bucket(CONFIGS)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
macro_rules! golden {
($path:literal) => {
include_str!(concat!("../../ecstore/tests/fixtures/bucket-config-goldens/", $path, ".in.xml"))
};
}
let target_request = serde_json::json!({
"endpoint": target.address(), "credentials": {"accessKey": FAKE_ACCESS_KEY, "secretKey": FAKE_SECRET_KEY},
"targetbucket": "interop-dr", "type": "replication", "secure": false,
"deploymentID": "0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c"
});
let response = send_set_replication_target_request(env, CONFIGS, false, target_request).await?;
if !response.status().is_success() {
return Err(format!("set remote target failed with {}", response.status()).into());
}
let target_arn: String = response.json().await?;
let replication = golden!("replication/rustfs").replace("arn:rustfs:replication:", "arn:minio:replication:");
if !replication.contains(&format!("<Bucket>{target_arn}</Bucket>")) {
return Err("registered target does not match the golden replication destination".into());
}
for (query, xml) in [
("versioning", golden!("versioning/minio")),
("notification", golden!("notification/rustfs")),
("replication", replication.as_str()),
("lifecycle", golden!("lifecycle/minio")),
("object-lock", golden!("object_lock/rustfs")),
("encryption", golden!("encryption/minio")),
("tagging", golden!("tagging/rustfs")),
("cors", golden!("cors/rustfs")),
("logging", golden!("logging/rustfs")),
("website", golden!("website/rustfs")),
("accelerate", golden!("accelerate/rustfs")),
("requestPayment", golden!("request_payment/rustfs")),
("publicAccessBlock", golden!("public_access_block/rustfs")),
] {
put_xml(env, query, xml).await?;
}
client
.put_bucket_acl()
.bucket(CONFIGS)
.acl(BucketCannedAcl::Private)
.send()
.await?;
let acl = client.get_bucket_acl().bucket(CONFIGS).send().await?;
let owner = acl.owner().and_then(|owner| owner.id()).ok_or("ACL owner missing")?;
let [grant] = acl.grants() else { return Err("expected exactly one ACL grant".into()) };
let grantee = grant.grantee().ok_or("missing ACL grantee")?;
if owner.is_empty()
|| grant.permission().map(|p| p.as_str()) != Some("FULL_CONTROL")
|| grantee.r#type().as_str() != "CanonicalUser"
|| grantee.id().is_some()
{
return Err("GET ACL changed its canonical owner grant".into());
}
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Deny", "Principal": "*", "Action": "s3:DeleteObject",
"Resource": format!("arn:aws:s3:::{CONFIGS}/protected/*"),
}],
});
client
.put_bucket_policy()
.bucket(CONFIGS)
.policy(policy.to_string())
.send()
.await?;
set_bucket_quota(env, CONFIGS, 16 * 1024 * 1024).await?;
let migration = OdmSourceSpec::for_fake_source(target, "migration-source");
for (path, body) in [
("/iceberg/v1/buckets/persistence-warehouse".to_owned(), None),
(format!("{ODM_ADMIN_ROUTE}/persistence-migration"), Some(migration.to_json().to_string())),
] {
let (status, _) = admin_request(&env.url, Method::PUT, &path, body, &env.access_key, &env.secret_key).await?;
if !status.is_success() {
return Err(format!("configuring {path} failed with {status}").into());
}
}
Ok(())
}
async fn write_objects(client: &Client) -> Result<(), Error> {
for key in ["put", "tagged"] {
client
.put_object()
.bucket(OBJECTS)
.key(key)
.metadata("purpose", "byte-parity")
.body(ByteStream::from_static(b"persisted object contents"))
.send()
.await?;
}
client
.copy_object()
.bucket(OBJECTS)
.key("copy")
.copy_source(format!("{OBJECTS}/put"))
.send()
.await?;
let tags = Tagging::builder()
.tag_set(Tag::builder().key("type").value("persisted").build()?)
.build()?;
client
.put_object_tagging()
.bucket(OBJECTS)
.key("tagged")
.tagging(tags)
.send()
.await?;
write_multipart(client, OBJECTS, "multipart", &[vec![0x31; FIRST_PART_SIZE], vec![0x32; 100]]).await?;
Ok(())
}
fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
let b = bucket_metadata(volume, CONFIGS)?;
let configs = BTreeMap::from([
("acl", b.bucket_acl_config_json),
("accelerate", b.accelerate_config_xml),
("cors", b.cors_config_xml),
("encryption", b.encryption_config_xml),
("lifecycle", b.lifecycle_config_xml),
("logging", b.logging_config_xml),
("notification", b.notification_config_xml),
("object_lock", b.object_lock_config_xml),
("public_access_block", b.public_access_block_config_xml),
("request_payment", b.request_payment_config_xml),
("tagging", b.tagging_config_xml),
("versioning", b.versioning_config_xml),
("website", b.website_config_xml),
("durability", b.durability_config_json),
("policy", b.policy_config_json),
("quota", b.quota_config_json),
("replication", b.replication_config_xml),
("targets", b.bucket_targets_config_json),
("table", bucket_metadata(volume, "persistence-warehouse")?.table_bucket_config_json),
("odm", bucket_metadata(volume, "persistence-migration")?.on_demand_migration_config_json),
]);
let mut objects = BTreeMap::new();
for key in ["put", "copy", "tagged"] {
let (_, fields) = read_object(volume, key)?;
objects.insert(key, fields);
}
let (multipart, _) = read_object(volume, "multipart")?;
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
return Err("xl.meta does not describe both uploaded parts".into());
}
let directory = volume
.join(OBJECTS)
.join("multipart")
.join(multipart.data_dir.ok_or("missing multipart data directory")?.to_string());
let mut multipart_files = BTreeMap::new();
for entry in std::fs::read_dir(directory)? {
let entry = entry?;
let name = entry.file_name().into_string().map_err(|_| "invalid multipart file name")?;
if name.starts_with("part.") {
let bytes = std::fs::read(entry.path())?;
if bytes.is_empty() {
return Err(format!("empty multipart file {name}").into());
}
multipart_files.insert(name, bytes);
}
}
for name in ["part.1", "part.2", "part.1.meta", "part.2.meta"] {
if !multipart_files.contains_key(name) {
return Err(format!("missing multipart file {name}").into());
}
}
Ok(Snapshot {
configs,
objects,
multipart_files,
})
}
fn save_xml_samples(root: &Path, side: &str, snapshot: &Snapshot) -> Result<(), Error> {
let directory = root.join(side);
std::fs::create_dir_all(&directory)?;
let families: [&str; 13] = [
"accelerate",
"cors",
"encryption",
"lifecycle",
"logging",
"notification",
"object_lock",
"public_access_block",
"replication",
"request_payment",
"tagging",
"versioning",
"website",
];
for family in families {
let bytes = snapshot
.configs
.get(family)
.ok_or_else(|| format!("missing XML family {family}"))?;
std::fs::write(directory.join(format!("{family}.xml")), bytes)?;
}
Ok(())
}
#[tokio::test]
async fn persistence_byte_parity() -> Result<(), Error> {
let reference = PathBuf::from(
std::env::var_os("RUSTFS_PERSISTENCE_REFERENCE_BINARY").ok_or("missing pinned RUSTFS_PERSISTENCE_REFERENCE_BINARY")?,
);
let candidate = rustfs_binary_path();
if binary_version(&reference).await? == binary_version(&candidate).await? {
return Err("reference and candidate binaries report identical build identities".into());
}
// The quota writer records the process wall clock inside its JSON document. Keep raw bytes
// comparable while the SDK's signature clock and both servers' monotonic timers advance normally.
let clock_library = std::env::var("RUSTFS_PERSISTENCE_CLOCK_LIBRARY")?;
if !Path::new(&clock_library).is_absolute() || !Path::new(&clock_library).is_file() {
return Err("RUSTFS_PERSISTENCE_CLOCK_LIBRARY must name an absolute libfaketime path".into());
}
let clock = time::OffsetDateTime::now_utc()
.format(time::macros::format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"))?;
let target = FakeS3Target::start().await?;
target.create_bucket("migration-source");
target.create_bucket_with_object_lock("interop-dr");
let reference = capture(&reference, &target, &clock, &clock_library).await?;
let candidate = capture(&candidate, &target, &clock, &clock_library).await?;
if let Some(directory) = std::env::var_os("RUSTFS_PERSISTENCE_XML_DIR") {
save_xml_samples(Path::new(&directory), "reference", &reference)?;
save_xml_samples(Path::new(&directory), "candidate", &candidate)?;
}
if reference.configs.len() != 20 || candidate.configs.len() != 20 {
return Err("expected 13 XML and seven JSON fields on each side".into());
}
for (&field, expected) in &reference.configs {
// Canned ACLs acknowledge the write and return one owner grant without persisting an ACL document.
if expected.is_empty() != (field == "acl") || candidate.configs.get(field) != Some(expected) {
return Err(format!("incorrect or different persisted bucket config: {field}").into());
}
}
for (key, expected) in &reference.objects {
if candidate.objects.get(key) != Some(expected) {
return Err(format!("different persisted xl.meta fields: {key}").into());
}
}
if candidate.multipart_files != reference.multipart_files {
return Err("different multipart part files or part metadata".into());
}
eprintln!(
"Compared {} bucket fields (canned ACL empty), {} full object records and multipart part files",
reference.configs.len(),
reference.objects.len()
);
Ok(())
}
#[test]
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
let original = fields(&sample)?;
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
sample["V2Obj"]["MTime"] = serde_json::json!(2);
assert_eq!(fields(&sample)?, original);
sample["V2Obj"]["Future"] = serde_json::Value::Null;
let with_nil = fields(&sample)?;
assert_ne!(with_nil, original);
sample["V2Obj"]["Future"] = serde_json::json!(false);
let with_value = fields(&sample)?;
assert_ne!(with_value, with_nil);
sample["FutureEnvelope"] = serde_json::json!(1);
assert_ne!(fields(&sample)?, with_value);
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
Ok(())
}
@@ -465,7 +465,7 @@ pub(crate) async fn set_replication_target_with_options(
Ok(arn)
}
pub(crate) async fn send_set_replication_target_request(
async fn send_set_replication_target_request(
source_env: &RustFSTestEnvironment,
source_bucket: &str,
update: bool,
-6
View File
@@ -57,12 +57,6 @@ pub(crate) mod replication_extension {
pub(crate) use super::BucketTargetSys;
}
#[cfg(test)]
pub(crate) mod persistence_parity {
pub(crate) use super::RUSTFS_META_BUCKET;
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
}
#[cfg(test)]
pub(crate) mod upgrade_compatibility {
pub(crate) use super::{DiskOption, Endpoint, RUSTFS_META_MULTIPART_BUCKET, new_disk};
+15 -127
View File
@@ -46,12 +46,9 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -145,41 +142,6 @@ fn current_source_revision() -> Result<String, BoxError> {
Ok(revision.to_string())
}
pub(crate) struct UpgradeSourceCapabilities {
supports_delete_recovery: bool,
supports_multipart_publication: bool,
pub(crate) requires_list_parts_read_quorum: bool,
}
impl UpgradeSourceCapabilities {
fn from_revision(revision: &str) -> Result<Self, BoxError> {
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
match revision {
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
supports_delete_recovery: false,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_4_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: true,
requires_list_parts_read_quorum: true,
}),
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
}
}
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
Self::from_revision(&binary_source_revision(binary).await?)
}
}
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
if !output.status.success() {
@@ -263,25 +225,6 @@ mod g09_evidence_tests {
use super::*;
use tempfile::TempDir;
#[test]
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
(RC2_COMMIT, false, false, false),
(RC5_COMMIT, false, false, false),
(STABLE_1_0_1_COMMIT, false, false, false),
(STABLE_1_0_4_COMMIT, true, false, false),
(STABLE_1_0_5_COMMIT, true, true, true),
(STABLE_1_0_6_COMMIT, true, true, true),
] {
let source = UpgradeSourceCapabilities::from_revision(revision)?;
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
}
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
Ok(())
}
#[test]
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
let binary = Path::new("rustfs");
@@ -422,7 +365,7 @@ async fn complete_pending_multipart(
.await?)
}
pub(crate) async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
let pending = stage_multipart(client, bucket, key, parts).await?;
complete_pending_multipart(client, bucket, key, &pending).await?;
Ok(())
@@ -1068,7 +1011,8 @@ async fn exercise_mixed_cluster(
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
);
// Use a separate old-writer key so the original seed still spans the entire rollout.
// 1.0.4 already implements the delete decision protocol. Use a separate
// old-writer key so the original seed still spans the entire rollout.
if previous_has_delete_recovery {
let key = format!(".upgrade-delete/{phase}/previous-writer");
previous_client
@@ -2349,7 +2293,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let previous_binary = source_binary()?;
// Bind the expected protocol to the verified release source, never to the
// DELETE result itself. The older CI source pins retain their 503 contract.
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
let evidence_context = g09_evidence_context(&previous_binary).await?;
let current_binary = rustfs_binary_path();
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2371,21 +2315,17 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.stop_node(0)?;
cluster.start_node_from_binary(0, &current_binary).await?;
let mut mixed_deleted_keys = Vec::new();
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
let mixed_completion = if source.supports_multipart_publication {
let clients = cluster.create_all_clients()?;
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
} else {
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
None
};
// One controlled out-of-policy attempt proves fail-closed behavior below
// native write quorum; positive MPU writers remain paused while mixed.
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
cluster.stop_node(node_idx)?;
cluster.start_node_from_binary(node_idx, &current_binary).await?;
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
}
@@ -2395,13 +2335,6 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let clients = cluster.create_all_clients()?;
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
if let Some(mixed) = mixed_completion {
assert_eq!(
completed_pending.version_id, mixed.version_id,
"full upgrade changed the completed version"
);
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
}
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
@@ -2518,7 +2451,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"phases": phases,
"objects_per_phase": objects_per_phase,
"mixed_load": "plain-put",
"mixed_delete_protocol_supported": source.supports_delete_recovery,
"mixed_delete_protocol_supported": previous_has_delete_recovery,
"mixed_deleted_keys": mixed_deleted_keys,
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
@@ -2528,7 +2461,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"all nodes list every mixed-version object after homogeneous-current convergence",
"the previous-release seed survives the baseline restart and every upgrade phase",
"previous readers read every current-native MPU body after quiescent rollback",
"the same previous-release upload remains readable after full upgrade and rollback"
"same legacy upload resumes with identical body, ETag and version after full upgrade"
],
}),
)?;
@@ -2550,13 +2483,9 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"previous-release writer publishes objects readable by current node",
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
if source.supports_multipart_publication {
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
} else {
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
},
"fully upgraded readers see the same body, ETag and sole version after replay",
if source.supports_delete_recovery {
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
"the same legacy upload completes and replays once fully upgraded without adding a version",
if previous_has_delete_recovery {
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
} else {
"modern DELETE fails closed and preserves every byte while old peers remain"
@@ -2903,7 +2832,7 @@ async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, ser
env.start_rustfs_server_from_binary(binary, vec![], server_env).await
}
pub(crate) async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
let path = format!("/rustfs/admin/v3/quota/{bucket}");
let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
@@ -3457,8 +3386,6 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
#[tokio::test]
#[ignore = "requires a pinned previous RustFS release binary"]
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
use sha2::Digest;
init_logging();
let previous_binary = source_binary()?;
let evidence_context = g09_evidence_context(&previous_binary).await?;
@@ -3496,16 +3423,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
let multipart_bytes = multipart_parts.concat();
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
// This release can finish an upload staged by the current gateway.
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
Some((pending, parts.concat()))
} else {
None
};
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
let old_client = env.create_s3_client();
@@ -3521,16 +3438,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
let rollback_multipart = if let Some((pending, expected)) = inflight {
assert_pending_parts(&old_client, &pending).await?;
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(body, expected);
Some((pending.part_sizes, body))
} else {
None
};
// A PUT on the rolled-back release must still honour the encryption
// configuration it decoded out of the current build's metadata blob.
let post_rollback_key = "rollback/written-after-rollback";
@@ -3571,25 +3478,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
post_rollback_bytes
);
if let Some((part_sizes, rollback_read)) = rollback_multipart {
let (_, current_read) = read_object(&current_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(current_read, rollback_read);
if let Some(context) = &evidence_context {
let evidence = serde_json::json!({
"previous_revision": context.previous_revision,
"current_revision": context.current_revision,
"part_sizes": part_sizes,
"bytes": current_read.len(),
"rollback_read_sha256": format!("{:x}", sha2::Sha256::digest(&rollback_read)),
"roll_forward_read_sha256": format!("{:x}", sha2::Sha256::digest(&current_read)),
});
std::fs::write(
context.directory.join("inflight-multipart-rollback.json"),
serde_json::to_vec_pretty(&evidence)?,
)?;
}
}
if let Some(context) = evidence_context.as_ref() {
write_g09_evidence(
context,
+1
View File
@@ -258,6 +258,7 @@ windows-sys = { workspace = true, features = [
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
aws-smithy-async.workspace = true
rustfs-s3-ops.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
+36 -16
View File
@@ -25,8 +25,10 @@
//! Not responsible for: validating configurations, touching the bucket
//! metadata store, or choosing a canonical shape.
//!
//! The tests exercise the production bucket configuration codec through `tests/storage_api.rs`.
//! The lifecycle date cases also pin the results of the legacy codec. The DTO/codec
//! Upstream: the bucket-config `deserialize`/`serialize` pair, reached through the
//! integration-test facade `tests/storage_api.rs`, and the configuration types they are
//! generic over (`BucketConfigXml`: the gateway persistence shapes for the families that
//! have moved, the s3s DTOs for the rest). Downstream: the DTO/codec
//! replacement (T1.4) and the persisted-bytes comparison (T3.7), which must
//! keep every golden green without regenerating it. Regeneration is reserved
//! to `scripts/gen_bucket_config_goldens.sh`, which sets
@@ -504,12 +506,13 @@ mod bucket_config_goldens {
mod bucket_config_lifecycle_date_compat {
use super::*;
// Captured from the 5761ddfe reader and writer. The reader preserves the offset and
// nanoseconds. The writer emits the local clock fields at millisecond precision with a Z.
const DATES: [(&str, &str); 3] = [
("2023-11-14T22:13:20.123456789Z", "2023-11-14T22:13:20.123Z"),
("2023-11-14T22:13:20.123456789+05:30", "2023-11-14T22:13:20.123Z"),
("2025-01-02T00:00:00+02:00", "2025-01-02T00:00:00.000Z"),
use rustfs_lifecycle::core::format_lifecycle_date;
use time::OffsetDateTime;
const DATES: [&str; 3] = [
"2023-11-14T22:13:20.123456789Z",
"2023-11-14T22:13:20.123456789+05:30",
"2025-01-02T00:00:00+02:00",
];
fn document(date: &str) -> Vec<u8> {
@@ -519,17 +522,33 @@ mod bucket_config_lifecycle_date_compat {
.into_bytes()
}
fn legacy_read(input: &[u8]) -> s3s::dto::BucketLifecycleConfiguration {
let mut reader = s3s::xml::Deserializer::new(input);
let value = s3s::xml::Deserialize::deserialize(&mut reader).expect("the actual old lifecycle reader accepts the date");
reader.expect_eof().expect("the old reader consumes the entire date fixture");
value
}
#[test]
fn lifecycle_xml_reader_preserves_legacy_date_precision_and_offset() {
for (date, _) in DATES {
let new: PersistedLifecycleConfiguration = deserialize(&document(date)).expect("read persisted lifecycle date");
for date in DATES {
let input = document(date);
let old = legacy_read(&input);
let new: PersistedLifecycleConfiguration = deserialize(&input).expect("read persisted lifecycle date");
let old_date = |value: &s3s::dto::Timestamp| {
format_lifecycle_date(OffsetDateTime::from(value.clone())).expect("format the original old-reader date")
};
assert_eq!(
[
new.expiry_updated_at.clone(),
new.rules[0].expiration.as_ref().and_then(|value| value.date.clone()),
new.rules[0].transitions.as_ref().unwrap()[0].date.clone(),
],
[Some(date.to_owned()), Some(date.to_owned()), Some(date.to_owned())],
[
Some(old_date(old.expiry_updated_at.as_ref().unwrap())),
Some(old_date(old.rules[0].expiration.as_ref().unwrap().date.as_ref().unwrap())),
Some(old_date(old.rules[0].transitions.as_ref().unwrap()[0].date.as_ref().unwrap())),
],
"the lifecycle reader must retain the old date representation for {date}"
);
}
@@ -537,10 +556,11 @@ mod bucket_config_lifecycle_date_compat {
#[test]
fn lifecycle_xml_writer_preserves_legacy_date_bytes() {
for (date, written_date) in DATES {
let expected = format!(
"<LifecycleConfiguration><ExpiryUpdatedAt>{written_date}</ExpiryUpdatedAt><Rule><Expiration><Date>{written_date}</Date></Expiration><ID>date-compat</ID><Status>Enabled</Status><Transition><Date>{written_date}</Date><StorageClass>GLACIER</StorageClass></Transition></Rule></LifecycleConfiguration>"
);
for date in DATES {
let old = legacy_read(&document(date));
let mut old_xml = Vec::new();
s3s::xml::Serialize::serialize(&old, &mut s3s::xml::Serializer::new(&mut old_xml))
.expect("the actual old lifecycle writer emits the date");
let new = PersistedLifecycleConfiguration {
expiry_updated_at: Some(date.to_owned()),
rules: vec![PersistedLifecycleRule {
@@ -565,7 +585,7 @@ mod bucket_config_lifecycle_date_compat {
};
assert_eq!(
serialize(&new).expect("write persisted lifecycle date"),
expected.as_bytes(),
old_xml,
"legacy date bytes for {date}"
);
}
+1 -1
View File
@@ -6,7 +6,7 @@ Applies to `crates/kms/`.
When changing key-management behavior, verify compatibility with:
- `rustfs/src/storage/sse.rs`
- `rustfs/src/storage/ecfs.rs`
- `rustfs/src/admin/handlers/kms.rs`
- `rustfs/src/admin/handlers/kms_dynamic.rs`
- `rustfs/src/admin/handlers/kms_keys.rs`
+55 -54
View File
@@ -28,7 +28,6 @@ use tracing_subscriber::{
use super::http2_ping::HYPER_HTTP2_CLIENT_TARGET;
pub const HTTP_SERVER_LOG_TARGET: &str = "rustfs::server::http";
const S3_DEPENDENCY_TARGETS: [&str; 2] = ["s3s", "rustfs_gateway"];
/// Pyroscope emits raw reqwest errors from its background session manager.
/// Those errors can include the configured endpoint, so they must never reach
@@ -45,13 +44,12 @@ pub(super) fn pyroscope_log_filter() -> FilterFn {
filter_fn(allows_non_pyroscope_dependency_event)
}
/// Keep verbose protocol events out of logs and OTLP while preserving diagnostic spans.
pub(super) fn s3_dependency_log_filter() -> FilterFn {
/// s3s verbose events include response bodies and expected signatures. Keep
/// diagnostics spans, but never let these events reach logs or OTLP exporters.
pub(super) fn s3s_log_filter() -> FilterFn {
filter_fn(|metadata| {
!(metadata.is_event()
&& S3_DEPENDENCY_TARGETS
.iter()
.any(|target| directive_applies_to_target(target, metadata.target()))
&& (metadata.target() == "s3s" || metadata.target().starts_with("s3s::"))
&& *metadata.level() > tracing::Level::INFO)
})
}
@@ -62,7 +60,7 @@ pub(super) fn s3_dependency_log_filter() -> FilterFn {
/// `RUST_LOG` environment variable takes precedence over `logger_level`.
/// For non-verbose levels (`info`, `warn`, `error`), noisy dependency targets
/// are automatically suppressed based on the effective log configuration.
/// Transport internals are disabled, while S3 protocol warnings remain visible. The
/// Transport internals are disabled, while `s3s` remains visible at WARN. The
/// diagnostic HTTP/2 PING feature adds back only Hyper's client target at DEBUG
/// so its exact keepalive timeout event reaches the metrics layer.
///
@@ -181,14 +179,33 @@ fn effective_level_for_target<'a>(rust_log: &'a str, target: &str) -> Option<&'a
best_match.map(|(_, _, level)| level)
}
fn should_demote_target(target: &str, logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
fn should_demote_http_request_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
if let Some(level) = default_level {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
if let Some(rust_log) = rust_log {
if let Some(level) = effective_level_for_target(rust_log, target) {
if let Some(level) = effective_level_for_target(rust_log, HTTP_SERVER_LOG_TARGET) {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
return false;
}
let level = logger_level.trim().to_ascii_lowercase();
matches!(level.as_str(), "info" | "warn")
}
fn should_demote_s3s_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
if let Some(level) = default_level {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
if let Some(rust_log) = rust_log {
if let Some(level) = effective_level_for_target(rust_log, "s3s") {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
@@ -242,16 +259,17 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
("tower", LevelFilter::OFF),
];
if should_demote_target(HTTP_SERVER_LOG_TARGET, logger_level, default_level, rust_log_env.as_deref()) {
if should_demote_http_request_logs(logger_level, default_level, rust_log_env.as_deref()) {
// HTTP request logs are demoted to WARN to reduce volume in production,
// but only when the effective log level is not stricter than WARN.
directives.push((HTTP_SERVER_LOG_TARGET, LevelFilter::WARN));
}
for target in S3_DEPENDENCY_TARGETS {
if should_demote_target(target, logger_level, default_level, rust_log_env.as_deref()) {
directives.push((target, LevelFilter::WARN));
}
if should_demote_s3s_logs(logger_level, default_level, rust_log_env.as_deref()) {
// s3s instruments each authentication request through multiple
// nested INFO spans. Keep warnings and errors in normal production
// logs without promoting it when the effective base is stricter.
directives.push(("s3s", LevelFilter::WARN));
}
for (crate_name, level) in directives {
@@ -345,14 +363,14 @@ mod tests {
}
#[test]
fn s3_dependency_filter_blocks_verbose_events_despite_explicit_directives() {
fn s3s_filter_blocks_credentials_despite_explicit_verbose_directives() {
use tracing_subscriber::prelude::*;
for directive in ["trace", concat!("trace,", "s3s::service=trace,", "s3s::ops::signature=debug")] {
let writer = SharedWriter::default();
let captured = Arc::clone(&writer.0);
let subscriber = tracing_subscriber::registry()
.with(EnvFilter::new(directive))
.with(s3_dependency_log_filter())
.with(s3s_log_filter())
.with(
tracing_subscriber::fmt::layer()
.without_time()
@@ -366,10 +384,6 @@ mod tests {
tracing::debug!(target: "s3s::service", response = "secret-response", "response dump");
tracing::trace!(target: "s3s", response = "secret-trace", "response dump");
tracing::debug!(target: "s3s::ops::signature", expected = "secret-signature", "signature mismatch");
tracing::debug!(target: "rustfs_gateway::service", "secret-gateway-debug");
tracing::trace!(target: "rustfs_gateway", "secret-gateway-trace");
tracing::info!(target: "rustfs_gateway::service", "visible-gateway-info");
tracing::debug!(target: "rustfs_gateway_other", "visible-gateway-neighbor");
tracing::info!(target: "s3s::service", "visible-info");
tracing::warn!(target: "s3s::service", "visible-warning");
tracing::debug!(target: "rustfs", "visible-rustfs-debug");
@@ -379,8 +393,6 @@ mod tests {
assert!(!output.contains("secret"), "{output}");
for expected in [
"safe-operation-span",
"visible-gateway-info",
"visible-gateway-neighbor",
"visible-info",
"visible-warning",
"visible-rustfs-debug",
@@ -433,29 +445,22 @@ mod tests {
#[test]
fn test_should_demote_http_request_logs() {
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, None));
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "warn", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "off", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("ERROR")));
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, Some("WARN")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("foo=warn")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs=error")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs::server=error")));
assert!(!should_demote_target(
HTTP_SERVER_LOG_TARGET,
"info",
None,
Some(&http_log_directive("error"))
));
assert!(!should_demote_target(
HTTP_SERVER_LOG_TARGET,
assert!(should_demote_http_request_logs("info", None, None));
assert!(should_demote_http_request_logs("warn", None, None));
assert!(!should_demote_http_request_logs("error", None, None));
assert!(!should_demote_http_request_logs("off", None, None));
assert!(!should_demote_http_request_logs("info", None, Some("ERROR")));
assert!(should_demote_http_request_logs("error", None, Some("WARN")));
assert!(!should_demote_http_request_logs("info", None, Some("foo=warn")));
assert!(!should_demote_http_request_logs("info", None, Some("rustfs=error")));
assert!(!should_demote_http_request_logs("info", None, Some("rustfs::server=error")));
assert!(!should_demote_http_request_logs("info", None, Some(&http_log_directive("error"))));
assert!(!should_demote_http_request_logs(
"info",
None,
Some(&format!("WARN,{}", http_log_directive("error")))
));
assert!(should_demote_target(
HTTP_SERVER_LOG_TARGET,
assert!(should_demote_http_request_logs(
"error",
None,
Some(&format!("WARN,{}", http_log_directive("warn")))
@@ -464,15 +469,15 @@ mod tests {
#[test]
fn test_should_demote_s3s_logs() {
assert!(should_demote_target("s3s", "info", None, None));
assert!(should_demote_target("s3s", "warn", None, None));
assert!(!should_demote_target("s3s", "error", None, None));
assert!(!should_demote_target("s3s", "off", None, None));
assert!(!should_demote_target("s3s", "info", None, Some("ERROR")));
assert!(should_demote_target("s3s", "error", None, Some("WARN")));
assert!(!should_demote_target("s3s", "info", None, Some("foo=warn")));
assert!(!should_demote_target("s3s", "info", None, Some("s3s=error")));
assert!(should_demote_target("s3s", "error", None, Some("WARN,s3s=warn")));
assert!(should_demote_s3s_logs("info", None, None));
assert!(should_demote_s3s_logs("warn", None, None));
assert!(!should_demote_s3s_logs("error", None, None));
assert!(!should_demote_s3s_logs("off", None, None));
assert!(!should_demote_s3s_logs("info", None, Some("ERROR")));
assert!(should_demote_s3s_logs("error", None, Some("WARN")));
assert!(!should_demote_s3s_logs("info", None, Some("foo=warn")));
assert!(!should_demote_s3s_logs("info", None, Some("s3s=error")));
assert!(should_demote_s3s_logs("error", None, Some("WARN,s3s=warn")));
}
#[test]
@@ -614,14 +619,10 @@ mod tests {
let output = capture_with_filter(build_env_filter("info", None), || {
tracing::info!(target: "s3s::auth", "s3s info must be suppressed");
tracing::warn!(target: "s3s::auth", "s3s warning must remain");
tracing::info!(target: "rustfs_gateway::service", "gateway info must be suppressed");
tracing::warn!(target: "rustfs_gateway::service", "gateway warning must remain");
});
assert!(!output.contains("s3s info must be suppressed"), "{output}");
assert!(output.contains("s3s warning must remain"), "{output}");
assert!(!output.contains("gateway info must be suppressed"), "{output}");
assert!(output.contains("gateway warning must remain"), "{output}");
});
}
+3 -3
View File
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
use crate::config::OtelConfig;
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3_dependency_log_filter};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::rolling::{RollingAppender, Rotation};
use metrics::counter;
use rustfs_config::observability::{
@@ -423,7 +423,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
.with(env_filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3_dependency_log_filter())
.with(s3s_log_filter())
.with(ErrorLayer::default())
.with(fmt_layer)
.try_init()
@@ -537,7 +537,7 @@ fn init_file_logging_internal(
.with(env_filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3_dependency_log_filter())
.with(s3s_log_filter())
.with(ErrorLayer::default())
.with(file_layer)
.with(stdout_layer)
+4 -4
View File
@@ -39,7 +39,7 @@
use crate::cleaner::types::FileMatchMode;
use crate::config::OtelConfig;
use crate::global::set_observability_metric_enabled;
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3_dependency_log_filter};
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
use crate::telemetry::local::{build_json_log_layer, open_rolling_appender, spawn_cleanup_task};
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -319,7 +319,7 @@ pub(super) fn init_observability_http(
.with(filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3_dependency_log_filter())
.with(s3s_log_filter())
.with(ErrorLayer::default())
.with(file_layer_opt)
.with(stdout_layer_opt)
@@ -1361,7 +1361,7 @@ mod tests {
let subscriber = tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(s3_dependency_log_filter())
.with(s3s_log_filter())
.with(
tracing_subscriber::fmt::layer()
.with_ansi(false)
@@ -1385,7 +1385,7 @@ mod tests {
tracing_subscriber::registry()
.with(build_env_filter("info", None))
.with(pyroscope_log_filter())
.with(s3_dependency_log_filter())
.with(s3s_log_filter())
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
.try_init()
.expect("install isolated LogTracer subscriber");
+1 -1
View File
@@ -13,7 +13,7 @@ Applies to `crates/policy/`.
- Validate compatibility with IAM and request-auth call sites:
- `crates/iam/`
- `rustfs/src/auth.rs`
- `rustfs/src/storage/authorization/`
- `rustfs/src/storage/access.rs`
## Error and Observability
+4
View File
@@ -70,6 +70,10 @@ url.workspace = true
urlencoding = { workspace = true }
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
# The remote error code table is checked against the legacy client's.
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
[lints]
workspace = true
+15 -18
View File
@@ -106,7 +106,7 @@ impl fmt::Display for RemoteErrorCode {
}
}
/// The sorted wire codes recognized by the legacy client at revision 5761ddfe.
/// The codes the legacy client (s3s at the pinned revision) recognized, sorted.
const RECOGNIZED_CODES: &[&str] = &[
"AccessControlListNotSupported",
"AccessDenied",
@@ -352,19 +352,15 @@ const RECOGNIZED_CODES: &[&str] = &[
#[cfg(test)]
mod tests {
use super::*;
use sha2::{Digest, Sha256};
#[test]
fn recognizes_exactly_the_codes_the_legacy_client_named() {
assert_eq!(RECOGNIZED_CODES.len(), 239);
assert!(RECOGNIZED_CODES.windows(2).all(|pair| pair[0] < pair[1]), "the table must stay sorted");
// SHA-256 of the sorted, newline-separated wire names in S3_ERROR_CODE_MAP at 5761ddfe.
let digest = Sha256::digest(RECOGNIZED_CODES.join("\n"));
assert_eq!(
hex_simd::encode_to_string(digest.as_slice(), hex_simd::AsciiCase::Lower),
"6fa6be09aa53ae2e05c5c98c10c43dae79713b07a2bdde492dbb6a776888978d"
);
for code in RECOGNIZED_CODES {
let legacy = s3s::S3ErrorCode::from_bytes(code.as_bytes()).expect("utf-8");
assert!(!matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{code} is not named by the legacy client");
assert_eq!(legacy.as_str(), *code);
let parsed = RemoteErrorCode::from_wire(code);
assert!(!parsed.is_custom(), "{code}");
assert_eq!(parsed.as_str(), *code);
@@ -373,18 +369,19 @@ mod tests {
#[test]
fn parses_like_the_legacy_client() {
for (text, expected, custom) in [
("NoSuchKey", "NoSuchKey", false),
("nosuchkey", "NoSuchKey", false),
("NOSUCHKEY", "NoSuchKey", false),
("PermanentRedirect", "PermanentRedirect", false),
("SomethingNew", "SomethingNew", true),
("", "", true),
("No Such Key", "No Such Key", true),
for text in [
"NoSuchKey",
"nosuchkey",
"NOSUCHKEY",
"PermanentRedirect",
"SomethingNew",
"",
"No Such Key",
] {
let legacy = s3s::S3ErrorCode::from_bytes(text.as_bytes()).expect("utf-8");
let parsed = RemoteErrorCode::from_wire(text);
assert_eq!(parsed.is_custom(), custom, "{text:?}");
assert_eq!(parsed.as_str(), expected, "{text:?}");
assert_eq!(parsed.is_custom(), matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{text:?}");
assert_eq!(parsed.as_str(), legacy.as_str(), "{text:?}");
}
assert_eq!(RemoteErrorCode::from_wire("nosuchkey"), RemoteErrorCode::NO_SUCH_KEY);
}
+2 -4
View File
@@ -164,10 +164,10 @@ impl From<legacy_s3s::S3Error> for S3Error {
/// reference only, so the whole error travels inside and this wrapper presents
/// the cause's own text and chain, as if the cause itself had been moved.
#[derive(Debug)]
pub(super) struct LegacyCause(legacy_s3s::S3Error);
struct LegacyCause(legacy_s3s::S3Error);
impl LegacyCause {
pub(super) fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
self.0.source()
}
}
@@ -378,8 +378,6 @@ mod tests {
legacy.set_source(Box::new(io::Error::other("read quorum")));
let mut ours: S3Error = legacy.into();
assert!(ours.source().is_some_and(|cause| cause.is::<io::Error>()));
assert!(std::error::Error::source(&ours).is_some_and(|cause| cause.is::<io::Error>()));
ours.set_message("retry later".to_owned());
let back: legacy_s3s::S3Error = ours.into();
assert_eq!(back.code(), &legacy_s3s::S3ErrorCode::ServiceUnavailable);
+2 -7
View File
@@ -157,12 +157,7 @@ impl S3Error {
/// The cause, if one was attached.
pub fn source(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
let source = self.0.source.as_deref()?;
#[cfg(any(test, feature = "compat-s3s"))]
if let Some(legacy) = source.downcast_ref::<crate::compat_s3s::LegacyCause>() {
return legacy.cause();
}
Some(source)
self.0.source.as_deref()
}
/// The explicit status if one was set, otherwise the code's own status;
@@ -207,7 +202,7 @@ impl fmt::Display for S3Error {
impl std::error::Error for S3Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
S3Error::source(self).map(|source| source as _)
self.0.source.as_deref().map(|source| source as _)
}
}
+1
View File
@@ -109,6 +109,7 @@ bytes.workspace = true
hex-simd.workspace = true
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
opentelemetry-proto.workspace = true
opentelemetry_sdk.workspace = true
prost.workspace = true
+337 -90
View File
@@ -1341,9 +1341,32 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() {
#[test]
fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after() {
// Seven-field tuple written with the pinned legacy scanner codec.
let old_info = b"\x97\xa6bucket\x07\xc0\xc3\xc0\xc0\x81\xaabad-object\x0b";
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(old_info).expect("decode the historical cache info tuple");
#[derive(Serialize)]
struct OldDataUsageCacheInfo {
name: String,
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
}
let old_info = OldDataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
last_update: None,
skip_healing: true,
lifecycle: None,
replication: None,
failed_objects: HashMap::from([("bad-object".to_string(), 11)]),
};
let mut buf = Vec::new();
old_info
.serialize(&mut rmp_serde::Serializer::new(&mut buf))
.expect("Failed to serialize old cache info");
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(&buf).expect("Failed to deserialize old cache info");
assert_eq!(decoded.name, "bucket");
assert_eq!(decoded.next_cycle, 7);
@@ -1368,13 +1391,13 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<()>,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
scan_resume_after: Option<String>,
scan_checkpoint: Option<DataUsageScanCheckpoint>,
pending_heals: Vec<PendingScannerHeal>,
object_lock: Option<()>,
object_lock: Option<Arc<s3s::dto::ObjectLockConfiguration>>,
}
#[derive(Deserialize)]
@@ -3537,21 +3560,22 @@ const LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD: &[u8] = &[
0x62, 0x6c, 0x65, 0x64, 0xc0,
];
// Project the seven original fields out of the production map. Keep the nested
// MessagePack arrays and maps opaque so their bytes can be checked against the old writer.
// Keep the old tuple carrier and payload explicit when the production field changes type.
#[derive(serde::Serialize, serde::Deserialize)]
struct LegacyLifecycleCacheInfo {
name: String,
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Value,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
}
struct LifecycleCacheOracleCase {
label: &'static str,
old_tuple: Vec<u8>,
old_payload: Vec<u8>,
current: DataUsageCacheInfo,
current_map: Vec<u8>,
frozen_tuple: &'static [u8],
@@ -3560,6 +3584,7 @@ struct LifecycleCacheOracleCase {
}
fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
use s3s::dto as legacy;
use time::OffsetDateTime;
let expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_456_789)
.expect("the legacy expiration fixture has an exact timestamp");
@@ -3568,6 +3593,136 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
let expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_789_123)
.expect("the legacy expiry update fixture has an exact timestamp");
let absent = legacy::LifecycleRule {
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
status: legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::ENABLED),
transitions: None,
};
let mut empty = absent.clone();
empty.id = Some(String::new());
empty.prefix = Some(String::new());
empty.status = legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::DISABLED);
empty.abort_incomplete_multipart_upload = Some(legacy::AbortIncompleteMultipartUpload {
days_after_initiation: None,
});
empty.del_marker_expiration = Some(legacy::DelMarkerExpiration { days: None });
empty.expiration = Some(legacy::LifecycleExpiration {
days: Some(0),
..Default::default()
});
empty.filter = Some(legacy::LifecycleRuleFilter {
and: Some(legacy::LifecycleRuleAndOperator {
tags: Some(Vec::new()),
..Default::default()
}),
tag: Some(legacy::Tag { key: None, value: None }),
..Default::default()
});
empty.noncurrent_version_expiration = Some(legacy::NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(-2),
});
empty.noncurrent_version_transitions = Some(Vec::new());
empty.transitions = Some(Vec::new());
let mut absent_tags = absent.clone();
absent_tags.filter = Some(legacy::LifecycleRuleFilter {
and: Some(legacy::LifecycleRuleAndOperator::default()),
..Default::default()
});
// Cached values must survive independently of lifecycle policy validation.
let complete = legacy::BucketLifecycleConfiguration {
expiry_updated_at: Some(expiry_updated_at.into()),
rules: vec![
legacy::LifecycleRule {
abort_incomplete_multipart_upload: Some(legacy::AbortIncompleteMultipartUpload {
days_after_initiation: Some(0),
}),
del_marker_expiration: Some(legacy::DelMarkerExpiration { days: Some(-1) }),
expiration: Some(legacy::LifecycleExpiration {
date: Some(expiration_date.into()),
days: Some(-3),
expired_object_delete_marker: Some(false),
expired_object_all_versions: Some(false),
}),
filter: Some(legacy::LifecycleRuleFilter {
cached_tags: Default::default(),
and: Some(legacy::LifecycleRuleAndOperator {
prefix: Some(String::new()),
tags: Some(vec![
legacy::Tag {
key: Some(String::new()),
value: Some(String::new()),
},
legacy::Tag {
key: None,
value: Some("value-without-key".to_string()),
},
legacy::Tag {
key: Some("key-without-value".to_string()),
value: None,
},
]),
object_size_greater_than: Some(-1),
object_size_less_than: Some(4096),
}),
prefix: Some(String::new()),
tag: Some(legacy::Tag {
key: Some("kind".to_string()),
value: Some("archive".to_string()),
}),
object_size_greater_than: Some(0),
object_size_less_than: Some(8192),
}),
id: Some("complete-legacy-rule".to_string()),
noncurrent_version_expiration: Some(legacy::NoncurrentVersionExpiration {
noncurrent_days: Some(-7),
newer_noncurrent_versions: Some(0),
}),
noncurrent_version_transitions: Some(vec![
legacy::NoncurrentVersionTransition {
noncurrent_days: Some(0),
newer_noncurrent_versions: Some(-1),
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
},
legacy::NoncurrentVersionTransition {
noncurrent_days: None,
newer_noncurrent_versions: None,
storage_class: None,
},
]),
prefix: Some("legacy-prefix/".to_string()),
status: legacy::ExpirationStatus::from_static("LegacyUnknownStatus"),
transitions: Some(vec![
legacy::Transition {
date: Some(transition_date.into()),
days: Some(0),
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
},
legacy::Transition {
date: None,
days: None,
storage_class: Some(legacy::TransitionStorageClass::from_static("")),
},
legacy::Transition {
date: None,
days: None,
storage_class: None,
},
]),
},
empty,
absent_tags,
absent,
],
};
use rustfs_gateway_types::persistence as persisted;
let domain_absent = persisted::PersistedLifecycleRule {
abort_incomplete_multipart_upload: None,
@@ -3714,12 +3869,17 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
(
"none",
None,
None,
LEGACY_LIFECYCLE_NONE_TUPLE,
LEGACY_LIFECYCLE_NONE_MAP,
LEGACY_LIFECYCLE_NONE_PAYLOAD,
),
(
"empty",
Some(Arc::new(legacy::BucketLifecycleConfiguration {
rules: Vec::new(),
expiry_updated_at: None,
})),
Some(Arc::new(persisted::PersistedLifecycleConfiguration {
rules: Vec::new(),
expiry_updated_at: None,
@@ -3730,6 +3890,7 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
),
(
"nonempty",
Some(Arc::new(complete)),
Some(Arc::new(domain_complete)),
LEGACY_LIFECYCLE_NONEMPTY_TUPLE,
LEGACY_LIFECYCLE_NONEMPTY_MAP,
@@ -3738,17 +3899,35 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
];
cases
.into_iter()
.map(|(label, lifecycle, frozen_tuple, frozen_map, frozen_payload)| {
let current = DataUsageCacheInfo {
.map(|(label, lifecycle, domain, frozen_tuple, frozen_map, frozen_payload)| {
let old = LegacyLifecycleCacheInfo {
name: "legacy-lifecycle-bucket".to_string(),
next_cycle: 7,
last_update: None,
skip_healing: true,
lifecycle,
replication: None,
failed_objects: HashMap::new(),
};
let mut old_tuple = Vec::new();
old.serialize(&mut rmp_serde::Serializer::new(&mut old_tuple))
.expect("the real old tuple carrier must encode");
let old_payload = rmp_serde::to_vec(&old.lifecycle).expect("the real old lifecycle payload must encode");
let current = DataUsageCacheInfo {
name: old.name.clone(),
next_cycle: old.next_cycle,
last_update: old.last_update,
skip_healing: old.skip_healing,
lifecycle: domain,
replication: None,
failed_objects: old.failed_objects,
..Default::default()
};
let current_map = rmp_serde::to_vec(&current).expect("the actual production map writer must encode the fixture");
LifecycleCacheOracleCase {
label,
old_tuple,
old_payload,
current,
current_map,
frozen_tuple,
@@ -3761,36 +3940,57 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
#[test]
fn test_lifecycle_cache_legacy_msgpack_oracle() {
use time::OffsetDateTime;
for case in lifecycle_cache_oracle_cases() {
let LifecycleCacheOracleCase {
label,
current,
current_map,
frozen_tuple,
frozen_map,
frozen_payload,
} = case;
let label = case.label;
let old_tuple = case.old_tuple;
let old_payload = case.old_payload;
let current = case.current;
let current_map = case.current_map;
let frozen_tuple = case.frozen_tuple;
let frozen_map = case.frozen_map;
let frozen_payload = case.frozen_payload;
println!("legacy_lifecycle_cache {label}: tuple={old_tuple:02x?}; map={current_map:02x?}; payload={old_payload:02x?}");
let old_from_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&current_map).expect("the actual old reader must read the production map");
if label == "nonempty" {
let lifecycle = old_from_current
.lifecycle
.as_ref()
.expect("the old reader must retain the nonempty lifecycle");
let expiration = lifecycle.rules[0]
.expiration
.as_ref()
.expect("the complete rule has expiration");
let transition = &lifecycle.rules[0]
.transitions
.as_ref()
.expect("the complete rule has transitions")[0];
let expected_expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000)
.expect("the legacy millisecond expiration date is representable");
let expected_transition_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_001_987_000_000)
.expect("the legacy millisecond transition date is representable");
let expected_expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_000_000)
.expect("the legacy millisecond expiry update is representable");
assert_eq!(
(
expiration.date.clone().map(OffsetDateTime::from),
transition.date.clone().map(OffsetDateTime::from),
lifecycle.expiry_updated_at.clone().map(OffsetDateTime::from),
),
(
Some(expected_expiration_date),
Some(expected_transition_date),
Some(expected_expiry_updated_at),
),
"cache serialization must retain the fixed legacy millisecond precision for Date and ExpiryUpdatedAt"
);
}
let frozen_tuple_read: DataUsageCacheInfo =
rmp_serde::from_slice(frozen_tuple).expect("the production reader must accept the fixed legacy tuple bytes");
let frozen_map_read: DataUsageCacheInfo =
rmp_serde::from_slice(frozen_map).expect("the production reader must accept the fixed legacy map bytes");
if label == "nonempty" {
let lifecycle = frozen_map_read.lifecycle.as_ref().expect("nonempty lifecycle");
let rule = &lifecycle.rules[0];
assert_eq!(
[
rule.expiration.as_ref().expect("expiration").date.as_deref(),
rule.transitions.as_ref().expect("transitions")[0].date.as_deref(),
lifecycle.expiry_updated_at.as_deref(),
],
[
Some("2023-11-14T22:13:20.123Z"),
Some("2023-11-14T22:13:21.987Z"),
Some("2023-11-14T22:13:22.456Z"),
],
"legacy timestamps retain millisecond precision",
);
}
assert_eq!(
[
rmp_serde::to_vec(&frozen_tuple_read).expect("encode the fixed legacy tuple after production read"),
@@ -3814,21 +4014,21 @@ fn test_lifecycle_cache_legacy_msgpack_oracle() {
fn test_lifecycle_cache_legacy_carrier_round_trips() {
for case in lifecycle_cache_oracle_cases() {
let label = case.label;
let old_tuple = case.frozen_tuple;
let old_tuple = case.old_tuple;
let current_map = case.current_map;
let from_old: DataUsageCacheInfo =
rmp_serde::from_slice(old_tuple).expect("the production reader must read the old tuple");
rmp_serde::from_slice(&old_tuple).expect("the production reader must read the old tuple");
let from_old_map = rmp_serde::to_vec(&from_old).expect("the production writer must retain the old tuple payload");
let old_via_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&from_old_map).expect("project the old fields after an old-tuple load");
rmp_serde::from_slice(&from_old_map).expect("the old reader must read the production map after an old-tuple load");
let old_from_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&current_map).expect("project the old fields after a production write");
rmp_serde::from_slice(&current_map).expect("the actual old reader must read the production map");
assert_eq!(
[
rmp_serde::to_vec(&old_via_current).expect("encode the old reader's complete tuple after production reload"),
rmp_serde::to_vec(&old_from_current).expect("encode the old reader's complete tuple after production write"),
],
[old_tuple, old_tuple],
[old_tuple.clone(), old_tuple],
"both cache directions must preserve the full old carrier for {label}"
);
}
@@ -3851,12 +4051,32 @@ fn test_lifecycle_cache_current_map_round_trip() {
#[test]
fn test_lifecycle_cache_strict_cached_tags_contract() {
// The pinned legacy writer produced the same empty map before and after test_tags initialized its cache.
use s3s::dto as legacy;
type LegacyPayload = Option<Arc<legacy::BucketLifecycleConfiguration>>;
let old: LegacyPayload = rmp_serde::from_slice(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD)
.expect("the frozen payload must initialize the real old configuration");
let filter = old.as_ref().expect("nonempty old payload").rules[0]
.filter
.as_ref()
.expect("first old rule has a filter");
let object_tags = [(String::new(), String::new()), ("kind".to_owned(), "archive".to_owned())]
.into_iter()
.collect();
// The real old inherent method initializes its OnceLock even if matching fails.
let _ = filter.test_tags(&object_tags);
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD))
.expect("the production reader must accept the fixed payload");
assert_eq!(
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the cached tags"),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
[
rmp_serde::to_vec(&old).expect("encode the initialized old cache"),
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the new cache")
],
[
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
],
"an initialized old CachedTags must still write the same empty map as the new carrier"
);
let slot = LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD
@@ -3869,14 +4089,22 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
bytes.extend_from_slice(&LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD[slot + 1..]);
bytes
};
// The pinned reader discarded populated String/String maps and rejected the other shapes below.
// A historical String/String map is consumed by both real readers, then
// rewritten empty. The original populated input is deliberately not the oracle.
let populated = input_with_cache(&[0x81, 0xa3, b'e', b'n', b'v', 0xa4, b'p', b'r', b'o', b'd']);
let old: LegacyPayload = rmp_serde::from_slice(&populated).expect("old reader accepts String/String cached tags");
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(populated.as_slice()))
.expect("new reader accepts String/String cached tags");
assert_eq!(
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the populated cache"),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
"populated cached tags are discarded on rewrite",
[
rmp_serde::to_vec(&old).expect("rewrite the old populated cache"),
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the new populated cache")
],
[
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
],
"both readers discard populated cached tags and both writers emit an empty map"
);
for (label, replacement) in [
@@ -3887,8 +4115,13 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
("nonstring-value", &[0x81, 0xa1, b'k', 0x01][..]),
] {
let input = input_with_cache(replacement);
let old = rmp_serde::from_slice::<LegacyPayload>(&input);
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(input.as_slice()));
assert!(new.is_err(), "must reject {label} cached tags");
assert_eq!(
[old.is_err(), new.is_err()],
[true, true],
"both real readers must reject {label} cached tags"
);
}
}
@@ -3900,59 +4133,73 @@ fn object_lock_cache_matches_the_legacy_dto_bytes() {
use rustfs_gateway_types::persistence::{
PersistedDefaultRetention, PersistedObjectLockConfiguration, PersistedObjectLockRule,
};
// Captured from the minio-enabled s3s DTO at 5761ddfe4c509fcac87e3b99fd5bd9e176b442c0.
let cases = [
(b"\x92\xc0\xc0".as_slice(), None, None),
(b"\x92\xa7Enabled\xc0".as_slice(), Some("Enabled"), None),
(
b"\x92\xa8Disabled\x91\xc0".as_slice(),
Some("Disabled"),
Some(PersistedObjectLockRule { default_retention: None }),
),
(
b"\x92\xa7Enabled\x91\x93\x1e\xaaGOVERNANCE\xc0".as_slice(),
Some("Enabled"),
Some(PersistedObjectLockRule {
default_retention: Some(PersistedDefaultRetention {
use s3s::dto as legacy;
let legacy_cases = vec![
legacy::ObjectLockConfiguration {
object_lock_enabled: None,
rule: None,
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: None,
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from("Disabled".to_owned())),
rule: Some(legacy::ObjectLockRule { default_retention: None }),
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: Some(legacy::ObjectLockRule {
default_retention: Some(legacy::DefaultRetention {
days: Some(30),
mode: Some("GOVERNANCE".to_owned()),
mode: Some(legacy::ObjectLockRetentionMode::from_static(legacy::ObjectLockRetentionMode::GOVERNANCE)),
years: None,
}),
}),
),
(
b"\x92\xa7Enabled\x91\x93\xc0\xaecompliance-ish\x02".as_slice(),
Some("Enabled"),
Some(PersistedObjectLockRule {
default_retention: Some(PersistedDefaultRetention {
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: Some(legacy::ObjectLockRule {
default_retention: Some(legacy::DefaultRetention {
days: None,
mode: Some("compliance-ish".to_owned()),
mode: Some(legacy::ObjectLockRetentionMode::from("compliance-ish".to_owned())),
years: Some(2),
}),
}),
),
},
];
for (old_bytes, enabled, rule) in cases {
let current = Some(Arc::new(PersistedObjectLockConfiguration {
object_lock_enabled: enabled.map(str::to_owned),
rule,
}));
for legacy_config in legacy_cases {
let persisted = PersistedObjectLockConfiguration {
object_lock_enabled: legacy_config
.object_lock_enabled
.as_ref()
.map(|value| value.as_str().to_owned()),
rule: legacy_config.rule.as_ref().map(|rule| PersistedObjectLockRule {
default_retention: rule.default_retention.as_ref().map(|retention| PersistedDefaultRetention {
mode: retention.mode.as_ref().map(|value| value.as_str().to_owned()),
days: retention.days,
years: retention.years,
}),
}),
};
let old_bytes = rmp_serde::to_vec(&Some(Arc::new(legacy_config))).expect("encode the legacy DTO");
let current = Some(Arc::new(persisted.clone()));
let new_bytes = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&current)).expect("encode the new cache");
assert_eq!(new_bytes, old_bytes, "must reproduce the original bytes for {current:?}");
let decoded =
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes)).expect("decode the legacy bytes");
assert_eq!(decoded, current);
assert_eq!(new_bytes, old_bytes, "the new writer must produce the legacy bytes for {persisted:?}");
let decoded = crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes.as_slice()))
.expect("decode the legacy bytes");
assert_eq!(decoded.as_deref(), Some(&persisted));
}
let absent = b"\xc0";
assert_eq!(
rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent"),
absent
);
assert!(
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(&absent[..]))
.expect("decode absent")
.is_none()
);
let absent_old = rmp_serde::to_vec(&Option::<Arc<legacy::ObjectLockConfiguration>>::None).expect("encode absent");
let absent_new = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent");
assert_eq!(absent_new, absent_old);
let decoded =
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(absent_old.as_slice())).expect("decode absent");
assert!(decoded.is_none());
}
#[test]
@@ -9,7 +9,7 @@ This page is a pointer, not a route table. The machine-checked matrix in `route_
| Prefix | Behavior | Rule |
|---|---|---|
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_router` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_route` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
| `/minio/admin` | Compatibility alias accepted by `S3Router::is_match`; `canonicalize_admin_path` rewrites it to `/rustfs/admin` immediately before route lookup (`rustfs/src/admin/router.rs`) | Never register routes twice; preserve canonicalization |
| `/iceberg/v1` | Table catalog prefix registered by `register_table_catalog_route` (`rustfs/src/admin/handlers/table_catalog/routes.rs`) and accepted by `is_admin_path` | Stays outside `/rustfs/admin`; table actions are authorized per handler |
| `/health`, `/health/ready` | Public health endpoints, registered only when `ENV_HEALTH_ENDPOINT_ENABLE` allows | Preserve the unauthenticated bypass |
@@ -1,7 +1,7 @@
# MinIO ↔ RustFS Router Compatibility (Exceptions Only)
**Use this when:** a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different.
**Source of truth:** S3 plane: the registered operation families in `rustfs/src/server/gateway_handlers/mod.rs`. Admin plane: `make_admin_router` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
**Source of truth:** S3 plane: the `s3s::S3` trait impl in `rustfs/src/storage/ecfs.rs`. Admin plane: `make_admin_route` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
This document lists only exceptions. Anything not listed here is implemented with MinIO-equivalent behavior. For the s3tests-level claim see [s3-compatibility-matrix.md](s3-compatibility-matrix.md).
@@ -18,7 +18,7 @@ Admin paths are relative to the canonical `/rustfs/admin` prefix; `/minio/admin`
## S3 Data Plane
Run `python3 scripts/s3-tests/api_coverage.py` for the registered gateway operations and the missing operations in the pinned S3 reference. The following table records compatibility exceptions.
All `s3s::S3` trait methods are implemented in `rustfs/src/storage/ecfs.rs` except the following.
| S3 operation | Status | Detail |
|---|---|---|
+1 -1
View File
@@ -97,7 +97,7 @@ Standard S3 areas that must not be described as complete:
| Area | Status | Evidence |
|---|---|---|
| Bucket access logging | Handlers exist (`GetBucketLogging` and `PutBucketLogging` in `rustfs/src/server/gateway_handlers/bucket_core.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
| Bucket access logging | Handlers exist (`get_bucket_logging`, `put_bucket_logging` in `rustfs/src/storage/ecfs.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
| POST Object form upload checksum handling | Not yet passing | `unimplemented_tests.txt` |
| Bucket ownership controls | No handler | `unimplemented_tests.txt` |
| Multipart upload listing and part lookup compatibility edge cases | Not part of default gate; ListParts returns HTTP 503 `SlowDownRead` when no directory was listed and any participant is unavailable, instead of treating one missing directory as an empty upload | `excluded_tests.txt`; `collect_list_parts_results_rejects_unavailable_empty_upload_dirs` |
+1 -1
View File
@@ -1,7 +1,7 @@
# Audit Entries for Authorization Denials
**Use this when:** you consume the audit stream and need to know which authorization denials appear in it, how many, and how to tune the limit.
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/authorization/decision.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/access.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
With audit enabled (`RUSTFS_AUDIT_ENABLE=true`), a request refused by the authorization layer produces an audit entry in addition to the `s3_authorization_denied` log event. The S3 response is unchanged: a bare `Access Denied`.
-17
View File
@@ -1,17 +0,0 @@
# S3 stack upgrade and rollback
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
## Roll back to the previous release
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
## Client-visible error change
| Conformance case | What clients see |
| --- | --- |
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
+1 -1
View File
@@ -13,7 +13,7 @@ Every fixed RustFS GitHub Security Advisory maps to at least one named regressio
| [GHSA-r5qv-rc46-hv8q](https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q) | Internode RPC authentication must fail closed | rustfs/rustfs#4402 | `ghsa_r5qv_resolve_shared_secret_rejects_default_fallback`, `ghsa_r5qv_verify_rpc_signature_fails_closed_on_missing_or_invalid_auth` (`crates/ecstore/src/cluster/rpc/http_auth.rs`) | unit |
| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with the shared root secret (intentionally unfixed) | n/a; tests pin the by-design behaviour and must flip red -> green when m77q is fixed | `test_ghsa_m77q_sts_session_token_signed_with_root_secret`, `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) | unit |
| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope; a non-owner with `CreateServiceAccountAdminAction` could parent a service account to root | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope`, `ghsa_5354_scope_guard_preserves_non_root_self_scope`, and the `add_service_account_parent_within_scope` invariant they pin (`rustfs/src/admin/handlers/service_account.rs`) | unit |
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` in `rustfs/src/storage/authorization/tests/actions.rs` covers `versioned_read_action` in `rustfs/src/storage/authorization_actions.rs` | unit |
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit |
| [GHSA-v9cp-qfw9-9pfp](https://github.com/rustfs/rustfs/security/advisories/GHSA-v9cp-qfw9-9pfp) | `ForAllValues:`/`ForAnyValue:` negated string operators applied negation to the aggregate instead of the per-value predicate | fixed, GHSA private-fork merge | `ghsa_v9cp_for_all_values_not_equals_partial_overlap`, `ghsa_v9cp_for_any_value_not_equals_partial_overlap` and the absent-key/positive-quantifier cases beside them (`crates/policy/tests/quantified_negation.rs`); the value set must partially overlap the policy set, since contained or disjoint sets cannot tell the quantifiers apart | crate test |
| [GHSA-6r96-hmgc-726c](https://github.com/rustfs/rustfs/security/advisories/GHSA-6r96-hmgc-726c) | Request headers must not populate server-derived IAM condition keys (`userid`, `groups`, `jwt:`/`ldap:` claims) | fixed, GHSA private-fork merge | `ghsa_6r96_identity_condition_keys_ignore_spoofed_headers`, `ghsa_6r96_claim_condition_keys_ignore_spoofed_headers`, and `test_request_headers_still_reach_conditions`, which keeps the reserved set from growing too broad (`rustfs/src/auth.rs`) | unit |
| [GHSA-x298-9x87-fvjq](https://github.com/rustfs/rustfs/security/advisories/GHSA-x298-9x87-fvjq) | Anonymous ListObjectVersions -> `s3:ListBucket` fallback must reach the same public-access gates as a direct grant | fixed, GHSA private-fork merge | `ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled` (`crates/e2e_test/src/anonymous_access_test.rs`); asserts 200 before the public-access block is applied so it proves the gate, not a broken fallback | e2e (`e2e-smoke`) |
+38 -12
View File
@@ -1868,8 +1868,6 @@ mod tests {
peer_topology_complete, query_peer_heal_status, query_peer_replacement_recovery_status, read_cluster_heal_status,
reject_heal_admission, validate_heal_request_mode, validate_heal_target, validate_mrf_legacy_responsibility_action,
};
use crate::admin::operation::test_support::TestRequest;
use crate::admin::router::dispatch;
use crate::storage::rpc::node_service::heal::{
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
};
@@ -2155,19 +2153,33 @@ mod tests {
#[tokio::test]
async fn test_nested_heal_routes_still_require_authentication() {
use s3s::route::S3Route;
let mut router = super::S3Router::new(false);
super::register_heal_route(&mut router).expect("register production Heal routes");
for prefix in ["/rustfs/admin", "/minio/admin"] {
for object in ["dir/object.bin", "dir%2Fobject.bin", "literal%252Fslash"] {
let request = TestRequest::new(http::Method::POST, format!("{prefix}/v3/heal/test-bucket/{object}")).build();
let mut req = s3s::S3Request {
input: s3s::Body::empty(),
method: http::Method::POST,
uri: format!("{prefix}/v3/heal/test-bucket/{object}").parse().expect("Heal URI"),
headers: http::HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let err = router
.admit(request.method(), request.uri(), request.headers(), false)
.check_access(&mut req)
.await
.expect_err("router must require a signature");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
let err = router
.call(req)
.await
.expect_err("handler must independently require authentication");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
assert!(err.to_string().contains("authentication required"));
}
}
@@ -2175,20 +2187,34 @@ mod tests {
#[tokio::test]
async fn replacement_recovery_diagnostics_require_authentication() {
use s3s::route::S3Route;
let mut router = super::S3Router::new(false);
super::register_heal_route(&mut router).expect("register production Heal routes");
for prefix in ["/rustfs/admin", "/minio/admin"] {
let path = format!("{prefix}/v4/heal/replacement-recovery");
assert!(router.contains_compatible_route(http::Method::GET, &path));
let request = TestRequest::new(http::Method::GET, path).build();
let mut req = s3s::S3Request {
input: s3s::Body::empty(),
method: http::Method::GET,
uri: path.parse().expect("replacement diagnostics URI"),
headers: http::HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let err = router
.admit(request.method(), request.uri(), request.headers(), false)
.check_access(&mut req)
.await
.expect_err("diagnostics router must require a signature");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
let err = router
.call(req)
.await
.expect_err("diagnostics handler must authenticate before inspecting local records");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
assert!(err.to_string().contains("authentication required"));
}
}
+51 -19
View File
@@ -90,47 +90,79 @@ impl Operation for HealthCheckHandler {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::operation::test_support::TestRequest;
use crate::admin::router::dispatch;
use http_body_util::BodyExt as _;
use s3s::route::S3Route as _;
use serial_test::serial;
use temp_env::with_var;
async fn call_real_health_handler(method: Method) -> (StatusCode, HeaderMap, Body) {
fn health_request(method: Method) -> s3s::S3Request<s3s::Body> {
s3s::S3Request {
input: s3s::Body::empty(),
method,
uri: HEALTH_PREFIX.parse().expect("health URI"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
async fn call_real_health_handler(method: Method) -> s3s::S3Response<s3s::Body> {
let mut router = S3Router::new(false);
router
.insert(method.clone(), HEALTH_PREFIX, AdminOperation(&HealthCheckHandler {}))
.expect("health route should register");
let request = TestRequest::new(method, HEALTH_PREFIX).build();
dispatch(&router, request.into_envelope(), Body::empty())
router
.call(health_request(method))
.await
.expect("health handler should return a response")
.into_parts()
}
#[tokio::test]
async fn real_health_handler_get_and_head_preserve_wire_branches() {
let (status, headers, body) = call_real_health_handler(Method::GET).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
let payload: serde_json::Value = serde_json::from_slice(&body.collect().await.expect("GET body should read").to_bytes())
.expect("GET health response should be JSON");
let get = call_real_health_handler(Method::GET).await;
assert_eq!(get.status, Some(StatusCode::OK));
assert_eq!(
get.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
Some("application/json")
);
let payload: serde_json::Value =
serde_json::from_slice(&get.output.collect().await.expect("GET body should read").to_bytes())
.expect("GET health response should be JSON");
assert_eq!(payload["status"], "ok");
let (status, headers, body) = call_real_health_handler(Method::HEAD).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
assert!(body.collect().await.expect("HEAD body should read").to_bytes().is_empty());
let head = call_real_health_handler(Method::HEAD).await;
assert_eq!(head.status, Some(StatusCode::OK));
assert_eq!(
head.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
Some("application/json")
);
assert!(
head.output
.collect()
.await
.expect("HEAD body should read")
.to_bytes()
.is_empty()
);
}
#[tokio::test]
async fn real_health_handler_rejects_unsupported_methods_with_allow_header() {
let (status, headers, body) = call_real_health_handler(Method::POST).await;
let response = call_real_health_handler(Method::POST).await;
assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED);
assert_eq!(headers.get(http::header::ALLOW).and_then(|value| value.to_str().ok()), Some("GET, HEAD"));
assert_eq!(response.status, Some(StatusCode::METHOD_NOT_ALLOWED));
assert_eq!(
body.collect().await.expect("error body should read").to_bytes(),
response
.headers
.get(http::header::ALLOW)
.and_then(|value| value.to_str().ok()),
Some("GET, HEAD")
);
assert_eq!(
response.output.collect().await.expect("error body should read").to_bytes(),
bytes::Bytes::from_static(b"Method Not Allowed")
);
}
+9 -9
View File
@@ -332,7 +332,7 @@ fn is_lock_timeout(err: &crate::storage_api::error::StorageError) -> bool {
fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError {
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) {
return ApiError {
code: S3ErrorCode::InternalError,
code: S3ErrorCode::InternalError.into(),
message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(),
source: Some(Box::new(err)),
};
@@ -341,7 +341,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// movement gate has a queued writer: the same transient contention.
if is_lock_timeout(&err) || matches!(err, crate::storage_api::error::StorageError::SlowDown) {
return ApiError {
code: S3ErrorCode::ServiceUnavailable,
code: S3ErrorCode::ServiceUnavailable.into(),
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -350,7 +350,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// should say so, not just "Invalid Request".
if matches!(err, crate::storage_api::error::StorageError::DecommissionAlreadyRunning) {
return ApiError {
code: S3ErrorCode::InvalidRequest,
code: S3ErrorCode::InvalidRequest.into(),
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -1261,14 +1261,14 @@ mod pools_handler_tests {
#[test]
fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
let err = crate::error::ApiError {
code: rustfs_s3_types::S3ErrorCode::InvalidRequest,
code: rustfs_s3_types::S3ErrorCode::InvalidRequest.into(),
message: "decommission already running".to_string(),
source: None,
};
let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]");
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest);
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest.into());
assert_eq!(
err.message,
"admin start decommission failed for pools [1, 3]: decommission already running"
@@ -1281,12 +1281,12 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
assert!(err.source.is_some());
let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed"));
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(unrelated.message, "We encountered an internal error, please try again.");
}
@@ -1299,7 +1299,7 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
assert!(err.message.contains("data movement gate (decommission activation)"), "{}", err.message);
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(
@@ -1313,7 +1313,7 @@ mod pools_handler_tests {
fn test_decommission_start_api_error_reports_a_refused_preflight_heal_as_retryable() {
let err = decommission_start_api_error(crate::storage_api::error::StorageError::SlowDown);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
}
+4 -5
View File
@@ -926,11 +926,10 @@ impl Operation for RebalanceStatus {
"admin rebalance state"
);
// Preserve the idle 404 contract without triggering the custom-route error logger.
let body = format!(
r#"<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchResource</Code><Message>{REBALANCE_NOT_STARTED_MESSAGE}</Message></Error>"#
);
let headers = HeaderMap::from_iter([(CONTENT_TYPE, HeaderValue::from_static("application/xml"))]);
return Ok(AdminResponse::with_headers(StatusCode::NOT_FOUND, Body::from(body), headers));
let (parts, body) = s3s::S3Error::with_message(s3s::S3ErrorCode::NoSuchResource, REBALANCE_NOT_STARTED_MESSAGE)
.to_http_response()?
.into_parts();
return Ok(AdminResponse::with_headers(parts.status, body.into(), parts.headers));
}
error!(
+25 -49
View File
@@ -48,6 +48,7 @@ use rustfs_policy::{
};
use rustfs_s3_types::{Body, S3Error, S3ErrorCode, S3Result, s3_error};
use rustfs_utils::MaskedAccessKey;
use s3s::dto::{AssumeRoleOutput, Credentials, Timestamp};
use serde::Deserialize;
use serde_json::Value;
use serde_urlencoded::from_bytes;
@@ -336,35 +337,33 @@ async fn handle_assume_role(
warn!("site replication STS hook failed, err: {err}");
}
let expiration = new_cred
.expiration
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600)));
let output = assume_role_response_xml(&new_cred, expiration)?;
let resp = AssumeRoleOutput {
credentials: Some(Credentials {
access_key_id: new_cred.access_key,
expiration: Timestamp::from(
new_cred
.expiration
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600))),
),
secret_access_key: new_cred.secret_key,
session_token: new_cred.session_token,
}),
..Default::default()
};
// getAssumeRoleCredentials
let output = serialize_assume_role_output(&resp)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize assume role output failed: {e}")))?;
Ok(AdminResponse::new(StatusCode::OK, Body::from(output)))
}
fn assume_role_response_xml(credentials: &rustfs_credentials::Credentials, expiration: OffsetDateTime) -> S3Result<String> {
// The STS codec writes local clock fields with millisecond precision and a four-digit year.
if !(0..=9999).contains(&expiration.year()) {
return Err(s3_error!(InternalError, "cannot serialize STS expiration year"));
}
let expiration = expiration
.format(time::macros::format_description!(
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
))
.map_err(|_| s3_error!(InternalError, "cannot serialize STS expiration"))?;
Ok(format!(
concat!(
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
"<AccessKeyId>{}</AccessKeyId><Expiration>{}</Expiration>",
"<SecretAccessKey>{}</SecretAccessKey><SessionToken>{}</SessionToken>",
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
),
xml_escape(&credentials.access_key),
expiration,
xml_escape(&credentials.secret_key),
xml_escape(&credentials.session_token),
))
/// The STS answer is an s3s response body, so the s3s encoder writes it; it
/// is not a bucket configuration and never goes through the bucket-config codec.
fn serialize_assume_role_output(output: &AssumeRoleOutput) -> s3s::xml::SerResult<Vec<u8>> {
let mut buf = Vec::with_capacity(256);
s3s::xml::Serialize::serialize(output, &mut s3s::xml::Serializer::new(&mut buf))?;
Ok(buf)
}
/// Session claim marking a session that presented a second factor.
@@ -580,29 +579,6 @@ mod tests {
}
}
#[test]
fn assume_role_xml_matches_the_pinned_sts_response() {
let credentials = rustfs_credentials::Credentials {
access_key: "access<&".to_owned(),
secret_key: "synthetic>'".to_owned(),
session_token: "quoted\"token".to_owned(),
..Default::default()
};
let expiration = time::macros::datetime!(2024-02-29 11:22:33.123456789 +05:30);
let answer = assume_role_response_xml(&credentials, expiration).expect("valid timestamp");
// Captured from the pinned 5761ddfe STS writer, including field order and escaping.
assert_eq!(
answer,
concat!(
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
"<AccessKeyId>access&lt;&amp;</AccessKeyId><Expiration>2024-02-29T11:22:33.123Z</Expiration>",
"<SecretAccessKey>synthetic&gt;&apos;</SecretAccessKey><SessionToken>quoted&quot;token</SessionToken>",
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
)
);
assert!(assume_role_response_xml(&credentials, time::macros::datetime!(-1-01-01 00:00 UTC)).is_err());
}
#[test]
fn test_xml_escape() {
assert_eq!(xml_escape("hello"), "hello");
+17 -1
View File
@@ -45,8 +45,24 @@ use handlers::{
user,
};
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
/// Builds the control-plane router and binds it to this server.
/// Create admin router
///
/// # Arguments
/// * `console_enabled` - Whether the console is enabled
///
/// # Returns
/// An instance of S3Route for admin operations
pub fn make_admin_route(
console_enabled: bool,
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
) -> std::io::Result<impl S3Route> {
make_admin_router(console_enabled, server_ctx)
}
/// The admin router with every route registered, bound to `server_ctx`: what the legacy stack
/// mounts as its custom route and the gateway stack's control-plane host dispatches through.
pub(crate) fn make_admin_router(
console_enabled: bool,
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
+331 -232
View File
File diff suppressed because it is too large Load Diff
@@ -77,7 +77,7 @@ fn uri_for(template: &str, bucket: &str) -> String {
async fn send(router: &S3Router<AdminOperation>, mut req: S3Request<Body>) -> S3Result<S3Response<Body>> {
router.check_access(&mut req).await?;
router.call(req).await.map_err(Into::into)
router.call(req).await
}
fn error_code(result: S3Result<S3Response<Body>>, context: &str) -> S3ErrorCode {
@@ -222,7 +222,7 @@ async fn extension_routes_refuse_anonymous_callers_before_dispatch() {
.check_access(&mut req)
.await
.expect_err("an anonymous extension request is refused");
assert_eq!(S3ErrorCode::from(error.code().clone()), S3ErrorCode::AccessDenied, "{method} {uri}");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{method} {uri}");
}
}
+1 -1
View File
@@ -37,7 +37,7 @@ use crate::server::collect_dependency_readiness_report as collect_runtime_depend
use crate::shared_types::DependencyReadiness;
use rustfs_data_usage::DataUsageInfo;
use rustfs_madmin::{InfoMessage, StorageInfo};
use rustfs_s3_types::S3ErrorCode;
use s3s::S3ErrorCode;
use std::collections::HashSet;
use std::sync::Arc;
use tracing::{debug, error, info, warn};
-76
View File
@@ -1,76 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Operations without a notification event must emit one use-case audit entry.
use crate::guard_sources::SourceSet;
use rustfs_s3_ops::S3Operation;
fn is_read_only(op: S3Operation) -> bool {
let name = op.as_str().trim_start_matches("s3:");
["Get", "Head", "List", "Select"]
.iter()
.any(|prefix| name.starts_with(prefix))
}
fn audit_sites(use_cases: &SourceSet, op: S3Operation) -> usize {
let helper = format!("OperationHelper::audit_only_from_view(&envelope.audit_view(), S3Operation::{op:?}");
use_cases.count_ignoring_whitespace(&format!("audit_only_operation(envelope, S3Operation::{op:?},"))
+ use_cases.count_ignoring_whitespace(&format!("{helper})"))
+ use_cases.count_ignoring_whitespace(&format!("{helper},)"))
}
#[test]
fn mutating_operations_without_a_notification_event_are_audited_once() {
let mut expected: Vec<S3Operation> = S3Operation::ALL
.into_iter()
.filter(|op| !is_read_only(*op) && op.to_event_name().is_none())
.collect();
// AbortMultipartUpload has an internal event but no handler-built helper.
expected.push(S3Operation::AbortMultipartUpload);
assert_eq!(expected.len(), 26, "an operation was added or removed: {expected:?}");
let use_cases = SourceSet::production("src/app/mod.rs", "src/app");
for op in expected {
assert_eq!(audit_sites(&use_cases, op), 1, "expected one audit site for {op:?}");
}
}
const OWNED_HELPER: &str =
"let helper = OperationHelper::audit_only_from_view(\n &envelope.audit_view(),\n S3Operation::UploadPart,\n);";
const OWNED_WRAPPER: &str =
"RequestEnvelope::audit_only_operation(\n envelope,\n S3Operation::UploadPart,\n StatusCode::OK,\n run,\n)";
fn count_sites(source: &str) -> usize {
let root = tempfile::tempdir().unwrap();
let app = root.path().join("src/app");
std::fs::create_dir_all(&app).unwrap();
std::fs::write(app.join("mod.rs"), source).unwrap();
let sources = SourceSet::production_under(root.path(), "src/app/mod.rs", "src/app");
audit_sites(&sources, S3Operation::UploadPart)
}
#[test]
fn the_guard_accepts_exactly_one_audit_place() {
assert_eq!(count_sites(OWNED_HELPER), 1);
assert_eq!(count_sites(OWNED_WRAPPER), 1);
}
#[test]
fn the_guard_refuses_an_operation_audited_nowhere_or_twice() {
assert_eq!(count_sites(""), 0);
assert_eq!(count_sites(&format!("{OWNED_HELPER}\n{OWNED_WRAPPER}")), 2);
assert_eq!(count_sites(&format!("#[cfg(test)]\nmod tests {{\n{OWNED_HELPER}\n}}")), 0);
assert_eq!(count_sites(&OWNED_WRAPPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
assert_eq!(count_sites(&OWNED_HELPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
}
+20 -31
View File
@@ -472,7 +472,7 @@ mod tests {
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
use crate::app::storage_api::bucket_usecase::s3::{
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response,
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3, S3Request, S3Response,
XmlSerialize, XmlSerializer,
};
use crate::app::storage_api::test::StoragePutObjReader;
@@ -1185,11 +1185,10 @@ mod tests {
service: None,
trailing_headers: None,
};
let get_env = crate::app::RequestEnvelope::from_s3s(&get);
let authorized_generation =
super::super::storage_api::bucket_usecase::access::load_bucket_generation_from_store(
&store,
&get_env,
&get,
&input.bucket,
)
.await
@@ -1215,16 +1214,12 @@ mod tests {
assert_ne!(replacement, old.incarnation_id());
sys.remove(&input.bucket);
let mut without_source = get.clone();
let mut without_source_env = crate::app::RequestEnvelope::from_s3s(&without_source);
super::super::storage_api::bucket_usecase::access::prepare_odm_read_generation(
&store,
&mut without_source_env,
&mut without_source,
&input.bucket,
)
.await;
if let Some(guard) = without_source_env.odm_read_generation().cloned() {
without_source.extensions.insert(guard);
}
for state_incarnation in [old.incarnation_id(), replacement] {
sys.apply_for_incarnation(&input.bucket, state_incarnation, Some(old.config()))
.await;
@@ -1320,34 +1315,28 @@ mod tests {
};
for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
let request_marker = input.marker.clone().unwrap_or_default();
let list_req = S3Request {
input: input.clone(),
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let envelope = crate::app::RequestEnvelope::from_s3s(&list_req);
let gateway_input =
rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::input_from_s3s(
list_req.input,
)
.expect("convert ListObjectsInput");
let gateway_output = tokio::time::timeout(
let response = tokio::time::timeout(
Duration::from_secs(10),
DefaultBucketUsecase::from_global().execute_list_objects(envelope, gateway_input),
S3::list_objects(
&crate::storage::ecfs::FS::new(),
S3Request {
input: input.clone(),
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
},
),
)
.await
.expect("v1 pagination must finish")
.expect("list-through must not change v1 listing");
let output = rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::output_to_s3s(
gateway_output,
)
.expect("convert ListObjectsOutput");
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
let output = response.output;
let contents = output.contents.as_ref().expect("local page contents");
assert_eq!(contents.len(), 1);
assert_eq!(contents[0].key.as_deref(), Some(expected_key));
+4 -6
View File
@@ -98,13 +98,14 @@ use rustfs_policy::policy::{
{BucketPolicy, BucketPolicyArgs, Effect},
};
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::{Region, S3Error, S3ErrorCode, S3Result};
use rustfs_s3_types::Region;
use rustfs_targets::{
EventName,
arn::{ARN, TargetID, TargetIDError},
};
use rustfs_utils::http::get_header;
use rustfs_utils::obj::extract_user_defined_metadata;
use s3s::{S3Error, S3ErrorCode, S3Result};
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
@@ -906,9 +907,7 @@ mod tests {
.expect_err("full queue rejects immediately");
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
let response = s3s::S3Error::from(error)
.to_http_response()
.expect("serialize admission error");
let response = error.to_http_response().expect("serialize admission error");
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
use http_body_util::BodyExt as _;
@@ -1001,8 +1000,7 @@ mod tests {
},
);
let envelope = RequestEnvelope::from_s3s(&req);
assert!(envelope.req_info().expect("req info should be present").is_owner);
assert!(req_info_ref(&req).expect("req info should be present").is_owner);
}
#[tokio::test]
-11
View File
@@ -67,17 +67,6 @@ impl DefaultBucketUsecase {
mod tests {
use super::*;
#[tokio::test]
async fn execute_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultBucketUsecase::without_context();
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
let error = usecase
.execute_put_bucket_acl(envelope, "test-bucket", false)
.await
.unwrap_err();
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
let input = gateway_dto::GetBucketAclInput {
+1 -72
View File
@@ -116,7 +116,7 @@ impl DefaultBucketUsecase {
async fn list_objects_page(&self, envelope: &mut RequestEnvelope, request: &PageRequest) -> S3Result<Page> {
let PageRequest { bucket, params, .. } = request;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
let store = get_validated_store(bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED).as_deref() == Some("true");
let table_list_access = envelope.table_list_access().cloned();
@@ -375,77 +375,6 @@ mod tests {
use crate::app::bucket_usecase::test_support::build_request;
use rustfs_gateway_types::BucketName;
#[test]
#[serial_test::serial]
fn list_apis_report_storage_read_quorum_as_unavailable() {
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore};
run_large_stack_test("list-quorum-surface", || async {
let usecase = DefaultBucketUsecase::with_context(Some(shared_gating_ambient().await));
let store = shared_gating_ecstore().await;
let bucket = BucketName::new(format!("list-quorum-{}", uuid::Uuid::new_v4().simple())).unwrap();
store
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
.await
.unwrap();
let set = &store.pools[0].disk_set[0];
let saved_disks = {
let mut disks = set.disks.write().await;
let saved = disks.clone();
disks.iter_mut().skip(1).for_each(|disk| *disk = None);
saved
};
let envelope = || owned_envelope(http::Method::GET);
let results = [
usecase
.execute_list_objects(
envelope(),
gateway_dto::ListObjectsInput {
bucket: bucket.clone(),
..Default::default()
},
)
.await
.map(drop),
usecase
.execute_list_objects_v2(
envelope(),
gateway_dto::ListObjectsV2Input {
bucket: bucket.clone(),
..Default::default()
},
)
.await
.map(drop),
usecase
.execute_list_object_versions(
envelope(),
gateway_dto::ListObjectVersionsInput {
bucket,
..Default::default()
},
)
.await
.map(drop),
];
*set.disks.write().await = saved_disks;
for result in results {
let error = result.expect_err("cannot list below storage read quorum");
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
let source = error.source().expect("retain storage diagnostics");
assert!(
matches!(
source.downcast_ref::<StorageError>(),
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
),
"{source}"
);
}
});
}
fn object(name: &str) -> ObjectInfo {
ObjectInfo {
name: name.to_owned(),
+1 -1
View File
@@ -626,7 +626,7 @@ impl DefaultBucketUsecase {
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
let store = get_validated_store(&bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
@@ -23,7 +23,8 @@ use crate::app::storage_api::s3::dto::{
FilterRule, LambdaFunctionConfiguration, NotificationConfiguration, NotificationConfigurationFilter,
PutBucketNotificationConfigurationInput, QueueConfiguration, S3KeyFilter, TopicConfiguration,
};
use crate::app::storage_api::s3::{S3ErrorCode, S3Request};
use crate::app::storage_api::s3::{S3, S3Access, S3ErrorCode, S3Request};
use crate::app::storage_api::test::ecfs::FS;
use crate::runtime_sources::{NotifyInterface, ServerContextSlot};
use http::Method;
use rustfs_notify::{EventArgs, NotificationError};
@@ -214,76 +215,6 @@ fn notification_request(
request
}
fn notification_filters(filter: Option<NotificationConfigurationFilter>) -> Vec<(Option<String>, Option<String>)> {
filter
.and_then(|f| f.key)
.and_then(|k| k.filter_rules)
.unwrap_or_default()
.into_iter()
.map(|rule| (rule.name.map(|n| n.as_str().to_owned()), rule.value))
.collect()
}
fn notification_destinations(configuration: NotificationConfiguration) -> Vec<NotificationDestination> {
let mut destinations = Vec::new();
for cfg in configuration.queue_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "QueueConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.queue_arn,
filters: notification_filters(cfg.filter),
});
}
for cfg in configuration.topic_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "TopicConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.topic_arn,
filters: notification_filters(cfg.filter),
});
}
for cfg in configuration.lambda_function_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "LambdaFunctionConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.lambda_function_arn,
filters: notification_filters(cfg.filter),
});
}
destinations
}
async fn authorize_notification_put(
slot: &Arc<ServerContextSlot>,
envelope: &mut RequestEnvelope,
bucket: &str,
) -> rustfs_s3_types::S3Result<()> {
use rustfs_policy::policy::action::{Action, S3Action};
crate::app::storage_api::owned_parity::authorize_bucket_config_mutation(
slot,
envelope,
bucket,
Action::S3Action(S3Action::PutBucketNotificationAction),
)
.await
}
async fn put_notification(
slot: &Arc<ServerContextSlot>,
mut envelope: RequestEnvelope,
input: PutBucketNotificationConfigurationInput,
) -> rustfs_s3_types::S3Result<gateway_dto::PutBucketNotificationConfigurationOutput> {
authorize_notification_put(slot, &mut envelope, &input.bucket).await?;
let prepared = crate::storage::persisted_from_s3s(&input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
let destinations = notification_destinations(input.notification_configuration);
DefaultBucketUsecase::with_context(slot.app_context())
.execute_put_bucket_notification_configuration(envelope, &input.bucket, destinations, prepared)
.await
}
fn check_notification_write_order() {
run_large_stack_test("notification-write-order", || async {
let ambient = shared_gating_ambient().await;
@@ -319,10 +250,13 @@ fn check_notification_write_order() {
.map_or_else(|| "eu-west-2".to_owned(), |region| region.to_string());
let slot = ServerContextSlot::new();
assert!(slot.install(context.clone()));
let fs = FS::with_server_ctx(slot.clone());
let audit = MutationCompletionProbe::install(&bucket);
let request = notification_request(&bucket, configuration(), &slot);
let envelope = RequestEnvelope::from_s3s(&request);
let result = observe_reload(&bucket, calls.clone(), put_notification(&slot, envelope, request.input)).await;
let mut request = notification_request(&bucket, configuration(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut request)
.await
.unwrap();
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
let expected = expected_rules();
let mut timeline = vec![Observation::Rules(
Callback::Validate,
@@ -354,7 +288,7 @@ fn check_notification_write_order() {
};
if let Some(message) = error_message {
let error = result.expect_err("injected callback failure");
assert_eq!(error.code().as_str(), S3ErrorCode::InternalError.as_str());
assert_eq!(error.code(), &S3ErrorCode::InternalError);
assert_eq!(error.message(), Some(message));
} else {
result.expect("configuration written");
@@ -373,13 +307,15 @@ fn check_notification_write_order() {
assert_eq!(*recorder.active_rules.lock().unwrap(), expected);
let mut invalid = configuration();
invalid.lambda_function_configurations.as_mut().unwrap()[0].lambda_function_arn = "invalid-arn".into();
let invalid = notification_request(&bucket, invalid, &slot);
let invalid_env = RequestEnvelope::from_s3s(&invalid);
let mut invalid = notification_request(&bucket, invalid, &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut invalid)
.await
.unwrap();
calls.lock().unwrap().clear();
let err = observe_reload(&bucket, calls.clone(), put_notification(&slot, invalid_env, invalid.input))
let err = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, invalid))
.await
.expect_err("the last destination has an invalid ARN");
assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str());
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert!(
err.message()
.unwrap()
@@ -401,9 +337,11 @@ fn check_notification_write_order() {
assert_eq!(output.queue_configurations[0].events[1].as_str(), "s3:Future:Unrecognized");
calls.lock().unwrap().clear();
let empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
let empty_env = RequestEnvelope::from_s3s(&empty);
observe_reload(&bucket, calls.clone(), put_notification(&slot, empty_env, empty.input))
let mut empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut empty)
.await
.unwrap();
observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, empty))
.await
.unwrap();
assert_eq!(
@@ -479,14 +417,16 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let request = notification_request(&bucket, configuration(), &slot);
let mut envelope = RequestEnvelope::from_s3s(&request);
authorize_notification_put(&slot, &mut envelope, &bucket).await.unwrap();
assert_eq!(bucket_config_mutation_incarnation(&envelope, &bucket).unwrap(), Some(first));
let fs = FS::with_server_ctx(slot.clone());
let mut request = notification_request(&bucket, configuration(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut request)
.await
.unwrap();
assert_eq!(bucket_config_mutation_incarnation(&request, &bucket).unwrap(), Some(first));
let mismatch = DefaultBucketUsecase::without_context()
.execute_put_bucket_notification_configuration(
envelope.fork_for_authorization(),
RequestEnvelope::from_s3s(&request),
"other-bucket",
vec![NotificationDestination {
kind: "QueueConfiguration",
@@ -509,22 +449,9 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
.await
.unwrap();
let prepared =
crate::storage::persisted_from_s3s(&request.input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
let destinations = notification_destinations(request.input.notification_configuration);
let result = observe_reload(
&bucket,
calls.clone(),
DefaultBucketUsecase::with_context(slot.app_context()).execute_put_bucket_notification_configuration(
envelope,
&bucket,
destinations,
prepared,
),
)
.await;
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
assert_eq!(read_notification_xml(&bucket).await, OLD_XML);
assert_eq!(result.unwrap_err().code().as_str(), S3ErrorCode::NoSuchBucket.as_str());
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::NoSuchBucket);
assert!(
matches!(calls.lock().unwrap().as_slice(), [Observation::Rules(Callback::Validate, _, _, xml)] if xml == OLD_XML)
);
@@ -633,24 +633,6 @@ mod tests {
assert!(err.message().is_some_and(|message| message.contains("peer reload failed")));
}
#[tokio::test]
async fn execute_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultBucketUsecase::without_context();
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
let error = usecase
.execute_put_object_lock_configuration(
envelope,
"test-bucket",
Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".into()),
rule: None,
}),
)
.await
.unwrap_err();
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_get_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
let input = gateway_dto::GetObjectLockConfigurationInput {
+27 -12
View File
@@ -243,19 +243,27 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::build_request;
use crate::app::bucket_usecase::test_support::{build_request, s3_op_total};
use crate::app::storage_api::s3::S3ErrorCode;
use crate::app::storage_api::s3::dto::{DeleteBucketPolicyInput, PutBucketPolicyInput};
use http::Method;
#[tokio::test]
async fn execute_delete_bucket_policy_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::DELETE);
let usecase = DefaultBucketUsecase::without_context();
async fn execute_delete_bucket_policy_records_s3_operation_before_store_lookup() {
let input = DeleteBucketPolicyInput::builder()
.bucket("test-bucket".to_string())
.build()
.unwrap();
let err = usecase
.execute_delete_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket")
let req = build_request(input, Method::DELETE);
let fs = crate::storage::ecfs::FS::new();
let before = s3_op_total(S3Operation::DeleteBucketPolicy);
let err = crate::app::storage_api::s3::S3::delete_bucket_policy(&fs, req)
.await
.unwrap_err();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::DeleteBucketPolicy), before + 1);
}
fn bucket(name: &str) -> rustfs_gateway_types::BucketName {
@@ -292,13 +300,20 @@ mod tests {
#[tokio::test]
async fn execute_put_bucket_policy_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let input = PutBucketPolicyInput::builder()
.bucket("test-bucket".to_string())
.policy("{}".to_string())
.build()
.unwrap();
let err = usecase
.execute_put_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket", "{}")
let req = build_request(input, Method::PUT);
let fs = crate::storage::ecfs::FS::new();
let before = s3_op_total(S3Operation::PutBucketPolicy);
let err = crate::app::storage_api::s3::S3::put_bucket_policy(&fs, req)
.await
.unwrap_err();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::PutBucketPolicy), before + 1);
}
}
+4 -10
View File
@@ -211,17 +211,11 @@ mod tests {
let mut req = build_request(input, Method::PUT);
req.extensions.insert(ReqInfo::default());
req.extensions.insert(slot.clone());
let mut envelope = RequestEnvelope::from_s3s(&req);
authorize_bucket_config_mutation(
&slot,
&mut envelope,
&bucket,
Action::S3Action(S3Action::PutBucketTaggingAction),
)
.await
.expect("permission allows tag updates");
authorize_bucket_config_mutation(&slot, &mut req, &bucket, Action::S3Action(S3Action::PutBucketTaggingAction))
.await
.expect("permission allows tag updates");
let result = usecase
.execute_put_bucket_tagging(envelope, &req.input.bucket, persisted)
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, persisted)
.await;
if valid {
result.expect("valid tags are persisted");
+29
View File
@@ -461,6 +461,35 @@ impl RequestEnvelope {
pub fn residual_extensions(&self) -> &Extensions {
&self.residual_extensions
}
/// The payloads the connection, request and signature layers attached, back in the extensions
/// they arrive in, for an edge that hands this request to a handler still taking those.
pub(crate) fn transport_extensions(&self) -> Extensions {
let mut extensions = self.residual_extensions.clone();
extensions.insert(self.remote_addr.map(RemoteAddr));
if let Some(client_info) = &self.client_info {
extensions.insert(client_info.clone());
}
if let Some(native_tls) = self.native_tls_transport {
extensions.insert(NativeTlsTransport(native_tls));
}
if let Some(request_context) = &self.request_context {
extensions.insert(request_context.clone());
}
if let Some(server_context) = &self.server_context {
extensions.insert(Arc::clone(server_context));
}
if let Some(body_read_control) = &self.body_read_control {
extensions.insert(body_read_control.clone());
}
if let Some(verified) = self.verified_sigv4 {
extensions.insert(verified);
}
if let Some(verified) = self.verified_presigned {
extensions.insert(verified);
}
extensions
}
}
#[cfg_attr(
@@ -28,6 +28,7 @@ use super::storage_api::test::contract::{
namespace::NamespaceLocking as _,
object::{ObjectIO as _, ObjectOperations as _},
};
use super::storage_api::test::ecfs::FS;
use super::storage_api::test::object_utils::to_s3s_etag;
use super::storage_api::test::runtime::{
MockWarmBackend, MockWarmOp, get_global_transition_state, register_mock_tier as register_mock_tier_util,
@@ -16,6 +16,7 @@
use super::common::*;
use super::*;
use crate::app::storage_api::s3::S3 as _;
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
@@ -161,7 +162,8 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
.unwrap(),
Method::PUT,
);
put_lifecycle(req)
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("Failed to update lifecycle configuration for delete marker cleanup");
@@ -197,7 +199,10 @@ async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
Method::PUT,
);
put_lifecycle(req).await.expect("Failed to update lifecycle configuration");
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("Failed to update lifecycle configuration");
assert!(
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
@@ -226,7 +231,8 @@ async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_obje
Method::PUT,
);
put_lifecycle(req)
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("ExpiredObjectDeleteMarker should be accepted on object-lock bucket");
}
@@ -252,9 +258,11 @@ async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_obj
Method::PUT,
);
let err = put_lifecycle(req)
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket");
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket")
.into();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -286,9 +294,11 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
Method::PUT,
);
let err = put_lifecycle(req)
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket");
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket")
.into();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -341,9 +351,11 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
Method::PUT,
);
let err = put_lifecycle(req)
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect_err("zero-day expiration must be rejected with InvalidArgument");
.expect_err("zero-day expiration must be rejected with InvalidArgument")
.into();
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -459,6 +471,7 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
),
];
let fs = FS::new();
let handled_count = || {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
@@ -498,8 +511,8 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
.extensions
.insert(parity_test_support::bucket_config_mutation(&readable));
}
let error = put_lifecycle(request).await.unwrap_err();
assert_eq!(error.code().as_str(), code.as_str(), "{name}: {error}");
let error = fs.put_bucket_lifecycle_configuration(request).await.unwrap_err();
assert_eq!(error.code(), &code, "{name}: {error}");
if let Some(message) = message {
assert_eq!(error.message(), Some(message), "{name}");
}
@@ -520,7 +533,9 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
let before = handled_count();
let probe = MutationCompletionProbe::install(&readable);
put_lifecycle(request(&readable, Some(("Disabled", false)))).await.unwrap();
fs.put_bucket_lifecycle_configuration(request(&readable, Some(("Disabled", false))))
.await
.unwrap();
assert_audit(&probe);
assert_eq!(handled_count() - before, 1, "success counts once");
@@ -532,26 +547,3 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
assert_eq!(rules[0].expiration.as_ref().unwrap().date.as_deref(), Some("2099-01-01T00:00:00.000Z"));
assert_eq!(rules[1].expiration.as_ref().unwrap().days, Some(30));
}
async fn put_lifecycle(req: S3Request<PutBucketLifecycleConfigurationInput>) -> rustfs_s3_types::S3Result<()> {
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::PutBucketLifecycleConfiguration);
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
let prepared = match req.input.lifecycle_configuration {
None => crate::app::bucket_usecase::PreparedLifecycleConfiguration::Missing,
Some(cfg) => {
if crate::app::bucket_usecase::validate_lifecycle_rule_status(cfg.rules.iter().map(|rule| rule.status.as_str()))
.is_err()
{
crate::app::bucket_usecase::PreparedLifecycleConfiguration::InvalidStatus
} else {
crate::app::bucket_usecase::PreparedLifecycleConfiguration::Prepared(
crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s(cfg),
)
}
}
};
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_lifecycle_configuration(envelope, &req.input.bucket, prepared)
.await
.map(|_| ())
}
@@ -28,10 +28,11 @@ use super::*;
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn object_lock_handlers_schedule_replication() {
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
use crate::app::storage_api::s3::S3;
use std::sync::atomic::Ordering;
let (_disk_paths, ecstore) = setup_test_env().await;
let object_meta = crate::app::object_meta::ObjectMetaUsecase::with_context(None);
let fs = FS::new();
let bucket = format!("test-lock-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "locked/object.txt";
@@ -51,7 +52,7 @@ async fn object_lock_handlers_schedule_replication() {
// PutObjectRetention must compute a replication decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let _retention_input = PutObjectRetentionInput::builder()
let retention_input = PutObjectRetentionInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.retention(Some(ObjectLockRetention {
@@ -60,18 +61,7 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
let req = build_request((), Method::PUT);
object_meta
.put_object_retention_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLockRetention {
mode: Some(ObjectLockRetentionMode::COMPLIANCE.to_owned()),
retain_until: Some(time::macros::datetime!(2030-01-01 00:00:00 UTC)),
}),
)
fs.put_object_retention(build_request(retention_input, Method::PUT))
.await
.expect("put_object_retention should succeed");
assert_eq!(
@@ -83,7 +73,7 @@ async fn object_lock_handlers_schedule_replication() {
// A bucket without a replication configuration can skip the decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let _no_replication_input = PutObjectLegalHoldInput::builder()
let no_replication_input = PutObjectLegalHoldInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.legal_hold(Some(ObjectLockLegalHold {
@@ -91,17 +81,7 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
let req = build_request((), Method::PUT);
object_meta
.put_object_legal_hold_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
ObjectLockLegalHoldStatus::OFF.to_owned(),
))),
)
fs.put_object_legal_hold(build_request(no_replication_input, Method::PUT))
.await
.expect("legal hold without replication should succeed");
assert_eq!(
@@ -113,7 +93,7 @@ async fn object_lock_handlers_schedule_replication() {
// A configured bucket must still evaluate the metadata replication decision.
install_delete_replication_config(&bucket, true, false, None).await;
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let _legal_hold_input = PutObjectLegalHoldInput::builder()
let legal_hold_input = PutObjectLegalHoldInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.legal_hold(Some(ObjectLockLegalHold {
@@ -121,17 +101,7 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
let req = build_request((), Method::PUT);
object_meta
.put_object_legal_hold_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
ObjectLockLegalHoldStatus::ON.to_owned(),
))),
)
fs.put_object_legal_hold(build_request(legal_hold_input, Method::PUT))
.await
.expect("put_object_legal_hold should succeed");
assert_eq!(
+1 -1
View File
@@ -29,7 +29,7 @@ use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::storage_api::s3::dto::{EncodingType, ListObjectVersionsInput, ListObjectsV2Input};
use crate::app::storage_api::s3::xml;
use crate::storage::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::access::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::authorization::AuthorizationRequest;
use http::{HeaderMap, Uri};
use rustfs_policy::policy::action::S3Action;
-2
View File
@@ -22,8 +22,6 @@
pub mod admin_usecase;
mod answered;
#[cfg(test)]
mod audit_coverage_test;
pub(crate) mod bucket_list_through;
pub mod bucket_usecase;
pub mod context;
+9 -16
View File
@@ -50,7 +50,8 @@ use super::storage_api::multipart_usecase::io::{
EtagAlgorithm, HashReader, WriteEncryption, WritePlan, compression_metadata_value,
};
#[cfg(test)]
use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag};
use super::storage_api::multipart_usecase::object_utils::to_s3s_etag;
use super::storage_api::multipart_usecase::object_utils::to_s3s_xml_etag;
use super::storage_api::multipart_usecase::options::{
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata,
@@ -103,7 +104,6 @@ use futures::StreamExt;
use http::{HeaderMap, Uri};
use metrics::counter;
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
#[cfg(test)]
@@ -115,6 +115,7 @@ use rustfs_utils::http::{
headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
insert_str,
};
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
@@ -192,13 +193,12 @@ fn multipart_max_total_object_size(metadata: &HashMap<String, String>) -> S3Resu
let value = get_consistent_str(metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
"multipart size capability metadata is missing or inconsistent",
"multipart size capability metadata is missing or inconsistent".to_string(),
)
})?;
value
.parse::<u64>()
.map(Some)
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid"))
value.parse::<u64>().map(Some).map_err(|_| {
S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid".to_string())
})
}
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
@@ -291,11 +291,7 @@ impl DefaultMultipartUsecase {
/// or completed".
fn multipart_upload_api_error(err: StorageError) -> S3Error {
if matches!(err, StorageError::MalformedUploadID(_)) {
let mut error = S3Error::new(S3ErrorCode::NoSuchUpload);
if let Some(message) = error.code().default_message() {
error.set_message(message);
}
return error;
return S3Error::new(S3ErrorCode::NoSuchUpload);
}
ApiError::from(err).into()
}
@@ -549,7 +545,7 @@ mod tests {
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
.await
.expect_err("reserved key must be rejected before accessing storage");
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
let complete = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_owned())
@@ -597,9 +593,6 @@ mod tests {
fn multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload() {
let malformed = multipart_upload_api_error(StorageError::MalformedUploadID("bucket/object-not-an-id".to_string()));
assert_eq!(malformed.code(), &S3ErrorCode::NoSuchUpload);
let legacy = crate::app::storage_api::s3::S3Error::new(crate::app::storage_api::s3::S3ErrorCode::NoSuchUpload);
assert_eq!(malformed.message(), legacy.message());
assert_eq!(malformed.status_code(), legacy.status_code());
let unknown = multipart_upload_api_error(StorageError::InvalidUploadID(
"bucket".to_string(),
@@ -61,7 +61,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
req.uri = format!("/{bucket}/{key}").parse().expect("fixture URI");
req.extensions.insert(slot.clone());
req.extensions.insert(request_context.clone());
RequestEnvelope::from_s3s(&req)
req
};
for name in [&bucket, &other] {
store
@@ -143,7 +143,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
(wrong, S3ErrorCode::InternalError),
] {
let err = usecase
.execute_abort_multipart_upload(req, Ok(target()))
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&req), Ok(target()))
.await
.expect_err("Abort requires current generation");
assert_eq!(err.code(), &code);
@@ -162,7 +162,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
.expect("capture current bucket");
current.install_bucket_generation(captured);
usecase
.execute_abort_multipart_upload(current.fork_for_authorization(), Ok(target()))
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&current), Ok(target()))
.await
.expect("current generation allows Abort");
assert!(matches!(
@@ -182,7 +182,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
.expect("create legacy-path upload");
usecase
.execute_abort_multipart_upload(
current,
RequestEnvelope::from_s3s(&current),
Ok(AbortMultipartUploadTarget::Legacy {
bucket: bucket.clone(),
key: key.to_owned(),
+116 -129
View File
@@ -294,8 +294,10 @@ impl DefaultMultipartUsecase {
reject_veeam_sosapi_mutation(|| self.object_store(), &bucket, &key).await?;
// The input member and the header are one value on the wire; both are checked, as the
// legacy guard checks both.
let specific_if_none_match = if_none_match.as_deref().is_some_and(|condition| condition.trim() != "*");
reject_non_wildcard_write_if_none_match(specific_if_none_match, headers)?;
if if_none_match.as_deref().is_some_and(|condition| condition.trim() != "*") {
return Err(S3Error::new(S3ErrorCode::NotImplemented));
}
reject_non_wildcard_write_if_none_match(None, headers)?;
validate_table_catalog_object_mutation(&bucket, &key).await?;
if if_match.is_some() || if_none_match.is_some() {
@@ -734,7 +736,6 @@ impl DefaultMultipartUsecase {
}
}
#[cfg(test)]
#[cfg(test)]
impl DefaultMultipartUsecase {
/// CompleteMultipartUpload for an s3s request: its facts in an envelope and its input across
@@ -743,41 +744,9 @@ impl DefaultMultipartUsecase {
&self,
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CompleteMultipartUploadInput>,
) -> S3Result<CompleteMultipartUploadOutput> {
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::complete_multipart_upload;
let envelope = RequestEnvelope::from_s3s(&req);
let mut s3s_input = req.input;
let has_upload = s3s_input.multipart_upload.is_some();
s3s_input.multipart_upload.get_or_insert_with(Default::default);
let tags: Vec<Option<String>> = s3s_input
.multipart_upload
.iter_mut()
.flat_map(|upload| upload.parts.iter_mut().flatten())
.map(|part| {
part.part_number.get_or_insert(0);
part.e_tag.take().map(|tag| tag.value().to_owned())
})
.collect();
for checksum in [
&mut s3s_input.checksum_crc32,
&mut s3s_input.checksum_crc32c,
&mut s3s_input.checksum_crc64nvme,
&mut s3s_input.checksum_md5,
&mut s3s_input.checksum_sha1,
&mut s3s_input.checksum_sha256,
&mut s3s_input.checksum_sha512,
&mut s3s_input.checksum_xxhash128,
&mut s3s_input.checksum_xxhash3,
&mut s3s_input.checksum_xxhash64,
] {
*checksum = None;
}
let mut input = complete_multipart_upload::input_from_s3s(s3s_input).map_err(crate::app::dto_values::seam_error)?;
for (part, tag) in input.multipart_upload.parts.iter_mut().zip(tags) {
part.e_tag = tag.map(rustfs_gateway_types::ETag::from_stored);
}
let parts = std::mem::take(&mut input.multipart_upload);
let parts = has_upload.then_some(parts);
let (input, parts) =
crate::storage::ecfs::complete_multipart_upload_input(req.input).map_err(crate::app::dto_values::seam_error)?;
match parts {
Some(parts) => {
let mut input = input;
@@ -1124,18 +1093,23 @@ mod tests {
crate::app::gating_test_env::run_large_stack_test("ssec-completion-checksums", || async {
let (_store, bucket) =
crate::app::gating_test_env::durable_quota_test_bucket("ssec-completion-checksums", 16_384).await;
let versioning_req = build_request((), Method::PUT);
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_versioning(
RequestEnvelope::from_s3s(&versioning_req),
&bucket,
Ok(rustfs_gateway_types::persistence::PersistedVersioningConfiguration {
status: Some("Enabled".to_owned()),
..Default::default()
}),
)
.await
.expect("enable versioning for encrypted multipart listing");
let fs = crate::storage::ecfs::FS::new();
crate::app::storage_api::s3::S3::put_bucket_versioning(
&fs,
build_request(
crate::app::storage_api::s3::dto::PutBucketVersioningInput::builder()
.bucket(bucket.clone())
.versioning_configuration(crate::app::storage_api::s3::dto::VersioningConfiguration {
status: Some(crate::app::storage_api::s3::dto::BucketVersioningStatus::from_static("Enabled")),
..Default::default()
})
.build()
.expect("bucket versioning input"),
Method::PUT,
),
)
.await
.expect("enable versioning for encrypted multipart listing");
let usecase = DefaultMultipartUsecase::from_global();
let key = "customer-encrypted";
let customer_key = "KioqKioqKioqKioqKioqKioqKioqKioqKioqKioqKio=";
@@ -1253,9 +1227,9 @@ mod tests {
let etag = etag_to_s3s(&response.e_tag.expect("completed encrypted object ETag"));
let version_id = response.version_id.expect("completed object version");
assert_ne!(version_id, "null");
assert_multipart_etag_listings(&bucket, key, &etag, &version_id).await;
let head = crate::app::test_seams::head_object(
&crate::app::object::DefaultObjectUsecase::from_global(),
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
let head = crate::app::storage_api::s3::S3::head_object(
&fs,
build_request(
crate::app::storage_api::s3::dto::HeadObjectInput {
bucket: bucket.clone(),
@@ -1285,10 +1259,15 @@ mod tests {
});
}
async fn assert_multipart_etag_listings(bucket: &str, key: &str, etag: &ETag, version_id: &str) {
async fn assert_multipart_etag_listings(
fs: &crate::storage::ecfs::FS,
bucket: &str,
key: &str,
etag: &ETag,
version_id: &str,
) {
use crate::app::storage_api::s3::dto::{ListObjectVersionsInput, ListObjectsInput, ListObjectsV2Input};
use crate::app::storage_api::s3::xml::{Serialize, Serializer};
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::{list_object_versions, list_objects};
fn assert_xml_etag<T: Serialize>(output: &T, etag: &ETag) {
let mut bytes = Vec::new();
@@ -1302,25 +1281,21 @@ mod tests {
);
}
let bucket_usecase = crate::app::bucket_usecase::DefaultBucketUsecase::from_global();
// Listings carry neither the SSE-C key nor a per-object decryption request.
let versions_req = build_request(
ListObjectVersionsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
);
let versions_env = RequestEnvelope::from_s3s(&versions_req);
let versions_in = list_object_versions::input_from_s3s(versions_req.input).expect("convert ListObjectVersionsInput");
let versions = list_object_versions::output_to_s3s(
bucket_usecase
.execute_list_object_versions(versions_env, versions_in)
.await
.expect("keyless version listing"),
let versions = crate::app::storage_api::s3::S3::list_object_versions(
fs,
build_request(
ListObjectVersionsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
),
)
.expect("convert ListObjectVersionsOutput");
.await
.expect("keyless version listing")
.output;
let entries = versions.versions.as_ref().expect("completed version must be listed");
assert_eq!(entries.len(), 1, "completion replay must preserve a single version");
assert_eq!(entries[0].key.as_deref(), Some(key));
@@ -1328,25 +1303,22 @@ mod tests {
assert_eq!(entries[0].e_tag.as_ref(), Some(etag));
assert_xml_etag(&versions, etag);
let v1_req = build_request(
ListObjectsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
);
let v1_env = RequestEnvelope::from_s3s(&v1_req);
let v1_in = list_objects::input_from_s3s(v1_req.input).expect("convert ListObjectsInput");
let v1 = list_objects::output_to_s3s(
bucket_usecase
.execute_list_objects(v1_env, v1_in)
.await
.expect("keyless V1 listing"),
let v1 = crate::app::storage_api::s3::S3::list_objects(
fs,
build_request(
ListObjectsInput {
bucket: bucket.to_string(),
prefix: Some(key.to_string()),
..Default::default()
},
Method::GET,
),
)
.expect("convert ListObjectsOutput");
let v2 = crate::app::test_seams::list_objects_v2(
&bucket_usecase,
.await
.expect("keyless V1 listing")
.output;
let v2 = crate::app::storage_api::s3::S3::list_objects_v2(
fs,
build_request(
ListObjectsV2Input {
bucket: bucket.to_string(),
@@ -1410,41 +1382,52 @@ mod tests {
)
.await
.expect("create versioned Object Lock bucket");
let fs = crate::storage::ecfs::FS::new();
let key = "kms-encrypted";
let enc_req = build_request((), Method::PUT);
let enc_cfg = crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
apply_server_side_encryption_by_default: Some(
crate::app::storage_api::s3::dto::ServerSideEncryptionByDefault {
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
kms_master_key_id: Some(kms_key.key_id.clone()),
},
),
..Default::default()
}],
};
let prepared = crate::storage::persisted_from_s3s(&enc_cfg).map_err(Into::into);
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_encryption(RequestEnvelope::from_s3s(&enc_req), &bucket, prepared)
.await
.expect("configure bucket-default SSE-KMS");
let usecase = DefaultMultipartUsecase::from_global();
let upload_id = usecase
.execute_create_multipart_upload_from_s3s(build_request(
crate::app::storage_api::s3::S3::put_bucket_encryption(
&fs,
build_request(
crate::app::storage_api::s3::dto::PutBucketEncryptionInput::builder()
.bucket(bucket.clone())
.server_side_encryption_configuration(
crate::app::storage_api::s3::dto::ServerSideEncryptionConfiguration {
rules: vec![crate::app::storage_api::s3::dto::ServerSideEncryptionRule {
apply_server_side_encryption_by_default: Some(
crate::app::storage_api::s3::dto::ServerSideEncryptionByDefault {
sse_algorithm: ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
kms_master_key_id: Some(kms_key.key_id.clone()),
},
),
..Default::default()
}],
},
)
.build()
.expect("bucket encryption input"),
Method::PUT,
),
)
.await
.expect("configure bucket-default SSE-KMS");
let upload_id = crate::app::storage_api::s3::S3::create_multipart_upload(
&fs,
build_request(
CreateMultipartUploadInput {
bucket: bucket.clone(),
key: key.to_string(),
..Default::default()
},
Method::POST,
))
.await
.expect("create KMS multipart upload")
.output
.upload_id
.expect("KMS upload id");
let part = usecase
.execute_upload_part(build_request(
),
)
.await
.expect("create KMS multipart upload")
.output
.upload_id
.expect("KMS upload id");
let part = crate::app::storage_api::s3::S3::upload_part(
&fs,
build_request(
UploadPartInput {
bucket: bucket.clone(),
key: key.to_string(),
@@ -1455,10 +1438,11 @@ mod tests {
..Default::default()
},
Method::PUT,
))
.await
.expect("upload encrypted KMS part")
.output;
),
)
.await
.expect("upload encrypted KMS part")
.output;
let complete = CompleteMultipartUploadInput {
bucket: bucket.clone(),
key: key.to_string(),
@@ -1473,16 +1457,19 @@ mod tests {
..Default::default()
};
for _ in 0..2 {
let response = usecase
.complete_for_s3s_request(build_request(complete.clone(), Method::POST))
.await
.expect("complete and replay KMS multipart upload");
let etag = etag_to_s3s(&response.e_tag.expect("KMS completed ETag"));
let response = crate::app::storage_api::s3::S3::complete_multipart_upload(
&fs,
build_request(complete.clone(), Method::POST),
)
.await
.expect("complete and replay KMS multipart upload")
.output;
let etag = response.e_tag.expect("KMS completed ETag");
let version_id = response.version_id.expect("KMS completed version");
assert_ne!(version_id, "null");
assert_multipart_etag_listings(&bucket, key, &etag, &version_id).await;
let head = crate::app::test_seams::head_object(
&crate::app::object::DefaultObjectUsecase::from_global(),
assert_multipart_etag_listings(&fs, &bucket, key, &etag, &version_id).await;
let head = crate::app::storage_api::s3::S3::head_object(
&fs,
build_request(
crate::app::storage_api::s3::dto::HeadObjectInput {
bucket: bucket.clone(),
+20 -82
View File
@@ -455,58 +455,6 @@ impl DefaultMultipartUsecase {
}
}
#[cfg(test)]
impl CreateMultipartUploadFields {
pub(crate) fn from_legacy(input: crate::app::storage_api::s3::dto::CreateMultipartUploadInput) -> Self {
Self {
bucket: input.bucket,
key: input.key,
metadata: input.metadata,
tagging: input.tagging,
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
ssekms_key_id: input.ssekms_key_id,
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
}
}
}
#[cfg(test)]
impl DefaultMultipartUsecase {
pub(crate) async fn execute_create_multipart_upload_from_s3s(
&self,
mut req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CreateMultipartUploadInput>,
) -> crate::app::storage_api::s3::S3Result<
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::CreateMultipartUploadOutput>,
> {
use crate::app::storage_api::s3::{S3Response, dto::CreateMultipartUploadOutput};
let envelope = RequestEnvelope::from_s3s(&req);
let query_version = req.input.version_id.take();
let output = self
.execute_create_multipart_upload(envelope, CreateMultipartUploadFields::from_legacy(req.input), query_version)
.await?;
let mut response = S3Response::new(CreateMultipartUploadOutput {
bucket: Some(output.bucket),
key: Some(output.key),
upload_id: Some(output.upload_id),
server_side_encryption: output.server_side_encryption.map(Into::into),
sse_customer_algorithm: output.sse_customer_algorithm,
sse_customer_key_md5: output.sse_customer_key_md5,
ssekms_key_id: output.ssekms_key_id,
checksum_algorithm: output.checksum_algorithm.map(Into::into),
checksum_type: output.checksum_type.map(Into::into),
..Default::default()
});
response.headers.extend(output.response_headers);
Ok(response)
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
@@ -783,19 +731,11 @@ mod tests {
.expect("create bucket");
}
let make_request = |target_bucket: &str| {
let mut req = request(target_bucket);
let envelope = RequestEnvelope::from_s3s(&req);
let query_version = req.input.version_id.take();
let fields = CreateMultipartUploadFields::from_legacy(req.input);
(envelope, fields, query_version)
};
let (mut stale_env, stale_fields, stale_ver) = make_request(&bucket);
let guard = load_bucket_generation_from_store(&store, &stale_env, &bucket)
let mut stale = request(&bucket);
let guard = load_bucket_generation_from_store(&store, &stale, &bucket)
.await
.expect("capture original incarnation");
stale_env.install_bucket_generation(guard);
stale.install_bucket_generation(guard);
let original = store.bucket_incarnation_id(&bucket).await.expect("old bucket generation");
store
@@ -809,18 +749,18 @@ mod tests {
let current = store.bucket_incarnation_id(&bucket).await.expect("new bucket generation");
assert_ne!(original, current);
let (missing_env, missing_fields, missing_ver) = make_request(&bucket);
let (mut wrong_env, wrong_fields, wrong_ver) = make_request(&bucket);
let guard = load_bucket_generation_from_store(&store, &wrong_env, &other)
let missing = request(&bucket);
let mut wrong = request(&bucket);
let guard = load_bucket_generation_from_store(&store, &wrong, &other)
.await
.expect("capture the other bucket");
wrong_env.install_bucket_generation(guard);
for (envelope, fields, query_version, code) in [
(stale_env, stale_fields, stale_ver, S3ErrorCode::NoSuchBucket),
(missing_env, missing_fields, missing_ver, S3ErrorCode::InternalError),
(wrong_env, wrong_fields, wrong_ver, S3ErrorCode::InternalError),
wrong.install_bucket_generation(guard);
for (request, code) in [
(stale, S3ErrorCode::NoSuchBucket),
(missing, S3ErrorCode::InternalError),
(wrong, S3ErrorCode::InternalError),
] {
let result = usecase.execute_create_multipart_upload(envelope, fields, query_version).await;
let result = usecase.execute_create_multipart_upload_from_s3s(request).await;
assert!(
store
.list_multipart_uploads(&bucket, "", None, None, None, 100)
@@ -829,20 +769,18 @@ mod tests {
.uploads
.is_empty()
);
let Err(error) = result else {
panic!("creation requires the target incarnation");
};
assert_eq!(error.code().as_str(), code.as_str());
let error = result.expect_err("creation requires the target incarnation");
assert_eq!(error.code(), &code);
}
let probe = crate::app::storage_api::multipart_usecase::helper::MutationCompletionProbe::install(&bucket);
let (mut allowed_env, allowed_fields, allowed_ver) = make_request(&bucket);
let guard = load_bucket_generation_from_store(&store, &allowed_env, &bucket)
let mut allowed = request(&bucket);
let guard = load_bucket_generation_from_store(&store, &allowed, &bucket)
.await
.expect("capture current incarnation");
allowed_env.install_bucket_generation(guard);
let output = usecase
.execute_create_multipart_upload(allowed_env, allowed_fields, allowed_ver)
allowed.install_bucket_generation(guard);
let response = usecase
.execute_create_multipart_upload_from_s3s(allowed)
.await
.expect("a current guard allows creation");
let uploads = store
@@ -851,7 +789,7 @@ mod tests {
.expect("list durable uploads")
.uploads;
assert_eq!(uploads.len(), 1);
assert_eq!(uploads[0].upload_id.as_str(), output.upload_id.as_str());
assert_eq!(Some(uploads[0].upload_id.as_str()), response.output.upload_id.as_deref());
assert_eq!(probe.audits.lock().unwrap().len(), 1);
assert!(probe.events.lock().unwrap().is_empty());
});
+7 -4
View File
@@ -26,8 +26,11 @@ use rustfs_s3_types::S3Result;
use rustfs_s3_types::{S3Error, S3ErrorCode};
pub(crate) enum ListPartsTarget {
#[allow(dead_code)]
Legacy { bucket: String, key: String, upload_id: String },
Legacy {
bucket: String,
key: String,
upload_id: String,
},
Gateway {
bucket: BucketName,
key: ObjectKey,
@@ -79,7 +82,7 @@ where
Ok::<_, StorageError>(page.recorded_owner.take().map(|owner| RecordedListParts { owner, page }))
})
.await
.map_err(multipart_upload_api_error)?
.map_err(|error| S3Error::from(multipart_upload_api_error(error)))?
.map_err(|refusal| S3Error::with_message(S3ErrorCode::NoSuchUpload, refusal.reason()))?;
Ok(record.page)
}
@@ -198,7 +201,7 @@ impl DefaultMultipartUsecase {
ListPartsTarget::Legacy { upload_id, .. } => store
.list_object_parts(bucket, key, upload_id, params.part_number_marker, params.max_parts, &opts)
.await
.map_err(multipart_upload_api_error),
.map_err(|error| multipart_upload_api_error(error).into()),
ListPartsTarget::Gateway {
bucket: bucket_name,
key: object_key,
@@ -165,7 +165,7 @@ fn list_parts_resolves_real_owners_and_retains_storage_errors() {
storage_error,
StorageError::InvalidUploadID(..) | StorageError::MalformedUploadID(..)
));
let expected: LegacyS3Error = ApiError::from(storage_error).into();
let expected = multipart_upload_api_error(storage_error);
for target in [
gateway_fields(&bucket, key, id, None, None),
ListPartsFields {
@@ -217,11 +217,10 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
let req = || {
let mut req = build_request((), Method::GET);
req.extensions.insert(slot.clone());
RequestEnvelope::from_s3s(&req)
req
};
let capability = |query: &str| {
let mut request = build_request((), Method::GET);
request.extensions.insert(slot.clone());
let mut request = req();
request.uri = format!("/fixture?{query}").parse().expect("fixture URI");
RequestEnvelope::from_s3s(&request)
};
@@ -270,7 +269,7 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
temp_env::async_with_vars([(rustfs_config::ENV_VEEAM_SOSAPI_ENABLE, Some("true"))], async {
let err = execute_counted(
&usecase,
veeam_request,
RequestEnvelope::from_s3s(&veeam_request),
gateway_fields(
&bucket,
".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml",
@@ -295,8 +294,13 @@ fn list_parts_applies_the_authorized_generation_before_lookup() {
if let Some(guard) = guard {
request.install_bucket_generation(guard);
}
let result =
execute_counted(&usecase, request, gateway_fields(&bucket, key, &upload.upload_id, marker, None), lookups).await;
let result = execute_counted(
&usecase,
RequestEnvelope::from_s3s(&request),
gateway_fields(&bucket, key, &upload.upload_id, marker, None),
lookups,
)
.await;
assert_eq!(result.as_ref().err().map(|err| err.code().as_str()), code);
if let Ok(page) = result {
assert!(page.upload_id == upload.upload_id);
@@ -43,7 +43,7 @@ impl RecordedUpload for PartUpload {
async fn lookup_part_upload(store: &ECStore, bucket: &str, key: &str, id: &str, opts: &ObjectOptions) -> S3Result<MultipartInfo> {
let result = store.get_multipart_info(bucket, key, id, opts).await;
result.map_err(multipart_upload_api_error)
result.map_err(|error| multipart_upload_api_error(error).into())
}
pub(super) async fn lookup_recorded_part_upload(
@@ -56,7 +56,7 @@ pub(super) async fn lookup_recorded_part_upload(
let result = store.lookup_multipart_part_binding(bucket, key, id, opts).await;
#[cfg(test)]
let result = tests::observe_lookup(result);
match result.map_err(multipart_upload_api_error) {
match result.map_err(|error| S3Error::from(multipart_upload_api_error(error))) {
Ok((info, owner)) => Ok(owner.map(|owner| PartUpload { owner, info })),
Err(error) if error.code() == &S3ErrorCode::NoSuchUpload => Ok(None),
Err(error) => Err(error),
@@ -148,18 +148,6 @@ fn normalize_presigned_part_checksums(req: &mut RequestEnvelope, input: &mut Upl
}
impl DefaultMultipartUsecase {
/// Records the gateway audit. The protocol client records its own projected response.
pub(crate) async fn execute_upload_part_with_audit(
&self,
envelope: RequestEnvelope,
input: UploadPartWriteInput,
) -> S3Result<UploadPartWriteOutput> {
RequestEnvelope::audit_only_operation(envelope, S3Operation::UploadPart, http::StatusCode::OK, |envelope| {
self.execute_upload_part_owned(envelope, input)
})
.await
}
#[instrument(name = "execute_upload_part", level = "debug", skip(self, req, input))]
#[hotpath::measure(impl_type = "MultipartUsecase")]
pub(crate) async fn execute_upload_part_owned(
@@ -513,72 +501,6 @@ impl DefaultMultipartUsecase {
}
}
#[cfg(test)]
impl UploadPartWriteInput {
pub(crate) fn from_legacy(
input: crate::app::storage_api::s3::dto::UploadPartInput,
trailers: Option<crate::app::storage_api::s3::TrailingHeaders>,
) -> Self {
Self {
target: UploadPartTarget::Legacy {
bucket: input.bucket,
key: input.key,
upload_id: input.upload_id,
},
part_number: input.part_number,
content_length: input.content_length,
body: input.body.map(crate::app::object::legacy_put_body),
trailing_headers: crate::app::trailer_adapter::trailer_source(trailers),
frame_trailers: None,
content_md5: input.content_md5,
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
checksum_crc32: input.checksum_crc32,
checksum_crc32c: input.checksum_crc32c,
checksum_crc64nvme: input.checksum_crc64nvme,
checksum_sha1: input.checksum_sha1,
checksum_sha256: input.checksum_sha256,
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
}
}
}
#[cfg(test)]
impl DefaultMultipartUsecase {
pub async fn execute_upload_part(
&self,
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::UploadPartInput>,
) -> crate::app::storage_api::s3::S3Result<
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::UploadPartOutput>,
> {
use crate::app::storage_api::s3::{
S3Response,
dto::{ServerSideEncryption, UploadPartOutput},
};
let envelope = RequestEnvelope::from_s3s(&req);
let input = UploadPartWriteInput::from_legacy(req.input, req.trailing_headers);
let output = self.execute_upload_part_with_audit(envelope, input).await?;
let mut response = S3Response::new(UploadPartOutput {
e_tag: output
.e_tag
.map(|tag| crate::app::storage_api::test::object_utils::to_s3s_etag(&tag)),
checksum_crc32: output.checksum_crc32,
checksum_crc32c: output.checksum_crc32c,
checksum_crc64nvme: output.checksum_crc64nvme,
checksum_sha1: output.checksum_sha1,
checksum_sha256: output.checksum_sha256,
server_side_encryption: output.server_side_encryption.map(ServerSideEncryption::from),
ssekms_key_id: output.ssekms_key_id,
sse_customer_algorithm: output.sse_customer_algorithm,
sse_customer_key_md5: output.sse_customer_key_md5,
..Default::default()
});
response.headers = output.headers;
Ok(response)
}
}
#[cfg(test)]
mod tests {
use super::super::test_support::*;
@@ -104,7 +104,7 @@ async fn native_claim_checks_order_ownership_and_one_active_lookup() {
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
LOOKUP
.scope((Cell::new(0), mode), async {
let result = usecase.execute_upload_part_with_audit(envelope, input).await;
let result = usecase.execute_upload_part_owned(envelope, input).await;
assert_eq!(
result.as_ref().err().map(|err| err.code().as_str()),
expected,
@@ -214,7 +214,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
let envelope = RequestEnvelope::from_s3s(&build_request((), Method::PUT));
LOOKUP
.scope((Cell::new(0), mode), async {
let result = Box::pin(usecase.execute_upload_part_copy_with_audit(envelope, input)).await;
let result = Box::pin(usecase.execute_upload_part_copy_owned(envelope, input)).await;
assert_eq!(result.as_ref().err().map(|error| error.code().as_str()), expected, "{mode:?}");
assert_eq!(LOOKUP.with(|(calls, _)| calls.get()), 1, "{mode:?}");
if expected == Some("NoSuchUpload") {
@@ -233,8 +233,7 @@ fn native_copy_part_rejects_unproved_owner_without_changing_parts() {
} else if matches!(mode, LookupResult::Outage) {
let Err(error) = result else { panic!("expected read quorum failure") };
let actual: crate::app::storage_api::s3::S3Error = error.into();
let expected: crate::app::storage_api::s3::S3Error =
ApiError::from(StorageError::ErasureReadQuorum).into();
let expected = multipart_upload_api_error(StorageError::ErasureReadQuorum);
assert_eq!(actual.code(), expected.code());
assert_eq!(actual.status_code(), expected.status_code());
assert_eq!(actual.message(), expected.message());
@@ -13,14 +13,14 @@
// limitations under the License.
//! UploadPartCopy through one owned storage implementation and its legacy adapter.
//! Gateway and protocol clients supply request facts. Storage owns the multipart upload.
//! Protocol clients and the legacy FS edge supply request facts; multipart ownership stays in storage.
use super::*;
#[cfg(test)]
use crate::app::storage_api::s3::dto::CopySource;
use crate::app::storage_api::s3::dto::{
CopyPartResult, CopySource, ETag, ServerSideEncryption, Timestamp, UploadPartCopyInput, UploadPartCopyOutput,
CopyPartResult, ETag, ServerSideEncryption, Timestamp, UploadPartCopyInput, UploadPartCopyOutput,
};
#[cfg(test)]
use crate::app::storage_api::s3::{S3Request, S3Response};
use crate::app::{RequestEnvelope, object_usecase::CopySourceCondition};
use rustfs_gateway_types::CopySourceValue;
@@ -100,7 +100,6 @@ fn validate_copy_source_range_not_exceeds(range_spec: &HTTPRangeSpec, object_siz
}
impl DefaultMultipartUsecase {
#[cfg(test)]
pub async fn execute_upload_part_copy(
&self,
req: S3Request<UploadPartCopyInput>,
@@ -130,7 +129,7 @@ impl DefaultMultipartUsecase {
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
};
let output = self.execute_upload_part_copy_with_audit(envelope, input).await?;
let output = self.execute_upload_part_copy_owned(envelope, input).await?;
Ok(S3Response::new(UploadPartCopyOutput {
copy_part_result: Some(CopyPartResult {
e_tag: output.e_tag.as_deref().map(to_s3s_xml_etag),
@@ -152,18 +151,6 @@ impl DefaultMultipartUsecase {
}
/// Copies plaintext into an existing upload without changing its recorded owner.
/// Records the gateway audit. The protocol client records its own projected response.
pub(crate) async fn execute_upload_part_copy_with_audit(
&self,
envelope: RequestEnvelope,
input: UploadPartCopyWriteInput,
) -> S3Result<UploadPartCopyWriteOutput> {
RequestEnvelope::audit_only_operation(envelope, S3Operation::UploadPartCopy, http::StatusCode::OK, |envelope| {
Box::pin(self.execute_upload_part_copy_owned(envelope, input))
})
.await
}
#[instrument(level = "debug", skip(self, envelope, input))]
#[hotpath::measure(impl_type = "MultipartUsecase")]
pub(crate) async fn execute_upload_part_copy_owned(
@@ -450,7 +437,7 @@ impl DefaultMultipartUsecase {
.user_defined
.contains_key("x-amz-server-side-encryption-customer-algorithm");
let ssekms_key_id = match server_side_encryption.as_ref() {
Some(sse) if sse.as_str() == rustfs_utils::http::headers::AMZ_ENCRYPTION_KMS => mp_info
Some(sse) if sse.as_str() == ServerSideEncryption::AWS_KMS => mp_info
.user_defined
.get("x-amz-server-side-encryption-aws-kms-key-id")
.map(|s| s.to_string()),
+3 -3
View File
@@ -162,7 +162,7 @@ impl DefaultObjectUsecase {
envelope.uri().query(),
envelope.verified_presigned().is_some(),
)?;
reject_non_wildcard_write_if_none_match(input.specific_if_none_match, envelope.headers())?;
reject_non_wildcard_write_if_none_match_native(input.specific_if_none_match, envelope.headers())?;
if let Some(context) = &self.context {
let _ = context.object_store();
}
@@ -1051,8 +1051,8 @@ mod tests {
.expect_err("short copy source must fail before destination write succeeds");
let api_error = ApiError::from(err);
assert_eq!(api_error.code, rustfs_s3_types::S3ErrorCode::ServiceUnavailable);
assert_ne!(api_error.code, rustfs_s3_types::S3ErrorCode::IncompleteBody);
assert_eq!(api_error.code, S3ErrorCode::ServiceUnavailable);
assert_ne!(api_error.code, S3ErrorCode::IncompleteBody);
}
#[tokio::test]
+1 -51
View File
@@ -99,57 +99,7 @@ impl CopyObjectWriteInput {
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::CopyObjectInput>,
) -> (crate::app::RequestEnvelope, Self) {
use crate::app::object_usecase::CopySourceCondition;
use crate::app::storage_api::s3::dto::{CopySource, ETag, ETagCondition};
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
let input = req.input;
let condition = |value: ETagCondition| match value {
ETagCondition::Any => CopySourceCondition::Any,
ETagCondition::ETag(ETag::Strong(value)) => CopySourceCondition::Strong(value),
ETagCondition::ETag(ETag::Weak(_)) => CopySourceCondition::Weak,
};
let copy_source = match &input.copy_source {
CopySource::AccessPoint { .. } | CopySource::Outpost { .. } => None,
CopySource::Bucket { bucket, key, version_id } => Some(rustfs_gateway_types::CopySourceValue::Bucket {
bucket: bucket.to_string(),
key: key.to_string(),
version_id: version_id.as_ref().map(|v| v.to_string()),
}),
};
let write_input = CopyObjectWriteInput {
copy_source,
bucket: input.bucket,
key: input.key,
version_id: input.version_id,
specific_if_none_match: matches!(input.if_none_match, Some(ETagCondition::ETag(_))),
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
ssekms_key_id: input.ssekms_key_id,
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
copy_source_sse_customer_algorithm: input.copy_source_sse_customer_algorithm,
copy_source_sse_customer_key: input.copy_source_sse_customer_key,
copy_source_sse_customer_key_md5: input.copy_source_sse_customer_key_md5,
metadata_directive: input.metadata_directive.map(|value| value.as_str().to_owned()),
metadata: input.metadata,
tagging: input.tagging,
tagging_directive: input.tagging_directive.map(|value| value.as_str().to_owned()),
copy_source_if_match: input.copy_source_if_match.map(condition),
copy_source_if_none_match: input.copy_source_if_none_match.map(condition),
cache_control: input.cache_control,
content_disposition: input.content_disposition,
content_encoding: input.content_encoding,
content_language: input.content_language,
content_type: input.content_type,
expires: input.expires,
website_redirect_location: input.website_redirect_location,
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
};
(envelope, write_input)
(envelope, crate::storage::ecfs::copy_input_from_legacy(req.input))
}
}
+5 -68
View File
@@ -503,7 +503,7 @@ fn project_delete_objects_pre_stat_error(
error: ApiError,
) -> S3Result<DeleteBatchError> {
// Bucket loss invalidates the shared request, not just one object.
if error.code == rustfs_s3_types::S3ErrorCode::NoSuchBucket {
if error.code == S3ErrorCode::NoSuchBucket {
return Err(error.into());
}
Ok(DeleteBatchError {
@@ -676,9 +676,10 @@ impl DefaultObjectUsecase {
let key = normalize_delete_object_key(&obj_id.key);
// A rejected wire key has no authorization resource. Its normalized text cannot become a target.
let key_error = if obj_id.key_too_long {
let code = rustfs_s3_types::S3ErrorCode::InvalidArgument;
let message = ApiError::error_code_to_message(&code);
Some((code, message))
Some((
S3ErrorCode::InvalidArgument,
ApiError::error_code_to_message(&S3ErrorCode::InvalidArgument),
))
} else {
validate_object_key(&key, "DeleteObjects")
.err()
@@ -4459,67 +4460,3 @@ mod bulk_denial_audit_wiring_tests {
);
}
}
#[cfg(test)]
impl From<crate::app::storage_api::s3::dto::DeleteObjectsInput> for DeleteBatchInput {
fn from(input: crate::app::storage_api::s3::dto::DeleteObjectsInput) -> Self {
Self {
bucket: input.bucket,
objects: input.delete.objects.into_iter().map(Into::into).collect(),
quiet: input.delete.quiet,
}
}
}
#[cfg(test)]
impl From<crate::app::storage_api::s3::dto::ObjectIdentifier> for DeleteBatchEntry {
fn from(object: crate::app::storage_api::s3::dto::ObjectIdentifier) -> Self {
use crate::app::storage_api::s3::dto::ETag;
Self {
key: object.key,
key_too_long: false,
version_id: object.version_id,
e_tag: object.e_tag.map(|tag| match tag {
ETag::Strong(value) => DeleteBatchETag { value, weak: false },
ETag::Weak(value) => DeleteBatchETag { value, weak: true },
}),
_last_modified_time: object.last_modified_time.map(|value| {
let instant = time::OffsetDateTime::from(value);
(instant.unix_timestamp(), instant.nanosecond())
}),
_size: object.size,
}
}
}
#[cfg(test)]
impl From<DeleteBatchOutput> for crate::app::storage_api::s3::dto::DeleteObjectsOutput {
fn from(output: DeleteBatchOutput) -> Self {
use crate::app::storage_api::s3::dto::{DeletedObject, Error};
Self {
deleted: output.deleted.map(|entries| {
entries
.into_iter()
.map(|entry| DeletedObject {
delete_marker: entry.delete_marker,
delete_marker_version_id: entry.delete_marker_version_id,
key: entry.key,
version_id: entry.version_id,
})
.collect()
}),
errors: output.errors.map(|entries| {
entries
.into_iter()
.map(|entry| Error {
code: entry.code,
key: entry.key,
message: entry.message,
version_id: entry.version_id,
})
.collect()
}),
request_charged: None,
}
}
}
+6 -52
View File
@@ -139,30 +139,6 @@ fn extract_archive_incomplete_body(remaining: u64) -> std::io::Error {
std::io::Error::new(std::io::ErrorKind::UnexpectedEof, rustfs_rio::IncompleteBody { remaining })
}
/// Settle the transport checksum and trailers before giving the final frame to the TAR parser.
/// A parser that rejects that frame may never poll the body again.
fn archive_transport_chunks(
chunks: super::request_body::UploadChunks,
expected_length: u64,
) -> impl Stream<Item = std::io::Result<Bytes>> {
futures::stream::try_unfold((chunks.fuse(), expected_length), |(mut chunks, remaining)| async move {
let Some(chunk) = chunks.next().await else {
return Ok(None);
};
let chunk = chunk.map_err(s3s_body_error_to_io)?;
let overlong = || std::io::Error::new(std::io::ErrorKind::InvalidData, "archive body exceeded expected Content-Length");
let remaining = remaining.checked_sub(chunk.len() as u64).ok_or_else(overlong)?;
if remaining == 0 {
while let Some(trailing) = chunks.next().await {
if !trailing.map_err(s3s_body_error_to_io)?.is_empty() {
return Err(overlong());
}
}
}
Ok(Some((chunk, (chunks, remaining))))
})
}
impl<R: AsyncRead> AsyncRead for ExtractArchiveEtagReader<R> {
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
let mut this = self.project();
@@ -1250,7 +1226,7 @@ pub fn normalize_extract_entry_key(path: &str, prefix: Option<&str>, is_dir: boo
fn map_extract_archive_error(err: std::io::Error) -> S3Error {
let api_error = ApiError::from(err);
if api_error.code != rustfs_s3_types::S3ErrorCode::InternalError {
if api_error.code != S3ErrorCode::InternalError {
return api_error.into();
}
@@ -1323,7 +1299,7 @@ fn is_empty_extract_entry_path(path: &str) -> bool {
fn validate_extract_member_key(path: &str, limits: ArchiveLimits) -> Result<(), ExtractEntryError> {
validate_put_object_extract_entry_path(path, limits).map_err(ExtractEntryError::Recoverable)?;
validate_new_object_key(path, "PUT").map_err(|err| ExtractEntryError::Recoverable(err.into()))
validate_new_object_key(path, "PUT").map_err(ExtractEntryError::Recoverable)
}
fn record_extract_pax_metadata_bytes(
@@ -2228,9 +2204,8 @@ impl DefaultObjectUsecase {
// Uses workload profile configuration (enabled by default) to select appropriate buffer size.
// Buffer sizes range from 32KB to 4MB depending on file size and configured workload profile.
let buffer_size = get_buffer_size_opt_in(size);
let expected_archive_length = u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent.into()))?;
let chunks = Box::pin(archive_transport_chunks(body, expected_archive_length));
let body = tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(chunks));
let body =
tokio::io::BufReader::with_capacity(buffer_size, StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))));
let md5hex = if let Some(base64_md5) = content_md5 {
let md5 = base64_simd::STANDARD
@@ -2251,6 +2226,7 @@ impl DefaultObjectUsecase {
return Err(ApiError::from(err).into());
}
let expected_archive_length = u64::try_from(size).map_err(|_| S3Error::new(S3ErrorCode::UnexpectedContent.into()))?;
let archive_upload_state = Arc::new(Mutex::new(ExtractArchiveUploadState::default()));
let extract_limits = put_object_extract_limits();
let tracked_archive =
@@ -3666,7 +3642,7 @@ mod tests {
] {
let error = map_extract_archive_error(error);
assert_eq!(error.code(), &code);
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code.into()).as_str()));
assert_eq!(error.message(), Some(ApiError::error_code_to_message(&code).as_str()));
assert!(error.source().is_some());
}
let error = map_extract_archive_error(io::Error::new(io::ErrorKind::InvalidData, "invalid TAR header"));
@@ -4317,28 +4293,6 @@ mod tests {
.expect("fixture member should parse")
}
#[tokio::test]
async fn archive_transport_checks_eof_before_releasing_the_final_frame() {
let data = || Ok(Bytes::from_static(b"ab"));
let late_error: StdError = Box::new(std::io::Error::other("trailer verification failed"));
for (tail, expected) in [
(Ok(Bytes::new()), None),
(data(), Some("archive body exceeded expected Content-Length")),
(Err(late_error), Some("trailer verification failed")),
] {
let body = Box::new(futures::stream::iter([data(), data(), tail]));
let mut chunks = Box::pin(archive_transport_chunks(body, 4));
assert_eq!(chunks.next().await.unwrap().unwrap(), "ab");
let last = chunks.next().await.unwrap();
if let Some(message) = expected {
assert_eq!(last.unwrap_err().to_string(), message);
} else {
assert_eq!(last.unwrap(), "ab");
}
assert!(chunks.next().await.is_none());
}
}
#[tokio::test]
async fn archive_etag_reader_validates_raw_eof_before_returning_final_byte() {
let payload = b"decoder-consumed-exact-body".to_vec();
+37 -16
View File
@@ -929,6 +929,7 @@ pin_project! {
struct GetObjectReaderStream<R> {
#[pin]
reader: Option<R>,
primed_byte: Option<u8>,
capacity: usize,
strategy: &'static str,
buffer_source: &'static str,
@@ -993,6 +994,7 @@ where
}
Self {
reader: Some(reader),
primed_byte: None,
capacity,
strategy,
buffer_source,
@@ -1437,9 +1439,14 @@ where
Some(reader) => reader,
None => return Poll::Ready(None),
};
let read_capacity = (*this.capacity).min(*this.remaining);
let mut buf = BytesMut::with_capacity(read_capacity);
let poll_read = poll_read_buf(reader, cx, &mut buf);
let (poll_read, buf) = if let Some(byte) = this.primed_byte.take() {
(Poll::Ready(Ok(1)), BytesMut::from(&[byte][..]))
} else {
let read_capacity = (*this.capacity).min(*this.remaining);
let mut buf = BytesMut::with_capacity(read_capacity);
let poll_read = poll_read_buf(reader, cx, &mut buf);
(poll_read, buf)
};
let result: Poll<Option<Self::Item>> = match poll_read {
Poll::Ready(Ok(bytes_read)) if bytes_read > 0 => {
@@ -2514,19 +2521,21 @@ impl DefaultObjectUsecase {
lifecycle,
resume,
);
let mut prefix = [0_u8; 1];
let prefix_len = if expected == 0 {
0
let primed_byte = if expected == 0 {
None
} else {
let mut prefix = [0_u8; 1];
reader
.read_exact(&mut prefix)
.await
.map_err(|error| map_get_object_reader_error(StorageError::from(error)))?;
1
Some(prefix[0])
};
let reader = std::io::Cursor::new(prefix).take(prefix_len).chain(reader);
let stream = GetObjectReaderStream::new(reader, stream_buffer_size, expected, stream_strategy.as_str(), buffer_source)
.with_diagnostics(bucket, key, request_id);
let mut stream =
GetObjectReaderStream::new(reader, stream_buffer_size, expected, stream_strategy.as_str(), buffer_source)
.with_diagnostics(bucket, key, request_id);
// Keep the preflight byte in the stream until the transport asks for it.
stream.primed_byte = primed_byte;
let blob = GetObjectBody::Stream(Box::pin(stream));
if let Some(handoff_start) = handoff_start {
rustfs_io_metrics::record_get_object_response_handoff(
@@ -2602,9 +2611,8 @@ impl DefaultObjectUsecase {
};
return Err(with_delete_marker_read_headers(S3Error::new(code), info, input.version_id.as_deref()));
}
let result = check_preconditions(envelope.headers(), info)
.map_err(S3Error::from)
.and_then(|()| Self::validate_get_object_part_number(part_number, info));
let result =
check_preconditions(envelope.headers(), info).and_then(|()| Self::validate_get_object_part_number(part_number, info));
if result.is_err() {
let principal = SseKmsPrincipal::from_request(envelope);
crate::storage::sse::validate_conditional_read(
@@ -9010,7 +9018,7 @@ mod tests {
let error = result.expect_err("a read quorum failure before the first byte must reject response construction");
assert_eq!(error.code(), &S3ErrorCode::Custom("SlowDownRead".into()));
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
assert_eq!(error.message(), Some("Resource requested is unreadable, please reduce your request rate"));
assert_eq!(error.message(), Some("The object could not be read from storage."));
}
#[tokio::test]
@@ -9037,7 +9045,20 @@ mod tests {
.expect("the first byte should be available before response handoff");
assert_eq!(reads.load(AtomicOrdering::Relaxed), 1, "response construction must prime one byte");
let mut received = Vec::new();
let first = body
.next()
.await
.expect("the primed byte should be present")
.expect("the first chunk should be readable");
assert_eq!(&first[..], b"h");
let first = first.try_into_mut().expect("the first chunk should own its buffer");
assert!(first.capacity() < 5, "the preflight byte must not reserve the remaining body capacity");
assert_eq!(
reads.load(AtomicOrdering::Relaxed),
1,
"emitting the primed byte must not poll storage again"
);
let mut received = first.to_vec();
while let Some(chunk) = body.next().await {
received.extend_from_slice(&chunk.expect("the primed body should remain readable"));
}
@@ -12314,7 +12335,7 @@ mod tests {
fn parse_get_object_part_number_rejects_above_s3_max() {
let err = parse_part_number_i32_to_usize(Some(10001), "GET").expect_err("partNumber above S3 max must fail");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.message(), Some("GET: partNumber must be between 1 and 10000"));
}
-36
View File
@@ -36,42 +36,6 @@ pub(crate) struct GetObjectFields {
pub(crate) sse_customer_key_md5: Option<String>,
}
impl GetObjectFields {
/// The conditional-read wire format permits a wildcard, a quoted tag, or a bare S3 tag.
pub(crate) fn valid_etag_condition(value: &[u8]) -> bool {
match value {
b"*" => true,
[b'"', tag @ .., b'"'] | [b'W', b'/', b'"', tag @ .., b'"'] => tag.iter().all(|b| matches!(*b, b'\t' | b' '..=b'~')),
bare => !bare.is_empty() && bare.iter().all(|b| b.is_ascii_alphanumeric() || *b == b'-'),
}
}
pub(crate) fn valid_date_condition(value: &str) -> bool {
time::PrimitiveDateTime::parse(value, super::RFC1123).is_ok()
}
/// Object Lambda and website reads accept the historical single-range wire grammar.
pub(crate) fn parse_range(value: &str) -> Option<ByteRange> {
use rustfs_gateway_types::RangeParse;
let spec = value.strip_prefix("bytes=")?;
if !spec.bytes().all(|b| b.is_ascii_digit() || b == b'-') {
return None;
}
if spec == "-" {
// An omitted suffix length historically selects a zero-length suffix.
return Some(ByteRange::Suffix { length: 0 });
}
let RangeParse::One(range) = RangeParse::parse(value) else {
return None;
};
match range {
ByteRange::From { first } if first > i64::MAX as u64 => None,
ByteRange::FromTo { last, .. } if last > i64::MAX as u64 => None,
_ => Some(range),
}
}
}
/// Body and ETag retain their native representation until the final edge.
#[derive(Debug, Default)]
pub(crate) struct GetObjectOutputFacts {
@@ -20,73 +20,6 @@ use crate::app::storage_api::s3::dto::Range;
use crate::app::storage_api::test::ReqInfo;
use http::Method;
#[test]
fn website_and_object_lambda_ranges_match_the_legacy_wire_reader() {
for value in [
"bytes=0-9",
"bytes=0-",
"bytes=-7",
"bytes=-0",
"bytes=-",
"bytes=-18446744073709551615",
"bytes=9223372036854775807-",
"bytes=9223372036854775808-",
"bytes=0-9223372036854775808",
"bytes=-18446744073709551616",
"bytes=10-5",
"bytes=1-2 ",
"bytes=1 -2",
"bytes=1-2,3-4",
"bytes=-+1",
"Bytes=1-2",
"bytes=",
] {
let expected = Range::parse(value).ok().map(|range| match range {
Range::Int { first, last: Some(last) } => rustfs_gateway_types::ByteRange::FromTo { first, last },
Range::Int { first, last: None } => rustfs_gateway_types::ByteRange::From { first },
Range::Suffix { length } => rustfs_gateway_types::ByteRange::Suffix { length },
});
assert_eq!(GetObjectFields::parse_range(value), expected, "{value}");
}
}
#[test]
fn website_and_object_lambda_conditions_match_the_legacy_wire_reader() {
use s3s::dto::{ETagCondition, Timestamp, TimestampFormat};
for value in [
b"*".as_slice(),
b"abc-2",
b"\"\"",
b"W/\"x\"",
b"\"a \tb\"",
b"\"a\"b\"",
b"",
b"bad etag",
b"w/\"x\"",
b"\"\xff\"",
b"\"\x7f\"",
b"\"x",
] {
assert_eq!(
GetObjectFields::valid_etag_condition(value),
ETagCondition::parse_http_header(value).is_ok(),
"{value:?}"
);
}
for value in [
"Sun, 06 Nov 1994 08:49:37 GMT",
"Sunday, 06-Nov-94 08:49:37 GMT",
"2025-01-01",
"Sun, 32 Nov 1994 08:49:37 GMT",
] {
assert_eq!(
GetObjectFields::valid_date_condition(value),
Timestamp::parse(TimestampFormat::HttpDate, value).is_ok(),
"{value}"
);
}
}
fn get_request(input: GetObjectInput) -> S3Request<GetObjectInput> {
let info = ReqInfo {
bucket: Some(input.bucket.clone()),
-5
View File
@@ -45,11 +45,6 @@ pub(crate) struct ResponseEtag {
}
impl ResponseEtag {
pub(crate) fn to_http_header(&self) -> Result<http::HeaderValue, http::header::InvalidHeaderValue> {
let prefix = if self.weak { "W/" } else { "" };
http::HeaderValue::from_str(&format!("{prefix}\"{}\"", self.opaque))
}
pub(super) fn from_stored(value: &str) -> Self {
let weak = value.starts_with("W/\"");
let etag = stored_etag(value);
+17 -7
View File
@@ -22,7 +22,7 @@
//! multipart handlers' policy steps against the same storage contract.
use super::*;
use rustfs_s3_types::{S3Error, S3ErrorCode};
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Response, S3Result};
use crate::app::object_data_cache::invalidate_object_data_cache_after_complete_multipart_success;
use crate::app::runtime_sources::current_app_context;
@@ -80,8 +80,7 @@ pub(crate) struct InternalPutContext {
const INTERNAL_PUT_METHOD_NAME: &str = "PUT";
fn api_error_from_s3(err: impl Into<S3Error>) -> ApiError {
let err = err.into();
fn api_error_from_s3(err: S3Error) -> ApiError {
ApiError {
code: err.code().clone(),
message: err.message().unwrap_or_default().to_string(),
@@ -140,7 +139,7 @@ async fn validate_internal_write_target(key: &str, bucket: &str, headers: &Heade
headers.get("content-type").and_then(|value| value.to_str().ok()),
headers.get("content-encoding").and_then(|value| value.to_str().ok()),
)
.map_err(api_error_from_s3)
.map_err(|err| api_error_from_s3(err.into()))
}
fn internal_events_wanted() -> bool {
@@ -214,7 +213,17 @@ impl InternalPutObjectEvent {
}
}
pub(super) fn publish(self, headers: &HeaderMap) {
/// Publish the event when `result` is a success; failures publish nothing.
pub(super) fn complete<T>(self, result: &S3Result<S3Response<T>>) {
if let Ok(response) = result {
self.complete_owned(&Ok(()), &response.headers);
}
}
pub(super) fn complete_owned<T>(self, result: &rustfs_s3_types::S3Result<T>, headers: &HeaderMap) {
if result.is_err() {
return;
}
let Self {
builder,
notify,
@@ -334,7 +343,7 @@ impl DefaultObjectUsecase {
.put_object_core(write, super::native_body::from_io_stream(body), None, start_time)
.await
.map_err(|error| ApiError {
code: error.code().clone(),
code: error.code().clone().into(),
message: error.message().unwrap_or_default().to_owned(),
source: None,
})?;
@@ -758,7 +767,8 @@ impl DefaultObjectUsecase {
if versioned && let Some(version_id) = obj_info.version_id {
event = event.version_id(version_id.to_string());
}
event.publish(&HeaderMap::new());
let result: S3Result<S3Response<()>> = Ok(S3Response::new(()));
event.complete(&result);
}
Ok(obj_info)
}
+2 -3
View File
@@ -79,6 +79,8 @@ use super::storage_api::object_usecase::data_usage::{
record_bucket_object_write_unknown_previous_memory,
};
use super::storage_api::object_usecase::deadlock_detector;
#[cfg(test)]
use super::storage_api::object_usecase::ecfs::FS;
use super::storage_api::object_usecase::error::{
Error as EcstoreError, StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
};
@@ -237,14 +239,11 @@ pub(crate) mod post_body;
pub(crate) mod post_headers;
mod put;
mod put_input;
#[cfg(test)]
pub(crate) use native_body::from_legacy as legacy_put_body;
pub(crate) use native_body::{BodyChunks, FrameTrailers};
pub(crate) use put_input::{PutObjectWriteInput, PutObjectWriteOutput};
pub(crate) mod request_body;
mod restore;
#[cfg(test)]
mod s3s_compat;
pub(crate) mod shared;
#[cfg(test)]
mod test_support;
+2 -9
View File
@@ -15,13 +15,9 @@
//! Upload frame/chunk boundary. Protocol edges supply a native body; storage consumes chunks.
//! This adapter preserves trailers and source errors without applying another length check.
use crate::app::storage_api::request_body::StdError;
#[cfg(test)]
use crate::app::storage_api::request_body::{ByteStream, StreamingBlob};
use crate::app::storage_api::request_body::{ByteStream, StdError, StreamingBlob};
use bytes::Bytes;
use futures::Stream;
#[cfg(test)]
use futures::StreamExt;
use futures::{Stream, StreamExt};
use http::HeaderMap;
use http_body::Body as _;
use parking_lot::Mutex;
@@ -34,7 +30,6 @@ use std::sync::Arc;
use std::task::{Context, Poll};
/// The legacy edge preserves the producer's length hint without enforcing it a second time.
#[cfg(test)]
pub(crate) fn from_legacy(body: StreamingBlob) -> Body {
let stream = LegacyPayload { body, ended: false };
Body::from_payload(Payload::Stream(Box::pin(stream)))
@@ -54,13 +49,11 @@ impl std::error::Error for ProducerSource {
}
}
#[cfg(test)]
struct LegacyPayload {
body: StreamingBlob,
ended: bool,
}
#[cfg(test)]
impl PayloadStream for LegacyPayload {
fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<PayloadRead, StreamError>> {
if self.ended {
@@ -32,7 +32,7 @@
//! `odm-source-etag`.
use super::*;
use rustfs_s3_types::S3ErrorCode;
use crate::app::storage_api::s3::S3ErrorCode;
use crate::app::storage_api::multipart_usecase::contract::multipart::CompletePart;
use crate::on_demand_migration::{
+5 -7
View File
@@ -150,7 +150,7 @@ pub(crate) fn reject_oversize_single_upload(size: i64) -> S3Result<()> {
if u64::try_from(size).is_ok_and(|size| size > rustfs_config::MAX_SINGLE_PUT_OBJECT_SIZE) {
return Err(S3Error::with_message(
S3ErrorCode::EntityTooLarge,
ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge),
ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge),
));
}
Ok(())
@@ -1174,10 +1174,8 @@ impl PutObjectCompletion {
match self {
Self::S3(helper) => Self::S3(helper.complete_owned(result, status, headers)),
Self::Internal(event) => {
if result.is_ok()
&& let Some(event) = event
{
event.publish(headers);
if let Some(event) = event {
event.complete_owned(result, headers);
}
Self::Internal(None)
}
@@ -1324,7 +1322,7 @@ impl DefaultObjectUsecase {
{
return Err(rustfs_s3_types::s3_error!(InvalidStorageClass));
}
reject_non_wildcard_write_if_none_match(input.specific_if_none_match, envelope.headers())?;
reject_non_wildcard_write_if_none_match_native(input.specific_if_none_match, envelope.headers())?;
// An authorized inbound replication PUT must store the replica verbatim.
// Legacy snowball-extracted members may still carry the auto-extract
// metadata, which replication replays as a header. Do not interpret that
@@ -5669,7 +5667,7 @@ mod oversize_single_upload_tests {
assert_eq!(*err.code(), S3ErrorCode::EntityTooLarge);
assert_eq!(
err.message(),
Some(ApiError::error_code_to_message(&rustfs_s3_types::S3ErrorCode::EntityTooLarge).as_str())
Some(ApiError::error_code_to_message(&S3ErrorCode::EntityTooLarge).as_str())
);
}
}
-45
View File
@@ -91,51 +91,6 @@ impl PutObjectWriteInput {
#[cfg(test)]
impl PutObjectWriteInput {
pub(crate) fn from_legacy(
input: crate::app::storage_api::s3::PutObjectInput,
trailing_headers: Option<crate::app::storage_api::s3::TrailingHeaders>,
) -> Self {
use crate::app::storage_api::s3::dto::ETagCondition;
Self {
bucket: input.bucket,
key: input.key,
body: input.body.map(crate::app::object::legacy_put_body),
post_archive: None,
frame_trailers: None,
trailing_headers: crate::app::trailer_adapter::trailer_source(trailing_headers),
content_length: input.content_length,
metadata: input.metadata,
metadata_headers: None,
checksum_headers: None,
specific_if_none_match: matches!(input.if_none_match, Some(ETagCondition::ETag(_))),
object_lock_retain_until_date: input.object_lock_retain_until_date.map(time::OffsetDateTime::from),
cache_control: input.cache_control,
content_disposition: input.content_disposition,
content_encoding: input.content_encoding,
content_language: input.content_language,
content_type: input.content_type,
expires: input.expires,
tagging: input.tagging,
version_id: input.version_id,
server_side_encryption: input.server_side_encryption.map(|value| value.as_str().to_owned()),
sse_customer_algorithm: input.sse_customer_algorithm,
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
ssekms_key_id: input.ssekms_key_id,
content_md5: input.content_md5,
object_lock_legal_hold_status: input.object_lock_legal_hold_status.map(|value| value.as_str().to_owned()),
object_lock_mode: input.object_lock_mode.map(|value| value.as_str().to_owned()),
storage_class: input.storage_class.map(|value| value.as_str().to_owned()),
website_redirect_location: input.website_redirect_location,
checksum_algorithm: input.checksum_algorithm.map(|value| value.as_str().to_owned()),
checksum_crc32: input.checksum_crc32,
checksum_crc32c: input.checksum_crc32c,
checksum_sha1: input.checksum_sha1,
checksum_sha256: input.checksum_sha256,
checksum_crc64nvme: input.checksum_crc64nvme,
}
}
/// A test's s3s fixture request as the envelope and input a gateway handler hands the use case.
pub(crate) fn from_test_request(
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::PutObjectInput>,
+1 -2
View File
@@ -13,11 +13,10 @@
// limitations under the License.
use super::*;
use crate::app::storage_api::s3::{Body as S3Body, S3Error, StreamingBlob};
use crate::app::storage_api::s3::{Body as S3Body, S3Error, S3ErrorCode, StreamingBlob};
use crate::error::ApiError;
use futures::{StreamExt, poll};
use http_body_util::StreamBody;
use rustfs_s3_types::S3ErrorCode;
use std::io;
use tokio::io::AsyncReadExt;
use tokio::sync::mpsc;
-36
View File
@@ -539,42 +539,6 @@ fn restore_output() -> RestoreObjectOutput {
}
}
#[cfg(test)]
impl DefaultObjectUsecase {
/// Projects a legacy RestoreObject request onto the shared restore path.
pub async fn execute_restore_object(
&self,
req: crate::app::storage_api::s3::S3Request<crate::app::storage_api::s3::dto::RestoreObjectInput>,
) -> crate::app::storage_api::s3::S3Result<
crate::app::storage_api::s3::S3Response<crate::app::storage_api::s3::dto::RestoreObjectOutput>,
> {
use crate::app::storage_api::object_usecase::bucket::restore_request_from_wire;
use crate::app::storage_api::s3::{
S3Response,
dto::{RequestCharged, RestoreObjectOutput},
};
let envelope = RequestEnvelope::from_s3s(&req);
let input = req.input;
self.execute_restore_object_owned(
envelope,
input.bucket,
input.key,
input.version_id,
input.restore_request.map(restore_request_from_wire),
)
.await
.map(|output| {
S3Response::new(RestoreObjectOutput {
request_charged: output
.request_charged
.map(|value| RequestCharged::from(value.as_str().to_owned())),
restore_output_path: output.restore_output_path,
})
})
.map_err(Into::into)
}
}
#[cfg(test)]
mod tests {
use super::*;
-181
View File
@@ -1,181 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Compatibility conversions between owned Get/Head object facts and s3s DTOs.
//!
//! Retained for `admin/router.rs` (object-lambda), `app/website.rs`, and `app/test_seams` until
//! T4.2 removes the remaining `s3s` dependency from `admin/router.rs`.
use crate::app::object_usecase::{GetObjectBody, GetObjectFields, GetObjectOutputFacts};
#[cfg(test)]
use crate::app::object_usecase::{HeadObjectFields, HeadObjectOutputFacts};
use rustfs_gateway_stream::{BoxPayloadStream, PayloadRead, StreamError, StreamErrorKind};
use rustfs_gateway_types::ByteRange;
use rustfs_gateway_types::compat::ConversionError;
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::get_object;
#[cfg(test)]
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::head_object;
use s3s::dto::{ETag, GetObjectInput, GetObjectOutput, Range, StreamingBlob};
#[cfg(test)]
use s3s::dto::{HeadObjectInput, HeadObjectOutput};
use std::pin::Pin;
use std::sync::{Mutex, PoisonError};
use std::task::{Context, Poll};
#[cfg(test)]
impl HeadObjectFields {
pub(crate) fn from_s3s(input: HeadObjectInput) -> Self {
Self {
bucket: input.bucket,
key: input.key,
version_id: input.version_id,
part_number: input.part_number,
range: input.range.map(|range| match range {
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
Range::Int { first, last: None } => ByteRange::From { first },
Range::Suffix { length } => ByteRange::Suffix { length },
}),
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
}
}
}
#[cfg(test)]
impl HeadObjectOutputFacts {
pub(crate) fn into_s3s(self) -> Result<HeadObjectOutput, ConversionError> {
let mut output = head_object::output_to_s3s(self.dto)?;
output.e_tag = self.etag.map(|etag| {
if etag.weak {
ETag::Weak(etag.opaque)
} else {
ETag::Strong(etag.opaque)
}
});
Ok(output)
}
}
impl GetObjectFields {
pub(crate) fn from_s3s(input: GetObjectInput) -> Self {
Self {
bucket: input.bucket,
key: input.key,
version_id: input.version_id,
part_number: input.part_number,
range: input.range.map(|range| match range {
Range::Int { first, last: Some(last) } => ByteRange::FromTo { first, last },
Range::Int { first, last: None } => ByteRange::From { first },
Range::Suffix { length } => ByteRange::Suffix { length },
}),
sse_customer_key: input.sse_customer_key,
sse_customer_key_md5: input.sse_customer_key_md5,
}
}
}
impl GetObjectOutputFacts {
pub(crate) fn into_s3s(self) -> Result<GetObjectOutput, ConversionError> {
let mut output = get_object::output_to_s3s(self.dto)?;
output.e_tag = self.etag.map(|etag| {
if etag.weak {
ETag::Weak(etag.opaque)
} else {
ETag::Strong(etag.opaque)
}
});
output.body = self.body.map(GetObjectBody::into_s3s);
Ok(output)
}
}
impl GetObjectBody {
pub(crate) fn into_s3s(self) -> StreamingBlob {
match self {
Self::Bytes(bytes) => StreamingBlob::from_bytes(bytes),
Self::Stream(payload) => StreamingBlob::new(LegacyGetStream {
payload: Mutex::new(payload),
done: false,
}),
}
}
}
// The legacy body requires Sync. Polling uses exclusive access, not a mutex lock.
struct LegacyGetStream {
payload: Mutex<BoxPayloadStream>,
done: bool,
}
impl futures::Stream for LegacyGetStream {
type Item = Result<bytes::Bytes, Box<dyn std::error::Error + Send + Sync>>;
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
let this = self.get_mut();
if this.done {
return Poll::Ready(None);
}
let event = std::task::ready!(
this.payload
.get_mut()
.unwrap_or_else(PoisonError::into_inner)
.as_mut()
.poll_read(cx)
);
Poll::Ready(match event {
Ok(PayloadRead::Chunk(bytes)) => Some(Ok(bytes)),
Ok(PayloadRead::Eof { trailers }) => {
this.done = true;
if trailers.is_empty() {
None
} else {
Some(Err(Box::new(ConversionError {
field: "body",
reason: "legacy GetObject cannot encode response trailers",
})))
}
}
Err(error) => {
this.done = true;
Some(Err(legacy_get_body_error(error)))
}
})
}
}
fn legacy_get_body_error(error: StreamError) -> Box<dyn std::error::Error + Send + Sync> {
if matches!(error.kind(), StreamErrorKind::Upstream(_)) {
let StreamErrorKind::Upstream(source) = error.into_kind() else {
unreachable!("the owned error kind was checked above")
};
source
} else {
Box::new(error)
}
}
impl s3s::stream::ByteStream for LegacyGetStream {
fn remaining_length(&self) -> s3s::stream::RemainingLength {
match self
.payload
.lock()
.unwrap_or_else(PoisonError::into_inner)
.len_hint()
.and_then(|length| usize::try_from(length).ok())
{
Some(length) => s3s::stream::RemainingLength::new_exact(length),
None => s3s::stream::RemainingLength::unknown(),
}
}
}
+31 -7
View File
@@ -1030,6 +1030,12 @@ pub(in crate::app) fn object_lock_checks_required_for_state(state: &metadata_sys
pub(crate) const ON_DEMAND_MIGRATION_HEADER: http::HeaderName = http::HeaderName::from_static("x-rustfs-on-demand-migration");
pub(crate) const ON_DEMAND_MIGRATION_SOURCE: HeaderValue = HeaderValue::from_static("source");
/// Response header marking a `ListObjectsV2` that could not consult the source
/// and was answered from local state alone (rustfs/backlog#2164).
pub(crate) const ON_DEMAND_MIGRATION_LIST_HEADER: http::HeaderName =
http::HeaderName::from_static("x-rustfs-on-demand-migration-list");
pub(crate) const ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY: HeaderValue = HeaderValue::from_static("local_only");
/// Custom S3 error code for a source failure surfaced under
/// `policy.source_error = propagate`; carried on HTTP 424.
pub(crate) const ODM_SOURCE_UNAVAILABLE_CODE: &str = "SourceUnavailable";
@@ -1103,6 +1109,12 @@ pub(crate) fn mark_on_demand_migration_response(headers: &mut HeaderMap) {
headers.insert(ON_DEMAND_MIGRATION_HEADER, ON_DEMAND_MIGRATION_SOURCE);
}
/// Marks a merged listing that fell back to local state because the source
/// could not be consulted (rustfs/backlog#2164).
pub(crate) fn mark_on_demand_migration_list_local_only(headers: &mut HeaderMap) {
headers.insert(ON_DEMAND_MIGRATION_LIST_HEADER, ON_DEMAND_MIGRATION_LIST_LOCAL_ONLY);
}
/// Evaluates the request's conditional headers (`If-Match`, `If-None-Match`,
/// `If-Modified-Since`, `If-Unmodified-Since`) against the source's view of
/// the object, with the same S3 semantics the local path applies to
@@ -1136,14 +1148,21 @@ pub(crate) fn odm_check_source_preconditions(headers: &HeaderMap, head: &SourceH
user_defined: metadata.into(),
..Default::default()
};
Ok(check_preconditions(headers, &info)?)
check_preconditions(headers, &info)
}
/// S3 documents write-side `If-None-Match` only as `*` (create-only).
/// Reject unsupported ETag comparisons with 501 NotImplemented.
/// Treating it as an ETag comparison would let a write overwrite an existing key
/// while still satisfying a policy that requires `s3:if-none-match`.
pub(crate) fn reject_non_wildcard_write_if_none_match(specific_if_none_match: bool, headers: &HeaderMap) -> S3Result<()> {
pub(crate) fn reject_non_wildcard_write_if_none_match(
if_none_match: Option<&s3s::dto::ETagCondition>,
headers: &HeaderMap,
) -> S3Result<()> {
reject_non_wildcard_write_if_none_match_native(matches!(if_none_match, Some(s3s::dto::ETagCondition::ETag(_))), headers)
}
pub(crate) fn reject_non_wildcard_write_if_none_match_native(specific_if_none_match: bool, headers: &HeaderMap) -> S3Result<()> {
// POST parses conditional DTO fields from the form, so HTTP headers also need validation.
if specific_if_none_match
|| headers.get_all(http::header::IF_NONE_MATCH).iter().any(|value| {
@@ -1161,6 +1180,7 @@ pub(crate) fn reject_non_wildcard_write_if_none_match(specific_if_none_match: bo
#[cfg(test)]
mod tests {
use super::*;
use crate::app::storage_api::s3::dto::ETag;
use http::{HeaderMap, HeaderValue};
use rustfs_gateway_types::persistence::{
PersistedBucketEncryptionRule as ServerSideEncryptionRule, PersistedEncryptionByDefault as ServerSideEncryptionByDefault,
@@ -1170,8 +1190,12 @@ mod tests {
#[test]
fn reject_non_wildcard_write_if_none_match_allows_only_wildcard() {
let headers = HeaderMap::new();
assert!(reject_non_wildcard_write_if_none_match(false, &headers).is_ok());
let err = reject_non_wildcard_write_if_none_match(true, &headers).expect_err("an ETag If-None-Match must be rejected");
assert!(reject_non_wildcard_write_if_none_match(None, &headers).is_ok());
assert!(reject_non_wildcard_write_if_none_match(Some(&s3s::dto::ETagCondition::Any), &headers).is_ok());
let etag = s3s::dto::ETagCondition::ETag(ETag::Strong("9b2cf535f27731c974343645a3985328".to_owned()));
let err =
reject_non_wildcard_write_if_none_match(Some(&etag), &headers).expect_err("an ETag If-None-Match must be rejected");
assert_eq!(*err.code(), S3ErrorCode::NotImplemented);
assert_eq!(err.status_code(), Some(http::StatusCode::NOT_IMPLEMENTED));
assert_eq!(
@@ -1185,12 +1209,12 @@ mod tests {
for value in ["*", " * ", "", " "] {
let mut headers = HeaderMap::new();
headers.insert(http::header::IF_NONE_MATCH, http::HeaderValue::from_str(value).expect("valid header"));
assert!(reject_non_wildcard_write_if_none_match(false, &headers).is_ok(), "{value:?}");
assert!(reject_non_wildcard_write_if_none_match(None, &headers).is_ok(), "{value:?}");
}
for value in ["\"different-etag\"", "W/\"weak-etag\"", "unquoted-etag"] {
let mut headers = HeaderMap::new();
headers.insert(http::header::IF_NONE_MATCH, http::HeaderValue::from_str(value).expect("valid header"));
let err = reject_non_wildcard_write_if_none_match(false, &headers)
let err = reject_non_wildcard_write_if_none_match(None, &headers)
.expect_err("header-only ETag precondition must be rejected");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
@@ -1205,7 +1229,7 @@ mod tests {
let mut headers = HeaderMap::new();
headers.append(http::header::IF_NONE_MATCH, http::HeaderValue::from_static("*"));
headers.append(http::header::IF_NONE_MATCH, value);
let err = reject_non_wildcard_write_if_none_match(false, &headers)
let err = reject_non_wildcard_write_if_none_match(Some(&s3s::dto::ETagCondition::Any), &headers)
.expect_err("a form wildcard must not hide an unsupported raw HTTP precondition");
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
}
+33 -21
View File
@@ -614,7 +614,7 @@ fn veeam_sosapi_http_contract() {
crate::app::gating_test_env::run_large_stack_test("veeam-sosapi-http", || async {
temp_env::async_with_vars([(rustfs_config::ENV_VEEAM_SOSAPI_ENABLE, None::<&str>)], async {
use crate::app::runtime_sources::{AppContext, ServerContextSlot};
use std::time::Duration;
use crate::app::storage_api::s3::{Body, S3ServiceBuilder, SimpleAuth};
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions};
use crate::app::storage_api::test::contract::list::ListOperations as _;
use crate::app::storage_api::test::contract::object::ObjectOperations as _;
@@ -626,18 +626,17 @@ fn veeam_sosapi_http_contract() {
let ambient = crate::app::gating_test_env::shared_gating_ambient().await;
let context = Arc::new(AppContext::new(Arc::clone(&store), ambient.iam(), ambient.kms()));
let slot = ServerContextSlot::new();
assert!(slot.install(Arc::clone(&context)));
let gateway = crate::server::gateway_stack::TestGateway::over(&context, slot);
struct Owner<'a>(&'a crate::server::gateway_stack::TestGateway);
impl Owner<'_> {
async fn call(
&self,
(method, uri, headers, body): (Method, String, Vec<(&'static str, String)>, Bytes),
) -> Result<http::Response<rustfs_gateway::Body>, std::convert::Infallible> {
self.0.signed(method, &uri, headers, body).await
}
}
let owner = Owner(&gateway);
assert!(slot.install(context));
let fs = super::super::FS::with_server_ctx(slot);
let owner = S3ServiceBuilder::new(fs.clone()).build();
let mut anonymous_builder = S3ServiceBuilder::new(fs.clone());
anonymous_builder.set_auth(SimpleAuth::from_single("test-access", "test-secret"));
anonymous_builder.set_access(fs.clone());
let anonymous = anonymous_builder.build();
let mut bad_auth_builder = S3ServiceBuilder::new(fs.clone());
bad_auth_builder.set_auth(SimpleAuth::from_single("test-access", "test-secret"));
bad_auth_builder.set_access(fs);
let bad_auth = bad_auth_builder.build();
let bucket = format!("veeam-sosapi-{}", super::super::Uuid::new_v4());
store
@@ -645,8 +644,19 @@ fn veeam_sosapi_http_contract() {
.await
.expect("create sosapi bucket");
let owner_call = |method: Method, uri: String, headers: Vec<(&'static str, String)>, body: Bytes| {
(method, uri, headers, body)
let owner_call = |method: Method, uri: String, headers: Vec<(&str, String)>, body: Bytes| {
let mut builder = http::Request::builder().method(method).uri(uri).header("host", "localhost");
for (name, value) in headers {
builder = builder.header(name, value);
}
builder
.extension(crate::storage::req_info::ReqInfo {
cred: Some(rustfs_credentials::Credentials::default()),
is_owner: true,
..Default::default()
})
.body(Body::from(body))
.expect("HTTP request")
};
let system_uri = format!("/{bucket}/{SYSTEM_KEY}");
@@ -1133,16 +1143,17 @@ fn veeam_sosapi_http_contract() {
.expect("list versions");
assert!(versions.objects.is_empty());
let anon = gateway
let anon = anonymous
.call(
http::Request::builder()
.method(Method::GET)
.uri(system_uri.clone())
.header("host", "localhost")
.body(rustfs_gateway::Body::empty())
.body(Body::empty())
.expect("anon request"),
)
.await;
.await
.expect("anonymous GET");
let anon_status = anon.status();
let anon_body = BodyExt::collect(anon.into_body()).await.expect("anon body").to_bytes();
let anon_xml = String::from_utf8_lossy(&anon_body);
@@ -1152,7 +1163,7 @@ fn veeam_sosapi_http_contract() {
let now = OffsetDateTime::now_utc();
let scope_date = format!("{:04}{:02}{:02}", now.year(), u8::from(now.month()), now.day());
let amz_date = format!("{scope_date}T{:02}{:02}{:02}Z", now.hour(), now.minute(), now.second());
let bad = gateway
let bad = bad_auth
.call(
http::Request::builder()
.method(Method::GET)
@@ -1166,10 +1177,11 @@ fn veeam_sosapi_http_contract() {
"AWS4-HMAC-SHA256 Credential=bad/{scope_date}/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=0000000000000000000000000000000000000000000000000000000000000000"
),
)
.body(rustfs_gateway::Body::empty())
.body(Body::empty())
.expect("bad auth request"),
)
.await;
.await
.expect("bad auth GET");
let bad_status = bad.status();
let bad_body = BodyExt::collect(bad.into_body()).await.expect("bad body").to_bytes();
let bad_xml = String::from_utf8_lossy(&bad_body);
+1 -1
View File
@@ -69,7 +69,7 @@ impl ObjectMetaUsecase {
.get_bucket_info(bucket, &BucketOptions::default())
.await
.map_err(ApiError::from)?;
return Err(err);
return Err(err.into());
}
let opts: ObjectOptions = get_opts(bucket, key, version_id.clone(), None, envelope.headers())
+1 -44
View File
@@ -101,7 +101,7 @@ mod tests {
use http::{Extensions, HeaderMap, Method, Uri};
use rustfs_gateway_types::dto::{
DeleteObjectTaggingInput, GetObjectAclInput, GetObjectLegalHoldInput, GetObjectRetentionInput, GetObjectTaggingInput,
PutObjectAclInput, Tag,
PutObjectAclInput,
};
use rustfs_gateway_types::{BucketName, ObjectKey};
@@ -185,27 +185,6 @@ mod tests {
)
.await
.map(drop),
usecase
.put_object_retention_fields(envelope(), bucket.as_str(), key.as_str(), None, None)
.await
.map(drop),
usecase
.put_object_legal_hold_fields(envelope(), bucket.as_str(), key.as_str(), None, None)
.await
.map(drop),
usecase
.put_object_tagging_fields(
envelope(),
bucket.as_str(),
key.as_str(),
None,
vec![Tag {
key: Some("project".into()),
value: Some("test".into()),
}],
)
.await
.map(drop),
usecase
.execute_delete_object_tagging(
envelope(),
@@ -224,26 +203,4 @@ mod tests {
assert_eq!(error.code(), &S3ErrorCode::InternalError, "use case {index}");
}
}
#[tokio::test]
async fn invalid_put_tags_are_rejected_before_version_and_store_lookup() {
let usecase = ObjectMetaUsecase::with_context(None);
let tag = |key: Option<&str>, value: Option<&str>| Tag {
key: key.map(str::to_owned),
value: value.map(str::to_owned),
};
let too_many = (0..11).map(|index| tag(Some(&format!("k{index}")), Some("value"))).collect();
for (tags, expected) in [
(too_many, "Cannot have more than 10 tags per object"),
(vec![tag(Some(""), Some("v1"))], "Tag key cannot be empty"),
(vec![tag(None, Some("value"))], "Tag key cannot be empty"),
(vec![tag(Some("key"), None)], "Tag value cannot be null"),
] {
let error = usecase
.put_object_tagging_fields(envelope(), "test-bucket", "test-key", Some("invalid".into()), tags)
.await
.unwrap_err();
assert_eq!(error.code(), &S3ErrorCode::InvalidTag);
assert_eq!(error.message(), Some(expected));
}
}
}
+276 -93
View File
@@ -19,16 +19,19 @@
use crate::app::RequestEnvelope;
use crate::app::storage_api::context::MultipartStagingOperation;
use crate::app::storage_api::owned_parity::FS;
use crate::app::storage_api::owned_parity::parity_test_support as support;
use crate::app::storage_api::owned_parity::{DenialAuditor, DenialRateLimiter};
use crate::app::storage_api::owned_parity::{
PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource, RequestedObjectLegalHold,
RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
ObjectTagConditions, PostObjectPolicyHeaders, PostObjectRequestMarker, ReqInfo, RequestedCopySource,
RequestedObjectLegalHold, RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags,
TableDataPlanePublicationGuards,
};
use crate::app::storage_api::owned_parity::{authorize_create_bucket, authorize_delete_bucket};
use crate::app::storage_api::s3::{
BucketLocationConstraint, CreateBucketConfiguration, CreateBucketInput, CreateMultipartUploadInput, DeleteObjectInput,
ObjectLockRetention, PutObjectInput, S3Error, S3ErrorCode, S3Request,
ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, PutObjectInput, S3Access, S3Error, S3ErrorCode,
S3Request, Tag, Tagging,
};
use crate::auth::facts::{AuthzPrincipal, AuthzTarget};
use crate::storage::storage_api::contract::bucket::{BucketOperations as _, MakeBucketOptions};
@@ -341,19 +344,31 @@ async fn owned_authorization_reloads_tags_when_version_changes() {
let mut owned = crate::app::RequestEnvelope::from_s3s(&legacy);
for (index, allowed, expected_tag) in [(0, true, "allowed"), (1, false, "denied"), (0, true, "allowed")] {
let version = &versions[index];
legacy.extensions.get_mut::<ReqInfo>().expect("legacy info").version_id = Some(version.clone());
owned.req_info_mut().expect("owned info").version_id = Some(version.clone());
let action = Action::S3Action(S3Action::PutObjectAction);
let legacy_result = crate::app::storage_api::owned_parity::authorize_request(&mut legacy, action)
.await
.map_err(S3Error::from);
let owned_result = crate::app::authorize_owned_request(&mut owned, action, None).await;
if allowed {
legacy_result.expect("legacy version allowed");
owned_result.expect("owned version allowed");
} else {
assert_eq!(legacy_result.expect_err("legacy version denied").code(), &S3ErrorCode::AccessDenied);
assert_eq!(
owned_result.expect_err("owned version denied").code(),
&rustfs_s3_types::S3ErrorCode::AccessDenied
);
}
let old = legacy.extensions.get::<ObjectTagConditions>().expect("legacy cache");
let new = owned.object_tag_conditions().expect("owned cache");
assert_eq!(support::tag_conditions_parts(old).2, Some(version.as_str()));
assert_eq!(support::tag_conditions_parts(new).2, Some(version.as_str()));
assert_eq!(
support::tag_conditions_parts(old).3.get("ExistingObjectTag/classification"),
Some(&vec![expected_tag.to_owned()])
);
assert_eq!(
support::tag_conditions_parts(new).3.get("ExistingObjectTag/classification"),
Some(&vec![expected_tag.to_owned()])
@@ -371,6 +386,7 @@ async fn table_publication_guard_timeout_blocks_put_delete_and_multipart_authori
let store = crate::app::gating_test_env::shared_gating_ecstore().await;
let server_ctx = ServerContextSlot::new();
assert!(server_ctx.install(Arc::new(AppContext::new(Arc::clone(&store), Arc::new(UnreadyIam), Arc::new(TestKms)))));
let fs = FS::with_server_ctx(server_ctx);
let bucket = format!("publication-timeout-{}", uuid::Uuid::new_v4());
store
.make_bucket(&bucket, &MakeBucketOptions::default())
@@ -421,46 +437,44 @@ async fn table_publication_guard_timeout_blocks_put_delete_and_multipart_authori
.expect("DELETE input should build"),
Method::DELETE,
);
ensure_req_info(&mut put_req);
ensure_req_info(&mut delete_req);
delete_req.extensions.insert(fs.server_ctx().clone());
ensure_req_info(&mut put_req);
ensure_req_info(&mut multipart_req);
put_req.extensions.insert(server_ctx.clone());
delete_req.extensions.insert(server_ctx.clone());
multipart_req.extensions.insert(server_ctx);
let mut owned_put = crate::app::RequestEnvelope::from_s3s(&put_req);
let owned_info = owned_put.req_info_mut().expect("owned request info");
put_req.extensions.insert(fs.server_ctx().clone());
multipart_req.extensions.insert(fs.server_ctx().clone());
let mut owned_req = crate::app::RequestEnvelope::from_s3s(&put_req);
let owned_info = owned_req.req_info_mut().expect("owned request info");
owned_info.bucket = Some(bucket.clone());
owned_info.object = Some("object".to_string());
let mut owned_delete = crate::app::RequestEnvelope::from_s3s(&delete_req);
let delete_info = owned_delete.req_info_mut().expect("owned delete info");
delete_info.bucket = Some(bucket.clone());
delete_info.object = Some("object".to_string());
let mut owned_multipart = crate::app::RequestEnvelope::from_s3s(&multipart_req);
let multipart_info = owned_multipart.req_info_mut().expect("owned multipart info");
multipart_info.bucket = Some(bucket.clone());
multipart_info.object = Some("object".to_string());
let owned_err = crate::app::authorize_owned_request(&mut owned_req, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect_err("owned PUT must wait for the publication writer");
assert_eq!(owned_err.code(), &rustfs_s3_types::S3ErrorCode::ServiceUnavailable);
assert!(owned_req.table_publication_guards().is_none());
for (operation, err) in [
(
"PutObject",
crate::app::authorize_owned_request(&mut owned_put, Action::S3Action(S3Action::PutObjectAction), None)
fs.put_object(&mut put_req)
.await
.expect_err("owned PUT must wait for the publication writer"),
.expect_err("PUT must wait for the publication writer"),
),
(
"DeleteObject",
crate::app::authorize_owned_request(&mut owned_delete, Action::S3Action(S3Action::DeleteObjectAction), None)
fs.delete_object(&mut delete_req)
.await
.expect_err("owned DELETE must wait for the publication writer"),
.expect_err("DELETE must wait for the publication writer"),
),
(
"CreateMultipartUpload",
crate::app::authorize_owned_request(&mut owned_multipart, Action::S3Action(S3Action::PutObjectAction), None)
fs.create_multipart_upload(&mut multipart_req)
.await
.expect_err("owned multipart initialization must wait for the publication writer"),
.expect_err("multipart initialization must wait for the publication writer"),
),
] {
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::ServiceUnavailable, "{operation}: {err}");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable, "{operation}: {err}");
let source = err.source().expect("retain the typed publication lock failure");
let storage = source
.downcast_ref::<crate::storage_api::error::StorageError>()
@@ -469,7 +483,7 @@ async fn table_publication_guard_timeout_blocks_put_delete_and_multipart_authori
storage,
crate::storage_api::error::StorageError::Lock(rustfs_lock::LockError::Timeout { .. })
));
let response = S3Error::from(err).to_http_response().expect("serialize publication failure");
let response = err.to_http_response().expect("serialize publication failure");
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
let body = response.into_body().collect().await.expect("read error XML").to_bytes();
@@ -478,32 +492,45 @@ async fn table_publication_guard_timeout_blocks_put_delete_and_multipart_authori
assert!(!body.contains(&bucket));
assert!(!body.contains(&lock_object));
}
assert!(owned_put.table_publication_guards().is_none());
assert!(owned_delete.table_publication_guards().is_none());
assert!(owned_multipart.table_publication_guards().is_none());
for extensions in [&put_req.extensions, &delete_req.extensions, &multipart_req.extensions] {
assert!(extensions.get::<TableDataPlanePublicationGuards>().is_none());
}
assert!(writer.release());
for _ in 0..2 {
for req in [&mut owned_put, &mut owned_multipart] {
crate::app::authorize_owned_request(req, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect("owned PUT may retry after publication finishes");
}
crate::app::authorize_owned_request(&mut owned_delete, Action::S3Action(S3Action::DeleteObjectAction), None)
crate::app::authorize_owned_request(&mut owned_req, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect("owned DELETE may retry after publication finishes");
for req in [&owned_put, &owned_delete, &owned_multipart] {
let retained = req
.table_publication_guards()
.expect("owned request retains the publication guard");
.expect("owned PUT may retry after publication finishes");
let retained = owned_req
.table_publication_guards()
.expect("owned request retains the publication guard");
{
let (held, guards) = support::publication_guard_holds(retained, &(bucket.clone(), lock_object.clone()));
assert!(held);
assert_eq!(guards, 1, "repeated owned admission must reuse its retained guard");
}
fs.put_object(&mut put_req)
.await
.expect("PUT may retry after publication finishes");
fs.create_multipart_upload(&mut multipart_req)
.await
.expect("multipart initialization may retry after publication finishes");
fs.delete_object(&mut delete_req)
.await
.expect("DELETE may retry after publication finishes");
for extensions in [&put_req.extensions, &delete_req.extensions, &multipart_req.extensions] {
let retained = extensions
.get::<TableDataPlanePublicationGuards>()
.expect("successful admission retains the guard");
let (held, guards) = support::publication_guard_holds(retained, &(bucket.clone(), lock_object.clone()));
assert!(held);
assert_eq!(guards, 1, "repeated admission must reuse its retained guard");
}
}
drop(owned_put);
drop(owned_multipart);
drop(owned_delete);
drop(put_req);
drop(multipart_req);
drop(delete_req);
drop(owned_req);
lock.get_write_lock(Duration::from_secs(1))
.await
.expect("dropping the request must release its publication guard");
@@ -589,18 +616,19 @@ async fn owned_and_legacy_create_bucket_authorization_use_location_constraint()
bucket: Some("location-policy-bucket".to_owned()),
..Default::default()
});
let location = legacy
.input
.create_bucket_configuration
.as_ref()
.and_then(|c| c.location_constraint.as_ref())
.map(|l| l.as_str().to_owned());
let location = support::requested_bucket_location_constraint(&legacy.input);
let mut owned = crate::app::RequestEnvelope::from_s3s(&legacy);
let action = Action::S3Action(S3Action::CreateBucketAction);
let legacy_result = support::authorize_request_with_location_constraint(&mut legacy, action, location.clone()).await;
let owned_result = crate::app::authorize_owned_request(&mut owned, action, location).await;
if allowed {
legacy_result.expect("legacy allows the matched location");
owned_result.expect("owned allows the matched location");
} else {
assert_eq!(
legacy_result.expect_err("legacy denies the mismatched location").code(),
&S3ErrorCode::AccessDenied
);
assert_eq!(
owned_result.expect_err("owned denies the mismatched location").code(),
&rustfs_s3_types::S3ErrorCode::AccessDenied,
@@ -608,6 +636,9 @@ async fn owned_and_legacy_create_bucket_authorization_use_location_constraint()
}
}
// The whole composite hook: the legacy `S3Access` hook against what the gateway edge runs on
// the facts it decodes, with the same decision and the same bucket handed on, both ways.
let fs = FS::with_server_ctx(Arc::clone(&slot));
for (bucket, region, object_lock, allowed) in [
("location-policy-bucket", Some("eu-west-1"), false, true),
("location-policy-bucket", Some("us-west-2"), false, false),
@@ -640,14 +671,19 @@ async fn owned_and_legacy_create_bucket_authorization_use_location_constraint()
let location = crate::app::bucket_usecase::requested_location_constraint(&input).map(str::to_owned);
let lock = input.object_lock_enabled_for_bucket.is_some_and(|enabled| enabled);
let legacy_result = fs.create_bucket(&mut legacy).await;
let owned_result = authorize_create_bucket(&mut owned, input.bucket.as_str(), location, lock).await;
let case = format!("{bucket} {region:?} lock={object_lock}");
assert_eq!(legacy_result.is_ok(), allowed, "legacy: {case}");
assert_eq!(owned_result.is_ok(), allowed, "owned: {case}");
if let Err(owned_error) = &owned_result {
assert_eq!(owned_error.code().as_str(), "AccessDenied", "{case}");
if let (Err(legacy_error), Err(owned_error)) = (&legacy_result, &owned_result) {
assert_eq!(legacy_error.code().as_str(), owned_error.code().as_str(), "{case}");
}
assert_eq!(owned.req_info().and_then(|info| info.bucket.clone()), Some(bucket.to_owned()), "{case}");
let handed_on = |req_info: Option<&ReqInfo>| req_info.and_then(|info| info.bucket.clone());
assert_eq!(handed_on(legacy.extensions.get::<ReqInfo>()), Some(bucket.to_owned()), "{case}");
assert_eq!(handed_on(owned.req_info()), Some(bucket.to_owned()), "{case}");
assert_eq!(legacy.headers, *owned.headers(), "{case}");
}
}
@@ -661,6 +697,7 @@ async fn owned_and_legacy_delete_bucket_authorization_agree() {
br#"{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":"s3:DeleteBucket","Resource":["arn:aws:s3:::plain-delete-bucket","arn:aws:s3:::force-delete-bucket"]},{"Effect":"Allow","Action":"s3:ForceDeleteBucket","Resource":"arn:aws:s3:::force-delete-bucket"}]}"#,
)
.await;
let fs = FS::with_server_ctx(Arc::clone(&slot));
for (bucket, header, refusal) in [
("plain-delete-bucket", None, None),
("plain-delete-bucket", Some(("x-rustfs-force-delete", "false")), Some("AccessDenied")),
@@ -685,11 +722,16 @@ async fn owned_and_legacy_delete_bucket_authorization_agree() {
..Default::default()
});
let mut owned = crate::app::RequestEnvelope::from_s3s(&legacy);
let legacy_result = fs.delete_bucket(&mut legacy).await;
let owned_result = authorize_delete_bucket(&mut owned, bucket).await;
let case = format!("{bucket} {header:?}");
assert_eq!(legacy_result.as_ref().err().map(|error| error.code().as_str()), refusal, "legacy: {case}");
assert_eq!(owned_result.as_ref().err().map(|error| error.code().as_str()), refusal, "owned: {case}");
assert_eq!(owned.req_info().and_then(|info| info.bucket.clone()), Some(bucket.to_owned()), "{case}");
let handed_on = |req_info: Option<&ReqInfo>| req_info.and_then(|info| info.bucket.clone());
assert_eq!(handed_on(legacy.extensions.get::<ReqInfo>()), Some(bucket.to_owned()), "{case}");
assert_eq!(handed_on(owned.req_info()), Some(bucket.to_owned()), "{case}");
}
}
@@ -706,8 +748,11 @@ fn owned_and_legacy_conditions_read_the_same_request_body_policy_inputs() {
};
let conditions_of = |req: &S3Request<()>, action| {
let owned = crate::app::RequestEnvelope::from_s3s(req);
support::authorization_conditions(&owned, &principal, &support::request_authz_facts(&owned, target(action)))
.expect("owned conditions")
let legacy = support::authorization_conditions(req, &principal, &support::request_authz_facts(req, target(action)))
.expect("legacy conditions");
let owned = support::authorization_conditions(&owned, &principal, &support::request_authz_facts(&owned, target(action)))
.expect("owned conditions");
(legacy, owned)
};
let values = |conditions: &HashMap<String, Vec<String>>, key: &str| conditions.get(key).cloned();
@@ -719,7 +764,10 @@ fn owned_and_legacy_conditions_read_the_same_request_body_policy_inputs() {
form.insert("x-amz-tagging", HeaderValue::from_static("source=form"));
post.extensions.insert(PostObjectPolicyHeaders(form));
post.extensions.insert(RequestedCopySource("src-bucket/src-key".to_owned()));
let owned = conditions_of(&post, S3Action::PutObjectAction);
let (legacy, owned) = conditions_of(&post, S3Action::PutObjectAction);
for key in ["RequestObjectTag/source", "RequestObjectTagKeys", "x-amz-copy-source"] {
assert_eq!(values(&owned, key), values(&legacy, key), "{key}");
}
assert_eq!(values(&owned, "RequestObjectTag/source"), Some(vec!["form".to_owned()]));
assert_eq!(values(&owned, "x-amz-copy-source"), Some(vec!["src-bucket/src-key".to_owned()]));
@@ -728,10 +776,20 @@ fn owned_and_legacy_conditions_read_the_same_request_body_policy_inputs() {
body.headers.insert("x-amz-tagging", HeaderValue::from_static("body=header"));
body.headers
.insert("x-amz-object-lock-legal-hold", HeaderValue::from_static("OFF"));
body.extensions.insert(RequestedObjectLegalHold(Some("ON".to_owned())));
body.extensions
.insert(RequestedObjectTags(vec![("body".to_owned(), "xml".to_owned())]));
let owned = conditions_of(&body, S3Action::PutObjectAction);
let legal_hold = ObjectLockLegalHold {
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)),
};
body.extensions.insert(RequestedObjectLegalHold::from(&legal_hold));
body.extensions.insert(RequestedObjectTags::from(&Tagging {
tag_set: vec![Tag {
key: Some("body".to_owned()),
value: Some("xml".to_owned()),
}],
}));
let (legacy, owned) = conditions_of(&body, S3Action::PutObjectAction);
for key in ["object-lock-legal-hold", "RequestObjectTag/body", "RequestObjectTagKeys"] {
assert_eq!(values(&owned, key), values(&legacy, key), "{key}");
}
assert_eq!(values(&owned, "object-lock-legal-hold"), Some(vec!["ON".to_owned()]));
assert_eq!(values(&owned, "RequestObjectTag/body"), Some(vec!["xml".to_owned()]));
}
@@ -760,22 +818,52 @@ fn installed_body_inputs_describe_the_request_the_same_on_both_forms() {
legacy
.headers
.insert("x-amz-object-lock-mode", HeaderValue::from_static("COMPLIANCE"));
let tagging = Tagging {
tag_set: vec![
Tag {
key: Some("class".to_owned()),
value: Some("public".to_owned()),
},
// A tag without a value names no condition, on either form.
Tag {
key: Some("incomplete".to_owned()),
value: None,
},
],
};
let retention = ObjectLockRetention {
mode: Some("GOVERNANCE".to_owned().into()),
retain_until_date: None,
};
let mut owned = RequestEnvelope::from_s3s(&legacy);
owned.install_requested_object_tags(RequestedObjectTags(vec![("class".to_owned(), "public".to_owned())]));
legacy.install_requested_object_tags(RequestedObjectTags::from(&tagging));
owned.install_requested_object_tags(RequestedObjectTags::from(&tagging));
legacy.install_requested_legal_hold(RequestedObjectLegalHold::default());
owned.install_requested_legal_hold(RequestedObjectLegalHold::default());
owned.install_requested_retention(RequestedObjectLockRetention {
mode: Some("GOVERNANCE".to_owned()),
retain_until: None,
});
legacy.install_requested_retention(RequestedObjectLockRetention::from(&retention));
owned.install_requested_retention(RequestedObjectLockRetention::from(&retention));
for action in [
S3Action::PutObjectTaggingAction,
S3Action::PutObjectLegalHoldAction,
S3Action::PutObjectRetentionAction,
] {
let legacy_conditions =
support::authorization_conditions(&legacy, &principal, &support::request_authz_facts(&legacy, target(action)))
.expect("legacy conditions");
let owned_conditions =
support::authorization_conditions(&owned, &principal, &support::request_authz_facts(&owned, target(action)))
.expect("owned conditions");
for key in [
"RequestObjectTag/class",
"RequestObjectTag/incomplete",
"RequestObjectTag/from",
"RequestObjectTagKeys",
"object-lock-legal-hold",
"object-lock-mode",
] {
assert_eq!(owned_conditions.get(key), legacy_conditions.get(key), "{action:?} {key}");
}
// The installed body, not the headers, describes the request.
assert_eq!(owned_conditions.get("RequestObjectTagKeys"), Some(&vec!["class".to_owned()]));
assert_eq!(owned_conditions.get("RequestObjectTag/from"), None);
@@ -821,14 +909,20 @@ fn create_bucket_location_constraint_conditions_use_only_xml() {
),
..Default::default()
};
let location = req
.input
.create_bucket_configuration
.as_ref()
.and_then(|c| c.location_constraint.as_ref())
.map(|l| l.as_str().to_owned());
let location = support::requested_bucket_location_constraint(&req.input);
let principal = AuthzPrincipal::new(&credentials, "");
let owned = crate::app::RequestEnvelope::from_s3s(&req);
let facts = support::request_authz_facts(
&req,
AuthzTarget {
action: Action::S3Action(S3Action::CreateBucketAction),
bucket: "",
object: "",
version_id: None,
location_constraint: location.as_deref(),
},
);
let conditions = support::authorization_conditions(&req, &principal, &facts).expect("CreateBucket conditions");
let owned_facts = support::request_authz_facts(
&owned,
AuthzTarget {
@@ -841,8 +935,9 @@ fn create_bucket_location_constraint_conditions_use_only_xml() {
);
let owned_conditions =
support::authorization_conditions(&owned, &principal, &owned_facts).expect("owned CreateBucket conditions");
assert_eq!(owned_conditions.get("LocationConstraint"), conditions.get("LocationConstraint"));
assert_eq!(
owned_conditions
conditions
.get("LocationConstraint")
.and_then(|values| values.first())
.map(String::as_str),
@@ -853,6 +948,18 @@ fn create_bucket_location_constraint_conditions_use_only_xml() {
S3Action::PutBucketVersioningAction,
S3Action::GetObjectAction,
] {
let facts = support::request_authz_facts(
&req,
AuthzTarget {
action: Action::S3Action(action),
bucket: "",
object: "",
version_id: None,
location_constraint: expected,
},
);
let conditions = support::authorization_conditions(&req, &principal, &facts).expect("non-CreateBucket conditions");
assert!(!conditions.contains_key("LocationConstraint"), "{action:?}");
let owned_facts = support::request_authz_facts(
&owned,
AuthzTarget {
@@ -913,6 +1020,9 @@ async fn request_slot_keeps_bucket_policy_bound_to_its_store() {
});
request_a.extensions.insert(slot_a);
let mut owned_a = crate::app::RequestEnvelope::from_s3s(&request_a);
crate::app::storage_api::owned_parity::authorize_request(&mut request_a, Action::S3Action(S3Action::PutObjectAction))
.await
.expect("store A policy should authorize store A request");
crate::app::authorize_owned_request(&mut owned_a, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect("store A policy should authorize owned request");
@@ -925,6 +1035,12 @@ async fn request_slot_keeps_bucket_policy_bound_to_its_store() {
});
request_b.extensions.insert(slot_b);
let mut owned_b = crate::app::RequestEnvelope::from_s3s(&request_b);
let err =
crate::app::storage_api::owned_parity::authorize_request(&mut request_b, Action::S3Action(S3Action::PutObjectAction))
.await
.map_err(S3Error::from)
.expect_err("store B policy must deny store B request");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let owned_err = crate::app::authorize_owned_request(&mut owned_b, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect_err("store B policy must deny owned request");
@@ -951,6 +1067,9 @@ async fn request_slot_keeps_bucket_policy_bound_to_its_store() {
});
allowed_request_b.extensions.insert(slot_b);
let mut owned_allowed_b = crate::app::RequestEnvelope::from_s3s(&allowed_request_b);
crate::app::storage_api::owned_parity::authorize_request(&mut allowed_request_b, Action::S3Action(S3Action::PutObjectAction))
.await
.expect("store B policy should allow before its public access block is installed");
crate::app::authorize_owned_request(&mut owned_allowed_b, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect("store B policy should authorize owned request before public access block");
@@ -974,6 +1093,14 @@ async fn request_slot_keeps_bucket_policy_bound_to_its_store() {
});
restricted_request_b.extensions.insert(slot_b);
let mut owned_restricted_b = crate::app::RequestEnvelope::from_s3s(&restricted_request_b);
let err = crate::app::storage_api::owned_parity::authorize_request(
&mut restricted_request_b,
Action::S3Action(S3Action::PutObjectAction),
)
.await
.map_err(S3Error::from)
.expect_err("store B public access block must restrict store B request");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let owned_err =
crate::app::authorize_owned_request(&mut owned_restricted_b, Action::S3Action(S3Action::PutObjectAction), None)
.await
@@ -1017,10 +1144,18 @@ async fn request_slot_keeps_bucket_policy_bound_to_its_store() {
});
tagged_request_b.extensions.insert(slot_b);
let mut owned_tagged_b = crate::app::RequestEnvelope::from_s3s(&tagged_request_b);
crate::app::storage_api::owned_parity::authorize_request(&mut tagged_request_b, Action::S3Action(S3Action::PutObjectAction))
.await
.expect("store B existing object tags should authorize store B request");
crate::app::authorize_owned_request(&mut owned_tagged_b, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect("store B existing object tags should authorize owned request");
let legacy_tags = tagged_request_b
.extensions
.get::<ObjectTagConditions>()
.expect("legacy tag cache");
let owned_tags = owned_tagged_b.object_tag_conditions().expect("owned tag cache");
assert_eq!(support::tag_conditions_parts(owned_tags), support::tag_conditions_parts(legacy_tags));
assert_eq!(
support::tag_conditions_parts(owned_tags)
.3
@@ -1032,8 +1167,8 @@ async fn request_slot_keeps_bucket_policy_bound_to_its_store() {
unready_request.extensions.insert(ServerContextSlot::new());
ensure_req_info(&mut unready_request);
let mut owned_unready = crate::app::RequestEnvelope::from_s3s(&unready_request);
let err = support::request_object_store(&owned_unready).expect_err("an unready request slot must not use an ambient store");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
let err = support::request_object_store(&unready_request).expect_err("an unready request slot must not use an ambient store");
assert_eq!(err.code(), &S3ErrorCode::InternalError);
let owned_err = crate::app::authorize_owned_request(&mut owned_unready, Action::S3Action(S3Action::PutObjectAction), None)
.await
.expect_err("unready request slot must not fall back to an ambient store");
@@ -1071,6 +1206,13 @@ fn get_object() -> rustfs_policy::policy::action::Action {
#[tokio::test]
async fn owned_entry_points_reject_duplicate_signed_metadata_like_the_legacy_edge() {
let query = "x-amz-meta-a=1&x-amz-meta-a=2";
let mut legacy = signed_zip_request(query);
let legacy_err = crate::app::storage_api::owned_parity::authorize_request(&mut legacy, get_object())
.await
.map_err(S3Error::from)
.expect_err("legacy rejects a repeated metadata key");
assert_eq!(legacy_err.code(), &S3ErrorCode::InvalidArgument);
let mut owned = RequestEnvelope::from_s3s(&signed_zip_request(query));
let owned_err = crate::app::authorize_owned_request(&mut owned, get_object(), None)
.await
@@ -1087,13 +1229,21 @@ async fn owned_entry_points_reject_duplicate_signed_metadata_like_the_legacy_edg
#[tokio::test]
async fn owned_entry_points_hoist_signed_metadata_into_the_headers_policy_reads() {
let query = "x-amz-meta-a=1";
let mut legacy = signed_zip_request(query);
// Neither request carries ReqInfo, so both stop at the core after the hoist.
let legacy_err = crate::app::storage_api::owned_parity::authorize_request(&mut legacy, get_object())
.await
.map_err(S3Error::from)
.expect_err("no ReqInfo");
assert_eq!(legacy_err.code(), &S3ErrorCode::InternalError);
let mut owned = RequestEnvelope::from_s3s(&signed_zip_request(query));
let owned_err = crate::app::authorize_owned_request(&mut owned, get_object(), None)
.await
.expect_err("no ReqInfo");
assert_eq!(owned_err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(owned.headers().get("x-amz-meta-a"), Some(&HeaderValue::from_static("1")));
assert_eq!(legacy.headers.get("x-amz-meta-a"), Some(&HeaderValue::from_static("1")));
assert_eq!(owned.headers(), &legacy.headers, "both edges evaluate the same headers");
}
#[tokio::test]
@@ -1428,7 +1578,9 @@ async fn owned_and_legacy_entry_points_reach_the_same_decisions() {
facts: |req| {
req.headers
.insert("x-amz-object-lock-legal-hold", HeaderValue::from_static("OFF"));
req.extensions.insert(RequestedObjectLegalHold(Some("ON".to_owned())));
req.extensions.insert(RequestedObjectLegalHold::from(&ObjectLockLegalHold {
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::ON)),
}));
},
allowed: true,
},
@@ -1441,7 +1593,9 @@ async fn owned_and_legacy_entry_points_reach_the_same_decisions() {
facts: |req| {
req.headers
.insert("x-amz-object-lock-legal-hold", HeaderValue::from_static("ON"));
req.extensions.insert(RequestedObjectLegalHold(Some("OFF".to_owned())));
req.extensions.insert(RequestedObjectLegalHold::from(&ObjectLockLegalHold {
status: Some(ObjectLockLegalHoldStatus::from_static(ObjectLockLegalHoldStatus::OFF)),
}));
},
allowed: false,
},
@@ -1453,8 +1607,12 @@ async fn owned_and_legacy_entry_points_reach_the_same_decisions() {
action: S3Action::PutObjectTaggingAction,
facts: |req| {
req.headers.insert("x-amz-tagging", HeaderValue::from_static("class=secret"));
req.extensions
.insert(RequestedObjectTags(vec![("class".to_owned(), "public".to_owned())]));
req.extensions.insert(RequestedObjectTags::from(&Tagging {
tag_set: vec![Tag {
key: Some("class".to_owned()),
value: Some("public".to_owned()),
}],
}));
},
allowed: true,
},
@@ -1466,8 +1624,12 @@ async fn owned_and_legacy_entry_points_reach_the_same_decisions() {
action: S3Action::PutObjectTaggingAction,
facts: |req| {
req.headers.insert("x-amz-tagging", HeaderValue::from_static("class=public"));
req.extensions
.insert(RequestedObjectTags(vec![("class".to_owned(), "secret".to_owned())]));
req.extensions.insert(RequestedObjectTags::from(&Tagging {
tag_set: vec![Tag {
key: Some("class".to_owned()),
value: Some("secret".to_owned()),
}],
}));
},
allowed: false,
},
@@ -1585,6 +1747,9 @@ async fn owned_and_legacy_entry_points_reach_the_same_decisions() {
// member is authorized: it must reach the same decision.
let mut owned_shape = RequestEnvelope::from_s3s(&legacy).fork_for_authorization();
let action = Action::S3Action(case.action);
let legacy_result = crate::app::storage_api::owned_parity::authorize_request(&mut legacy, action)
.await
.map_err(S3Error::from);
let owned_result = crate::app::authorize_owned_request(&mut owned, action, None).await;
let shape_result = crate::app::storage_api::owned_parity::authorize_object_request(
&mut owned_shape,
@@ -1595,6 +1760,10 @@ async fn owned_and_legacy_entry_points_reach_the_same_decisions() {
)
.await
.map_err(S3Error::from);
let legacy_ok = match &legacy_result {
Ok(()) => case.allowed,
Err(err) => !case.allowed && err.code() == &S3ErrorCode::AccessDenied,
};
let owned_ok = match &owned_result {
Ok(()) => case.allowed,
Err(err) => !case.allowed && err.code() == &rustfs_s3_types::S3ErrorCode::AccessDenied,
@@ -1603,9 +1772,9 @@ async fn owned_and_legacy_entry_points_reach_the_same_decisions() {
Ok(()) => case.allowed,
Err(err) => !case.allowed && err.code() == &S3ErrorCode::AccessDenied,
};
if !owned_ok || !shape_ok {
if !legacy_ok || !owned_ok || !shape_ok {
mismatches.push(format!(
"{} (expected allowed={}): owned {owned_result:?}, object shape {shape_result:?}",
"{} (expected allowed={}): legacy {legacy_result:?}, owned {owned_result:?}, object shape {shape_result:?}",
case.name, case.allowed
));
}
@@ -1619,10 +1788,7 @@ fn retain(req: &mut S3Request<()>, mode: &'static str, days: i64) {
mode: Some(mode.to_owned().into()),
..Default::default()
};
req.extensions.insert(RequestedObjectLockRetention {
mode: retention.mode.map(|m| m.as_str().to_owned()),
retain_until: retention.retain_until_date.map(time::OffsetDateTime::from),
});
req.extensions.insert(RequestedObjectLockRetention::from(&retention));
req.extensions.insert(RequestedObjectLockRetainUntil(
time::OffsetDateTime::now_utc() + time::Duration::days(days),
));
@@ -1737,6 +1903,9 @@ async fn owned_and_legacy_bucket_config_mutations_hand_on_the_same_decision_and_
let mut owned = RequestEnvelope::from_s3s(&legacy);
let action = Action::S3Action(action);
let legacy_result = authorize_bucket_config_mutation(&slot, &mut legacy, bucket, action)
.await
.map_err(S3Error::from);
let owned_result = authorize_bucket_config_mutation(&slot, &mut owned, bucket, action)
.await
.map_err(S3Error::from);
@@ -1745,6 +1914,10 @@ async fn owned_and_legacy_bucket_config_mutations_hand_on_the_same_decision_and_
Err(S3ErrorCode::AccessDenied) => MutationOutcome::Denied,
Err(code) => MutationOutcome::Failed(code.as_str().to_owned()),
};
let legacy_outcome = outcome(
legacy_result.map_err(|err| err.code().clone()),
bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone().into()),
);
let owned_outcome = outcome(
owned_result.map_err(|err| err.code().clone()),
bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone().into()),
@@ -1753,11 +1926,18 @@ async fn owned_and_legacy_bucket_config_mutations_hand_on_the_same_decision_and_
let refused_without_snapshot = |snapshot: Result<Option<uuid::Uuid>, S3ErrorCode>| {
matches!(expected, MutationOutcome::Allowed(_)) || snapshot == Err(S3ErrorCode::InternalError)
};
let both_refused_without_snapshot =
refused_without_snapshot(bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone().into()));
let bucket_recorded = owned.req_info().and_then(|info| info.bucket.as_deref()) == Some(bucket);
if owned_outcome != expected || !both_refused_without_snapshot || !bucket_recorded {
mismatches.push(format!("{name}: expected {expected:?}, owned {owned_outcome:?}"));
let both_refused_without_snapshot = refused_without_snapshot(
bucket_config_mutation_incarnation(&legacy, bucket).map_err(|err| err.code().clone().into()),
) && refused_without_snapshot(
bucket_config_mutation_incarnation(&owned, bucket).map_err(|err| err.code().clone().into()),
);
let bucket_recorded = [legacy.extensions.get::<ReqInfo>(), owned.req_info()]
.iter()
.all(|info| info.and_then(|info| info.bucket.as_deref()) == Some(bucket));
if legacy_outcome != expected || owned_outcome != expected || !both_refused_without_snapshot || !bucket_recorded {
mismatches.push(format!(
"{name}: expected {expected:?}, legacy {legacy_outcome:?}, owned {owned_outcome:?}"
));
}
}
assert!(mismatches.is_empty(), "bucket config mutation mismatches:\n{}", mismatches.join("\n"));
@@ -1989,9 +2169,12 @@ async fn owned_and_legacy_write_checks_hand_on_the_same_decision_and_state() {
});
facts(&mut legacy);
let mut owned = RequestEnvelope::from_s3s(&legacy);
let legacy_outcome = run_write_shape(shape, &mut legacy).await;
let owned_outcome = run_write_shape(shape, &mut owned).await;
if owned_outcome != expected {
mismatches.push(format!("{name}: expected {expected:?}, owned {owned_outcome:?}"));
if legacy_outcome != expected || owned_outcome != expected {
mismatches.push(format!(
"{name}: expected {expected:?}, legacy {legacy_outcome:?}, owned {owned_outcome:?}"
));
}
}
assert!(mismatches.is_empty(), "write check mismatches:\n{}", mismatches.join("\n"));
+3 -3
View File
@@ -4,8 +4,8 @@ use super::storage_api::select_object::options::get_opts;
use super::storage_api::select_object::request_context::spawn_traced;
use super::storage_api::select_object::sse::{SseKmsPrincipal, authorize_sse_kms_object_read, project_sse_read_response_headers};
use super::storage_api::select_object::{
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
validate_ssec_for_read,
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
validate_sse_headers_for_read, validate_ssec_for_read,
};
use crate::app::runtime_sources::current_s3select_db;
use crate::app::{Answered, RequestEnvelope};
@@ -874,7 +874,7 @@ async fn prepare_select_object_snapshot(
let opts = get_opts(&input.bucket, &input.key, None, None, headers)
.await
.map_err(ApiError::from)?;
let store = get_validated_store(&input.bucket).await?;
let store = get_validated_store_owned(&input.bucket).await?;
let snapshot = store
.prepare_select_object_snapshot(&input.bucket, &input.key, headers, &opts)
.await
-4
View File
@@ -18,16 +18,12 @@
//! The select use case validates it before building the engine.
//! The s3s conversion is deleted with the legacy edge (rustfs/backlog#2783).
#[cfg(test)]
use rustfs_gateway_types::compat::ConversionError;
#[cfg(test)]
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::shapes::input_serialization::input_serialization_from_s3s;
#[cfg(test)]
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::shapes::output_serialization::output_serialization_from_s3s;
use rustfs_s3select_api::request as engine;
/// Converts the decoded request, keeping the SSE-C members as the client sent them.
#[cfg(test)]
pub(crate) fn select_input_from_s3s(
input: s3s::dto::SelectObjectContentInput,
) -> Result<engine::SelectObjectContentInput, ConversionError> {
+30 -24
View File
@@ -26,11 +26,11 @@ pub(crate) fn EndpointServerPools(
/// Streaming body types the request-body adapters implement against.
pub(crate) mod request_body {
pub(crate) use rustfs_s3_types::StdError;
#[cfg(test)]
pub(crate) use s3s::StdError;
pub(crate) use s3s::dto::StreamingBlob;
pub(crate) use s3s::stream::ByteStream;
#[cfg(test)]
pub(crate) use s3s::stream::{ByteStream, DynByteStream, RemainingLength};
pub(crate) use s3s::stream::{DynByteStream, RemainingLength};
}
#[cfg(feature = "diagnostic-read-version-caller")]
@@ -54,9 +54,12 @@ pub(crate) mod s3 {
PersistedOptionalReplicationStatus, PersistedReplicationConfiguration, PersistedReplicationDestination,
PersistedReplicationFilter, PersistedReplicationRule, PersistedReplicationTag,
};
pub(crate) use rustfs_s3_types::StdError;
#[cfg(any(test, all(any(feature = "cuobject", feature = "nixl"), target_os = "linux")))]
pub(crate) use s3s::Body;
pub(crate) use s3s::S3Response;
pub(crate) use s3s::StdError;
#[cfg(test)]
pub(crate) use s3s::access::S3Access;
#[cfg(test)]
pub(crate) use s3s::auth::{Credentials, SimpleAuth};
#[cfg(test)]
@@ -65,7 +68,7 @@ pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::dto::{
BucketLocationConstraint, CreateBucketConfiguration, CreateBucketInput, CreateMultipartUploadInput, DeleteObjectInput,
ObjectLockRetention,
ObjectLockLegalHold, ObjectLockLegalHoldStatus, ObjectLockRetention, Tag, Tagging,
};
#[cfg(test)]
pub(crate) use s3s::dto::{GetObjectInput, HeadObjectInput, ListObjectsV2Input, ListObjectsV2Output};
@@ -84,10 +87,8 @@ pub(crate) mod s3 {
#[cfg(test)]
pub(crate) use s3s::xml::{Serialize as XmlSerialize, Serializer as XmlSerializer};
#[cfg(test)]
pub(crate) use s3s::{BodySizeLimitExceeded, S3, TrailingHeaders};
pub(crate) use s3s::{S3Error, S3ErrorCode, S3Result};
#[cfg(test)]
pub(crate) use s3s::{S3Request, S3Response};
pub(crate) use s3s::{BodySizeLimitExceeded, S3};
pub(crate) use s3s::{S3Error, S3ErrorCode, S3Request, S3Result, TrailingHeaders};
}
pub(crate) mod admin {
@@ -392,7 +393,6 @@ pub(crate) mod bucket {
}
// The legacy edge keeps values that its decoder permits before domain validation.
#[cfg(test)]
pub(crate) fn restore_request_from_wire(request: s3s::dto::RestoreRequest) -> RestoreRequest {
use rustfs_gateway_types::dto;
@@ -1569,7 +1569,9 @@ pub(crate) mod bucket {
}
pub(crate) mod tagging {
pub(crate) use crate::storage::storage_api::ecstore_bucket::tagging::decode_tags;
pub(crate) fn decode_tags(tags: &str) -> Vec<s3s::dto::Tag> {
crate::storage::storage_api::decode_tags(tags)
}
}
pub(crate) mod target {
@@ -1611,6 +1613,11 @@ pub(crate) mod deadlock_detector {
pub(crate) use crate::storage::storage_api::deadlock_detector_consumer::{DeadlockDetector, get_deadlock_detector};
}
#[cfg(test)]
pub(crate) mod ecfs {
pub(crate) use crate::storage::storage_api::FS;
}
pub(crate) mod error {
pub(crate) use crate::storage::storage_api::{
StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found,
@@ -1816,12 +1823,12 @@ pub(crate) mod object_usecase {
}
}
#[cfg(test)]
pub(crate) use super::object_utils;
pub(crate) use super::{
access, bucket, compression, concurrency, data_usage, deadlock_detector, error, head_prefix, helper, io, options,
request_context, s3_api, set_disk, sse, storage_class, timeout_wrapper,
};
#[cfg(test)]
pub(crate) use super::{ecfs, object_utils};
pub(crate) use crate::storage::storage_api::{
DEFAULT_EC_BLOCK_SIZE, EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX, EC_BLOCK_SIZE_HINT_METADATA_KEY,
EC_BLOCK_SIZE_HINT_REASON_HEADER, EC_BLOCK_SIZE_HINT_STATUS_HEADER, EC_BLOCK_SIZE_RESPONSE_HEADER, ECStore,
@@ -1882,10 +1889,8 @@ pub(crate) mod multipart_usecase {
}
}
#[cfg(test)]
pub(crate) use super::object_utils;
pub(crate) use super::{
access, bucket, compression, data_usage, error, helper, io, options, request_context, s3_api, set_disk, sse,
access, bucket, compression, data_usage, error, helper, io, object_utils, options, request_context, s3_api, set_disk, sse,
};
pub(crate) use crate::storage::storage_api::{
DEFAULT_EC_BLOCK_SIZE, EC_BLOCK_SIZE_HINT_INTERNAL_SUFFIX, EC_BLOCK_SIZE_HINT_METADATA_KEY,
@@ -1900,8 +1905,8 @@ pub(crate) mod select_object {
#[cfg(test)]
pub(crate) use crate::storage::storage_api::StorageError;
pub(crate) use crate::storage::storage_api::{
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store, validate_sse_headers_for_read,
validate_ssec_for_read,
StoragePrepareSelectObjectSnapshotError, StorageSelectObjectSnapshot, get_validated_store_owned,
validate_sse_headers_for_read, validate_ssec_for_read,
};
}
@@ -1909,11 +1914,12 @@ pub(crate) mod select_object {
#[cfg(test)]
pub(crate) mod owned_parity {
pub(crate) use crate::storage::storage_api::owned_parity_consumer::{
AuthorizationRequest, DenialAuditor, DenialRateLimiter, DenialScope, PostObjectPolicyHeaders, PostObjectRequestMarker,
ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold, RequestedObjectLockRetainUntil,
RequestedObjectLockRetention, RequestedObjectTags, SUPPRESSED_DENIALS_TAG, authorization_denial_entry,
authorization_denial_entry_for_view, authorize_bucket_config_mutation, authorize_create_bucket, authorize_delete_bucket,
authorize_object_request, authorize_replication_only_put_headers, authorize_staged_multipart_part,
AuthorizationRequest, DenialAuditor, DenialRateLimiter, DenialScope, FS, ObjectTagConditions, PostObjectPolicyHeaders,
PostObjectRequestMarker, ReqInfo, RequestContext, RequestedCopySource, RequestedObjectLegalHold,
RequestedObjectLockRetainUntil, RequestedObjectLockRetention, RequestedObjectTags, SUPPRESSED_DENIALS_TAG,
TableDataPlanePublicationGuards, authorization_denial_entry, authorization_denial_entry_for_view,
authorize_bucket_config_mutation, authorize_create_bucket, authorize_delete_bucket, authorize_object_request,
authorize_replication_only_put_headers, authorize_request, authorize_staged_multipart_part,
authorize_write_etag_condition, bucket_config_mutation_incarnation, parity_test_support, replication_request_authorized,
};
}
@@ -1981,7 +1987,7 @@ pub(crate) mod test {
RecordingKmsKeyAuthorizer, TestAuthorizationHooks, sse_kms_object_metadata,
};
}
pub(crate) use super::{bucket, object_utils, runtime};
pub(crate) use super::{bucket, ecfs, object_utils, runtime};
pub(crate) mod data_usage {
pub(crate) use super::super::data_usage::*;
}
+4 -14
View File
@@ -23,20 +23,10 @@ pub(crate) async fn list_objects_v2(
usecase: &DefaultBucketUsecase,
req: S3Request<ListObjectsV2Input>,
) -> S3Result<S3Response<ListObjectsV2Output>> {
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects_v2 as ops;
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
let input = ops::input_from_s3s(req.input).map_err(crate::app::dto_values::seam_error)?;
let mut output = list_objects_v2_owned(usecase, envelope, input).await?;
let migration_list = output.rustfs_on_demand_migration_list.take();
let s3s_output = ops::output_to_s3s(output).map_err(crate::app::dto_values::seam_error)?;
let mut response = S3Response::new(s3s_output);
if let Some(value) = migration_list {
response.headers.insert(
"x-rustfs-on-demand-migration-list",
http::HeaderValue::from_str(&value).expect("valid migration list header"),
);
}
Ok(response)
crate::storage::ecfs::list_objects_v2_through_legacy_edge(req, |envelope, input| {
list_objects_v2_owned(usecase, envelope, input)
})
.await
}
/// Calls `DefaultBucketUsecase::execute_list_objects_v2` with owned inputs.
-6
View File
@@ -15,19 +15,14 @@
//! The single bridge from the HTTP framework's trailer handle to the
//! framework-neutral `rustfs_rio::TrailerSource`.
#[cfg(test)]
use crate::app::storage_api::s3::TrailingHeaders;
#[cfg(test)]
use rustfs_rio::{SharedTrailerSource, TrailerSource, TrailerValue};
#[cfg(test)]
use std::sync::Arc;
// RUSTFS_COMPAT_TODO(s3gate-trailer-adapter): s3s still decodes aws-chunked bodies and publishes their trailers through its own handle, so the object write path adapts that handle to rio's TrailerSource here. Remove after the gateway stack replaces s3s as the request body decoder.
#[cfg(test)]
#[derive(Clone, Debug)]
struct S3sTrailerSource(TrailingHeaders);
#[cfg(test)]
impl TrailerSource for S3sTrailerSource {
fn lookup(&self, name: &str) -> TrailerValue {
// s3s fills the handle after the final chunk has been decoded and
@@ -44,7 +39,6 @@ impl TrailerSource for S3sTrailerSource {
}
/// Adapts the request's trailer handle, if the request declared one.
#[cfg(test)]
pub(crate) fn trailer_source(trailing_headers: Option<TrailingHeaders>) -> Option<SharedTrailerSource> {
trailing_headers.map(|headers| Arc::new(S3sTrailerSource(headers)) as SharedTrailerSource)
}

Some files were not shown because too many files have changed in this diff Show More