Compare commits

..

31 Commits

Author SHA1 Message Date
cxymds 38aa318fb2 fix(iam): revalidate STS sessions during password rotation 2026-11-20 18:51:32 +08:00
cxymds e7c5d7c198 fix(iam): verify the parent password for Console STS rotations 2026-11-20 17:28:52 +08:00
overtrue e37aa27e40 fix(test): encode rollback digests with the workspace hex encoder 2026-11-20 14:54:34 +08:00
overtrue 6e417a458e test(upgrade): exercise the official 1.0.5 gateway baseline 2026-11-20 14:50:43 +08:00
overtrue 60ed7f1c40 test(upgrade): select mixed storage contracts by source revision 2026-11-20 14:32:11 +08:00
overtrue 7f2bd6b99b test(protocols): match the released error for a missing directory marker 2026-11-20 14:23:19 +08:00
overtrue a798df7906 refactor(protocols): prepare part copy routing and storage from one upload ID 2026-11-20 14:23:19 +08:00
overtrue 793632c0da docs(operations): document rollback after the S3 stack switch 2026-11-20 14:23:18 +08:00
overtrue 301133c1c8 test(gateway): check the release torrent error for boundary keys 2026-11-20 14:23:18 +08:00
overtrue a642ba2a89 chore(ci): tighten the s3s import limit to 33 files 2026-11-20 13:04:07 +08:00
overtrue d893d70b2e refactor(storage): return native errors from S3 storage helpers 2026-11-20 13:03:42 +08:00
overtrue cf15e4d1b6 refactor(http): verify control body checksums with the native hashers 2026-11-20 12:35:23 +08:00
overtrue 5a3e94bbbb chore(ci): ratchet the remaining s3s imports to 34 files 2026-11-20 12:21:40 +08:00
overtrue 29b389bf80 refactor(errors): keep API error codes native through storage handling 2026-11-20 12:21:15 +08:00
overtrue f94e8cae07 refactor(storage): use native listing and object lock inputs 2026-11-20 12:17:42 +08:00
overtrue 64675785fe refactor(bucket): return native S3 errors from admission and config writes 2026-11-20 12:01:48 +08:00
overtrue 5b703b61f9 test(ci): align storage guard fixtures with the surviving helper rule 2026-11-20 12:00:53 +08:00
overtrue 23420e357d refactor(multipart): use native S3 errors for upload validation and storage failures 2026-11-20 11:43:15 +08:00
overtrue 9da1edd8d6 test(e2e): compare complete persisted version maps 2026-11-20 11:39:25 +08:00
overtrue d87c058a4d fix(protocols): audit multipart uploads exactly once 2026-11-20 11:12:35 +08:00
overtrue 54cda17e58 refactor(storage): publish internal write events without legacy S3 responses 2026-11-20 11:12:35 +08:00
overtrue 49f6e815e7 refactor(multipart): retire legacy request envelopes from production 2026-11-20 11:12:35 +08:00
overtrue 58bbb17300 fix(gateway): preserve the missing torrent response for invalid keys 2026-11-20 11:01:04 +08:00
overtrue 310b977f05 test(e2e): compare release and candidate persistence bytes 2026-11-20 10:36:43 +08:00
overtrue bbe396219f refactor(storage): validate tags and website settings with native S3 types 2026-11-20 10:33:22 +08:00
overtrue e286691371 test(admin): exercise health and heal on the native router 2026-11-20 10:32:58 +08:00
overtrue 9ff399afee test(scanner): freeze legacy cache compatibility fixtures 2026-11-20 10:23:41 +08:00
overtrue 23c8c35710 refactor(storage): use native listing errors and write preconditions 2026-11-20 10:08:44 +08:00
Chris An 9c44c5ea6b refactor(admin): serve Object Lambda through native GetObject (#9529)
* test(storage): restore POST form metadata policy fixture

* test(storage): follow the current PUT path in contract checks

* fix(storage): verify archive transport EOF before parsing the final frame

* test(rustfs): restore main verification after gateway migration

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* refactor(storage): retire the legacy S3 storage edge

* fix(storage): bound the audited multipart copy future and trace LIST quorum causes

* test(storage): pin legacy codec answers and trim test dependencies

* refactor(admin): confine the legacy route adapter to parity tests

* fix(deps): align bindgen and prettyplease on syn 3 for FIPS builds

* refactor(admin): keep Object Lambda authorization and responses on the owned request path

* fix(storage): expose the typed legacy cause on owned errors

* refactor(admin): read and return native GetObject fields for Object Lambda

* ci: ratchet the legacy S3 footprint after the control-plane migration

* refactor(admin): remove the unused envelope-to-extensions adapter

* test(admin): guard the Object Lambda response token checks at their definition

* refactor(admin): serialize AssumeRole credentials without the legacy DTO

* test(admin): assert native range and error types at the test boundary

* test(admin): compare errors in the legacy route parity tests

* refactor(notify): initialize event rules from persisted notification types

* refactor(admin): return the idle rebalance response on the owned HTTP path

* refactor(storage): keep the persisted-to-legacy XML oracle in tests

* test(admin): compare extension refusals using the native error code

* fix(obs): filter verbose native gateway protocol events at every sink

* test(storage): preserve stream lengths and request facts on retired seams

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:08:35 +08:00
Chris An 31e0798637 refactor(storage): retire the legacy S3 storage edge (#9527)
* test(storage): restore POST form metadata policy fixture

* test(storage): follow the current PUT path in contract checks

* fix(storage): verify archive transport EOF before parsing the final frame

* test(rustfs): restore main verification after gateway migration

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* refactor(storage): retire the legacy S3 storage edge

* fix(storage): bound the audited multipart copy future and trace LIST quorum causes

* fix(deps): align bindgen and prettyplease on syn 3 for FIPS builds

* fix(storage): expose the typed legacy cause on owned errors

* test(storage): preserve stream lengths and request facts on retired seams

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:07:52 +08:00
Chris An 868af666e9 fix(storage): preserve archive errors and restore gateway CI checks (#9525)
* test(storage): restore POST form metadata policy fixture

* test(storage): follow the current PUT path in contract checks

* fix(storage): verify archive transport EOF before parsing the final frame

* test(rustfs): restore main verification after gateway migration

Co-Authored-By: heihutu <heihutu@gmail.com>
Co-Authored-By: zhi22915 <qiuzgang@gmail.com>

* fix(deps): align bindgen and prettyplease on syn 3 for FIPS builds

---------

Co-authored-by: Hauser Zhi <housemecn@gmail.com>
Co-authored-by: heihutu <heihutu@gmail.com>
Co-authored-by: zhi22915 <qiuzgang@gmail.com>
2026-11-20 10:05:46 +08:00
208 changed files with 4489 additions and 11929 deletions
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=24a8cfaa1b9125bd69d7d4a7ff752a8ab8ed350c761d90ae8472ef52393e2686
sha256-linux=014257197fbbfbb48cab2864da9e10d96ee877d478fa29d119f8186532ea293d
sha256-darwin=9ccb03f4139e99ec11a63810523d504df39191b86e74b28a72dd9a075682293b
sha256-linux=5ef052b03a03afbd00ebfcbbdcb754bbde414b7e6151aae3e2ee78024ce1d0b6
+8 -2
View File
@@ -114,8 +114,14 @@
}
],
"upgrade_sources": {
"basis": "Exactly the two previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"basis": "The three previous-release sources pinned in .github/workflows/e2e-upgrade.yml are exercised today (source.version / source.asset / source.sha256). No other N-1/N-2 compatibility is claimed, and these digests are not inferred from tags.",
"sources": [
{
"version": "1.0.5",
"role": "gateway compatibility baseline",
"artifact_name": "rustfs-linux-x86_64-gnu-v1.0.5.zip",
"artifact_sha256": "5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0"
},
{
"version": "1.0.1",
"role": "N-1",
@@ -130,6 +136,6 @@
}
],
"rollback_supported": false,
"rollback_basis": "Only the bucket-config-rollback case (rollback_to_previous_release_reads_current_bucket_metadata) is exercised. Full data-plane rollback is not certified and is never a default claim."
"rollback_basis": "The release upgrade lane checks bucket-config rollback. The hosted compatibility workflow also checks in-flight multipart rollback to 1.0.5. Full data-plane rollback is not certified and is never a default claim."
}
}
+2 -2
View File
@@ -13,6 +13,6 @@
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=49
s3_error_lines=95
files=33
s3_error_lines=89
ecstore_files=0
+12 -2
View File
@@ -1091,9 +1091,12 @@ jobs:
- name: Verify E2E archive identity
run: python3 scripts/ci_test_artifact.py verify --configuration e2e --archive "$RUNNER_TEMP/e2e-tests.tar.zst"
- name: Install network fault-injection tools
- name: Install network fault-injection tools and the persistence test clock
run: |
sudo apt-get install -y iptables
sudo apt-get install -y iptables libfaketime
clock_library="$(dpkg-query -L libfaketime | grep -E '/libfaketime[.]so[.]1$')"
test -f "$clock_library"
echo "RUSTFS_PERSISTENCE_CLOCK_LIBRARY=$clock_library" >> "$GITHUB_ENV"
sudo -n iptables --version
# The endpoint-blackhole heal scenario needs CAP_NET_ADMIN. Containerised
# runners can run iptables but not touch the rule set; the test then logs
@@ -1200,8 +1203,14 @@ jobs:
# default-filter in .config/nextest.toml is the single wiring mechanism —
# extend that filter, never add ad-hoc e2e jobs here. Reuses the downloaded
# debug binary; each test spawns its own rustfs server on a random port.
- name: Download pinned release for persistence byte parity
id: persistence-reference
uses: ./.github/actions/reference-binary
- name: Run e2e full suite
env:
RUSTFS_PERSISTENCE_REFERENCE_BINARY: ${{ steps.persistence-reference.outputs.path }}
RUSTFS_PERSISTENCE_XML_DIR: ${{ runner.temp }}/rustfs-persistence-xml
CARGO_BIN_EXE_rustfs: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
RUSTFS_E2E_STARTUP_CAS_BINARY: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs
RUSTFS_E2E_STARTUP_CAS_BUILD_MANIFEST: ${{ runner.temp }}/rustfs-startup-cas-input/rustfs.e2e-startup-cas-build.json
@@ -1220,6 +1229,7 @@ jobs:
path: |
target/nextest/e2e-full/junit.xml
${{ runner.temp }}/rustfs-e2e-full-list.json
${{ runner.temp }}/rustfs-persistence-xml
retention-days: 7
- name: Upload startup CAS evidence
+39 -7
View File
@@ -90,11 +90,16 @@ jobs:
- version: 1.0.1
asset: rustfs-linux-x86_64-gnu-v1.0.1.zip
sha256: 9742e183cb0dfb0d48fa909797af572fe47b7f827569036baa4156e0d3958ee5
- version: 1.0.5
asset: rustfs-linux-x86_64-gnu-v1.0.5.zip
sha256: 5addae2f74756bf80104d0975e6347cda843bc2daffdfcd1f0cb80118b95cbd0
revision: 34e6449ed90e3d3a106f6655342e0b3127f33082
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
UPGRADE_SOURCE_VERSION: ${{ matrix.source.version }}
UPGRADE_SOURCE_ASSET: ${{ matrix.source.asset }}
UPGRADE_SOURCE_SHA256: ${{ matrix.source.sha256 }}
UPGRADE_SOURCE_REVISION: ${{ matrix.source.revision }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
@@ -132,6 +137,13 @@ jobs:
unzip -q "$archive" -d "$SOURCE_DIR"
chmod +x "$SOURCE_DIR/rustfs"
test -x "$SOURCE_DIR/rustfs"
if [[ -n "$UPGRADE_SOURCE_REVISION" ]]; then
mkdir -p "$RUNNER_TEMP/rustfs-upgrade-logs"
version_log="$RUNNER_TEMP/rustfs-upgrade-logs/source-version.txt"
"$SOURCE_DIR/rustfs" --version | tee "$version_log"
grep -Fx "rustfs $UPGRADE_SOURCE_VERSION" "$version_log"
grep -Eq "^git commit[[:space:]]*:[[:space:]]*$UPGRADE_SOURCE_REVISION$" "$version_log"
fi
echo "RUSTFS_UPGRADE_SOURCE_BINARY=$SOURCE_DIR/rustfs" >> "$GITHUB_ENV"
echo "RUSTFS_E2E_LOG_DIR=$RUNNER_TEMP/rustfs-upgrade-logs" >> "$GITHUB_ENV"
@@ -151,12 +163,14 @@ jobs:
multipart-layout-upgrade:direct_upgrade_from_previous_release_preserves_multipart_layouts
multipart-layout-baseline:previous_release_baseline_replicates_multipart_layouts
)
if [[ "$UPGRADE_SOURCE_VERSION" == "1.0.0-rc.5" ]]; then
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects)
else
cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects)
fi
case "$UPGRADE_SOURCE_VERSION" in
1.0.0-rc.5)
cases+=(odm-config-rollback:rc5_rollback_requires_restoring_odm_configuration)
cases+=(checkpoint-upgrade:rc5_checkpoint_upgrade_rewinds_signed_progress_without_losing_objects) ;;
1.0.1) cases+=(legacy-delete-reservation:stable_upgrade_deletes_foreign_reservations_without_losing_live_objects) ;;
1.0.5) ;; # The bucket rollback case also completes an in-flight multipart upload.
*) echo "::error::Unknown upgrade source $UPGRADE_SOURCE_VERSION"; exit 1 ;;
esac
status=0
for entry in "${cases[@]}"; do
artifact="${entry%%:*}"
@@ -178,7 +192,7 @@ jobs:
exit "$status"
- name: Run mixed-version MPU recovery
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' }}
if: ${{ !cancelled() && steps.verify_source.outcome == 'success' && matrix.source.version != '1.0.5' }}
timeout-minutes: 15
env:
NO_PROXY: 127.0.0.1,localhost,::1
@@ -199,6 +213,24 @@ jobs:
"$mpu_test" -- --exact --nocapture --test-threads=1 \
2>&1 | tee "$RUSTFS_E2E_LOG_DIR/test.log"
- name: Verify gateway baseline G09 evidence
if: ${{ success() && matrix.source.version == '1.0.5' }}
shell: bash
run: |
for field in mixed_version_reader_evidence mixed_version_writer_evidence rollback_payload_evidence; do
directory=mixed-version-upgrade
if [[ "$field" == rollback_payload_evidence ]]; then directory=bucket-config-rollback; fi
jq -e --arg field "$field" --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.gate == "G09" and .field == $field and .current_revision == $current and .previous_revision == $previous
and ($field != "rollback_payload_evidence" or .rollback_payload_replayed == true)' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/$directory/G09-$field.json"
done
jq -e --arg current "$GITHUB_SHA" --arg previous "$UPGRADE_SOURCE_REVISION" \
'.current_revision == $current and .previous_revision == $previous
and .part_sizes == [5242880, 5242880, 4096] and .bytes == (.part_sizes | add)
and (.rollback_read_sha256 | test("^[a-f0-9]{64}$")) and .rollback_read_sha256 == .roll_forward_read_sha256' \
"$RUNNER_TEMP/rustfs-upgrade-g09-evidence/bucket-config-rollback/inflight-multipart-rollback.json"
- name: Upload scanner/heal G09 evidence
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+1 -1
View File
@@ -13,7 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deployment
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, run the previous release binary on the same data.
- **The S3 protocol stack is now rustfs/gateway**, and it is the only S3 stack. The legacy s3s-based stack is removed, and so are the `RUSTFS_S3_STACK` environment variable and the `gateway-stack` cargo feature. There is no on-disk format change. `RUSTFS_S3_STACK` no longer selects anything: a server started with it set, to any value, refuses to start and prints `RUSTFS_S3_STACK is no longer supported; the gateway stack is the only S3 stack since <version>`, where `<version>` is the release's version. Remove the variable from service units, container environments, Helm values and orchestration files before upgrading. To roll back, follow the [S3 stack rollback procedure](docs/operations/s3-stack-rollback.md).
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
Generated
+11 -13
View File
@@ -1648,7 +1648,7 @@ dependencies = [
"cexpr",
"clang-sys",
"log",
"prettyplease 0.2.37",
"prettyplease 0.3.0",
"proc-macro2",
"quote",
"regex",
@@ -4238,6 +4238,7 @@ dependencies = [
"s3s",
"same-file",
"serde",
"serde-value",
"serde_json",
"serde_urlencoded",
"sha1 0.11.0",
@@ -10237,7 +10238,6 @@ dependencies = [
"rustix",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
"serial_test",
@@ -10323,7 +10323,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway"
version = "0.66.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"arc-swap",
"bytes",
@@ -10370,7 +10370,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-core"
version = "0.63.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bytes",
"futures-core",
@@ -10391,7 +10391,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-dialect-rustfs-admin"
version = "0.13.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bytes",
"form_urlencoded",
@@ -10405,7 +10405,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-http"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10423,7 +10423,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-macros"
version = "0.1.4"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"proc-macro2",
"quote",
@@ -10433,7 +10433,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-sig"
version = "0.15.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"hmac 0.13.0",
"http 1.5.0",
@@ -10448,7 +10448,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-stream"
version = "0.2.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bitflags 2.13.2",
"bytes",
@@ -10459,7 +10459,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-types"
version = "0.42.0+aws.2026-10-02"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"bytes",
"crc-fast",
@@ -10482,7 +10482,7 @@ dependencies = [
[[package]]
name = "rustfs-gateway-xml"
version = "0.4.0"
source = "git+https://github.com/rustfs/gateway.git?rev=d32f17219d75989167a05183dd3d04922615cbd5#d32f17219d75989167a05183dd3d04922615cbd5"
source = "git+https://github.com/rustfs/gateway.git?rev=60dcd2a55713dbcc9dbd6e165922e1a0df4473d7#60dcd2a55713dbcc9dbd6e165922e1a0df4473d7"
dependencies = [
"quick-xml 0.41.0",
]
@@ -11236,7 +11236,6 @@ dependencies = [
"rustfs-utils",
"rustls",
"rustls-pki-types",
"s3s",
"serde",
"serde_json",
"sha2 0.11.1",
@@ -11376,7 +11375,6 @@ dependencies = [
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-utils",
"s3s",
"serde",
"serde_json",
"serial_test",
+7 -6
View File
@@ -152,16 +152,16 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.2.0", default-features = false }
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.42.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.66.0", default-features = false }
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.66.0", default-features = false }
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "d32f17219d75989167a05183dd3d04922615cbd5", version = "0.13.0", default-features = false }
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", default-features = false }
rustfs-gateway-core = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.63.0", default-features = false }
rustfs-gateway-dialect-rustfs-admin = { git = "https://github.com/rustfs/gateway.git", rev = "60dcd2a55713dbcc9dbd6e165922e1a0df4473d7", version = "0.13.0", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
@@ -222,6 +222,7 @@ quick-xml = "0.42.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
serde = { version = "1.0.229" }
serde-value = "0.7.0"
serde_ignored = { version = "0.1" }
serde_with = { version = "3", default-features = false, features = ["macros", "std"] }
serde_json = { version = "1.0.154" }
@@ -171,70 +171,6 @@ impl DataUsageBucketObservation {
}
impl DataUsageInfo {
/// Hide values that cannot be bound to the current storage-owned identity.
///
/// `incarnations` contains canonical non-nil identities read from storage;
/// a missing entry means identity was unavailable. Neither legacy values
/// without identity nor metadata read failures establish confirmed zero.
/// Changed responses are display-only and cannot authorize quota or writes.
/// If retained counters overflow, all values remain unknown.
/// Returns the number of canonical values removed.
pub fn retain_current_bucket_usage_for_admin(&mut self, incarnations: &HashMap<String, String>) -> usize {
let original_count = self.buckets_usage.len();
let original_observation_count = self.usage_snapshot_bucket_observations.len();
let matches_current = |bucket: &String, usage: &BucketUsageInfo| {
incarnations.get(bucket).is_some_and(|current| {
!current.is_empty()
&& current != "00000000-0000-0000-0000-000000000000"
&& usage.bucket_incarnation.as_ref() == Some(current)
})
};
self.buckets_usage.retain(|bucket, usage| matches_current(bucket, usage));
self.usage_snapshot_bucket_observations.retain(|bucket, observation| {
matches_current(bucket, &observation.usage)
&& observation
.sources
.iter()
.all(|source| incarnations.get(bucket) == Some(&source.bucket_incarnation))
});
if original_count == self.buckets_usage.len()
&& original_observation_count == self.usage_snapshot_bucket_observations.len()
{
return 0;
}
let totals = self.buckets_usage.values().try_fold([0_u64; 4], |total, usage| {
Some([
total[0].checked_add(usage.size)?,
total[1].checked_add(usage.objects_count)?,
total[2].checked_add(usage.versions_count)?,
total[3].checked_add(usage.delete_markers_count)?,
])
});
let [size, objects, versions, markers] = totals.unwrap_or_else(|| {
self.buckets_usage.clear();
self.usage_snapshot_bucket_observations.clear();
[0; 4]
});
self.bucket_sizes = self
.buckets_usage
.iter()
.map(|(bucket, usage)| (bucket.clone(), usage.size))
.collect();
self.buckets_count = u64::try_from(self.buckets_usage.len()).unwrap_or(u64::MAX);
self.objects_total_size = size;
self.objects_total_count = objects;
self.versions_total_count = versions;
self.delete_markers_total_count = markers;
self.replication_info.clear();
self.tier_stats = None;
self.unknown_tier_stats = None;
self.usage_snapshot_complete = false;
self.usage_snapshot_partial = true;
self.usage_snapshot_converged = Some(false);
self.usage_snapshot_admin_projection = true;
original_count.saturating_sub(self.buckets_usage.len())
}
/// Overlay validated companions on an owned admin response only.
///
/// `incarnations` must come from current storage metadata, not the companion.
@@ -321,110 +321,3 @@ fn provenance_and_companions_change_candidate_bytes_and_round_trip_as_maps() {
let restored: DataUsageInfo = rmp_serde::from_slice(&named).expect("read named map with additive fields");
assert_eq!(restored, canonical);
}
#[test]
fn current_identity_filter_preserves_an_unchanged_complete_snapshot() {
let mut candidate = snapshot();
let original = candidate.clone();
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&incarnations()), 0);
assert_eq!(candidate, original);
assert!(candidate.is_complete_bucket_usage_snapshot());
}
#[test]
fn current_identity_filter_hides_unbound_values_without_confirming_zero() {
for identity in [
None,
Some(""),
Some("corrupt"),
Some("00000000-0000-0000-0000-000000000000"),
Some("22222222-2222-4222-8222-222222222222"),
] {
let mut candidate = snapshot();
candidate.usage_snapshot_bucket_observations.clear();
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.bucket_incarnation = identity.map(str::to_string);
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&incarnations()),
1,
"identity {identity:?}"
);
assert!(!candidate.buckets_usage.contains_key("bucket"));
assert!(!candidate.bucket_sizes.contains_key("bucket"));
assert_eq!(candidate.usage_snapshot_converged, Some(false));
assert!(candidate.usage_snapshot_admin_projection);
assert!(!candidate.is_complete_bucket_usage_snapshot());
assert!(!candidate.is_valid_partial_snapshot());
}
for current in ["", "00000000-0000-0000-0000-000000000000"] {
let mut candidate = snapshot();
candidate
.buckets_usage
.get_mut("bucket")
.expect("canonical")
.bucket_incarnation = Some(current.to_string());
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("bucket".to_string(), current.to_string())])),
1
);
assert!(candidate.usage_snapshot_admin_projection);
}
let mut candidate = snapshot();
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&HashMap::new()), 1);
assert!(candidate.usage_snapshot_bucket_observations.is_empty());
assert_eq!(candidate.usage_snapshot_converged, Some(false));
}
#[test]
fn current_identity_filter_keeps_healthy_buckets_and_recalculates_partial_totals() {
let mut candidate = snapshot();
candidate.buckets_usage.insert("cold".to_string(), usage(7));
candidate.buckets_count = 2;
candidate.bucket_sizes.insert("cold".to_string(), 7);
candidate.calculate_totals();
candidate.replication_info.insert("target".to_string(), Default::default());
candidate.tier_stats = Some(Default::default());
candidate.unknown_tier_stats = Some(Default::default());
assert_eq!(
candidate.retain_current_bucket_usage_for_admin(&HashMap::from([("cold".to_string(), INCARNATION.to_string())])),
1
);
assert_eq!(candidate.buckets_usage.len(), 1);
assert_eq!(candidate.buckets_usage["cold"].size, 7);
assert_eq!(candidate.bucket_sizes, HashMap::from([("cold".to_string(), 7)]));
assert_eq!(
(
candidate.buckets_count,
candidate.objects_total_size,
candidate.objects_total_count,
candidate.versions_total_count
),
(1, 7, 7, 7)
);
assert!(candidate.replication_info.is_empty());
assert!(candidate.tier_stats.is_none());
assert!(candidate.unknown_tier_stats.is_none());
assert!(candidate.usage_snapshot_admin_projection);
assert_eq!(candidate.usage_snapshot_converged, Some(false));
}
#[test]
fn current_identity_filter_overflow_leaves_totals_unknown() {
let mut candidate = snapshot();
candidate.buckets_usage.insert("large".to_string(), usage(u64::MAX));
candidate.buckets_usage.insert("small".to_string(), usage(1));
candidate.buckets_count = 3;
let current = HashMap::from([
("large".to_string(), INCARNATION.to_string()),
("small".to_string(), INCARNATION.to_string()),
]);
assert_eq!(candidate.retain_current_bucket_usage_for_admin(&current), 3);
assert!(candidate.buckets_usage.is_empty());
assert!(candidate.bucket_sizes.is_empty());
assert_eq!(candidate.usage_snapshot_converged, Some(false));
assert!(candidate.usage_snapshot_admin_projection);
assert!(!candidate.is_complete_bucket_usage_snapshot());
}
+2 -1
View File
@@ -129,7 +129,6 @@ md-5 = { workspace = true }
prost.workspace = true
sha2 = { workspace = true }
rustfs-s3-types = { workspace = true }
s3s = { workspace = true, features = ["minio"] }
zstd.workspace = true
time = { workspace = true, features = ["parsing", "formatting", "macros", "serde"] }
rcgen.workspace = true
@@ -139,6 +138,8 @@ rustls = { workspace = true, default-features = false, features = ["aws-lc-rs",
clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
serde-value.workspace = true
s3s = { workspace = true, features = ["minio"] }
async-compression = { workspace = true, features = ["tokio", "bzip2", "lz4", "xz"] }
aws-config = { workspace = true }
aws-sdk-sts = { workspace = true, default-features = false, features = ["default-https-client", "rt-tokio"] }
+2
View File
@@ -211,6 +211,8 @@ the wiring source of truth. Committed test-ID digests under
**Endpoint blackhole scenario skipped** — `heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_after_target_endpoint_blackhole` installs a loopback `iptables` DROP rule and therefore needs `CAP_NET_ADMIN` (root or passwordless `sudo -n iptables`). A host where `iptables` is missing or cannot read the OUTPUT chain (typical inside an unprivileged container, where the nf_tables backend reports "Permission denied" even under `sudo`) logs a `heal_interruption_skipped` warning and returns without exercising heal. Set `RUSTFS_E2E_REQUIRE_NET_FAULT_INJECTION=1` on lanes that do provision the capability so a broken runner fails instead of skipping.
The full suite compares persisted bucket configurations and object records against the release pinned in `.config/reference-binary.toml`. Set `RUSTFS_PERSISTENCE_REFERENCE_BINARY` to the verified executable and `RUSTFS_PERSISTENCE_CLOCK_LIBRARY` to the absolute path of libfaketime (`libfaketime.so.1` on Linux or `libfaketime.1.dylib` on macOS). To retain the two sets of 13 XML documents for the gateway golden corpus, set `RUSTFS_PERSISTENCE_XML_DIR` to an output directory. CI includes them in the full suite's JUnit artifact.
**Reproduce a CI failure locally** — run the exact profile/lane:
```bash
+52 -1
View File
@@ -34,7 +34,10 @@
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
use crate::common::{
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
init_logging, local_http_client, parse_assume_role_credentials,
};
use hmac::{Hmac, KeyInit as _, Mac};
use http::header::HOST;
use rustfs_s3_types::Body;
@@ -396,6 +399,54 @@ mod tests {
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
env.stop_server();
drop(env);
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
}
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let user = "console-password-user";
let old_password = uuid::Uuid::new_v4().to_string();
let new_password = uuid::Uuid::new_v4().to_string();
admin_create_user(&env, user, &old_password).await?;
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
.await?;
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
let session_request = |method, path, body| {
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
};
let password_request = |current: &str| {
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
};
let (status, body) =
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
assert_eq!(
status,
reqwest::StatusCode::BAD_REQUEST,
"the session secret must not be accepted as the IAM password"
);
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
parse_assume_role_credentials(&response)?;
env.stop_server();
Ok(())
}
@@ -137,6 +137,40 @@ async fn test_bucket_policy_authenticated_user() -> Result<(), Box<dyn std::erro
.await
.map_err(|e| format!("DeleteObject failed: {}", e))?;
for (allowed, may_read) in [("s3:GetBucketPolicy", true), ("s3:PutBucketPolicy", false)] {
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": { "AWS": [user_access] },
"Action": allowed,
"Resource": format!("arn:aws:s3:::{bucket_name}")
}]
})
.to_string();
admin_client
.put_bucket_policy()
.bucket(bucket_name)
.policy(&policy)
.send()
.await?;
let read = user_client.get_bucket_policy().bucket(bucket_name).send().await;
let write = user_client
.put_bucket_policy()
.bucket(bucket_name)
.policy(&policy)
.send()
.await;
if may_read {
read?;
let denied = write.expect_err("GetBucketPolicy must not grant PutBucketPolicy");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
} else {
let denied = read.expect_err("PutBucketPolicy must not grant GetBucketPolicy");
assert_eq!(denied.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
write?;
}
}
info!("Test Passed!");
Ok(())
}
+26 -14
View File
@@ -31,7 +31,7 @@ use super::harness::{
use crate::common::{
AdminTransport, admin_add_canned_policy_via, admin_attach_user_policy_via, admin_create_user_via, init_logging,
};
use crate::upgrade_compatibility_test::wait_for_upgrade_write_readiness;
use crate::upgrade_compatibility_test::{UpgradeSourceCapabilities, wait_for_upgrade_write_readiness};
use aws_sdk_s3::Client;
use aws_sdk_s3::error::ProvideErrorMetadata;
use std::path::{Path, PathBuf};
@@ -427,7 +427,12 @@ async fn assert_history_and_iam(dist: &DistCluster, seed: &UpgradeSeed, context:
Ok(())
}
async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &UpgradeSeed, previous: &Path) -> TestResult {
async fn compare_degraded_legacy_list_parts(
dist: &mut DistCluster,
seed: &UpgradeSeed,
previous: &Path,
source: &UpgradeSourceCapabilities,
) -> TestResult {
let decoded = String::from_utf8(base64_simd::URL_SAFE_NO_PAD.decode_to_vec(seed.multipart_upload.as_bytes())?)?;
let (_, upload_dir) = decoded.split_once('.').ok_or("legacy upload ID lacks deployment delimiter")?;
let hash = sha256_hex(format!("{}/legacy-multipart.bin", seed.history_bucket).as_bytes());
@@ -489,16 +494,22 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
// Three online disks retain metadata quorum but have no part directory;
// the fourth disk is offline. A legacy empty response is not a proof that
// the unavailable participant has no part.
let legacy = legacy.expect("previous coordinator should expose its legacy empty-list behavior");
assert!(legacy.parts().is_empty());
let current = current.expect_err("current coordinator must not infer empty from missing directories and offline disks");
assert_eq!(current.raw_response().expect("current S3 response").status().as_u16(), 503);
// Since #9149, SlowDownRead is reserved for GET/HEAD object reads; a
// ListParts read-quorum failure is reported as ServiceUnavailable.
assert_eq!(
current.as_service_error().and_then(|error| error.meta().code()),
Some("ServiceUnavailable")
);
let mut quorum_errors =
vec![current.expect_err("current coordinator must not infer empty from missing directories and offline disks")];
if source.requires_list_parts_read_quorum {
quorum_errors.push(legacy.expect_err("previous coordinator must require evidence from unavailable disks"));
} else {
assert!(
legacy
.expect("previous coordinator should expose its legacy empty-list behavior")
.parts()
.is_empty()
);
}
for error in quorum_errors {
assert_eq!(error.raw_response().expect("S3 response").status().as_u16(), 503);
assert_eq!(error.as_service_error().and_then(|error| error.meta().code()), Some("ServiceUnavailable"));
}
dist.cluster.start_node_from_binary(3, previous).await?;
wait_for_upgrade_write_readiness(&clients, &seed.history_bucket, "mixed-list-restored", CREDENTIAL_TIMEOUT).await?;
for (path, before) in original_files {
@@ -520,7 +531,7 @@ async fn compare_degraded_legacy_list_parts(dist: &mut DistCluster, seed: &Upgra
assert_eq!(listed.parts()[0].e_tag(), Some(seed.multipart_etag.as_str()));
}
eprintln!(
"MPU mixed degraded: one offline disk, three missing part directories; previous returned empty, current returned 503 SlowDownRead; original part restored on both"
"MPU mixed degraded: one offline disk, three missing part directories; checked each coordinator against its release contract; original part restored on both"
);
Ok(())
}
@@ -592,6 +603,7 @@ async fn replace_node_and_wait_for_write_readiness(dist: &mut DistCluster, seed:
async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> TestResult {
init_logging();
let previous = source_binary()?;
let source = UpgradeSourceCapabilities::from_binary(&previous).await?;
let mut dist = DistCluster::new_stopped(DistLayout::FourNodeFourDisk).await?;
dist.cluster.set_env("RUSTFS_STORAGE_CLASS_RRS", "EC:0");
capture_upgrade_logs(&mut dist, "rolling-upgrade")?;
@@ -601,7 +613,7 @@ async fn four_node_rolling_upgrade_preserves_history_and_iam_credentials() -> Te
replace_node_and_wait_for_write_readiness(&mut dist, &seed, 0).await?;
assert_history_and_iam(&dist, &seed, "one-current-node").await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous).await?;
compare_degraded_legacy_list_parts(&mut dist, &seed, &previous, &source).await?;
for node_idx in [1, 2] {
replace_node_and_wait_for_write_readiness(&mut dist, &seed, node_idx).await?;
@@ -27,7 +27,7 @@
//! is process-global, so two stacks in one process would read each other's buckets.
mod compare;
mod reference;
pub(crate) mod reference;
mod scenarios;
use crate::common::{RustFSTestEnvironment, init_logging, rustfs_binary_path, workspace_root};
@@ -78,7 +78,7 @@ fn is_executable(_metadata: &std::fs::Metadata) -> bool {
/// What `<binary> --version` printed, trimmed. Fails when the binary does not run, exits
/// non-zero, or prints nothing, because an empty version would report nothing as observed.
pub(super) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
pub(crate) async fn binary_version(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
if !output.status.success() {
return Err(format!("{} --version failed with {}", binary.display(), output.status).into());
+3
View File
@@ -37,6 +37,9 @@ pub mod fake_s3_target;
#[cfg(test)]
mod dual_stack_differential;
#[cfg(test)]
mod persistence_parity;
// On-demand migration (backlog#2147): shared two-server environment, admin
// wrappers, and the harness self-test (backlog#2151). Behavior scenarios are
// added by later ODM tasks.
@@ -1543,6 +1543,30 @@ async fn test_anonymous_post_object_honors_success_action_status() -> Result<(),
let uploaded = get_out.body.collect().await?.into_bytes();
assert_eq!(uploaded.as_ref(), expected_body.as_slice(), "uploaded object body should match");
// An empty redirect is rejected when rendering the response, after storage succeeds.
for (field, value, expected_status) in [
("success_action_status", "200", reqwest::StatusCode::OK),
("success_action_status", "204", reqwest::StatusCode::NO_CONTENT),
("success_action_redirect", "", reqwest::StatusCode::BAD_REQUEST),
] {
let key = format!("{field}-{value}.txt");
let form = reqwest::multipart::Form::new()
.text("key", key.clone())
.text(field, value)
.part("file", reqwest::multipart::Part::bytes(expected_body.clone()).file_name("upload.txt"));
let response = local_http_client()
.post(format!("{}/{bucket}", env.url))
.multipart(form)
.send()
.await?;
assert_eq!(response.status(), expected_status);
if value.is_empty() {
assert!(response.text().await?.contains("<Code>InvalidArgument</Code>"));
}
let stored = admin_client.get_object().bucket(bucket).key(key).send().await?;
assert_eq!(stored.body.collect().await?.into_bytes().as_ref(), expected_body.as_slice());
}
Ok(())
}
+449
View File
@@ -0,0 +1,449 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Compare bytes produced by S3 writes to independent, single-drive volumes.
use crate::common::{RustFSTestEnvironment, admin_request, rustfs_binary_path, signed_s3_request_with_headers};
use crate::dual_stack_differential::reference::binary_version;
use crate::fake_s3_target::{FAKE_ACCESS_KEY, FAKE_SECRET_KEY, FakeS3Target};
use crate::on_demand_migration::common::{ODM_ADMIN_ROUTE, ODM_MODULE_SWITCH_ENV, OdmSourceSpec};
use crate::replication_extension_test::send_set_replication_target_request;
use crate::storage_api::persistence_parity::{BucketMetadata, RUSTFS_META_BUCKET};
use crate::upgrade_compatibility_test::{set_bucket_quota, write_multipart};
use aws_sdk_s3::Client;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{BucketCannedAcl, Tag, Tagging};
use http::{HeaderMap, Method};
use md5::{Digest, Md5};
use rustfs_filemeta::{FileMeta, MetaObject};
use serde_value::Value;
use std::collections::BTreeMap;
use std::path::{Path, PathBuf};
type Error = Box<dyn std::error::Error + Send + Sync>;
const CONFIGS: &str = "persisted-configs";
const OBJECTS: &str = "persisted-objects";
const FIRST_PART_SIZE: usize = 5 * 1024 * 1024;
const LOGS: &str = "rustfs-access-logs";
fn bucket_metadata(volume: &Path, bucket: &str) -> Result<BucketMetadata, Error> {
let key = BucketMetadata::new(bucket).save_file_path();
let object = volume.join(RUSTFS_META_BUCKET).join(&key);
let meta = FileMeta::load(&std::fs::read(object.join("xl.meta"))?)?;
let info = meta.into_fileinfo(RUSTFS_META_BUCKET, &key, "", true, false, true)?;
info.validate_for_metadata_read()?;
let [part] = info.parts.as_slice() else { return Err("bucket metadata must have one part".into()) };
let size = usize::try_from(info.size)?;
if info.erasure.data_blocks != 1 || size != part.size || size > info.erasure.block_size {
return Err("bucket metadata exceeds the single-drive, single-stripe fixture".into());
}
let framed = if info.inline_data() {
info.data.ok_or("missing inline bucket metadata")?.to_vec()
} else {
let directory = info.data_dir.ok_or("missing bucket metadata data directory")?;
std::fs::read(object.join(directory.to_string()).join(format!("part.{}", part.number)))?
};
let algorithm = info.erasure.get_checksum_info(part.number).algorithm;
let (digest, payload) = framed.split_at_checked(algorithm.size()).ok_or("truncated bucket metadata")?;
if digest.is_empty() || payload.len() != size || algorithm.hash_encode(payload).as_ref() != digest {
return Err("invalid bucket metadata shard checksum or size".into());
}
BucketMetadata::check_header(payload)?;
let fields = &payload[4..];
if bucket == CONFIGS {
// The production decoder defaults a missing field to empty. Require the ACL field on disk
// so dropping it cannot look like the expected zero-byte document.
let keys: BTreeMap<&str, serde::de::IgnoredAny> = rmp_serde::from_slice(fields)?;
if !keys.contains_key("BucketAclConfigJSON") {
return Err("missing BucketAclConfigJSON field".into());
}
}
Ok(BucketMetadata::unmarshal(fields)?)
}
type VersionFields = BTreeMap<String, Value>;
fn read_object(volume: &Path, key: &str) -> Result<(MetaObject, VersionFields), Error> {
let meta = FileMeta::load(&std::fs::read(volume.join(OBJECTS).join(key).join("xl.meta"))?)?;
let [version] = meta.versions.as_slice() else {
return Err(format!("{key}: expected one version").into());
};
let object = version.parse_version_meta()?.object.ok_or("missing V2 object")?;
if object.mod_time.is_none() || object.data_dir.is_none_or(|id| id.is_nil()) {
return Err(format!("{key}: missing modification time or data directory").into());
}
Ok((object, nonvolatile_fields(&version.meta, key)?))
}
fn nonvolatile_fields(raw: &[u8], key: &str) -> Result<VersionFields, Error> {
// The production reader skips unknown keys and supplies defaults for some missing keys.
// Keep the raw maps so both the version envelope and V2Obj preserve every field and value.
let mut fields: VersionFields = rmp_serde::from_slice(raw)?;
let Some(Value::Map(object_fields)) = fields.get_mut("V2Obj") else {
return Err(format!("{key}: missing raw V2Obj map").into());
};
for volatile in ["MTime", "DDir"] {
let value = object_fields
.get_mut(&Value::String(volatile.to_owned()))
.ok_or_else(|| format!("{key}: missing {volatile}"))?;
*value = Value::Unit;
}
Ok(fields)
}
async fn put_xml(env: &RustFSTestEnvironment, query: &str, xml: &str) -> Result<(), Error> {
let mut headers = HeaderMap::new();
headers.insert("content-md5", base64_simd::STANDARD.encode_to_string(Md5::digest(xml)).parse()?);
let response = signed_s3_request_with_headers(
Method::PUT,
&format!("{}/{CONFIGS}?{query}", env.url),
Some(xml.to_owned()),
Some("application/xml"),
&env.access_key,
&env.secret_key,
&headers,
)
.await?;
if !response.status().is_success() {
return Err(format!("PUT ?{query} failed with {}", response.status()).into());
}
Ok(())
}
struct Snapshot {
configs: BTreeMap<&'static str, Vec<u8>>,
objects: BTreeMap<&'static str, VersionFields>,
multipart_files: BTreeMap<String, Vec<u8>>,
}
async fn capture(binary: &Path, target: &FakeS3Target, clock: &str, clock_library: &str) -> Result<Snapshot, Error> {
let mut env = RustFSTestEnvironment::new().await?;
let queue = tempfile::tempdir()?;
let clock_loader = if cfg!(target_os = "macos") {
"DYLD_INSERT_LIBRARIES"
} else {
"LD_PRELOAD"
};
let options = [
(clock_loader, clock_library),
("DYLD_FORCE_FLAT_NAMESPACE", "1"),
("FAKETIME", clock),
("FAKETIME_DONT_FAKE_MONOTONIC", "1"),
("TZ", "UTC"),
("RUSTFS_NOTIFY_ENABLE", "true"),
("RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY", "on"),
("RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY", target.endpoint()),
(
"RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY",
queue.path().to_str().ok_or("non-UTF8 queue path")?,
),
("RUSTFS_OUTBOUND_ALLOW_ORIGINS", target.endpoint()),
("RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET", "true"),
(ODM_MODULE_SWITCH_ENV, "true"),
("RUSTFS_NEW_BUCKET_DURABILITY_MODE", "strict"),
("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
];
env.start_rustfs_server_from_binary_without_cleanup(binary, vec![], &options)
.await?;
let client = env.create_s3_client();
write_configs(&env, &client, target).await?;
write_objects(&client).await?;
env.stop_server();
read_snapshot(Path::new(&env.temp_dir))
}
async fn write_configs(env: &RustFSTestEnvironment, client: &Client, target: &FakeS3Target) -> Result<(), Error> {
env.create_test_bucket(LOGS).await?;
env.create_test_bucket(OBJECTS).await?;
env.create_test_bucket("persistence-warehouse").await?;
env.create_test_bucket("persistence-migration").await?;
client
.create_bucket()
.bucket(CONFIGS)
.object_lock_enabled_for_bucket(true)
.send()
.await?;
macro_rules! golden {
($path:literal) => {
include_str!(concat!("../../ecstore/tests/fixtures/bucket-config-goldens/", $path, ".in.xml"))
};
}
let target_request = serde_json::json!({
"endpoint": target.address(), "credentials": {"accessKey": FAKE_ACCESS_KEY, "secretKey": FAKE_SECRET_KEY},
"targetbucket": "interop-dr", "type": "replication", "secure": false,
"deploymentID": "0b6f3c2a-5e6a-4d2b-9c1e-7a8f9d0e1b2c"
});
let response = send_set_replication_target_request(env, CONFIGS, false, target_request).await?;
if !response.status().is_success() {
return Err(format!("set remote target failed with {}", response.status()).into());
}
let target_arn: String = response.json().await?;
let replication = golden!("replication/rustfs").replace("arn:rustfs:replication:", "arn:minio:replication:");
if !replication.contains(&format!("<Bucket>{target_arn}</Bucket>")) {
return Err("registered target does not match the golden replication destination".into());
}
for (query, xml) in [
("versioning", golden!("versioning/minio")),
("notification", golden!("notification/rustfs")),
("replication", replication.as_str()),
("lifecycle", golden!("lifecycle/minio")),
("object-lock", golden!("object_lock/rustfs")),
("encryption", golden!("encryption/minio")),
("tagging", golden!("tagging/rustfs")),
("cors", golden!("cors/rustfs")),
("logging", golden!("logging/rustfs")),
("website", golden!("website/rustfs")),
("accelerate", golden!("accelerate/rustfs")),
("requestPayment", golden!("request_payment/rustfs")),
("publicAccessBlock", golden!("public_access_block/rustfs")),
] {
put_xml(env, query, xml).await?;
}
client
.put_bucket_acl()
.bucket(CONFIGS)
.acl(BucketCannedAcl::Private)
.send()
.await?;
let acl = client.get_bucket_acl().bucket(CONFIGS).send().await?;
let owner = acl.owner().and_then(|owner| owner.id()).ok_or("ACL owner missing")?;
let [grant] = acl.grants() else { return Err("expected exactly one ACL grant".into()) };
let grantee = grant.grantee().ok_or("missing ACL grantee")?;
if owner.is_empty()
|| grant.permission().map(|p| p.as_str()) != Some("FULL_CONTROL")
|| grantee.r#type().as_str() != "CanonicalUser"
|| grantee.id().is_some()
{
return Err("GET ACL changed its canonical owner grant".into());
}
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Deny", "Principal": "*", "Action": "s3:DeleteObject",
"Resource": format!("arn:aws:s3:::{CONFIGS}/protected/*"),
}],
});
client
.put_bucket_policy()
.bucket(CONFIGS)
.policy(policy.to_string())
.send()
.await?;
set_bucket_quota(env, CONFIGS, 16 * 1024 * 1024).await?;
let migration = OdmSourceSpec::for_fake_source(target, "migration-source");
for (path, body) in [
("/iceberg/v1/buckets/persistence-warehouse".to_owned(), None),
(format!("{ODM_ADMIN_ROUTE}/persistence-migration"), Some(migration.to_json().to_string())),
] {
let (status, _) = admin_request(&env.url, Method::PUT, &path, body, &env.access_key, &env.secret_key).await?;
if !status.is_success() {
return Err(format!("configuring {path} failed with {status}").into());
}
}
Ok(())
}
async fn write_objects(client: &Client) -> Result<(), Error> {
for key in ["put", "tagged"] {
client
.put_object()
.bucket(OBJECTS)
.key(key)
.metadata("purpose", "byte-parity")
.body(ByteStream::from_static(b"persisted object contents"))
.send()
.await?;
}
client
.copy_object()
.bucket(OBJECTS)
.key("copy")
.copy_source(format!("{OBJECTS}/put"))
.send()
.await?;
let tags = Tagging::builder()
.tag_set(Tag::builder().key("type").value("persisted").build()?)
.build()?;
client
.put_object_tagging()
.bucket(OBJECTS)
.key("tagged")
.tagging(tags)
.send()
.await?;
write_multipart(client, OBJECTS, "multipart", &[vec![0x31; FIRST_PART_SIZE], vec![0x32; 100]]).await?;
Ok(())
}
fn read_snapshot(volume: &Path) -> Result<Snapshot, Error> {
let b = bucket_metadata(volume, CONFIGS)?;
let configs = BTreeMap::from([
("acl", b.bucket_acl_config_json),
("accelerate", b.accelerate_config_xml),
("cors", b.cors_config_xml),
("encryption", b.encryption_config_xml),
("lifecycle", b.lifecycle_config_xml),
("logging", b.logging_config_xml),
("notification", b.notification_config_xml),
("object_lock", b.object_lock_config_xml),
("public_access_block", b.public_access_block_config_xml),
("request_payment", b.request_payment_config_xml),
("tagging", b.tagging_config_xml),
("versioning", b.versioning_config_xml),
("website", b.website_config_xml),
("durability", b.durability_config_json),
("policy", b.policy_config_json),
("quota", b.quota_config_json),
("replication", b.replication_config_xml),
("targets", b.bucket_targets_config_json),
("table", bucket_metadata(volume, "persistence-warehouse")?.table_bucket_config_json),
("odm", bucket_metadata(volume, "persistence-migration")?.on_demand_migration_config_json),
]);
let mut objects = BTreeMap::new();
for key in ["put", "copy", "tagged"] {
let (_, fields) = read_object(volume, key)?;
objects.insert(key, fields);
}
let (multipart, _) = read_object(volume, "multipart")?;
if multipart.part_numbers != [1, 2] || multipart.part_sizes != [FIRST_PART_SIZE, 100] {
return Err("xl.meta does not describe both uploaded parts".into());
}
let directory = volume
.join(OBJECTS)
.join("multipart")
.join(multipart.data_dir.ok_or("missing multipart data directory")?.to_string());
let mut multipart_files = BTreeMap::new();
for entry in std::fs::read_dir(directory)? {
let entry = entry?;
let name = entry.file_name().into_string().map_err(|_| "invalid multipart file name")?;
if name.starts_with("part.") {
let bytes = std::fs::read(entry.path())?;
if bytes.is_empty() {
return Err(format!("empty multipart file {name}").into());
}
multipart_files.insert(name, bytes);
}
}
for name in ["part.1", "part.2", "part.1.meta", "part.2.meta"] {
if !multipart_files.contains_key(name) {
return Err(format!("missing multipart file {name}").into());
}
}
Ok(Snapshot {
configs,
objects,
multipart_files,
})
}
fn save_xml_samples(root: &Path, side: &str, snapshot: &Snapshot) -> Result<(), Error> {
let directory = root.join(side);
std::fs::create_dir_all(&directory)?;
let families: [&str; 13] = [
"accelerate",
"cors",
"encryption",
"lifecycle",
"logging",
"notification",
"object_lock",
"public_access_block",
"replication",
"request_payment",
"tagging",
"versioning",
"website",
];
for family in families {
let bytes = snapshot
.configs
.get(family)
.ok_or_else(|| format!("missing XML family {family}"))?;
std::fs::write(directory.join(format!("{family}.xml")), bytes)?;
}
Ok(())
}
#[tokio::test]
async fn persistence_byte_parity() -> Result<(), Error> {
let reference = PathBuf::from(
std::env::var_os("RUSTFS_PERSISTENCE_REFERENCE_BINARY").ok_or("missing pinned RUSTFS_PERSISTENCE_REFERENCE_BINARY")?,
);
let candidate = rustfs_binary_path();
if binary_version(&reference).await? == binary_version(&candidate).await? {
return Err("reference and candidate binaries report identical build identities".into());
}
// The quota writer records the process wall clock inside its JSON document. Keep raw bytes
// comparable while the SDK's signature clock and both servers' monotonic timers advance normally.
let clock_library = std::env::var("RUSTFS_PERSISTENCE_CLOCK_LIBRARY")?;
if !Path::new(&clock_library).is_absolute() || !Path::new(&clock_library).is_file() {
return Err("RUSTFS_PERSISTENCE_CLOCK_LIBRARY must name an absolute libfaketime path".into());
}
let clock = time::OffsetDateTime::now_utc()
.format(time::macros::format_description!("[year]-[month]-[day] [hour]:[minute]:[second]"))?;
let target = FakeS3Target::start().await?;
target.create_bucket("migration-source");
target.create_bucket_with_object_lock("interop-dr");
let reference = capture(&reference, &target, &clock, &clock_library).await?;
let candidate = capture(&candidate, &target, &clock, &clock_library).await?;
if let Some(directory) = std::env::var_os("RUSTFS_PERSISTENCE_XML_DIR") {
save_xml_samples(Path::new(&directory), "reference", &reference)?;
save_xml_samples(Path::new(&directory), "candidate", &candidate)?;
}
if reference.configs.len() != 20 || candidate.configs.len() != 20 {
return Err("expected 13 XML and seven JSON fields on each side".into());
}
for (&field, expected) in &reference.configs {
// Canned ACLs acknowledge the write and return one owner grant without persisting an ACL document.
if expected.is_empty() != (field == "acl") || candidate.configs.get(field) != Some(expected) {
return Err(format!("incorrect or different persisted bucket config: {field}").into());
}
}
for (key, expected) in &reference.objects {
if candidate.objects.get(key) != Some(expected) {
return Err(format!("different persisted xl.meta fields: {key}").into());
}
}
if candidate.multipart_files != reference.multipart_files {
return Err("different multipart part files or part metadata".into());
}
eprintln!(
"Compared {} bucket fields (canned ACL empty), {} full object records and multipart part files",
reference.configs.len(),
reference.objects.len()
);
Ok(())
}
#[test]
fn version_comparison_preserves_unknown_fields_and_absent_values() -> Result<(), Error> {
let mut sample = serde_json::json!({"Type": 1, "v": 1, "V2Obj": {"DDir": vec![1; 16], "MTime": 1}});
let fields = |value: &serde_json::Value| nonvolatile_fields(&rmp_serde::to_vec(value)?, "fixture");
let original = fields(&sample)?;
sample["V2Obj"]["DDir"] = serde_json::json!(vec![2; 16]);
sample["V2Obj"]["MTime"] = serde_json::json!(2);
assert_eq!(fields(&sample)?, original);
sample["V2Obj"]["Future"] = serde_json::Value::Null;
let with_nil = fields(&sample)?;
assert_ne!(with_nil, original);
sample["V2Obj"]["Future"] = serde_json::json!(false);
let with_value = fields(&sample)?;
assert_ne!(with_value, with_nil);
sample["FutureEnvelope"] = serde_json::json!(1);
assert_ne!(fields(&sample)?, with_value);
assert!(fields(&serde_json::json!({"V2Obj": {"MTime": 1}})).is_err());
Ok(())
}
@@ -465,7 +465,7 @@ pub(crate) async fn set_replication_target_with_options(
Ok(arn)
}
async fn send_set_replication_target_request(
pub(crate) async fn send_set_replication_target_request(
source_env: &RustFSTestEnvironment,
source_bucket: &str,
update: bool,
+6
View File
@@ -57,6 +57,12 @@ pub(crate) mod replication_extension {
pub(crate) use super::BucketTargetSys;
}
#[cfg(test)]
pub(crate) mod persistence_parity {
pub(crate) use super::RUSTFS_META_BUCKET;
pub(crate) use rustfs_ecstore::api::bucket::metadata::BucketMetadata;
}
#[cfg(test)]
pub(crate) mod upgrade_compatibility {
pub(crate) use super::{DiskOption, Endpoint, RUSTFS_META_MULTIPART_BUCKET, new_disk};
+127 -15
View File
@@ -46,9 +46,12 @@ type BoxError = Box<dyn std::error::Error + Send + Sync>;
const SOURCE_BINARY_ENV: &str = "RUSTFS_UPGRADE_SOURCE_BINARY";
const G09_EVIDENCE_DIR_ENV: &str = "RUSTFS_SCANNER_HEAL_G09_EVIDENCE_DIR";
const RC2_COMMIT: &str = "e9f53180273b32a9c2b0aa8b5a384047bc7e875c";
const RC5_COMMIT: &str = "40a2470feb567201165a5b809b7598bb4b1f68f5";
const STABLE_1_0_1_COMMIT: &str = "6de965ae3c965a78ff819fbcd7acd4aa44177d92";
const STABLE_1_0_4_COMMIT: &str = "46337942d55f3a8e6a97b147ca98d9ea8115555d";
const STABLE_1_0_5_COMMIT: &str = "34e6449ed90e3d3a106f6655342e0b3127f33082";
const STABLE_1_0_6_COMMIT: &str = "de9009d882644effc2628e777650025c3d8e7c65";
const SSE_MASTER_KEY_ENV: &str = "RUSTFS_SSE_S3_MASTER_KEY";
const SSE_MASTER_KEY: &str = "QkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkJCQkI=";
const PLAIN_BUCKET: &str = "upgrade-plain-data";
@@ -142,6 +145,41 @@ fn current_source_revision() -> Result<String, BoxError> {
Ok(revision.to_string())
}
pub(crate) struct UpgradeSourceCapabilities {
supports_delete_recovery: bool,
supports_multipart_publication: bool,
pub(crate) requires_list_parts_read_quorum: bool,
}
impl UpgradeSourceCapabilities {
fn from_revision(revision: &str) -> Result<Self, BoxError> {
// 1.0.4 supports the delete decision protocol (#8468). Version 1.0.5 also
// shipped native MPU publication (#8919) and strict ListParts evidence (#9037, #9149).
match revision {
RC2_COMMIT | RC5_COMMIT | STABLE_1_0_1_COMMIT => Ok(Self {
supports_delete_recovery: false,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_4_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: false,
requires_list_parts_read_quorum: false,
}),
STABLE_1_0_5_COMMIT | STABLE_1_0_6_COMMIT => Ok(Self {
supports_delete_recovery: true,
supports_multipart_publication: true,
requires_list_parts_read_quorum: true,
}),
_ => Err(format!("no pinned upgrade expectations for source revision {revision}").into()),
}
}
pub(crate) async fn from_binary(binary: &Path) -> Result<Self, BoxError> {
Self::from_revision(&binary_source_revision(binary).await?)
}
}
async fn binary_source_revision(binary: &Path) -> Result<String, BoxError> {
let output = tokio::process::Command::new(binary).arg("--version").output().await?;
if !output.status.success() {
@@ -225,6 +263,25 @@ mod g09_evidence_tests {
use super::*;
use tempfile::TempDir;
#[test]
fn pinned_upgrade_sources_keep_distinct_delete_and_list_parts_contracts() -> TestResult {
for (revision, delete_recovery, multipart_publication, list_parts_quorum) in [
(RC2_COMMIT, false, false, false),
(RC5_COMMIT, false, false, false),
(STABLE_1_0_1_COMMIT, false, false, false),
(STABLE_1_0_4_COMMIT, true, false, false),
(STABLE_1_0_5_COMMIT, true, true, true),
(STABLE_1_0_6_COMMIT, true, true, true),
] {
let source = UpgradeSourceCapabilities::from_revision(revision)?;
assert_eq!(source.supports_delete_recovery, delete_recovery, "{revision}");
assert_eq!(source.supports_multipart_publication, multipart_publication, "{revision}");
assert_eq!(source.requires_list_parts_read_quorum, list_parts_quorum, "{revision}");
}
assert!(UpgradeSourceCapabilities::from_revision("ffffffffffffffffffffffffffffffffffffffff").is_err());
Ok(())
}
#[test]
fn source_revision_parser_requires_lowercase_forty_byte_sha() -> TestResult {
let binary = Path::new("rustfs");
@@ -365,7 +422,7 @@ async fn complete_pending_multipart(
.await?)
}
async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
pub(crate) async fn write_multipart(client: &Client, bucket: &str, key: &str, parts: &[Vec<u8>]) -> TestResult {
let pending = stage_multipart(client, bucket, key, parts).await?;
complete_pending_multipart(client, bucket, key, &pending).await?;
Ok(())
@@ -1011,8 +1068,7 @@ async fn exercise_mixed_cluster(
vec![u8::try_from(MULTIPART_WORKERS - 1)?; 64 * 1024]
);
// 1.0.4 already implements the delete decision protocol. Use a separate
// old-writer key so the original seed still spans the entire rollout.
// Use a separate old-writer key so the original seed still spans the entire rollout.
if previous_has_delete_recovery {
let key = format!(".upgrade-delete/{phase}/previous-writer");
previous_client
@@ -2293,7 +2349,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let previous_binary = source_binary()?;
// Bind the expected protocol to the verified release source, never to the
// DELETE result itself. The older CI source pins retain their 503 contract.
let previous_has_delete_recovery = binary_source_revision(&previous_binary).await? == STABLE_1_0_4_COMMIT;
let source = UpgradeSourceCapabilities::from_binary(&previous_binary).await?;
let evidence_context = g09_evidence_context(&previous_binary).await?;
let current_binary = rustfs_binary_path();
let mut cluster = RustFSTestClusterEnvironment::new(MIXED_NODE_COUNT).await?;
@@ -2315,17 +2371,21 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
cluster.stop_node(0)?;
cluster.start_node_from_binary(0, &current_binary).await?;
let mut mixed_deleted_keys = Vec::new();
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, previous_has_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, "one-current-node", 0, 1, source.supports_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
// One controlled out-of-policy attempt proves fail-closed behavior below
// native write quorum; positive MPU writers remain paused while mixed.
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
let mixed_completion = if source.supports_multipart_publication {
let clients = cluster.create_all_clients()?;
Some(finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?)
} else {
assert_mixed_pending_refusal(&cluster, &pending, &sources).await?;
None
};
for (node_idx, phase) in [(1, "two-current-nodes"), (2, "one-previous-node")] {
cluster.stop_node(node_idx)?;
cluster.start_node_from_binary(node_idx, &current_binary).await?;
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, previous_has_delete_recovery).await? {
if let Some(key) = exercise_mixed_cluster(&cluster, phase, 0, 3, source.supports_delete_recovery).await? {
mixed_deleted_keys.push(key);
}
}
@@ -2335,6 +2395,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
let clients = cluster.create_all_clients()?;
wait_for_drive_readmission(&clients, MIXED_BUCKET, "homogeneous-current", DRIVE_READMISSION_TIMEOUT).await?;
let completed_pending = finish_pending_multipart(&clients, &pending, pending_parts.concat()).await?;
if let Some(mixed) = mixed_completion {
assert_eq!(
completed_pending.version_id, mixed.version_id,
"full upgrade changed the completed version"
);
assert_eq!(completed_pending.etag, mixed.etag, "full upgrade changed the completed ETag");
}
exercise_homogeneous_multipart(&cluster, "homogeneous-current").await?;
verify_load_bodies(&clients, "homogeneous-previous", "multipart").await?;
assert_mixed_deletes_visible(&clients, &mixed_deleted_keys).await?;
@@ -2451,7 +2518,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"phases": phases,
"objects_per_phase": objects_per_phase,
"mixed_load": "plain-put",
"mixed_delete_protocol_supported": previous_has_delete_recovery,
"mixed_delete_protocol_supported": source.supports_delete_recovery,
"mixed_deleted_keys": mixed_deleted_keys,
"homogeneous_multipart_phases": ["homogeneous-previous", "homogeneous-current", "homogeneous-rollback"],
"multipart_objects_per_homogeneous_phase": MULTIPART_WORKERS * MULTIPART_UPLOADS_PER_WORKER,
@@ -2461,7 +2528,7 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"all nodes list every mixed-version object after homogeneous-current convergence",
"the previous-release seed survives the baseline restart and every upgrade phase",
"previous readers read every current-native MPU body after quiescent rollback",
"same legacy upload resumes with identical body, ETag and version after full upgrade"
"the same previous-release upload remains readable after full upgrade and rollback"
],
}),
)?;
@@ -2483,9 +2550,13 @@ async fn rolling_upgrade_from_previous_release_preserves_mixed_version_contracts
"previous-release writer publishes objects readable by current node",
"mixed phases retain 16x16 plain PUT writers while positive multipart writers are paused",
"16x16 multipart load runs in homogeneous previous, current and rollback phases",
"one-current-node Complete returns typed 503 and preserves actual source authority and parts",
"the same legacy upload completes and replays once fully upgraded without adding a version",
if previous_has_delete_recovery {
if source.supports_multipart_publication {
"one-current-node Complete and replay return the same version and ETag, and all readers agree on the body"
} else {
"one-current-node Complete returns typed 503 and preserves actual source authority and parts"
},
"fully upgraded readers see the same body, ETag and sole version after replay",
if source.supports_delete_recovery {
"mixed-version DELETE succeeds and stays absent through every reader after upgrade and rollback"
} else {
"modern DELETE fails closed and preserves every byte while old peers remain"
@@ -2832,7 +2903,7 @@ async fn restart_from_binary(env: &mut RustFSTestEnvironment, binary: &Path, ser
env.start_rustfs_server_from_binary(binary, vec![], server_env).await
}
async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
pub(crate) async fn set_bucket_quota(env: &RustFSTestEnvironment, bucket: &str, quota_bytes: u64) -> TestResult {
let path = format!("/rustfs/admin/v3/quota/{bucket}");
let body = serde_json::json!({ "quota": quota_bytes, "quota_type": "HARD" }).to_string();
let deadline = Instant::now() + QUOTA_READINESS_TIMEOUT;
@@ -3386,6 +3457,8 @@ async fn direct_upgrade_from_previous_release_preserves_bucket_configuration() -
#[tokio::test]
#[ignore = "requires a pinned previous RustFS release binary"]
async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestResult {
use sha2::Digest;
init_logging();
let previous_binary = source_binary()?;
let evidence_context = g09_evidence_context(&previous_binary).await?;
@@ -3423,6 +3496,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
let multipart_bytes = multipart_parts.concat();
write_multipart(&new_client, ROLLBACK_BUCKET, multipart_key, &multipart_parts).await?;
// This release can finish an upload staged by the current gateway.
let inflight = if binary_source_revision(&previous_binary).await? == STABLE_1_0_5_COMMIT {
env.create_test_bucket(PENDING_MULTIPART_BUCKET).await?;
let parts = [vec![1; LAYOUT_PART_SIZE], vec![2; LAYOUT_PART_SIZE], vec![3; 4096]];
let pending = stage_multipart(&new_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &parts).await?;
Some((pending, parts.concat()))
} else {
None
};
restart_from_binary(&mut env, &previous_binary, &server_env).await?;
let old_client = env.create_s3_client();
@@ -3438,6 +3521,16 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
assert_eq!(encryption, Some(ServerSideEncryption::Aes256));
assert_eq!(body, multipart_bytes, "the multipart object did not survive the rollback");
let rollback_multipart = if let Some((pending, expected)) = inflight {
assert_pending_parts(&old_client, &pending).await?;
complete_pending_multipart(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, &pending).await?;
let (_, body) = read_object(&old_client, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(body, expected);
Some((pending.part_sizes, body))
} else {
None
};
// A PUT on the rolled-back release must still honour the encryption
// configuration it decoded out of the current build's metadata blob.
let post_rollback_key = "rollback/written-after-rollback";
@@ -3478,6 +3571,25 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
post_rollback_bytes
);
if let Some((part_sizes, rollback_read)) = rollback_multipart {
let (_, current_read) = read_object(&current_again, PENDING_MULTIPART_BUCKET, PENDING_MULTIPART_KEY, None).await?;
assert_eq!(current_read, rollback_read);
if let Some(context) = &evidence_context {
let evidence = serde_json::json!({
"previous_revision": context.previous_revision,
"current_revision": context.current_revision,
"part_sizes": part_sizes,
"bytes": current_read.len(),
"rollback_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(&rollback_read), hex_simd::AsciiCase::Lower),
"roll_forward_read_sha256": hex_simd::encode_to_string(sha2::Sha256::digest(&current_read), hex_simd::AsciiCase::Lower),
});
std::fs::write(
context.directory.join("inflight-multipart-rollback.json"),
serde_json::to_vec_pretty(&evidence)?,
)?;
}
}
if let Some(context) = evidence_context.as_ref() {
write_g09_evidence(
context,
-1
View File
@@ -258,7 +258,6 @@ windows-sys = { workspace = true, features = [
windows-sys = { workspace = true, features = ["Win32_System_Ioctl"] }
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
aws-smithy-async.workspace = true
rustfs-s3-ops.workspace = true
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
+2 -2
View File
@@ -454,8 +454,8 @@ pub mod data_usage {
DATA_USAGE_CACHE_NAME, apply_bucket_usage_memory_overlay, compute_bucket_usage,
init_compression_total_memory_from_backend, invalidate_admin_data_usage_snapshot_cache,
invalidate_data_usage_snapshot_cache, live_bucket_usage_computations, load_admin_data_usage_from_backend_cached,
load_admin_data_usage_with_memory_overlay, load_bucket_usage_from_backend_cached, load_compression_total_from_memory,
load_data_usage_from_backend, load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
load_bucket_usage_from_backend_cached, load_compression_total_from_memory, load_data_usage_from_backend,
load_data_usage_from_backend_cached, quota_object_size, record_bucket_delete_marker_memory,
record_bucket_object_delete_memory, record_bucket_object_version_write_memory, record_bucket_object_write_memory,
record_bucket_object_write_unknown_previous_memory, record_compression_total_memory,
refresh_bucket_usage_from_object_layer, refresh_versioned_bucket_usage_from_object_layer,
+21 -116
View File
@@ -1463,36 +1463,22 @@ async fn load_admin_data_usage_from_backend(store: Arc<ECStore>) -> Result<DataU
let observed = load_observed_data_usage_snapshot(store.clone()).await.ok().flatten();
let (selected, selected_is_current_format) =
select_admin_data_usage_snapshot(authoritative, source.is_authoritative(), observed);
Ok(normalize_loaded_data_usage(selected, selected_is_current_format).await.0)
}
async fn project_admin_data_usage_for_current_buckets(
store: Arc<ECStore>,
mut response: DataUsageInfo,
with_memory_overlay: bool,
) -> DataUsageInfo {
let mut response = normalize_loaded_data_usage(selected, selected_is_current_format).await.0;
use futures::StreamExt as _;
let snapshot_measured = response.is_complete_bucket_usage_snapshot() || response.is_valid_partial_snapshot();
let overlay_authoritative = with_memory_overlay && snapshot_measured && !runtime_sources::setup_is_dist_erasure().await;
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
let mut incarnations = HashMap::new();
{
// Identity reads happen after releasing snapshot and refresh locks,
// including on cache hits. Metadata transaction reads linearize each
// bucket against creation/deletion without relying on peer notices.
// The common deadline cancels unfinished work and bounds added latency.
let mut buckets = response
.buckets_usage
// Display-only identity reads must not serialize every bucket behind a
// slow metadata lock while holding the shared admin refresh permit.
// The common deadline bounds their total added latency; dropping this
// stream cancels all unfinished reads without spawning detached work.
// Own the names and store across awaits so this future remains Send
// when composed into spawned metrics collectors.
let buckets = response
.usage_snapshot_bucket_observations
.keys()
.chain(response.usage_snapshot_bucket_observations.keys())
.cloned()
.collect::<HashSet<_>>();
if overlay_authoritative {
// Copy only names while holding the memory lock, then release it
// before storage I/O. A later new entry has no identity proof and
// will remain unknown in this response.
buckets.extend(memory_cache().read().await.keys().cloned());
}
.collect::<Vec<_>>();
let mut identities = futures::stream::iter(buckets)
.map(move |bucket| {
let store = store.clone();
@@ -1506,6 +1492,7 @@ async fn project_admin_data_usage_for_current_buckets(
let Ok(Some((bucket, identity))) = tokio::time::timeout_at(deadline, identities.next()).await else {
break;
};
// Unavailable, timed-out or recreated buckets retain canonical values.
if let Ok(incarnation) = identity
&& !incarnation.is_nil()
{
@@ -1514,17 +1501,7 @@ async fn project_admin_data_usage_for_current_buckets(
}
}
response.project_bucket_observations_for_admin(&incarnations);
if apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await {
response.usage_snapshot_complete = false;
response.usage_snapshot_partial = true;
response.usage_snapshot_converged = Some(false);
response.usage_snapshot_admin_projection = true;
response.replication_info.clear();
response.tier_stats = None;
response.unknown_tier_stats = None;
}
response.retain_current_bucket_usage_for_admin(&incarnations);
response
Ok(response)
}
fn discard_incomplete_bucket_usage(data_usage_info: &mut DataUsageInfo) {
@@ -1713,7 +1690,11 @@ async fn project_data_usage_from_backend_cached<T>(
}
}
async fn load_admin_data_usage_snapshot_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
/// Load the freshest structurally complete snapshot for authenticated admin
/// observability. A scan raced by namespace activity may be selected here, but
/// never by [`load_data_usage_from_backend_cached`], which remains the
/// converged source for quota admission.
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let ttl = Duration::from_secs(DATA_USAGE_CACHE_TTL_SECS);
loop {
@@ -1762,38 +1743,6 @@ async fn load_admin_data_usage_snapshot_cached(store: Arc<ECStore>) -> Result<Da
}
}
/// Load cached scanner statistics for authenticated admin observability.
///
/// Each response verifies bucket identities against current storage metadata,
/// including cache hits. Missing, failed, or timed-out identity reads leave that
/// bucket's usage absent (unknown). Legacy values without an incarnation stay
/// unknown until a scanner snapshot certifies their identity. Per-bucket reads
/// linearize independently; this is not an atomic namespace snapshot.
/// Identity reads have a shared deadline and do not outlive cancellation. The
/// unchanged cached baseline remains independent of quota and persistence.
///
/// # Errors
/// Returns the cached snapshot loading error when no baseline can be loaded.
pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
Ok(project_admin_data_usage_for_current_buckets(store, response, false).await)
}
/// Load the admin view with the existing authoritative single-process overlay.
///
/// Only a complete or valid partial scanner snapshot can receive memory values.
/// The final identity check excludes statistics from retired or unverified buckets.
/// Distributed nodes continue to use scanner statistics without local totals.
/// Cancellation, unknown values, and per-bucket read semantics are the same as
/// [`load_admin_data_usage_from_backend_cached`]. No overlay is persisted.
///
/// # Errors
/// Returns the cached snapshot loading error when no baseline can be loaded.
pub async fn load_admin_data_usage_with_memory_overlay(store: Arc<ECStore>) -> Result<DataUsageInfo, Error> {
let response = load_admin_data_usage_snapshot_cached(store.clone()).await?;
Ok(project_admin_data_usage_for_current_buckets(store, response, true).await)
}
/// Invalidate the process-local persisted usage snapshot cache after a durable
/// scanner write.
pub async fn invalidate_data_usage_snapshot_cache() {
@@ -2856,17 +2805,16 @@ async fn replace_bucket_usage_memory_from_info_if_generation(data_usage_info: &D
*cache = next_cache;
}
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) -> bool {
async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mut DataUsageInfo, authoritative: bool) {
if !authoritative {
return false;
return;
}
let cache = memory_cache().read().await;
if cache.is_empty() {
return false;
return;
}
let was_complete = data_usage_info.usage_snapshot_complete;
let persisted_update = data_usage_info.last_update;
let mut changed = false;
let mut added_bucket = false;
@@ -2903,7 +2851,6 @@ async fn apply_bucket_usage_memory_overlay_if_authoritative(data_usage_info: &mu
data_usage_info.buckets_count = data_usage_info.buckets_usage.len() as u64;
data_usage_info.calculate_totals();
}
changed || was_complete != data_usage_info.usage_snapshot_complete
}
pub async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut DataUsageInfo) {
@@ -3797,23 +3744,6 @@ mod tests {
*cache_updating().write().await = false;
}
async fn bind_admin_usage_test_identities(store: &Arc<ECStore>, snapshot: &mut DataUsageInfo) {
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
for (bucket, usage) in &mut snapshot.buckets_usage {
store
.make_bucket(bucket, &crate::storage_api_contracts::bucket::MakeBucketOptions::default())
.await
.expect("create a real bucket for current admin identity validation");
usage.bucket_incarnation = Some(
store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("read the storage-owned fixture identity")
.to_string(),
);
}
}
fn data_usage_info_for_test(bucket: &str, objects_count: u64, size: u64, last_update: SystemTime) -> DataUsageInfo {
let mut info = DataUsageInfo {
last_update: Some(last_update),
@@ -4349,11 +4279,7 @@ mod tests {
use crate::config::com::save_config;
let (_dirs, store) = isolated_store_over_temp_disks().await;
let (mut authoritative, mut observed) = usage_snapshots_with_missing_observed_set();
bind_admin_usage_test_identities(&store, &mut authoritative).await;
for (bucket, usage) in &mut observed.buckets_usage {
usage.bucket_incarnation = authoritative.buckets_usage[bucket].bucket_incarnation.clone();
}
let (authoritative, observed) = usage_snapshots_with_missing_observed_set();
let primary_bytes = serde_json::to_vec(&authoritative).expect("encode complete usage");
let observed_bytes = serde_json::to_vec(&observed).expect("encode partial observation");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), primary_bytes.clone())
@@ -4402,11 +4328,6 @@ mod tests {
reconciled.scanner_epoch = Some(4);
reconciled.scanner_cycle = Some(12);
reconciled.usage_snapshot_converged = Some(true);
reconciled
.buckets_usage
.get_mut("large-bucket")
.expect("reconciled bucket")
.bucket_incarnation = authoritative.buckets_usage["large-bucket"].bucket_incarnation.clone();
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -4928,12 +4849,6 @@ mod tests {
backup.scanner_epoch = Some(4);
backup.scanner_cycle = Some(10);
backup.usage_snapshot_converged = Some(true);
bind_admin_usage_test_identities(&store, &mut current).await;
backup
.buckets_usage
.get_mut("existing")
.expect("backup bucket")
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
for (path, info) in [
(DATA_USAGE_OBJ_NAME_PATH.as_str(), &current),
(DATA_USAGE_OBJ_BACKUP_PATH.as_str(), &backup),
@@ -4956,11 +4871,6 @@ mod tests {
after_delete.scanner_epoch = current.scanner_epoch;
after_delete.scanner_cycle = current.scanner_cycle;
after_delete.usage_snapshot_converged = Some(true);
after_delete
.buckets_usage
.get_mut("existing")
.expect("post-delete bucket")
.bucket_incarnation = current.buckets_usage["existing"].bucket_incarnation.clone();
let mut stale_observation =
data_usage_info_for_test("retired-incarnation", 99, 999, SystemTime::UNIX_EPOCH + Duration::from_secs(4));
stale_observation.scanner_epoch = Some(4);
@@ -5022,7 +4932,6 @@ mod tests {
authoritative.scanner_epoch = Some(7);
authoritative.scanner_cycle = Some(10);
authoritative.usage_snapshot_converged = Some(true);
bind_admin_usage_test_identities(&store, &mut authoritative).await;
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
@@ -5037,10 +4946,6 @@ mod tests {
info.scanner_cycle = Some(cycle);
info.usage_snapshot_converged = Some(false);
info.usage_snapshot_authoritative_baseline = Some(authoritative.snapshot_identity());
info.buckets_usage
.get_mut("bucket")
.expect("observed bucket")
.bucket_incarnation = authoritative.buckets_usage["bucket"].bucket_incarnation.clone();
info
};
let first = observation(7, 11, 200, 11, 1_100);
@@ -221,7 +221,7 @@ async fn companion_admin_cache_isolated_from_quota_memory_and_persistence() {
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_buckets() {
async fn blocked_companion_identity_keeps_canonical_without_delaying_healthy_buckets() {
use crate::bucket::metadata_sys::{
acquire_metadata_transaction_guard, init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks,
};
@@ -280,11 +280,8 @@ async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_bucke
.await
.expect("optional projection must return before the blocked metadata lock's long timeout")
.expect("canonical admin response stays available");
assert!(
!response.buckets_usage.contains_key(blocked),
"an unavailable current identity cannot bind the canonical value"
);
assert!(!response.bucket_sizes.contains_key(blocked));
assert_eq!(response.buckets_usage[blocked].size, 100);
assert_eq!(response.buckets_usage[blocked].usage_value_state, Some(BucketUsageValueState::Carried));
assert_eq!(response.buckets_usage[healthy].size, 250);
assert_eq!(response.buckets_usage[healthy].usage_value_state, Some(BucketUsageValueState::Observed));
assert!(response.usage_snapshot_admin_projection);
@@ -296,12 +293,13 @@ async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_bucke
.is_ok()
);
drop(blocked_guard);
invalidate_data_usage_snapshot_cache().await;
let recovered = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("refresh after lock recovery");
assert_eq!(
recovered.buckets_usage[blocked].size, 250,
"unavailable identity can recover on a cache hit"
"unavailable identity can recover on the next refresh"
);
assert_eq!(recovered.buckets_usage[healthy].size, 250);
let quota = load_data_usage_from_backend_cached(store)
@@ -310,249 +308,3 @@ async fn blocked_companion_identity_stays_unknown_without_delaying_healthy_bucke
assert_eq!(quota.buckets_usage[blocked].size, 100);
assert_eq!(quota.buckets_usage[healthy].size, 100);
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn cached_canonical_admin_usage_rechecks_current_identity_without_peer_invalidation() {
use crate::bucket::metadata::{load_bucket_incarnation, save_bucket_incarnation};
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "usage-recreated-cache";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create identity fixture");
let old_identity = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("original identity");
let mut canonical = companion_snapshot(bucket, old_identity);
canonical.usage_snapshot_bucket_observations.clear();
canonical.usage_snapshot_converged = Some(true);
canonical.usage_snapshot_pending_buckets.clear();
canonical.usage_snapshot_pending_bucket_count = 0;
canonical
.buckets_usage
.get_mut(bucket)
.expect("canonical bucket")
.objects_count = 107;
canonical.calculate_totals();
let original_bytes = serde_json::to_vec(&canonical).expect("encode canonical old bucket");
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), original_bytes.clone())
.await
.expect("persist canonical root");
invalidate_data_usage_snapshot_cache().await;
let first = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("prime raw admin cache");
assert_eq!(first.buckets_usage[bucket].objects_count, 107);
assert!(first.is_complete_bucket_usage_snapshot());
let cache_generation = admin_data_usage_snapshot_generation();
let new_identity = uuid::Uuid::new_v4();
// Simulate the storage-owned identity changing on another node while this
// process has not received its invalidation notice. Do not clear the cache.
save_bucket_incarnation(store.clone(), bucket, new_identity)
.await
.expect("publish changed disk identity");
assert_eq!(
load_bucket_incarnation(store.clone(), bucket)
.await
.expect("read new sidecar"),
Some(new_identity)
);
let second = load_admin_data_usage_from_backend_cached(store.clone())
.await
.expect("read raw cache with fresh identity");
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
assert!(
!second.buckets_usage.contains_key(bucket),
"same name cannot retain the predecessor's 107"
);
assert!(!second.bucket_sizes.contains_key(bucket));
assert_eq!(second.usage_snapshot_converged, Some(false));
assert!(second.usage_snapshot_admin_projection);
assert!(reject_admin_usage_projection(&second).is_err());
assert!(certified_bucket_usage(&second, bucket, new_identity).is_none());
assert!(parse_usage_snapshot(&serde_json::to_vec(&second).expect("encode display response")).is_err());
let quota = load_data_usage_from_backend_cached(store.clone())
.await
.expect("independent quota baseline");
assert_eq!(quota.buckets_usage[bucket].objects_count, 107);
assert_eq!(
read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
.await
.expect("unchanged persisted root"),
original_bytes
);
// Rebinding a cached raw baseline is transient: recovery does not need to
// reload or permanently blank a healthy bucket's statistics.
save_bucket_incarnation(store.clone(), bucket, old_identity)
.await
.expect("restore fixture identity");
let recovered = load_admin_data_usage_from_backend_cached(store)
.await
.expect("recover on another cache hit");
assert_eq!(recovered.buckets_usage[bucket].objects_count, 107);
assert!(recovered.is_complete_bucket_usage_snapshot());
assert_eq!(admin_data_usage_snapshot_generation(), cache_generation);
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_memory_overlay_requires_a_measured_scanner_baseline() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
assert!(!runtime_sources::setup_is_dist_erasure().await);
let bucket = "usage-unmeasured-overlay";
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
let mut baseline = data_usage_info_for_test(bucket, 1, 100, SystemTime::UNIX_EPOCH);
let usage = baseline.buckets_usage.get_mut(bucket).unwrap();
usage.bucket_incarnation = Some(incarnation.to_string());
let memory_usage = BucketUsageInfo {
size: 150,
..usage.clone()
};
clear_usage_memory_cache_for_test().await;
replace_bucket_usage_memory_from_authoritative(bucket, memory_usage, SystemTime::now()).await;
// A missing snapshot and a mismatched bucket count both leave usage unknown.
for (buckets_count, expected_size) in [(None, None), (Some(2), None), (Some(1), Some(150))] {
if let Some(count) = buckets_count {
baseline.buckets_count = count;
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&baseline).unwrap())
.await
.unwrap();
}
invalidate_data_usage_snapshot_cache().await;
let response = load_admin_data_usage_with_memory_overlay(store.clone()).await.unwrap();
assert_eq!(
response.buckets_usage.get(bucket).map(|usage| usage.size),
expected_size,
"baseline count {buckets_count:?}"
);
assert_eq!(response.bucket_sizes.get(bucket).copied(), expected_size);
assert_eq!(response.objects_total_size, expected_size.unwrap_or_default());
assert_eq!(response.usage_snapshot_admin_projection, expected_size.is_some());
assert!(!response.is_complete_bucket_usage_snapshot());
assert!(!response.is_valid_partial_snapshot());
}
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_memory_overlay_cannot_reintroduce_a_predecessor_identity() {
use crate::bucket::metadata::save_bucket_incarnation;
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let bucket = "usage-recreated-overlay";
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create overlay identity fixture");
let old_identity = store
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("original identity");
let mut canonical = companion_snapshot(bucket, old_identity);
canonical.usage_snapshot_bucket_observations.clear();
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&canonical).expect("encode canonical root"),
)
.await
.expect("persist overlay fixture");
invalidate_data_usage_snapshot_cache().await;
let cached_usage = canonical.buckets_usage[bucket].clone();
replace_bucket_usage_memory_from_authoritative(bucket, cached_usage, SystemTime::now()).await;
let new_identity = uuid::Uuid::new_v4();
save_bucket_incarnation(store.clone(), bucket, new_identity)
.await
.expect("publish a replacement identity");
let response = load_admin_data_usage_with_memory_overlay(store.clone())
.await
.expect("project after memory overlay");
assert!(!response.buckets_usage.contains_key(bucket));
assert_eq!(response.usage_snapshot_converged, Some(false));
assert!(response.usage_snapshot_admin_projection);
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
#[serial]
async fn admin_overlay_flags_only_change_keeps_identity_bound_cold_usage() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::config::com::save_config;
use crate::storage_api_contracts::bucket::MakeBucketOptions;
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
assert!(
!runtime_sources::setup_is_dist_erasure().await,
"exercise the production single-process overlay"
);
let cold = "usage-overlay-cold";
let unverified = "usage-overlay-unverified";
for bucket in [cold, unverified] {
store
.make_bucket(bucket, &MakeBucketOptions::default())
.await
.expect("create overlay fixture bucket");
}
let identity = store
.bucket_incarnation_id_from_disk(cold)
.await
.expect("cold bucket identity");
let mut canonical = companion_snapshot(cold, identity);
canonical.usage_snapshot_bucket_observations.clear();
canonical.usage_snapshot_converged = Some(true);
save_config(
store.clone(),
DATA_USAGE_OBJ_NAME_PATH.as_str(),
serde_json::to_vec(&canonical).expect("encode cold canonical value"),
)
.await
.expect("persist cold baseline");
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
memory_cache().write().await.insert(
unverified.to_string(),
cached_bucket_usage_from_backend(BucketUsageInfo::default(), SystemTime::UNIX_EPOCH, false),
);
record_bucket_object_write_memory(unverified, None, 42).await;
{
let cache = memory_cache().read().await;
assert!(cache[unverified].dirty);
assert!(!cache[unverified].authoritative);
}
let response = load_admin_data_usage_with_memory_overlay(store)
.await
.expect("identity-filtered flags-only overlay");
assert_eq!(response.buckets_usage[cold].size, 100);
assert_eq!(response.objects_total_size, 100);
assert_eq!(response.bucket_sizes[cold], 100);
assert!(!response.buckets_usage.contains_key(unverified));
assert!(response.usage_snapshot_admin_projection);
assert_eq!(response.usage_snapshot_converged, Some(false));
assert!(!response.is_complete_bucket_usage_snapshot());
assert!(!response.is_valid_partial_snapshot());
clear_usage_memory_cache_for_test().await;
invalidate_data_usage_snapshot_cache().await;
}
@@ -1033,26 +1033,16 @@ mod tests {
#[tokio::test]
#[serial]
async fn server_info_reads_complete_observation_without_promoting_authority() {
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
use crate::bucket::metadata_sys::test_support::isolated_store_over_temp_disks;
use crate::config::com::{read_config, save_config};
use crate::data_usage::{
DATA_USAGE_OBJ_NAME_PATH, DATA_USAGE_OBSERVED_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache,
load_data_usage_from_backend_cached,
};
use crate::storage_api_contracts::bucket::{BucketOperations as _, MakeBucketOptions};
use rustfs_data_usage::{BucketUsageInfo, DataUsageInfo};
use std::time::{Duration, SystemTime};
let (_dirs, store) = isolated_store_over_temp_disks().await;
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
store
.make_bucket("bucket", &MakeBucketOptions::default())
.await
.expect("create a real observed bucket");
let incarnation = store
.bucket_incarnation_id_from_disk("bucket")
.await
.expect("read current storage-owned incarnation");
let primary = DataUsageInfo {
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
scanner_epoch: Some(1),
@@ -1076,7 +1066,6 @@ mod tests {
versions_count: 4,
delete_markers_count: 1,
size: 6,
bucket_incarnation: Some(incarnation.to_string()),
..Default::default()
},
)]
@@ -1128,46 +1117,6 @@ mod tests {
invalidate_data_usage_snapshot_cache().await;
}
#[test]
fn identity_filtered_usage_remains_unknown_in_server_info() {
let mut info = rustfs_data_usage::DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
buckets_count: 1,
objects_total_count: 107,
objects_total_size: 1024,
buckets_usage: [(
"bucket".to_string(),
rustfs_data_usage::BucketUsageInfo {
objects_count: 107,
size: 1024,
bucket_incarnation: Some("11111111-1111-4111-8111-111111111111".to_string()),
..Default::default()
},
)]
.into_iter()
.collect(),
usage_snapshot_complete: true,
..Default::default()
};
let current =
std::collections::HashMap::from([("bucket".to_string(), "22222222-2222-4222-8222-222222222222".to_string())]);
assert_eq!(info.retain_current_bucket_usage_for_admin(&current), 1);
assert!(info.buckets_usage.is_empty());
assert!(info.usage_snapshot_admin_projection);
assert_eq!(info.usage_snapshot_converged, Some(false));
let mut buckets = rustfs_madmin::Buckets::default();
let mut objects = rustfs_madmin::Objects::default();
let mut versions = rustfs_madmin::Versions::default();
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
let mut usage = rustfs_madmin::Usage::default();
apply_data_usage_result(Ok(info), &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
assert_eq!(buckets.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(objects.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(versions.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(delete_markers.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
assert_eq!(usage.error.as_deref(), Some(DATA_USAGE_UNAVAILABLE_ERROR));
}
#[test]
fn incomplete_data_usage_is_unavailable_in_server_info() {
let mut buckets = rustfs_madmin::Buckets::default();
+16 -36
View File
@@ -25,10 +25,8 @@
//! Not responsible for: validating configurations, touching the bucket
//! metadata store, or choosing a canonical shape.
//!
//! Upstream: the bucket-config `deserialize`/`serialize` pair, reached through the
//! integration-test facade `tests/storage_api.rs`, and the configuration types they are
//! generic over (`BucketConfigXml`: the gateway persistence shapes for the families that
//! have moved, the s3s DTOs for the rest). Downstream: the DTO/codec
//! The tests exercise the production bucket configuration codec through `tests/storage_api.rs`.
//! The lifecycle date cases also pin the results of the legacy codec. The DTO/codec
//! replacement (T1.4) and the persisted-bytes comparison (T3.7), which must
//! keep every golden green without regenerating it. Regeneration is reserved
//! to `scripts/gen_bucket_config_goldens.sh`, which sets
@@ -506,13 +504,12 @@ mod bucket_config_goldens {
mod bucket_config_lifecycle_date_compat {
use super::*;
use rustfs_lifecycle::core::format_lifecycle_date;
use time::OffsetDateTime;
const DATES: [&str; 3] = [
"2023-11-14T22:13:20.123456789Z",
"2023-11-14T22:13:20.123456789+05:30",
"2025-01-02T00:00:00+02:00",
// Captured from the 5761ddfe reader and writer. The reader preserves the offset and
// nanoseconds. The writer emits the local clock fields at millisecond precision with a Z.
const DATES: [(&str, &str); 3] = [
("2023-11-14T22:13:20.123456789Z", "2023-11-14T22:13:20.123Z"),
("2023-11-14T22:13:20.123456789+05:30", "2023-11-14T22:13:20.123Z"),
("2025-01-02T00:00:00+02:00", "2025-01-02T00:00:00.000Z"),
];
fn document(date: &str) -> Vec<u8> {
@@ -522,33 +519,17 @@ mod bucket_config_lifecycle_date_compat {
.into_bytes()
}
fn legacy_read(input: &[u8]) -> s3s::dto::BucketLifecycleConfiguration {
let mut reader = s3s::xml::Deserializer::new(input);
let value = s3s::xml::Deserialize::deserialize(&mut reader).expect("the actual old lifecycle reader accepts the date");
reader.expect_eof().expect("the old reader consumes the entire date fixture");
value
}
#[test]
fn lifecycle_xml_reader_preserves_legacy_date_precision_and_offset() {
for date in DATES {
let input = document(date);
let old = legacy_read(&input);
let new: PersistedLifecycleConfiguration = deserialize(&input).expect("read persisted lifecycle date");
let old_date = |value: &s3s::dto::Timestamp| {
format_lifecycle_date(OffsetDateTime::from(value.clone())).expect("format the original old-reader date")
};
for (date, _) in DATES {
let new: PersistedLifecycleConfiguration = deserialize(&document(date)).expect("read persisted lifecycle date");
assert_eq!(
[
new.expiry_updated_at.clone(),
new.rules[0].expiration.as_ref().and_then(|value| value.date.clone()),
new.rules[0].transitions.as_ref().unwrap()[0].date.clone(),
],
[
Some(old_date(old.expiry_updated_at.as_ref().unwrap())),
Some(old_date(old.rules[0].expiration.as_ref().unwrap().date.as_ref().unwrap())),
Some(old_date(old.rules[0].transitions.as_ref().unwrap()[0].date.as_ref().unwrap())),
],
[Some(date.to_owned()), Some(date.to_owned()), Some(date.to_owned())],
"the lifecycle reader must retain the old date representation for {date}"
);
}
@@ -556,11 +537,10 @@ mod bucket_config_lifecycle_date_compat {
#[test]
fn lifecycle_xml_writer_preserves_legacy_date_bytes() {
for date in DATES {
let old = legacy_read(&document(date));
let mut old_xml = Vec::new();
s3s::xml::Serialize::serialize(&old, &mut s3s::xml::Serializer::new(&mut old_xml))
.expect("the actual old lifecycle writer emits the date");
for (date, written_date) in DATES {
let expected = format!(
"<LifecycleConfiguration><ExpiryUpdatedAt>{written_date}</ExpiryUpdatedAt><Rule><Expiration><Date>{written_date}</Date></Expiration><ID>date-compat</ID><Status>Enabled</Status><Transition><Date>{written_date}</Date><StorageClass>GLACIER</StorageClass></Transition></Rule></LifecycleConfiguration>"
);
let new = PersistedLifecycleConfiguration {
expiry_updated_at: Some(date.to_owned()),
rules: vec![PersistedLifecycleRule {
@@ -585,7 +565,7 @@ mod bucket_config_lifecycle_date_compat {
};
assert_eq!(
serialize(&new).expect("write persisted lifecycle date"),
old_xml,
expected.as_bytes(),
"legacy date bytes for {date}"
);
}
+47 -1
View File
@@ -1786,7 +1786,7 @@ where
}
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
self.update_user_secret_key_with_guard(access_key, secret_key, None)
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
.await
.map(|(updated_at, status, _guard)| (updated_at, status))
}
@@ -1796,6 +1796,7 @@ where
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(
OffsetDateTime,
AccountStatus,
@@ -1824,6 +1825,51 @@ where
}
}
// Lock order is parent, then the presenting STS credential. Deletion and
// session refresh use the same STS lock. Release it before the revocation sweep.
let _session_guard = match session {
Some(session) => {
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
let current = self
.api
.load_user_identity(&session.access_key, UserType::Sts)
.await
.map_err(|error| {
if is_err_no_such_user(&error) {
Error::IAMActionNotAllowed
} else {
error
}
})?;
let parent = if current.credentials.parent_user.is_empty() {
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
} else {
Some(current.credentials.parent_user.as_str())
};
use subtle::ConstantTimeEq as _;
if !session.is_valid()
|| !current.credentials.is_valid()
|| !current.credentials.is_temp()
|| current.credentials.is_service_account()
|| current.credentials.access_key != session.access_key
|| parent != Some(access_key)
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|| !bool::from(
current
.credentials
.session_token
.as_bytes()
.ct_eq(session.session_token.as_bytes()),
)
|| current.sts_generation != u.sts_generation
{
return Err(Error::IAMActionNotAllowed);
}
Some(guard)
}
None => None,
};
let mut cred = u.credentials.clone();
cred.secret_key = secret_key.to_string();
+183 -6
View File
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
}
/// Change an enabled regular caller's secret only if its signing secret is
/// still current under the parent mutation lock. Status and policies are kept.
/// Change an enabled regular user's secret only if the supplied current
/// secret still matches under the parent mutation lock. Status and policies are kept.
/// Persistence and revocation continue after caller cancellation.
pub async fn change_own_secret_key(
&self,
@@ -1433,7 +1433,27 @@ impl<T: Store> IamSys<T> {
expected_secret: &str,
secret_key: &str,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
.await
}
/// Change a local IAM password using an authenticated STS session and its parent password.
///
/// The parent and session mutation locks protect revalidation through the secret write.
/// Persistence and session revocation continue after request cancellation.
/// A revocation failure is returned after the new secret has been persisted.
///
/// # Errors
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
/// incorrect current password. Storage and revocation errors are propagated.
pub async fn change_own_secret_key_from_session(
&self,
access_key: &str,
expected_secret: &str,
secret_key: &str,
session: &Credentials,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
.await
}
@@ -1442,16 +1462,18 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
let iam = self.clone();
let access_key = access_key.to_string();
let secret_key = secret_key.to_string();
let expected_secret = expected_secret.map(str::to_string);
let session = session.cloned();
#[cfg(test)]
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
let operation = async move {
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
.await
};
#[cfg(test)]
@@ -1471,6 +1493,7 @@ impl<T: Store> IamSys<T> {
access_key: &str,
secret_key: &str,
expected_secret: Option<&str>,
session: Option<&Credentials>,
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
if !is_access_key_valid(access_key) {
return Err(IamError::InvalidAccessKeyLength);
@@ -1482,7 +1505,7 @@ impl<T: Store> IamSys<T> {
let (updated_at, status, parent_guard) = self
.store
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
.await?;
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
drop(parent_guard);
@@ -4815,6 +4838,160 @@ mod tests {
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
}
#[tokio::test]
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
ensure_test_global_credentials();
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
let store = StsTestMockStore::new(false);
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
let parent = "session-change-parent";
let password = "currentParentSecret123";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: password.to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "password-session");
let owner = iam.get_user(parent).await.expect("parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
let pending = {
let iam = Arc::clone(&iam);
let session = session.clone();
tokio::spawn(async move {
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
.await
})
};
if mutation == "revoke" {
iam.delete_temp_account(&session.access_key, false)
.await
.expect("revoke presenting session");
} else {
let mut saved = store.saved_sts_users.lock().expect("saved identities");
let current = saved.get_mut(&session.access_key).expect("durable session");
match mutation {
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
"disable" => current.credentials.status = "off".to_string(),
"parent" => current.credentials.parent_user = "another-parent".to_string(),
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
"token" => current.credentials.session_token = "replacement-token".to_string(),
"generation" => {
assert!(current.sts_generation.take().is_some());
}
_ => unreachable!("known fixture mutation"),
}
}
drop(guard);
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
.await
.expect("queued rotation finishes")
.expect("rotation task")
.expect_err("stale session must fail");
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
let saved = store.saved_sts_users.lock().expect("saved identities");
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
}
}
#[tokio::test]
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
let iam = test_iam_sys().await;
let parent = "stale-password-parent";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: "cachedParentSecret123".to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "fresh-password-session");
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
iam.store
.api
.saved_sts_users
.lock()
.expect("saved identities")
.get_mut(parent)
.expect("durable parent")
.credentials
.secret_key = "durableParentSecret123".to_string();
assert_eq!(
iam.store
.get_user(parent)
.await
.expect("stale cached parent")
.credentials
.secret_key,
"cachedParentSecret123"
);
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
crate::LOAD_USER_NOTIFICATION_PROBE
.scope(
probe,
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
)
.await
.expect("durable current password authorizes the change");
assert_eq!(
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
"replacementParentSecret123"
);
}
#[tokio::test]
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
let iam = test_iam_sys().await;
let parent = "current-session-parent";
let password = "currentParentSecret123";
iam.create_user(
parent,
&AddOrUpdateUserReq {
secret_key: password.to_string(),
policy: None,
status: rustfs_madmin::AccountStatus::Enabled,
},
)
.await
.expect("create parent");
let session = rotation_test_session(parent, "current-password-session");
let owner = iam.get_user(parent).await.expect("parent").credentials;
iam.set_temp_user_from_parent(&session, &owner)
.await
.expect("publish session");
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
let (_, _, revoked) = tokio::time::timeout(
std::time::Duration::from_secs(5),
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
probe,
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
),
)
.await
.expect("rotation must release its session lock before revocation")
.expect("rotate current session");
assert_eq!(revoked, 1);
assert!(iam.get_user(&session.access_key).await.is_none());
assert_eq!(
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
"replacementParentSecret123"
);
}
#[tokio::test]
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
ensure_test_global_credentials();
+1 -1
View File
@@ -6,7 +6,7 @@ Applies to `crates/kms/`.
When changing key-management behavior, verify compatibility with:
- `rustfs/src/storage/ecfs.rs`
- `rustfs/src/storage/sse.rs`
- `rustfs/src/admin/handlers/kms.rs`
- `rustfs/src/admin/handlers/kms_dynamic.rs`
- `rustfs/src/admin/handlers/kms_keys.rs`
+54 -55
View File
@@ -28,6 +28,7 @@ use tracing_subscriber::{
use super::http2_ping::HYPER_HTTP2_CLIENT_TARGET;
pub const HTTP_SERVER_LOG_TARGET: &str = "rustfs::server::http";
const S3_DEPENDENCY_TARGETS: [&str; 2] = ["s3s", "rustfs_gateway"];
/// Pyroscope emits raw reqwest errors from its background session manager.
/// Those errors can include the configured endpoint, so they must never reach
@@ -44,12 +45,13 @@ pub(super) fn pyroscope_log_filter() -> FilterFn {
filter_fn(allows_non_pyroscope_dependency_event)
}
/// s3s verbose events include response bodies and expected signatures. Keep
/// diagnostics spans, but never let these events reach logs or OTLP exporters.
pub(super) fn s3s_log_filter() -> FilterFn {
/// Keep verbose protocol events out of logs and OTLP while preserving diagnostic spans.
pub(super) fn s3_dependency_log_filter() -> FilterFn {
filter_fn(|metadata| {
!(metadata.is_event()
&& (metadata.target() == "s3s" || metadata.target().starts_with("s3s::"))
&& S3_DEPENDENCY_TARGETS
.iter()
.any(|target| directive_applies_to_target(target, metadata.target()))
&& *metadata.level() > tracing::Level::INFO)
})
}
@@ -60,7 +62,7 @@ pub(super) fn s3s_log_filter() -> FilterFn {
/// `RUST_LOG` environment variable takes precedence over `logger_level`.
/// For non-verbose levels (`info`, `warn`, `error`), noisy dependency targets
/// are automatically suppressed based on the effective log configuration.
/// Transport internals are disabled, while `s3s` remains visible at WARN. The
/// Transport internals are disabled, while S3 protocol warnings remain visible. The
/// diagnostic HTTP/2 PING feature adds back only Hyper's client target at DEBUG
/// so its exact keepalive timeout event reaches the metrics layer.
///
@@ -179,33 +181,14 @@ fn effective_level_for_target<'a>(rust_log: &'a str, target: &str) -> Option<&'a
best_match.map(|(_, _, level)| level)
}
fn should_demote_http_request_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
fn should_demote_target(target: &str, logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
if let Some(level) = default_level {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
if let Some(rust_log) = rust_log {
if let Some(level) = effective_level_for_target(rust_log, HTTP_SERVER_LOG_TARGET) {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
return false;
}
let level = logger_level.trim().to_ascii_lowercase();
matches!(level.as_str(), "info" | "warn")
}
fn should_demote_s3s_logs(logger_level: &str, default_level: Option<&str>, rust_log: Option<&str>) -> bool {
if let Some(level) = default_level {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
if let Some(rust_log) = rust_log {
if let Some(level) = effective_level_for_target(rust_log, "s3s") {
if let Some(level) = effective_level_for_target(rust_log, target) {
let level = level.trim().to_ascii_lowercase();
return matches!(level.as_str(), "info" | "warn");
}
@@ -259,17 +242,16 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
("tower", LevelFilter::OFF),
];
if should_demote_http_request_logs(logger_level, default_level, rust_log_env.as_deref()) {
if should_demote_target(HTTP_SERVER_LOG_TARGET, logger_level, default_level, rust_log_env.as_deref()) {
// HTTP request logs are demoted to WARN to reduce volume in production,
// but only when the effective log level is not stricter than WARN.
directives.push((HTTP_SERVER_LOG_TARGET, LevelFilter::WARN));
}
if should_demote_s3s_logs(logger_level, default_level, rust_log_env.as_deref()) {
// s3s instruments each authentication request through multiple
// nested INFO spans. Keep warnings and errors in normal production
// logs without promoting it when the effective base is stricter.
directives.push(("s3s", LevelFilter::WARN));
for target in S3_DEPENDENCY_TARGETS {
if should_demote_target(target, logger_level, default_level, rust_log_env.as_deref()) {
directives.push((target, LevelFilter::WARN));
}
}
for (crate_name, level) in directives {
@@ -363,14 +345,14 @@ mod tests {
}
#[test]
fn s3s_filter_blocks_credentials_despite_explicit_verbose_directives() {
fn s3_dependency_filter_blocks_verbose_events_despite_explicit_directives() {
use tracing_subscriber::prelude::*;
for directive in ["trace", concat!("trace,", "s3s::service=trace,", "s3s::ops::signature=debug")] {
let writer = SharedWriter::default();
let captured = Arc::clone(&writer.0);
let subscriber = tracing_subscriber::registry()
.with(EnvFilter::new(directive))
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(
tracing_subscriber::fmt::layer()
.without_time()
@@ -384,6 +366,10 @@ mod tests {
tracing::debug!(target: "s3s::service", response = "secret-response", "response dump");
tracing::trace!(target: "s3s", response = "secret-trace", "response dump");
tracing::debug!(target: "s3s::ops::signature", expected = "secret-signature", "signature mismatch");
tracing::debug!(target: "rustfs_gateway::service", "secret-gateway-debug");
tracing::trace!(target: "rustfs_gateway", "secret-gateway-trace");
tracing::info!(target: "rustfs_gateway::service", "visible-gateway-info");
tracing::debug!(target: "rustfs_gateway_other", "visible-gateway-neighbor");
tracing::info!(target: "s3s::service", "visible-info");
tracing::warn!(target: "s3s::service", "visible-warning");
tracing::debug!(target: "rustfs", "visible-rustfs-debug");
@@ -393,6 +379,8 @@ mod tests {
assert!(!output.contains("secret"), "{output}");
for expected in [
"safe-operation-span",
"visible-gateway-info",
"visible-gateway-neighbor",
"visible-info",
"visible-warning",
"visible-rustfs-debug",
@@ -445,22 +433,29 @@ mod tests {
#[test]
fn test_should_demote_http_request_logs() {
assert!(should_demote_http_request_logs("info", None, None));
assert!(should_demote_http_request_logs("warn", None, None));
assert!(!should_demote_http_request_logs("error", None, None));
assert!(!should_demote_http_request_logs("off", None, None));
assert!(!should_demote_http_request_logs("info", None, Some("ERROR")));
assert!(should_demote_http_request_logs("error", None, Some("WARN")));
assert!(!should_demote_http_request_logs("info", None, Some("foo=warn")));
assert!(!should_demote_http_request_logs("info", None, Some("rustfs=error")));
assert!(!should_demote_http_request_logs("info", None, Some("rustfs::server=error")));
assert!(!should_demote_http_request_logs("info", None, Some(&http_log_directive("error"))));
assert!(!should_demote_http_request_logs(
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, None));
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "warn", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "off", None, None));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("ERROR")));
assert!(should_demote_target(HTTP_SERVER_LOG_TARGET, "error", None, Some("WARN")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("foo=warn")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs=error")));
assert!(!should_demote_target(HTTP_SERVER_LOG_TARGET, "info", None, Some("rustfs::server=error")));
assert!(!should_demote_target(
HTTP_SERVER_LOG_TARGET,
"info",
None,
Some(&http_log_directive("error"))
));
assert!(!should_demote_target(
HTTP_SERVER_LOG_TARGET,
"info",
None,
Some(&format!("WARN,{}", http_log_directive("error")))
));
assert!(should_demote_http_request_logs(
assert!(should_demote_target(
HTTP_SERVER_LOG_TARGET,
"error",
None,
Some(&format!("WARN,{}", http_log_directive("warn")))
@@ -469,15 +464,15 @@ mod tests {
#[test]
fn test_should_demote_s3s_logs() {
assert!(should_demote_s3s_logs("info", None, None));
assert!(should_demote_s3s_logs("warn", None, None));
assert!(!should_demote_s3s_logs("error", None, None));
assert!(!should_demote_s3s_logs("off", None, None));
assert!(!should_demote_s3s_logs("info", None, Some("ERROR")));
assert!(should_demote_s3s_logs("error", None, Some("WARN")));
assert!(!should_demote_s3s_logs("info", None, Some("foo=warn")));
assert!(!should_demote_s3s_logs("info", None, Some("s3s=error")));
assert!(should_demote_s3s_logs("error", None, Some("WARN,s3s=warn")));
assert!(should_demote_target("s3s", "info", None, None));
assert!(should_demote_target("s3s", "warn", None, None));
assert!(!should_demote_target("s3s", "error", None, None));
assert!(!should_demote_target("s3s", "off", None, None));
assert!(!should_demote_target("s3s", "info", None, Some("ERROR")));
assert!(should_demote_target("s3s", "error", None, Some("WARN")));
assert!(!should_demote_target("s3s", "info", None, Some("foo=warn")));
assert!(!should_demote_target("s3s", "info", None, Some("s3s=error")));
assert!(should_demote_target("s3s", "error", None, Some("WARN,s3s=warn")));
}
#[test]
@@ -619,10 +614,14 @@ mod tests {
let output = capture_with_filter(build_env_filter("info", None), || {
tracing::info!(target: "s3s::auth", "s3s info must be suppressed");
tracing::warn!(target: "s3s::auth", "s3s warning must remain");
tracing::info!(target: "rustfs_gateway::service", "gateway info must be suppressed");
tracing::warn!(target: "rustfs_gateway::service", "gateway warning must remain");
});
assert!(!output.contains("s3s info must be suppressed"), "{output}");
assert!(output.contains("s3s warning must remain"), "{output}");
assert!(!output.contains("gateway info must be suppressed"), "{output}");
assert!(output.contains("gateway warning must remain"), "{output}");
});
}
+3 -3
View File
@@ -36,7 +36,7 @@ use crate::cleaner::LogCleaner;
use crate::cleaner::types::{CompressionAlgorithm, FileMatchMode};
use crate::config::OtelConfig;
use crate::global::{METRIC_LOG_CLEANER_RUN_FAILURES_TOTAL, METRIC_LOG_CLEANER_RUNS_TOTAL, set_observability_metric_enabled};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::filter::{build_env_filter, effective_verbose_logging, pyroscope_log_filter, s3_dependency_log_filter};
use crate::telemetry::rolling::{RollingAppender, Rotation};
use metrics::counter;
use rustfs_config::observability::{
@@ -423,7 +423,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
.with(env_filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(ErrorLayer::default())
.with(fmt_layer)
.try_init()
@@ -537,7 +537,7 @@ fn init_file_logging_internal(
.with(env_filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(ErrorLayer::default())
.with(file_layer)
.with(stdout_layer)
+4 -4
View File
@@ -39,7 +39,7 @@
use crate::cleaner::types::FileMatchMode;
use crate::config::OtelConfig;
use crate::global::set_observability_metric_enabled;
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3s_log_filter};
use crate::telemetry::filter::{build_env_filter, pyroscope_log_filter, s3_dependency_log_filter};
use crate::telemetry::guard::{OtelGuard, ProfilingAgent};
use crate::telemetry::local::{build_json_log_layer, open_rolling_appender, spawn_cleanup_task};
use crate::telemetry::recorder::{Recorder, install_process_global_recorder};
@@ -319,7 +319,7 @@ pub(super) fn init_observability_http(
.with(filter)
.with(super::http2_ping_timeout_layer())
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(ErrorLayer::default())
.with(file_layer_opt)
.with(stdout_layer_opt)
@@ -1361,7 +1361,7 @@ mod tests {
let subscriber = tracing_subscriber::registry()
.with(env_filter)
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(
tracing_subscriber::fmt::layer()
.with_ansi(false)
@@ -1385,7 +1385,7 @@ mod tests {
tracing_subscriber::registry()
.with(build_env_filter("info", None))
.with(pyroscope_log_filter())
.with(s3s_log_filter())
.with(s3_dependency_log_filter())
.with(tracing_subscriber::fmt::layer().with_ansi(false).without_time())
.try_init()
.expect("install isolated LogTracer subscriber");
+1 -1
View File
@@ -13,7 +13,7 @@ Applies to `crates/policy/`.
- Validate compatibility with IAM and request-auth call sites:
- `crates/iam/`
- `rustfs/src/auth.rs`
- `rustfs/src/storage/access.rs`
- `rustfs/src/storage/authorization/`
## Error and Observability
-4
View File
@@ -70,10 +70,6 @@ url.workspace = true
urlencoding = { workspace = true }
uuid = { workspace = true, features = ["v4", "fast-rng", "serde", "macro-diagnostics"] }
# The remote error code table is checked against the legacy client's.
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
[lints]
workspace = true
+18 -15
View File
@@ -106,7 +106,7 @@ impl fmt::Display for RemoteErrorCode {
}
}
/// The codes the legacy client (s3s at the pinned revision) recognized, sorted.
/// The sorted wire codes recognized by the legacy client at revision 5761ddfe.
const RECOGNIZED_CODES: &[&str] = &[
"AccessControlListNotSupported",
"AccessDenied",
@@ -352,15 +352,19 @@ const RECOGNIZED_CODES: &[&str] = &[
#[cfg(test)]
mod tests {
use super::*;
use sha2::{Digest, Sha256};
#[test]
fn recognizes_exactly_the_codes_the_legacy_client_named() {
assert_eq!(RECOGNIZED_CODES.len(), 239);
assert!(RECOGNIZED_CODES.windows(2).all(|pair| pair[0] < pair[1]), "the table must stay sorted");
// SHA-256 of the sorted, newline-separated wire names in S3_ERROR_CODE_MAP at 5761ddfe.
let digest = Sha256::digest(RECOGNIZED_CODES.join("\n"));
assert_eq!(
hex_simd::encode_to_string(digest.as_slice(), hex_simd::AsciiCase::Lower),
"6fa6be09aa53ae2e05c5c98c10c43dae79713b07a2bdde492dbb6a776888978d"
);
for code in RECOGNIZED_CODES {
let legacy = s3s::S3ErrorCode::from_bytes(code.as_bytes()).expect("utf-8");
assert!(!matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{code} is not named by the legacy client");
assert_eq!(legacy.as_str(), *code);
let parsed = RemoteErrorCode::from_wire(code);
assert!(!parsed.is_custom(), "{code}");
assert_eq!(parsed.as_str(), *code);
@@ -369,19 +373,18 @@ mod tests {
#[test]
fn parses_like_the_legacy_client() {
for text in [
"NoSuchKey",
"nosuchkey",
"NOSUCHKEY",
"PermanentRedirect",
"SomethingNew",
"",
"No Such Key",
for (text, expected, custom) in [
("NoSuchKey", "NoSuchKey", false),
("nosuchkey", "NoSuchKey", false),
("NOSUCHKEY", "NoSuchKey", false),
("PermanentRedirect", "PermanentRedirect", false),
("SomethingNew", "SomethingNew", true),
("", "", true),
("No Such Key", "No Such Key", true),
] {
let legacy = s3s::S3ErrorCode::from_bytes(text.as_bytes()).expect("utf-8");
let parsed = RemoteErrorCode::from_wire(text);
assert_eq!(parsed.is_custom(), matches!(legacy, s3s::S3ErrorCode::Custom(_)), "{text:?}");
assert_eq!(parsed.as_str(), legacy.as_str(), "{text:?}");
assert_eq!(parsed.is_custom(), custom, "{text:?}");
assert_eq!(parsed.as_str(), expected, "{text:?}");
}
assert_eq!(RemoteErrorCode::from_wire("nosuchkey"), RemoteErrorCode::NO_SUCH_KEY);
}
+4 -2
View File
@@ -164,10 +164,10 @@ impl From<legacy_s3s::S3Error> for S3Error {
/// reference only, so the whole error travels inside and this wrapper presents
/// the cause's own text and chain, as if the cause itself had been moved.
#[derive(Debug)]
struct LegacyCause(legacy_s3s::S3Error);
pub(super) struct LegacyCause(legacy_s3s::S3Error);
impl LegacyCause {
fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
pub(super) fn cause(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
self.0.source()
}
}
@@ -378,6 +378,8 @@ mod tests {
legacy.set_source(Box::new(io::Error::other("read quorum")));
let mut ours: S3Error = legacy.into();
assert!(ours.source().is_some_and(|cause| cause.is::<io::Error>()));
assert!(std::error::Error::source(&ours).is_some_and(|cause| cause.is::<io::Error>()));
ours.set_message("retry later".to_owned());
let back: legacy_s3s::S3Error = ours.into();
assert_eq!(back.code(), &legacy_s3s::S3ErrorCode::ServiceUnavailable);
+7 -2
View File
@@ -157,7 +157,12 @@ impl S3Error {
/// The cause, if one was attached.
pub fn source(&self) -> Option<&(dyn std::error::Error + Send + Sync + 'static)> {
self.0.source.as_deref()
let source = self.0.source.as_deref()?;
#[cfg(any(test, feature = "compat-s3s"))]
if let Some(legacy) = source.downcast_ref::<crate::compat_s3s::LegacyCause>() {
return legacy.cause();
}
Some(source)
}
/// The explicit status if one was set, otherwise the code's own status;
@@ -202,7 +207,7 @@ impl fmt::Display for S3Error {
impl std::error::Error for S3Error {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
self.0.source.as_deref().map(|source| source as _)
S3Error::source(self).map(|source| source as _)
}
}
-1
View File
@@ -15,7 +15,6 @@
pub mod init_time;
pub mod last_minute;
pub mod metrics;
pub mod scan_observation;
pub mod startup;
pub mod usage_candidate;
+1 -40
View File
@@ -13,9 +13,6 @@
// limitations under the License.
use crate::last_minute::{AccElem, LastMinuteLatency};
use crate::scan_observation::{
ScannerMetadataReadGuard, ScannerMetadataReadOutcome, ScannerScanObservation, ScannerScanObservationSnapshot,
};
use crate::usage_candidate::{
ScannerUsageCandidateCount, ScannerUsageCandidateKind, ScannerUsageCandidateOutcome, USAGE_CANDIDATE_COUNTER_SLOTS,
};
@@ -980,7 +977,6 @@ impl ScannerUsagePublicationObservation {
pub struct Metrics {
startup: crate::startup::ScannerStartupObservation,
scan_observation: ScannerScanObservation,
operations: Vec<AtomicU64>,
latency: Vec<LockedLastMinuteLatency>,
actions: Vec<AtomicU64>,
@@ -1635,9 +1631,6 @@ pub struct ScannerMetricsReport {
pub lifecycle_transition: ScannerLifecycleTransitionSnapshot,
#[serde(default)]
pub usage_freshness: ScannerUsageFreshnessSnapshot,
/// Node-local, process-lifetime work observations. Older nodes have no evidence.
#[serde(default)]
pub scan_observation: ScannerScanObservationSnapshot,
/// Process-lifetime candidate-construction results. Missing on older reports
/// means unobserved, not accepted or healthy. Publication is tracked separately.
#[serde(default)]
@@ -2132,38 +2125,9 @@ impl Metrics {
&self.startup
}
pub fn scan_observation(&self) -> &ScannerScanObservation {
&self.scan_observation
}
/// Start after scanner node admission, before polling the disk read future.
/// Disk-permit and RPC waits remain included; decoding is excluded. Dropping
/// an unfinished guard observes cancellation, not physical I/O completion.
pub fn start_scanner_metadata_read(&self) -> ScannerMetadataReadGuard<'_> {
self.scan_observation.start_metadata_read();
ScannerMetadataReadGuard {
metrics: self,
started: std::time::Instant::now(),
outcome: ScannerMetadataReadOutcome::Cancelled,
bytes: 0,
}
}
pub(crate) fn finish_scanner_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
let bytes = if outcome == ScannerMetadataReadOutcome::Success {
bytes
} else {
0
};
self.scan_observation.finish_metadata_read(outcome, bytes, elapsed);
self.operations[Metric::ReadMetadata as usize].fetch_add(1, Ordering::Relaxed);
self.latency[Metric::ReadMetadata as usize].add_size(elapsed, bytes);
}
pub fn new() -> Self {
Self {
startup: crate::startup::ScannerStartupObservation::default(),
scan_observation: ScannerScanObservation::default(),
operations: (0..Metric::Last as usize).map(|_| AtomicU64::new(0)).collect(),
// Each slot gets its own fresh LockedLastMinuteLatency so that
// different metrics never accidentally share state.
@@ -2769,8 +2733,6 @@ impl Metrics {
/// walked the whole namespace, so an operator can tell why scoped
/// convergence is not happening.
pub fn record_scanner_scope_fallback(&self, reason: impl Into<String>) {
let reason = reason.into();
self.scan_observation.record_scope_fallback(&reason);
self.scanner_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
self.scanner_scope_fallback_last_unix_secs
.store(unix_now_secs(), Ordering::Relaxed);
@@ -2778,7 +2740,7 @@ impl Metrics {
Ok(guard) => guard,
Err(poisoned) => poisoned.into_inner(),
};
*last_reason = reason;
*last_reason = reason.into();
}
pub fn record_scanner_usage_durable_success(&self) {
@@ -3913,7 +3875,6 @@ impl Metrics {
failed: self.scanner_transition_failed.load(Ordering::Relaxed),
};
m.usage_candidates = self.usage_candidate_counts();
m.scan_observation = self.scan_observation.snapshot();
let usage_save_result = ScannerUsageSaveResult::from_code(self.scanner_usage_last_save_result.load(Ordering::Relaxed));
m.usage_freshness = ScannerUsageFreshnessSnapshot {
dirty_pending_buckets: self.scanner_dirty_usage_pending_buckets.load(Ordering::Relaxed),
@@ -1,675 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Node-local, process-lifetime observations of scanner work. Counters are
//! independently loaded, so concurrent reports need not balance exactly.
//! These observations never grant cache reuse or publication authority.
use crate::metrics::Metrics;
use serde::{Deserialize, Serialize};
use std::{
collections::BTreeMap,
sync::atomic::{AtomicU64, Ordering},
time::{Duration, Instant},
};
pub const PREFIX_REUSE_BLOCKERS: [&str; 8] = [
"missing_producer_identity",
"missing_durable_journal_replay",
"restart_gap",
"generation_gap",
"overflow",
"stale_ack_generation",
"missing_cold_zero_walk_oracle",
"distributed_without_peer_invalidation",
];
const SCOPE_FALLBACK_REASONS: [&str; 6] = [
"dirty_snapshot_incomplete",
"dirty_generation_exhausted",
"dirty_bucket_overflow",
"no_listed_dirty_buckets",
"baseline_unavailable",
"peer_scope_unavailable",
];
/// A storage read result, independent of later metadata decoding or ILM work.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerMetadataReadOutcome {
Success,
NotFound,
Error,
Cancelled,
}
/// The existing path that actually copied a cached subtree instead of walking it.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseKind {
CleanPrefix,
CoveredNamespace,
SampledNamespace,
ClassifiedNamespace,
}
impl ScannerCacheReuseKind {
const ALL: [Self; 4] = [
Self::CleanPrefix,
Self::CoveredNamespace,
Self::SampledNamespace,
Self::ClassifiedNamespace,
];
fn label(self) -> &'static str {
match self {
Self::CleanPrefix => "clean_prefix",
Self::CoveredNamespace => "covered_namespace",
Self::SampledNamespace => "sampled_namespace",
Self::ClassifiedNamespace => "classified_namespace",
}
}
}
/// Completed bucket-level reuse work, independent of durable publication.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerBucketReuseKind {
ScopedColdBucket,
CurrentRoot,
}
impl ScannerBucketReuseKind {
const ALL: [Self; 2] = [Self::ScopedColdBucket, Self::CurrentRoot];
fn label(self) -> &'static str {
match self {
Self::ScopedColdBucket => "scoped_cold_bucket",
Self::CurrentRoot => "current_root",
}
}
}
/// A production folder gate, not a diagnostic preflight.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseGate {
CleanPrefix,
CoveredNamespace,
NamespaceSampling,
}
impl ScannerCacheReuseGate {
const ALL: [Self; 3] = [Self::CleanPrefix, Self::CoveredNamespace, Self::NamespaceSampling];
fn label(self) -> &'static str {
match self {
Self::CleanPrefix => "clean_prefix",
Self::CoveredNamespace => "covered_namespace",
Self::NamespaceSampling => "namespace_sampling",
}
}
}
/// The first existing condition reached by a folder reuse gate.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerCacheReuseDecision {
MissingScope,
NoResumeSweep,
DeepScan,
ForwardSweep,
ActiveReplication,
NotRoot,
CompactedParent,
MissingCachedChild,
InvalidPrefix,
DirtyPrefix,
ErasureHealthRequired,
MissingCoverage,
CachedFailures,
UncompactedChild,
SelectedForWalk,
Eligible,
}
impl ScannerCacheReuseDecision {
const ALL: [Self; 16] = [
Self::MissingScope,
Self::NoResumeSweep,
Self::DeepScan,
Self::ForwardSweep,
Self::ActiveReplication,
Self::NotRoot,
Self::CompactedParent,
Self::MissingCachedChild,
Self::InvalidPrefix,
Self::DirtyPrefix,
Self::ErasureHealthRequired,
Self::MissingCoverage,
Self::CachedFailures,
Self::UncompactedChild,
Self::SelectedForWalk,
Self::Eligible,
];
fn label(self) -> &'static str {
match self {
Self::MissingScope => "missing_scope",
Self::NoResumeSweep => "no_resume_sweep",
Self::DeepScan => "deep_scan",
Self::ForwardSweep => "forward_sweep",
Self::ActiveReplication => "active_replication",
Self::NotRoot => "not_root",
Self::CompactedParent => "compacted_parent",
Self::MissingCachedChild => "missing_cached_child",
Self::InvalidPrefix => "invalid_prefix",
Self::DirtyPrefix => "dirty_prefix",
Self::ErasureHealthRequired => "erasure_health_required",
Self::MissingCoverage => "missing_coverage",
Self::CachedFailures => "cached_failures",
Self::UncompactedChild => "uncompacted_child",
Self::SelectedForWalk => "selected_for_walk",
Self::Eligible => "eligible",
}
}
fn not_evaluated(self) -> bool {
matches!(
self,
Self::MissingScope | Self::NoResumeSweep | Self::DeepScan | Self::ForwardSweep | Self::ActiveReplication
)
}
}
/// Scope admission and end-of-cycle evidence are separate observations. An
/// activated completion proof does not mean this cycle used prefix reuse.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerPrefixReuseStage {
ScopeResolution,
CycleCompletion,
}
/// The first existing baseline validation check that rejected a snapshot.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ScannerBaselineRejection {
Missing,
Decode,
Incomplete,
Partial,
Marks,
Epoch,
SourceCount,
PlanMissing,
PlanMismatch,
SourceIndex,
SourceMembership,
SourceDuplicate,
SetIncomplete,
SetTombstone,
SetEpoch,
SetCycle,
SetPlan,
SourceCoverage,
AuthoritativeIdentity,
NotNewer,
}
impl ScannerBaselineRejection {
const ALL: [Self; 20] = [
Self::Missing,
Self::Decode,
Self::Incomplete,
Self::Partial,
Self::Marks,
Self::Epoch,
Self::SourceCount,
Self::PlanMissing,
Self::PlanMismatch,
Self::SourceIndex,
Self::SourceMembership,
Self::SourceDuplicate,
Self::SetIncomplete,
Self::SetTombstone,
Self::SetEpoch,
Self::SetCycle,
Self::SetPlan,
Self::SourceCoverage,
Self::AuthoritativeIdentity,
Self::NotNewer,
];
fn label(self) -> &'static str {
match self {
Self::Missing => "missing",
Self::Decode => "decode",
Self::Incomplete => "incomplete",
Self::Partial => "partial",
Self::Marks => "marks",
Self::Epoch => "epoch",
Self::SourceCount => "source_count",
Self::PlanMissing => "plan_missing",
Self::PlanMismatch => "plan_mismatch",
Self::SourceIndex => "source_index",
Self::SourceMembership => "source_membership",
Self::SourceDuplicate => "source_duplicate",
Self::SetIncomplete => "set_incomplete",
Self::SetTombstone => "set_tombstone",
Self::SetEpoch => "set_epoch",
Self::SetCycle => "set_cycle",
Self::SetPlan => "set_plan",
Self::SourceCoverage => "source_coverage",
Self::AuthoritativeIdentity => "authoritative_identity",
Self::NotNewer => "not_newer",
}
}
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerMetadataReadSnapshot {
pub attempts: u64,
pub succeeded: u64,
pub not_found: u64,
pub errors: u64,
pub cancelled: u64,
/// Bytes returned by successful storage reads, including corrupt metadata.
pub bytes: u64,
/// Elapsed read-future time, including failed and cancelled reads.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerCacheReuseSnapshot {
/// Completed subtree reuse decisions; not cache entries or objects skipped.
pub count: u64,
/// Time spent copying and attaching the reused subtree, excluding listing.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerBucketReuseSnapshot {
/// Cold bucket copies or current roots successfully handed to the collector.
pub count: u64,
/// Cold copy/attachment time or current-root channel-send time; excludes validation.
pub elapsed_nanos: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerCacheReuseDecisionSnapshot {
/// Policy prerequisites that stopped the gate before cache proof checks.
pub not_evaluated: BTreeMap<String, u64>,
/// First failed cache/work condition; later conditions are not evaluated.
pub blocked: BTreeMap<String, u64>,
/// Admitted candidates, which may still need EC classification before copying.
pub eligible: u64,
}
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerPrefixReuseSnapshot {
pub evaluated: u64,
pub activated: u64,
/// All failed checks are counted; one evaluation can have several blockers.
pub blockers: BTreeMap<String, u64>,
}
/// Missing on an older node means unobserved, not zero work or healthy reuse.
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
#[serde(default)]
pub struct ScannerScanObservationSnapshot {
pub metadata_reads: ScannerMetadataReadSnapshot,
pub cache_reuse: BTreeMap<String, ScannerCacheReuseSnapshot>,
pub bucket_reuse: BTreeMap<String, ScannerBucketReuseSnapshot>,
pub reuse_decisions: BTreeMap<String, ScannerCacheReuseDecisionSnapshot>,
/// Clean root children still walked because EC scans also check health.
pub erasure_clean_prefix_walks: u64,
pub prefix_scope_resolution: ScannerPrefixReuseSnapshot,
pub prefix_cycle_completion: ScannerPrefixReuseSnapshot,
pub scope_fallbacks: BTreeMap<String, u64>,
/// Rejections only for cycles with a baseline_unavailable scope fallback.
pub authoritative_baseline_rejections: BTreeMap<String, u64>,
pub observed_baseline_rejections: BTreeMap<String, u64>,
}
#[derive(Default)]
pub struct ScannerScanObservation {
metadata: [AtomicU64; 7],
cache_count: [AtomicU64; 4],
cache_nanos: [AtomicU64; 4],
bucket_count: [AtomicU64; 2],
bucket_nanos: [AtomicU64; 2],
reuse_decisions: [[AtomicU64; ScannerCacheReuseDecision::ALL.len()]; 3],
erasure_clean_prefix_walks: AtomicU64,
prefix_evaluated: [AtomicU64; 2],
prefix_activated: [AtomicU64; 2],
prefix_blockers: [[AtomicU64; PREFIX_REUSE_BLOCKERS.len()]; 2],
scope_fallbacks: [AtomicU64; SCOPE_FALLBACK_REASONS.len()],
baseline_rejections: [[AtomicU64; ScannerBaselineRejection::ALL.len()]; 2],
}
fn duration_nanos(duration: Duration) -> u64 {
u64::try_from(duration.as_nanos()).unwrap_or(u64::MAX)
}
fn labelled_counts(labels: impl IntoIterator<Item = &'static str>, counters: &[AtomicU64]) -> BTreeMap<String, u64> {
labels
.into_iter()
.zip(counters)
.map(|(label, count)| (label.to_owned(), count.load(Ordering::Relaxed)))
.collect()
}
impl ScannerScanObservation {
pub(crate) fn start_metadata_read(&self) {
self.metadata[0].fetch_add(1, Ordering::Relaxed);
}
pub(crate) fn finish_metadata_read(&self, outcome: ScannerMetadataReadOutcome, bytes: u64, elapsed: Duration) {
self.metadata[1 + outcome as usize].fetch_add(1, Ordering::Relaxed);
if outcome == ScannerMetadataReadOutcome::Success {
self.metadata[5].fetch_add(bytes, Ordering::Relaxed);
}
self.metadata[6].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
pub fn record_cache_reuse(&self, kind: ScannerCacheReuseKind, elapsed: Duration) {
self.cache_count[kind as usize].fetch_add(1, Ordering::Relaxed);
self.cache_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
/// Records completed reuse work only; it grants no publication authority.
pub fn record_bucket_reuse(&self, kind: ScannerBucketReuseKind, count: u64, elapsed: Duration) {
self.bucket_count[kind as usize].fetch_add(count, Ordering::Relaxed);
self.bucket_nanos[kind as usize].fetch_add(duration_nanos(elapsed), Ordering::Relaxed);
}
/// Records one first-condition decision without inspecting later prerequisites.
pub fn record_cache_reuse_decision(&self, gate: ScannerCacheReuseGate, decision: ScannerCacheReuseDecision) {
self.reuse_decisions[gate as usize][decision as usize].fetch_add(1, Ordering::Relaxed);
}
pub fn record_erasure_clean_prefix_walk(&self) {
self.erasure_clean_prefix_walks.fetch_add(1, Ordering::Relaxed);
}
pub fn record_prefix_reuse(&self, stage: ScannerPrefixReuseStage, activated: bool, blockers: &[Option<&str>; 8]) {
let stage = stage as usize;
self.prefix_evaluated[stage].fetch_add(1, Ordering::Relaxed);
if activated {
self.prefix_activated[stage].fetch_add(1, Ordering::Relaxed);
}
for blocker in blockers.iter().flatten() {
if let Some(index) = PREFIX_REUSE_BLOCKERS.iter().position(|label| label == blocker) {
self.prefix_blockers[stage][index].fetch_add(1, Ordering::Relaxed);
}
}
}
pub(crate) fn record_scope_fallback(&self, reason: &str) {
if let Some(index) = SCOPE_FALLBACK_REASONS.iter().position(|label| *label == reason) {
self.scope_fallbacks[index].fetch_add(1, Ordering::Relaxed);
}
}
pub fn record_baseline_rejection(&self, authoritative: ScannerBaselineRejection, observed: Option<ScannerBaselineRejection>) {
self.baseline_rejections[0][authoritative as usize].fetch_add(1, Ordering::Relaxed);
if let Some(observed) = observed {
self.baseline_rejections[1][observed as usize].fetch_add(1, Ordering::Relaxed);
}
}
pub fn snapshot(&self) -> ScannerScanObservationSnapshot {
let prefix = |stage: ScannerPrefixReuseStage| ScannerPrefixReuseSnapshot {
evaluated: self.prefix_evaluated[stage as usize].load(Ordering::Relaxed),
activated: self.prefix_activated[stage as usize].load(Ordering::Relaxed),
blockers: labelled_counts(PREFIX_REUSE_BLOCKERS, &self.prefix_blockers[stage as usize]),
};
ScannerScanObservationSnapshot {
metadata_reads: ScannerMetadataReadSnapshot {
attempts: self.metadata[0].load(Ordering::Relaxed),
succeeded: self.metadata[1].load(Ordering::Relaxed),
not_found: self.metadata[2].load(Ordering::Relaxed),
errors: self.metadata[3].load(Ordering::Relaxed),
cancelled: self.metadata[4].load(Ordering::Relaxed),
bytes: self.metadata[5].load(Ordering::Relaxed),
elapsed_nanos: self.metadata[6].load(Ordering::Relaxed),
},
cache_reuse: ScannerCacheReuseKind::ALL
.into_iter()
.map(|kind| {
(
kind.label().to_owned(),
ScannerCacheReuseSnapshot {
count: self.cache_count[kind as usize].load(Ordering::Relaxed),
elapsed_nanos: self.cache_nanos[kind as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
bucket_reuse: ScannerBucketReuseKind::ALL
.into_iter()
.map(|kind| {
(
kind.label().to_owned(),
ScannerBucketReuseSnapshot {
count: self.bucket_count[kind as usize].load(Ordering::Relaxed),
elapsed_nanos: self.bucket_nanos[kind as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
reuse_decisions: ScannerCacheReuseGate::ALL
.into_iter()
.map(|gate| {
let counts = &self.reuse_decisions[gate as usize];
let decisions = |not_evaluated| {
ScannerCacheReuseDecision::ALL
.into_iter()
.filter(|decision| {
*decision != ScannerCacheReuseDecision::Eligible && decision.not_evaluated() == not_evaluated
})
.map(|decision| (decision.label().to_owned(), counts[decision as usize].load(Ordering::Relaxed)))
.collect()
};
(
gate.label().to_owned(),
ScannerCacheReuseDecisionSnapshot {
not_evaluated: decisions(true),
blocked: decisions(false),
eligible: counts[ScannerCacheReuseDecision::Eligible as usize].load(Ordering::Relaxed),
},
)
})
.collect(),
erasure_clean_prefix_walks: self.erasure_clean_prefix_walks.load(Ordering::Relaxed),
prefix_scope_resolution: prefix(ScannerPrefixReuseStage::ScopeResolution),
prefix_cycle_completion: prefix(ScannerPrefixReuseStage::CycleCompletion),
scope_fallbacks: labelled_counts(SCOPE_FALLBACK_REASONS, &self.scope_fallbacks),
authoritative_baseline_rejections: labelled_counts(
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
&self.baseline_rejections[0],
),
observed_baseline_rejections: labelled_counts(
ScannerBaselineRejection::ALL.map(ScannerBaselineRejection::label),
&self.baseline_rejections[1],
),
}
}
}
/// Dropping an unfinished read records cancellation. The guard observes the
/// read future only; it cannot cancel it or change its storage result.
#[must_use]
pub struct ScannerMetadataReadGuard<'a> {
pub(crate) metrics: &'a Metrics,
pub(crate) started: Instant,
pub(crate) outcome: ScannerMetadataReadOutcome,
pub(crate) bytes: u64,
}
impl ScannerMetadataReadGuard<'_> {
pub fn finish(mut self, outcome: ScannerMetadataReadOutcome, bytes: u64) {
self.outcome = outcome;
self.bytes = bytes;
}
}
impl Drop for ScannerMetadataReadGuard<'_> {
fn drop(&mut self) {
self.metrics
.finish_scanner_metadata_read(self.outcome, self.bytes, self.started.elapsed());
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
#[tokio::test]
async fn metadata_read_reports_storage_results_and_existing_read_metric() {
let metrics = Metrics::new();
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Success, 127);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Success, 0);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::NotFound, 0);
metrics
.start_scanner_metadata_read()
.finish(ScannerMetadataReadOutcome::Error, 0);
let report = metrics.report().await;
let reads = &report.scan_observation.metadata_reads;
assert_eq!(
(reads.attempts, reads.succeeded, reads.not_found, reads.errors, reads.cancelled),
(4, 2, 1, 1, 0)
);
assert_eq!(reads.bytes, 127);
assert_eq!(report.life_time_ops["read_metadata"], 4);
assert_eq!(report.last_minute.actions["read_metadata"].count, 4);
assert_eq!(report.last_minute.actions["read_metadata"].bytes, 127);
}
#[tokio::test]
async fn metadata_read_cancellation_balances_an_actually_polled_future_once() {
let metrics = Arc::new(Metrics::new());
let worker_metrics = Arc::clone(&metrics);
let (started, admitted) = tokio::sync::oneshot::channel();
let worker = tokio::spawn(async move {
let read = worker_metrics.start_scanner_metadata_read();
started.send(()).expect("observe started read");
std::future::pending::<()>().await;
read.finish(ScannerMetadataReadOutcome::Success, 99);
});
tokio::time::timeout(Duration::from_secs(5), admitted)
.await
.expect("read starts within deadline")
.expect("read started");
let in_flight = metrics.scan_observation().snapshot().metadata_reads;
assert_eq!((in_flight.attempts, in_flight.cancelled, in_flight.succeeded), (1, 0, 0));
worker.abort();
assert!(worker.await.expect_err("read task was aborted").is_cancelled());
let report = metrics.report().await;
let reads = &report.scan_observation.metadata_reads;
assert_eq!((reads.attempts, reads.cancelled, reads.succeeded, reads.bytes), (1, 1, 0, 0));
assert_eq!(report.life_time_ops["read_metadata"], 1);
assert_eq!(report.last_minute.actions["read_metadata"].count, 1);
assert_eq!(
Duration::from_nanos(reads.elapsed_nanos).as_secs(),
report.last_minute.actions["read_metadata"].acc_time
);
}
#[tokio::test]
async fn observation_report_keeps_stages_reasons_and_copy_costs_separate() {
let metrics = Metrics::new();
let observation = metrics.scan_observation();
observation.record_cache_reuse(ScannerCacheReuseKind::CleanPrefix, Duration::from_nanos(321));
observation.record_bucket_reuse(ScannerBucketReuseKind::ScopedColdBucket, 2, Duration::from_nanos(654));
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, Duration::from_nanos(987));
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::MissingScope);
observation
.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, ScannerCacheReuseDecision::ErasureHealthRequired);
observation.record_cache_reuse_decision(ScannerCacheReuseGate::CoveredNamespace, ScannerCacheReuseDecision::Eligible);
observation.record_erasure_clean_prefix_walk();
observation.record_prefix_reuse(
ScannerPrefixReuseStage::ScopeResolution,
false,
&[
Some("missing_producer_identity"),
Some("missing_durable_journal_replay"),
None,
None,
None,
None,
None,
None,
],
);
observation.record_prefix_reuse(ScannerPrefixReuseStage::CycleCompletion, true, &[None; 8]);
metrics.record_scanner_scope_fallback("baseline_unavailable");
observation.record_baseline_rejection(ScannerBaselineRejection::Epoch, Some(ScannerBaselineRejection::Missing));
let report = metrics.report().await;
let snapshot = &report.scan_observation;
assert_eq!(snapshot.cache_reuse["clean_prefix"].count, 1);
assert_eq!(snapshot.cache_reuse["clean_prefix"].elapsed_nanos, 321);
assert_eq!(snapshot.cache_reuse["sampled_namespace"].count, 0);
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].count, 2);
assert_eq!(snapshot.bucket_reuse["scoped_cold_bucket"].elapsed_nanos, 654);
assert_eq!(snapshot.bucket_reuse["current_root"].count, 1);
assert_eq!(snapshot.bucket_reuse["current_root"].elapsed_nanos, 987);
assert_eq!(snapshot.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"], 1);
assert_eq!(snapshot.reuse_decisions["clean_prefix"].blocked["erasure_health_required"], 1);
assert_eq!(snapshot.reuse_decisions["covered_namespace"].eligible, 1);
let mut prior_observations = serde_json::to_value(snapshot).expect("encode observations");
let fields = prior_observations.as_object_mut().expect("observation object");
fields.remove("bucket_reuse");
fields.remove("reuse_decisions");
let prior: ScannerScanObservationSnapshot =
serde_json::from_value(prior_observations).expect("read earlier observations");
assert!(prior.bucket_reuse.is_empty());
assert!(prior.reuse_decisions.is_empty());
assert_eq!(prior.cache_reuse["clean_prefix"].count, 1);
assert_eq!(snapshot.erasure_clean_prefix_walks, 1);
assert_eq!(
(snapshot.prefix_scope_resolution.evaluated, snapshot.prefix_scope_resolution.activated),
(1, 0)
);
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_producer_identity"], 1);
assert_eq!(snapshot.prefix_scope_resolution.blockers["missing_durable_journal_replay"], 1);
assert_eq!(
(snapshot.prefix_cycle_completion.evaluated, snapshot.prefix_cycle_completion.activated),
(1, 1)
);
assert_eq!(
snapshot.scope_fallbacks["baseline_unavailable"],
report.usage_freshness.scope_fallback_total
);
assert_eq!(snapshot.authoritative_baseline_rejections["epoch"], 1);
assert_eq!(snapshot.observed_baseline_rejections["missing"], 1);
let mut legacy = serde_json::to_value(&report).expect("encode report");
legacy
.as_object_mut()
.expect("report is an object")
.remove("scan_observation");
let decoded: crate::metrics::ScannerMetricsReport = serde_json::from_value(legacy).expect("read legacy report");
assert!(decoded.scan_observation.cache_reuse.is_empty(), "legacy counters are unobserved");
assert_eq!(decoded.scan_observation.metadata_reads.attempts, 0);
}
}
-1
View File
@@ -109,7 +109,6 @@ bytes.workspace = true
hex-simd.workspace = true
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
opentelemetry-proto.workspace = true
opentelemetry_sdk.workspace = true
prost.workspace = true
-26
View File
@@ -206,32 +206,6 @@ pub(crate) struct DataUsageCacheRevisions {
backup: Option<DataUsageCacheRevision>,
}
/// In-memory provenance of the decoded cache and the revisions observed by its loader.
#[derive(Clone, Debug)]
pub(crate) struct DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions,
loaded_from_main: bool,
}
impl DataUsageCacheBaselineRevisions {
pub(crate) fn revisions(&self) -> &DataUsageCacheRevisions {
&self.revisions
}
/// A backup may catch up to an unchanged primary without changing the decoded baseline.
/// Both reads must have decoded that primary; an ETag on a corrupt primary is insufficient.
pub(crate) fn permits_unchanged_baseline_since(&self, initial: &Self) -> bool {
if self.revisions == initial.revisions {
return true;
}
self.loaded_from_main
&& initial.loaded_from_main
&& matches!(&initial.revisions.main, DataUsageCacheRevision::Etag(etag) if !etag.is_empty())
&& self.revisions.main == initial.revisions.main
&& self.revisions.backup.as_ref() == Some(&self.revisions.main)
}
}
pub static DATA_USAGE_BUCKET: LazyLock<String> =
LazyLock::new(|| format!("{RUSTFS_META_BUCKET}{SLASH_SEPARATOR}{BUCKET_META_PREFIX}"));
@@ -70,7 +70,6 @@ struct DataUsageCacheLoadResult {
cache: DataUsageCache,
main_revision: DataUsageCacheRevision,
backup_revision: Option<DataUsageCacheRevision>,
loaded_from_main: bool,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -107,18 +106,6 @@ impl DataUsageCache {
store: Arc<S>,
name: &str,
) -> StorageResult<DataUsageCacheRevisions> {
self.load_with_baseline_revisions(store, name)
.await
.map(|baseline| baseline.revisions)
}
/// Loads cache revisions with provenance for unchanged-primary backup reconciliation.
/// Missing or rebuilt caches and a decoded backup do not establish a decoded primary.
pub(crate) async fn load_with_baseline_revisions<S: ScannerObjectIO>(
&mut self,
store: Arc<S>,
name: &str,
) -> StorageResult<DataUsageCacheBaselineRevisions> {
let backup_name = format!("{name}.bkp");
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
let loaded = Self::load_cache(store.clone(), name).await?;
@@ -146,10 +133,7 @@ impl DataUsageCache {
let main = loaded.main_revision;
*self = loaded.cache;
Ok(DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions { main, backup },
loaded_from_main: loaded.loaded_from_main,
})
Ok(DataUsageCacheRevisions { main, backup })
}
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
@@ -172,13 +156,10 @@ impl DataUsageCache {
cache,
revision: Some(main_revision),
} => {
// Legacy-location fallback carries Missing for the primary path.
let loaded_from_main = matches!(&main_revision, DataUsageCacheRevision::Etag(_));
return Ok(DataUsageCacheLoadResult {
cache: *cache,
main_revision,
backup_revision: None,
loaded_from_main,
});
}
DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => {
@@ -216,7 +197,6 @@ impl DataUsageCache {
cache: *cache,
main_revision,
backup_revision: Some(backup_revision),
loaded_from_main: false,
});
}
DataUsageCacheLoadState::Loaded
@@ -240,7 +220,6 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?,
backup_revision,
loaded_from_main: false,
});
}
(DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing)
@@ -260,7 +239,6 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?,
backup_revision,
loaded_from_main: false,
});
}
_ => {}
+90 -433
View File
@@ -772,102 +772,6 @@ async fn corrupt_primary_cache_loads_valid_backup() {
assert_eq!(store.puts.load(Ordering::SeqCst), 0);
}
#[test]
fn baseline_revisions_allow_only_a_decoded_unchanged_primary_backup_catchup() {
let etag = |value: &str| DataUsageCacheRevision::Etag(value.to_owned());
let initial = DataUsageCacheBaselineRevisions {
revisions: DataUsageCacheRevisions {
main: etag("primary"),
backup: Some(etag("old-backup")),
},
loaded_from_main: true,
};
assert!(initial.permits_unchanged_baseline_since(&initial));
let mut caught_up = initial.clone();
caught_up.revisions.backup = Some(etag("primary"));
assert!(caught_up.permits_unchanged_baseline_since(&initial));
assert!(
!initial.permits_unchanged_baseline_since(&caught_up),
"backup rollback changes the baseline"
);
for backup in [None, Some(DataUsageCacheRevision::Missing)] {
let mut missing_backup = initial.clone();
missing_backup.revisions.backup = backup;
assert!(caught_up.permits_unchanged_baseline_since(&missing_backup));
assert!(!missing_backup.permits_unchanged_baseline_since(&initial));
}
for (initial_from_main, current_from_main) in [(false, true), (true, false), (false, false)] {
let mut baseline = initial.clone();
baseline.loaded_from_main = initial_from_main;
let mut current = caught_up.clone();
current.loaded_from_main = current_from_main;
assert!(
!current.permits_unchanged_baseline_since(&baseline),
"backup fallback grants no primary proof"
);
current.revisions = baseline.revisions.clone();
assert!(
current.permits_unchanged_baseline_since(&baseline),
"exact revisions retain existing admission"
);
}
let mut changed_main = caught_up.clone();
changed_main.revisions.main = etag("new-primary");
changed_main.revisions.backup = Some(etag("new-primary"));
assert!(!changed_main.permits_unchanged_baseline_since(&initial));
let mut changed_backup = caught_up;
changed_backup.revisions.backup = Some(etag("unrelated-backup"));
assert!(!changed_backup.permits_unchanged_baseline_since(&initial));
for main in [DataUsageCacheRevision::Missing, etag("")] {
let mut baseline = initial.clone();
baseline.revisions.main = main.clone();
let mut current = baseline.clone();
current.revisions.backup = Some(main);
assert!(
!current.permits_unchanged_baseline_since(&baseline),
"missing or empty primary ETag is not proof"
);
}
}
#[tokio::test]
async fn baseline_revisions_record_the_actual_decoded_cache_source() {
let mut expected = DataUsageCache::default();
expected.info.name = "decoded".to_owned();
let valid = expected.marshal_msg().expect("encode valid cache");
for (main, backup, from_main, name) in [
(valid.clone(), Some(valid.clone()), true, "decoded"),
(b"corrupt".to_vec(), Some(valid.clone()), false, "decoded"),
(b"corrupt".to_vec(), None, false, ""),
(Vec::new(), None, false, ""),
] {
let store = Arc::new(CacheReadStore::new(CacheReadBody::Bytes(main), backup));
let mut loaded = DataUsageCache::default();
let baseline = loaded
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
.await
.expect("load actual cache source");
assert_eq!(baseline.loaded_from_main, from_main);
assert_eq!(loaded.info.name, name);
let mut raw_loaded = DataUsageCache::default();
let raw = raw_loaded
.load_with_revisions(store, DATA_USAGE_CACHE_NAME)
.await
.expect("load compatible raw revisions");
assert_eq!(baseline.revisions(), &raw, "raw loader retains its revision contract");
}
let mut legacy_store = CacheReadStore::new(CacheReadBody::Bytes(Vec::new()), None);
legacy_store.legacy = Some(valid);
let mut legacy_cache = DataUsageCache::default();
let legacy = legacy_cache
.load_with_baseline_revisions(Arc::new(legacy_store), "legacy-cache")
.await
.expect("load cache from the legacy location");
assert_eq!(legacy_cache.info.name, "decoded");
assert!(!legacy.loaded_from_main);
assert_eq!(legacy.revisions.main, DataUsageCacheRevision::Missing);
}
#[tokio::test]
async fn corrupt_cache_without_valid_backup_rebuilds_with_cas_revisions() {
for backup in [None, Some(b"also-not-msgpack".to_vec())] {
@@ -1437,32 +1341,9 @@ fn test_data_usage_cache_info_deserialize_defaults_scan_resume_after() {
#[test]
fn test_data_usage_cache_info_unmarshal_old_msgpack_defaults_scan_resume_after() {
#[derive(Serialize)]
struct OldDataUsageCacheInfo {
name: String,
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
}
let old_info = OldDataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
last_update: None,
skip_healing: true,
lifecycle: None,
replication: None,
failed_objects: HashMap::from([("bad-object".to_string(), 11)]),
};
let mut buf = Vec::new();
old_info
.serialize(&mut rmp_serde::Serializer::new(&mut buf))
.expect("Failed to serialize old cache info");
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(&buf).expect("Failed to deserialize old cache info");
// Seven-field tuple written with the pinned legacy scanner codec.
let old_info = b"\x97\xa6bucket\x07\xc0\xc3\xc0\xc0\x81\xaabad-object\x0b";
let decoded: DataUsageCacheInfo = rmp_serde::from_slice(old_info).expect("decode the historical cache info tuple");
assert_eq!(decoded.name, "bucket");
assert_eq!(decoded.next_cycle, 7);
@@ -1487,13 +1368,13 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
lifecycle: Option<()>,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
scan_resume_after: Option<String>,
scan_checkpoint: Option<DataUsageScanCheckpoint>,
pending_heals: Vec<PendingScannerHeal>,
object_lock: Option<Arc<s3s::dto::ObjectLockConfiguration>>,
object_lock: Option<()>,
}
#[derive(Deserialize)]
@@ -3656,22 +3537,21 @@ const LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD: &[u8] = &[
0x62, 0x6c, 0x65, 0x64, 0xc0,
];
// Keep the old tuple carrier and payload explicit when the production field changes type.
// Project the seven original fields out of the production map. Keep the nested
// MessagePack arrays and maps opaque so their bytes can be checked against the old writer.
#[derive(serde::Serialize, serde::Deserialize)]
struct LegacyLifecycleCacheInfo {
name: String,
next_cycle: u64,
last_update: Option<SystemTime>,
skip_healing: bool,
lifecycle: Option<Arc<s3s::dto::BucketLifecycleConfiguration>>,
lifecycle: Value,
replication: Option<Arc<ReplicationConfig>>,
failed_objects: HashMap<String, u64>,
}
struct LifecycleCacheOracleCase {
label: &'static str,
old_tuple: Vec<u8>,
old_payload: Vec<u8>,
current: DataUsageCacheInfo,
current_map: Vec<u8>,
frozen_tuple: &'static [u8],
@@ -3680,7 +3560,6 @@ struct LifecycleCacheOracleCase {
}
fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
use s3s::dto as legacy;
use time::OffsetDateTime;
let expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_456_789)
.expect("the legacy expiration fixture has an exact timestamp");
@@ -3689,136 +3568,6 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
let expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_789_123)
.expect("the legacy expiry update fixture has an exact timestamp");
let absent = legacy::LifecycleRule {
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
expiration: None,
filter: None,
id: None,
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
status: legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::ENABLED),
transitions: None,
};
let mut empty = absent.clone();
empty.id = Some(String::new());
empty.prefix = Some(String::new());
empty.status = legacy::ExpirationStatus::from_static(legacy::ExpirationStatus::DISABLED);
empty.abort_incomplete_multipart_upload = Some(legacy::AbortIncompleteMultipartUpload {
days_after_initiation: None,
});
empty.del_marker_expiration = Some(legacy::DelMarkerExpiration { days: None });
empty.expiration = Some(legacy::LifecycleExpiration {
days: Some(0),
..Default::default()
});
empty.filter = Some(legacy::LifecycleRuleFilter {
and: Some(legacy::LifecycleRuleAndOperator {
tags: Some(Vec::new()),
..Default::default()
}),
tag: Some(legacy::Tag { key: None, value: None }),
..Default::default()
});
empty.noncurrent_version_expiration = Some(legacy::NoncurrentVersionExpiration {
noncurrent_days: None,
newer_noncurrent_versions: Some(-2),
});
empty.noncurrent_version_transitions = Some(Vec::new());
empty.transitions = Some(Vec::new());
let mut absent_tags = absent.clone();
absent_tags.filter = Some(legacy::LifecycleRuleFilter {
and: Some(legacy::LifecycleRuleAndOperator::default()),
..Default::default()
});
// Cached values must survive independently of lifecycle policy validation.
let complete = legacy::BucketLifecycleConfiguration {
expiry_updated_at: Some(expiry_updated_at.into()),
rules: vec![
legacy::LifecycleRule {
abort_incomplete_multipart_upload: Some(legacy::AbortIncompleteMultipartUpload {
days_after_initiation: Some(0),
}),
del_marker_expiration: Some(legacy::DelMarkerExpiration { days: Some(-1) }),
expiration: Some(legacy::LifecycleExpiration {
date: Some(expiration_date.into()),
days: Some(-3),
expired_object_delete_marker: Some(false),
expired_object_all_versions: Some(false),
}),
filter: Some(legacy::LifecycleRuleFilter {
cached_tags: Default::default(),
and: Some(legacy::LifecycleRuleAndOperator {
prefix: Some(String::new()),
tags: Some(vec![
legacy::Tag {
key: Some(String::new()),
value: Some(String::new()),
},
legacy::Tag {
key: None,
value: Some("value-without-key".to_string()),
},
legacy::Tag {
key: Some("key-without-value".to_string()),
value: None,
},
]),
object_size_greater_than: Some(-1),
object_size_less_than: Some(4096),
}),
prefix: Some(String::new()),
tag: Some(legacy::Tag {
key: Some("kind".to_string()),
value: Some("archive".to_string()),
}),
object_size_greater_than: Some(0),
object_size_less_than: Some(8192),
}),
id: Some("complete-legacy-rule".to_string()),
noncurrent_version_expiration: Some(legacy::NoncurrentVersionExpiration {
noncurrent_days: Some(-7),
newer_noncurrent_versions: Some(0),
}),
noncurrent_version_transitions: Some(vec![
legacy::NoncurrentVersionTransition {
noncurrent_days: Some(0),
newer_noncurrent_versions: Some(-1),
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
},
legacy::NoncurrentVersionTransition {
noncurrent_days: None,
newer_noncurrent_versions: None,
storage_class: None,
},
]),
prefix: Some("legacy-prefix/".to_string()),
status: legacy::ExpirationStatus::from_static("LegacyUnknownStatus"),
transitions: Some(vec![
legacy::Transition {
date: Some(transition_date.into()),
days: Some(0),
storage_class: Some(legacy::TransitionStorageClass::from_static("LEGACY_UNKNOWN_CLASS")),
},
legacy::Transition {
date: None,
days: None,
storage_class: Some(legacy::TransitionStorageClass::from_static("")),
},
legacy::Transition {
date: None,
days: None,
storage_class: None,
},
]),
},
empty,
absent_tags,
absent,
],
};
use rustfs_gateway_types::persistence as persisted;
let domain_absent = persisted::PersistedLifecycleRule {
abort_incomplete_multipart_upload: None,
@@ -3965,17 +3714,12 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
(
"none",
None,
None,
LEGACY_LIFECYCLE_NONE_TUPLE,
LEGACY_LIFECYCLE_NONE_MAP,
LEGACY_LIFECYCLE_NONE_PAYLOAD,
),
(
"empty",
Some(Arc::new(legacy::BucketLifecycleConfiguration {
rules: Vec::new(),
expiry_updated_at: None,
})),
Some(Arc::new(persisted::PersistedLifecycleConfiguration {
rules: Vec::new(),
expiry_updated_at: None,
@@ -3986,7 +3730,6 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
),
(
"nonempty",
Some(Arc::new(complete)),
Some(Arc::new(domain_complete)),
LEGACY_LIFECYCLE_NONEMPTY_TUPLE,
LEGACY_LIFECYCLE_NONEMPTY_MAP,
@@ -3995,35 +3738,17 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
];
cases
.into_iter()
.map(|(label, lifecycle, domain, frozen_tuple, frozen_map, frozen_payload)| {
let old = LegacyLifecycleCacheInfo {
.map(|(label, lifecycle, frozen_tuple, frozen_map, frozen_payload)| {
let current = DataUsageCacheInfo {
name: "legacy-lifecycle-bucket".to_string(),
next_cycle: 7,
last_update: None,
skip_healing: true,
lifecycle,
replication: None,
failed_objects: HashMap::new(),
};
let mut old_tuple = Vec::new();
old.serialize(&mut rmp_serde::Serializer::new(&mut old_tuple))
.expect("the real old tuple carrier must encode");
let old_payload = rmp_serde::to_vec(&old.lifecycle).expect("the real old lifecycle payload must encode");
let current = DataUsageCacheInfo {
name: old.name.clone(),
next_cycle: old.next_cycle,
last_update: old.last_update,
skip_healing: old.skip_healing,
lifecycle: domain,
replication: None,
failed_objects: old.failed_objects,
..Default::default()
};
let current_map = rmp_serde::to_vec(&current).expect("the actual production map writer must encode the fixture");
LifecycleCacheOracleCase {
label,
old_tuple,
old_payload,
current,
current_map,
frozen_tuple,
@@ -4036,57 +3761,36 @@ fn lifecycle_cache_oracle_cases() -> Vec<LifecycleCacheOracleCase> {
#[test]
fn test_lifecycle_cache_legacy_msgpack_oracle() {
use time::OffsetDateTime;
for case in lifecycle_cache_oracle_cases() {
let label = case.label;
let old_tuple = case.old_tuple;
let old_payload = case.old_payload;
let current = case.current;
let current_map = case.current_map;
let frozen_tuple = case.frozen_tuple;
let frozen_map = case.frozen_map;
let frozen_payload = case.frozen_payload;
println!("legacy_lifecycle_cache {label}: tuple={old_tuple:02x?}; map={current_map:02x?}; payload={old_payload:02x?}");
let old_from_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&current_map).expect("the actual old reader must read the production map");
if label == "nonempty" {
let lifecycle = old_from_current
.lifecycle
.as_ref()
.expect("the old reader must retain the nonempty lifecycle");
let expiration = lifecycle.rules[0]
.expiration
.as_ref()
.expect("the complete rule has expiration");
let transition = &lifecycle.rules[0]
.transitions
.as_ref()
.expect("the complete rule has transitions")[0];
let expected_expiration_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_000_123_000_000)
.expect("the legacy millisecond expiration date is representable");
let expected_transition_date = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_001_987_000_000)
.expect("the legacy millisecond transition date is representable");
let expected_expiry_updated_at = OffsetDateTime::from_unix_timestamp_nanos(1_700_000_002_456_000_000)
.expect("the legacy millisecond expiry update is representable");
assert_eq!(
(
expiration.date.clone().map(OffsetDateTime::from),
transition.date.clone().map(OffsetDateTime::from),
lifecycle.expiry_updated_at.clone().map(OffsetDateTime::from),
),
(
Some(expected_expiration_date),
Some(expected_transition_date),
Some(expected_expiry_updated_at),
),
"cache serialization must retain the fixed legacy millisecond precision for Date and ExpiryUpdatedAt"
);
}
let LifecycleCacheOracleCase {
label,
current,
current_map,
frozen_tuple,
frozen_map,
frozen_payload,
} = case;
let frozen_tuple_read: DataUsageCacheInfo =
rmp_serde::from_slice(frozen_tuple).expect("the production reader must accept the fixed legacy tuple bytes");
let frozen_map_read: DataUsageCacheInfo =
rmp_serde::from_slice(frozen_map).expect("the production reader must accept the fixed legacy map bytes");
if label == "nonempty" {
let lifecycle = frozen_map_read.lifecycle.as_ref().expect("nonempty lifecycle");
let rule = &lifecycle.rules[0];
assert_eq!(
[
rule.expiration.as_ref().expect("expiration").date.as_deref(),
rule.transitions.as_ref().expect("transitions")[0].date.as_deref(),
lifecycle.expiry_updated_at.as_deref(),
],
[
Some("2023-11-14T22:13:20.123Z"),
Some("2023-11-14T22:13:21.987Z"),
Some("2023-11-14T22:13:22.456Z"),
],
"legacy timestamps retain millisecond precision",
);
}
assert_eq!(
[
rmp_serde::to_vec(&frozen_tuple_read).expect("encode the fixed legacy tuple after production read"),
@@ -4110,21 +3814,21 @@ fn test_lifecycle_cache_legacy_msgpack_oracle() {
fn test_lifecycle_cache_legacy_carrier_round_trips() {
for case in lifecycle_cache_oracle_cases() {
let label = case.label;
let old_tuple = case.old_tuple;
let old_tuple = case.frozen_tuple;
let current_map = case.current_map;
let from_old: DataUsageCacheInfo =
rmp_serde::from_slice(&old_tuple).expect("the production reader must read the old tuple");
rmp_serde::from_slice(old_tuple).expect("the production reader must read the old tuple");
let from_old_map = rmp_serde::to_vec(&from_old).expect("the production writer must retain the old tuple payload");
let old_via_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&from_old_map).expect("the old reader must read the production map after an old-tuple load");
rmp_serde::from_slice(&from_old_map).expect("project the old fields after an old-tuple load");
let old_from_current: LegacyLifecycleCacheInfo =
rmp_serde::from_slice(&current_map).expect("the actual old reader must read the production map");
rmp_serde::from_slice(&current_map).expect("project the old fields after a production write");
assert_eq!(
[
rmp_serde::to_vec(&old_via_current).expect("encode the old reader's complete tuple after production reload"),
rmp_serde::to_vec(&old_from_current).expect("encode the old reader's complete tuple after production write"),
],
[old_tuple.clone(), old_tuple],
[old_tuple, old_tuple],
"both cache directions must preserve the full old carrier for {label}"
);
}
@@ -4147,32 +3851,12 @@ fn test_lifecycle_cache_current_map_round_trip() {
#[test]
fn test_lifecycle_cache_strict_cached_tags_contract() {
use s3s::dto as legacy;
type LegacyPayload = Option<Arc<legacy::BucketLifecycleConfiguration>>;
let old: LegacyPayload = rmp_serde::from_slice(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD)
.expect("the frozen payload must initialize the real old configuration");
let filter = old.as_ref().expect("nonempty old payload").rules[0]
.filter
.as_ref()
.expect("first old rule has a filter");
let object_tags = [(String::new(), String::new()), ("kind".to_owned(), "archive".to_owned())]
.into_iter()
.collect();
// The real old inherent method initializes its OnceLock even if matching fails.
let _ = filter.test_tags(&object_tags);
// The pinned legacy writer produced the same empty map before and after test_tags initialized its cache.
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD))
.expect("the production reader must accept the fixed payload");
assert_eq!(
[
rmp_serde::to_vec(&old).expect("encode the initialized old cache"),
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the new cache")
],
[
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
],
"an initialized old CachedTags must still write the same empty map as the new carrier"
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("encode the cached tags"),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
);
let slot = LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD
@@ -4185,22 +3869,14 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
bytes.extend_from_slice(&LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD[slot + 1..]);
bytes
};
// A historical String/String map is consumed by both real readers, then
// rewritten empty. The original populated input is deliberately not the oracle.
// The pinned reader discarded populated String/String maps and rejected the other shapes below.
let populated = input_with_cache(&[0x81, 0xa3, b'e', b'n', b'v', 0xa4, b'p', b'r', b'o', b'd']);
let old: LegacyPayload = rmp_serde::from_slice(&populated).expect("old reader accepts String/String cached tags");
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(populated.as_slice()))
.expect("new reader accepts String/String cached tags");
assert_eq!(
[
rmp_serde::to_vec(&old).expect("rewrite the old populated cache"),
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the new populated cache")
],
[
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec(),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD.to_vec()
],
"both readers discard populated cached tags and both writers emit an empty map"
rmp_serde::to_vec(&crate::lifecycle_cache::LifecycleCache(&new)).expect("rewrite the populated cache"),
LEGACY_LIFECYCLE_NONEMPTY_PAYLOAD,
"populated cached tags are discarded on rewrite",
);
for (label, replacement) in [
@@ -4211,13 +3887,8 @@ fn test_lifecycle_cache_strict_cached_tags_contract() {
("nonstring-value", &[0x81, 0xa1, b'k', 0x01][..]),
] {
let input = input_with_cache(replacement);
let old = rmp_serde::from_slice::<LegacyPayload>(&input);
let new = crate::lifecycle_cache::deserialize(&mut rmp_serde::Deserializer::new(input.as_slice()));
assert_eq!(
[old.is_err(), new.is_err()],
[true, true],
"both real readers must reject {label} cached tags"
);
assert!(new.is_err(), "must reject {label} cached tags");
}
}
@@ -4229,73 +3900,59 @@ fn object_lock_cache_matches_the_legacy_dto_bytes() {
use rustfs_gateway_types::persistence::{
PersistedDefaultRetention, PersistedObjectLockConfiguration, PersistedObjectLockRule,
};
use s3s::dto as legacy;
let legacy_cases = vec![
legacy::ObjectLockConfiguration {
object_lock_enabled: None,
rule: None,
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: None,
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from("Disabled".to_owned())),
rule: Some(legacy::ObjectLockRule { default_retention: None }),
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: Some(legacy::ObjectLockRule {
default_retention: Some(legacy::DefaultRetention {
// Captured from the minio-enabled s3s DTO at 5761ddfe4c509fcac87e3b99fd5bd9e176b442c0.
let cases = [
(b"\x92\xc0\xc0".as_slice(), None, None),
(b"\x92\xa7Enabled\xc0".as_slice(), Some("Enabled"), None),
(
b"\x92\xa8Disabled\x91\xc0".as_slice(),
Some("Disabled"),
Some(PersistedObjectLockRule { default_retention: None }),
),
(
b"\x92\xa7Enabled\x91\x93\x1e\xaaGOVERNANCE\xc0".as_slice(),
Some("Enabled"),
Some(PersistedObjectLockRule {
default_retention: Some(PersistedDefaultRetention {
days: Some(30),
mode: Some(legacy::ObjectLockRetentionMode::from_static(legacy::ObjectLockRetentionMode::GOVERNANCE)),
mode: Some("GOVERNANCE".to_owned()),
years: None,
}),
}),
},
legacy::ObjectLockConfiguration {
object_lock_enabled: Some(legacy::ObjectLockEnabled::from_static(legacy::ObjectLockEnabled::ENABLED)),
rule: Some(legacy::ObjectLockRule {
default_retention: Some(legacy::DefaultRetention {
),
(
b"\x92\xa7Enabled\x91\x93\xc0\xaecompliance-ish\x02".as_slice(),
Some("Enabled"),
Some(PersistedObjectLockRule {
default_retention: Some(PersistedDefaultRetention {
days: None,
mode: Some(legacy::ObjectLockRetentionMode::from("compliance-ish".to_owned())),
mode: Some("compliance-ish".to_owned()),
years: Some(2),
}),
}),
},
),
];
for legacy_config in legacy_cases {
let persisted = PersistedObjectLockConfiguration {
object_lock_enabled: legacy_config
.object_lock_enabled
.as_ref()
.map(|value| value.as_str().to_owned()),
rule: legacy_config.rule.as_ref().map(|rule| PersistedObjectLockRule {
default_retention: rule.default_retention.as_ref().map(|retention| PersistedDefaultRetention {
mode: retention.mode.as_ref().map(|value| value.as_str().to_owned()),
days: retention.days,
years: retention.years,
}),
}),
};
let old_bytes = rmp_serde::to_vec(&Some(Arc::new(legacy_config))).expect("encode the legacy DTO");
let current = Some(Arc::new(persisted.clone()));
for (old_bytes, enabled, rule) in cases {
let current = Some(Arc::new(PersistedObjectLockConfiguration {
object_lock_enabled: enabled.map(str::to_owned),
rule,
}));
let new_bytes = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&current)).expect("encode the new cache");
assert_eq!(new_bytes, old_bytes, "the new writer must produce the legacy bytes for {persisted:?}");
let decoded = crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes.as_slice()))
.expect("decode the legacy bytes");
assert_eq!(decoded.as_deref(), Some(&persisted));
assert_eq!(new_bytes, old_bytes, "must reproduce the original bytes for {current:?}");
let decoded =
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(old_bytes)).expect("decode the legacy bytes");
assert_eq!(decoded, current);
}
let absent_old = rmp_serde::to_vec(&Option::<Arc<legacy::ObjectLockConfiguration>>::None).expect("encode absent");
let absent_new = rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent");
assert_eq!(absent_new, absent_old);
let decoded =
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(absent_old.as_slice())).expect("decode absent");
assert!(decoded.is_none());
let absent = b"\xc0";
assert_eq!(
rmp_serde::to_vec(&crate::object_lock_cache::ObjectLockCache(&None)).expect("encode absent"),
absent
);
assert!(
crate::object_lock_cache::deserialize(&mut rmp_serde::Deserializer::new(&absent[..]))
.expect("decode absent")
.is_none()
);
}
#[test]
+9 -8
View File
@@ -248,13 +248,13 @@ const CLEAN_IDLE_BACKOFF_FACTOR: u32 = 2;
/// otherwise idle instance whose clean-idle backoff had doubled a 60 s
/// interval), which defeats the fast path it is meant to protect.
///
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] protects
/// against a persistently hot bucket driving an unbroken full-scan loop, so
/// retries can start small: 5 s, 10 s, 20 s … Superseded retries cap this
/// schedule at the ordinary cycle interval and preserve an operator-selected
/// nominal cadence. Preflight deferrals retain the generic
/// [`SCANNER_RETRY_MAX_INTERVAL`] cap so a temporarily unavailable peer cannot
/// drive a tight retry loop.
/// The exponential growth in [`ScannerRetryBackoff::retry_interval`] is
/// what protects against a persistently hot bucket driving an unbroken
/// full-scan loop, so it can start small: 5 s, 10 s, 20 s … capped by
/// [`SCANNER_RETRY_MAX_INTERVAL`]. A one-off race recovers in seconds; a
/// genuinely hot bucket still reaches minute-scale backoff within a handful of
/// cycles. Preflight deferrals use the same bounded schedule so a temporarily
/// unavailable peer cannot drive a tight retry loop.
const SCANNER_RETRY_BASE_INTERVAL: Duration = Duration::from_secs(5);
const SCANNER_RETRY_MAX_INTERVAL: Duration = Duration::from_secs(30 * 60);
/// Probe cadence while data movement pauses the scanner. Recovery retries use
@@ -3528,7 +3528,8 @@ where
));
let mut wait_plan =
scanner_cycle_wait_plan(&runtime_config, clean_idle_backoff, backoff_enabled, randomized_cycle_delay_for);
let superseded_retry_interval = scanner_superseded_retry_interval(superseded_backoff, &runtime_config);
let superseded_retry_interval =
scanner_superseded_retry_interval(superseded_backoff, &runtime_config, maintenance_features);
let deferred_retry_interval = deferred_backoff.retry_interval(runtime_config.cycle_interval);
let scope_fallback_retry_interval = scanner_scope_fallback_retry_interval(scope_fallback_backoff, &runtime_config);
let mut convergence_retry_interval = superseded_retry_interval
+10 -7
View File
@@ -192,16 +192,19 @@ impl ScannerRetryBackoff {
pub(super) fn scanner_superseded_retry_interval(
backoff: ScannerRetryBackoff,
runtime_config: &ScannerRuntimeConfig,
maintenance_features: ScannerMaintenanceFeatures,
) -> Option<Duration> {
let cycle_interval = runtime_config.cycle_interval.max(Duration::from_secs(1));
let retry_interval = backoff.retry_interval(cycle_interval)?;
// Preserve an operator-selected nominal cadence during convergence.
let mut retry_interval = backoff.retry_interval(runtime_config.cycle_interval)?;
// Preserve an operator-selected duty-cycle floor during convergence.
if runtime_config.cycle_interval_source != ScannerRuntimeConfigSource::Default {
return Some(cycle_interval);
retry_interval = retry_interval.max(runtime_config.cycle_interval);
}
// A complete but non-authoritative walk must not postpone convergence
// beyond the ordinary cadence, even without lifecycle or replication.
Some(retry_interval.min(cycle_interval))
// A complete but non-authoritative walk must not postpone lifecycle or
// replication evaluation beyond its ordinary maintenance cadence.
if maintenance_features.needs_regular_cycle() {
retry_interval = retry_interval.min(runtime_config.cycle_interval.max(Duration::from_secs(1)));
}
Some(retry_interval)
}
/// Pacing for the cycle after a full-namespace scope fallback while dirty
+38 -73
View File
@@ -8947,7 +8947,7 @@ fn clean_idle_backoff_grows_to_cap() {
}
#[test]
fn retry_backoff_grows_caps_and_resets_after_convergence() {
fn superseded_retry_backoff_grows_caps_and_resets_after_convergence() {
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(backoff.retry_interval(Duration::from_secs(24 * 60 * 60)), None);
@@ -8971,7 +8971,7 @@ fn retry_backoff_grows_caps_and_resets_after_convergence() {
}
#[test]
fn retry_backoff_respects_a_faster_configured_cycle() {
fn superseded_retry_backoff_respects_a_faster_configured_cycle() {
let mut backoff = ScannerRetryBackoff::default();
backoff.record_retryable_cycle(true);
@@ -8983,7 +8983,7 @@ fn retry_backoff_respects_a_faster_configured_cycle() {
}
#[test]
fn retry_backoff_grows_from_the_default_cycle() {
fn superseded_retry_backoff_grows_from_the_default_cycle() {
let mut backoff = ScannerRetryBackoff::default();
// The first race after a write burst retries in seconds, not a whole
// cycle, while repeated supersedes still climb toward the cap.
@@ -9003,7 +9003,7 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
..Default::default()
};
assert_eq!(
scanner_superseded_retry_interval(backoff, &default_config),
scanner_superseded_retry_interval(backoff, &default_config, ScannerMaintenanceFeatures::default()),
Some(SCANNER_RETRY_BASE_INTERVAL),
"the default adaptive cadence must retain fast convergence"
);
@@ -9013,24 +9013,24 @@ fn superseded_retry_preserves_explicit_cycle_cadence() {
ScannerRuntimeConfigSource::Config,
ScannerRuntimeConfigSource::ScannerCompatConfig,
] {
for cycle_secs in [1, 3, 600, 6 * 60 * 60] {
let runtime_config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(cycle_secs),
cycle_interval_source: source,
let runtime_config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(6 * 60 * 60),
cycle_interval_source: source,
..Default::default()
};
for features in [
ScannerMaintenanceFeatures::default(),
ScannerMaintenanceFeatures {
lifecycle: true,
..Default::default()
};
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
for _ in 0..40 {
backoff.record_retryable_cycle(true);
assert_eq!(
scanner_superseded_retry_interval(backoff, &runtime_config),
Some(runtime_config.cycle_interval),
"{source:?} cycle cadence must hold throughout convergence retries"
);
}
backoff.record_retryable_cycle(false);
assert_eq!(scanner_superseded_retry_interval(backoff, &runtime_config), None);
},
] {
assert_eq!(
scanner_superseded_retry_interval(backoff, &runtime_config, features),
Some(runtime_config.cycle_interval),
"{source:?} cycle cadence must not be shortened by convergence retries"
);
}
}
}
@@ -9051,7 +9051,7 @@ fn superseded_retry_does_not_delay_required_maintenance() {
for consecutive_cycles in 1..=12 {
backoff.record_retryable_cycle(true);
assert_eq!(
scanner_superseded_retry_interval(backoff, &config),
scanner_superseded_retry_interval(backoff, &config, features),
Some(config.cycle_interval),
"superseded cycle {consecutive_cycles} must preserve regular maintenance"
);
@@ -9137,50 +9137,19 @@ fn scope_fallback_retry_preserves_explicit_and_fast_cycle_cadence() {
}
#[test]
fn superseded_retry_caps_at_the_default_cycle_and_resets_after_convergence() {
fn superseded_retry_only_caps_regular_maintenance() {
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(60),
..Default::default()
};
let mut backoff = ScannerRetryBackoff::default();
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
for expected in [5, 10, 20, 40, 60, 60, 60, 60] {
backoff.record_retryable_cycle(true);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(Duration::from_secs(expected)));
}
backoff.consecutive_cycles = u32::MAX;
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
backoff.record_retryable_cycle(true);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), Some(config.cycle_interval));
backoff.record_retryable_cycle(false);
assert_eq!(scanner_superseded_retry_interval(backoff, &config), None);
}
#[test]
fn superseded_retry_caps_fast_cycles_without_changing_the_generic_retry_cap() {
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..40 {
for _ in 0..6 {
backoff.record_retryable_cycle(true);
}
assert_eq!(
backoff.retry_interval(Duration::from_secs(60)),
Some(Duration::from_secs(30 * 60)),
"deferred and transient retries must retain the generic cap"
scanner_superseded_retry_interval(backoff, &config, ScannerMaintenanceFeatures::default()),
Some(Duration::from_secs(160)),
);
for cycle_secs in [0, 1, 3, 60, 6 * 60 * 60] {
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(cycle_secs),
..Default::default()
};
assert_eq!(
scanner_superseded_retry_interval(backoff, &config),
Some(Duration::from_secs(cycle_secs.max(1))),
);
}
}
#[test]
fn superseded_retry_timer_keeps_required_maintenance_full_scans() {
for features in [
ScannerMaintenanceFeatures {
lifecycle: true,
@@ -9195,23 +9164,28 @@ fn superseded_retry_timer_keeps_required_maintenance_full_scans() {
..Default::default()
},
] {
assert!(features.requires_full_scan(Some(7), 7, ScannerCycleWakeReason::Timer, true));
assert_eq!(scanner_superseded_retry_interval(backoff, &config, features), Some(config.cycle_interval));
assert_eq!(scanner_superseded_retry_interval(ScannerRetryBackoff::default(), &config, features), None);
}
}
#[tokio::test(start_paused = true)]
#[serial]
async fn superseded_retry_wait_keeps_convergence_fencing_at_the_default_cycle() {
async fn superseded_maintenance_wait_keeps_convergence_fencing() {
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
let config = ScannerRuntimeConfig {
cycle_interval: Duration::from_secs(60),
..Default::default()
};
let features = ScannerMaintenanceFeatures {
lifecycle: true,
..Default::default()
};
let mut backoff = ScannerRetryBackoff::default();
for _ in 0..12 {
backoff.record_retryable_cycle(true);
}
let interval = scanner_superseded_retry_interval(backoff, &config);
let interval = scanner_superseded_retry_interval(backoff, &config, features);
assert_eq!(interval, Some(config.cycle_interval));
let generations = ScannerCycleObservedGenerations::for_wait(
&config,
@@ -9226,28 +9200,19 @@ async fn superseded_retry_wait_keeps_convergence_fencing_at_the_default_cycle()
let mut seen = Some(BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]));
let changed = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 8, 3))]);
let started = tokio::time::Instant::now();
let mut wait = Box::pin(wait_for_next_scanner_cycle_with_activity(
let reason = wait_for_next_scanner_cycle_with_activity(
&ctx,
interval.expect("superseded retry must retain its convergence interval"),
interval.expect("maintenance must retain its convergence interval"),
Some(Duration::from_secs(1)),
&mut seen,
generations,
|| false,
|| std::future::ready(Ok(changed.clone())),
));
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
crate::scanner_io::record_dirty_usage_bucket("photos");
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
for _ in 0..3 {
tokio::time::advance(Duration::from_secs(15)).await;
crate::scanner_io::record_dirty_usage_bucket("photos");
assert!(matches!(futures::poll!(&mut wait), Poll::Pending));
}
let reason = wait.await;
)
.await;
assert_eq!(reason, ScannerCycleWakeReason::Timer);
assert_eq!(started.elapsed(), config.cycle_interval);
assert_eq!(seen, Some(changed));
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
}
#[test]
+44 -105
View File
@@ -59,7 +59,6 @@ use rustfs_scanner_metrics::metrics::{
CloseDiskGuard, IlmAction, Metric, Metrics, ScannerReplicationRepairKind, ScannerSourceWorkUpdate, ScannerWorkSource,
UpdateCurrentPathFn, current_path_updater, global_metrics,
};
use rustfs_scanner_metrics::scan_observation::{ScannerCacheReuseDecision, ScannerCacheReuseGate, ScannerCacheReuseKind};
use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
use sha2::{Digest, Sha256};
use time::OffsetDateTime;
@@ -1061,51 +1060,38 @@ impl FolderScanner {
child_hash: &DataUsageHash,
abandoned_children: &DataUsageHashMap,
) -> bool {
let decision = if let Some(prefix_scan_scope) = &self.prefix_scan_scope {
if folder.parent.is_some() || folder.name != self.old_cache.info.name {
ScannerCacheReuseDecision::NotRoot
} else if into.compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !abandoned_children.contains(&child_hash.0) {
ScannerCacheReuseDecision::MissingCachedChild
} else if let Some(entry) = child
.name
.strip_prefix(folder.name.as_str())
.and_then(|entry| entry.strip_prefix('/'))
{
if entry.is_empty() || entry.contains('/') {
ScannerCacheReuseDecision::InvalidPrefix
} else if prefix_scan_scope.contains(entry) {
ScannerCacheReuseDecision::DirtyPrefix
} else if self.is_erasure_mode {
// EC scans must still perform independent object-health work.
global_metrics().scan_observation().record_erasure_clean_prefix_walk();
ScannerCacheReuseDecision::ErasureHealthRequired
} else {
ScannerCacheReuseDecision::Eligible
}
} else {
ScannerCacheReuseDecision::InvalidPrefix
}
} else {
ScannerCacheReuseDecision::MissingScope
let Some(prefix_scan_scope) = &self.prefix_scan_scope else {
return false;
};
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::CleanPrefix, decision);
decision == ScannerCacheReuseDecision::Eligible
// Erasure-mode usage scans also perform probabilistic object-health
// work. Reusing a clean subtree here would silently suppress that
// independent maintenance path, so prefix reuse is limited to the
// non-erasure data-usage scanner.
if self.is_erasure_mode
|| folder.parent.is_some()
|| folder.name != self.old_cache.info.name
|| into.compacted
|| !abandoned_children.contains(&clone_cache_value(&child_hash.0))
{
return false;
}
let Some(entry) = child
.name
.strip_prefix(folder.name.as_str())
.and_then(|entry| entry.strip_prefix('/'))
else {
return false;
};
!entry.is_empty() && !entry.contains('/') && !prefix_scan_scope.contains(entry)
}
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
if let Some(updates) = &mut self.usage_updates {
updates.cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
}
into.add_child(child_hash);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::CleanPrefix, started.elapsed());
}
fn failed_retry_suppressed(&self, path: &str) -> bool {
@@ -1292,60 +1278,14 @@ impl FolderScanner {
}
fn can_reuse_covered_namespace(&self, name: &str, compacted: bool) -> bool {
let decision = if self.old_cache.info.scan_progress.is_none() {
ScannerCacheReuseDecision::NoResumeSweep
} else if compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !self
.resume_frontier
.as_deref()
.is_some_and(|frontier| scan_frontier_covers_path(name, frontier))
{
ScannerCacheReuseDecision::MissingCoverage
} else if self.old_cache.find(name).is_none() {
ScannerCacheReuseDecision::MissingCachedChild
} else if self.resume_subtree_has_failures(name) {
ScannerCacheReuseDecision::CachedFailures
} else {
ScannerCacheReuseDecision::Eligible
};
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::CoveredNamespace, decision);
decision == ScannerCacheReuseDecision::Eligible
}
// None preserves work that cannot use sampling; false requests this cycle's rescan.
fn sampled_namespace_reuse(&self, hash: &DataUsageHash, compacted: bool, forward_sweep: bool) -> Option<bool> {
let decision = if self.scan_mode == HealScanMode::Deep {
ScannerCacheReuseDecision::DeepScan
} else if forward_sweep {
ScannerCacheReuseDecision::ForwardSweep
} else if self
.old_cache
.info
.replication
.as_ref()
.is_some_and(|config| config.has_active_rules("", true))
{
ScannerCacheReuseDecision::ActiveReplication
} else if compacted {
ScannerCacheReuseDecision::CompactedParent
} else if !self.old_cache.is_compacted(hash) {
ScannerCacheReuseDecision::UncompactedChild
} else if hash.mod_(self.old_cache.info.next_cycle as u32, data_usage_update_dir_cycles()) {
ScannerCacheReuseDecision::SelectedForWalk
} else {
ScannerCacheReuseDecision::Eligible
};
global_metrics()
.scan_observation()
.record_cache_reuse_decision(ScannerCacheReuseGate::NamespaceSampling, decision);
match decision {
ScannerCacheReuseDecision::Eligible => Some(true),
ScannerCacheReuseDecision::SelectedForWalk => Some(false),
_ => None,
}
self.old_cache.info.scan_progress.is_some()
&& !compacted
&& self
.resume_frontier
.as_deref()
.is_some_and(|frontier| scan_frontier_covers_path(name, frontier))
&& self.old_cache.find(name).is_some()
&& !self.resume_subtree_has_failures(name)
}
fn resume_subtree_has_failures(&self, folder: &str) -> bool {
@@ -2137,11 +2077,7 @@ impl FolderScanner {
// The caller stitches this cached child into its
// parent; do not clone a growing child set into the
// unused provisional accumulator as well.
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &this_hash, &folder.parent);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::ClassifiedNamespace, started.elapsed());
self.budget.record_entries_visited(pending_entry_progress);
self.finish_raw_enumeration_parent(&folder.name);
done_folder();
@@ -2728,12 +2664,8 @@ impl FolderScanner {
FolderScanMode::Namespace
};
if child_mode == FolderScanMode::Namespace && reuse_covered {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::CoveredNamespace, started.elapsed());
continue;
}
@@ -2773,15 +2705,22 @@ impl FolderScanner {
}
FolderScanSource::Existing => {
// Usage sampling is not proof that a Deep check ran.
if let Some(reuse) = self.sampled_namespace_reuse(&h, into.compacted, forward_sweep) {
if reuse {
if self.scan_mode != HealScanMode::Deep
&& !forward_sweep
&& !self
.old_cache
.info
.replication
.as_ref()
.is_some_and(|config| config.has_active_rules("", true))
&& !into.compacted
&& self.old_cache.is_compacted(&h)
{
let next_cycle = self.old_cache.info.next_cycle as u32;
if !h.mod_(next_cycle, data_usage_update_dir_cycles()) {
if child_mode == FolderScanMode::Namespace {
let started = Instant::now();
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
global_metrics()
.scan_observation()
.record_cache_reuse(ScannerCacheReuseKind::SampledNamespace, started.elapsed());
self.record_scan_resume_hint(&folder_item.name);
continue;
}
-191
View File
@@ -3103,112 +3103,6 @@ async fn test_scan_folder_non_erasure_metadata_keeps_namespace_descent() {
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Directories));
}
#[tokio::test]
#[serial]
async fn cache_reuse_gates_report_only_the_first_production_condition() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir);
scanner.old_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/child",
"bucket",
DataUsageEntry {
compacted: true,
objects: 2,
..Default::default()
},
);
let hash = hash_path("bucket/child");
let folder = CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
};
let child = CachedFolder {
name: "bucket/child".to_string(),
parent: Some(hash_path("bucket")),
object_heal_prob_div: 1,
};
let abandoned = HashSet::from([hash.0.clone()]);
let before = global_metrics().scan_observation().snapshot();
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
let missing_scope = global_metrics().scan_observation().snapshot();
assert_eq!(
missing_scope.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"],
before.reuse_decisions["clean_prefix"].not_evaluated["missing_scope"] + 1
);
scanner.prefix_scan_scope = ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["dirty".to_string()]));
scanner.is_erasure_mode = true;
assert!(!scanner.should_reuse_clean_root_child(&folder, &DataUsageEntry::default(), &child, &hash, &abandoned));
let erasure = global_metrics().scan_observation().snapshot();
assert_eq!(
erasure.reuse_decisions["clean_prefix"].blocked["erasure_health_required"],
before.reuse_decisions["clean_prefix"].blocked["erasure_health_required"] + 1
);
assert_eq!(erasure.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
});
assert!(!scanner.can_reuse_covered_namespace("bucket/child", true));
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
scanner.resume_frontier = Some("bucket/child".to_string());
scanner.resume_failed_paths = vec!["bucket/child/failed".to_string()];
assert!(!scanner.can_reuse_covered_namespace("bucket/child", false));
scanner.resume_failed_paths.clear();
assert!(scanner.can_reuse_covered_namespace("bucket/child", false));
let covered = global_metrics().scan_observation().snapshot();
let reasons = &covered.reuse_decisions["covered_namespace"];
let prior = &before.reuse_decisions["covered_namespace"];
assert_eq!(reasons.not_evaluated["no_resume_sweep"], prior.not_evaluated["no_resume_sweep"] + 1);
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
assert_eq!(reasons.blocked["missing_coverage"], prior.blocked["missing_coverage"] + 1);
assert_eq!(reasons.blocked["cached_failures"], prior.blocked["cached_failures"] + 1);
assert_eq!(reasons.eligible, prior.eligible + 1);
with_var(ENV_DATA_USAGE_UPDATE_DIR_CYCLES, Some("16"), || {
scanner.scan_mode = HealScanMode::Deep;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
scanner.scan_mode = HealScanMode::Normal;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, true), None);
let replication: ReplicationConfig = serde_json::from_value(serde_json::json!({
"config": { "role": "", "rules": [{ "destination": { "bucket": "arn:test" }, "id": "status", "prefix": "", "priority": 1, "status": "Enabled" }] },
"remotes": crate::storage_api::EcstoreBucketTargets::default(),
})).expect("active replication fixture");
scanner.old_cache.info.replication = Some(Arc::new(replication));
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
scanner.old_cache.info.replication = None;
assert_eq!(scanner.sampled_namespace_reuse(&hash, true, false), None);
let rescan_cycle = (0..16)
.find(|cycle| hash.mod_(*cycle, 16))
.expect("hash is visited in one sampling cycle");
scanner.old_cache.info.next_cycle = u64::from(rescan_cycle);
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(false));
scanner.old_cache.info.next_cycle = u64::from((rescan_cycle + 1) % 16);
assert_eq!(scanner.sampled_namespace_reuse(&hash, false, false), Some(true));
});
let after = global_metrics().scan_observation().snapshot();
let reasons = &after.reuse_decisions["namespace_sampling"];
let prior = &before.reuse_decisions["namespace_sampling"];
for label in ["deep_scan", "forward_sweep", "active_replication"] {
assert_eq!(
reasons.not_evaluated[label],
prior.not_evaluated[label] + 1,
"first sampling condition {label}"
);
}
assert_eq!(reasons.blocked["compacted_parent"], prior.blocked["compacted_parent"] + 1);
assert_eq!(reasons.blocked["selected_for_walk"], prior.blocked["selected_for_walk"] + 1);
assert_eq!(reasons.eligible, prior.eligible + 1);
for (kind, reuse) in &after.cache_reuse {
assert_eq!(reuse.count, before.cache_reuse[kind].count, "gate admission alone does not copy {kind}");
}
}
#[tokio::test]
#[serial]
async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entries() {
@@ -3250,21 +3144,12 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
parent: None,
object_heal_prob_div: 1,
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
.await
.expect("scoped root scan should finish successfully");
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count + 1);
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks);
assert_eq!(
observed.reuse_decisions["clean_prefix"].eligible,
before.reuse_decisions["clean_prefix"].eligible + 1
);
let clean = scanner
.new_cache
.size_recursive("bucket/clean")
@@ -3282,78 +3167,6 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
assert_eq!((bucket.size, bucket.objects), (17, 3));
}
#[tokio::test]
#[serial]
async fn scoped_cache_observations_distinguish_checkpoint_and_periodic_sampling() {
for covered in [true, false] {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
tokio::fs::create_dir_all(temp_dir.join("bucket/child"))
.await
.expect("create cached namespace");
scanner.old_cache.info.name = "bucket".to_string();
scanner.new_cache.info.name = "bucket".to_string();
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
scanner.old_cache.replace(
"bucket/child",
"bucket",
DataUsageEntry {
size: 17,
objects: 5,
compacted: true,
..Default::default()
},
);
let kind = if covered {
scanner.old_cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: crate::DataUsageScanPlanDigest([1; 32]),
requested_plan: crate::DataUsageScanPlanDigest([1; 32]),
});
scanner.resume_frontier = Some("bucket/child".to_string());
"covered_namespace"
} else {
let hash = hash_path("bucket/child");
let cycle = (0..data_usage_update_dir_cycles())
.find(|cycle| !hash.mod_(*cycle, data_usage_update_dir_cycles()))
.expect("a cycle samples other work");
scanner.old_cache.info.next_cycle = u64::from(cycle);
"sampled_namespace"
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(
CancellationToken::new(),
CachedFolder {
name: "bucket".to_string(),
parent: None,
object_heal_prob_div: 1,
},
&mut root,
)
.await
.expect("cached namespace reuses accounting");
let summary = scanner.new_cache.size_recursive("bucket").expect("complete bucket totals");
assert_eq!((summary.size, summary.objects), (17, 5));
let after = global_metrics().scan_observation().snapshot();
for label in [
"clean_prefix",
"covered_namespace",
"sampled_namespace",
"classified_namespace",
] {
assert_eq!(
after.cache_reuse[label].count,
before.cache_reuse[label].count + u64::from(label == kind),
"reuse reason {label}"
);
}
assert_eq!(after.metadata_reads.attempts, before.metadata_reads.attempts);
let gate = if covered { "covered_namespace" } else { "namespace_sampling" };
assert_eq!(after.reuse_decisions[gate].eligible, before.reuse_decisions[gate].eligible + 1);
}
}
async fn scan_hot_cold_segment_fixture(scoped: bool) -> (DataUsageEntry, Vec<String>) {
let (mut scanner, temp_dir) = build_test_scanner().await;
let visits = metadata_visits::observe(&mut scanner);
@@ -3465,7 +3278,6 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
parent: None,
object_heal_prob_div: 1,
};
let before = global_metrics().scan_observation().snapshot();
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
@@ -3477,9 +3289,6 @@ async fn scoped_root_scan_preserves_erasure_health_walks() {
.size_recursive("bucket/clean")
.expect("erasure scan should visit the clean entry");
assert_eq!((clean.size, clean.objects), (0, 0));
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(observed.erasure_clean_prefix_walks, before.erasure_clean_prefix_walks + 1);
assert_eq!(observed.cache_reuse["clean_prefix"].count, before.cache_reuse["clean_prefix"].count);
}
#[tokio::test]
@@ -774,7 +774,6 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
},
);
scanner.budget = budget.clone();
let before = global_metrics().scan_observation().snapshot();
scanner
.scan_folder(
budget.token(),
@@ -791,15 +790,6 @@ async fn erasure_candidate_covered_namespace_reuses_totals_after_classification(
assert_eq!(budget.progress(), (0, 1), "only the parent consumes namespace quota");
assert_eq!(charged_ops(&root), 4, "list both directories once without re-reading covered metadata");
assert_eq!(scanner.new_cache.find(&folder.name).expect("reused candidate").objects, 1);
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(
observed.cache_reuse["classified_namespace"].count,
before.cache_reuse["classified_namespace"].count + 1
);
assert_eq!(
observed.metadata_reads.attempts, before.metadata_reads.attempts,
"classification retains the existing zero metadata read path"
);
}
#[tokio::test]
@@ -879,7 +869,6 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
},
);
scanner.budget = budget.clone();
let before = global_metrics().scan_observation().snapshot();
scanner
.scan_folder(
budget.token(),
@@ -900,15 +889,6 @@ async fn erasure_candidate_preserves_compacted_namespace_sampling_after_classifi
"sampling still requires full current classification, without metadata re-read"
);
assert_eq!(scanner.new_cache.find(&folder.name).expect("sampled candidate").objects, 1);
let observed = global_metrics().scan_observation().snapshot();
assert_eq!(
observed.cache_reuse["classified_namespace"].count,
before.cache_reuse["classified_namespace"].count + 1
);
assert_eq!(
observed.metadata_reads.attempts, before.metadata_reads.attempts,
"classification retains the existing zero metadata read path"
);
}
#[tokio::test]
+46 -133
View File
@@ -40,7 +40,6 @@ use rustfs_lock::{LockError, NamespaceLockGuard};
use rustfs_scanner_metrics::metrics::{
Metric, Metrics, emit_scan_bucket_drive_complete, emit_scan_bucket_drive_partial, global_metrics,
};
use rustfs_scanner_metrics::scan_observation::{PREFIX_REUSE_BLOCKERS, ScannerBaselineRejection, ScannerPrefixReuseStage};
use rustfs_utils::path::path_join_buf;
use sha2::{Digest as _, Sha256};
use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet};
@@ -113,7 +112,16 @@ pub(crate) const SCANNER_SEGMENT_ACTIVATION_PROOF_INPUTS: [&str; 9] = [
"durable_dirty_producer_journal",
"ack_generation_guard",
];
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = PREFIX_REUSE_BLOCKERS;
pub(crate) const SCANNER_SEGMENT_ACTIVATION_FAIL_CLOSED_CHECKS: [&str; 8] = [
"missing_producer_identity",
"missing_durable_journal_replay",
"restart_gap",
"generation_gap",
"overflow",
"stale_ack_generation",
"missing_cold_zero_walk_oracle",
"distributed_without_peer_invalidation",
];
pub type DirtyUsageBuckets = HashMap<String, u64>;
@@ -189,7 +197,6 @@ struct VerifiedRemoteDirtyUsage {
}
struct ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Option<ScannerSegmentReuseActivationPreflight>,
scope: ScannerBucketScanScope,
remote_dirty_usage_acknowledgements: Vec<crate::scanner::ScannerDirtyUsageAcknowledgement>,
distributed_segment_invalidation_evidence: Option<DistributedSegmentInvalidationEvidence>,
@@ -229,13 +236,6 @@ pub(crate) struct ScannerSegmentReuseActivationPreflight {
}
impl ScannerSegmentReuseActivationPreflight {
fn record_observation(&self, stage: ScannerPrefixReuseStage) {
global_metrics().scan_observation().record_prefix_reuse(
stage,
self.scanner_segment_reuse_activated,
&self.fail_closed_blockers,
);
}
#[cfg(test)]
pub(crate) fn fail_closed_blockers(&self) -> impl Iterator<Item = &'static str> + '_ {
self.fail_closed_blockers.iter().filter_map(|blocker| *blocker)
@@ -319,7 +319,6 @@ fn resolve_remote_dirty_usage_scope(
baseline_proof: ScannerCacheBaselineProof<'_>,
) -> ScannerBucketScopeResolutionResult {
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: None,
scope,
remote_dirty_usage_acknowledgements: Vec::new(),
distributed_segment_invalidation_evidence: None,
@@ -411,7 +410,6 @@ fn resolve_remote_dirty_usage_scope(
};
ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
scope,
remote_dirty_usage_acknowledgements: scoped_acknowledgements,
distributed_segment_invalidation_evidence,
@@ -429,81 +427,43 @@ fn complete_scanner_cache_snapshot_plan_digest(
proof: ScannerCacheBaselineProof<'_>,
expected_converged: Option<bool>,
) -> Option<DataUsageScanPlanDigest> {
validate_scanner_cache_snapshot_plan_digest(snapshot, proof, expected_converged).ok()
}
fn validate_scanner_cache_snapshot_plan_digest(
snapshot: &DataUsageInfo,
proof: ScannerCacheBaselineProof<'_>,
expected_converged: Option<bool>,
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineRejection> {
// The observational mark disqualifies only the authoritative slot: an
// observed candidate carries it by construction.
let marks_accepted = match expected_converged {
Some(expected) => snapshot.usage_snapshot_converged == Some(expected),
None => snapshot.usage_snapshot_converged.is_some() && !snapshot.usage_snapshot_observational,
};
if !snapshot.is_complete_bucket_usage_snapshot() {
return Err(ScannerBaselineRejection::Incomplete);
}
if snapshot.usage_snapshot_partial {
return Err(ScannerBaselineRejection::Partial);
}
if !marks_accepted {
return Err(ScannerBaselineRejection::Marks);
}
if snapshot.scanner_epoch != Some(proof.leader_epoch) {
return Err(ScannerBaselineRejection::Epoch);
}
if snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len() {
return Err(ScannerBaselineRejection::SourceCount);
if !snapshot.is_complete_bucket_usage_snapshot()
|| snapshot.usage_snapshot_partial
|| !marks_accepted
|| snapshot.scanner_epoch != Some(proof.leader_epoch)
|| snapshot.usage_snapshot_set_states.len() != proof.expected_sources.len()
{
return None;
}
// Completed maintenance also covers ordinary usage. Keep its exact stored
// proof for cache reuse, and reject mixtures of different set work proofs.
let baseline_plan_digest = DataUsageScanPlanDigest(
snapshot
.usage_snapshot_set_states
.first()
.and_then(|state| state.scan_plan_digest)
.ok_or(ScannerBaselineRejection::PlanMissing)?,
);
let baseline_plan_digest = DataUsageScanPlanDigest(snapshot.usage_snapshot_set_states.first()?.scan_plan_digest?);
if !scanner_plan_digest_is_work_variant(proof.scan_plan_digest, baseline_plan_digest) {
return Err(ScannerBaselineRejection::PlanMismatch);
return None;
}
let mut states = HashSet::with_capacity(snapshot.usage_snapshot_set_states.len());
for state in &snapshot.usage_snapshot_set_states {
let source = DataUsageCacheSource::new(
usize::try_from(state.pool_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
usize::try_from(state.set_index).map_err(|_| ScannerBaselineRejection::SourceIndex)?,
);
if !proof.expected_sources.contains(&source) {
return Err(ScannerBaselineRejection::SourceMembership);
}
if !states.insert(source) {
return Err(ScannerBaselineRejection::SourceDuplicate);
}
if !state.complete {
return Err(ScannerBaselineRejection::SetIncomplete);
}
if state.tombstone {
return Err(ScannerBaselineRejection::SetTombstone);
}
if state.scanner_epoch != Some(proof.leader_epoch) {
return Err(ScannerBaselineRejection::SetEpoch);
}
if state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle) {
return Err(ScannerBaselineRejection::SetCycle);
}
if state.scan_plan_digest != Some(baseline_plan_digest.0) {
return Err(ScannerBaselineRejection::SetPlan);
let source = DataUsageCacheSource::new(usize::try_from(state.pool_index).ok()?, usize::try_from(state.set_index).ok()?);
if !proof.expected_sources.contains(&source)
|| !states.insert(source)
|| !state.complete
|| state.tombstone
|| state.scanner_epoch != Some(proof.leader_epoch)
|| state.scanner_cycle.is_none_or(|cycle| cycle > proof.want_cycle)
|| state.scan_plan_digest != Some(baseline_plan_digest.0)
{
return None;
}
}
if states != *proof.expected_sources {
return Err(ScannerBaselineRejection::SourceCoverage);
}
Ok(baseline_plan_digest)
(states == *proof.expected_sources).then_some(baseline_plan_digest)
}
/// Whether `digest` is `structural` under any work flag: the same bucket
@@ -533,35 +493,10 @@ fn scanner_baseline_pending_buckets(proof: &ScannerCacheBaselineProof<'_>) -> Ve
}
fn complete_scanner_cache_baseline_plan_digest(proof: ScannerCacheBaselineProof<'_>) -> Option<DataUsageScanPlanDigest> {
validate_scanner_cache_baseline_plan_digest(proof).ok()
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
struct ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection,
observed: Option<ScannerBaselineRejection>,
}
fn validate_scanner_cache_baseline_plan_digest(
proof: ScannerCacheBaselineProof<'_>,
) -> std::result::Result<DataUsageScanPlanDigest, ScannerBaselineValidationFailure> {
let authoritative_error = |authoritative| ScannerBaselineValidationFailure {
authoritative,
observed: None,
};
let authoritative_data = proof
.authoritative_data
.ok_or_else(|| authoritative_error(ScannerBaselineRejection::Missing))?;
let authoritative = serde_json::from_slice::<DataUsageInfo>(authoritative_data)
.map_err(|_| authoritative_error(ScannerBaselineRejection::Decode))?;
let authoritative_rejection = match validate_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
Ok(validated_digest) => return Ok(validated_digest),
Err(reason) => reason,
};
let observed_error = |observed| ScannerBaselineValidationFailure {
authoritative: authoritative_rejection,
observed: Some(observed),
};
let authoritative = serde_json::from_slice::<DataUsageInfo>(proof.authoritative_data?).ok()?;
if let Some(validated_digest) = complete_scanner_cache_snapshot_plan_digest(&authoritative, proof, None) {
return Some(validated_digest);
}
// A complete but superseded observation may reuse its per-set cache only
// when it was explicitly tied to the durable authoritative baseline. It
@@ -571,18 +506,14 @@ fn validate_scanner_cache_baseline_plan_digest(
&& !authoritative.usage_snapshot_observational)
|| crate::scanner::data_usage_info_is_bootstrap_pending(&authoritative);
if !authoritative_has_identity {
return Err(observed_error(ScannerBaselineRejection::AuthoritativeIdentity));
return None;
}
let observed_data = proof
.observed_candidate_data
.ok_or_else(|| observed_error(ScannerBaselineRejection::Missing))?;
let observed =
serde_json::from_slice::<DataUsageInfo>(observed_data).map_err(|_| observed_error(ScannerBaselineRejection::Decode))?;
let observed = serde_json::from_slice::<DataUsageInfo>(proof.observed_candidate_data?).ok()?;
if !observed_data_usage_is_newer(&observed, &authoritative) {
return Err(observed_error(ScannerBaselineRejection::NotNewer));
return None;
}
validate_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false)).map_err(observed_error)
complete_scanner_cache_snapshot_plan_digest(&observed, proof, Some(false))
}
fn scanner_segment_reuse_baseline_producer_evidence(
@@ -727,12 +658,6 @@ impl ScannerScopeFallbackReason {
}
}
#[derive(Clone, Copy)]
struct ScannerScopeFallback {
reason: ScannerScopeFallbackReason,
baseline_rejection: Option<ScannerBaselineValidationFailure>,
}
/// Classify a resolved full-namespace scope against the preconditions
/// `resolve_scanner_bucket_scan_scope` checks, in the same order. Returns
/// `None` when the cycle was required to walk everything or was scoped.
@@ -743,24 +668,18 @@ fn scanner_scope_fallback_reason(
all_buckets: &[BucketInfo],
baseline_proof: ScannerCacheBaselineProof<'_>,
distributed: bool,
) -> Option<ScannerScopeFallback> {
let fallback = |reason| {
Some(ScannerScopeFallback {
reason,
baseline_rejection: None,
})
};
) -> Option<ScannerScopeFallbackReason> {
if !scope_eligible || !resolved_scope.is_default() {
return None;
}
if dirty_usage_snapshot.generation == u64::MAX {
return fallback(ScannerScopeFallbackReason::DirtyGenerationExhausted);
return Some(ScannerScopeFallbackReason::DirtyGenerationExhausted);
}
if !dirty_usage_snapshot.covers_listed_pending {
return fallback(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
return Some(ScannerScopeFallbackReason::DirtySnapshotIncomplete);
}
if dirty_usage_snapshot.buckets.len() > crate::SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES {
return fallback(ScannerScopeFallbackReason::DirtyBucketOverflow);
return Some(ScannerScopeFallbackReason::DirtyBucketOverflow);
}
// Peers may contribute dirty buckets the local snapshot does not hold.
if !distributed {
@@ -770,19 +689,13 @@ fn scanner_scope_fallback_reason(
.keys()
.any(|bucket| listed.contains(bucket.as_str()))
{
return fallback(ScannerScopeFallbackReason::NoListedDirtyBuckets);
return Some(ScannerScopeFallbackReason::NoListedDirtyBuckets);
}
}
if let Err(failure) = validate_scanner_cache_baseline_plan_digest(baseline_proof) {
return Some(ScannerScopeFallback {
reason: ScannerScopeFallbackReason::BaselineUnavailable,
baseline_rejection: Some(failure),
});
if complete_scanner_cache_baseline_plan_digest(baseline_proof).is_none() {
return Some(ScannerScopeFallbackReason::BaselineUnavailable);
}
distributed.then_some(ScannerScopeFallback {
reason: ScannerScopeFallbackReason::PeerScopeUnavailable,
baseline_rejection: None,
})
distributed.then_some(ScannerScopeFallbackReason::PeerScopeUnavailable)
}
fn scanner_segment_reuse_activation_preflight() -> ScannerSegmentReuseActivationPreflight {
+7 -11
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use super::*;
/// scanner cache locks and the cache snapshot persist/publish path.
use crate::data_usage_define::{DataUsageCacheBaselineRevisions, clone_cache_value};
use crate::data_usage_define::clone_cache_value;
use crate::{ScannerConfigObjectDelete, ScannerObjectIO};
use rustfs_scanner_metrics::usage_candidate::{
ScannerUsageCandidateKind as CandidateKind, ScannerUsageCandidateOutcome as CandidateOutcome,
@@ -1494,7 +1494,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
store: Arc<SetDisks>,
updates: &mpsc::Sender<DataUsageCache>,
mut cache_snapshot: DataUsageCache,
initial_revisions: Option<&DataUsageCacheBaselineRevisions>,
initial_revisions: Option<&DataUsageCacheRevisions>,
cache_cycle_floor: &AtomicU64,
expected_publication_epoch: u64,
) -> Option<SystemTime> {
@@ -1536,10 +1536,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
};
let mut persisted = DataUsageCache::default();
let current_baseline = match persisted
.load_with_baseline_revisions(store.clone(), DATA_USAGE_CACHE_NAME)
.await
{
let revisions = match persisted.load_with_revisions(store.clone(), DATA_USAGE_CACHE_NAME).await {
Ok(revisions) => revisions,
Err(err) => {
error!(
@@ -1561,7 +1558,6 @@ pub(super) async fn persist_and_publish_cache_snapshot(
return None;
}
};
let revisions = current_baseline.revisions();
let Some(scan_plan_digest) = cache_snapshot.info.scan_plan_digest else {
log_missing_set_snapshot_metadata(&store, &cache_snapshot.info, "plan_digest");
return None;
@@ -1625,9 +1621,9 @@ pub(super) async fn persist_and_publish_cache_snapshot(
{
cache_snapshot = persisted;
} else {
// Another execution must not advance the baseline. A backup catching
// up to the same decoded primary changes no baseline content.
if !initial_revisions.is_some_and(|initial| current_baseline.permits_unchanged_baseline_since(initial)) {
// A later execution may have completed while this scan was walking.
// Only replace the cache revision from which this scan started.
if initial_revisions != Some(&revisions) {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
@@ -1666,7 +1662,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
let done_save = Metrics::time(Metric::SaveUsage);
if let Err(e) = cache_snapshot
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, revisions, expected_publication_epoch)
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
.await
{
error!(
+3 -48
View File
@@ -14,7 +14,6 @@
use super::*;
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
use crate::data_usage_define::clone_cache_value;
use rustfs_scanner_metrics::scan_observation::{ScannerBucketReuseKind, ScannerScanObservation};
pub(super) async fn collect_completed_bucket_results(
ctx: CancellationToken,
@@ -143,32 +142,6 @@ pub(super) struct PreparedScopedSetScan {
pub(super) buckets: Vec<BucketInfo>,
pub(super) cache: DataUsageCache,
pub(super) cold_bucket_reuse_proof: Option<ScopedColdBucketReuseProof>,
cold_bucket_reuse_count: u64,
cold_bucket_reuse_elapsed: Duration,
}
impl PreparedScopedSetScan {
pub(super) fn record_cold_bucket_reuse(&self, observation: &ScannerScanObservation) {
observation.record_bucket_reuse(
ScannerBucketReuseKind::ScopedColdBucket,
self.cold_bucket_reuse_count,
self.cold_bucket_reuse_elapsed,
);
}
}
/// Counts only roots delivered to the collector; cancellation or channel failure is not reuse.
pub(super) async fn send_current_cache_root_entry(
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
root: DataUsageEntryInfo,
cache: &DataUsageCache,
pending_maintenance_work: &AtomicBool,
observation: &ScannerScanObservation,
) -> std::result::Result<(), ScannerError> {
let started = Instant::now();
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await?;
observation.record_bucket_reuse(ScannerBucketReuseKind::CurrentRoot, 1, started.elapsed());
Ok(())
}
pub(super) struct ScopedColdBucketReuseProof {
@@ -252,8 +225,6 @@ pub(super) fn prepare_scoped_set_scan(
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let mut current_bucket_names = HashSet::with_capacity(all_buckets.len());
let mut cold_bucket_reuse_count = 0u64;
let mut cold_bucket_reuse_elapsed = Duration::ZERO;
for bucket in all_buckets {
if !current_bucket_names.insert(bucket.name.as_str()) {
return None;
@@ -262,14 +233,11 @@ pub(super) fn prepare_scoped_set_scan(
cache.info.scan_bucket_value_origins.remove(&bucket.name);
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
} else {
let started = Instant::now();
cache.copy_with_children(
old_cache,
&rustfs_data_usage::hash_path(&bucket.name),
&Some(rustfs_data_usage::hash_path(DATA_USAGE_ROOT)),
);
cold_bucket_reuse_count = cold_bucket_reuse_count.saturating_add(1);
cold_bucket_reuse_elapsed = cold_bucket_reuse_elapsed.saturating_add(started.elapsed());
}
}
@@ -281,8 +249,6 @@ pub(super) fn prepare_scoped_set_scan(
.collect(),
cache,
cold_bucket_reuse_proof,
cold_bucket_reuse_count,
cold_bucket_reuse_elapsed,
})
}
@@ -459,10 +425,7 @@ impl ScannerIOCache for SetDisks {
.ok_or_else(|| StorageError::other("scanner cache publication is blocked by data movement"))?,
};
let mut old_cache = DataUsageCache::default();
let initial_revisions = match old_cache
.load_with_baseline_revisions(self.clone(), DATA_USAGE_CACHE_NAME)
.await
{
let initial_revisions = match old_cache.load_with_revisions(self.clone(), DATA_USAGE_CACHE_NAME).await {
Ok(revisions) => Some(revisions),
Err(e) => {
warn!(
@@ -592,8 +555,6 @@ impl ScannerIOCache for SetDisks {
})
});
let mut scoped_cache = scoped_scan.map(|mut prepared| {
// The copies are completed work even if this turn later fails publication.
prepared.record_cold_bucket_reuse(global_metrics().scan_observation());
buckets = prepared.buckets;
prepared.cache.info.scan_coverage_digest = Some(bucket_coverage_digest);
prepared.cache
@@ -1597,14 +1558,8 @@ impl ScannerIOCache for SetDisks {
}
record_completed_usage_watch(&usage_watch_completions_clone, source, &bucket.name, bucket_usage_watch.as_ref()).await;
if let Err(e) =
send_current_cache_root_entry(
&bucket_result_tx_clone,
*root,
&cache,
&pending_maintenance_work_clone,
global_metrics().scan_observation(),
)
.await
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
+3 -20
View File
@@ -357,7 +357,6 @@ where
S: ScannerStorage,
{
let default_result = |scope: ScannerBucketScanScope| ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: None,
scope,
remote_dirty_usage_acknowledgements: Vec::new(),
distributed_segment_invalidation_evidence: None,
@@ -507,7 +506,7 @@ where
distributed,
resolution.baseline_proof,
);
let scope = scoped_scan_scope_from_dirty_buckets(
default_result(scoped_scan_scope_from_dirty_buckets(
resolution.requested_scope,
dirty_buckets,
(!distributed).then_some(resolution.dirty_usage_snapshot.scopes.as_ref()),
@@ -515,11 +514,7 @@ where
segment_reuse_activation_preflight.scanner_segment_reuse_activated,
resolution.all_buckets,
resolution.baseline_proof,
);
ScannerBucketScopeResolutionResult {
segment_reuse_activation_preflight: Some(segment_reuse_activation_preflight),
..default_result(scope)
}
))
}
/// Runs once between the bucket listing and the dirty-usage snapshot, the
@@ -764,9 +759,6 @@ where
},
)
.await;
if let Some(preflight) = scope_resolution.segment_reuse_activation_preflight {
preflight.record_observation(ScannerPrefixReuseStage::ScopeResolution);
}
let remote_dirty_usage_acknowledgements = scope_resolution.remote_dirty_usage_acknowledgements;
let distributed_segment_invalidation_evidence = scope_resolution.distributed_segment_invalidation_evidence;
let scan_scope = scope_resolution.scope;
@@ -778,14 +770,8 @@ where
baseline_proof,
distributed,
);
if let Some(fallback) = scope_fallback {
let reason = fallback.reason;
if let Some(reason) = scope_fallback {
global_metrics().record_scanner_scope_fallback(reason.as_str());
if let Some(failure) = fallback.baseline_rejection {
global_metrics()
.scan_observation()
.record_baseline_rejection(failure.authoritative, failure.observed);
}
tracing::info!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_SET_STATE,
@@ -799,7 +785,6 @@ where
"Scanner cycle walks every bucket because its dirty-bucket scope was not admitted"
);
}
let scope_fallback = scope_fallback.map(|fallback| fallback.reason);
let segment_invalidation_proof = dirty_usage_producer_evidence(&dirty_usage_snapshot).segment_invalidation_proof();
#[cfg(test)]
if let Some(observer) = resolved_scope_observer {
@@ -876,7 +861,6 @@ where
None,
false,
);
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
return Ok(ScannerCycleResult::new(status, dirty_usage_clear)
.with_publication_epoch(publication_epoch)
.with_activity_digest(activity_digest)
@@ -1223,7 +1207,6 @@ where
distributed_segment_invalidation_evidence,
cold_zero_walk_oracle,
);
segment_reuse_activation_preflight.record_observation(ScannerPrefixReuseStage::CycleCompletion);
let pending_maintenance_work = pending_maintenance_work_for_cycle(&pending_maintenance_work, &results);
let observed_cycle_floor = cache_cycle_floor.load(Ordering::Acquire);
let required_cycle_floor = (observed_cycle_floor > want_cycle).then_some(observed_cycle_floor);
+2 -15
View File
@@ -15,7 +15,6 @@ use super::*;
use crate::UNKNOWN_TIER;
/// ScannerIODisk implementation for Disk: get_size and the per-disk bucket scan.
use crate::data_usage_define::clone_cache_value;
use rustfs_scanner_metrics::scan_observation::ScannerMetadataReadOutcome;
///
/// Seed [`SizeSummary::tier_stats`] from the cached tier-name list.
@@ -63,20 +62,8 @@ impl ScannerIODisk for Disk {
ensure_scanner_node_operation_active(cancellation.as_ref())?;
let metadata_object_path = item.object_path();
let metadata_result = crate::workload_admission::with_scanner_metadata_io(async {
let read = global_metrics().start_scanner_metadata_read();
let result = self.read_metadata(&item.bucket, &metadata_object_path).await;
let (outcome, bytes) = match &result {
Ok(data) => (ScannerMetadataReadOutcome::Success, u64::try_from(data.len()).unwrap_or(u64::MAX)),
Err(e) if DiskError::is_err_object_not_found(e) || DiskError::is_err_version_not_found(e) => {
(ScannerMetadataReadOutcome::NotFound, 0)
}
Err(_) => (ScannerMetadataReadOutcome::Error, 0),
};
read.finish(outcome, bytes);
result
})
.await;
let metadata_result =
crate::workload_admission::with_scanner_metadata_io(self.read_metadata(&item.bucket, &metadata_object_path)).await;
ensure_scanner_node_operation_active(cancellation.as_ref())?;
let data = match metadata_result {
Ok(data) => data,
+7 -558
View File
@@ -27,10 +27,10 @@ use crate::storage_api::owner::{
};
use crate::storage_api::scan::{BucketOperations as _, DeleteBucketOptions, MakeBucketOptions, ObjectIO as _};
use crate::{
BUCKET_META_PREFIX, DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints,
InstanceContext, PoolEndpoints, ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER,
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
new_disk, path2_bucket_object_with_base_path, save_config,
DATA_USAGE_BLOOM_NAME_PATH, DiskOption, ECStore, Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints,
ScannerObjectOptions, ScannerPutObjReader, UNKNOWN_TIER, init_bucket_metadata_sys_for_scanner_tests,
init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx, new_disk, path2_bucket_object_with_base_path,
save_config,
};
use rustfs_concurrency::{
AdmissionState, WorkloadAdmissionRegistrySnapshot, WorkloadAdmissionSnapshot, WorkloadAdmissionSnapshotProvider,
@@ -104,7 +104,6 @@ fn scanner_activity_preflight_defers_a_temporarily_offline_peer() {
#[test]
fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
let before = global_metrics().scan_observation().snapshot();
let preflight = scanner_segment_reuse_activation_preflight();
assert!(preflight.production_activation);
@@ -125,9 +124,6 @@ fn scanner_segment_reuse_activation_preflight_reports_release_gate_inputs() {
"distributed_without_peer_invalidation",
]
);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated);
}
#[test]
@@ -2390,254 +2386,6 @@ async fn set_cache_send_failure_logs_receiver_and_snapshot_without_changing_retu
assert_set_cache_context(set_cache_event(&events, "publish_failed"), Some((0, 0)));
}
#[tokio::test]
#[serial]
async fn set_snapshot_accepts_backup_catchup_to_the_same_decoded_primary() {
use crate::data_usage_define::read_config_with_revision;
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read decoded primary bytes");
let primary_body = primary_body.expect("primary exists");
let mut older = cache.clone();
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode older backup"))
.await
.expect("seed a lagging backup");
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture primary with lagging backup");
save_config(Arc::clone(&set), &backup_path, primary_body)
.await
.expect("backup catches up without changing primary");
let current = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the reconciled backup");
assert_ne!(initial.revisions(), current.revisions());
assert!(current.permits_unchanged_baseline_since(&initial));
let mut candidate = cache;
candidate.info.next_cycle += 1;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
candidate.replace(
"photos",
DATA_USAGE_ROOT,
DataUsageEntry {
size: 20,
objects: 1,
..Default::default()
},
);
let cycle = candidate.info.next_cycle;
let cycle_floor = AtomicU64::new(cycle);
let (tx, mut rx) = mpsc::channel(1);
assert!(
persist_and_publish_cache_snapshot(Arc::clone(&set), &tx, candidate, Some(&initial), &cycle_floor, epoch)
.await
.is_some(),
"content-preserving backup catchup must not veto the new snapshot"
);
let published = rx.try_recv().expect("new snapshot reached the collector");
assert_eq!(published.info.next_cycle, cycle);
assert_eq!(published.find("photos").expect("new bucket entry").size, 20);
let (primary, primary_revision) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read new primary");
let (backup, backup_revision) = read_config_with_revision(set, &backup_path).await.expect("read new backup");
assert_eq!(primary, backup, "backup CAS must use the reconciled current revision");
assert_eq!(primary_revision, backup_revision);
let persisted = DataUsageCache::unmarshal(&primary.expect("new primary body")).expect("decode new primary");
assert_eq!(persisted.info.next_cycle, cycle);
assert_eq!(persisted.find("photos").expect("persisted new entry").size, 20);
}
#[tokio::test]
#[serial]
async fn set_snapshot_backup_catchup_keeps_revision_provenance_and_epoch_fences() {
use crate::data_usage_define::read_config_with_revision;
for case in ["different_backup", "different_primary", "corrupt_primary", "changed_epoch"] {
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read original primary");
let primary_body = primary_body.expect("original primary exists");
let mut older = cache.clone();
older.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, older.marshal_msg().expect("encode lagging backup"))
.await
.expect("seed lagging backup");
if case == "corrupt_primary" {
save_config(Arc::clone(&set), &primary_path, b"corrupt-primary".to_vec())
.await
.expect("inject primary corruption before the initial load");
}
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture original baseline source and revisions");
let mut replacement = cache.clone();
replacement.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
match case {
"different_backup" => {
save_config(Arc::clone(&set), &backup_path, replacement.marshal_msg().expect("encode changed backup"))
.await
.expect("backup changes to different contents");
}
"different_primary" => {
let body = replacement.marshal_msg().expect("encode changed primary");
save_config(Arc::clone(&set), &primary_path, body.clone())
.await
.expect("primary advances");
save_config(Arc::clone(&set), &backup_path, body)
.await
.expect("backup follows changed primary");
}
"corrupt_primary" => {
save_config(Arc::clone(&set), &backup_path, b"corrupt-primary".to_vec())
.await
.expect("backup matches the corrupt primary ETag without a decoded primary proof");
}
"changed_epoch" => {
save_config(Arc::clone(&set), &backup_path, primary_body)
.await
.expect("backup catches up before epoch refusal");
}
_ => unreachable!("fixed test cases"),
}
let before_primary = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("current primary");
let before_backup = read_config_with_revision(Arc::clone(&set), &backup_path)
.await
.expect("current backup");
let mut candidate = cache;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
let cycle_floor = AtomicU64::new(candidate.info.next_cycle);
let requested_epoch = if case == "changed_epoch" {
epoch.checked_add(1).expect("fixture epoch can advance")
} else {
epoch
};
let (tx, mut rx) = mpsc::channel(1);
let (result, events) = capture_set_cache_logs(persist_and_publish_cache_snapshot(
Arc::clone(&set),
&tx,
candidate,
Some(&initial),
&cycle_floor,
requested_epoch,
))
.await;
assert!(result.is_none(), "{case} must still refuse publication");
assert!(
matches!(rx.try_recv(), Err(mpsc::error::TryRecvError::Empty)),
"{case} forwards no snapshot"
);
let expected_state = if case == "changed_epoch" {
"save_failed"
} else {
"scan_baseline_revision_changed"
};
assert_set_cache_context(set_cache_event(&events, expected_state), Some((0, 0)));
assert_eq!(
read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("protected primary"),
before_primary,
"{case} does not overwrite primary"
);
assert_eq!(
read_config_with_revision(set, &backup_path).await.expect("protected backup"),
before_backup,
"{case} does not overwrite backup"
);
}
}
#[tokio::test]
#[serial]
async fn set_cache_cas_preserves_updates_after_backup_catchup_validation() {
use crate::data_usage_define::read_config_with_revision;
for changed_copy in ["primary", "backup"] {
let (_temp_dir, store, cache, epoch) = set_cache_log_fixture().await;
let set = Arc::clone(&store.pools[0].disk_set[0]);
let primary_path = path_join_buf(&[BUCKET_META_PREFIX, DATA_USAGE_CACHE_NAME]);
let backup_path = format!("{primary_path}.bkp");
let (primary_body, _) = read_config_with_revision(Arc::clone(&set), &primary_path)
.await
.expect("read initial primary");
let mut lagging = cache.clone();
lagging.info.scan_execution_digest = Some(DataUsageScanPlanDigest([8; 32]));
save_config(Arc::clone(&set), &backup_path, lagging.marshal_msg().expect("lagging backup bytes"))
.await
.expect("seed backup lag");
let mut loaded = DataUsageCache::default();
let initial = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("read initial baseline");
save_config(Arc::clone(&set), &backup_path, primary_body.expect("initial primary exists"))
.await
.expect("finish backup catchup");
let current = loaded
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("read reconciled baseline");
assert!(current.permits_unchanged_baseline_since(&initial));
let changed_path = if changed_copy == "primary" {
&primary_path
} else {
&backup_path
};
let mut concurrent = cache.clone();
concurrent.info.scan_execution_digest = Some(DataUsageScanPlanDigest([9; 32]));
let concurrent_body = concurrent.marshal_msg().expect("encode concurrent update");
save_config(Arc::clone(&set), changed_path, concurrent_body.clone())
.await
.expect("copy changes after the baseline validation");
let mut candidate = cache;
candidate.info.next_cycle += 1;
candidate.info.scan_execution_digest = Some(DataUsageScanPlanDigest([3; 32]));
let result = candidate
.save_with_revisions_for_epoch(Arc::clone(&set), DATA_USAGE_CACHE_NAME, current.revisions(), epoch)
.await;
if changed_copy == "primary" {
assert!(result.is_err(), "primary CAS refuses the intervening update");
} else {
result.expect("primary remains durable when the best-effort backup CAS refuses an update");
}
let (protected, _) = read_config_with_revision(Arc::clone(&set), changed_path)
.await
.expect("read protected concurrent copy");
assert_eq!(
protected,
Some(concurrent_body),
"{changed_copy} CAS must not overwrite the concurrent update"
);
let (primary, _) = read_config_with_revision(set, &primary_path)
.await
.expect("read resulting primary");
let primary = DataUsageCache::unmarshal(&primary.expect("primary exists")).expect("primary remains decodable");
assert_eq!(
primary.info.scan_execution_digest,
Some(DataUsageScanPlanDigest([if changed_copy == "primary" { 9 } else { 3 }; 32])),
);
}
}
#[tokio::test]
#[serial]
async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers() {
@@ -2652,7 +2400,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
.expect("seed legacy set cache");
let mut persisted = DataUsageCache::default();
let initial = persisted
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the shared starting revision");
let mut fresh = legacy.clone();
@@ -2678,7 +2426,7 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
assert_eq!(published.find("photos").expect("published bucket").size, 20);
assert_eq!(published.info.scan_execution_digest, fresh.info.scan_execution_digest);
let current = persisted
.load_with_baseline_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.load_with_revisions(Arc::clone(&set), DATA_USAGE_CACHE_NAME)
.await
.expect("capture the current revision for the unidentified execution");
@@ -2969,130 +2717,6 @@ fn scoped_scan_requires_a_complete_authoritative_baseline_with_exact_set_provena
);
}
#[test]
fn scoped_scan_baseline_rejections_preserve_the_existing_first_failed_check() {
let source = DataUsageCacheSource::new(1, 2);
let expected_sources = HashSet::from([source]);
let digest = DataUsageScanPlanDigest([9; 32]);
let baseline = complete_usage_baseline(source, digest, 7, 11);
let snapshot: DataUsageInfo = serde_json::from_slice(&baseline).expect("decode complete baseline");
let proof = ScannerCacheBaselineProof {
authoritative_data: Some(&baseline),
observed_candidate_data: None,
expected_sources: &expected_sources,
leader_epoch: 11,
want_cycle: 8,
scan_plan_digest: digest,
};
type RejectionCase = (ScannerBaselineRejection, fn(&mut DataUsageInfo));
let cases: [RejectionCase; 13] = [
(ScannerBaselineRejection::Incomplete, |s| s.usage_snapshot_complete = false),
(ScannerBaselineRejection::Partial, |s| s.usage_snapshot_partial = true),
(ScannerBaselineRejection::Marks, |s| s.usage_snapshot_converged = None),
(ScannerBaselineRejection::Epoch, |s| s.scanner_epoch = Some(10)),
(ScannerBaselineRejection::SourceCount, |s| s.usage_snapshot_set_states.clear()),
(ScannerBaselineRejection::PlanMissing, |s| {
s.usage_snapshot_set_states[0].scan_plan_digest = None
}),
(ScannerBaselineRejection::PlanMismatch, |s| {
s.usage_snapshot_set_states[0].scan_plan_digest = Some([8; 32])
}),
(ScannerBaselineRejection::SourceMembership, |s| {
s.usage_snapshot_set_states[0].set_index = 3
}),
(ScannerBaselineRejection::SetIncomplete, |s| {
s.usage_snapshot_set_states[0].complete = false
}),
(ScannerBaselineRejection::SetTombstone, |s| {
s.usage_snapshot_set_states[0].tombstone = true
}),
(ScannerBaselineRejection::SetEpoch, |s| {
s.usage_snapshot_set_states[0].scanner_epoch = None
}),
(ScannerBaselineRejection::SetCycle, |s| {
s.usage_snapshot_set_states[0].scanner_cycle = Some(9)
}),
(ScannerBaselineRejection::SetCycle, |s| {
s.usage_snapshot_set_states[0].scanner_cycle = None
}),
];
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&snapshot, proof, None), Ok(digest));
for (expected, mutate) in cases {
let mut invalid = snapshot.clone();
mutate(&mut invalid);
assert_eq!(validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None), Err(expected));
assert_eq!(complete_scanner_cache_snapshot_plan_digest(&invalid, proof, None), None);
}
let mut invalid = snapshot.clone();
invalid.usage_snapshot_partial = true;
invalid.scanner_epoch = Some(10);
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&invalid, proof, None),
Err(ScannerBaselineRejection::Partial)
);
let two_sources = HashSet::from([source, DataUsageCacheSource::new(1, 3)]);
let mut two_sets = snapshot.clone();
two_sets
.usage_snapshot_set_states
.push(two_sets.usage_snapshot_set_states[0].clone());
let two_set_proof = ScannerCacheBaselineProof {
expected_sources: &two_sources,
..proof
};
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
Err(ScannerBaselineRejection::SourceDuplicate)
);
two_sets.usage_snapshot_set_states[1].set_index = 3;
two_sets.usage_snapshot_set_states[1].scan_plan_digest = Some([8; 32]);
assert_eq!(
validate_scanner_cache_snapshot_plan_digest(&two_sets, two_set_proof, None),
Err(ScannerBaselineRejection::SetPlan)
);
let undecodable = Bytes::from_static(b"not-json");
for (data, expected) in [
(None, ScannerBaselineRejection::Missing),
(Some(&undecodable), ScannerBaselineRejection::Decode),
] {
assert_eq!(
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
authoritative_data: data,
..proof
}),
Err(ScannerBaselineValidationFailure {
authoritative: expected,
observed: None
})
);
}
let mut wrong_epoch = snapshot;
wrong_epoch.scanner_epoch = Some(10);
let wrong_epoch = Bytes::from(serde_json::to_vec(&wrong_epoch).expect("encode stale epoch baseline"));
let stale_proof = ScannerCacheBaselineProof {
authoritative_data: Some(&wrong_epoch),
..proof
};
assert_eq!(
validate_scanner_cache_baseline_plan_digest(stale_proof),
Err(ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection::Epoch,
observed: Some(ScannerBaselineRejection::Missing),
})
);
assert_eq!(
validate_scanner_cache_baseline_plan_digest(ScannerCacheBaselineProof {
observed_candidate_data: Some(&undecodable),
..stale_proof
}),
Err(ScannerBaselineValidationFailure {
authoritative: ScannerBaselineRejection::Epoch,
observed: Some(ScannerBaselineRejection::Decode)
})
);
}
#[test]
fn scoped_scan_accepts_only_a_complete_observation_tied_to_the_authoritative_baseline() {
let source = DataUsageCacheSource::new(1, 2);
@@ -3526,12 +3150,6 @@ fn distributed_segment_reuse_activation_keeps_remote_dirty_buckets_at_bucket_sco
result.scope.prefix_scope_for("photos").is_some(),
"durable local producer proof may activate local segment reuse after distributed ACK capability evidence"
);
assert!(
result
.segment_reuse_activation_preflight
.expect("actual scope proof is retained for observation")
.scanner_segment_reuse_activated
);
assert!(
result.scope.prefix_scope_for("archive").is_none(),
"remote dirty usage has only bucket-granularity evidence and must not be narrowed to a local prefix"
@@ -3950,10 +3568,6 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
.checked_flatten("stable")
.expect("selected bucket placeholder should exist");
assert_eq!((stable.size, stable.objects), (0, 0));
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
prepared.record_cold_bucket_reuse(&observation);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
assert!(prepared.cache.find("stable/prefix").is_none());
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
assert!(prepared.cache.find("deleted").is_none());
@@ -3965,124 +3579,6 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
}
#[test]
fn cold_bucket_reuse_observation_rejects_unbound_or_failed_preparation() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20)], baseline_digest);
old_cache.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "dirty"]);
let incarnations = old_cache.info.scan_bucket_incarnations.clone();
let buckets = vec![
bucket_info_with_created_time("stable"),
bucket_info_with_created_time("dirty"),
];
let scope = ScannerBucketScanScope {
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
selected_bucket_prefixes: None,
baseline_scan_plan_digest: Some(baseline_digest),
};
let generation = ScannerSetCacheGeneration {
want_cycle: 8,
leader_epoch: 11,
tier_registry_generation: 13,
source: DataUsageCacheSource::new(1, 2),
scan_plan_digest: DataUsageScanPlanDigest([2; 32]),
};
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
let prepared = prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation, Some(&incarnations))
.expect("bound complete cache permits a cold bucket copy");
prepared.record_cold_bucket_reuse(&observation);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 1);
for case in ["incomplete", "duplicate", "nil_incarnation"] {
let mut invalid_cache = old_cache.clone();
let mut invalid_buckets = buckets.clone();
let mut invalid_incarnations = incarnations.clone();
match case {
"incomplete" => invalid_cache.info.snapshot_complete = false,
"duplicate" => invalid_buckets.push(buckets[0].clone()),
"nil_incarnation" => {
invalid_incarnations.insert("stable".to_string(), Uuid::nil());
}
_ => unreachable!("fixed test cases"),
}
let prepared = prepare_scoped_set_scan(
&invalid_cache,
&invalid_buckets,
&invalid_buckets,
&scope,
generation,
Some(&invalid_incarnations),
);
assert!(prepared.is_none(), "{case} must not return a reusable prepared cache");
if let Some(prepared) = prepared {
prepared.record_cold_bucket_reuse(&observation);
}
assert_eq!(
observation.snapshot().bucket_reuse["scoped_cold_bucket"].count,
1,
"{case} does not count completed reuse"
);
}
}
#[tokio::test]
async fn current_root_reuse_counts_only_successful_collector_delivery() {
use rustfs_scanner_metrics::scan_observation::ScannerScanObservation;
let observation = ScannerScanObservation::default();
let cache = DataUsageCache::default();
let pending = AtomicBool::new(false);
let root = || DataUsageEntryInfo {
name: "bucket".to_string(),
parent: DATA_USAGE_ROOT.to_string(),
entry: DataUsageEntry {
size: 10,
objects: 2,
..Default::default()
},
bucket_incarnation: Some(Uuid::from_u128(7)),
tier_registry_generation: None,
value_origin: None,
};
let (sender, mut receiver) = mpsc::channel(1);
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
.await
.expect("current root is delivered to the collector");
let delivered = receiver.recv().await.expect("collector received the root");
assert_eq!((delivered.entry.size, delivered.entry.objects), (10, 2));
assert_eq!(observation.snapshot().bucket_reuse["current_root"].count, 1);
drop(receiver);
assert!(
super::io_cache::send_current_cache_root_entry(&sender, root(), &cache, &pending, &observation)
.await
.is_err()
);
assert_eq!(
observation.snapshot().bucket_reuse["current_root"].count,
1,
"closed collector is not completed reuse"
);
let (sender, _receiver) = mpsc::channel(1);
sender.send(root()).await.expect("fill bounded collector channel");
{
let mut send = Box::pin(super::io_cache::send_current_cache_root_entry(
&sender,
root(),
&cache,
&pending,
&observation,
));
assert!(futures::poll!(send.as_mut()).is_pending(), "full collector keeps reuse unfinished");
}
assert_eq!(
observation.snapshot().bucket_reuse["current_root"].count,
1,
"dropping unfinished delivery is not reuse"
);
assert_eq!(observation.snapshot().bucket_reuse["scoped_cold_bucket"].count, 0);
}
#[test]
fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
@@ -4124,12 +3620,6 @@ fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
)
.expect("matching bucket incarnations should authorize cold bucket reuse");
let observation = rustfs_scanner_metrics::scan_observation::ScannerScanObservation::default();
prepared.record_cold_bucket_reuse(&observation);
let observed = observation.snapshot();
assert_eq!(observed.bucket_reuse["scoped_cold_bucket"].count, 1);
assert_eq!(observed.bucket_reuse["current_root"].count, 0);
assert!(observed.cache_reuse.values().all(|reuse| reuse.count == 0));
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
let stable = prepared
.cache
@@ -5135,36 +4625,12 @@ async fn get_size_treats_missing_metadata_as_skip_file() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
let err = disk
.get_size(item.clone())
.get_size(item)
.await
.expect_err("missing metadata should be skipped instead of reported as a scanner failure");
assert!(matches!(err, StorageError::Io(ref io) if io.to_string() == SCANNER_SKIP_FILE_ERROR));
let missing = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(missing.attempts, before.attempts + 1);
assert_eq!(missing.not_found, before.not_found + 1);
assert_eq!(missing.bytes, before.bytes);
#[cfg(unix)]
{
tokio::fs::create_dir_all(&object_dir)
.await
.expect("recreate object directory");
std::os::unix::fs::symlink(STORAGE_FORMAT_FILE, &metadata_path).expect("inject a metadata symlink loop");
let err = disk
.get_size(item)
.await
.expect_err("symlink loop must remain a read failure");
assert!(is_scanner_metadata_transient_error(&err), "read error: {err:?}");
let failed = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(failed.attempts, missing.attempts + 1);
assert_eq!(failed.errors, missing.errors + 1);
assert_eq!(failed.succeeded, before.succeeded);
assert_eq!(failed.bytes, before.bytes);
}
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -5215,7 +4681,6 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
for contents in [b"".as_slice(), b"not-valid-filemeta".as_slice()] {
tokio::fs::write(&metadata_path, contents)
.await
@@ -5227,16 +4692,6 @@ async fn get_size_marks_corrupt_metadata_for_heal() {
assert!(is_scanner_metadata_corrupt_error(&err));
}
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(reads.attempts, before.attempts + 2);
assert_eq!(
reads.succeeded,
before.succeeded + 2,
"storage reads succeed before decoding rejects corruption"
);
assert_eq!(reads.errors, before.errors);
assert_eq!(reads.bytes, before.bytes + 18);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -5309,17 +4764,11 @@ async fn get_size_counts_delete_markers_separately_from_versions() {
debug: false,
};
let before = global_metrics().scan_observation().snapshot().metadata_reads;
let expected_bytes = tokio::fs::metadata(&metadata_path).await.expect("stat stored metadata").len();
let summary = disk.get_size(item).await.expect("scanner should read versioned metadata");
assert_eq!(summary.versions, 2);
assert_eq!(summary.delete_markers, 1);
assert_eq!(summary.total_size, 30);
let reads = global_metrics().scan_observation().snapshot().metadata_reads;
assert_eq!(reads.attempts, before.attempts + 1);
assert_eq!(reads.succeeded, before.succeeded + 1);
assert_eq!(reads.bytes, before.bytes + expected_bytes);
let _ = tokio::fs::remove_dir_all(&temp_dir).await;
}
@@ -183,11 +183,9 @@ async fn dirty_entry_removed_during_listing_still_forces_a_full_scan() {
);
assert_eq!(second.buckets_usage["hot-bucket"].objects_count, 2);
assert_eq!(second.buckets_usage["cold-bucket"].objects_count, 2);
let report = global_metrics().report().await;
let freshness = report.usage_freshness;
let freshness = global_metrics().report().await.usage_freshness;
assert_eq!(freshness.scope_fallback_total, fallbacks_before + 1);
assert_eq!(freshness.last_scope_fallback_reason, "dirty_snapshot_incomplete");
assert_eq!(report.scan_observation.scope_fallbacks["dirty_snapshot_incomplete"], 1);
clear_dirty_usage_buckets_for_tests();
}
@@ -201,53 +199,11 @@ async fn required_full_scan_is_not_reported_as_a_scope_fallback() {
wait_for_namespace_commit_tails(store.as_ref()).await;
super::io_cycle::set_bucket_listing_window_hook_for_tests(|| record_dirty_usage_bucket("created-after-listing"));
let before = global_metrics().scan_observation().snapshot();
let (result, _, scope) = run_cycle(&store, 2, Some(baseline), true).await;
assert!(scope.is_default());
assert_eq!(result.pending_dirty_usage_buckets(), 1);
assert_eq!(result.scope_fallback(), None, "scheduled maintenance walks are not degradations");
assert_eq!(result.completed_scope_fallback(), None);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(after.scope_fallbacks, before.scope_fallbacks);
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
clear_dirty_usage_buckets_for_tests();
}
#[tokio::test]
#[serial]
async fn baseline_unavailable_cycle_records_real_scope_and_baseline_rejections_once() {
let (_temp_dir, store) = setup_local_scanner_store(1, 1).await;
let _baseline = converged_baseline(&store).await;
put_fixture_object(&store, "hot-bucket", "second").await;
record_dirty_usage_bucket("hot-bucket");
wait_for_namespace_commit_tails(store.as_ref()).await;
let before = global_metrics().scan_observation().snapshot();
let (result, snapshot, scope) = run_cycle(&store, 2, Some(Bytes::from_static(b"not-json")), false).await;
assert!(scope.is_default());
assert_eq!(result.status, ScannerCycleStatus::Complete);
assert_eq!(result.scope_fallback(), Some(ScannerScopeFallbackReason::BaselineUnavailable));
assert_eq!(snapshot.buckets_usage["hot-bucket"].objects_count, 2);
let after = global_metrics().scan_observation().snapshot();
assert_eq!(
after.scope_fallbacks["baseline_unavailable"],
before.scope_fallbacks["baseline_unavailable"] + 1
);
assert_eq!(
after.authoritative_baseline_rejections["decode"],
before.authoritative_baseline_rejections["decode"] + 1
);
assert_eq!(
after.observed_baseline_rejections, before.observed_baseline_rejections,
"an undecodable authority never admits an observation"
);
assert_eq!(after.prefix_scope_resolution.evaluated, before.prefix_scope_resolution.evaluated + 1);
assert_eq!(after.prefix_scope_resolution.activated, before.prefix_scope_resolution.activated);
assert_eq!(
after.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"],
before.prefix_scope_resolution.blockers["missing_cold_zero_walk_oracle"] + 1
);
assert_eq!(after.prefix_cycle_completion.evaluated, before.prefix_cycle_completion.evaluated + 1);
clear_dirty_usage_buckets_for_tests();
}
@@ -1,12 +1,12 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! A bucket whose walk stops early can publish its last certified value as
//! pending. This fixture tests usage-scan status, not the outer scanner cycle
//! outcome: budget exhaustion may leave complete validated usage while the
//! coordinator records `Partial`. A continuation under a newer plan exercises
//! `mixed_coverage` (rustfs/rustfs#8494). A bucket that never certified keeps
//! the usage scan incomplete.
//! A bucket whose walk stops early without exhausting the cycle budget (here
//! a sweep resumed under a newer plan, the `mixed_coverage` case of
//! rustfs/rustfs#8494) no longer makes the whole cycle `Incomplete`. The set
//! publishes the bucket's last certified value, the cycle completes with the
//! bucket pending, and the next cycle walks it again. A bucket that never
//! certified keeps the cycle incomplete.
use super::*;
@@ -38,7 +38,8 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
..Default::default()
},
);
// Real checkpoint commits need the runtime drain grace after the count budget stops the walk.
// These cycles persist real bucket checkpoints and source caches. Keep
// CI storage latency inside the production drain window after a count stop.
budget.use_production_drain_grace_for_tests();
let (updates, mut receiver) = mpsc::channel(1);
let result = tokio::time::timeout(
@@ -48,7 +49,7 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
ScannerCycleRequest {
owner_ctx: ctx.clone(),
ctx: budget.token(),
budget: Arc::clone(&budget),
budget,
updates,
want_cycle: cycle,
leader_epoch: LEADER_EPOCH,
@@ -68,13 +69,6 @@ async fn run_cycle(store: &Arc<ECStore>, cycle: u64, options: CycleOptions) -> (
.await
.expect("cycle should finish within the test deadline")
.expect("cycle should succeed");
if options.max_objects.is_some() {
assert_eq!(
budget.reason(),
Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects),
"a count-limited fixture must stop for its object budget"
);
}
let snapshot = receiver.recv().await;
assert!(receiver.recv().await.is_none(), "a cycle publishes at most one terminal snapshot");
(result, snapshot)
@@ -131,8 +125,9 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
assert!(result.clear_verified_usage().is_empty());
assert!(dirty_usage_buckets_for_tests().is_empty());
// Cycle 3: the budget stops the walk and saves a checkpoint. Usage can
// carry the certified value as pending; the outer coordinator records Partial.
// Cycle 3: the object budget stops the hot bucket's walk and leaves a
// checkpoint under this cycle's plan. Its certified value is carried, so
// budget exhaustion leaves the bucket pending without blocking the cycle.
for object in ["p4/third", "p5/third"] {
put_fixture_object(&store, "hot-bucket", object).await;
}
@@ -149,38 +144,12 @@ async fn partial_bucket_with_a_certified_value_completes_the_cycle_as_pending()
},
)
.await;
assert_eq!(
result.status,
ScannerCycleStatus::Complete,
"certified carried usage remains complete after a budget stop"
);
let third = third.expect("validated carried usage publishes after the budget stop");
let third = third.expect("a carried partial cycle publishes");
assert_eq!(result.status, ScannerCycleStatus::Complete);
assert_eq!(result.pending_dirty_usage_buckets(), 1);
assert!(!third.usage_snapshot_observational);
assert_eq!(third.usage_snapshot_converged, Some(false));
assert_eq!(third.usage_snapshot_pending_buckets, vec!["hot-bucket".to_string()]);
assert_eq!(third.usage_snapshot_pending_bucket_count, 1);
assert!(third.is_complete_bucket_usage_snapshot());
let carried_hot = &third.buckets_usage["hot-bucket"];
let certified_hot = &second.buckets_usage["hot-bucket"];
assert_eq!(carried_hot.usage_value_state, Some(rustfs_data_usage::BucketUsageValueState::Carried));
assert!(certified_hot.usage_certification.is_some(), "cycle 2 certified the prior hot value");
assert_eq!(
carried_hot.usage_certification, certified_hot.usage_certification,
"carrying must not mint a new certificate"
);
assert_eq!(carried_hot.objects_count, 4);
assert_eq!(carried_hot.size, certified_hot.size);
assert_eq!(third.buckets_usage["cold-bucket"].objects_count, 1);
assert_eq!(third.buckets_usage["idle-bucket"].objects_count, 1);
assert_eq!(third.objects_total_count, 6);
// Only hot was dirty in this scoped scan; it received no fresh certification or ACK.
assert!(result.clear_verified_usage().is_empty());
assert_eq!(
dirty_usage_buckets_for_tests().keys().cloned().collect::<Vec<_>>(),
vec!["hot-bucket".to_string()],
"the budget-stopped hot bucket retains its dirty responsibility"
);
assert_eq!(third.buckets_usage["hot-bucket"].objects_count, 4);
let set = Arc::clone(&store.pools[0].disk_set[0]);
let mut interrupted = DataUsageCache::default();
interrupted
@@ -9,7 +9,7 @@ This page is a pointer, not a route table. The machine-checked matrix in `route_
| Prefix | Behavior | Rule |
|---|---|---|
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_route` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
| `/rustfs/admin` | Canonical admin prefix used by `make_admin_router` (`rustfs/src/admin/mod.rs`) | The only registered admin prefix |
| `/minio/admin` | Compatibility alias accepted by `S3Router::is_match`; `canonicalize_admin_path` rewrites it to `/rustfs/admin` immediately before route lookup (`rustfs/src/admin/router.rs`) | Never register routes twice; preserve canonicalization |
| `/iceberg/v1` | Table catalog prefix registered by `register_table_catalog_route` (`rustfs/src/admin/handlers/table_catalog/routes.rs`) and accepted by `is_admin_path` | Stays outside `/rustfs/admin`; table actions are authorized per handler |
| `/health`, `/health/ready` | Public health endpoints, registered only when `ENV_HEALTH_ENDPOINT_ENABLE` allows | Preserve the unauthenticated bypass |
@@ -1,7 +1,7 @@
# MinIO ↔ RustFS Router Compatibility (Exceptions Only)
**Use this when:** a client or `mc` call that works against MinIO fails against RustFS and you need to know whether the endpoint is missing, stubbed, or deliberately different.
**Source of truth:** S3 plane: the `s3s::S3` trait impl in `rustfs/src/storage/ecfs.rs`. Admin plane: `make_admin_route` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
**Source of truth:** S3 plane: the registered operation families in `rustfs/src/server/gateway_handlers/mod.rs`. Admin plane: `make_admin_router` in `rustfs/src/admin/mod.rs`, the registration inventory `rustfs/src/admin/route_registration_test.rs`, and the route/action guardrail [admin-route-action-snapshot.md](admin-route-action-snapshot.md).
This document lists only exceptions. Anything not listed here is implemented with MinIO-equivalent behavior. For the s3tests-level claim see [s3-compatibility-matrix.md](s3-compatibility-matrix.md).
@@ -18,7 +18,7 @@ Admin paths are relative to the canonical `/rustfs/admin` prefix; `/minio/admin`
## S3 Data Plane
All `s3s::S3` trait methods are implemented in `rustfs/src/storage/ecfs.rs` except the following.
Run `python3 scripts/s3-tests/api_coverage.py` for the registered gateway operations and the missing operations in the pinned S3 reference. The following table records compatibility exceptions.
| S3 operation | Status | Detail |
|---|---|---|
+1 -1
View File
@@ -97,7 +97,7 @@ Standard S3 areas that must not be described as complete:
| Area | Status | Evidence |
|---|---|---|
| Bucket access logging | Handlers exist (`get_bucket_logging`, `put_bucket_logging` in `rustfs/src/storage/ecfs.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
| Bucket access logging | Handlers exist (`GetBucketLogging` and `PutBucketLogging` in `rustfs/src/server/gateway_handlers/bucket_core.rs`); the `test_*bucket_logging*` s3tests cases are still listed as unimplemented | `unimplemented_tests.txt` |
| POST Object form upload checksum handling | Not yet passing | `unimplemented_tests.txt` |
| Bucket ownership controls | No handler | `unimplemented_tests.txt` |
| Multipart upload listing and part lookup compatibility edge cases | Not part of default gate; ListParts returns HTTP 503 `SlowDownRead` when no directory was listed and any participant is unavailable, instead of treating one missing directory as an empty upload | `excluded_tests.txt`; `collect_list_parts_results_rejects_unavailable_empty_upload_dirs` |
+1 -1
View File
@@ -1,7 +1,7 @@
# Audit Entries for Authorization Denials
**Use this when:** you consume the audit stream and need to know which authorization denials appear in it, how many, and how to tune the limit.
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/access.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
**Source of truth:** `rustfs/src/storage/denial_audit.rs` (`DenialAuditor`, `DenialRateLimiter`), `rustfs/src/storage/authorization/decision.rs` (`DenialContext`), `crates/config/src/constants/env.rs` (`ENV_AUDIT_DENIAL_RATE`, `ENV_AUDIT_DENIAL_ANONYMOUS_RATE`).
With audit enabled (`RUSTFS_AUDIT_ENABLE=true`), a request refused by the authorization layer produces an audit entry in addition to the `s3_authorization_denied` log event. The S3 response is unchanged: a bare `Access Denied`.
+17
View File
@@ -0,0 +1,17 @@
# S3 stack upgrade and rollback
The gateway release serves S3 with rustfs/gateway. Remove `RUSTFS_S3_STACK` from service units, container environments and Helm values before upgrading. Even an empty setting prevents startup. Keep the previous release binary or image digest with the deployment configuration.
## Roll back to the previous release
1. Check the target release against the [storage format version floors](rolling-restart.md#version-floors-that-break-mixed-version-fleets). If the cluster has written data or metadata that the target cannot read, stop here. The S3 stack switch itself requires no data migration.
2. Stop one node, restore the previous RustFS binary or image and start it with the same volumes and configuration. Follow the [rolling restart procedure](rolling-restart.md#rolling-restart-procedure). Wait for the node to report ready and for its peers to report its drives online before replacing another node.
3. Through the restored node, read an object written after the upgrade and compare its contents or checksum. Read a bucket configuration used by the application. If an upload was in progress, list its parts and compare their numbers, sizes and ETags with the client's record. Pause the rollout if any check fails.
## Client-visible error change
| Conformance case | What clients see |
| --- | --- |
| `c-sig-0598` (R-053) | For an access key the S3 parser cannot read, both versions return HTTP 403. The gateway returns the XML error code `AccessDenied`. The legacy binary returns `InvalidAccessKeyId`. Clients that classify these errors by code need to handle both during a rollout. |
The case is recorded in the [pinned gateway conformance rulings](https://github.com/rustfs/gateway/blob/60dcd2a55713dbcc9dbd6e165922e1a0df4473d7/conformance/rulings/rustfs-profile.toml#L298-L304).
@@ -898,42 +898,6 @@ done
The `aggregated.scanner` payload preserves the same scanner progress, checkpoint, pacing, source work, maintenance control, lifecycle expiry, and lifecycle transition fields used by the local scanner status, but only for the responding node; `by_host.*.scanner` keeps that node's host view. Compare the per-node artifacts externally to find old active paths, partial checkpoints, pacing pressure, source-level control pressure, or downstream queue admission problems across the deployment.
## Reading Cached Bucket Statistics
Admin usage reads cache the selected scanner baseline for the existing 30-second TTL, then verify every returned bucket against uncached storage-owned incarnation metadata. Identity reads use at most four concurrent operations and a shared two-second deadline after releasing snapshot/refresh locks. Each bucket read linearizes independently against metadata transactions; the response is not an atomic snapshot of the entire namespace. Delayed peer invalidation cannot authorize a predecessor's statistics for a recreated bucket.
Missing, failed, timed-out, nil, mismatched, or legacy absent identities leave the bucket value absent (unknown), rather than returning the predecessor's value or a confirmed zero. Legacy statistics become visible after a scanner snapshot supplies the current identity. Account-info and usage display apply the existing single-process memory overlay before the final identity filter; distributed local counters are not promoted. When a projection omits values or changes an overlay, totals are partial and `usage_snapshot_converged=false`, and per-tier/replication aggregates are unavailable. Verified cold bucket values remain visible. Such response-only projections cannot be persisted, used for quota admission, or acknowledge dirty work.
## Reading Scanner Work Observations
`GET /rustfs/admin/v3/scanner/status` exposes `metrics.scan_observation` for the responding node. Capture timestamped snapshots on every node, including the scanner leader, and compare their deltas over the same workload window. These process-lifetime counters reset on restart; a missing field on an older node means unobserved. They are loaded independently and may briefly disagree while operations finish. This telemetry does not change scan cadence, cached usage reads, quota authority, dirty acknowledgement, or maintenance requirements.
| Field | Meaning |
|---|---|
| `metadata_reads.attempts` | Disk metadata read futures started after scanner node admission. Invalid paths and operations cancelled before starting the read do not count. |
| `metadata_reads.succeeded`, `not_found`, `errors`, `cancelled` | Storage outcomes, including object/version disappearance and dropped read futures. Successful storage reads can still contain corrupt metadata; decoding and lifecycle failures do not become read errors. Cancellation observes the future, not proof that physical I/O stopped. |
| `metadata_reads.bytes` | Total returned buffer lengths for successful reads, including inline data and corrupt metadata. Failed reads contribute no known bytes; this is not physical disk traffic. |
| `metadata_reads.elapsed_nanos` | Sum of read-future elapsed time, including disk-permit or RPC waits, failures, and cancellation. Folder pacing, scanner node admission, decoding, and lifecycle work are outside this interval. Concurrent reads overlap, so the sum is not cycle wall time. |
| `cache_reuse.clean_prefix` | Clean root subtrees copied under an admitted prefix scope by the non-erasure scanner. |
| `cache_reuse.covered_namespace` | Namespace subtrees copied from validated checkpoint coverage. |
| `cache_reuse.sampled_namespace` | Cached namespace subtrees copied when the existing periodic sampling selected other work. |
| `cache_reuse.classified_namespace` | EC candidate namespace subtrees copied only after the current listing finished classifying them. Health/classification work still runs. |
| `cache_reuse.*.count`, `elapsed_nanos` | Completed subtree reuse decisions and time spent copying/attaching their cache entries. Counts are not objects skipped or a cache hit ratio; times exclude classification/listing and are not total cache maintenance cost. |
| `bucket_reuse.scoped_cold_bucket` | Whole cold buckets copied and attached to a successfully prepared scoped set cache. Count is copied buckets, not plans selected; later turn publication can still fail. Time covers copying/attachment only. |
| `bucket_reuse.current_root` | Existing current bucket roots successfully sent to the local collector after watch, namespace lease and publication epoch checks. Time covers channel send only; failure or cancellation contributes no completed reuse. Collector acceptance and durable publication remain separate. |
| `reuse_decisions.*.not_evaluated`, `blocked`, `eligible` | First conditions reached by production clean-prefix, checkpoint-coverage and namespace-sampling gates. Policy exclusions (`missing_scope`, `no_resume_sweep`, `deep_scan`, `forward_sweep`, `active_replication`) precede cache proof checks. `blocked` reports the first failed work/cache condition; later checks are not repeated for diagnostics. `eligible` does not certify a copy: an EC candidate can still require classification. |
| `erasure_clean_prefix_walks` | Clean root children that still require walking because EC scans perform independent object-health work. |
| `prefix_scope_resolution.evaluated`, `activated`, `blockers` | Proof evaluations retained by the final scope resolver, admitted proof gates, and every failed prerequisite. An admitted proof does not guarantee a reusable child exists; actual subtree copies appear in `cache_reuse`. Startup/required full scans, early scope rejection, and discarded peer/ACK resolutions do not count here. |
| `prefix_cycle_completion.evaluated`, `activated`, `blockers` | End-of-cycle proof evaluations for subsequent reuse. An activated completion proof does not mean this cycle reused prefixes or that its later publication/persistence succeeded. One evaluation can increment several blocker counters. |
| `scope_fallbacks` | Per-reason breakdown of the existing `usage_freshness.scope_fallback_total`; required full scans are excluded. |
| `authoritative_baseline_rejections`, `observed_baseline_rejections` | First failed validation checks for `baseline_unavailable` fallback cycles, preserving the authoritative rejection when the observed fallback also fails. Missing/undecodable authority never reaches observed validation; `authoritative_identity` means it could not anchor an observed fallback. |
`freshness` describes the responding node's completed-cycle age, not whether every bucket has incorporated new writes. Read `metrics.usage_freshness` for node-local live pending work, guardrail vetoes, durable success and publication state; read `metrics.usage_publication` for the confirmed snapshot's pending count and publication class/outcome, and `cycle_schedule` for effective retry intervals and consecutive superseded cycles. Missing or unknown publication data does not establish zero pending work. A follower or a single node's empty dirty map cannot establish cluster-wide convergence.
Whole cold bucket copies, current-root reuse and the four subtree categories have different prerequisites and are not interchangeable hit rates. EC clean prefixes retain health walks. Deep or required full work must run, but a completed cache matching that exact work digest may satisfy a retry of the same cycle. A normal workload need not exercise current-root reuse or every subtree category; use controlled tests for positive emission evidence and capture real gate reasons when a VM window has zero copies.
The existing `life_time_ops.read_metadata` and `last_minute.actions.read_metadata` now include finished and cancelled read attempts; the latter reports count, accumulated whole seconds, and returned bytes in its existing window. Use `metadata_reads.elapsed_nanos` for subsecond read costs. Use read-count and byte deltas to assess work growth, and elapsed-time deltas to assess read-future cost. Compare those with reused-subtree deltas, scope fallback reasons, cycle wall time, pacing waits, and source work before attributing freshness delays to metadata I/O or proposing a scan optimization.
## Reading Lifecycle Transition Status
`metrics.lifecycle_transition`:
+1 -1
View File
@@ -13,7 +13,7 @@ Every fixed RustFS GitHub Security Advisory maps to at least one named regressio
| [GHSA-r5qv-rc46-hv8q](https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q) | Internode RPC authentication must fail closed | rustfs/rustfs#4402 | `ghsa_r5qv_resolve_shared_secret_rejects_default_fallback`, `ghsa_r5qv_verify_rpc_signature_fails_closed_on_missing_or_invalid_auth` (`crates/ecstore/src/cluster/rpc/http_auth.rs`) | unit |
| [GHSA-m77q-r63m-pj89](https://github.com/rustfs/rustfs/security/advisories/GHSA-m77q-r63m-pj89) | STS JWTs signed with the shared root secret (intentionally unfixed) | n/a; tests pin the by-design behaviour and must flip red -> green when m77q is fixed | `test_ghsa_m77q_sts_session_token_signed_with_root_secret`, `test_created_sts_credentials_authorize_with_session_token_claims` (`crates/iam/src/sys.rs`); `token_signing_key` doc (`crates/iam/src/root_credentials.rs`) | unit |
| [GHSA-5354-r3w2-34m8](https://github.com/rustfs/rustfs/security/advisories/GHSA-5354-r3w2-34m8) | Service-account parent must stay within caller scope; a non-owner with `CreateServiceAccountAdminAction` could parent a service account to root | rustfs/rustfs#5141 | `ghsa_5354_non_owner_service_account_parent_confined_to_scope`, `ghsa_5354_scope_guard_preserves_non_root_self_scope`, and the `add_service_account_parent_within_scope` invariant they pin (`rustfs/src/admin/handlers/service_account.rs`) | unit |
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` and the `versioned_read_action` helper it pins (`rustfs/src/storage/access.rs`) | unit |
| [GHSA-3ppv-fx5m-m749](https://github.com/rustfs/rustfs/security/advisories/GHSA-3ppv-fx5m-m749) | Versioned reads (`get_object`, CopyObject source, UploadPartCopy source) authorize against `s3:GetObjectVersion`, not `s3:GetObject` | rustfs/rustfs#5142 | `ghsa_3ppv_versioned_read_selects_get_object_version_action` in `rustfs/src/storage/authorization/tests/actions.rs` covers `versioned_read_action` in `rustfs/src/storage/authorization_actions.rs` | unit |
| [GHSA-v9cp-qfw9-9pfp](https://github.com/rustfs/rustfs/security/advisories/GHSA-v9cp-qfw9-9pfp) | `ForAllValues:`/`ForAnyValue:` negated string operators applied negation to the aggregate instead of the per-value predicate | fixed, GHSA private-fork merge | `ghsa_v9cp_for_all_values_not_equals_partial_overlap`, `ghsa_v9cp_for_any_value_not_equals_partial_overlap` and the absent-key/positive-quantifier cases beside them (`crates/policy/tests/quantified_negation.rs`); the value set must partially overlap the policy set, since contained or disjoint sets cannot tell the quantifiers apart | crate test |
| [GHSA-6r96-hmgc-726c](https://github.com/rustfs/rustfs/security/advisories/GHSA-6r96-hmgc-726c) | Request headers must not populate server-derived IAM condition keys (`userid`, `groups`, `jwt:`/`ldap:` claims) | fixed, GHSA private-fork merge | `ghsa_6r96_identity_condition_keys_ignore_spoofed_headers`, `ghsa_6r96_claim_condition_keys_ignore_spoofed_headers`, and `test_request_headers_still_reach_conditions`, which keeps the reserved set from growing too broad (`rustfs/src/auth.rs`) | unit |
| [GHSA-x298-9x87-fvjq](https://github.com/rustfs/rustfs/security/advisories/GHSA-x298-9x87-fvjq) | Anonymous ListObjectVersions -> `s3:ListBucket` fallback must reach the same public-access gates as a direct grant | fixed, GHSA private-fork merge | `ghsa_x298_anonymous_list_object_versions_denied_when_restrict_public_buckets_enabled` (`crates/e2e_test/src/anonymous_access_test.rs`); asserts 200 before the public-access block is applied so it proves the gate, not a broken fallback | e2e (`e2e-smoke`) |
+39 -31
View File
@@ -195,27 +195,14 @@ impl Operation for ChangeOwnPasswordHandler {
let request: ChangePasswordRequest = serde_json::from_slice(&body)
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
let iam_store =
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
// Reached only if the identity was deleted between authentication
// and this lookup.
emit_audit(
&audit,
AccountAuditRecord::failure(
AccountAuditOperation::ChangeOwnPassword,
&caller.access_key,
caller.identity_type,
AccountAuditFailure::Internal,
),
);
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
// An STS session signs with its own secret. A direct IAM request must
// have been signed with the same password the caller is replacing.
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
&caller.credentials.secret_key
} else {
&request.current_secret_key
};
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
{
if !constant_time_eq(expected_signing_secret, &signing_secret) {
warn!(
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_ACCOUNT,
@@ -256,8 +243,10 @@ impl Operation for ChangeOwnPasswordHandler {
}
let access_key = caller.access_key.clone();
let new_secret_key = request.new_secret_key.clone();
let current_secret_key = request.current_secret_key;
let new_secret_key = request.new_secret_key;
let identity_type = caller.identity_type;
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
let session_access_key = caller.session_access_key.clone();
let audit_for_task = audit.clone();
@@ -268,16 +257,35 @@ impl Operation for ChangeOwnPasswordHandler {
let iam_store =
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
let (updated_at, status, revoked) = iam_store
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
.await
.map_err(|error| {
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
} else {
iam_error_to_s3_error(error)
}
})?;
let rotation = match session_credentials.as_ref() {
Some(session) => {
iam_store
.change_own_secret_key_from_session(&access_key, &current_secret_key, &new_secret_key, session)
.await
}
None => {
iam_store
.change_own_secret_key(&access_key, &current_secret_key, &new_secret_key)
.await
}
};
let (updated_at, status, revoked) = rotation.map_err(|error| {
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
emit_audit(
&audit_for_task,
AccountAuditRecord::failure(
AccountAuditOperation::ChangeOwnPassword,
&access_key,
identity_type,
AccountAuditFailure::InvalidCurrentSecret,
)
.with_session_access_key(session_access_key.as_deref()),
);
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
} else {
iam_error_to_s3_error(error)
}
})?;
emit_audit(
&audit_for_task,
+3 -2
View File
@@ -20,7 +20,7 @@ use crate::admin::service::caller_identity::oidc_profile_fields;
use crate::admin::storage_api::bucket::versioning_sys::BucketVersioningSys;
use crate::admin::storage_api::contract::admin::StorageAdminApi;
use crate::admin::storage_api::contract::bucket::{BucketOperations, BucketOptions};
use crate::admin::storage_api::data_usage::load_admin_data_usage_with_memory_overlay;
use crate::admin::storage_api::data_usage::{apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached};
use crate::admin::storage_api::metadata_sys;
use crate::auth::get_condition_values;
use crate::server::ADMIN_PREFIX;
@@ -278,7 +278,8 @@ impl Operation for AccountInfoHandler {
// process-local absolute counters to cluster-wide bucket totals.
// This path never triggers a live full-version listing
// (rustfs/backlog#1306); freshness is owned by the scanner.
let data_usage_info = map_data_usage_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
let mut data_usage_info = map_data_usage_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
apply_bucket_usage_memory_overlay(&mut data_usage_info).await;
account_info.usage_snapshot_converged = data_usage_info.usage_snapshot_converged;
for bucket in buckets.iter() {
+12 -38
View File
@@ -1868,6 +1868,8 @@ mod tests {
peer_topology_complete, query_peer_heal_status, query_peer_replacement_recovery_status, read_cluster_heal_status,
reject_heal_admission, validate_heal_request_mode, validate_heal_target, validate_mrf_legacy_responsibility_action,
};
use crate::admin::operation::test_support::TestRequest;
use crate::admin::router::dispatch;
use crate::storage::rpc::node_service::heal::{
NodeHealProgress, NodeHealStatusSnapshot, NodeReplacementRecoveryStatusSnapshot, encode_node_replacement_recovery_status,
};
@@ -2153,33 +2155,19 @@ mod tests {
#[tokio::test]
async fn test_nested_heal_routes_still_require_authentication() {
use s3s::route::S3Route;
let mut router = super::S3Router::new(false);
super::register_heal_route(&mut router).expect("register production Heal routes");
for prefix in ["/rustfs/admin", "/minio/admin"] {
for object in ["dir/object.bin", "dir%2Fobject.bin", "literal%252Fslash"] {
let mut req = s3s::S3Request {
input: s3s::Body::empty(),
method: http::Method::POST,
uri: format!("{prefix}/v3/heal/test-bucket/{object}").parse().expect("Heal URI"),
headers: http::HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let request = TestRequest::new(http::Method::POST, format!("{prefix}/v3/heal/test-bucket/{object}")).build();
let err = router
.check_access(&mut req)
.await
.admit(request.method(), request.uri(), request.headers(), false)
.expect_err("router must require a signature");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
let err = router
.call(req)
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
.await
.expect_err("handler must independently require authentication");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert!(err.to_string().contains("authentication required"));
}
}
@@ -2187,34 +2175,20 @@ mod tests {
#[tokio::test]
async fn replacement_recovery_diagnostics_require_authentication() {
use s3s::route::S3Route;
let mut router = super::S3Router::new(false);
super::register_heal_route(&mut router).expect("register production Heal routes");
for prefix in ["/rustfs/admin", "/minio/admin"] {
let path = format!("{prefix}/v4/heal/replacement-recovery");
assert!(router.contains_compatible_route(http::Method::GET, &path));
let mut req = s3s::S3Request {
input: s3s::Body::empty(),
method: http::Method::GET,
uri: path.parse().expect("replacement diagnostics URI"),
headers: http::HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let request = TestRequest::new(http::Method::GET, path).build();
let err = router
.check_access(&mut req)
.await
.admit(request.method(), request.uri(), request.headers(), false)
.expect_err("diagnostics router must require a signature");
assert_eq!(err.code(), &S3ErrorCode::AccessDenied.into());
let err = router
.call(req)
assert_eq!(err.code(), &S3ErrorCode::AccessDenied);
let err = dispatch(&router, request.into_envelope(), rustfs_s3_types::Body::empty())
.await
.expect_err("diagnostics handler must authenticate before inspecting local records");
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest.into());
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
assert!(err.to_string().contains("authentication required"));
}
}
+19 -51
View File
@@ -90,79 +90,47 @@ impl Operation for HealthCheckHandler {
#[cfg(test)]
mod tests {
use super::*;
use crate::admin::operation::test_support::TestRequest;
use crate::admin::router::dispatch;
use http_body_util::BodyExt as _;
use s3s::route::S3Route as _;
use serial_test::serial;
use temp_env::with_var;
fn health_request(method: Method) -> s3s::S3Request<s3s::Body> {
s3s::S3Request {
input: s3s::Body::empty(),
method,
uri: HEALTH_PREFIX.parse().expect("health URI"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
async fn call_real_health_handler(method: Method) -> s3s::S3Response<s3s::Body> {
async fn call_real_health_handler(method: Method) -> (StatusCode, HeaderMap, Body) {
let mut router = S3Router::new(false);
router
.insert(method.clone(), HEALTH_PREFIX, AdminOperation(&HealthCheckHandler {}))
.expect("health route should register");
router
.call(health_request(method))
let request = TestRequest::new(method, HEALTH_PREFIX).build();
dispatch(&router, request.into_envelope(), Body::empty())
.await
.expect("health handler should return a response")
.into_parts()
}
#[tokio::test]
async fn real_health_handler_get_and_head_preserve_wire_branches() {
let get = call_real_health_handler(Method::GET).await;
assert_eq!(get.status, Some(StatusCode::OK));
assert_eq!(
get.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
Some("application/json")
);
let payload: serde_json::Value =
serde_json::from_slice(&get.output.collect().await.expect("GET body should read").to_bytes())
.expect("GET health response should be JSON");
let (status, headers, body) = call_real_health_handler(Method::GET).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
let payload: serde_json::Value = serde_json::from_slice(&body.collect().await.expect("GET body should read").to_bytes())
.expect("GET health response should be JSON");
assert_eq!(payload["status"], "ok");
let head = call_real_health_handler(Method::HEAD).await;
assert_eq!(head.status, Some(StatusCode::OK));
assert_eq!(
head.headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()),
Some("application/json")
);
assert!(
head.output
.collect()
.await
.expect("HEAD body should read")
.to_bytes()
.is_empty()
);
let (status, headers, body) = call_real_health_handler(Method::HEAD).await;
assert_eq!(status, StatusCode::OK);
assert_eq!(headers.get(CONTENT_TYPE).and_then(|value| value.to_str().ok()), Some("application/json"));
assert!(body.collect().await.expect("HEAD body should read").to_bytes().is_empty());
}
#[tokio::test]
async fn real_health_handler_rejects_unsupported_methods_with_allow_header() {
let response = call_real_health_handler(Method::POST).await;
let (status, headers, body) = call_real_health_handler(Method::POST).await;
assert_eq!(response.status, Some(StatusCode::METHOD_NOT_ALLOWED));
assert_eq!(status, StatusCode::METHOD_NOT_ALLOWED);
assert_eq!(headers.get(http::header::ALLOW).and_then(|value| value.to_str().ok()), Some("GET, HEAD"));
assert_eq!(
response
.headers
.get(http::header::ALLOW)
.and_then(|value| value.to_str().ok()),
Some("GET, HEAD")
);
assert_eq!(
response.output.collect().await.expect("error body should read").to_bytes(),
body.collect().await.expect("error body should read").to_bytes(),
bytes::Bytes::from_static(b"Method Not Allowed")
);
}
+9 -9
View File
@@ -332,7 +332,7 @@ fn is_lock_timeout(err: &crate::storage_api::error::StorageError) -> bool {
fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError {
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) {
return ApiError {
code: S3ErrorCode::InternalError.into(),
code: S3ErrorCode::InternalError,
message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(),
source: Some(Box::new(err)),
};
@@ -341,7 +341,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// movement gate has a queued writer: the same transient contention.
if is_lock_timeout(&err) || matches!(err, crate::storage_api::error::StorageError::SlowDown) {
return ApiError {
code: S3ErrorCode::ServiceUnavailable.into(),
code: S3ErrorCode::ServiceUnavailable,
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -350,7 +350,7 @@ fn decommission_start_api_error(err: crate::storage_api::error::StorageError) ->
// should say so, not just "Invalid Request".
if matches!(err, crate::storage_api::error::StorageError::DecommissionAlreadyRunning) {
return ApiError {
code: S3ErrorCode::InvalidRequest.into(),
code: S3ErrorCode::InvalidRequest,
message: err.to_string(),
source: Some(Box::new(err)),
};
@@ -1261,14 +1261,14 @@ mod pools_handler_tests {
#[test]
fn test_contextualize_admin_pool_api_error_preserves_code_and_adds_pool_context() {
let err = crate::error::ApiError {
code: rustfs_s3_types::S3ErrorCode::InvalidRequest.into(),
code: rustfs_s3_types::S3ErrorCode::InvalidRequest,
message: "decommission already running".to_string(),
source: None,
};
let err = contextualize_admin_pool_api_error(err, "start decommission", "pools [1, 3]");
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest.into());
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InvalidRequest);
assert_eq!(
err.message,
"admin start decommission failed for pools [1, 3]: decommission already running"
@@ -1281,12 +1281,12 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(err.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
assert!(err.source.is_some());
let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed"));
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError.into());
assert_eq!(unrelated.code, rustfs_s3_types::S3ErrorCode::InternalError);
assert_eq!(unrelated.message, "We encountered an internal error, please try again.");
}
@@ -1299,7 +1299,7 @@ mod pools_handler_tests {
let err = decommission_start_api_error(err);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
assert!(err.message.contains("data movement gate (decommission activation)"), "{}", err.message);
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(
@@ -1313,7 +1313,7 @@ mod pools_handler_tests {
fn test_decommission_start_api_error_reports_a_refused_preflight_heal_as_retryable() {
let err = decommission_start_api_error(crate::storage_api::error::StorageError::SlowDown);
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable.into());
assert_eq!(err.code, S3ErrorCode::ServiceUnavailable);
let err = rustfs_s3_types::S3Error::from(contextualize_admin_pool_api_error(err, "start decommission", "pools [0]"));
assert_eq!(err.status_code(), Some(http::StatusCode::SERVICE_UNAVAILABLE));
}
+5 -4
View File
@@ -926,10 +926,11 @@ impl Operation for RebalanceStatus {
"admin rebalance state"
);
// Preserve the idle 404 contract without triggering the custom-route error logger.
let (parts, body) = s3s::S3Error::with_message(s3s::S3ErrorCode::NoSuchResource, REBALANCE_NOT_STARTED_MESSAGE)
.to_http_response()?
.into_parts();
return Ok(AdminResponse::with_headers(parts.status, body.into(), parts.headers));
let body = format!(
r#"<?xml version="1.0" encoding="UTF-8"?><Error><Code>NoSuchResource</Code><Message>{REBALANCE_NOT_STARTED_MESSAGE}</Message></Error>"#
);
let headers = HeaderMap::from_iter([(CONTENT_TYPE, HeaderValue::from_static("application/xml"))]);
return Ok(AdminResponse::with_headers(StatusCode::NOT_FOUND, Body::from(body), headers));
}
error!(
+49 -25
View File
@@ -48,7 +48,6 @@ use rustfs_policy::{
};
use rustfs_s3_types::{Body, S3Error, S3ErrorCode, S3Result, s3_error};
use rustfs_utils::MaskedAccessKey;
use s3s::dto::{AssumeRoleOutput, Credentials, Timestamp};
use serde::Deserialize;
use serde_json::Value;
use serde_urlencoded::from_bytes;
@@ -337,33 +336,35 @@ async fn handle_assume_role(
warn!("site replication STS hook failed, err: {err}");
}
let resp = AssumeRoleOutput {
credentials: Some(Credentials {
access_key_id: new_cred.access_key,
expiration: Timestamp::from(
new_cred
.expiration
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600))),
),
secret_access_key: new_cred.secret_key,
session_token: new_cred.session_token,
}),
..Default::default()
};
// getAssumeRoleCredentials
let output = serialize_assume_role_output(&resp)
.map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, format!("serialize assume role output failed: {e}")))?;
let expiration = new_cred
.expiration
.unwrap_or_else(|| OffsetDateTime::now_utc().saturating_add(Duration::seconds(3600)));
let output = assume_role_response_xml(&new_cred, expiration)?;
Ok(AdminResponse::new(StatusCode::OK, Body::from(output)))
}
/// The STS answer is an s3s response body, so the s3s encoder writes it; it
/// is not a bucket configuration and never goes through the bucket-config codec.
fn serialize_assume_role_output(output: &AssumeRoleOutput) -> s3s::xml::SerResult<Vec<u8>> {
let mut buf = Vec::with_capacity(256);
s3s::xml::Serialize::serialize(output, &mut s3s::xml::Serializer::new(&mut buf))?;
Ok(buf)
fn assume_role_response_xml(credentials: &rustfs_credentials::Credentials, expiration: OffsetDateTime) -> S3Result<String> {
// The STS codec writes local clock fields with millisecond precision and a four-digit year.
if !(0..=9999).contains(&expiration.year()) {
return Err(s3_error!(InternalError, "cannot serialize STS expiration year"));
}
let expiration = expiration
.format(time::macros::format_description!(
"[year]-[month]-[day]T[hour]:[minute]:[second].[subsecond digits:3]Z"
))
.map_err(|_| s3_error!(InternalError, "cannot serialize STS expiration"))?;
Ok(format!(
concat!(
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
"<AccessKeyId>{}</AccessKeyId><Expiration>{}</Expiration>",
"<SecretAccessKey>{}</SecretAccessKey><SessionToken>{}</SessionToken>",
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
),
xml_escape(&credentials.access_key),
expiration,
xml_escape(&credentials.secret_key),
xml_escape(&credentials.session_token),
))
}
/// Session claim marking a session that presented a second factor.
@@ -579,6 +580,29 @@ mod tests {
}
}
#[test]
fn assume_role_xml_matches_the_pinned_sts_response() {
let credentials = rustfs_credentials::Credentials {
access_key: "access<&".to_owned(),
secret_key: "synthetic>'".to_owned(),
session_token: "quoted\"token".to_owned(),
..Default::default()
};
let expiration = time::macros::datetime!(2024-02-29 11:22:33.123456789 +05:30);
let answer = assume_role_response_xml(&credentials, expiration).expect("valid timestamp");
// Captured from the pinned 5761ddfe STS writer, including field order and escaping.
assert_eq!(
answer,
concat!(
"<AssumeRoleResponse xmlns=\"https://sts.amazonaws.com/doc/2011-06-15/\"><AssumeRoleResult><Credentials>",
"<AccessKeyId>access&lt;&amp;</AccessKeyId><Expiration>2024-02-29T11:22:33.123Z</Expiration>",
"<SecretAccessKey>synthetic&gt;&apos;</SecretAccessKey><SessionToken>quoted&quot;token</SessionToken>",
"</Credentials></AssumeRoleResult></AssumeRoleResponse>"
)
);
assert!(assume_role_response_xml(&credentials, time::macros::datetime!(-1-01-01 00:00 UTC)).is_err());
}
#[test]
fn test_xml_escape() {
assert_eq!(xml_escape("hello"), "hello");
+1 -17
View File
@@ -45,24 +45,8 @@ use handlers::{
user,
};
use router::{AdminOperation, S3Router};
use s3s::route::S3Route;
/// Create admin router
///
/// # Arguments
/// * `console_enabled` - Whether the console is enabled
///
/// # Returns
/// An instance of S3Route for admin operations
pub fn make_admin_route(
console_enabled: bool,
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
) -> std::io::Result<impl S3Route> {
make_admin_router(console_enabled, server_ctx)
}
/// The admin router with every route registered, bound to `server_ctx`: what the legacy stack
/// mounts as its custom route and the gateway stack's control-plane host dispatches through.
/// Builds the control-plane router and binds it to this server.
pub(crate) fn make_admin_router(
console_enabled: bool,
server_ctx: std::sync::Arc<crate::admin::runtime_sources::ServerContextSlot>,
+232 -331
View File
File diff suppressed because it is too large Load Diff
@@ -77,7 +77,7 @@ fn uri_for(template: &str, bucket: &str) -> String {
async fn send(router: &S3Router<AdminOperation>, mut req: S3Request<Body>) -> S3Result<S3Response<Body>> {
router.check_access(&mut req).await?;
router.call(req).await
router.call(req).await.map_err(Into::into)
}
fn error_code(result: S3Result<S3Response<Body>>, context: &str) -> S3ErrorCode {
@@ -222,7 +222,7 @@ async fn extension_routes_refuse_anonymous_callers_before_dispatch() {
.check_access(&mut req)
.await
.expect_err("an anonymous extension request is refused");
assert_eq!(error.code(), &S3ErrorCode::AccessDenied, "{method} {uri}");
assert_eq!(S3ErrorCode::from(error.code().clone()), S3ErrorCode::AccessDenied, "{method} {uri}");
}
}
+23 -1
View File
@@ -87,6 +87,8 @@ pub(crate) enum CredentialMutationDenial {
ServiceAccount,
/// A derived credential whose parent could not be determined.
UnresolvedParent,
/// A storage delegation has no account-management authority.
TableStorageSession,
}
impl CredentialMutationDenial {
@@ -97,6 +99,7 @@ impl CredentialMutationDenial {
}
Self::FederatedIdentity => "federated identities are managed by their identity provider",
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
}
}
@@ -187,7 +190,7 @@ impl CallerIdentity {
// root", and an unresolved parent must never fall through to a
// permissive answer.
//
// These three denials apply to both capabilities: a machine credential
// These denials apply to both capabilities: a machine credential
// must not take over its parent, a federated identity is owned by its
// IdP, and an unresolvable parent fails closed.
let shared_denial = if unresolved_parent {
@@ -196,6 +199,11 @@ impl CallerIdentity {
Some(CredentialMutationDenial::ServiceAccount)
} else if federated {
Some(CredentialMutationDenial::FederatedIdentity)
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
.iter()
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
{
Some(CredentialMutationDenial::TableStorageSession)
} else {
None
};
@@ -501,6 +509,20 @@ mod tests {
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
}
#[test]
fn table_storage_sessions_cannot_manage_parent_credentials() {
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
let mut credentials = sts_session("TABLEKEY", "table-owner");
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
let caller = CallerIdentity::from_credentials(credentials, false);
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
assert!(!caller.mutability().password);
assert!(caller.ensure_credential_mutation_allowed().is_err());
assert!(caller.ensure_mfa_management_allowed().is_err());
}
}
#[test]
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
// Fail closed: an unresolvable parent must not be treated as a
+4 -6
View File
@@ -945,6 +945,10 @@ pub(crate) static ERR_TIER_RESERVED_NAME: AdminErrorRef = AdminErrorRef(|| &ecst
pub(crate) mod data_usage {
use std::sync::Arc;
pub(crate) async fn apply_bucket_usage_memory_overlay(data_usage_info: &mut rustfs_data_usage::DataUsageInfo) {
crate::storage::storage_api::ecstore_data_usage::apply_bucket_usage_memory_overlay(data_usage_info).await;
}
pub(crate) async fn load_data_usage_from_backend(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
@@ -956,12 +960,6 @@ pub(crate) mod data_usage {
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_from_backend_cached(store).await
}
pub(crate) async fn load_admin_data_usage_with_memory_overlay(
store: Arc<crate::storage::storage_api::ECStore>,
) -> Result<rustfs_data_usage::DataUsageInfo, crate::storage::storage_api::StorageError> {
crate::storage::storage_api::ecstore_data_usage::load_admin_data_usage_with_memory_overlay(store).await
}
}
pub(crate) mod access {
+23 -66
View File
@@ -21,7 +21,9 @@ use super::storage_api::admin_usecase::capacity::{
};
use super::storage_api::admin_usecase::contract::StorageAdminApi;
use super::storage_api::admin_usecase::contract::bucket::{BucketOperations as _, BucketOptions};
use super::storage_api::admin_usecase::data_usage::load_admin_data_usage_with_memory_overlay;
use super::storage_api::admin_usecase::data_usage::{
apply_bucket_usage_memory_overlay, load_admin_data_usage_from_backend_cached,
};
use super::storage_api::admin_usecase::{ECStore, EndpointServerPools};
use crate::app::runtime_sources::{
AppContext, current_app_context, current_endpoints_handle, current_object_store_handle_for_context,
@@ -35,7 +37,7 @@ use crate::server::collect_dependency_readiness_report as collect_runtime_depend
use crate::shared_types::DependencyReadiness;
use rustfs_data_usage::DataUsageInfo;
use rustfs_madmin::{InfoMessage, StorageInfo};
use s3s::S3ErrorCode;
use rustfs_s3_types::S3ErrorCode;
use std::collections::HashSet;
use std::sync::Arc;
use tracing::{debug, error, info, warn};
@@ -270,7 +272,7 @@ impl DefaultAdminUsecase {
info.buckets_usage.retain(|bucket, _| current_buckets.contains(bucket));
info.bucket_sizes.retain(|bucket, _| current_buckets.contains(bucket));
if info.buckets_usage.is_empty() && !current_buckets.is_empty() && !info.usage_snapshot_admin_projection {
if info.buckets_usage.is_empty() && !current_buckets.is_empty() {
*info = DataUsageInfo::default();
return;
}
@@ -285,14 +287,19 @@ impl DefaultAdminUsecase {
}
}
/// Keep already identity-bound display values without accepting a display
/// projection as a persistable or quota-authoritative snapshot.
fn present_identity_bound_data_usage_snapshot(
mut info: DataUsageInfo,
buckets: impl IntoIterator<Item = String>,
) -> DataUsageInfo {
let snapshot_measured =
info.usage_snapshot_admin_projection || info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot();
/// Overlay single-process counts on a persisted snapshot and narrow it
/// for display.
///
/// Whether the snapshot measured anything is decided before the overlay.
/// The overlay clears `usage_snapshot_complete` when it holds counts it
/// cannot reconcile with the snapshot, such as writes into a bucket that
/// has no verified baseline. That makes the totals unconverged, not the
/// scanned buckets unusable; treating it as structural incompleteness
/// blanked usage for every bucket for as long as one such bucket kept
/// receiving writes, because each write restarts its scanner fence.
async fn present_data_usage_snapshot(mut info: DataUsageInfo, buckets: impl IntoIterator<Item = String>) -> DataUsageInfo {
let snapshot_measured = info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot();
apply_bucket_usage_memory_overlay(&mut info).await;
Self::narrow_data_usage_snapshot_to_measured_buckets(&mut info, snapshot_measured, buckets);
info
}
@@ -338,7 +345,7 @@ impl DefaultAdminUsecase {
/// This path never triggers a live full-version listing
/// (rustfs/backlog#1306); freshness is owned by the scanner.
pub(crate) async fn query_data_usage_info_with_store(store: Arc<ECStore>) -> AdminUsecaseResult<DataUsageInfo> {
let info = Self::map_data_usage_load_result(load_admin_data_usage_with_memory_overlay(store.clone()).await)?;
let info = Self::map_data_usage_load_result(load_admin_data_usage_from_backend_cached(store.clone()).await)?;
let buckets = store
.list_bucket(&BucketOptions {
cached: true,
@@ -347,7 +354,7 @@ impl DefaultAdminUsecase {
})
.await
.map_err(|_| Self::app_error(S3ErrorCode::InternalError, "list_bucket failed"))?;
let mut info = Self::present_identity_bound_data_usage_snapshot(info, buckets.into_iter().map(|bucket| bucket.name));
let mut info = Self::present_data_usage_snapshot(info, buckets.into_iter().map(|bucket| bucket.name)).await;
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
@@ -972,52 +979,6 @@ mod tests {
assert_eq!(info.buckets_count, 0);
}
#[test]
fn identity_bound_partial_display_keeps_valid_buckets_and_unknown_state() {
use std::collections::HashMap;
let mut projected = DataUsageInfo {
last_update: Some(std::time::SystemTime::UNIX_EPOCH),
usage_snapshot_admin_projection: true,
usage_snapshot_partial: true,
usage_snapshot_converged: Some(false),
buckets_count: 1,
objects_total_count: 7,
objects_total_size: 70,
buckets_usage: HashMap::from([(
"cold".to_string(),
rustfs_data_usage::BucketUsageInfo {
size: 70,
objects_count: 7,
..Default::default()
},
)]),
bucket_sizes: HashMap::from([("cold".to_string(), 70)]),
..Default::default()
};
projected = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
projected,
["cold".to_string(), "recreated".to_string()],
);
assert_eq!(projected.buckets_usage["cold"].objects_count, 7);
assert!(!projected.buckets_usage.contains_key("recreated"));
assert_eq!(projected.objects_total_count, 7);
assert_eq!(projected.usage_snapshot_converged, Some(false));
assert!(!projected.is_complete_bucket_usage_snapshot());
assert!(!projected.is_valid_partial_snapshot());
projected.buckets_usage.clear();
projected.bucket_sizes.clear();
projected.calculate_totals();
projected.buckets_count = 0;
let unknown = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(projected, ["recreated".to_string()]);
assert!(unknown.buckets_usage.is_empty());
assert!(unknown.usage_snapshot_admin_projection);
assert_eq!(unknown.usage_snapshot_converged, Some(false));
assert!(!unknown.is_complete_bucket_usage_snapshot());
assert_eq!(unknown.last_update, Some(std::time::SystemTime::UNIX_EPOCH));
}
/// A write into a bucket without a verified baseline leaves process-local
/// counts the overlay cannot reconcile, so it clears
/// `usage_snapshot_complete`. That must flag the response non-converged,
@@ -1048,15 +1009,11 @@ mod tests {
crate::app::storage_api::test::data_usage::record_bucket_object_write_memory(unverified_bucket, None, 1_024).await;
// Exercise the production overlay, followed by the same identity-bound
// presentation function used by the admin query.
crate::app::storage_api::test::data_usage::apply_bucket_usage_memory_overlay(&mut persisted).await;
persisted.usage_snapshot_admin_projection = true;
persisted.usage_snapshot_converged = Some(false);
let info = DefaultAdminUsecase::present_identity_bound_data_usage_snapshot(
let info = DefaultAdminUsecase::present_data_usage_snapshot(
persisted,
[measured_bucket.to_string(), unverified_bucket.to_string()],
);
)
.await;
assert_eq!(
info.buckets_usage
+76
View File
@@ -0,0 +1,76 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Operations without a notification event must emit one use-case audit entry.
use crate::guard_sources::SourceSet;
use rustfs_s3_ops::S3Operation;
fn is_read_only(op: S3Operation) -> bool {
let name = op.as_str().trim_start_matches("s3:");
["Get", "Head", "List", "Select"]
.iter()
.any(|prefix| name.starts_with(prefix))
}
fn audit_sites(use_cases: &SourceSet, op: S3Operation) -> usize {
let helper = format!("OperationHelper::audit_only_from_view(&envelope.audit_view(), S3Operation::{op:?}");
use_cases.count_ignoring_whitespace(&format!("audit_only_operation(envelope, S3Operation::{op:?},"))
+ use_cases.count_ignoring_whitespace(&format!("{helper})"))
+ use_cases.count_ignoring_whitespace(&format!("{helper},)"))
}
#[test]
fn mutating_operations_without_a_notification_event_are_audited_once() {
let mut expected: Vec<S3Operation> = S3Operation::ALL
.into_iter()
.filter(|op| !is_read_only(*op) && op.to_event_name().is_none())
.collect();
// AbortMultipartUpload has an internal event but no handler-built helper.
expected.push(S3Operation::AbortMultipartUpload);
assert_eq!(expected.len(), 26, "an operation was added or removed: {expected:?}");
let use_cases = SourceSet::production("src/app/mod.rs", "src/app");
for op in expected {
assert_eq!(audit_sites(&use_cases, op), 1, "expected one audit site for {op:?}");
}
}
const OWNED_HELPER: &str =
"let helper = OperationHelper::audit_only_from_view(\n &envelope.audit_view(),\n S3Operation::UploadPart,\n);";
const OWNED_WRAPPER: &str =
"RequestEnvelope::audit_only_operation(\n envelope,\n S3Operation::UploadPart,\n StatusCode::OK,\n run,\n)";
fn count_sites(source: &str) -> usize {
let root = tempfile::tempdir().unwrap();
let app = root.path().join("src/app");
std::fs::create_dir_all(&app).unwrap();
std::fs::write(app.join("mod.rs"), source).unwrap();
let sources = SourceSet::production_under(root.path(), "src/app/mod.rs", "src/app");
audit_sites(&sources, S3Operation::UploadPart)
}
#[test]
fn the_guard_accepts_exactly_one_audit_place() {
assert_eq!(count_sites(OWNED_HELPER), 1);
assert_eq!(count_sites(OWNED_WRAPPER), 1);
}
#[test]
fn the_guard_refuses_an_operation_audited_nowhere_or_twice() {
assert_eq!(count_sites(""), 0);
assert_eq!(count_sites(&format!("{OWNED_HELPER}\n{OWNED_WRAPPER}")), 2);
assert_eq!(count_sites(&format!("#[cfg(test)]\nmod tests {{\n{OWNED_HELPER}\n}}")), 0);
assert_eq!(count_sites(&OWNED_WRAPPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
assert_eq!(count_sites(&OWNED_HELPER.replace("::UploadPart,", "::UploadPartCopy,")), 0);
}
+31 -20
View File
@@ -472,7 +472,7 @@ mod tests {
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ecstore};
use crate::app::storage_api::bucket_usecase::s3::{
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3, S3Request, S3Response,
GetObjectInput, HeadObjectInput, ListObjectsInput, ListObjectsV2Input, ListObjectsV2Output, S3Request, S3Response,
XmlSerialize, XmlSerializer,
};
use crate::app::storage_api::test::StoragePutObjReader;
@@ -1185,10 +1185,11 @@ mod tests {
service: None,
trailing_headers: None,
};
let get_env = crate::app::RequestEnvelope::from_s3s(&get);
let authorized_generation =
super::super::storage_api::bucket_usecase::access::load_bucket_generation_from_store(
&store,
&get,
&get_env,
&input.bucket,
)
.await
@@ -1214,12 +1215,16 @@ mod tests {
assert_ne!(replacement, old.incarnation_id());
sys.remove(&input.bucket);
let mut without_source = get.clone();
let mut without_source_env = crate::app::RequestEnvelope::from_s3s(&without_source);
super::super::storage_api::bucket_usecase::access::prepare_odm_read_generation(
&store,
&mut without_source,
&mut without_source_env,
&input.bucket,
)
.await;
if let Some(guard) = without_source_env.odm_read_generation().cloned() {
without_source.extensions.insert(guard);
}
for state_incarnation in [old.incarnation_id(), replacement] {
sys.apply_for_incarnation(&input.bucket, state_incarnation, Some(old.config()))
.await;
@@ -1315,28 +1320,34 @@ mod tests {
};
for (index, expected_key) in ["a&local", "z-local"].into_iter().enumerate() {
let request_marker = input.marker.clone().unwrap_or_default();
let response = tokio::time::timeout(
let list_req = S3Request {
input: input.clone(),
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
};
let envelope = crate::app::RequestEnvelope::from_s3s(&list_req);
let gateway_input =
rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::input_from_s3s(
list_req.input,
)
.expect("convert ListObjectsInput");
let gateway_output = tokio::time::timeout(
Duration::from_secs(10),
S3::list_objects(
&crate::storage::ecfs::FS::new(),
S3Request {
input: input.clone(),
method: http::Method::GET,
uri: http::Uri::from_static("/"),
headers: HeaderMap::new(),
extensions: http::Extensions::new(),
credentials: None,
region: None,
service: None,
trailing_headers: None,
},
),
DefaultBucketUsecase::from_global().execute_list_objects(envelope, gateway_input),
)
.await
.expect("v1 pagination must finish")
.expect("list-through must not change v1 listing");
assert!(!response.headers.contains_key("x-rustfs-on-demand-migration-list"));
let output = response.output;
let output = rustfs_gateway_types::compat::s3s_5761ddfe::generated::ops::list_objects::output_to_s3s(
gateway_output,
)
.expect("convert ListObjectsOutput");
let contents = output.contents.as_ref().expect("local page contents");
assert_eq!(contents.len(), 1);
assert_eq!(contents[0].key.as_deref(), Some(expected_key));
+6 -4
View File
@@ -98,14 +98,13 @@ use rustfs_policy::policy::{
{BucketPolicy, BucketPolicyArgs, Effect},
};
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::Region;
use rustfs_s3_types::{Region, S3Error, S3ErrorCode, S3Result};
use rustfs_targets::{
EventName,
arn::{ARN, TargetID, TargetIDError},
};
use rustfs_utils::http::get_header;
use rustfs_utils::obj::extract_user_defined_metadata;
use s3s::{S3Error, S3ErrorCode, S3Result};
use std::{
borrow::Cow,
collections::{HashMap, HashSet},
@@ -907,7 +906,9 @@ mod tests {
.expect_err("full queue rejects immediately");
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
let response = error.to_http_response().expect("serialize admission error");
let response = s3s::S3Error::from(error)
.to_http_response()
.expect("serialize admission error");
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
use http_body_util::BodyExt as _;
@@ -1000,7 +1001,8 @@ mod tests {
},
);
assert!(req_info_ref(&req).expect("req info should be present").is_owner);
let envelope = RequestEnvelope::from_s3s(&req);
assert!(envelope.req_info().expect("req info should be present").is_owner);
}
#[tokio::test]
+11
View File
@@ -67,6 +67,17 @@ impl DefaultBucketUsecase {
mod tests {
use super::*;
#[tokio::test]
async fn execute_put_bucket_acl_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultBucketUsecase::without_context();
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
let error = usecase
.execute_put_bucket_acl(envelope, "test-bucket", false)
.await
.unwrap_err();
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_get_bucket_acl_returns_internal_error_when_store_uninitialized() {
let input = gateway_dto::GetBucketAclInput {
+72 -1
View File
@@ -116,7 +116,7 @@ impl DefaultBucketUsecase {
async fn list_objects_page(&self, envelope: &mut RequestEnvelope, request: &PageRequest) -> S3Result<Page> {
let PageRequest { bucket, params, .. } = request;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
let store = get_validated_store(bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED).as_deref() == Some("true");
let table_list_access = envelope.table_list_access().cloned();
@@ -375,6 +375,77 @@ mod tests {
use crate::app::bucket_usecase::test_support::build_request;
use rustfs_gateway_types::BucketName;
#[test]
#[serial_test::serial]
fn list_apis_report_storage_read_quorum_as_unavailable() {
use crate::app::bucket_usecase::test_support::owned_envelope;
use crate::app::gating_test_env::{run_large_stack_test, shared_gating_ambient, shared_gating_ecstore};
run_large_stack_test("list-quorum-surface", || async {
let usecase = DefaultBucketUsecase::with_context(Some(shared_gating_ambient().await));
let store = shared_gating_ecstore().await;
let bucket = BucketName::new(format!("list-quorum-{}", uuid::Uuid::new_v4().simple())).unwrap();
store
.make_bucket(bucket.as_str(), &MakeBucketOptions::default())
.await
.unwrap();
let set = &store.pools[0].disk_set[0];
let saved_disks = {
let mut disks = set.disks.write().await;
let saved = disks.clone();
disks.iter_mut().skip(1).for_each(|disk| *disk = None);
saved
};
let envelope = || owned_envelope(http::Method::GET);
let results = [
usecase
.execute_list_objects(
envelope(),
gateway_dto::ListObjectsInput {
bucket: bucket.clone(),
..Default::default()
},
)
.await
.map(drop),
usecase
.execute_list_objects_v2(
envelope(),
gateway_dto::ListObjectsV2Input {
bucket: bucket.clone(),
..Default::default()
},
)
.await
.map(drop),
usecase
.execute_list_object_versions(
envelope(),
gateway_dto::ListObjectVersionsInput {
bucket,
..Default::default()
},
)
.await
.map(drop),
];
*set.disks.write().await = saved_disks;
for result in results {
let error = result.expect_err("cannot list below storage read quorum");
assert_eq!(error.code(), &S3ErrorCode::ServiceUnavailable);
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
assert_eq!(error.message(), Some("The service is unavailable. Please retry."));
let source = error.source().expect("retain storage diagnostics");
assert!(
matches!(
source.downcast_ref::<StorageError>(),
Some(StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _))
),
"{source}"
);
}
});
}
fn object(name: &str) -> ObjectInfo {
ObjectInfo {
name: name.to_owned(),
+1 -1
View File
@@ -626,7 +626,7 @@ impl DefaultBucketUsecase {
let params = parse_list_objects_v2_params(prefix, delimiter, max_keys, continuation_token, start_after)?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_ref(), envelope.uri().query())?;
validate_list_object_unordered_with_delimiter(params.delimiter.as_deref(), envelope.uri().query())?;
let store = get_validated_store(&bucket).await?;
let incl_deleted = get_header(envelope.headers(), rustfs_utils::http::SUFFIX_INCLUDE_DELETED)
@@ -23,8 +23,7 @@ use crate::app::storage_api::s3::dto::{
FilterRule, LambdaFunctionConfiguration, NotificationConfiguration, NotificationConfigurationFilter,
PutBucketNotificationConfigurationInput, QueueConfiguration, S3KeyFilter, TopicConfiguration,
};
use crate::app::storage_api::s3::{S3, S3Access, S3ErrorCode, S3Request};
use crate::app::storage_api::test::ecfs::FS;
use crate::app::storage_api::s3::{S3ErrorCode, S3Request};
use crate::runtime_sources::{NotifyInterface, ServerContextSlot};
use http::Method;
use rustfs_notify::{EventArgs, NotificationError};
@@ -215,6 +214,76 @@ fn notification_request(
request
}
fn notification_filters(filter: Option<NotificationConfigurationFilter>) -> Vec<(Option<String>, Option<String>)> {
filter
.and_then(|f| f.key)
.and_then(|k| k.filter_rules)
.unwrap_or_default()
.into_iter()
.map(|rule| (rule.name.map(|n| n.as_str().to_owned()), rule.value))
.collect()
}
fn notification_destinations(configuration: NotificationConfiguration) -> Vec<NotificationDestination> {
let mut destinations = Vec::new();
for cfg in configuration.queue_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "QueueConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.queue_arn,
filters: notification_filters(cfg.filter),
});
}
for cfg in configuration.topic_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "TopicConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.topic_arn,
filters: notification_filters(cfg.filter),
});
}
for cfg in configuration.lambda_function_configurations.into_iter().flatten() {
destinations.push(NotificationDestination {
kind: "LambdaFunctionConfiguration",
id: cfg.id,
events: cfg.events.into_iter().map(|event| event.as_ref().to_owned()).collect(),
target_arn: cfg.lambda_function_arn,
filters: notification_filters(cfg.filter),
});
}
destinations
}
async fn authorize_notification_put(
slot: &Arc<ServerContextSlot>,
envelope: &mut RequestEnvelope,
bucket: &str,
) -> rustfs_s3_types::S3Result<()> {
use rustfs_policy::policy::action::{Action, S3Action};
crate::app::storage_api::owned_parity::authorize_bucket_config_mutation(
slot,
envelope,
bucket,
Action::S3Action(S3Action::PutBucketNotificationAction),
)
.await
}
async fn put_notification(
slot: &Arc<ServerContextSlot>,
mut envelope: RequestEnvelope,
input: PutBucketNotificationConfigurationInput,
) -> rustfs_s3_types::S3Result<gateway_dto::PutBucketNotificationConfigurationOutput> {
authorize_notification_put(slot, &mut envelope, &input.bucket).await?;
let prepared = crate::storage::persisted_from_s3s(&input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
let destinations = notification_destinations(input.notification_configuration);
DefaultBucketUsecase::with_context(slot.app_context())
.execute_put_bucket_notification_configuration(envelope, &input.bucket, destinations, prepared)
.await
}
fn check_notification_write_order() {
run_large_stack_test("notification-write-order", || async {
let ambient = shared_gating_ambient().await;
@@ -250,13 +319,10 @@ fn check_notification_write_order() {
.map_or_else(|| "eu-west-2".to_owned(), |region| region.to_string());
let slot = ServerContextSlot::new();
assert!(slot.install(context.clone()));
let fs = FS::with_server_ctx(slot.clone());
let audit = MutationCompletionProbe::install(&bucket);
let mut request = notification_request(&bucket, configuration(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut request)
.await
.unwrap();
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
let request = notification_request(&bucket, configuration(), &slot);
let envelope = RequestEnvelope::from_s3s(&request);
let result = observe_reload(&bucket, calls.clone(), put_notification(&slot, envelope, request.input)).await;
let expected = expected_rules();
let mut timeline = vec![Observation::Rules(
Callback::Validate,
@@ -288,7 +354,7 @@ fn check_notification_write_order() {
};
if let Some(message) = error_message {
let error = result.expect_err("injected callback failure");
assert_eq!(error.code(), &S3ErrorCode::InternalError);
assert_eq!(error.code().as_str(), S3ErrorCode::InternalError.as_str());
assert_eq!(error.message(), Some(message));
} else {
result.expect("configuration written");
@@ -307,15 +373,13 @@ fn check_notification_write_order() {
assert_eq!(*recorder.active_rules.lock().unwrap(), expected);
let mut invalid = configuration();
invalid.lambda_function_configurations.as_mut().unwrap()[0].lambda_function_arn = "invalid-arn".into();
let mut invalid = notification_request(&bucket, invalid, &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut invalid)
.await
.unwrap();
let invalid = notification_request(&bucket, invalid, &slot);
let invalid_env = RequestEnvelope::from_s3s(&invalid);
calls.lock().unwrap().clear();
let err = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, invalid))
let err = observe_reload(&bucket, calls.clone(), put_notification(&slot, invalid_env, invalid.input))
.await
.expect_err("the last destination has an invalid ARN");
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(err.code().as_str(), S3ErrorCode::InvalidArgument.as_str());
assert!(
err.message()
.unwrap()
@@ -337,11 +401,9 @@ fn check_notification_write_order() {
assert_eq!(output.queue_configurations[0].events[1].as_str(), "s3:Future:Unrecognized");
calls.lock().unwrap().clear();
let mut empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut empty)
.await
.unwrap();
observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, empty))
let empty = notification_request(&bucket, NotificationConfiguration::default(), &slot);
let empty_env = RequestEnvelope::from_s3s(&empty);
observe_reload(&bucket, calls.clone(), put_notification(&slot, empty_env, empty.input))
.await
.unwrap();
assert_eq!(
@@ -417,16 +479,14 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
let context = Arc::new(app_context_from_current_environment(&ambient).with_test_notify_interface(recorder));
let slot = ServerContextSlot::new();
assert!(slot.install(context));
let fs = FS::with_server_ctx(slot.clone());
let mut request = notification_request(&bucket, configuration(), &slot);
S3Access::put_bucket_notification_configuration(&fs, &mut request)
.await
.unwrap();
assert_eq!(bucket_config_mutation_incarnation(&request, &bucket).unwrap(), Some(first));
let request = notification_request(&bucket, configuration(), &slot);
let mut envelope = RequestEnvelope::from_s3s(&request);
authorize_notification_put(&slot, &mut envelope, &bucket).await.unwrap();
assert_eq!(bucket_config_mutation_incarnation(&envelope, &bucket).unwrap(), Some(first));
let mismatch = DefaultBucketUsecase::without_context()
.execute_put_bucket_notification_configuration(
RequestEnvelope::from_s3s(&request),
envelope.fork_for_authorization(),
"other-bucket",
vec![NotificationDestination {
kind: "QueueConfiguration",
@@ -449,9 +509,22 @@ fn legacy_put_notification_does_not_write_after_the_bucket_is_recreated() {
.await
.unwrap();
let result = observe_reload(&bucket, calls.clone(), S3::put_bucket_notification_configuration(&fs, request)).await;
let prepared =
crate::storage::persisted_from_s3s(&request.input.notification_configuration).map_err(rustfs_s3_types::S3Error::from);
let destinations = notification_destinations(request.input.notification_configuration);
let result = observe_reload(
&bucket,
calls.clone(),
DefaultBucketUsecase::with_context(slot.app_context()).execute_put_bucket_notification_configuration(
envelope,
&bucket,
destinations,
prepared,
),
)
.await;
assert_eq!(read_notification_xml(&bucket).await, OLD_XML);
assert_eq!(result.unwrap_err().code(), &S3ErrorCode::NoSuchBucket);
assert_eq!(result.unwrap_err().code().as_str(), S3ErrorCode::NoSuchBucket.as_str());
assert!(
matches!(calls.lock().unwrap().as_slice(), [Observation::Rules(Callback::Validate, _, _, xml)] if xml == OLD_XML)
);
@@ -633,6 +633,24 @@ mod tests {
assert!(err.message().is_some_and(|message| message.contains("peer reload failed")));
}
#[tokio::test]
async fn execute_put_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
let usecase = DefaultBucketUsecase::without_context();
let envelope = crate::app::bucket_usecase::test_support::owned_envelope(http::Method::PUT);
let error = usecase
.execute_put_object_lock_configuration(
envelope,
"test-bucket",
Some(ObjectLockConfiguration {
object_lock_enabled: Some("Enabled".into()),
rule: None,
}),
)
.await
.unwrap_err();
assert_eq!(error.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
#[tokio::test]
async fn execute_get_object_lock_configuration_returns_internal_error_when_store_uninitialized() {
let input = gateway_dto::GetObjectLockConfigurationInput {
+12 -27
View File
@@ -243,27 +243,19 @@ impl DefaultBucketUsecase {
#[cfg(test)]
mod tests {
use super::*;
use crate::app::bucket_usecase::test_support::{build_request, s3_op_total};
use crate::app::storage_api::s3::S3ErrorCode;
use crate::app::storage_api::s3::dto::{DeleteBucketPolicyInput, PutBucketPolicyInput};
use crate::app::bucket_usecase::test_support::build_request;
use http::Method;
#[tokio::test]
async fn execute_delete_bucket_policy_records_s3_operation_before_store_lookup() {
let input = DeleteBucketPolicyInput::builder()
.bucket("test-bucket".to_string())
.build()
.unwrap();
async fn execute_delete_bucket_policy_returns_internal_error_when_store_uninitialized() {
let req = build_request((), Method::DELETE);
let usecase = DefaultBucketUsecase::without_context();
let req = build_request(input, Method::DELETE);
let fs = crate::storage::ecfs::FS::new();
let before = s3_op_total(S3Operation::DeleteBucketPolicy);
let err = crate::app::storage_api::s3::S3::delete_bucket_policy(&fs, req)
let err = usecase
.execute_delete_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket")
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::DeleteBucketPolicy), before + 1);
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
fn bucket(name: &str) -> rustfs_gateway_types::BucketName {
@@ -300,20 +292,13 @@ mod tests {
#[tokio::test]
async fn execute_put_bucket_policy_returns_internal_error_when_store_uninitialized() {
let input = PutBucketPolicyInput::builder()
.bucket("test-bucket".to_string())
.policy("{}".to_string())
.build()
.unwrap();
let req = build_request((), Method::PUT);
let usecase = DefaultBucketUsecase::without_context();
let req = build_request(input, Method::PUT);
let fs = crate::storage::ecfs::FS::new();
let before = s3_op_total(S3Operation::PutBucketPolicy);
let err = crate::app::storage_api::s3::S3::put_bucket_policy(&fs, req)
let err = usecase
.execute_put_bucket_policy(RequestEnvelope::from_s3s(&req), "test-bucket", "{}")
.await
.unwrap_err();
assert_eq!(err.code(), &S3ErrorCode::InternalError);
assert_eq!(s3_op_total(S3Operation::PutBucketPolicy), before + 1);
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InternalError);
}
}
+10 -4
View File
@@ -211,11 +211,17 @@ mod tests {
let mut req = build_request(input, Method::PUT);
req.extensions.insert(ReqInfo::default());
req.extensions.insert(slot.clone());
authorize_bucket_config_mutation(&slot, &mut req, &bucket, Action::S3Action(S3Action::PutBucketTaggingAction))
.await
.expect("permission allows tag updates");
let mut envelope = RequestEnvelope::from_s3s(&req);
authorize_bucket_config_mutation(
&slot,
&mut envelope,
&bucket,
Action::S3Action(S3Action::PutBucketTaggingAction),
)
.await
.expect("permission allows tag updates");
let result = usecase
.execute_put_bucket_tagging(RequestEnvelope::from_s3s(&req), &req.input.bucket, persisted)
.execute_put_bucket_tagging(envelope, &req.input.bucket, persisted)
.await;
if valid {
result.expect("valid tags are persisted");
-29
View File
@@ -461,35 +461,6 @@ impl RequestEnvelope {
pub fn residual_extensions(&self) -> &Extensions {
&self.residual_extensions
}
/// The payloads the connection, request and signature layers attached, back in the extensions
/// they arrive in, for an edge that hands this request to a handler still taking those.
pub(crate) fn transport_extensions(&self) -> Extensions {
let mut extensions = self.residual_extensions.clone();
extensions.insert(self.remote_addr.map(RemoteAddr));
if let Some(client_info) = &self.client_info {
extensions.insert(client_info.clone());
}
if let Some(native_tls) = self.native_tls_transport {
extensions.insert(NativeTlsTransport(native_tls));
}
if let Some(request_context) = &self.request_context {
extensions.insert(request_context.clone());
}
if let Some(server_context) = &self.server_context {
extensions.insert(Arc::clone(server_context));
}
if let Some(body_read_control) = &self.body_read_control {
extensions.insert(body_read_control.clone());
}
if let Some(verified) = self.verified_sigv4 {
extensions.insert(verified);
}
if let Some(verified) = self.verified_presigned {
extensions.insert(verified);
}
extensions
}
}
#[cfg_attr(
@@ -108,17 +108,6 @@ async fn persist_scanner_snapshot(ecstore: Arc<ECStore>, info: DataUsageInfo) {
rustfs_scanner::scanner::store_data_usage_in_backend(CancellationToken::new(), ecstore, receiver).await;
}
async fn bind_current_bucket_incarnations(ecstore: &ECStore, info: &mut DataUsageInfo) {
for (bucket, usage) in &mut info.buckets_usage {
let incarnation = ecstore
.bucket_incarnation_id_from_disk(bucket)
.await
.expect("load the current incarnation for the covered snapshot");
assert!(!incarnation.is_nil(), "covered snapshot must bind a non-nil bucket identity");
usage.bucket_incarnation = Some(incarnation.to_string());
}
}
#[tokio::test]
#[serial]
async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
@@ -185,7 +174,6 @@ async fn data_usage_endpoint_serves_snapshot_without_live_listing() {
seeded_info.bucket_sizes.insert(overlay_bucket.clone(), 0);
seeded_info.buckets_count = u64::try_from(seeded_info.buckets_usage.len()).expect("test bucket count should fit into u64");
seeded_info.calculate_totals();
bind_current_bucket_incarnations(&ecstore, &mut seeded_info).await;
persist_scanner_snapshot(ecstore.clone(), seeded_info).await;
record_bucket_object_write_memory(&overlay_bucket, None, 512).await;
@@ -353,7 +341,6 @@ async fn data_usage_endpoint_treats_legacy_partial_stats_as_unknown_until_covere
completed.bucket_sizes.insert(small_bucket.clone(), 57_344);
completed.buckets_count = u64::try_from(completed.buckets_usage.len()).expect("test bucket count should fit into u64");
completed.calculate_totals();
bind_current_bucket_incarnations(&ecstore, &mut completed).await;
store_data_usage_in_backend(completed, ecstore.clone())
.await
.expect("persist complete covered snapshot");
@@ -28,7 +28,6 @@ use super::storage_api::test::contract::{
namespace::NamespaceLocking as _,
object::{ObjectIO as _, ObjectOperations as _},
};
use super::storage_api::test::ecfs::FS;
use super::storage_api::test::object_utils::to_s3s_etag;
use super::storage_api::test::runtime::{
MockWarmBackend, MockWarmOp, get_global_transition_state, register_mock_tier as register_mock_tier_util,
@@ -16,7 +16,6 @@
use super::common::*;
use super::*;
use crate::app::storage_api::s3::S3 as _;
fn expiration_lifecycle_configuration(prefix: &str) -> BucketLifecycleConfiguration {
BucketLifecycleConfiguration {
@@ -162,8 +161,7 @@ async fn compensation_driven_delete_marker_still_honors_lifecycle_cleanup() {
.unwrap(),
Method::PUT,
);
FS::new()
.put_bucket_lifecycle_configuration(req)
put_lifecycle(req)
.await
.expect("Failed to update lifecycle configuration for delete marker cleanup");
@@ -199,10 +197,7 @@ async fn put_bucket_lifecycle_configuration_expires_existing_objects() {
Method::PUT,
);
FS::new()
.put_bucket_lifecycle_configuration(req)
.await
.expect("Failed to update lifecycle configuration");
put_lifecycle(req).await.expect("Failed to update lifecycle configuration");
assert!(
wait_for_delete_marker(&ecstore, bucket.as_str(), object, TRANSITION_WAIT_TIMEOUT).await,
@@ -231,8 +226,7 @@ async fn put_bucket_lifecycle_configuration_allows_expired_delete_marker_on_obje
Method::PUT,
);
FS::new()
.put_bucket_lifecycle_configuration(req)
put_lifecycle(req)
.await
.expect("ExpiredObjectDeleteMarker should be accepted on object-lock bucket");
}
@@ -258,11 +252,9 @@ async fn put_bucket_lifecycle_configuration_rejects_del_marker_expiration_on_obj
Method::PUT,
);
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
let err = put_lifecycle(req)
.await
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket")
.into();
.expect_err("DelMarkerExpiration must be rejected on object-lock bucket");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -294,11 +286,9 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
Method::PUT,
);
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
let err = put_lifecycle(req)
.await
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket")
.into();
.expect_err("DelMarkerExpiration with zero days must be rejected on object-lock bucket");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -351,11 +341,9 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
Method::PUT,
);
let err: rustfs_s3_types::S3Error = FS::new()
.put_bucket_lifecycle_configuration(req)
let err = put_lifecycle(req)
.await
.expect_err("zero-day expiration must be rejected with InvalidArgument")
.into();
.expect_err("zero-day expiration must be rejected with InvalidArgument");
assert_eq!(err.code(), &rustfs_s3_types::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
@@ -471,7 +459,6 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
),
];
let fs = FS::new();
let handled_count = || {
rustfs_io_metrics::s3_op_metrics_snapshot()
.into_iter()
@@ -511,8 +498,8 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
.extensions
.insert(parity_test_support::bucket_config_mutation(&readable));
}
let error = fs.put_bucket_lifecycle_configuration(request).await.unwrap_err();
assert_eq!(error.code(), &code, "{name}: {error}");
let error = put_lifecycle(request).await.unwrap_err();
assert_eq!(error.code().as_str(), code.as_str(), "{name}: {error}");
if let Some(message) = message {
assert_eq!(error.message(), Some(message), "{name}");
}
@@ -533,9 +520,7 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
let before = handled_count();
let probe = MutationCompletionProbe::install(&readable);
fs.put_bucket_lifecycle_configuration(request(&readable, Some(("Disabled", false))))
.await
.unwrap();
put_lifecycle(request(&readable, Some(("Disabled", false)))).await.unwrap();
assert_audit(&probe);
assert_eq!(handled_count() - before, 1, "success counts once");
@@ -547,3 +532,26 @@ async fn put_bucket_lifecycle_configuration_preserves_error_priority() {
assert_eq!(rules[0].expiration.as_ref().unwrap().date.as_deref(), Some("2099-01-01T00:00:00.000Z"));
assert_eq!(rules[1].expiration.as_ref().unwrap().days, Some(30));
}
async fn put_lifecycle(req: S3Request<PutBucketLifecycleConfigurationInput>) -> rustfs_s3_types::S3Result<()> {
rustfs_io_metrics::record_s3_op(rustfs_s3_ops::S3Operation::PutBucketLifecycleConfiguration);
let envelope = crate::app::RequestEnvelope::from_s3s(&req);
let prepared = match req.input.lifecycle_configuration {
None => crate::app::bucket_usecase::PreparedLifecycleConfiguration::Missing,
Some(cfg) => {
if crate::app::bucket_usecase::validate_lifecycle_rule_status(cfg.rules.iter().map(|rule| rule.status.as_str()))
.is_err()
{
crate::app::bucket_usecase::PreparedLifecycleConfiguration::InvalidStatus
} else {
crate::app::bucket_usecase::PreparedLifecycleConfiguration::Prepared(
crate::storage::s3_compat::lifecycle::lifecycle_configuration_from_s3s(cfg),
)
}
}
};
crate::app::bucket_usecase::DefaultBucketUsecase::from_global()
.execute_put_bucket_lifecycle_configuration(envelope, &req.input.bucket, prepared)
.await
.map(|_| ())
}
@@ -28,11 +28,10 @@ use super::*;
#[ignore = "global-state usecase integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn object_lock_handlers_schedule_replication() {
use super::super::storage_api::object_usecase::bucket::replication::MUST_REPLICATE_OBJECT_CALLS;
use crate::app::storage_api::s3::S3;
use std::sync::atomic::Ordering;
let (_disk_paths, ecstore) = setup_test_env().await;
let fs = FS::new();
let object_meta = crate::app::object_meta::ObjectMetaUsecase::with_context(None);
let bucket = format!("test-lock-repl-{}", &Uuid::new_v4().simple().to_string()[..8]);
let object = "locked/object.txt";
@@ -52,7 +51,7 @@ async fn object_lock_handlers_schedule_replication() {
// PutObjectRetention must compute a replication decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let retention_input = PutObjectRetentionInput::builder()
let _retention_input = PutObjectRetentionInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.retention(Some(ObjectLockRetention {
@@ -61,7 +60,18 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
fs.put_object_retention(build_request(retention_input, Method::PUT))
let req = build_request((), Method::PUT);
object_meta
.put_object_retention_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLockRetention {
mode: Some(ObjectLockRetentionMode::COMPLIANCE.to_owned()),
retain_until: Some(time::macros::datetime!(2030-01-01 00:00:00 UTC)),
}),
)
.await
.expect("put_object_retention should succeed");
assert_eq!(
@@ -73,7 +83,7 @@ async fn object_lock_handlers_schedule_replication() {
// A bucket without a replication configuration can skip the decision.
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let no_replication_input = PutObjectLegalHoldInput::builder()
let _no_replication_input = PutObjectLegalHoldInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.legal_hold(Some(ObjectLockLegalHold {
@@ -81,7 +91,17 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
fs.put_object_legal_hold(build_request(no_replication_input, Method::PUT))
let req = build_request((), Method::PUT);
object_meta
.put_object_legal_hold_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
ObjectLockLegalHoldStatus::OFF.to_owned(),
))),
)
.await
.expect("legal hold without replication should succeed");
assert_eq!(
@@ -93,7 +113,7 @@ async fn object_lock_handlers_schedule_replication() {
// A configured bucket must still evaluate the metadata replication decision.
install_delete_replication_config(&bucket, true, false, None).await;
MUST_REPLICATE_OBJECT_CALLS.store(0, Ordering::SeqCst);
let legal_hold_input = PutObjectLegalHoldInput::builder()
let _legal_hold_input = PutObjectLegalHoldInput::builder()
.bucket(bucket.clone())
.key(object.to_string())
.legal_hold(Some(ObjectLockLegalHold {
@@ -101,7 +121,17 @@ async fn object_lock_handlers_schedule_replication() {
}))
.build()
.unwrap();
fs.put_object_legal_hold(build_request(legal_hold_input, Method::PUT))
let req = build_request((), Method::PUT);
object_meta
.put_object_legal_hold_fields(
crate::app::RequestEnvelope::from_s3s(&req),
&bucket,
object,
None,
Some(crate::storage::request_markers::RequestedObjectLegalHold(Some(
ObjectLockLegalHoldStatus::ON.to_owned(),
))),
)
.await
.expect("put_object_legal_hold should succeed");
assert_eq!(
+1 -1
View File
@@ -29,7 +29,7 @@ use crate::app::RequestEnvelope;
use crate::app::bucket_usecase::DefaultBucketUsecase;
use crate::app::storage_api::s3::dto::{EncodingType, ListObjectVersionsInput, ListObjectsV2Input};
use crate::app::storage_api::s3::xml;
use crate::storage::access::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::access_owned::listing_multipart::authorize_bucket_listing;
use crate::storage::authorization::AuthorizationRequest;
use http::{HeaderMap, Uri};
use rustfs_policy::policy::action::S3Action;
+2
View File
@@ -22,6 +22,8 @@
pub mod admin_usecase;
mod answered;
#[cfg(test)]
mod audit_coverage_test;
pub(crate) mod bucket_list_through;
pub mod bucket_usecase;
pub mod context;
+16 -9
View File
@@ -50,8 +50,7 @@ use super::storage_api::multipart_usecase::io::{
EtagAlgorithm, HashReader, WriteEncryption, WritePlan, compression_metadata_value,
};
#[cfg(test)]
use super::storage_api::multipart_usecase::object_utils::to_s3s_etag;
use super::storage_api::multipart_usecase::object_utils::to_s3s_xml_etag;
use super::storage_api::multipart_usecase::object_utils::{to_s3s_etag, to_s3s_xml_etag};
use super::storage_api::multipart_usecase::options::{
copy_src_opts, extract_metadata_from_mime, get_complete_multipart_upload_opts_with_replication_authorization,
get_content_sha256_with_query, get_opts, has_replication_retention_update, namespace_reserved_user_metadata,
@@ -104,6 +103,7 @@ use futures::StreamExt;
use http::{HeaderMap, Uri};
use metrics::counter;
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result, s3_error};
use rustfs_targets::EventName;
use rustfs_utils::CompressionAlgorithm;
#[cfg(test)]
@@ -115,7 +115,6 @@ use rustfs_utils::http::{
headers::{AMZ_CHECKSUM_TYPE, AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS},
insert_str,
};
use s3s::{S3Error, S3ErrorCode, S3Result, s3_error};
use std::collections::{HashMap, HashSet};
use std::sync::Arc;
use std::time::Duration;
@@ -193,12 +192,13 @@ fn multipart_max_total_object_size(metadata: &HashMap<String, String>) -> S3Resu
let value = get_consistent_str(metadata, SUFFIX_MAX_TOTAL_OBJECT_SIZE).ok_or_else(|| {
S3Error::with_message(
S3ErrorCode::InvalidRequest,
"multipart size capability metadata is missing or inconsistent".to_string(),
"multipart size capability metadata is missing or inconsistent",
)
})?;
value.parse::<u64>().map(Some).map_err(|_| {
S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid".to_string())
})
value
.parse::<u64>()
.map(Some)
.map_err(|_| S3Error::with_message(S3ErrorCode::InvalidRequest, "multipart size capability metadata is invalid"))
}
async fn validate_table_catalog_object_mutation(bucket: &str, key: &str) -> S3Result<()> {
@@ -291,7 +291,11 @@ impl DefaultMultipartUsecase {
/// or completed".
fn multipart_upload_api_error(err: StorageError) -> S3Error {
if matches!(err, StorageError::MalformedUploadID(_)) {
return S3Error::new(S3ErrorCode::NoSuchUpload);
let mut error = S3Error::new(S3ErrorCode::NoSuchUpload);
if let Some(message) = error.code().default_message() {
error.set_message(message);
}
return error;
}
ApiError::from(err).into()
}
@@ -545,7 +549,7 @@ mod tests {
.execute_create_multipart_upload_from_s3s(build_request(create, Method::POST))
.await
.expect_err("reserved key must be rejected before accessing storage");
assert_eq!(error.code(), &S3ErrorCode::InvalidArgument);
assert_eq!(error.code(), &crate::app::storage_api::s3::S3ErrorCode::InvalidArgument);
let complete = CompleteMultipartUploadInput::builder()
.bucket("bucket".to_owned())
@@ -593,6 +597,9 @@ mod tests {
fn multipart_upload_api_error_reports_malformed_upload_id_as_no_such_upload() {
let malformed = multipart_upload_api_error(StorageError::MalformedUploadID("bucket/object-not-an-id".to_string()));
assert_eq!(malformed.code(), &S3ErrorCode::NoSuchUpload);
let legacy = crate::app::storage_api::s3::S3Error::new(crate::app::storage_api::s3::S3ErrorCode::NoSuchUpload);
assert_eq!(malformed.message(), legacy.message());
assert_eq!(malformed.status_code(), legacy.status_code());
let unknown = multipart_upload_api_error(StorageError::InvalidUploadID(
"bucket".to_string(),
@@ -61,7 +61,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
req.uri = format!("/{bucket}/{key}").parse().expect("fixture URI");
req.extensions.insert(slot.clone());
req.extensions.insert(request_context.clone());
req
RequestEnvelope::from_s3s(&req)
};
for name in [&bucket, &other] {
store
@@ -143,7 +143,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
(wrong, S3ErrorCode::InternalError),
] {
let err = usecase
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&req), Ok(target()))
.execute_abort_multipart_upload(req, Ok(target()))
.await
.expect_err("Abort requires current generation");
assert_eq!(err.code(), &code);
@@ -162,7 +162,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
.expect("capture current bucket");
current.install_bucket_generation(captured);
usecase
.execute_abort_multipart_upload(RequestEnvelope::from_s3s(&current), Ok(target()))
.execute_abort_multipart_upload(current.fork_for_authorization(), Ok(target()))
.await
.expect("current generation allows Abort");
assert!(matches!(
@@ -182,7 +182,7 @@ fn native_abort_requires_current_generation_and_audits_each_result() {
.expect("create legacy-path upload");
usecase
.execute_abort_multipart_upload(
RequestEnvelope::from_s3s(&current),
current,
Ok(AbortMultipartUploadTarget::Legacy {
bucket: bucket.clone(),
key: key.to_owned(),

Some files were not shown because too many files have changed in this diff Show More