Compare commits

..

154 Commits

Author SHA1 Message Date
overtrue e917fad829 test(e2e): admit the website stack interim test to the merged smoke selection 2026-11-12 04:29:24 +08:00
overtrue f06cf92891 Merge remote-tracking branch 'origin/gw/T2.5-gateway-stack' into gw/T2.11-website-interim
# Conflicts:
#	.config/e2e-smoke-selection.txt
2026-11-12 02:20:40 +08:00
overtrue 4d7354eaa9 test(server): show a non-website host stays on the gateway stack 2026-11-12 02:19:22 +08:00
overtrue 83f3fbf2cd Merge remote-tracking branch 'origin/main' into gw/T2.5-gateway-stack 2026-11-12 00:46:17 +08:00
overtrue f56477eb53 Merge remote-tracking branch 'origin/main' into gw/T2.5-gateway-stack
# Conflicts:
#	rustfs/src/storage/access.rs
#	rustfs/src/storage/storage_api.rs
2026-11-11 22:58:00 +08:00
overtrue 2f55ddb595 Merge remote-tracking branch 'origin/gw/T1.8-access-envelope' into gw/T2.5-gateway-stack
# Conflicts:
#	.config/make/pre-commit.mak
#	.config/test-count-baseline.txt
#	Cargo.lock
#	Cargo.toml
#	crates/gateway-adapter/Cargo.toml
#	crates/gateway-adapter/src/lib.rs
2026-11-11 20:53:15 +08:00
overtrue 44e147e8eb test(gateway): sign the probe request with a declared payload token
The probe request carried no x-amz-content-sha256 and was answered 400 before it reached the handler. It now declares an unsigned payload, and a probe that is not reached reports the answer it got.
2026-11-11 20:01:29 +08:00
overtrue dd760dfa56 fix(app): drop the PutObjectInput re-export main now provides 2026-11-11 19:37:21 +08:00
overtrue 7be29ee088 Merge remote-tracking branch 'origin/main' into gw/T1.8-access-envelope 2026-11-11 19:22:52 +08:00
overtrue e024c65494 Merge remote-tracking branch 'origin/gw/T2.5-gateway-stack' into gw/T2.11-website-interim 2026-11-11 18:14:41 +08:00
overtrue 4eb1731c42 test(e2e): observe website redirects instead of following them 2026-11-11 18:14:36 +08:00
overtrue 70a0267d01 Merge remote-tracking branch 'origin/gw/T1.8-access-envelope' into gw/T2.5-gateway-stack 2026-11-11 18:03:26 +08:00
overtrue 28ac9de62a fix(auth): keep the test-only credentials rebuild out of the lib build 2026-11-11 17:39:55 +08:00
overtrue 7dbd8e4bed Merge remote-tracking branch 'origin/main' into gw/T2.11-website-interim 2026-11-11 17:37:29 +08:00
overtrue 5126fdc4c2 Merge remote-tracking branch 'origin/gw/T2.5-gateway-stack' into gw/T2.11-website-interim 2026-11-11 17:37:29 +08:00
overtrue da6c626f2a fix(gateway): refuse an empty handed-over secret at the gateway edge
RequestCredentials no longer represents an absent secret as an empty string. The gateway edge maps the new constructor result, and a signed ListBuckets request is shown to reach its handler without the caller's secret, where a reader of it fails closed.
2026-11-11 17:35:13 +08:00
overtrue e2fee7cb2b Merge remote-tracking branch 'origin/gw/T1.8-access-envelope' into gw/T2.5-gateway-stack
# Conflicts:
#	rustfs/src/app/context/envelope.rs
2026-11-11 17:12:17 +08:00
overtrue a115748cda fix(auth): make an absent caller secret unrepresentable as a key
RequestCredentials::secret_key now returns Option<&str>: None when the edge kept the secret, and an empty handed-over secret is refused at construction. Secret comparisons go through secret_key_matches, which never matches an absent secret, and readers that need the value fail closed with an internal error.
2026-11-11 17:08:17 +08:00
overtrue 1f7f5b8532 fix(table-catalog): restore owned request lines the main merge reverted
The merge onto main replaced two lines with main's text: the warehouse location check again wrapped an already owned request in legacy_request_view, and an error conversion was re-added. Restore the branch's version of the file.
2026-11-11 16:44:45 +08:00
overtrue 9a0530af03 Merge remote-tracking branch 'origin/gw/T2.5-gateway-stack' into gw/T2.11-website-interim 2026-11-11 16:08:38 +08:00
overtrue 0c69936a99 test(e2e): admit the website stack interim test to the e2e selections 2026-11-11 16:08:28 +08:00
overtrue 0721822cfa Merge remote-tracking branch 'origin/gw/T1.8-access-envelope' into gw/T2.5-gateway-stack 2026-11-11 16:03:45 +08:00
overtrue 0b7fd7a367 Merge remote-tracking branch 'origin/gw/T2.5-gateway-stack' into gw/T2.5-gateway-stack 2026-11-11 15:55:44 +08:00
overtrue 494b0a789b Merge remote-tracking branch 'origin/gw/T2.5-gateway-stack' into gw/T2.11-website-interim 2026-11-11 15:55:29 +08:00
overtrue 6b10eb7008 test(e2e): report every stack difference, not the first 2026-11-11 15:55:24 +08:00
Chris An f3b0806547 fix(gateway): reconcile squashed foundation with main
Merge main after the foundation squash, preserving gateway branch changes
and importing the unchanged scanner prefix fix from main.

Verified exact tree preservation outside the two scanner files, whitespace,
release-boundary guard and self-test, footprint guard, and logging guard.
Rust compile, fmt and scoped tests unavailable locally: no Rust toolchain.
Test-count guard could not run because cargo metadata is unavailable.
2026-11-11 02:55:06 -05:00
overtrue 1fbcacb2b6 test(auth): cover multipart staging in the owned/legacy decision matrix 2026-11-11 15:53:57 +08:00
overtrue 7729e8c337 Merge remote-tracking branch 'origin/gw/T1.8-access-envelope' into gw/T2.5-gateway-stack
# Conflicts:
#	rustfs/src/storage/storage_api.rs
2026-11-11 15:53:25 +08:00
overtrue a537b6ea98 Merge remote-tracking branch 'origin/gw/T2.1-adapter-foundation' into gw/T2.5-gateway-stack 2026-11-11 15:53:11 +08:00
overtrue 3805f2970e Merge remote-tracking branch 'origin/main' into gw/T1.8-access-envelope 2026-11-11 15:42:39 +08:00
overtrue 62f0385769 Merge remote-tracking branch 'origin/gateway/integration' into gw/T1.8-access-envelope 2026-11-11 15:42:39 +08:00
overtrue c312102bab docs(gateway-adapter): state what install guarantees about the provider handle 2026-11-11 15:41:18 +08:00
overtrue 35e6c8077d Merge remote-tracking branch 'origin/main' into gw/T2.1-adapter-foundation 2026-11-11 15:40:56 +08:00
overtrue 007c3b204b Merge remote-tracking branch 'origin/gateway/integration' into gw/T2.1-adapter-foundation 2026-11-11 15:40:56 +08:00
overtrue 8ace49936b test(e2e): compare website and metadata listings across both stacks
One data directory is seeded and answered on the legacy stack, then
restarted and answered on the gateway stack: status, Location,
Content-Type, Vary, CORS headers and body bytes must match for the
website requests and the metadata listings. The legacy answers are held
to what a website must answer, and a plain ListObjectsV2 must differ, so
neither a shared failure nor a restart that kept legacy can pass.

The test joins the e2e-smoke profile. An embedded gateway-stack test
pins the website route on the stack's own feature lane.
2026-11-11 15:01:44 +08:00
overtrue d0499ae66c feat(server): keep website hosts and metadata listings on legacy
While the gateway stack is selected, the stack dispatcher now sends two
S3-shaped families to the legacy service, checked first as legacy checks
them: requests whose Host names a RUSTFS_WEBSITE_DOMAINS domain (R-064),
and the ?metadata=true ListObjectsV2 and ListObjectVersions extension.

Both predicates are the legacy metadata route's own rules, exposed with
s3s-free arguments, and each lives in its own file under
server/stack/ so the PR that migrates a family deletes only that file.
2026-11-11 15:01:43 +08:00
overtrue 1c3fe4747e refactor(server): box the route facts a deferred question carries
A question the table answers by itself is one byte; one that asks the core carries the request facts. Boxing them keeps the common, decided case small.
2026-11-11 15:00:12 +08:00
overtrue 032566507a test(auth): report every decision mismatch in the parity matrix 2026-11-11 14:35:50 +08:00
overtrue 42ab123d69 Merge remote-tracking branch 'origin/gateway/integration' into gw/T2.5-gateway-stack 2026-11-11 14:25:05 +08:00
overtrue 22f6c668da feat(server): wire the owned authorization core into the gateway stack
The route stage's decision core is now the owned authorization entry point: a RouteDecidable row resolves the caller with the function the legacy access check uses and asks authorize_owned_request about the request as it arrived. The host attaches the request line for it, since the gateway hands an authorizer the headers and the query but neither the method nor the target. A core that cannot answer stays Indeterminate.

The gateway stack refuses to start while Keystone is enabled: the pinned gateway admits a request that carries only X-Auth-Token as anonymous, while the RustFS code behind it reads the Keystone identity (rustfs/gateway#1586).

The embedded tests now observe the S3 request counter through the whole stack: one served ListBuckets is counted once under the label the legacy stack uses, and a request refused by authentication once under no operation. The two tests that pin how a failed credential lookup is answered name ledger entry R-068 and rustfs/gateway#1585.
2026-11-11 14:24:59 +08:00
overtrue 9829661a98 Merge remote-tracking branch 'origin/gw/T1.8-access-envelope' into gw/T2.5-gateway-stack
# Conflicts:
#	rustfs/src/storage/access.rs
#	rustfs/src/storage/storage_api.rs
2026-11-11 14:23:10 +08:00
overtrue ee3ef73630 test(gateway): pin how the gateway answers an unavailable credential backend
The pinned gateway's SigV4 authenticator answers a credential provider fault exactly as an unknown access key, so an unavailable IAM or a withdrawn root is 403 InvalidAccessKeyId, not a 5xx as on the legacy stack. The two tests asserted the 5xx this stack cannot answer; they now pin the observed refusal and that nothing is served, and the root rotation test checks the new key is known to the next request.
2026-11-11 13:48:59 +08:00
rustfs-cla-assistant[bot] 4c7cc8a943 Merge remote-tracking branch 'origin/main' into gateway/integration 2026-11-11 05:25:16 +00:00
overtrue 42ee5b151b test(auth): satisfy clippy on the parity test facade 2026-11-11 13:12:38 +08:00
overtrue 3cda284b4c test(auth): owned and legacy entry points reach the same decisions
Decision-level parity over IAM allow and explicit deny, owner, anonymous on private and public buckets, bucket-policy explicit deny, aws:SourceIp from the peer and from a verified client, and one allow and one deny per body-derived input (legal hold, tagging, retention mode and retain-until, resolved copy source, POST form fields).
2026-11-11 12:36:59 +08:00
overtrue fb053dfb85 test(auth): read storage-private parity facts through test accessors
The parity tests moved to the app layer can no longer read ObjectTagConditions and TableDataPlanePublicationGuards fields directly; parity_test_support exposes read-only accessors instead. Drop an unused import.
2026-11-11 12:32:50 +08:00
overtrue 9707aa4f1f ci(gateway): lint and test the gateway stack on every code change
The gateway-stack feature leg becomes its own job on the pull-request path and joins the Test and Lint aggregate, instead of a leg of the nightly-only feature matrix. Operation slices put their handler tests, their authorization census and their end-to-end refusal tests behind the feature, so a leg that ran only on dispatch would let them rot.

The job set changed, so the plan contract marker moves to 4: a plan from a base planner that predates this job is refused and the full matrix runs.
2026-11-11 12:23:02 +08:00
overtrue 197e682488 refactor(server): install the IAM adapters as one pair
The adapter crate now installs its credential provider and policy source together through IamAdapters::install, and takes the live root as a closure. The assembly passes a closure over the root credentials the legacy stack reads, and keeps refusing a missing or too-short configured root.
2026-11-11 12:09:55 +08:00
overtrue 0e0744cb49 Merge remote-tracking branch 'origin/gw/T2.1-adapter-foundation' into gw/T2.5-gateway-stack 2026-11-11 12:08:08 +08:00
overtrue 20872bba63 fix(gateway-adapter): effective secret length and unsplittable pair
Address the confirmation review of the T2.1 adapters
(rustfs/backlog#2760).

- One rule for every secret: its length after trailing NUL bytes are
  removed must be at least MIN_SECRET_KEY_LEN. HMAC zero-pads its key,
  so trailing NULs add nothing; interior NULs count. Applied to the root
  secret, the STS signing key and every looked-up credential.
- IamAdapters keeps both halves private and hands them to a builder
  together through the consuming install(builder, make_authenticator);
  compile-fail doctests show a mixed pair cannot be assembled.
- Any Fn() -> Option<Arc<RootCredentials>> is a RootSource, so the
  binary reads its live root accessor directly; RootCell is now a
  test-only helper.
2026-11-11 12:06:45 +08:00
overtrue 30d8aee534 feat(server): read the live root for the gateway stack on every request
The assembly builds both adapters over one live root source that reads the root credentials where the legacy stack reads them, per request, so a root change reaches both stacks at once. A missing or too-short configured root fails the assembly instead of starting a server without a root.
2026-11-11 11:55:23 +08:00
overtrue 05e6fd2085 Merge remote-tracking branch 'origin/gateway/integration' into gw/T2.5-gateway-stack 2026-11-11 11:52:39 +08:00
overtrue 3f861328d8 Merge remote-tracking branch 'origin/gw/T2.1-adapter-foundation' into gw/T2.5-gateway-stack 2026-11-11 11:52:39 +08:00
overtrue 210e1b214e fix(server): keep the empty Buckets element on the legacy ListBuckets
The compatibility seam writes a listing without buckets as an absent member, where the legacy edge has always answered an empty element. Clients index the answer by it, so the legacy edge restores it. Also drops a re-export the removed legacy output builder was the last user of.
2026-11-11 11:52:39 +08:00
Chris An 55d931353f ci(s3-tests): add reference parity lane against pinned 1.0.5 (#9288) 2026-11-10 22:50:17 -05:00
overtrue 0e1e2e0003 fix(gateway-adapter): share live root and refuse short secrets
Address the security review of the T2.1 adapters (rustfs/backlog#2760).

- Add RootCredentials (validated: root secret and STS signing key at
  least MIN_SECRET_KEY_LEN bytes), the RootSource trait and the settable
  RootCell. IamAdapters::new is now the only constructor of
  IamCredentials and IamPolicySource, so both read one live root source
  per request; no root source answers unavailable.
- A looked-up identity whose secret is shorter than 8 bytes is NotFound
  in the credential provider and DISABLED_PRINCIPAL in the policy
  source, whatever the lookup reports as valid.
- STS claims are read with the STS signing key from the root source
  instead of assuming it equals the root secret.
- Cover the off-status-alone branch, the OIDC owner exclusion, service
  accounts of root, the name live groups are read under, and another
  session's token.
2026-11-11 11:36:11 +08:00
overtrue b3e60a8882 test(auth): move owned parity tests to the app layer
Owned-versus-legacy parity, hoist and fail-closed tests now live in app/owned_authorization_test.rs beside the envelope's facts impl, so storage names no app type in test code either. Private storage steps are reached through a test-only parity_test_support module. Also fix the S3Error conversion CI reported in prepare_owned_authorization.
2026-11-11 11:26:52 +08:00
overtrue e6a1f542a1 Merge remote-tracking branch 'origin/gateway/integration' into gw/T1.8-access-envelope
# Conflicts:
#	.config/test-count-baseline.txt
2026-11-11 10:59:13 +08:00
overtrue 662781e45b Merge remote-tracking branch 'origin/gateway/integration' into gw/T2.5-gateway-stack 2026-11-11 10:59:06 +08:00
overtrue 7aceffc68c ci(gateway): keep gateway-stack out of release builds and lint it
The gateway stack refuses to compile without debug assertions, and a static guard keeps the feature, and any feature that enables it, out of the default and full feature lists and out of every release input. The feature matrix gains a gateway-stack leg for its lints and tests.
2026-11-11 10:58:56 +08:00
overtrue e0b53cc8d9 feat(server): serve ListBuckets on the gateway stack behind a feature
With the gateway-stack feature built in and RUSTFS_S3_STACK=gateway, the S3 service under the middleware is the rustfs-gateway service: the RustFS profile, SigV4 verification against RustFS IAM, an authorizer backed by an explicit per-operation table, explicit deadlines, and one registered handler. Every other S3 operation is answered 501, and admin, console, STS, health and profiling routes stay on the legacy router until they are migrated.

The stack is selected once at startup. An assembly failure fails startup, and a binary without the feature keeps refusing the selection.
2026-11-11 10:58:56 +08:00
overtrue db370e3241 refactor(app): move ListBuckets use case onto the request envelope
The use case takes a RequestEnvelope and the gateway's ListBucketsInput and returns the gateway's ListBucketsOutput, so both S3 stacks call one implementation. The legacy s3s method converts through the compatibility seam and calls it.

The per-request part of the legacy access check (caller resolution, the upload-size capability scoping and the license check) moves into two functions over owned facts, which the legacy check now calls and the gateway handlers will call too.
2026-11-11 10:58:56 +08:00
overtrue ffe6bc86d3 fix(auth): keep owned authorization in the app layer and fail closed
Move the RequestEnvelope impl of AuthorizationRequest and the owned entry points into the app layer so storage depends only on the request-facts trait (layer guard). The owned entry points now hoist signed x-amz-meta-* query parameters like the legacy edge and refuse actions whose decision needs a request-body input the envelope does not carry. Revert unrelated TODO rewording and redact PostObjectPolicyHeaders debug output.
2026-11-11 10:53:07 +08:00
Chris An fc09148f08 chore(gateway): record ignored microbenchmarks merged from main (#9290)
chore(gateway): record the ignored microbenchmarks merged from main

main added three throughput microbenchmarks that are ignored by design
(#9217): one each in rustfs-checksums, rustfs-rio and rustfs-utils. The
test-count ratchet treats a higher ignored count as a failure until the
row is raised, so raise those three rows to 1 and tighten the test rows
to the merged tree with --update.
2026-11-10 21:30:34 -05:00
rustfs-cla-assistant[bot] c9e1464a2a Merge remote-tracking branch 'origin/main' into gateway/integration 2026-11-11 02:06:08 +00:00
overtrue 5d6358a0a5 Merge remote-tracking branch 'origin/gateway/integration' into gw/T2.5-gateway-stack 2026-11-11 09:58:48 +08:00
Chris An 0e3028d29e ci(gateway): make the test-count ratchet a floor, not exact (#9278)
ci(gateway): make the test-count ratchet a floor instead of an exact baseline

Tests below a row still fail; tests above it, a member without a row and ignores below the row now pass with a note. Ignores above a row still fail. Adds contract tests wired into make script-tests.
2026-11-10 20:53:06 -05:00
overtrue dde2169c83 feat(gateway-adapter): add IAM-backed PolicySource
Add IamPolicySource, the gateway PolicySource answered by RustFS IAM
(rustfs/backlog#2760, policy half). One snapshot per request carries a
RustfsPrincipal: principal kind, account, effective username, parent
user, credential groups, live group memberships, token claims and
is_owner.

- Anonymous callers get an empty snapshot without asking IAM.
- IAM not ready, not initialized or failing is PolicyError::unavailable,
  never an empty snapshot.
- Unknown, disabled and session-kind-mismatched principals are refused
  with constant labels; unreadable tokens are malformed.
- Claims are read from the stored token (root secret for STS, the
  account's own secret for service accounts), and is_owner mirrors the
  legacy has_root_access rule.
- RustfsPrincipal holds no secret or session token and has no Debug.
2026-11-11 09:43:47 +08:00
overtrue 0b586c37e9 refactor(auth): share authorization core with owned entry points
Extract the policy decision behind authorize_request into a core that both the legacy s3s request and the RustFS-owned RequestEnvelope feed, and move the zip-create and table-catalog admin callers to the owned entry points (authorize_owned_request, authorize_owned_internal_object_request). Request-body policy inputs added on main since the candidate base (POST form headers, copy source, legal hold and tagging bodies) become named envelope fields so both edges reach the same decision. Refs rustfs/backlog#2749.
2026-11-11 09:40:02 +08:00
Chris An 365014a385 feat(scripts): add junit_diff.py for dual-stack JUnit comparison (#9275) 2026-11-10 19:55:24 -05:00
overtrue 7946b094f9 feat(gateway-adapter): add IAM-backed CredentialProvider
Add the ring-2 crate rustfs-gateway-adapter with IamCredentials, the
gateway CredentialProvider answered by RustFS IAM (rustfs/backlog#2760,
credentials half; PolicySource is deferred).

- Pin rustfs-gateway at the existing gateway rev with
  default-features = false so the server crate stays out of the build.
- IamKeyLookup is a narrow IAM trait implemented for IamSys so the
  adapter is testable without an in-memory IAM backend.
- Root credentials are injected at construction and resolve before IAM
  is ready.
- Mapping: keystone: prefix -> NotFound (deliberate change from the
  legacy empty secret); IAM not ready or not initialized -> Unavailable;
  unknown key or NoSuch{User,ServiceAccount,TempAccount,Account} ->
  NotFound; other IAM errors -> Backend; invalid or "off" principals
  resolve disabled; STS credentials carry a SessionBinding, and an STS
  credential without an expiry is a Backend fault rather than a
  never-ending session.
- No authorization in lookup; no Debug on secret-holding types; IAM
  error text is dropped after classification.
2026-11-11 07:19:17 +08:00
overtrue 443e8afee1 chore(gateway): merge main into integration
# Conflicts:
#	crates/ecstore/src/disk/local/delete_recovery.rs
#	crates/ecstore/src/store/bucket.rs
2026-11-10 21:40:26 +08:00
overtrue 782e3e5209 chore(gateway): merge main into integration 2026-11-10 08:28:05 +08:00
overtrue cfffb1700e chore(gateway): update test-count baseline after main sync
Main #9241 removed the node pool-metadata readiness cache with 8 tests
and added 6, so the rustfs row drops from 6440 to 6438. The ecstore and
s3select rows rise for tests added by #9237, #9239 and #9241.
2026-11-09 10:21:53 +08:00
rustfs-cla-assistant[bot] cd62f3ba4f Merge remote-tracking branch 'origin/main' into gateway/integration 2026-11-09 02:07:05 +00:00
overtrue 7bc419e368 chore(gateway): merge main into integration
# Conflicts:
#	.config/e2e-full-selection.txt
#	crates/ecstore/src/disk/local/delete_recovery.rs
2026-11-09 00:50:02 +08:00
overtrue 7269243de7 chore(gateway): merge main into integration
Port main's changes into integration's relocated code: the XML compatibility
body bound and presigned-guard tests into legacy_compat, the policy-condition
fixes (#9150, #9159, #9163) into auth facts and the owned admin request API,
and record native publications on main's new fused delete-recovery paths.
2026-11-08 22:26:59 +08:00
overtrue c4a7423f99 fix(ecstore): create parents before admitted truncate opens
Align the heal tests with removal-disabled preservation and local-only outcome
fields, and refresh the e2e-full selection for the synced tests.
2026-10-31 08:38:51 +08:00
overtrue 2db4ed244a fix(ecstore): scope the merged direct-writer test hook to test builds 2026-10-31 06:40:49 +08:00
overtrue 93a61df1cf chore(gateway): merge main into integration
Keep native publication recording in main's native_io helpers and resource
writers, and adapt merged tests to the owned Body, Bytes write input, and
fallible find_disk.
2026-10-31 06:15:49 +08:00
overtrue feec0e9e4a test(s3select): use the owned JSON input types in merged planner tests 2026-10-31 03:08:13 +08:00
overtrue 38baeeb11d chore(gateway): merge main into integration
Record native publications on the paths main's commit-fusion work opened
directly: the lazy truncate writer and the fused delete-recovery activation
and receipt renames. Keep extract_params_header in the shared audit builder.
2026-10-31 01:40:30 +08:00
overtrue 1af039c972 chore(gateway): merge main into integration 2026-10-30 20:14:52 +08:00
overtrue a2fc7c13b4 chore(gateway): merge main into integration 2026-10-30 18:41:32 +08:00
overtrue dbe01e5cb1 chore(gateway): raise test-count rows for tests merged from main 2026-10-30 14:35:18 +08:00
overtrue ffc27fa6db chore(gateway): merge main into integration 2026-10-30 14:29:25 +08:00
overtrue 7e8edab0d3 chore(gateway): merge main into integration 2026-10-30 11:23:05 +08:00
overtrue 22057ad4c5 chore(gateway): merge main into integration 2026-10-30 07:08:33 +08:00
overtrue 279a88ab56 chore(gateway): merge main into integration 2026-10-29 20:19:36 +08:00
overtrue b4abe64ada chore(gateway): merge main into integration
Sync main 84e24e767 into gateway/integration. Ports main-side additions to
the integration boundaries: the tier deletion diagnostics admin handlers move
to the owned admin request/response types, and the new lifecycle rule checks
in ecstore use the persisted lifecycle types instead of s3s DTOs. Keeps the
gateway git source in deny.toml while dropping the smithy-rs pin main retired,
and refreshes the test-count baseline for the merged tree.
2026-10-29 16:01:13 +08:00
overtrue add363f084 chore(gateway): merge main into integration
Sync main 9601a7c4b into gateway/integration. Ports main-side additions to the
owned admin boundary (plugin delegation/attempt handlers, notification target
edit tests), drops a new ecstore s3s::dto reference in favour of the persisted
lifecycle filter, and refreshes the e2e selection digests and test-count
baseline for the merged tree.
2026-10-29 07:31:24 +08:00
overtrue 1f2dbe36e5 test(ecstore): pin native history on cancellation under the cancellable queue
main moved the local write path onto the disk blocking queue, which skips an
operation once its waiter is dropped, so a borrowed write cancelled while it is
queued no longer creates its file. The integration-only native publication
test still expected the queued open to run. Assert the new execution model
instead (the open never runs) while keeping its safety claims: the Started
record is taken before queueing and the cancelled publication never regrants
native history.
2026-10-27 22:08:04 +08:00
overtrue 67fbafa13e chore(gateway): merge main into integration
Source: main 4e48c390e7
Target: gateway/integration 1000860cb8

Conflicts resolved, keeping both sides:
- crates/ecstore/src/disk/{fs,local}.rs: main's batched rollback writes,
  unified blocking write path and relocated open_file, with integration's
  native publication records restored at each publication point and the
  publication_history field kept beside main's multipart proof authority.
- crates/ecstore/src/set_disk/ops/multipart.rs: union of the imports.
- rustfs/src/storage/rpc/node_service.rs: main's MRF lifecycle reader
  capability branch; the heal capability keeps integration's restore
  copy-back probe.
- rustfs/src/admin/handlers/{audit,event,plugins_instances}.rs: main's
  target_read inventory snapshot on integration's owned request boundary;
  the new event.rs inventory test now builds owned requests.
- scripts/check_s3s_footprint.sh: integration's baseline-file version.
- .config/e2e-{full,smoke}-selection.txt: digests recomputed over the
  union of both heads' CI listings (full 700 darwin / 708 linux, smoke 232).

Also migrated main's new rebalance delete-all expiry test to the persisted
lifecycle types integration uses in ecstore, and raised the test-count
baseline to the merged counts (every decrease comes from main).
2026-10-27 08:38:17 +08:00
Chris An 1000860cb8 refactor(admin): drop the legacy target descriptor JSON response helper (#8968)
`target_descriptor::build_json_response` returned the s3s response and
had no production caller left once the event, audit and plugin instance
handlers moved to `owned_build_json_response`; it survived only for the
legacy/owned equivalence test. Remove it, give the owned helper its own
documentation, and make the test assert the owned response directly:
status, `application/json`, the echoed `x-request-id` when present and
only then, and the body bytes. The file no longer imports s3s, so the
footprint baseline drops from 66 to 65 files.
2026-10-27 00:15:37 +08:00
Chris An b783f404ad refactor(admin): migrate identity and cluster handlers to owned requests (#8966)
* refactor(admin): migrate identity and cluster handlers to owned requests

Move the T1.12 batch 4 admin handlers (user, service_account, oidc, tier,
system, pools, quota, kms_dynamic, kms_backup, rebalance, module_switch,
bucket_meta, cluster_snapshot, realtime, bucket_recovery, profile) from the
s3s request/response types to the owned AdminRequest/AdminResponse
boundary. All 90 impl Operation blocks now take AdminRequest and return
AdminResponse; handler logic, authorization order, error codes and response
bytes are unchanged. The handlers use the shared owned helpers
(owned_json_response, owned_admin_json_response, authorize_profile_envelope,
the envelope-form runtime sources and audit constructors) instead of local
copies.

profile.rs drops the s3s forms of authorize_profile_request and
profile_not_implemented_response now that profile_admin.rs is on the
envelope gate too; profile_not_implemented_response answers with an
AdminResponse. The realtime metrics stream moves from the s3s ByteStream
adapter to Body::from_stream, and the rebalance idle-404 document still
renders through s3s::S3Error::to_http_response with its body converted at
the edge.

The idp_compat delegators ImportIamV2 and AddServiceAccountLdap move to the
owned trait as well: with their targets on both traits, the method-call
delegation became ambiguous between the owned impl and the router blanket
impl, so they now take the owned request directly.

KmsAdminAudit::from_request loses its last production caller with this batch
and is deleted; its comparison test now asserts the envelope facts directly.

Lower the s3s footprint baseline to the measured 68 files.

Part of rustfs/backlog#2788 (rustfs/backlog#2734 T1.12).

* refactor(admin): remove the legacy admin helpers batch 4 superseded

Batch 4 moved the last production callers of the s3s-typed admin helpers
to their owned forms, which left `admin_json_response`, `json_response`,
`authenticate_request`, `authorize_admin_request`, `object_store_from_req`
and `object_store_from_extensions` unused, along with the `Body`,
`S3Request` and `S3Response` re-exports of the admin `s3` facade. Remove
them. The owned forms keep their documentation; the behavior tests of the
removed `json_response` now exercise `owned_json_response`, and the
compat-prefix test asserts the owned response directly.

Two tests existed only to compare a removed legacy helper with its owned
twin (`owned_json_response_matches_the_legacy_shape`) or to cover the
removed legacy gate (`authorize_admin_request_without_credentials_is_rejected`);
they go with the helpers, and the owned equivalents stay. The rustfs
test-count row drops from 5957 to 5955 and the s3s footprint from 68 to 66
files.
2026-10-26 22:23:47 +08:00
Chris An 11a991c777 refactor(admin): complete the owned admin request boundary (#8949)
* refactor(admin): complete the owned admin request boundary

Every type implementing the owned admin `Operation` now serves the legacy
router through one blanket impl that converts the s3s request and response
at that single edge, so handlers register with `AdminOperation(&Handler {})`
and need no adapter statics; `LegacyAdminAdapter` is removed.

`AdminRequest` gains the accessors handlers read today (`remote_addr`,
`client_info`, `request_context`, `req_info`, `req_info_mut`,
`install_req_info`, `envelope_mut`, `take_body`), the request helpers gain
envelope forms (`authenticate_envelope_credentials`,
`object_store_from_envelope`, the table-catalog `*_from_envelope` helpers,
`KmsAdminAudit::from_envelope`, `AccountAuditContext::from_envelope`,
`CallerIdentity::resolve_envelope`), `read_compatible_admin_body` accepts
either body type, and the owned `Body` gains `store_all_limited` and
`Default`. `admin::operation::test_support::TestRequest` builds owned
requests for handler tests. Unused accessors carry a per-method dead-code
expectation that each migrating handler batch removes.

No handler behavior changes; the remaining migrations follow in batches.

* chore(admin): add the T1.12 admin handler codemod

The script rewrites an admin handler file from the s3s request and response
types to the owned boundary (imports, signatures, request field and helper
calls, body access, response construction, and test request literals) and
leaves a TODO comment at every site it cannot rewrite mechanically. It is
idempotent and is removed once T1.12 completes.

* refactor(admin): migrate replication and KMS handlers to owned requests (#8951)

Move the T1.12 batch 2 admin handlers (site_replication, kms_keys, event,
replication, diagnostics, on_demand_migration, audit, account, kms_rekey,
kms_key_lifecycle, object_data_cache, tls_debug, integrity) from the s3s
request/response types to the owned AdminRequest/AdminResponse boundary.
Handler logic, authorization order, error codes and response bytes are
unchanged; the request/response plumbing now goes through the envelope
accessors and helpers introduced by the owned boundary.

The shared response helpers (utils::json_response, utils::empty_response,
handlers::admin_json_response, target_descriptor::build_json_response,
profile::authorize_profile_request) and the site replication repair
executor still answer with the s3s types, so the migrated files carry
owned-boundary forms of those helpers and convert the repair response at
its call edge. The streaming console-log body and the two devnull drains
move from the s3s byte stream to the owned Body stream/frame API.

Drop the dead_code expectations for the owned accessors and envelope
helpers this batch starts using, and lower the s3s footprint baseline to
the measured 99 files.

Part of rustfs/backlog#2788 (rustfs/backlog#2734 T1.12).

* refactor(admin): add owned forms of the shared admin response helpers

Handler batches were copying `json_response`, `empty_response`,
`admin_json_response`, and `build_json_response` into each migrated file
because the shared helpers still return the s3s response. Provide owned
forms next to each legacy helper (`owned_json_response`,
`owned_empty_response`, `owned_admin_json_response`,
`owned_build_json_response`, `authorize_profile_envelope`) with parity
tests against the legacy shape, derive `Debug` for `AdminResponse` so
tests can use `expect_err`, and fix three codemod rules: strip owned
string conversions in test credentials, annotate non-literal URI parses,
and leave a rebound request parameter immutable.

* refactor(admin): use the shared owned response helpers in batch 2 (#8954)

Replace the per-file owned-boundary copies of json_response,
empty_response, admin_json_response, build_json_response and
authorize_profile_request in the batch 2 admin handlers with the shared
owned_* helpers and authorize_profile_envelope that the boundary branch
now provides, and restore expect_err in the handler tests now that
AdminResponse implements Debug; the `.err().expect(..)` stand-ins were
failing the base branch's clippy step (clippy::err_expect under
-D warnings). The site replication repair call edge
keeps its conversion because the repair executor still answers with the
s3s response type.

Drop the dead_code expectations on the shared owned helpers this batch
now uses in non-test code.

Part of rustfs/backlog#2788 (rustfs/backlog#2734 T1.12).

* fix(admin): preserve legacy body limit error responses

* refactor(admin): move policy, ILM and batch 3 handlers to owned requests (#8956)

Batch 3 of T1.12 (rustfs/backlog#2788, parent rustfs/backlog#2734) moves
14 admin handler files to AdminRequest/AdminResponse and the owned
Operation trait: ilm_transition, policies, idp_compat, config_admin,
plugins_instances, profile_admin, group, heal, object_zip_download,
kms_key_metadata, durability, inspect_archive, sts and
gateway_key_inventory. Handler logic, authorization order, error codes
and response bytes are unchanged; test request literals use
test_support::TestRequest.

The handlers use the shared owned helpers (owned_json_response,
owned_admin_json_response, owned_build_json_response,
authorize_profile_envelope). The dead_code expectations on the
AdminRequest accessors and RequestEnvelope::uri_mut they now exercise
are removed. The legacy target_descriptor::build_json_response keeps
only its equivalence test caller and carries the matching expectation
until it retires.

object_zip_download builds the s3s request the S3 gate still reads from
the owned request facts; heal authenticates through
authenticate_envelope_credentials; the trace, inspect-archive and zip
bodies use Body::from_stream. extract_plugin_instance_filters takes the
URI so extensions.rs keeps calling it, and the two idp_compat
delegations stay on the legacy trait until their delegates migrate.

The s3s footprint baseline drops from 99 to 90 files.

* test(admin): anchor the replication diff body contract on the call

The source-contract test looked for `let body = read_compatible_admin_body(`
on one line; rustfmt wrapped that binding after the handler moved to the
owned request boundary, so the marker was no longer found. Anchor on the
call itself so formatting cannot change what the contract checks.

* refactor(admin): migrate table catalog and batch 1 to owned requests (#8955)

* fix(admin): preserve catalog body limit response bytes
2026-10-26 17:04:02 +08:00
Chris An 3f2b6f690a refactor(admin): migrate health and prefix usage to owned requests (#8887)
* refactor(admin): add owned health router boundary

* refactor(admin): migrate prefix usage to owned request
2026-10-26 07:34:31 +08:00
overtrue 88b626e7a0 chore(gateway): merge main into integration
Source: main 0f24511535
Target: gateway/integration 8cf6187ff4

Resolve the e2e-full selection digest conflict by recomputing both
platform digests over the merged nextest listing (685 darwin / 693
linux): main adds two rpc_fault_proxy unit tests, integration keeps its
dual-stack differential and anonymous-access additions. Raise the rustfs
test-count baseline row to the merged count (5932).
2026-10-25 09:58:02 +08:00
overtrue 8cf6187ff4 chore(gateway): merge main into integration 2026-10-25 01:50:03 +08:00
Chris a0a0cee9eb refactor(auth): derive policy conditions from AuthzFacts (#8442)
* refactor(auth): extract authorization facts

* test(e2e): update smoke selection baseline

* test(e2e): update full selection baseline
2026-10-24 16:56:01 +09:00
overtrue 80fff4b5c5 Merge gateway/integration (#8844) into the main sync
Only the s3s footprint baseline conflicted; s3_error_lines tightens to 289.
2026-10-24 09:30:22 +08:00
Chris d91dfcedd2 refactor(s3): migrate private multipart validation errors (#8844) 2026-10-24 09:17:48 +09:00
overtrue 79c4fedf8c Merge gateway/integration into the main sync
Brings in #8825, #8831, #8836 and #8839. Conflicts were import lists and test blocks (both sides kept), plus the new restore_copyback request flag next to main's data-movement field. Main replaced completed_multipart_object_part with prepare_complete_multipart_parts, so the stale import is dropped.

Main's xl.meta fast-path scanner (inspect_v2_object_slice) predates #8836 and classified a transitioned version with a pending v1 restore copyback as having no physical data dir, which made find_unshared_data_dir_for_version drop the copyback data dir. The scanner now reads the copyback marker with get_consistent_bytes semantics (both aliases, ASCII case-insensitive, empty or conflicting values count as absent), and uses_data_dir keeps main's allocation-free transition check plus #8836's copyback exclusion. find_unshared_data_dir_for_version_keeps_pending_copyback_data_dir failed on the plain merge and passes now.
2026-10-24 07:51:55 +08:00
Chris 73c846e09c refactor(storage): migrate option errors to domain types (#8839) 2026-10-24 05:50:10 +09:00
Chris 6938c467d0 fix(ecstore): finalize restore metadata after local copy-back (#8836)
* fix(ecstore): finalize restore metadata after local copy-back

* test(ecstore): ratchet added restore finalization cases

* fix(ecstore): keep restore copyback compatible with older peers

* fix(ecstore): gate restore copyback on fleet compatibility

* fix(ecstore): retain transition identity during restore copyback

* fix(ecstore): route committed restore copyback locally

* fix restore copyback cleanup

* fix(ecstore): honor restore probe test fallback
2026-10-24 02:34:29 +09:00
Chris 5323df7049 fix(protocols): clear clippy failures after T1.5 merge (#8831)
fix(protocols): clear clippy failures after merge
2026-10-23 17:37:26 +09:00
Chris 55b6fbd260 refactor(protocols): move StorageBackend to gateway types (#8825)
* refactor(protocols): move storage backend to gateway types

* test: update e2e test count baseline

* fix: preserve copy source keys and unknown lengths

* test: update protocols test count baseline

* fix(protocols): preserve typed SFTP backend errors

* test(protocols): use gateway list input in policy tests
2026-10-23 15:46:28 +09:00
overtrue dd677cc121 Merge main into gateway/integration
Picks up main through 44a73b0be. Conflicts: rustfs/src/error.rs keeps the rustfs-s3-types ApiError conversion and drops the helper main's #8816 replaced; scripts/check_s3s_footprint.sh keeps the integration script, whose baseline lives in .config (main only retuned a constant the integration script does not have). Main's new tests use the rustfs-s3-types error code in admin replication and reach s3s::BodySizeLimitExceeded through the app's test-only s3 facade, so the s3s file count stays at 118; s3_error_lines tightens to 302. The test-count baseline is regenerated; the e2e selections are unchanged.
2026-10-23 08:40:02 +08:00
overtrue 0c2f327974 Merge gateway/integration into the main sync
Brings in #8812 and #8818 from gateway/integration. #8812 predates main's heal error classifier (#8778); its two retryable variants (TooManyOpenFiles, OutdatedXLMeta) move into the classifier's transient arms for storage and disk errors, and its test is kept.
2026-10-23 07:22:20 +08:00
overtrue df42cf1ce8 test(e2e): use a non-default region for the differential negative control
Since main's region-aware CreateBucket and GetBucketLocation (#8739, #8743), us-east-1 answers an empty LocationConstraint and 200 for an owned re-create, so the control no longer reported its own region. Also refresh the e2e-full linux selection from the CI listing (682 tests).
2026-10-23 07:21:19 +08:00
Chris d290d46b34 test: reserve nextest capacity for progress probes (#8818) 2026-10-23 06:12:31 +09:00
Chris 0d297ed8cd fix(heal): retry transient metadata conflicts (#8812)
* fix(heal): retry transient metadata conflicts

* chore: update heal test count baseline
2026-10-23 01:47:55 +09:00
overtrue b8dffa2f1d Merge main into gateway/integration
Picks up the 20 main commits since 62b6f5f3f. The one conflict is the tier free-version recovery test setup: it keeps the integration-only fixture isolation (background recovery disabled while the fixture starts) and main's mock tier registration (#8804 skips scans with no tier). The e2e selections are unchanged by these commits; the test-count baseline is regenerated.
2026-10-22 20:17:09 +08:00
overtrue 62b6f5f3fc Merge main into gateway/integration
The scheduled sync has stopped on conflicts since 2026-10-19. Resolved by keeping both sides: main's restructured delete-rollback path in disk/local.rs with the integration-only native-publication record, main's per-instance region with the rustfs-s3-types Region, main's new lifecycle, rebalance and admin code ported to the persisted lifecycle types and rustfs-s3-types errors, both sides' new scanner tests, and the union of the access_consumer imports. The e2e smoke, full (darwin) and repl-nightly selections and the test-count baseline are regenerated from the merged tree.
2026-10-22 10:14:41 +09:00
Chris be0986115c refactor(s3select): run queries on gateway DTO request shapes (#8794)
* refactor(s3select): run queries on an engine request built from gateway DTO shapes

Part of T1.5 (rustfs/backlog#2746), batch 2.

* fix(s3select): keep the select request conversion in the app layer

The layer guard forbids app -> storage; the conversion moves from storage/s3_compat/select.rs to app/select_request.rs unchanged.
2026-10-22 08:16:48 +09:00
Chris 1bd91c8dad refactor(s3-client): move remote error codes and DTO enums off s3s (#8783)
Part of T1.5 (rustfs/backlog#2746), batch 1.
2026-10-22 06:15:04 +09:00
Chris dcb22bd4d5 refactor(ecstore): move bucket configs onto gateway persisted types (#8709)
* refactor(ecstore): move bucket configs onto gateway persisted types

Finish the ECStore batch of T1.4 (rustfs/backlog#2745). The remaining
eleven bucket configuration families (Website, CORS, Tagging, Versioning,
ObjectLock, Encryption, Notification, Logging, Accelerate, RequestPayment,
PublicAccessBlock) now read and write their stored XML through the
rustfs-gateway-types persistence and cors_tagging codecs instead of the
s3s codec, so the legacy s3s bucket-config codec and its
s3gate-metadata-xml compat marker are removed.

- ecstore: BucketConfigXml is implemented by the gateway codec for every
  family; metadata, metadata_sys, versioning, object lock, replication and
  lifecycle consumers take the persisted types. Region moves to
  rustfs_s3_types::Region, and the ETag conversion to s3s moves to the
  rustfs edge with its tests. Only warm_backend.rs and warm_backend_s3.rs
  still reference s3s, held for T1.5 (rustfs/backlog#2746).
- scanner: object lock cache, item actions and data usage use the
  persisted types.
- rustfs: the legacy S3 edge converts each family between s3s DTOs and the
  persisted types in storage/storage_api.rs by writing one codec's XML and
  reading it with the other; Tagging converts field by field.
- Bump the rustfs-gateway-types pin to 21283e87.

The 39 persisted-byte goldens are unchanged and pass, so a document the
old codec wrote reads back and re-encodes to the same bytes. s3s footprint:
files 181 -> 146, ecstore_files 28 -> 2, s3_error_lines unchanged.

* test(app): prepare the PutBucketEncryption config from the request

The uninitialized-store test passed an empty persisted configuration, which the use case now validates before it reaches the store check, so it answered MalformedXML instead of InternalError. Convert the request's configuration the way the S3 edge does.

* style(app): format the PutBucketEncryption test
2026-10-21 23:51:23 +09:00
Chris fa17718444 refactor(admin): move error types to rustfs-s3-types (#8440)
* refactor(protocols): move error types to rustfs-s3-types by codemod

T1.2 batch 3 (rustfs/backlog#2743): s3select-api, s3select-query and
the error items of protocols move off s3s by scripts/codemods, run with
--hold crates/protocols/src/sftp/errors.rs. That file stays on s3s: it
downcasts the backend error, whose runtime type is still s3s::S3Error.

- rustfs/src/protocols/client.rs: the two webdav session methods now
  return rustfs_s3_types::S3Result and convert the s3s error with From.
- codemod: a child module that calls s3_error! through `use super::*`
  gets its own `use rustfs_s3_types::s3_error;` once its parent imports
  that macro by name; otherwise it is listed as GLOB.
- s3-types: a table test pins code, status and the s3s error the legacy
  edge renders for every variant a moved call site names.
- s3_error_lines 1585 -> 1584 (a lifecycle comment reworded).

* refactor(admin): move error types to rustfs-s3-types (WIP, no build)

T1.2 batch 4 (rustfs/backlog#2743), work in progress: rustfs check
still fails with 16 errors (listed in the PR body).

- codemod: --facade PATH treats a crate-local re-export of s3s's error
  items like s3s for the four names (9 new self-tests, 9 mutants red).
- codemod output for rustfs/src/admin and rustfs/src/site_replication
  (--facade crate::storage_api::site_replication::s3); 6 glob children
  get an explicit rustfs_s3_types::s3_error import.
- admin/router.rs: the Operation trait returns rustfs_s3_types::S3Result.
- error.rs: From<ApiError> builds the rustfs-s3-types error; the s3s
  conversion is derived from it.

* fix(admin): convert error types at the s3s boundary

Batch 4 of T1.2 left the admin and site_replication modules returning
rustfs_s3_types::S3Error while their callers and callees on the storage
and auth side still speak s3s::S3Error. Convert at each edge with the
existing From impls in rustfs-s3-types (compat_s3s.rs) instead of
changing any code, message or status:

- admin/auth.rs, admin/handlers/object_zip_download.rs,
  admin/handlers/table_catalog/mod.rs: map the s3s result returned by
  check_key_valid / authorize_request / authorize_internal_object_request
  into the admin error type with .map_err(Into::into).
- storage/ecfs.rs: the s3s-facing side maps the admin site_replication
  gate results the other way.
- admin/handlers/pools.rs: ApiError.code stays s3s::S3ErrorCode, so the
  rustfs code is converted with .into() when the struct is built, in
  production and in the two tests that build or compare it.
- admin/handlers/heal.rs, admin/runtime_sources.rs: the router returns
  s3s errors (S3Route impl); the expected rustfs code is converted with
  .into() before the comparison.
- admin/handlers/table_catalog/mod.rs: iceberg_rest_error takes
  &'static str so the Cow<'static, str> custom code borrows it; every
  caller already passes an ICEBERG_ERROR_* constant or a &'static str.
- admin/handlers/quota.rs: drop a now same-type .into() after s3_error!.
- storage_api.rs, site_replication/mod.rs: drop re-exports and an import
  that no longer have users (clippy -D warnings).

cargo check -p rustfs --all-targets and cargo clippy -p rustfs
--all-targets -- -D warnings are clean; the s3s footprint baseline is
unchanged at files=172, s3_error_lines=303, ecstore_files=28.

Refs rustfs/backlog#2743, rustfs/backlog#2734.
2026-10-21 22:50:27 +09:00
overtrue 99abffc57a refactor(lifecycle): replace s3s domain types with Gateway DTOs
Keep lifecycle rules, restore requests, Object Lock, and scanner cached tags in domain types. Convert at the existing HTTP boundary while preserving legacy XML, scanner cache bytes, date ordering, and tombstones.

Linux test validation before the final import-path-only adjustment:

cargo nextest run -p rustfs-ecstore bucket_config_goldens
39 passed; 6211 skipped by selector.

cargo nextest run --lib -p rustfs -E 'test(/test_zero_rule_lifecycle_tombstone_recognition/) | test(/test_local_lifecycle_staleness_axis_selection/) | test(/test_lifecycle_staleness_preserves_legacy_submillisecond_axis/) | test(/test_merge_incoming_lifecycle/) | test(/test_lifecycle_expiry_statement_matrix/) | test(/test_lifecycle_expiry_subset_xml/) | test(/put_bucket_lifecycle_validation_errors_keep_their_s3_code/) | test(/execute_put_bucket_lifecycle_configuration_rejects_missing_configuration/) | test(/execute_restore_object_rejects_missing_restore_request/) | test(/execute_restore_object_rejects_select_type/) | test(/execute_restore_object_maps_failures_to_typed_s3_errors/) | test(/execute_restore_object_returns_internal_error_when_store_uninitialized/) | test(/restore_request_conversion_preserves_legacy_output_values/) | test(/restore_request_conversion_preserves_presence_and_raw_enums/) | test(/restore_request_conversion_distinguishes_absent_and_empty_containers/)'
25 passed; 5318 skipped by selector.

cargo nextest run --profile ci -p rustfs-ecstore -p rustfs-lifecycle -p rustfs-replication -p rustfs-scanner
7872 passed; 21 skipped under the existing CI profile.

Previously verified mutation controls: 39 production faults, 41 RED/GREEN pairs.

cargo clippy --locked -p rustfs -p rustfs-ecstore -p rustfs-lifecycle -p rustfs-scanner --all-targets -- -D warnings
Passed in 40m 45s before the import-path adjustment, then in 1m 59s on
the final source with warnings denied. The final adjustment imports the
existing helper through the object domain facade; fmt and the architecture
check also passed.

Part of rustfs/backlog#2745. The remaining ecstore configuration families follow separately.
2026-10-17 07:03:41 +08:00
overtrue fc061c7930 fix(ecstore): retry tier commit intent lock contention
Retry transient namespace lock timeouts within the existing three-attempt
CAS budget. Reload and verify the original mutation identity after
contention, and preserve the final typed error when attempts are exhausted.
Cover save/read contention, exhausted retries, replacement identities and
CAS reconciliation without changing the existing timeout or lock policy.

Validation:
- cargo nextest run --locked -p rustfs-ecstore --features test-util --lib -E 'test(tier_mutation)' --test-threads 1 --no-fail-fast: 52 passed.
- cargo nextest run --locked -p rustfs-ecstore --features test-util --lib -E 'test(intent_advance_) | test(=services::tier::tier::tests::caller_cancellation_after_durable_prepare_does_not_hide_the_mutation)' --test-threads 1 --no-fail-fast: 5 passed.
- cargo clippy --locked -p rustfs-ecstore --all-targets -- -D warnings: passed.
- cargo clippy --locked -p rustfs-ecstore --all-targets --features test-util -- -D warnings: passed.
- 27 mutation pairs failed at the intended assertions and passed after restoring the production code.
- bash scripts/check_test_count_ratchet.sh --update, then bash scripts/check_test_count_ratchet.sh: 52 crates, 22013 tests, 165 ignored; ecstore increases from 6581 to 6585.
2026-10-17 01:19:11 +08:00
overtrue d86bf1360f fix(ecstore): recover delete batches with missing versions
Treat an explicit non-null version as a no-op when it is absent from both recovery sources. Keep rejecting missing null versions, delete-marker mismatches and conflicting local generations.

Add returning-disk and partial-receipt regressions, extend journal boundary checks, and regenerate the ecstore test-count baseline. Production faults exercise the 24 added assertion sites and 7 semantic checks: 33 faults failed at the expected sites and passed after full restoration.

Validation (LANG=en_US.UTF-8, one build job):

cargo nextest run --locked -p rustfs-ecstore --lib -E 'test(delete_recovery) | test(quorum_delete_cleanup_coordinator_binds_single_and_batch) | test(crash_delete_retry_batch_reclaims_inline_and_reserved_external)' --test-threads 1 --no-fail-fast
Summary [  34.580s] 69 tests run: 69 passed, 6058 skipped; exit 0

cargo clippy --locked -p rustfs-ecstore --all-targets -- -D warnings
Finished `dev` profile [unoptimized + debuginfo] target(s) in 5m 37s; exit 0

All 19 final Quick Checks completed with exit 0:

- shellcheck --version && actionlint
- cargo fmt --all --check
- ./scripts/check_unsafe_code_allowances.sh
- ./scripts/check_layer_dependencies.sh
- ./scripts/check_architecture_migration_rules.sh
- ./scripts/check_logging_guardrails.sh
- ./scripts/check_error_other_format_ratchet.sh
- ./scripts/check_no_tokio_io_uring.sh
- ./scripts/check_extension_schema_boundaries.sh
- ./scripts/check_body_cache_whitelist.sh
- ./scripts/check_s3s_footprint.sh
- ./scripts/check_test_count_ratchet.sh
- ./scripts/check_fips_wording.sh
- ./scripts/check_embedded_secrets.sh
- make script-tests
- python3 ./scripts/test_object_generation_protocol_model.py
- python3 ./scripts/check_test_wiring.py
- ./scripts/check_no_planning_docs.sh
- ./scripts/check_uring_lane_lib_only.sh

test-count ratchet OK: 52 crates, 22009 tests, 165 ignored match .config/test-count-baseline.txt
PASS: cold stampede script self-test
model: states=51023, transitions=167520, terminal=732, chosen_candidates=2
multi-slot round schedules: 640
OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and Quick Checks are wired
2026-10-16 17:32:35 +08:00
overtrue f6097d1df2 fix(ci): restore integration test compilation
Gate the diagnostic test import with its feature, initialize the conditional
MetadataWriteStore fields explicitly, and decode the scanner replication fixture
through its existing cache boundary. Preserve the original fixture and assertions.

Validation was run on tree 544acb80c0 at 6003d4b8.
LANG=en_US.UTF-8; no source changes follow validation.

Quick checks: 19/19 passed.
- shellcheck --version && actionlint: exit 0
- cargo fmt --all --check: exit 0
- ./scripts/check_unsafe_code_allowances.sh: exit 0
- ./scripts/check_layer_dependencies.sh: exit 0
- ./scripts/check_architecture_migration_rules.sh: exit 0
- ./scripts/check_logging_guardrails.sh: exit 0
- ./scripts/check_error_other_format_ratchet.sh: exit 0
- ./scripts/check_no_tokio_io_uring.sh: exit 0
- ./scripts/check_extension_schema_boundaries.sh: exit 0
- ./scripts/check_body_cache_whitelist.sh: exit 0
- ./scripts/check_s3s_footprint.sh: exit 0
- ./scripts/check_test_count_ratchet.sh: exit 0
- ./scripts/check_fips_wording.sh: exit 0
- ./scripts/check_embedded_secrets.sh: exit 0
- make script-tests: exit 0
- python3 ./scripts/test_object_generation_protocol_model.py: exit 0
- python3 ./scripts/check_test_wiring.py: exit 0
- ./scripts/check_no_planning_docs.sh: exit 0
- ./scripts/check_uring_lane_lib_only.sh: exit 0

Darwin validation:
- cargo clippy --locked -p rustfs-ecstore -p rustfs-scanner --all-targets -- -D warnings: exit 0
- cargo nextest run --locked -p rustfs-ecstore -p rustfs-scanner --lib -E 'test(config::com::tests::save_config_preserves_retryable_system_volume_errors) | test(scanner_io::tests::replication_maintenance_refreshes_compacted_stats_across_cycles_and_retries)' --test-threads 1 --no-fail-fast: exit 0, 2/2 passed
- cargo clippy --locked -p rustfs-ecstore -p rustfs-scanner --all-targets --features rustfs-ecstore/diagnostic-read-version-caller,rustfs-scanner/diagnostic-read-version-caller -- -D warnings: exit 101
- cargo nextest run --locked -p rustfs-ecstore -p rustfs-scanner --lib --features rustfs-ecstore/diagnostic-read-version-caller,rustfs-scanner/diagnostic-read-version-caller -E 'test(config::com::tests::save_config_preserves_retryable_system_volume_errors) | test(scanner_io::tests::replication_maintenance_refreshes_compacted_stats_across_cycles_and_retries) | test(cluster::rpc::runtime_sources::tests::) | test(config::com::tests::diagnostic_config_write_retains_origin_and_api_for_results_and_cancellation)' --test-threads 1 --no-fail-fast: exit 0, 14/14 passed

Diagnostic strict Clippy remains red on four unchanged production lints:
runtime_sources.rs if_same_then_else and three redundant_locals reports in
set_disk/core/metadata_read.rs and set_disk/read.rs. Their source ranges and
feature wiring match the base; the base was not compiled separately.

Linux aarch64 validation (Rust/Clippy 1.99.0, UID 501, jobs=1):
- cargo clippy --all-targets -- -D warnings: exit 0; Finished dev profile in 68m 12s
- cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings: exit 0; Finished dev profile in 49m 45s
Source HEAD/tree and clean status matched before and after both commands.
No compiler or stack overrides were used. protoc 35.1 was installed from its
verified ARM64 release; flatc was absent and not invoked by the workspace build scripts.
The local environment differs from hosted CI.
2026-10-16 13:06:24 +08:00
rustfs-cla-assistant[bot] 7e5d32eecf Merge remote-tracking branch 'origin/main' into gateway/integration 2026-10-16 02:09:53 +00:00
overtrue 72da0e21c4 chore: merge main into gateway/integration
Keep the migrated compatibility adapters in their active modules while bringing in upstream fixes. Preserve native publication history on the new create, rename and borrowed-open paths.

Validation:
- `cargo nextest run --locked -p rustfs-ecstore --lib -E 'test(disk::os::native_creation_history_tests::) | test(disk::local::test::borrowed_open_cancellation_retains_started_history_before_execution) | test(disk::local::test::owned_direct_child_open_attempt_consumes_native_history)' --no-fail-fast`: 9/9 passed after restoring all 12 mutation cases at cea78f2590645eb0d303cd7cf56f4dd0d9c00e87; these source blobs are unchanged in this merge.
- `cargo nextest run --locked -p rustfs-s3select-query --lib -E 'test(dispatcher::json_document_schema::tests::)' --no-fail-fast`: 5/5 passed at 5421ba682d1329cc7ebeaac7c6a863da2cea1996, with the same tree.
- `cargo fmt --all --check`: passed.
- `./scripts/check_test_count_ratchet.sh --update`: 52 crates, 22007 tests, 165 ignored.
- Profile baselines use seven actual Darwin listings and seven actual Linux listings.
- Earlier conflict snapshot f5e0cdf0c58aed1ebe53b9243cf52f94832ab049: `cargo nextest run --locked -p rustfs-ecstore -p rustfs --lib -E 'test(disk::os::) | test(bucket::metadata_sys::) | test(server::legacy_compat::) | test(server::layer::tests::request_logging_watchdog) | test(server::http::tests::conditional_object_http::) | test(storage::helper::tests::operation_helper)' --no-fail-fast` passed 218/218 before the borrowed-open fix. This is historical evidence.
2026-10-16 09:49:08 +08:00
Chris 78a0cf321c refactor(replication): move the replication config onto gateway types (#8443)
* wip: T1.4 replication family

* refactor(replication): move the replication config onto gateway types

The replication configuration family leaves s3s: rustfs-replication no longer depends on s3s, and ECStore bucket metadata, the scanner and the rustfs app/admin layers hold rustfs_gateway_types::persistence::PersistedReplicationConfiguration. The persisted XML goes through the gateway persistence codec behind a new BucketConfigXml trait (the 39 byte goldens pass with unchanged fixtures); the legacy s3s edge converts once in storage/ecfs.rs through storage/s3_compat/replication.rs; the scanner data usage cache keeps its MessagePack bytes through a serde mirror pinned by bytes recorded from s3s 5761ddfe. rustfs-gateway-types 0.32.0 is pinned once (rev 772d3385). Refs rustfs/backlog#2745.

* test: use S3 serializer for list response tests
2026-10-16 06:19:32 +08:00
overtrue cbe38a120f Merge main into gateway/integration
Merge frozen main acd26afd8b and include
compatibility fixes and measured Darwin/Linux membership corrections
across six files.

Validation on candidate b4c3ab560a:
- All 20 local gates passed.
- Native smoke: 218 passed. Native full: 661 passed.
- Full passed on the same-head failed-only rerun; the initial failure
  remains recorded.
- Gateway remains report-only: 38 passed, 180 failed, under the existing
  exemption.

CI: https://github.com/rustfs/rustfs/actions/runs/38285788334
2026-10-13 03:26:16 +08:00
overtrue db0828aa1c test(ecstore): isolate rollback guard loss fixtures
Run the three rollback metadata-cleanup cases in separate test processes so reused /proc/self/fd aliases from earlier cases cannot invalidate their creation witnesses. Keep the original fixture, guards, assertions and io_uring test selection, and verify that each child runs the intended case successfully.

Update the generated test-count baseline for the added macro arm.

Validation: cargo check -p rustfs-ecstore --tests; cargo fmt --all --check; all 19 quick-check commands, including make script-tests and the two Python checks. Linux io_uring acceptance and fault controls remain pending.
2026-10-12 00:42:51 +08:00
overtrue 3ca889b099 fix(ecstore): box nested bucket rollback state
Keep the guarded rollback future behind a heap allocation so bucket creation does not store its nested state inline. The rollback body, all three fence references and pending-proof revocation order are unchanged.

The same three original Linux tests overflowed on both e611 CI attempts. Local format, ecstore test compilation and all 19 quick-check steps pass; the original Linux feature lanes still need to validate this candidate.
2026-10-11 20:32:00 +08:00
overtrue e6112c526f fix(ecstore): retain creation fences through rollback
Keep lifecycle, metadata-transaction, and namespace guards around rollback and metadata cleanup. Preserve the original publication error when a guard is lost. Build the timestamp retry fixture through the real interrupted producer so it checks a valid pending creation.

Validated source tree: 4e8e207999.
Two independent source reviews found no blocking regression.

Validation (LANG=en_US.UTF-8):
- bash -c 'shellcheck --version && actionlint' — exit 0
- cargo fmt --all --check — exit 0
- ./scripts/check_unsafe_code_allowances.sh — exit 0
- ./scripts/check_layer_dependencies.sh — exit 0
- ./scripts/check_architecture_migration_rules.sh — exit 0
- ./scripts/check_logging_guardrails.sh — exit 0
- ./scripts/check_error_other_format_ratchet.sh — exit 0
- ./scripts/check_no_tokio_io_uring.sh — exit 0
- ./scripts/check_extension_schema_boundaries.sh — exit 0
- ./scripts/check_body_cache_whitelist.sh — exit 0
- ./scripts/check_s3s_footprint.sh — exit 0
- ./scripts/check_test_count_ratchet.sh — exit 0
- ./scripts/check_fips_wording.sh — exit 0
- ./scripts/check_embedded_secrets.sh — exit 0
- make script-tests — exit 0
- python3 ./scripts/test_object_generation_protocol_model.py — exit 0
- python3 ./scripts/check_test_wiring.py — exit 0
- ./scripts/check_no_planning_docs.sh — exit 0
- ./scripts/check_uring_lane_lib_only.sh — exit 0
- cargo nextest run --profile ci -p rustfs-ecstore --lib --locked --no-fail-fast — 5894 passed, 0 skipped
- cargo check -p rustfs-ecstore --all-targets --locked — exit 0
- cargo nextest run --profile ci -p rustfs --lib --locked --no-fail-fast -E 'test(=admin::handlers::table_catalog::tests::cold_strong_registration_completes_while_other_bucket_backup_waits) | test(=admin::handlers::table_catalog::tests::commit_backend_borrowed_put_preserves_owner_authorization_and_precondition) | test(=admin::handlers::table_catalog::tests::disable_table_bucket_preserves_objects_and_can_be_reenabled) | test(=admin::handlers::table_catalog::tests::load_table_handler_negotiates_vended_credentials_without_breaking_metadata_only_callers) | test(=admin::handlers::table_catalog::tests::namespace_property_handler_updates_object_backed_catalog_and_maps_errors) | test(=app::object::get::tests::execute_get_object_resumes_from_relocated_pool_without_splicing_body) | test(=table_catalog::store::tests::catalog_writer_loss_prevents_recovery_storage_commit)' — 7 passed
- cargo nextest run --profile ci -p rustfs --lib --locked --no-fail-fast -E 'test(unsupported_remote_disk_)' — 2 passed

Additional fault-injection evidence remains incomplete. Timestamp M1 reached its original assertion; its restore run is being collected. M2–M5 and the three planned rollback faults have not run. An automated risk check interrupted the timestamp proof task; no assertions or validation settings were weakened.
2026-10-11 16:16:31 +08:00
Chris c5967686f1 fix(ecstore): bound bucket creation stack usage (#8466) 2026-10-11 00:55:29 -07:00
Chris 859de565ad refactor(rpc): serve internode rpc bodies with the owned Body (#8441)
* test(rpc): pin the wire bytes of every internode rpc response body

rustfs/backlog#2747 batch 4, step 1 of 2, on the s3s bodies the rpc paths
serve today. Each response body builder in storage/rpc/http_service.rs is
served through hyper's HTTP/1.1 server and the raw bytes the peer reads are
compared with goldens captured from this commit: status, content-length or
chunked framing, chunk boundaries, the terminating chunk, HEAD, and the cut
connection a failure after data produces. The capability responses carry a
per-process proof, so their length header and payload are checked against
the body they encode instead.

The read_file_stream response builder moves into read_file_stream_response
so the test reaches the same code the handler runs; behaviour is unchanged.

* refactor(rpc): serve internode rpc bodies with the owned Body

rustfs/backlog#2747 batch 4, step 2 of 2. The internode rpc HTTP paths
build their responses with rustfs_s3_types::Body: Body::from_stream
replaces Body::from(StreamingBlob::wrap(..)) at the six stream sites, and
the in-memory and empty bodies keep their constructors. InternodeRpcService
converts its responses into the wrapped service's response body through
From<Body>, so the server keeps composing it with the s3s service as
before. The wire-byte goldens from the previous commit pass unchanged;
storage/rpc no longer names s3s.

* fix(rpc): simplify wire test response builder type

Name the boxed response builder used by the capability wire test so
all-target Clippy can check it without a type-complexity warning.

Validation (LANG=en_US.UTF-8, CARGO_INCREMENTAL=0):
- cargo clippy -p rustfs --all-targets -- -D warnings: exit 0.
- cargo nextest run --profile ci -p rustfs --lib -E 'test(/storage::rpc::/)': 279 tests run, 279 passed, 4842 skipped.
- All 19 quick-check commands, including script-tests, both Python scripts and cargo fmt --all --check: exit 0.
- Footprint baseline regenerated: files=194, s3_error_lines=1584, ecstore_files=36; test-count baseline regenerated: 52 crates, 21228 tests, 164 ignored.
- All 54 Cargo.toml files parse without duplicate keys.

No assertion was added or changed by this lint fix. R1-R8 are retained
in the existing PR verification record.

* fix(ecstore): resume fenced Object Lock creation intents

* fix(ecstore): bound creation retry futures on the stack

* fix(ecstore): require native proof for creation retries
2026-10-11 12:34:14 +08:00
overtrue 1f7cc593e7 test(ecstore): isolate tier-free recovery fixture startup
The fixture constructor can take the production recovery lock and wait for 60 seconds before this test installs its queue and seeds metadata. Disable its background recovery only while creating this fixture, then restore the environment before calling the existing production recovery entrypoint.

Only the existing test setup changes. Its production entrypoint, default lock parameters, original assertions, 30-second timeout, cancellation and cleanup remain intact. No tests or assertions were added. Production and store/init.rs are byte-identical to the parent.

Independent acceptance ran on merge tree 8ec358cb42ba9ebb3bcfeafcef048a3f0366355b; the runner header reports parent HEAD, while before/after index and source hashes confirm this candidate. Worker task branch d8055766b8 has the same tree.

Every command used LANG=en_US.UTF-8. Task compilation also used CARGO_BUILD_JOBS=1, CARGO_INCREMENTAL=0, CARGO_NET_OFFLINE=true, an independent target, and empty RUSTC_WRAPPER/RUSTC_WORKSPACE_WRAPPER to keep compilers in its owned process group without changing repository configuration.

Command:
shellcheck --version && actionlint
Exit 0; observed output excerpt:
version: 0.11.0
Log SHA-256: fc9eec16b4191d058a295813ca9afbe9d2b37b59975da860170726be162a3924

Command:
cargo fmt --all --check
Exit 0; observed output excerpt:
(no stdout or stderr)
Log SHA-256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

Command:
./scripts/check_unsafe_code_allowances.sh
Exit 0; observed output excerpt:
(no stdout or stderr)
Log SHA-256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

Command:
./scripts/check_layer_dependencies.sh
Exit 0; observed output excerpt:
Layer dependency guard passed (no new reverse dependencies/cycles).
Log SHA-256: 69c6cb1896c0586b28d1167380d3260d99d8017a4265c8df5a2348f289e93a12

Command:
./scripts/check_architecture_migration_rules.sh
Exit 0; observed output excerpt:
Architecture migration rules passed.
Log SHA-256: 28790e70d070607ed4de0704f292a65030be92ec224ef106d4b72fd8296f896e

Command:
./scripts/check_logging_guardrails.sh
Exit 0; observed output excerpt:
✅ Logging guardrails check passed
Log SHA-256: 7adf41abff8ed8f7617decf86498888ddde78937099dd9ce47f6d359dfc2d317

Command:
./scripts/check_error_other_format_ratchet.sh
Exit 0; observed output excerpt:
error(format!) ratchet passed (total call sites: 579).
Log SHA-256: 66172aef18f150ba0605af122e26ffcf91f6dd86f43e38b05e2e982ce8d783ee

Command:
./scripts/check_no_tokio_io_uring.sh
Exit 0; observed output excerpt:
(no stdout or stderr)
Log SHA-256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

Command:
./scripts/check_extension_schema_boundaries.sh
Exit 0; observed output excerpt:
Extension schema boundaries passed.
Log SHA-256: f13d0acd0e2ce5d9815241b12b4dab7d01cc11b9b65bcf6dfb7812c878532a51

Command:
./scripts/check_body_cache_whitelist.sh
Exit 0; observed output excerpt:
Body-cache whitelist guard passed.
Log SHA-256: 6b828847b0383950ee6eef9cd846d0c322c945e47efd4c7d1ad881178810d6bb

Command:
./scripts/check_s3s_footprint.sh
Exit 0; observed output excerpt:
s3s footprint OK: files importing s3s is 195 (baseline files=195)
s3s footprint OK: s3s s3_error! lines is 1584 (baseline s3_error_lines=1584)
s3s footprint OK: ecstore files referencing s3s is 36 (baseline ecstore_files=36)
✅ s3s footprint baseline check passed
Log SHA-256: 47f53d04ef645618563fe1f22abc64667f19078f1974feedcca46e4f2c016844

Command:
./scripts/check_test_count_ratchet.sh
Exit 0; observed output excerpt:
test-count ratchet OK: 52 crates, 21340 tests, 164 ignored match .config/test-count-baseline.txt
Log SHA-256: 618a5e330e933de0c6ff484546df401457024a88bb5986d11d588794e33dab60

Command:
./scripts/check_fips_wording.sh
Exit 0; observed output excerpt:
FIPS wording guard passed (README.md and CHANGELOG.md contain no forbidden claims).
KMS wording guard passed (crates/kms claims no Vault KV2 Transit wrapping).
Log SHA-256: 99221cc9979c480ec4f46f50d76c52b18c72fe499f12e15e4f4a22a1b8bfb140

Command:
./scripts/check_embedded_secrets.sh
Exit 0; observed output excerpt:
Embedded secret guard passed (no private key material or provider credential literal in the tree).
Log SHA-256: 437a3ceba32094d2901889965ad191e411aad32cdb18bf0b8659e818011c406b

Command:
make script-tests
Exit 0; observed output excerpt:
test_cli_requires_the_repository_default_branch (__main__.SelfTests.test_cli_requires_the_repository_default_branch) ... ok
test_config_rejects_duplicate_and_invalid_entries (__main__.SelfTests.test_config_rejects_duplicate_and_invalid_entries) ... ok
test_freshness_boundaries (__main__.SelfTests.test_freshness_boundaries) ... ok
test_grace_requires_two_successful_queries_with_no_history (__main__.SelfTests.test_grace_requires_two_successful_queries_with_no_history) ... ok
test_invalid_api_evidence_fails_closed (__main__.SelfTests.test_invalid_api_evidence_fails_closed) ... ok
test_report_retains_every_workflow (__main__.SelfTests.test_report_retains_every_workflow) ... ok
test_requests_filter_schedule_default_branch_and_success_on_server (__main__.SelfTests.test_requests_filter_schedule_default_branch_and_success_on_server) ... ok
Ran 10 tests in 0.042s
OK
Ran 30 tests in 20.627s
OK
Ran 10 tests in 2.268s
OK
Ran 33 tests in 0.127s
OK
Ran 7 tests in 0.002s
OK
Ran 7 tests in 0.001s
OK
Ran 4 tests in 0.108s
OK
Ran 3 tests in 0.564s
OK
PASS: cold stampede script self-test
Log SHA-256: 75fda2d069a373c63667b908baaf99e6591aab9099a739b0c4c22ceb6da8d68c

Command:
python3 ./scripts/test_object_generation_protocol_model.py
Exit 0; observed output excerpt:
test_an_unavailable_decision_quorum_cannot_choose_a_candidate (__main__.ObjectGenerationProtocolModel.test_an_unavailable_decision_quorum_cannot_choose_a_candidate) ... ok
test_counter_allocation_requires_persistence_and_is_consumed_once (__main__.ObjectGenerationProtocolModel.test_counter_allocation_requires_persistence_and_is_consumed_once) ... ok
test_counter_restart_burns_persisted_allocations_before_or_after_issue (__main__.ObjectGenerationProtocolModel.test_counter_restart_burns_persisted_allocations_before_or_after_issue) ... ok
test_crash_after_promise_before_reply_does_not_restore_an_older_ballot (__main__.ObjectGenerationProtocolModel.test_crash_after_promise_before_reply_does_not_restore_an_older_ballot) ... ok
test_duplicate_or_unconfigured_voters_cannot_form_a_quorum (__main__.ObjectGenerationProtocolModel.test_duplicate_or_unconfigured_voters_cannot_form_a_quorum) ... ok
test_empty_or_unobserved_minority_slot_can_retry_without_allocating_another_revision (__main__.ObjectGenerationProtocolModel.test_empty_or_unobserved_minority_slot_can_retry_without_allocating_another_revision) ... ok
test_equal_ballot_cannot_accept_different_bytes_and_lower_ballot_is_rejected (__main__.ObjectGenerationProtocolModel.test_equal_ballot_cannot_accept_different_bytes_and_lower_ballot_is_rejected) ... ok
test_forgetting_tombstone_authority_allows_an_aba_create (__main__.ObjectGenerationProtocolModel.test_forgetting_tombstone_authority_allows_an_aba_create) ... ok
test_issuing_before_persistence_reuses_a_ballot_after_restart (__main__.ObjectGenerationProtocolModel.test_issuing_before_persistence_reuses_a_ballot_after_restart) ... ok
test_lost_ack_and_restart_preserve_the_chosen_operation (__main__.ObjectGenerationProtocolModel.test_lost_ack_and_restart_preserve_the_chosen_operation) ... ok
test_minority_accepted_work_is_adopted_and_finished (__main__.ObjectGenerationProtocolModel.test_minority_accepted_work_is_adopted_and_finished) ... ok
test_omitting_highest_accepted_adoption_has_a_double_choice_counterexample (__main__.ObjectGenerationProtocolModel.test_omitting_highest_accepted_adoption_has_a_double_choice_counterexample) ... ok
test_quorum_choice_does_not_require_an_uncontacted_disk_to_learn_it (__main__.ObjectGenerationProtocolModel.test_quorum_choice_does_not_require_an_uncontacted_disk_to_learn_it) ... ok
test_recovery_failure_or_stale_expected_head_does_not_allocate_a_successor (__main__.ObjectGenerationProtocolModel.test_recovery_failure_or_stale_expected_head_does_not_allocate_a_successor) ... ok
test_replacing_voters_with_empty_state_can_choose_a_conflicting_value (__main__.ObjectGenerationProtocolModel.test_replacing_voters_with_empty_state_can_choose_a_conflicting_value) ... ok
test_skipping_predecessor_recovery_chooses_a_broken_revision_chain (__main__.ObjectGenerationProtocolModel.test_skipping_predecessor_recovery_chooses_a_broken_revision_chain) ... ok
test_stale_publication_rollback_and_gc_cannot_change_the_winner (__main__.ObjectGenerationProtocolModel.test_stale_publication_rollback_and_gc_cannot_change_the_winner) ... ok
test_superseded_operation_identity_cannot_be_reused_for_different_input (__main__.ObjectGenerationProtocolModel.test_superseded_operation_identity_cannot_be_reused_for_different_input) ... ok
test_superseded_operation_retry_returns_its_original_successor (__main__.ObjectGenerationProtocolModel.test_superseded_operation_retry_returns_its_original_successor) ... ok
test_tombstone_and_null_marker_payloads_do_not_reset_authority_revision (__main__.ObjectGenerationProtocolModel.test_tombstone_and_null_marker_payloads_do_not_reset_authority_revision) ... ok
test_tombstone_rejects_a_delayed_never_present_create (__main__.ObjectGenerationProtocolModel.test_tombstone_rejects_a_delayed_never_present_create) ... ok
test_unlearned_predecessor_is_recovered_before_the_second_slot (__main__.ObjectGenerationProtocolModel.test_unlearned_predecessor_is_recovered_before_the_second_slot) ... ok
Ran 24 tests in 0.929s
OK
Log SHA-256: 0b4be926efb0609d497ae58930340153f01957b606206cbbdb3e0d47eb29d3a2

Command:
python3 ./scripts/check_test_wiring.py
Exit 0; observed output excerpt:
OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and Quick Checks are wired
Log SHA-256: 21f676f458451525535ab0ef73d6a9c0d9774e16d76a4d19e396ad34936dd39c

Command:
./scripts/check_no_planning_docs.sh
Exit 0; observed output excerpt:
(no stdout or stderr)
Log SHA-256: e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855

Command:
./scripts/check_uring_lane_lib_only.sh
Exit 0; observed output excerpt:
OK: no 'uring_'-matching test functions under crates/ecstore/tests; --lib is safe
Log SHA-256: 43f3f851e27ec2ab9e34d6b05fefdede38af992699c473904bfe6f53a46d52e3

Command:
cargo nextest run --profile ci -p rustfs-ecstore --lib --features rio-v2 -E 'test(=bucket::lifecycle::bucket_lifecycle_ops::tests::tier_free_version_recovery_production_entrypoint_enqueues_seeded_item)' --no-fail-fast
Exit 0; wall 234.399s.
Observed output:
    Finished `test` profile [unoptimized + debuginfo] target(s) in 3m 49s
    Starting 1 test across 1 binary (5871 tests skipped)
  SETUP PASS [   0.006s] ecstore-base-stack: sh -c 'echo RUST_MIN_STACK=4194304 >> "$NEXTEST_ENV"'
        PASS [   0.440s] rustfs-ecstore bucket::lifecycle::bucket_lifecycle_ops::tests::tier_free_version_recovery_production_entrypoint_enqueues_seeded_item
     Summary [   0.586s] 1 test run: 1 passed, 5871 skipped
Log SHA-256: 34e973f86e8dc2102c1a0e483e0ac802ce7df3d1b40a603932f62c0805c4de27

Command:
cargo nextest run --profile ci -p rustfs-ecstore --lib --features rio-v2 -E 'test(=bucket::lifecycle::bucket_lifecycle_ops::tests::tier_free_version_recovery_eight_coordinators_hold_one_leader_and_take_over) | test(=store::bucket::tests::creation_commit_backfill_runs_under_the_config_write_fence) | test(=store::bucket::tests::orphaned_bucket_recovery_requires_every_volume_absent_and_is_retryable)' --no-fail-fast
Exit 0; wall 4.010s.
Observed output:
    Finished `test` profile [unoptimized + debuginfo] target(s) in 0.43s
    Starting 3 tests across 1 binary (5869 tests skipped)
  SETUP PASS [   0.005s] ecstore-base-stack: sh -c 'echo RUST_MIN_STACK=4194304 >> "$NEXTEST_ENV"'
        PASS [   0.068s] rustfs-ecstore bucket::lifecycle::bucket_lifecycle_ops::tests::tier_free_version_recovery_eight_coordinators_hold_one_leader_and_take_over
        PASS [   0.637s] rustfs-ecstore store::bucket::tests::creation_commit_backfill_runs_under_the_config_write_fence
        PASS [   0.846s] rustfs-ecstore store::bucket::tests::orphaned_bucket_recovery_requires_every_volume_absent_and_is_retryable
     Summary [   0.987s] 3 tests run: 3 passed, 5869 skipped
Log SHA-256: b5da2a927ff18ca9aa217762f7186a6d4371bd9e8feedd7b77940e10b64dffac

Command:
cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
Exit 0; wall 1084.508s.
Observed output:
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 18m 03s
Log SHA-256: 28f98be7970faaa71cd118da1da4e6e2e3d23b9dda7e6449eaa7a37e741260b5

Compiled mutation command (unchanged for each red and restored run):
cargo nextest run --profile ci -p rustfs-ecstore --lib --features rio-v2 -E 'test(=bucket::lifecycle::bucket_lifecycle_ops::tests::tier_free_version_recovery_production_entrypoint_enqueues_seeded_item)' --no-fail-fast

M1: reverse only this test-local constructor flag from false to true. The mutant compiled, then failed the original recovered-item queue assertion after 30.481s. Restoring the entire observed candidate passed in 0.460s. Red log 55bf3b1535529ac0bcfb8071f9b56f9f40631007a85c315367e7e93786d65293; restored log 204d5fbe4574a7917f5e740a795d864fe28a7e7fb5e33d27b0a9cf68e2f17f52.

M2: replace only the actual awaited enqueue with false, consuming its original inputs. The mutant compiled, then failed the same original queue assertion after 30.516s. Restoring the entire observed candidate passed in 0.457s. Red log 89186330413215e231e2b06bf013cc31097ea8f4a106d91421a36d6c8986a7fd; restored log e848b42d2302401afa5dbcd9e37bf76d2e56a96ab6de5a488513b997f53c5e9b.

All temporary observation and mutation code was removed before final acceptance. Historical baseline and diagnostic failures remain recorded; they are not mutation credit. The separate Object Lock intent-retry diagnostic still fails, and these native macOS checks do not establish Linux CI recovery.
2026-10-09 19:29:50 +08:00
overtrue 1d5a50f094 fix: remove redundant pending closure in disk test
Pass the capture-free pending future factory as a function item in the existing slow-operation test. This fixes the redundant_closure error at disk_store.rs:2914 that stopped strict Clippy before the feature tests could run.

The single-line test edit preserves the factory's result, warning and timeout behavior. Reversing the replacement reproduces the parent file byte for byte. All 220 assertion lines and the production prefix are unchanged; no tests, assertions, guards, profiles or baselines were added or relaxed.

Original Linux CI evidence (run 37888552268 at 53245be9f):
- RIO job113685483992: strict Clippy exit101, redundant closure at disk_store.rs:2914; feature tests skipped.
- Workspace Test and Lint job113685484058: same strict Clippy exit101; nextest and doc tests skipped.
These are lint failures, not runtime mutation kills or known flakes.

Native macOS arm64 validation on tree7a5bf47e7533b29479bc85207416102f1337c566, LANG=en_US.UTF-8, one build job, CARGO_INCREMENTAL=0 and offline dependencies:

shellcheck --version && actionlint -> exit 0
cargo fmt --all --check -> exit 0
./scripts/check_unsafe_code_allowances.sh -> exit 0
./scripts/check_layer_dependencies.sh -> exit 0
./scripts/check_architecture_migration_rules.sh -> exit 0
./scripts/check_logging_guardrails.sh -> exit 0
./scripts/check_error_other_format_ratchet.sh -> exit 0
./scripts/check_no_tokio_io_uring.sh -> exit 0
./scripts/check_extension_schema_boundaries.sh -> exit 0
./scripts/check_body_cache_whitelist.sh -> exit 0
./scripts/check_s3s_footprint.sh -> exit 0
./scripts/check_test_count_ratchet.sh -> exit 0
./scripts/check_fips_wording.sh -> exit 0
./scripts/check_embedded_secrets.sh -> exit 0
make script-tests -> exit 0
python3 ./scripts/test_object_generation_protocol_model.py -> exit 0
python3 ./scripts/check_test_wiring.py -> exit 0
./scripts/check_no_planning_docs.sh -> exit 0
./scripts/check_uring_lane_lib_only.sh -> exit 0
All 19 checks: exit0, 312.161s.

$ cargo clippy -p rustfs -p rustfs-ecstore --all-targets --features rio-v2 -- -D warnings
Finished `dev` profile [unoptimized + debuginfo] target(s) in 20m 11s
exit0, 1213.210s total.

$ cargo nextest run --profile ci -p rustfs-ecstore --features rio-v2 slow_disk_operation_warns_once_while_pending_and_preserves_result --no-fail-fast
Finished `test` profile [unoptimized + debuginfo] target(s) in 4m 34s
PASS [0.026s] rustfs-ecstore disk::disk_store::tests::slow_disk_operation_warns_once_while_pending_and_preserves_result
Summary [   0.170s] 1 test run: 1 passed, 5943 skipped
exit0, 281.920s total; 5943 existing filter skips.

warning: linker stderr: ld: __eh_frame section too large (max 16MB) to encode dwarf unwind offsets in compact unwind table, performance of exception handling might be affected
This linker warning is retained; flags, profiles and allowances are unchanged.

54 tracked Cargo.toml files parsed without duplicate keys. git diff --check passed. No new assertions were introduced, so no new runtime mutation is claimed. Native compilation and the existing helper test do not substitute for actual Linux RIO execution; the new integration CI run still needs to recover before the RPC merge.

Preservation and output hashes:
parent file SHA256: 8e59ee745e3aaaa22848b943d3b10f6353b515956282cde1f3af2c394200c9ac
final file SHA256: b69ba3237061b161448ae8b62b615885d5770b98a5b4a2479bcfbd70f1bd1f17
unchanged production prefix SHA256: 7903a6e288252c420231242d7b58a25fa2a7140ea3dd430cbe7f0dde11369dcb
RIO Linux failure log SHA256:421addb9868fd88e4109a922311e54f91f0dfd70862f8ba7fcaba193482ed90b
Workspace Linux failure log SHA256:e0aea8d0f014c022252a9534968e083c98cfc1fd760da445ecd04ec3f522e26a
19 output SHA256:35a54f28d707956ed1edb29d492c4dfe8a9fe753d103b111b5a2f1b5c41c2b98
RIO Clippy output SHA256:d6046e3868f2b3992fe11cdd04e9e98eb50a87d1234d9a70dbfeb154ff61638d
slow-operation output SHA256:605b7b258d2fd09c6ef2ab2ffc90d433d0509480f1e45125d20c823759e83608

Refs rustfs/backlog#2734.
2026-10-09 14:44:39 +08:00
overtrue 53245be9f4 fix: repair integration checks and IPC test readiness
Wait for the RPC/locks test sockets to pass the existing owner and mode
checks inside the original 5s readiness window and 10ms poll interval.

Use the existing admin S3 carrier for the new recovery and quota error
calls. Regenerate the test-count baseline for main's 118 existing tests
across seven crates. Pair the private pagination helper's raw-scan inputs
and remove the redundant Copy clone from the retirement event.

Keep the original assertions, responses, authorization, locks and storage
operations. The pagination body and raw-scan expressions at all five calls
are byte-identical; reinserting only the Copy clone restores the entire
base heal file. All ignored baseline values and s3s counts stay unchanged.

Validation base: c47432e353
Validation tree: f081084bc5
Patch SHA-256: 2dcf0028691bfb2772edfb38e262829b04f4151832dcce823c9dd9f7434b52de
Trace production-prefix SHA-256: fd994f218a21429d62866f4efb8eee1ee57b58f4a51865aa35d05a54ef1ccb45

All commands used LANG=en_US.UTF-8, CARGO_BUILD_JOBS=1,
CARGO_INCREMENTAL=0 and CARGO_NET_OFFLINE=true in one exclusive target.
Every gate retained the same tree and six source hashes before and after.
54 Cargo.toml files parse without duplicate keys.

cargo clippy -p rustfs --all-targets -- -D warnings
    Finished `dev` profile [unoptimized + debuginfo] target(s) in 12m 50s
Exit: 0. Wall time: 770.855s.

The original 19 quick-check steps:
shellcheck --version && actionlint -> exit 0
cargo fmt --all --check -> exit 0
./scripts/check_unsafe_code_allowances.sh -> exit 0
./scripts/check_layer_dependencies.sh -> exit 0
./scripts/check_architecture_migration_rules.sh -> exit 0
./scripts/check_logging_guardrails.sh -> exit 0
./scripts/check_error_other_format_ratchet.sh -> exit 0
./scripts/check_no_tokio_io_uring.sh -> exit 0
./scripts/check_extension_schema_boundaries.sh -> exit 0
./scripts/check_body_cache_whitelist.sh -> exit 0
./scripts/check_s3s_footprint.sh -> exit 0
./scripts/check_test_count_ratchet.sh -> exit 0
./scripts/check_fips_wording.sh -> exit 0
./scripts/check_embedded_secrets.sh -> exit 0
make script-tests -> exit 0
python3 ./scripts/test_object_generation_protocol_model.py -> exit 0
python3 ./scripts/check_test_wiring.py -> exit 0
./scripts/check_no_planning_docs.sh -> exit 0
./scripts/check_uring_lane_lib_only.sh -> exit 0
Wall time: 286.337s.

cargo nextest run --profile ci -p rustfs --no-fail-fast
     Summary [ 217.017s] 5382 tests run: 5382 passed (4 leaky), 40 skipped
Exit: 0. Wall time: 1324.414s.

cargo nextest run --profile ci -p rustfs-s3-types -p rustfs-protocols --features rustfs-protocols/swift --no-fail-fast
     Summary [   2.189s] 543 tests run: 543 passed, 0 skipped
Exit: 0. Wall time: 123.192s.

cargo nextest run --profile ci -p rustfs-ecstore list_objects_paginate --no-fail-fast
     Summary [   0.170s] 8 tests run: 8 passed, 5921 skipped
Exit: 0. Wall time: 261.324s.

cargo nextest run --profile ci -p rustfs-heal durable_mrf_anchor_retires_only_after_exact_bucket_generation_is_deleted --no-fail-fast
     Summary [   0.805s] 1 test run: 1 passed, 835 skipped
Exit: 0. Wall time: 199.801s.

Baseline regeneration:
./scripts/check_test_count_ratchet.sh --update
wrote .config/test-count-baseline.txt: 52 crates, 21340 tests, 164 ignored
Test-only increases: ecstore48, heal3, iam14, io-metrics2, obs3, policy14,
rustfs34. Existing ignored values and all other rows are byte-identical.
s3s footprint OK: files importing s3s is 195 (baseline files=195)
s3s footprint OK: s3s s3_error! lines is 1584 (baseline s3_error_lines=1584)
s3s footprint OK: ecstore files referencing s3s is 36 (baseline ecstore_files=36)

Historical runtime red controls on f622 (same retained two assertions):
cargo nextest run --profile ci -p rustfs --lib -E 'test(=connect::diagnostics::trace_runtime::tests::local_top_rpc_reads_service_http_completions_and_signs_offline) | test(=connect::diagnostics::trace_runtime::tests::local_top_locks_reads_service_manager_and_signs_offline)' --no-fail-fast
Original path readiness with a temporary 500ms bind/seal gap:
     Summary [   0.863s] 2 tests run: 0 passed, 2 failed, 5113 skipped
Exit: 100.
Sealed readiness in that same gap:
     Summary [   5.621s] 2 tests run: 2 passed, 5113 skipped
Exit: 0.
Compiled M1: replace the helper predicate with path.exists():
     Summary [   0.870s] 2 tests run: 0 passed, 2 failed, 5113 skipped
Exit: 100.
Restore the sealed predicate byte-for-byte:
     Summary [   5.640s] 2 tests run: 2 passed, 5113 skipped
Exit: 0.
Both reds observe mode755 with owner_matches=true and fail at the original
RPC/locks unwraps with StateSecurity. M1 failure locations: trace_runtime.rs
2547:29 and 2713:29. No new assertions were introduced.

cargo nextest run --profile ci -p rustfs --lib -E 'test(connect::diagnostics::trace_runtime::tests::)' --no-fail-fast
     Summary [  20.748s] 28 tests run: 28 passed (1 leaky), 5087 skipped
Exit: 0. This security sweep is historical; the full final unit gate above
covers the actual c474 candidate.
The first disk-stopped compile ran no tests and is excluded from red credit.
All temporary seal instrumentation is removed from the final source.

Final unit cleanup markers: 4 leaky, 40 original skips (exit0). The ld
__eh_frame linker warning also appears verbatim in the original f622 run
and the earlier c474 run; there are no new source warnings.
2026-10-09 13:09:50 +08:00
rustfs-cla-assistant[bot] c47432e353 Merge remote-tracking branch 'origin/main' into gateway/integration 2026-10-09 02:20:19 +00:00
overtrue f622c1bce0 fix(test): compare legacy body wire parity as raw bytes
Lossy UTF-8 comparison concealed a changed binary response byte.
Compare the already captured Vec<u8> values directly.

Validation:
- Binary mutant: migrated payload 0x84 vs legacy 0x83; old comparator
  passed 1/1, raw comparator failed 0/1 (exit 100); restored green.
- Non-Swift body-drain mutant: untouched-body delegation test failed
  with empty vs payload (exit 100); restored handler without a diff.
- cargo nextest run --profile ci -p rustfs-s3-types -p rustfs-protocols
  --features rustfs-protocols/swift --no-fail-fast:
  543 tests run, 543 passed, 0 skipped after both mutants restored.
- cargo fmt --all --check and git diff --check passed.

Orchestrator acceptance (LANG=en_US.UTF-8):
cargo nextest run --profile ci -p rustfs-s3-types -p rustfs-protocols --features rustfs-protocols/swift --no-fail-fast
Summary [2.445s] 543 tests run: 543 passed, 0 skipped.

All shared quick checks on the final code tree:
shellcheck --version && actionlint -> exit 0
cargo fmt --all --check -> exit 0
./scripts/check_unsafe_code_allowances.sh -> exit 0
./scripts/check_layer_dependencies.sh -> exit 0
./scripts/check_architecture_migration_rules.sh -> exit 0
./scripts/check_logging_guardrails.sh -> exit 0
./scripts/check_error_other_format_ratchet.sh -> exit 0
./scripts/check_no_tokio_io_uring.sh -> exit 0
./scripts/check_extension_schema_boundaries.sh -> exit 0
./scripts/check_body_cache_whitelist.sh -> exit 0
./scripts/check_s3s_footprint.sh -> exit 0
./scripts/check_test_count_ratchet.sh -> exit 0
./scripts/check_fips_wording.sh -> exit 0
./scripts/check_embedded_secrets.sh -> exit 0
make script-tests -> exit 0
python3 ./scripts/test_object_generation_protocol_model.py -> exit 0
python3 ./scripts/check_test_wiring.py -> exit 0
./scripts/check_no_planning_docs.sh -> exit 0
./scripts/check_uring_lane_lib_only.sh -> exit 0

Final independent acceptance after the drive-test CI repair
Parent: a99d8f4f89
Tested code tree: 93cbd96bf0

Tested tree: 075ae5161c2b62a96812bc1e3c904a7663b4f2f8
shellcheck --version && actionlint -> exit 0
cargo fmt --all --check -> exit 0
./scripts/check_unsafe_code_allowances.sh -> exit 0
./scripts/check_layer_dependencies.sh -> exit 0
./scripts/check_architecture_migration_rules.sh -> exit 0
./scripts/check_logging_guardrails.sh -> exit 0
./scripts/check_error_other_format_ratchet.sh -> exit 0
./scripts/check_no_tokio_io_uring.sh -> exit 0
./scripts/check_extension_schema_boundaries.sh -> exit 0
./scripts/check_body_cache_whitelist.sh -> exit 0
./scripts/check_s3s_footprint.sh -> exit 0
./scripts/check_test_count_ratchet.sh -> exit 0
./scripts/check_fips_wording.sh -> exit 0
./scripts/check_embedded_secrets.sh -> exit 0
make script-tests -> exit 0
python3 ./scripts/test_object_generation_protocol_model.py -> exit 0
python3 ./scripts/check_test_wiring.py -> exit 0
./scripts/check_no_planning_docs.sh -> exit 0
./scripts/check_uring_lane_lib_only.sh -> exit 0

$ LANG=en_US.UTF-8 CARGO_BUILD_JOBS=1 CARGO_INCREMENTAL=0 cargo nextest run --profile ci -p rustfs-s3-types -p rustfs-protocols --features rustfs-protocols/swift --no-fail-fast
    Finished `test` profile [unoptimized + debuginfo] target(s) in 10m 08s
 Nextest run ID 5c7ff102-ec6d-479a-86e0-810859b28bbf with nextest profile: ci
    Starting 543 tests across 8 binaries
     Summary [   6.729s] 543 tests run: 543 passed, 0 skipped
exit_code=0

The only source change is the raw-byte wire comparator. Cargo manifests and the ratchet baselines remain unchanged from the independently validated parent. The invalid-byte and consumed-request mutations remain recorded above; their implementation changes were restored.
2026-10-09 08:42:46 +08:00
overtrue a99d8f4f89 test(connect): restore one-second drive request budgets
Apply the exact test-only repair already landed on main in 5972a09cc3. The filesystem probe and production CLI success test request one second and keep their existing one-second duration and parameter checks. All outcome, I/O, error, cleanup and resource-limit assertions remain.

Scope: rustfs/tests/connect_perf_drive.rs only. Patch-id: cb0a1dfc8f68a67727525adea7458e2cee7d09f8.

Validation environment: LANG=en_US.UTF-8, CARGO_BUILD_JOBS=1, CARGO_INCREMENTAL=0, exclusive APFS-cloned CARGO_TARGET_DIR.

Quick Checks: every command from .github/actions/quick-checks/action.yml, with captured output and exit status below.

$ shellcheck --version && actionlint
ShellCheck - shell script analysis tool
version: 0.11.0
license: GNU General Public License, version 3
website: https://www.shellcheck.net
exit_code=0

$ cargo fmt --all --check
exit_code=0

$ ./scripts/check_unsafe_code_allowances.sh
exit_code=0

$ ./scripts/check_layer_dependencies.sh
Layer dependency guard passed (no new reverse dependencies/cycles).
exit_code=0

$ ./scripts/check_architecture_migration_rules.sh
Architecture migration rules passed.
exit_code=0

$ ./scripts/check_logging_guardrails.sh
✅ Logging guardrails check passed
exit_code=0

$ ./scripts/check_error_other_format_ratchet.sh
error(format!) ratchet passed (total call sites: 582).
exit_code=0

$ ./scripts/check_no_tokio_io_uring.sh
exit_code=0

$ ./scripts/check_extension_schema_boundaries.sh
Extension schema boundaries passed.
exit_code=0

$ ./scripts/check_body_cache_whitelist.sh
Body-cache whitelist guard passed.
exit_code=0

$ ./scripts/check_s3s_footprint.sh
s3s footprint OK: files importing s3s is 195 (baseline files=195)
s3s footprint OK: s3s s3_error! lines is 1584 (baseline s3_error_lines=1584)
s3s footprint OK: ecstore files referencing s3s is 36 (baseline ecstore_files=36)
✅ s3s footprint baseline check passed
exit_code=0

$ ./scripts/check_test_count_ratchet.sh
test-count ratchet OK: 52 crates, 21222 tests, 164 ignored match .config/test-count-baseline.txt
exit_code=0

$ ./scripts/check_fips_wording.sh
FIPS wording guard passed (README.md and CHANGELOG.md contain no forbidden claims).
KMS wording guard passed (crates/kms claims no Vault KV2 Transit wrapping).
exit_code=0

$ ./scripts/check_embedded_secrets.sh
Embedded secret guard passed (no private key material or provider credential literal in the tree).
exit_code=0

$ make script-tests
Running script tests...
entrypoint credential policy tests passed
internode grpc A/B bench env tests passed
object batch benchmark enhanced tests passed
PASS package service script contract
hotpath warp A/B gate tests passed
hotpath warp ABBA tests passed
Scanner validation artifacts written to /var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp.1pFe6Kl1Yu/out
Scanner validation artifacts written to /var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp.1pFe6Kl1Yu/out-missing-pid
.............................
----------------------------------------------------------------------
Ran 29 tests in 181.286s

OK
{"decision":"verified","pending_gates":["G01","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G02"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"R-E"}
{"decision":"verified","pending_gates":["G01","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G02"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"R-E"}
{"decision":"verified","pending_gates":["G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G01"}
{"decision":"verified","pending_gates":["G01","G02","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G03"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"R-L"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G14"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G14"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G10"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"P1"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"P2"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"P3"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G05"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G06"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"R-D"}
Status-and-outcome release descriptor verified: /private/var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp8rwn1pe0/descriptor/release-bundle-status-outcome.json
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G05"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G06"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"R-D"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G07"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G09","G10","G11","G12","G13","G14","P1","P2","P3","P4","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"G08"}
{"decision":"verified","pending_gates":["G01","G02","G03","G04","G05","G06","G07","G08","G09","G10","G11","G12","G13","G14","P1","P2","P3","R-D","R-E","R-L"],"pending_lanes":["authority-coverage","checkpoint-and-crash","ec8-4-multiset","maintenance-producers","mixed-version-rollback","mrf-responsibility","scheduler-pressure","status-and-outcome"],"rejected_errors":[],"rejected_gate":null,"release_approved":false,"release_schema_capable":true,"schema":1,"verified_gate":"P4"}
exact 1MiB handoff ABBA tests passed
pinned paired ABBA bench tests passed
==> /var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp.mYmuI01hNq/bin/path_containment (-max_total_time=7, -seed=123456789, -artifact_prefix=/var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp.mYmuI01hNq/explicit-artifacts/path_containment/, corpus=/private/tmp/gw-orchestrator-2734/ci-drive/fuzz/corpus/path_containment)
-seed=123456789
target=path_containment
seed=123456789
max_total_time=7
git_revision=4d15b816236630b2c42f321ffc9ed3654edc94e3
git_dirty=true
runner_mode=prebuilt
==> /var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp.mYmuI01hNq/bin/path_containment (-max_total_time=1, -seed=2322617706, -artifact_prefix=/var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp.mYmuI01hNq/auto-artifacts/path_containment/, corpus=/private/tmp/gw-orchestrator-2734/ci-drive/fuzz/corpus/path_containment)
-seed=2322617706
==> cargo +nightly fuzz run path_containment (-max_total_time=9, -seed=987654321, -artifact_prefix=/var/folders/0w/s4wxsn2n4w9b6f3cd4xrz53w0000gn/T/tmp.mYmuI01hNq/cargo-artifacts/path_containment/)
+nightly
fuzz
run
path_containment
-seed=987654321
seed=987654321
runner_mode=cargo-fuzz
fuzz runner seed manifest tests passed
✅ scripts/python_bin.sh resolver checks passed
self-test ok: clean fixture with every exemption present passes
self-test ok: all 11 pattern families fire
self-test ok: moving a pinned source is reported
Embedded secret guard self-test passed.
self-test ok: p1-s3s-import: 'use s3s::s3_error;' counts every line, the rustfs_s3_types module's too (s3_error_lines=3)
self-test ok: p2-s3s-mixed-tree: a multi-line s3s use tree naming s3_error counts (s3_error_lines=3)
self-test ok: p3-s3s-glob: a crate-root glob of s3s, bare or inside a tree, counts (s3_error_lines=5)
self-test ok: p4-macro-use: '#[macro_use] extern crate s3s' counts (s3_error_lines=3)
self-test ok: p5-qualified-s3s: a qualified s3s::s3_error! line counts in a cleared file (s3_error_lines=2)
self-test ok: p6-exact-pass: a baseline equal to the tree passes (exit 0)
self-test ok: n1-rustfs-import: 'use rustfs_s3_types::s3_error;' alone counts nothing (s3_error_lines=1)
self-test ok: n2-rustfs-mixed-tree: a rustfs_s3_types tree plus an s3s tree without the macro counts nothing (s3_error_lines=1)
self-test ok: n3-defines: the file defining the macro counts nothing (s3_error_lines=1)
self-test ok: n4-qualified-rustfs: a qualified rustfs_s3_types::s3_error! line counts nothing (s3_error_lines=1)
self-test ok: n5-renamed-call: a call through a renamed macro is not text-matched (out of scope) (s3_error_lines=1)
self-test ok: n6-comment-string: a comment or string mentioning s3_error! in a cleared file counts nothing (s3_error_lines=1)
self-test ok: n7-e2e-excluded: crates/e2e_test is excluded (s3_error_lines=1)
self-test ok: n8-super-glob: an unrecognised import form ('use super::*;') counts (s3_error_lines=3)
self-test ok: n9-facade: a crate-local re-export counts (s3_error_lines=2)
self-test ok: n10-renamed-import: a renamed rustfs_s3_types import does not clear s3_error! lines (s3_error_lines=2)
self-test ok: n11-commented-import: a commented-out rustfs_s3_types import does not clear (s3_error_lines=2)
self-test ok: n12-rustfs-glob: a glob import of rustfs_s3_types does not clear (s3_error_lines=2)
self-test ok: n13-comment-only: a comment mentioning s3_error! in a file with no import counts (s3_error_lines=2)
self-test ok: f1-baseline-missing: a missing baseline file fails (exit 1)
self-test ok: f2-key-missing: a baseline without s3_error_lines fails (exit 1)
self-test ok: f3-rg-fails: a failing classification search fails the guard (exit 1)
✅ s3s footprint self-test passed
test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs (__main__.SelfTests.test_core_gate_rejects_missing_ignored_filtered_and_corrupt_inputs) ... ok
test_e2e_requires_registration (__main__.SelfTests.test_e2e_requires_registration) ... ok
test_fuzz_runtime_uses_matrix_target (__main__.SelfTests.test_fuzz_runtime_uses_matrix_target) ... ok
test_ilm_lane_keeps_the_measured_cargo_build_budget (__main__.SelfTests.test_ilm_lane_keeps_the_measured_cargo_build_budget) ... ok
test_profile_listing_binds_platform_digest (__main__.SelfTests.test_profile_listing_binds_platform_digest) ... ok
test_profile_listing_enforces_selection (__main__.SelfTests.test_profile_listing_enforces_selection) ... ok
test_quick_checks_commands_propagate_failure (__main__.SelfTests.test_quick_checks_commands_propagate_failure) ... ok
test_quick_checks_rejects_caller_and_execution_bypasses (__main__.SelfTests.test_quick_checks_rejects_caller_and_execution_bypasses) ... ok
test_runner_readiness_fails_closed (__main__.SelfTests.test_runner_readiness_fails_closed) ... ok
test_s3_runner_rejects_unbounded_failure_locals (__main__.SelfTests.test_s3_runner_rejects_unbounded_failure_locals) ... ok
test_scanner_heal_begin_requires_embedded_source_provenance (__main__.SelfTests.test_scanner_heal_begin_requires_embedded_source_provenance) ... ok
test_scanner_heal_case_does_not_approve_pending_release (__main__.SelfTests.test_scanner_heal_case_does_not_approve_pending_release) ... ok
test_scanner_heal_case_lane_is_required_for_runner_profile (__main__.SelfTests.test_scanner_heal_case_lane_is_required_for_runner_profile) ... ok
test_scanner_heal_case_only_schema_cannot_approve_release (__main__.SelfTests.test_scanner_heal_case_only_schema_cannot_approve_release) ... ok
test_scanner_heal_crash_case_rejects_restart_oracle (__main__.SelfTests.test_scanner_heal_crash_case_rejects_restart_oracle) ... ok
test_scanner_heal_ec84_case_rejects_wrong_erasure_geometry (__main__.SelfTests.test_scanner_heal_ec84_case_rejects_wrong_erasure_geometry) ... ok
test_scanner_heal_finish_collects_oracles_from_registry (__main__.SelfTests.test_scanner_heal_finish_collects_oracles_from_registry) ... ok
test_scanner_heal_multipool_case_accepts_deferred_optional_outage_write (__main__.SelfTests.test_scanner_heal_multipool_case_accepts_deferred_optional_outage_write) ... ok
test_scanner_heal_pending_gate_cannot_map_to_implemented_lane (__main__.SelfTests.test_scanner_heal_pending_gate_cannot_map_to_implemented_lane) ... ok
test_scanner_heal_receipts_reject_reuse_and_missing_builds (__main__.SelfTests.test_scanner_heal_receipts_reject_reuse_and_missing_builds) ... ok
test_scanner_heal_rejects_boolean_fractional_and_out_of_geometry_integers (__main__.SelfTests.test_scanner_heal_rejects_boolean_fractional_and_out_of_geometry_integers) ... ok
test_scanner_heal_rejects_broken_execution_and_artifacts (__main__.SelfTests.test_scanner_heal_rejects_broken_execution_and_artifacts) ... ok
test_scanner_heal_rejects_copied_junit_and_wrong_suite_build (__main__.SelfTests.test_scanner_heal_rejects_copied_junit_and_wrong_suite_build) ... ok
test_scanner_heal_release_bundle_accepts_complete_measured_evidence (__main__.SelfTests.test_scanner_heal_release_bundle_accepts_complete_measured_evidence) ... ok
test_scanner_heal_release_bundle_assembler_copies_measured_artifacts (__main__.SelfTests.test_scanner_heal_release_bundle_assembler_copies_measured_artifacts) ... ok
test_scanner_heal_release_bundle_assembler_merges_measured_descriptors (__main__.SelfTests.test_scanner_heal_release_bundle_assembler_merges_measured_descriptors) ... ok
test_scanner_heal_release_bundle_assembler_rejects_duplicate_descriptor_gate (__main__.SelfTests.test_scanner_heal_release_bundle_assembler_rejects_duplicate_descriptor_gate) ... ok
test_scanner_heal_release_bundle_assembler_rejects_fixture_descriptor (__main__.SelfTests.test_scanner_heal_release_bundle_assembler_rejects_fixture_descriptor) ... ok
test_scanner_heal_release_bundle_assembler_rejects_hash_mismatch (__main__.SelfTests.test_scanner_heal_release_bundle_assembler_rejects_hash_mismatch) ... ok
test_scanner_heal_release_bundle_assembler_rejects_mixed_descriptor_revisions (__main__.SelfTests.test_scanner_heal_release_bundle_assembler_rejects_mixed_descriptor_revisions) ... ok
test_scanner_heal_release_bundle_enforces_topology_duration_profile_and_versions (__main__.SelfTests.test_scanner_heal_release_bundle_enforces_topology_duration_profile_and_versions) ... ok
test_scanner_heal_release_bundle_fixture_writer_is_not_release_evidence (__main__.SelfTests.test_scanner_heal_release_bundle_fixture_writer_is_not_release_evidence) ... ok
test_scanner_heal_release_bundle_gate_accepts_single_measured_gate (__main__.SelfTests.test_scanner_heal_release_bundle_gate_accepts_single_measured_gate) ... ok
test_scanner_heal_release_bundle_gate_rejects_missing_field (__main__.SelfTests.test_scanner_heal_release_bundle_gate_rejects_missing_field) ... ok
test_scanner_heal_release_bundle_gate_rejects_unknown_gate (__main__.SelfTests.test_scanner_heal_release_bundle_gate_rejects_unknown_gate) ... ok
test_scanner_heal_release_bundle_rejects_synthetic_or_missing_fields (__main__.SelfTests.test_scanner_heal_release_bundle_rejects_synthetic_or_missing_fields) ... ok
test_scanner_heal_release_bundle_requires_domain_evidence (__main__.SelfTests.test_scanner_heal_release_bundle_requires_domain_evidence) ... ok
test_scanner_heal_release_bundle_requires_field_provenance (__main__.SelfTests.test_scanner_heal_release_bundle_requires_field_provenance) ... ok
test_scanner_heal_release_bundle_requires_json_artifact_provenance (__main__.SelfTests.test_scanner_heal_release_bundle_requires_json_artifact_provenance) ... ok
test_scanner_heal_release_bundle_requires_mixed_version_field_roles (__main__.SelfTests.test_scanner_heal_release_bundle_requires_mixed_version_field_roles) ... ok
test_scanner_heal_release_bundle_requires_same_gate_measurement_window (__main__.SelfTests.test_scanner_heal_release_bundle_requires_same_gate_measurement_window) ... ok
test_scanner_heal_release_consumes_multiple_registry_oracles (__main__.SelfTests.test_scanner_heal_release_consumes_multiple_registry_oracles) ... ok
test_scanner_heal_release_matrix_lanes_remain_pending_without_real_evidence (__main__.SelfTests.test_scanner_heal_release_matrix_lanes_remain_pending_without_real_evidence) ... ok
test_scanner_heal_release_requirements_cannot_be_cleared (__main__.SelfTests.test_scanner_heal_release_requirements_cannot_be_cleared) ... ok
test_scanner_heal_release_status_rejects_focused_case_run (__main__.SelfTests.test_scanner_heal_release_status_rejects_focused_case_run) ... ok
test_scanner_heal_release_status_rejects_synthetic_case (__main__.SelfTests.test_scanner_heal_release_status_rejects_synthetic_case) ... ok
test_scanner_heal_required_evidence_fields_cannot_be_removed (__main__.SelfTests.test_scanner_heal_required_evidence_fields_cannot_be_removed) ... ok
test_scheduled_alerts_require_completion_watchdog (__main__.SelfTests.test_scheduled_alerts_require_completion_watchdog) ... ok
test_update_profile_selection_changes_only_requested_platform (__main__.SelfTests.test_update_profile_selection_changes_only_requested_platform) ... ok
test_validate_includes_quick_checks (__main__.SelfTests.test_validate_includes_quick_checks) ... ok
test_vault_tests_require_cross_process_serialization (__main__.SelfTests.test_vault_tests_require_cross_process_serialization) ... ok
test_workflow_readiness_requires_dependency_probe (__main__.SelfTests.test_workflow_readiness_requires_dependency_probe) ... ok

----------------------------------------------------------------------
Ran 52 tests in 14.301s

OK
e2e-full selection OK: 1 tests, sha256=29e30e6ed2ed38d9e5b0a7cdd9536abf5c719f7021a02b578ae1381563a3c273
...................................
----------------------------------------------------------------------
Ran 35 tests in 1.161s

OK
...................
----------------------------------------------------------------------
Ran 19 tests in 17.397s

OK
..........
----------------------------------------------------------------------
Ran 10 tests in 0.183s

OK
test_actual_selector_bootstrap_uses_base_policy_and_fails_closed (__main__.SelfTests.test_actual_selector_bootstrap_uses_base_policy_and_fails_closed) ... ok
test_documentation_paths_do_not_hide_build_or_fixture_changes (__main__.SelfTests.test_documentation_paths_do_not_hide_build_or_fixture_changes) ... ok
test_e2e_scope_keeps_production_and_shared_configuration_full (__main__.SelfTests.test_e2e_scope_keeps_production_and_shared_configuration_full) ... ok
test_e2e_shortcut_rejects_new_direct_renamed_and_target_dependencies (__main__.SelfTests.test_e2e_shortcut_rejects_new_direct_renamed_and_target_dependencies) ... ok
test_e2e_shortcut_traverses_implicit_path_dependencies (__main__.SelfTests.test_e2e_shortcut_traverses_implicit_path_dependencies) ... ok
test_event_contract_requires_complete_candidate_and_optional_lanes (__main__.SelfTests.test_event_contract_requires_complete_candidate_and_optional_lanes) ... ok
test_every_wrong_result_missing_job_or_selection_fails_closed (__main__.SelfTests.test_every_wrong_result_missing_job_or_selection_fails_closed) ... ok
test_full_e2e_gate_preserves_workflow_branch_and_event_scope (__main__.SelfTests.test_full_e2e_gate_preserves_workflow_branch_and_event_scope) ... ok
test_git_range_includes_deleted_source_and_rename_origins (__main__.SelfTests.test_git_range_includes_deleted_source_and_rename_origins) ... ok
test_job_ids_and_display_names_cannot_hide_validation (__main__.SelfTests.test_job_ids_and_display_names_cannot_hide_validation) ... ok
test_report_only_stack_lane_mirrors_e2e_smoke_outside_the_aggregate (__main__.SelfTests.test_report_only_stack_lane_mirrors_e2e_smoke_outside_the_aggregate) ... ok
test_repository_wiring_and_missing_dependency_regression (__main__.SelfTests.test_repository_wiring_and_missing_dependency_regression) ... ok
test_verify_command_preserves_failures (__main__.SelfTests.test_verify_command_preserves_failures) ... ok

----------------------------------------------------------------------
Ran 13 tests in 3.280s

OK
security coverage self-test passed
test_api_failures_preserve_other_evidence_and_never_enter_grace (__main__.SelfTests.test_api_failures_preserve_other_evidence_and_never_enter_grace) ... ok
test_attempt_outcome_does_not_replace_recent_success (__main__.SelfTests.test_attempt_outcome_does_not_replace_recent_success) ... ok
test_cancelled_attempt_cannot_refresh_expired_success (__main__.SelfTests.test_cancelled_attempt_cannot_refresh_expired_success) ... ok
test_cli_requires_the_repository_default_branch (__main__.SelfTests.test_cli_requires_the_repository_default_branch) ... ok
test_config_rejects_duplicate_and_invalid_entries (__main__.SelfTests.test_config_rejects_duplicate_and_invalid_entries) ... ok
test_freshness_boundaries (__main__.SelfTests.test_freshness_boundaries) ... ok
test_grace_requires_two_successful_queries_with_no_history (__main__.SelfTests.test_grace_requires_two_successful_queries_with_no_history) ... ok
test_invalid_api_evidence_fails_closed (__main__.SelfTests.test_invalid_api_evidence_fails_closed) ... ok
test_report_retains_every_workflow (__main__.SelfTests.test_report_retains_every_workflow) ... ok
test_requests_filter_schedule_default_branch_and_success_on_server (__main__.SelfTests.test_requests_filter_schedule_default_branch_and_success_on_server) ... ok

----------------------------------------------------------------------
Ran 10 tests in 0.051s

OK
..............................
----------------------------------------------------------------------
Ran 30 tests in 22.354s

OK
..........
----------------------------------------------------------------------
Ran 10 tests in 2.969s

OK
.................................
----------------------------------------------------------------------
Ran 33 tests in 0.160s

OK
Online functional runner labels: pf-testing
.......
----------------------------------------------------------------------
Ran 7 tests in 0.002s

OK
.......
----------------------------------------------------------------------
Ran 7 tests in 0.001s

OK
....
----------------------------------------------------------------------
Ran 4 tests in 0.128s

OK
...
----------------------------------------------------------------------
Ran 3 tests in 0.547s

OK
PASS: cold stampede script self-test
exit_code=0

$ python3 ./scripts/test_object_generation_protocol_model.py
test_a_recovery_barrier_cannot_choose_an_empty_value (__main__.ObjectGenerationProtocolModel.test_a_recovery_barrier_cannot_choose_an_empty_value) ... ok
test_all_bounded_two_proposer_interleavings_choose_at_most_one_value (__main__.ObjectGenerationProtocolModel.test_all_bounded_two_proposer_interleavings_choose_at_most_one_value) ... ok
test_an_unavailable_decision_quorum_cannot_choose_a_candidate (__main__.ObjectGenerationProtocolModel.test_an_unavailable_decision_quorum_cannot_choose_a_candidate) ... ok
test_counter_allocation_requires_persistence_and_is_consumed_once (__main__.ObjectGenerationProtocolModel.test_counter_allocation_requires_persistence_and_is_consumed_once) ... ok
test_counter_restart_burns_persisted_allocations_before_or_after_issue (__main__.ObjectGenerationProtocolModel.test_counter_restart_burns_persisted_allocations_before_or_after_issue) ... ok
test_crash_after_promise_before_reply_does_not_restore_an_older_ballot (__main__.ObjectGenerationProtocolModel.test_crash_after_promise_before_reply_does_not_restore_an_older_ballot) ... ok
test_duplicate_or_unconfigured_voters_cannot_form_a_quorum (__main__.ObjectGenerationProtocolModel.test_duplicate_or_unconfigured_voters_cannot_form_a_quorum) ... ok
test_empty_or_unobserved_minority_slot_can_retry_without_allocating_another_revision (__main__.ObjectGenerationProtocolModel.test_empty_or_unobserved_minority_slot_can_retry_without_allocating_another_revision) ... ok
test_equal_ballot_cannot_accept_different_bytes_and_lower_ballot_is_rejected (__main__.ObjectGenerationProtocolModel.test_equal_ballot_cannot_accept_different_bytes_and_lower_ballot_is_rejected) ... ok
test_forgetting_tombstone_authority_allows_an_aba_create (__main__.ObjectGenerationProtocolModel.test_forgetting_tombstone_authority_allows_an_aba_create) ... ok
test_issuing_before_persistence_reuses_a_ballot_after_restart (__main__.ObjectGenerationProtocolModel.test_issuing_before_persistence_reuses_a_ballot_after_restart) ... ok
test_lost_ack_and_restart_preserve_the_chosen_operation (__main__.ObjectGenerationProtocolModel.test_lost_ack_and_restart_preserve_the_chosen_operation) ... ok
test_minority_accepted_work_is_adopted_and_finished (__main__.ObjectGenerationProtocolModel.test_minority_accepted_work_is_adopted_and_finished) ... ok
test_omitting_highest_accepted_adoption_has_a_double_choice_counterexample (__main__.ObjectGenerationProtocolModel.test_omitting_highest_accepted_adoption_has_a_double_choice_counterexample) ... ok
test_quorum_choice_does_not_require_an_uncontacted_disk_to_learn_it (__main__.ObjectGenerationProtocolModel.test_quorum_choice_does_not_require_an_uncontacted_disk_to_learn_it) ... ok
test_recovery_failure_or_stale_expected_head_does_not_allocate_a_successor (__main__.ObjectGenerationProtocolModel.test_recovery_failure_or_stale_expected_head_does_not_allocate_a_successor) ... ok
test_replacing_voters_with_empty_state_can_choose_a_conflicting_value (__main__.ObjectGenerationProtocolModel.test_replacing_voters_with_empty_state_can_choose_a_conflicting_value) ... ok
test_skipping_predecessor_recovery_chooses_a_broken_revision_chain (__main__.ObjectGenerationProtocolModel.test_skipping_predecessor_recovery_chooses_a_broken_revision_chain) ... ok
test_stale_publication_rollback_and_gc_cannot_change_the_winner (__main__.ObjectGenerationProtocolModel.test_stale_publication_rollback_and_gc_cannot_change_the_winner) ... ok
test_superseded_operation_identity_cannot_be_reused_for_different_input (__main__.ObjectGenerationProtocolModel.test_superseded_operation_identity_cannot_be_reused_for_different_input) ... ok
test_superseded_operation_retry_returns_its_original_successor (__main__.ObjectGenerationProtocolModel.test_superseded_operation_retry_returns_its_original_successor) ... ok
test_tombstone_and_null_marker_payloads_do_not_reset_authority_revision (__main__.ObjectGenerationProtocolModel.test_tombstone_and_null_marker_payloads_do_not_reset_authority_revision) ... ok
test_tombstone_rejects_a_delayed_never_present_create (__main__.ObjectGenerationProtocolModel.test_tombstone_rejects_a_delayed_never_present_create) ... ok
test_unlearned_predecessor_is_recovered_before_the_second_slot (__main__.ObjectGenerationProtocolModel.test_unlearned_predecessor_is_recovered_before_the_second_slot) ... ok

----------------------------------------------------------------------
Ran 24 tests in 0.958s

OK
model: states=51023, transitions=167520, terminal=732, chosen_candidates=2
negative control: double-choice trace=((0, 'prepare', 0), (0, 'prepare', 1), (0, 'prepare', 2), (0, 'accept', 0), (0, 'accept', 1), (0, 'accept', 2), (1, 'prepare', 0), (1, 'prepare', 1), (1, 'prepare', 2), (1, 'accept', 0), (1, 'accept', 1), (1, 'accept', 2))
multi-slot round schedules: 640
exit_code=0

$ python3 ./scripts/check_test_wiring.py
OK: e2e modules, runner selection, fuzz matrices, profiles, scheduled alerts, and Quick Checks are wired
exit_code=0

$ ./scripts/check_no_planning_docs.sh
exit_code=0

$ ./scripts/check_uring_lane_lib_only.sh
OK: no 'uring_'-matching test functions under crates/ecstore/tests; --lib is safe
exit_code=0

Initial complete gate
$ cargo nextest run --profile ci -p rustfs --test connect_perf_drive --no-fail-fast
 Nextest run ID e07fc5b9-c6c8-4cdd-ae61-54839bf24437 with nextest profile: ci
    Starting 10 tests across 1 binary
        PASS [   0.016s] rustfs::connect_perf_drive cancellation_and_single_collector_leave_no_scratch
        PASS [   0.011s] rustfs::connect_perf_drive cleanup_failure_overrides_cancellation_and_remains_attributed
        PASS [   0.009s] rustfs::connect_perf_drive consent_and_resource_budgets_fail_before_scratch_io
        PASS [   0.009s] rustfs::connect_perf_drive invalid_roots_and_existing_scratch_fail_without_clobbering
        PASS [  16.950s] rustfs::connect_perf_drive production_cli_measures_and_signs_exact_binary_provenance
        PASS [   4.025s] rustfs::connect_perf_drive production_cli_rejects_missing_l1_consent_and_limits_before_artifacts
        PASS [   0.008s] rustfs::connect_perf_drive read_only_root_and_full_drive_errors_are_classified
        PASS [   0.015s] rustfs::connect_perf_drive real_local_write_read_is_measured_and_scratch_is_removed
        PASS [   0.007s] rustfs::connect_perf_drive runtime_timeout_and_full_drive_use_schema_valid_failed_reason
        PASS [   0.035s] rustfs::connect_perf_drive successful_result_has_signed_bounded_private_offline_export
────────────
     Summary [  21.087s] 10 tests run: 10 passed, 0 skipped
exit_code=0

Restored complete gate
$ cargo nextest run --profile ci -p rustfs --test connect_perf_drive --no-fail-fast
 Nextest run ID 233b7307-5361-4eb7-94d5-b384f17d2d6a with nextest profile: ci
    Starting 10 tests across 1 binary
        PASS [   0.009s] rustfs::connect_perf_drive cancellation_and_single_collector_leave_no_scratch
        PASS [   0.008s] rustfs::connect_perf_drive cleanup_failure_overrides_cancellation_and_remains_attributed
        PASS [   0.008s] rustfs::connect_perf_drive consent_and_resource_budgets_fail_before_scratch_io
        PASS [   0.008s] rustfs::connect_perf_drive invalid_roots_and_existing_scratch_fail_without_clobbering
        PASS [  19.518s] rustfs::connect_perf_drive production_cli_measures_and_signs_exact_binary_provenance
        PASS [   4.146s] rustfs::connect_perf_drive production_cli_rejects_missing_l1_consent_and_limits_before_artifacts
        PASS [   0.009s] rustfs::connect_perf_drive read_only_root_and_full_drive_errors_are_classified
        PASS [   0.018s] rustfs::connect_perf_drive real_local_write_read_is_measured_and_scratch_is_removed
        PASS [   0.009s] rustfs::connect_perf_drive runtime_timeout_and_full_drive_use_schema_valid_failed_reason
        PASS [   0.054s] rustfs::connect_perf_drive successful_result_has_signed_bounded_private_offline_export
────────────
     Summary [  23.787s] 10 tests run: 10 passed, 0 skipped
exit_code=0

Mutation M1: success_measurement reports max(actual elapsed, requested budget + 1), forcing 1001 ms for the one-second request. The retained duration assertion at rustfs/tests/connect_perf_drive.rs:105 fails.
$ cargo nextest run --profile ci -p rustfs --test connect_perf_drive --no-fail-fast -E 'test(real_local_write_read_is_measured_and_scratch_is_removed)'
info: experimental features enabled: setup-scripts
   Compiling rustfs v1.0.1 (/private/tmp/gw-orchestrator-2734/ci-drive/rustfs)
warning: linker stderr: ld: __eh_frame section too large (max 16MB) to encode dwarf unwind offsets in compact unwind table, performance of exception handling might be affected
  |
  = note: `#[warn(linker_messages)]` on by default

warning: `rustfs` (bin "rustfs") generated 1 warning
warning: `rustfs` (bin "rustfs-cli") generated 1 warning (1 duplicate)
    Finished `test` profile [unoptimized + debuginfo] target(s) in 7m 06s
────────────
 Nextest run ID 9f618777-7bf6-4cbc-8803-dd7f50f449ed with nextest profile: ci
    Starting 1 test across 1 binary (9 tests skipped)
        FAIL [   0.016s] rustfs::connect_perf_drive real_local_write_read_is_measured_and_scratch_is_removed
  stdout ───

    running 1 test
    test real_local_write_read_is_measured_and_scratch_is_removed ... FAILED

    failures:

    failures:
        real_local_write_read_is_measured_and_scratch_is_removed

    test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 9 filtered out; finished in 0.01s

  stderr ───

    thread 'real_local_write_read_is_measured_and_scratch_is_removed' (106986846) panicked at rustfs/tests/connect_perf_drive.rs:105:5:
    assertion failed: (1..=1_000).contains(&data.duration_millis)
    note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

────────────
     Summary [   0.017s] 1 test run: 0 passed, 1 failed, 9 skipped
        FAIL [   0.016s] rustfs::connect_perf_drive real_local_write_read_is_measured_and_scratch_is_removed
error: test run failed
exit_code=100

Mutation M2: target_parameters reports requested duration + 1. The in-process assertion at rustfs/tests/connect_perf_drive.rs:111 and CLI JSON assertion at rustfs/tests/connect_perf_drive.rs:527 both reject 1001 against 1000.
$ cargo nextest run --profile ci -p rustfs --test connect_perf_drive --no-fail-fast -E 'test(real_local_write_read_is_measured_and_scratch_is_removed) | test(production_cli_measures_and_signs_exact_binary_provenance)'
info: experimental features enabled: setup-scripts
   Compiling rustfs v1.0.1 (/private/tmp/gw-orchestrator-2734/ci-drive/rustfs)
warning: linker stderr: ld: __eh_frame section too large (max 16MB) to encode dwarf unwind offsets in compact unwind table, performance of exception handling might be affected
  |
  = note: `#[warn(linker_messages)]` on by default

warning: `rustfs` (bin "rustfs") generated 1 warning
warning: `rustfs` (bin "rustfs-cli") generated 1 warning (1 duplicate)
    Finished `test` profile [unoptimized + debuginfo] target(s) in 7m 20s
────────────
 Nextest run ID b73d46ab-c910-4e1e-8063-7fc7e332ec15 with nextest profile: ci
    Starting 2 tests across 1 binary (8 tests skipped)
        FAIL [  13.968s] rustfs::connect_perf_drive production_cli_measures_and_signs_exact_binary_provenance
  stdout ───

    running 1 test
    test production_cli_measures_and_signs_exact_binary_provenance ... FAILED

    failures:

    failures:
        production_cli_measures_and_signs_exact_binary_provenance

    test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 9 filtered out; finished in 13.96s

  stderr ───

    thread 'production_cli_measures_and_signs_exact_binary_provenance' (107015462) panicked at rustfs/tests/connect_perf_drive.rs:527:5:
    assertion `left == right` failed
      left: Number(1001)
     right: 1000
    note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

        FAIL [   0.015s] rustfs::connect_perf_drive real_local_write_read_is_measured_and_scratch_is_removed
  stdout ───

    running 1 test
    test real_local_write_read_is_measured_and_scratch_is_removed ... FAILED

    failures:

    failures:
        real_local_write_read_is_measured_and_scratch_is_removed

    test result: FAILED. 0 passed; 1 failed; 0 ignored; 0 measured; 9 filtered out; finished in 0.01s

  stderr ───

    thread 'real_local_write_read_is_measured_and_scratch_is_removed' (107016667) panicked at rustfs/tests/connect_perf_drive.rs:111:5:
    assertion `left == right` failed
      left: 1001
     right: 1000
    note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace

────────────
     Summary [  13.985s] 2 tests run: 0 passed, 2 failed, 8 skipped
        FAIL [  13.968s] rustfs::connect_perf_drive production_cli_measures_and_signs_exact_binary_provenance
        FAIL [   0.015s] rustfs::connect_perf_drive real_local_write_read_is_measured_and_scratch_is_removed
error: test run failed
exit_code=100

Transient production mutations were restored byte-for-byte before the complete green rerun. Original/restored SHA256: 8d389e6d8836c041aab8c6a93e56addd65ba3f70b33264e1c74f639d0572a2da.

$ git diff --check

exit_code=0

$ git diff --cached --check

exit_code=0

Independent orchestrator acceptance
Tested code tree: e67c72fb01

Tested tree: 16fbbd2ea4
shellcheck --version && actionlint -> exit 0
cargo fmt --all --check -> exit 0
./scripts/check_unsafe_code_allowances.sh -> exit 0
./scripts/check_layer_dependencies.sh -> exit 0
./scripts/check_architecture_migration_rules.sh -> exit 0
./scripts/check_logging_guardrails.sh -> exit 0
./scripts/check_error_other_format_ratchet.sh -> exit 0
./scripts/check_no_tokio_io_uring.sh -> exit 0
./scripts/check_extension_schema_boundaries.sh -> exit 0
./scripts/check_body_cache_whitelist.sh -> exit 0
./scripts/check_s3s_footprint.sh -> exit 0
./scripts/check_test_count_ratchet.sh -> exit 0
./scripts/check_fips_wording.sh -> exit 0
./scripts/check_embedded_secrets.sh -> exit 0
make script-tests -> exit 0
python3 ./scripts/test_object_generation_protocol_model.py -> exit 0
python3 ./scripts/check_test_wiring.py -> exit 0
./scripts/check_no_planning_docs.sh -> exit 0
./scripts/check_uring_lane_lib_only.sh -> exit 0

$ LANG=en_US.UTF-8 CARGO_BUILD_JOBS=1 CARGO_INCREMENTAL=0 cargo nextest run --profile ci -p rustfs --test connect_perf_drive --no-fail-fast
info: experimental features enabled: setup-scripts
warning: linker stderr: ld: __eh_frame section too large (max 16MB) to encode dwarf unwind offsets in compact unwind table, performance of exception handling might be affected
  |
  = note: `#[warn(linker_messages)]` on by default

warning: `rustfs` (bin "rustfs-cli") generated 1 warning
warning: `rustfs` (bin "rustfs") generated 1 warning (1 duplicate)
    Finished `test` profile [unoptimized + debuginfo] target(s) in 0.69s
────────────
 Nextest run ID cd1edf4c-f1c0-456d-ad1a-76cc00ce05c9 with nextest profile: ci
    Starting 10 tests across 1 binary
        PASS [   0.012s] rustfs::connect_perf_drive cancellation_and_single_collector_leave_no_scratch
        PASS [   0.010s] rustfs::connect_perf_drive cleanup_failure_overrides_cancellation_and_remains_attributed
        PASS [   0.010s] rustfs::connect_perf_drive consent_and_resource_budgets_fail_before_scratch_io
        PASS [   0.009s] rustfs::connect_perf_drive invalid_roots_and_existing_scratch_fail_without_clobbering
        PASS [  20.328s] rustfs::connect_perf_drive production_cli_measures_and_signs_exact_binary_provenance
        PASS [   5.527s] rustfs::connect_perf_drive production_cli_rejects_missing_l1_consent_and_limits_before_artifacts
        PASS [   0.013s] rustfs::connect_perf_drive read_only_root_and_full_drive_errors_are_classified
        PASS [   0.019s] rustfs::connect_perf_drive real_local_write_read_is_measured_and_scratch_is_removed
        PASS [   0.011s] rustfs::connect_perf_drive runtime_timeout_and_full_drive_use_schema_valid_failed_reason
        PASS [   0.049s] rustfs::connect_perf_drive successful_result_has_signed_bounded_private_offline_export
────────────
     Summary [  25.991s] 10 tests run: 10 passed, 0 skipped

exit_code=0

All 54 tracked Cargo.toml files parse without duplicate keys. git diff --check passed. The production source remains unchanged; the two deliberate implementation mutations above were restored before this complete green run.
2026-10-09 08:21:15 +08:00
Chris 4d15b81623 refactor(protocols): move error types to rustfs-s3-types by codemod (#8439)
T1.2 batch 3 (rustfs/backlog#2743): s3select-api, s3select-query and
the error items of protocols move off s3s by scripts/codemods, run with
--hold crates/protocols/src/sftp/errors.rs. That file stays on s3s: it
downcasts the backend error, whose runtime type is still s3s::S3Error.

- rustfs/src/protocols/client.rs: the two webdav session methods now
  return rustfs_s3_types::S3Result and convert the s3s error with From.
- codemod: a child module that calls s3_error! through `use super::*`
  gets its own `use rustfs_s3_types::s3_error;` once its parent imports
  that macro by name; otherwise it is listed as GLOB.
- s3-types: a table test pins code, status and the s3s error the legacy
  edge renders for every variant a moved call site names.
- s3_error_lines 1585 -> 1584 (a lifecycle comment reworded).
2026-10-08 15:59:34 -07:00
Chris 836d30192f feat(s3-types): add Body::from_stream and move Swift off s3s::Body (#8438)
rustfs/backlog#2747 batch 3. `Body::from_stream` wraps a chunk stream of
unknown length over `StreamBody`/`BoxBody`: one data frame per chunk, the
first error ends the body, the size hint stays unknown, and dropping the
body drops the stream. Parity tests pin it to the s3s `StreamingBlob::wrap`
it replaces, at the `http_body` level and byte for byte through hyper.

The Swift handlers in rustfs-protocols build their responses as the owned
`Body`. `SwiftService` converts them into the wrapped S3 service's response
body type through `From<Body>`, so rustfs keeps serving `s3s::Body` at the
legacy edge unchanged. Request bodies stay Send-only (`UnsyncBoxBody`); the
two storage writers and the DLO/SLO segment streams get `Sync` from
`sync_wrapper::SyncStream`, which is what s3s's `http_body_unsync` mutex
provided before. The DTO and error uses of s3s in protocols are untouched
(T1.5 and T1.2).
2026-10-08 15:05:16 -07:00
Chris 860c61ee6d feat(app): add the RustFS-owned RequestEnvelope and its s3s constructor (#8437)
* feat(app): add the RustFS-owned RequestEnvelope and its s3s constructor

* ci: classify the s3s edge adapters in storage/s3_compat as interface
2026-10-08 11:58:55 -07:00
Chris f600af4704 test(e2e): compare two RUSTFS_S3_STACK servers over every S3 operation (#8435)
Add dual_stack_differential to e2e_test. It starts two rustfs processes
from the same binary, stack A on RUSTFS_S3_STACK=legacy and stack B on
RUSTFS_E2E_STACK_B (default legacy until rustfs/backlog#2734 T2.5),
sends both the same aws-sdk-s3 script covering the 73 methods of
impl S3 for FS (a normal request and a 4xx negative each), and compares
status, error code, an allowlist of response headers and a few typed
body fields. Error message text is never compared. Each run writes
<target>/dual-stack/<run>.json with compared_ops and the (op, field,
a, b) differences; any difference fails the test.

differential_detects_an_injected_difference starts B with
RUSTFS_REGION=eu-west-1 and requires GetBucketLocation to be reported.
The comparator is pure and unit-tested without a server.

Wire the module into the e2e-smoke filter and update the e2e-smoke and
e2e-full selection digests and the e2e_test test-count row.

Refs rustfs/backlog#2740
2026-10-08 10:47:10 -07:00
overtrue 9e747e8475 ci(footprint): count only s3_error! lines still on s3s's macro
s3_error_lines counted every line holding the text `s3_error!`, which
also matches the RustFS-owned `rustfs_s3_types::s3_error!`. A codemod
that moves call sites onto rustfs-s3-types therefore could not lower it
(rustfs/backlog#2743, decision 3; rustfs/rustfs#8432, gotcha 2).

The counter now applies a textual, per-file and per-line rule that errs
on the side of counting (full rule in the script header):
- occurrences spelled `rustfs_s3_types::s3_error!` never count;
- a line holding `s3s::s3_error!` always counts;
- any other `s3_error!` line counts unless its file is cleared, that is,
  it imports the macro from rustfs_s3_types by name on a line starting
  with the use item (or defines it with `macro_rules! s3_error`) and does
  not also bring s3s's macro in (`use s3s::s3_error`, a `use s3s::{..}`
  tree naming it or a crate-root `*`, `use s3s::*`, or
  `#[macro_use] extern crate s3s`);
- fail closed: a file matching neither form (`use super::*`, a
  crate-local re-export, a renamed or glob rustfs_s3_types import, a
  commented-out import, a comment-only mention) counts.

Census of the 1585 lines on 96fc06776: 1478 in 64 files that import the
macro from s3s; 107 in 15 files matching neither form (105 through
`use super::*`, 1 qualified `s3s::s3_error!`, 1 comment line); none in
files that import it from rustfs_s3_types or define it. Every line stays
counted, so the baseline remains s3_error_lines=1585 (files=202 and
ecstore_files=36 unchanged). Rewriting rustfs/src/auth.rs's import to
rustfs_s3_types lowers the new counter to 1567; the old one stayed 1585.

`--self-test` (wired into `make script-tests`) copies the script into
22 fixture trees: 6 positive cases and 16 negative ones (lines that must
not count, forms that must not clear a file, and a missing baseline,
missing key or failing classification search that must fail). Each tree
has a control line that must count, and each run gets s3s text on stdin.
Seven cases were red against the old counter. 19 mutants each turn their
case red: old definition, flipped import test, dropped fail-closed
branch, dropped e2e_test exclusion, dropped s3s veto, dropped qualified
s3s rule, dropped rustfs-qualified strip, unanchored rustfs import,
renamed import accepted, nested glob taken as root glob, single-line
s3s trees only, dropped s3s glob forms, dropped macro_use form, dropped
definition clearing, ignored classification rg errors, error-line search
without its path, classification searches without their path, missing
baseline skipped, missing key skipped.
2026-10-09 01:03:21 +08:00
Chris 96fc067765 refactor(ecstore): move S3ErrorCode to rustfs-s3-types by codemod (#8432)
* test(ecstore): pin the InvalidRange responses of new_getobjectreader

Both error branches of new_getobjectreader build an ErrorResponse with S3ErrorCode::InvalidRange, and no test observed either code. Pin the code, its wire name and the location fields of each branch before the code type moves to rustfs-s3-types (rustfs/backlog#2743), so a changed code on either branch fails a test. The ecstore test-count row rises by the two tests.

* chore(scripts): add the s3s-to-rustfs-s3-types error codemod

Task T1.2 of rustfs/backlog#2734 (rustfs/backlog#2743) moves S3Error, S3ErrorCode, S3Result and s3_error! from s3s to rustfs-s3-types by script, one crate closure per PR. The codemod rewrites use trees (splitting mixed s3s imports) and qualified code paths, never comments or literals, adds the rustfs-s3-types dependency, drops s3s only from a crate that no longer names it, refuses every file of the s3s edge allowlist, takes --hold for a file whose type is fixed outside the closure, and prints a per-crate census. A second run changes nothing. Its self-test runs under make script-tests.

* refactor(ecstore): take S3ErrorCode from rustfs-s3-types

Output of scripts/codemods/s3_error_to_s3_types.py over crates/ecstore, crates/lifecycle, crates/scanner and crates/replication (rustfs/backlog#2743 batches 1 and 2). The ErrorResponse code field, the two InvalidRange constructions in new_getobjectreader and an unused tier import now use rustfs_s3_types::S3ErrorCode; the variant names and wire names are unchanged. lifecycle, scanner and replication reference none of the four error names. warm_backend.rs and warm_backend_s3.rs are held: they compare against rustfs_s3_client::ErrorResponse.code, which stays the s3s type until s3-client moves.
2026-10-08 09:30:37 -07:00
Chris e809b4899e refactor(s3-types): own the HTTP Body and drop s3s from three crates (#8431)
Add rustfs_s3_types::Body, a BoxBody<Bytes, StdError> wrapper with
empty() and From<Bytes | Vec<u8> | String | &'static str>, and the
legacy From conversions to and from s3s::Body behind compat-s3s.

The signer is now generic over the request body: it only reads and
writes the request head, so it needs no body type and its callers keep
theirs. s3-client sends its requests with the owned Body, madmin signs a
bodiless head, and targets drops its two S3Request/S3Response header
wrappers in favour of extract_params_header.

rustfs-signer, rustfs-madmin and rustfs-targets no longer depend on
s3s; the s3s footprint baseline drops from 209 to 202 files.

Refs rustfs/backlog#2747, rustfs/backlog#2734
2026-10-08 08:43:06 -07:00
Chris 584f03f300 test(ecstore): reach the bucket-config codec through the test facade (#8428) 2026-10-08 07:12:19 -07:00
Chris d52d6fade7 ci: make the s3s footprint baseline exact and lower-only (#8403)
* ci: make the s3s footprint baseline exact and lower-only

The s3s footprint guard compared three counters against ceilings held in
the script itself, so a ceiling could sit above the tree and a drift was
fixable by raising it (rustfs/rustfs#5964 raised the s3_error! ceiling
to go green). The three values now live in
.config/s3s-footprint-baseline.txt as files, s3_error_lines and
ecstore_files, the script reads only that file, and a measurement that
differs from its baseline in either direction fails: growth prints
"s3s footprint grew: <measured> > <baseline>", shrinkage prints
"lower the baseline to <measured> in this PR". The counting method is
unchanged; the baseline is the current measurement of
gateway/integration (209 / 1585 / 36). The header records the end state:
once all three reach 0, T4.2 switches the script to failing on any
occurrence.

Refs rustfs/backlog#2739, rustfs/backlog#2734

* ci: add the s3s edge allowlist mode to the footprint guard (#8405)

`scripts/check_s3s_footprint.sh --mode allowlist` fails on every Rust
file that references s3s (`s3s::` or `s3s_sigv4`) and is not matched by
an entry of the new `.config/s3s-edge-allowlist.txt`; `--dry-run` lists
those files and exits 0. The allowlist carries the initial edge from the
spec, and the script rejects an entry that names or matches anything
under crates/ other than crates/e2e_test. The baseline mode and the
argument-less CI invocation are unchanged.

Refs rustfs/backlog#2741, rustfs/backlog#2734
2026-10-08 06:48:57 -07:00
Chris 3623d6db2d ci: raise the test-count baseline to the integration tree (#8426) 2026-10-08 06:48:40 -07:00
Chris 484f1254ca test(ecstore): pin the persisted bucket-config XML bytes for 13 families (#8408)
Before any DTO or codec is replaced, pin the exact bytes that
deserialize followed by serialize produces for every persisted
bucket-configuration XML family in its three source shapes (AWS
documentation, MinIO-written, RustFS-written), and the bytes RustFS
writes today for a typed value. Byte equality only: a structural
comparison would let a codec replacement change the persisted encoding
unnoticed.

The generator script refuses to overwrite existing goldens without
--force, refuses on dirty codec paths, cannot report success from its
write mode, and records the generating commit in the fixtures README.

Refs rustfs/backlog#2744, rustfs/backlog#2734
2026-10-08 21:25:56 +08:00
Chris c9c16efbb6 ci: add the per-crate test-count ratchet (#8404)
Quick Checks now fails any PR that lowers the number of test attributes
in a workspace crate or adds #[ignore], measured per crate against the
exact rows of .config/test-count-baseline.txt (rustfs/backlog#2734
section 1 rule 3). The counter widens the spec's literal
#[(tokio::)?test] to the tokio attributes that carry arguments and both
spellings of #[test_case(...)], and anchors at line start so
commented-out attributes do not count. The initial baseline covers 52
crates, 21077 tests and 164 ignored.

Refs rustfs/backlog#2738
Refs rustfs/backlog#2734
2026-10-08 21:25:42 +08:00
Chris f2a59073ca feat(s3-types): add the RustFS-owned S3 error type and s3_error! macro (#8406)
`rustfs-s3-types` now provides `S3ErrorCode` (generated by build.rs from
src/codes.txt, the grep-derived call surface, plus `Custom`), `S3Error`,
`S3Result`, `StdError` and the `s3_error` macro with the same call syntax
as s3s, so the T1.2 codemod is a `use` swap. Statuses are hand-written and
pinned to the s3s behaviour RustFS ships today by an oracle test that uses
s3s as a dev-dependency only. The `From` conversions for the legacy edge
live behind the off-by-default `compat-s3s` feature (the orphan rule
forbids them in rustfs/src/error.rs); `rustfs` enables it.

Refs rustfs/backlog#2742, rustfs/backlog#2734
2026-10-08 04:11:00 -07:00
Chris c1af94d1d1 refactor(server): group the s3s-compat tower layers under legacy_compat (#8410)
Move the ten tower layers that patch s3s behaviour out of
rustfs/src/server/layer.rs into rustfs/src/server/legacy_compat/, one file
per layer with its tests. Each file header names the s3s behaviour it
patches, the gateway mechanism that replaces it (Replaced by:), and the task
that removes it (Removed by: T2.12, rustfs/backlog#2771). The layers keep
their code, their order in external_service_stack!, and their behaviour; no
layer is deleted.

Add server::http::tests::stack_census, which renders the ordered service
chain of the real external_service_stack! expansion and compares it with a
committed constant, so a reorder, insertion or removal is a visible diff.

Route the two s3s error types the moved layers use through a
storage_api::server::legacy_compat shim so the split adds no s3s-importing
file, and lower the s3s footprint baselines to the measured values (files 209
to 208, s3_error! lines 1586 to 1585). Keep the server runtime-source bypass
guard covering the moved code.

Refs rustfs/backlog#2787, rustfs/backlog#2734
2026-10-08 02:57:47 -07:00
Chris f84bbfe28f feat(server): add the RUSTFS_S3_STACK selector and dual-stack e2e lane (#8411) 2026-10-07 21:01:39 -07:00
rustfs-cla-assistant[bot] 378a2ff7f8 Merge remote-tracking branch 'origin/main' into gateway/integration 2026-10-08 02:18:57 +00:00
rustfs-cla-assistant[bot] b88b53aa6a Merge remote-tracking branch 'origin/main' into gateway/integration 2026-10-07 21:18:17 +00:00
500 changed files with 23148 additions and 56296 deletions
-10
View File
@@ -1,10 +0,0 @@
# Rulings for the gateway conformance lane (.github/workflows/gateway-acceptance.yml).
#
# One row per case that fails on the GATEWAY stack for a reviewed reason: `case`; `ledger`, the
# rustfs/backlog#2684 entry that accepts the difference (R-0xx or K-n); a one-line `reason`; and
# `failure`, a prefix of the failure the ruling accepts, as the runner's report.json renders it
# (`[expect/status] at /expect/status: exchange #1, attempt 1: expected status 400, observed 403`).
# scripts/check_conformance_rulings.py fails the lane on a failed case without a row, on a ruled
# case whose failures do not start with its `failure`, and on a row whose case no longer fails or
# no longer exists. The legacy stack is a reference run and takes no rulings: the `rustfs` profile
# describes the gateway edge, not the legacy one.
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=8fef326a3fddd8a548e2cbcc6ea8159f831113595fb003eeebbd9aeb4ac23e48
sha256-linux=8b4381578377704ac8b1b5156128b83f036ec4aedc9d863fff25df15112420fc
sha256-darwin=0374e6b9b061dd5d622686301386232a4aa1cb22eedc18af70d2a7411bc66f8d
sha256-linux=70d8e59d9ba0512859bf1de0cb255b27b1bcf9f2d10cf12442762a025887397d
+2 -2
View File
@@ -1,2 +1,2 @@
sha256-darwin=dcc076f8a36320d2fe36b4d2f26e2d54c3b9bc2ff7e14c6e90241f4fd8cf4a33
sha256-linux=229ebc0cbc9b35863600ae60c6fbb548516efe2a2f41f4b5db503b3283c2f64f
sha256-darwin=a2cb67d91eb0d093928371d6e84a84b2dd41b2ba975a28094e81879bd65838ad
sha256-linux=82db0561cf9b111f11782d647d0859c9d2a44c5c68481b5e39c48505828bd392
+1 -1
View File
@@ -1 +1 @@
sha256=061d0a697967d2dc8f472ff885886e069b1cf4aeaa5f230e177b7236017c23b6
sha256=900861de706f0fcecb90dd328326c8583df61420400771b12eb1b37108c3e8d9
+4 -9
View File
@@ -70,10 +70,10 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
@echo "📦 Checking s3s footprint ratchet..."
./scripts/check_s3s_footprint.sh
.PHONY: release-feature-boundary-check
release-feature-boundary-check: ## Check no release-reachable feature enables a debug-only one
@echo "🚧 Checking release feature boundaries..."
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --features-only
.PHONY: gateway-stack-release-boundary-check
gateway-stack-release-boundary-check: ## Check no release build can enable the gateway-stack feature
@echo "🚧 Checking gateway-stack release boundary..."
./scripts/check_gateway_stack_release_boundary.sh
.PHONY: test-count-ratchet-check
test-count-ratchet-check: ## Check no workspace crate fell below its test floor or rose above its #[ignore] ceiling in the committed baseline
@@ -85,11 +85,6 @@ test-seams-check: ## Check tests outside an operation's owner call its use-case
@echo "🧪 Checking cross-owner test seams..."
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py
.PHONY: test-retirements-check
test-retirements-check: ## Check every test retired by a migration slice names a replacement that exists
@echo "🧪 Checking test retirement tables..."
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_retirements.py
.PHONY: fips-wording-check
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
@echo "📣 Checking cryptographic capability wording guard..."
+2 -2
View File
@@ -20,11 +20,11 @@ planning-docs-check: ## Check that no planning-type documents are committed
./scripts/check_no_planning_docs.sh
.PHONY: pre-commit
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
pre-commit: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check quick-check ## Run fast pre-commit checks without clippy/full tests
@echo "✅ All pre-commit checks passed!"
.PHONY: pre-pr
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check release-feature-boundary-check test-count-ratchet-check test-seams-check test-retirements-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
pre-pr: fmt-check unsafe-code-check architecture-migration-check logging-guardrails-check error-other-ratchet-check tokio-io-uring-check extension-schema-check body-cache-whitelist-check s3s-footprint-check gateway-stack-release-boundary-check test-count-ratchet-check test-seams-check fips-wording-check embedded-secrets-check test-wiring-check doc-paths-check planning-docs-check log-analyzer-rules-check offline-enrollment-e2e-check clippy-check test ## Run full pre-PR checks with clippy and tests
@echo "✅ All pre-PR checks passed!"
.PHONY: dev-check
+2 -12
View File
@@ -38,7 +38,6 @@ script-tests: ## Run shell script tests
./scripts/test_entrypoint_credentials.sh
./scripts/test_entrypoint_directories.sh
./scripts/test_internode_grpc_ab_bench.sh
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
./scripts/test_object_batch_bench_enhanced.sh
./scripts/test_package_service_scripts.sh
./scripts/test_hotpath_warp_ab_gate.sh
@@ -62,16 +61,12 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/test_check_doc_paths.py
./scripts/check_embedded_secrets.sh --self-test
./scripts/check_s3s_footprint.sh --self-test
./scripts/check_gateway_stack_release_boundary.sh --self-test
./scripts/test_check_test_count_ratchet.sh
./scripts/test_guard_source_set.sh
./scripts/test_split_guard_rules.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_wiring.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/s3_error_to_s3_types.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_bucket_usecase.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_ecfs_legacy_edge.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/codemods/split_lifecycle_transition_api_test.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_seams.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_test_retirements.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_e2e_binary.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_download_release_asset.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_reference_binary.py
@@ -84,7 +79,7 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/ci_gate.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/check_connect_release_boundary.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_artifact.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_group_proof.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_test_shards.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_merge_queue.py
$(RUSTFS_PYTHON_BIN) -m unittest discover -s scripts/ci -p 'test_*.py'
$(RUSTFS_PYTHON_BIN) ./scripts/check_security_coverage.py --self-test
@@ -99,13 +94,8 @@ script-tests: ## Run shell script tests
$(RUSTFS_PYTHON_BIN) ./scripts/test_retain_validation_evidence.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_ci_timing_report.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_junit_diff.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_dual_stack.py
$(RUSTFS_PYTHON_BIN) ./scripts/test_mint_stack_diff.py
$(RUSTFS_PYTHON_BIN) ./scripts/check_conformance_rulings.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/compat_matrix_report.py --self-test
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_report_compat.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_runner_tools.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_api_coverage.py
$(RUSTFS_PYTHON_BIN) ./scripts/s3-tests/test_reference_parity.py
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
+3 -18
View File
@@ -99,11 +99,6 @@ threads-required = "num-test-threads"
filter = '(package(rustfs-ecstore) & (test(/^core::pools_test::decommission_lock_order_tests::(active_decommission_migration_does_not_invert_capacity_and_delete_object(_tag)?_locks|decommission_moves_a_transitioned_version_expired_by_a_delete_all_rule_without_stalling)$/) | test(=services::integrity::tests::audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts))) | (package(rustfs-heal) & test(=heal::mrf_queue::partial_write::tests::durable_retry_submission_error_keeps_checkpoint_and_uses_backoff))'
threads-required = "num-test-threads"
# These eight-disk namespace-owner probes overlap durable cleanup with a lock contender. Reserve capacity so sibling disk fixtures cannot consume their lock and completion budgets.
[[profile.default.overrides]]
filter = 'package(rustfs-ecstore) & test(/^store::object::tests::(canceled_create_retains_usage_cleanup_namespace_owner|batch_delete_excludes_recovery_on_nonzero_pool|delete_excludes_recovery_with_(disjoint|identical|overlapping)_pool_lock_clients)$/)'
threads-required = "num-test-threads"
# These eight-disk decommission fixtures exercise concurrent retries,
# free-version cleanup, and real capacity probes under production lock deadlines.
# Reserve the run's capacity so unrelated IO cannot exhaust those budgets.
@@ -142,7 +137,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
setup = 'ecstore-base-stack'
[[profile.default.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
# These handler/store fixtures and the relocated-pool GET compose deep storage
@@ -410,11 +405,6 @@ threads-required = "num-test-threads"
filter = '(package(rustfs-ecstore) & (test(/^core::pools_test::decommission_lock_order_tests::(active_decommission_migration_does_not_invert_capacity_and_delete_object(_tag)?_locks|decommission_moves_a_transitioned_version_expired_by_a_delete_all_rule_without_stalling)$/) | test(=services::integrity::tests::audit_migration_and_ack_recovery_preserve_source_and_reject_conflicts))) | (package(rustfs-heal) & test(=heal::mrf_queue::partial_write::tests::durable_retry_submission_error_keeps_checkpoint_and_uses_backoff))'
threads-required = "num-test-threads"
# These eight-disk namespace-owner probes overlap durable cleanup with a lock contender. Reserve capacity so sibling disk fixtures cannot consume their lock and completion budgets.
[[profile.ci.overrides]]
filter = 'package(rustfs-ecstore) & test(/^store::object::tests::(canceled_create_retains_usage_cleanup_namespace_owner|batch_delete_excludes_recovery_on_nonzero_pool|delete_excludes_recovery_with_(disjoint|identical|overlapping)_pool_lock_clients)$/)'
threads-required = "num-test-threads"
# These eight-disk decommission fixtures exercise concurrent retries,
# free-version cleanup, and real capacity probes under production lock deadlines.
# Reserve the run's capacity so unrelated IO cannot exhaust those budgets.
@@ -465,7 +455,7 @@ filter = '(package(rustfs) & test(/^(admin::handlers::replication::target_repair
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::complete::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
filter = 'package(rustfs-ecstore) | package(rustfs-s3select-api) | package(rustfs-scanner) | (package(rustfs) & test(/^(app::multipart_usecase::tests::concurrent_completions_share_durable_bucket_quota_reservations|app::object::delete::tests::compressed_delete_requests_update_observed_usage_without_releasing_quota_floor|app::object::internal_put::tests::internal_multipart_roundtrip_completes_and_abort_leaves_nothing|app::object::restore::tests::execute_restore_object_maps_failures_to_typed_s3_errors|storage::access::tests::(delete_object_access_captures_authorized_bucket_incarnation|copy_operations_reject_recreated_source_bucket_after_authorization|request_slot_keeps_bucket_policy_bound_to_its_store))$/))'
setup = 'ecstore-base-stack'
[[profile.ci.scripts]]
@@ -731,7 +721,7 @@ test-group = 'ecstore-serial-flaky'
[profile.e2e-smoke]
default-filter = """
package(e2e_test) & (
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage)_test::|^fake_s3_target::/)
test(/^(delete_marker_migration_semantics|version_id_regression|list_objects_v2_pagination|list_object_versions_regression|list_objects_duplicates|list_buckets_double_slash|list_buckets_auth|list_buckets_iam_filter|leading_slash_key|special_chars|create_bucket_region|delete_objects_versioning|head_object_consistency|head_object_range|copy_object_metadata|copy_object_tagging|copy_source_invalid_date|content_encoding|compression|multipart_storage_class|multipart_upload_id|storage_class_capability|ssec_copy|anonymous_access|bucket_policy_check|presigned_negative|negative_sigv4|admin_auth|notification_webhook|tls_hot_reload|console_smoke|admin_iam_crud|admin_pools|sts_query_compat|data_usage|website_stack_interim)_test::|^fake_s3_target::/)
| test(/^replication_extension_test::(test_replication_check_succeeds_with_remote_target|test_replication_check_rejects_target_without_object_lock|test_set_remote_target_rejects_unversioned_source_bucket|test_replication_check_rejects_unversioned_source_bucket|test_replication_check_rejects_missing_replication_config|test_replication_check_rejects_invalid_bucket|test_set_remote_target_rejects_same_bucket_on_same_deployment|test_set_remote_target_rejects_unversioned_target_bucket|test_set_remote_target_update_requires_arn|test_set_remote_target_update_rejects_missing_target|test_set_remote_target_rejects_invalid_target_url|test_set_remote_target_rejects_self_signed_https_target_without_skip_tls_verify|test_set_remote_target_rejects_private_ca_https_target_without_ca_cert_pem|test_list_remote_targets_rejects_empty_bucket|test_list_remote_targets_rejects_invalid_bucket|test_remove_remote_target_rejects_missing_target|test_remove_remote_target_rejects_missing_arn|test_remove_remote_target_rejects_invalid_bucket|test_remove_remote_target_rejects_target_used_by_replication|test_delete_bucket_replication_removes_remote_target)$/)
| test(/^reliant::lifecycle::/)
| test(/^conditional_delete_objects_test::/)
@@ -956,10 +946,6 @@ path = "junit.xml"
# parallel-safe — the same property e2e-smoke relies on. The exceptions are the
# 4-disk reliability / degraded-read fault-injection tests, multi-node heal
# clusters, and the fixed-port Vault tests, all serialized below.
# `dual_stack_differential::reference::` holds hermetic unit tests of how the
# differential picks side A (shell-script stand-ins, no server, no reference
# binary). They already run on every pull request in e2e-smoke, so e2e-full
# excludes them to keep its selection digest unchanged.
[profile.e2e-full]
default-filter = """
package(e2e_test)
@@ -970,7 +956,6 @@ default-filter = """
& !test(/^replication_target_matrix_test::/)
& !test(/^on_demand_migration::(concurrency_test|fault_test|interop_test|real_source_test)::/)
& !test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)
& !test(/^dual_stack_differential::reference::/)
"""
fail-fast = false
+10 -11
View File
@@ -1,18 +1,17 @@
# Baseline for the direct s3s footprint, consumed only by
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4;
# comparison rules on rustfs/backlog#2765).
# Exact baseline for the direct s3s footprint, consumed only by
# scripts/check_s3s_footprint.sh (rustfs/backlog#1677; rustfs/backlog#2734 T0.4).
#
# Each line is a ceiling everywhere: a counter above it fails, because new code
# widened the s3s surface; route it through the gateway abstractions instead and
# never raise a value here. Pull requests and merge groups are additionally held
# to their base revision's measurement, so a slice that shrinks the surface need
# not edit this file. A counter below its line passes and the check prints the
# lowered line; the migration coordinator lowers the lines in batches.
# Each value is the exact measurement of the tree, not a ceiling. CI fails when a
# measured counter differs from its line here in either direction:
# - a larger measurement means new code widened the s3s surface: route it
# through the gateway abstractions instead; never raise a value here;
# - a smaller measurement means the PR shrank the surface: lower the matching
# value to the measured number in the same PR.
#
# files Rust files referencing s3s paths, repo-wide minus crates/e2e_test
# s3_error_lines lines holding s3_error!, same scope, minus those the script
# header's rule clears (rustfs_s3_types's macro); fails closed
# ecstore_files Rust files referencing s3s paths under crates/ecstore/src
files=63
s3_error_lines=209
files=65
s3_error_lines=217
ecstore_files=0
+1 -1
View File
@@ -12,7 +12,7 @@
# Regenerate with:
# scripts/check_test_count_ratchet.sh --update
e2e_test 1026 20
rustfs 6449 64
rustfs 6440 64
rustfs-audit 70 0
rustfs-checksums 28 1
rustfs-common 36 0
@@ -1,3 +0,0 @@
# Tests the bucket_access slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the bucket_configs slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the bucket_core slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the bucket_replication_lifecycle slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the cutover slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the listing_multipart slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the object_commit slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the object_meta slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the object_misc slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the object_read slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
@@ -1,3 +0,0 @@
# Tests the object_write slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the t211 slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-3
View File
@@ -1,3 +0,0 @@
# Tests the t212 slice retires (rustfs/backlog#2765); format and checks: scripts/check_test_retirements.py.
# <crate> | <module path>::<test fn> | <kind: test, differential or e2e> | <replacement> | <reason>
# The retired test must still exist when its entry is added; the change that deletes it deletes the entry.
-6
View File
@@ -140,12 +140,6 @@ With list syntax, use `- RUSTFS_OBS_LOG_DIRECTORY=/var/log/rustfs/` or quote the
---
## Local storage pool expansion
To append a second four-drive pool, set `RUSTFS_VOLUMES: "/data/rustfs0/{0...3} /data/rustfs1/{0...3}"` and keep the original pool's drives, order, and erasure set width. The container prepares each drive directory and passes one argument per pool to RustFS.
If an older image already formatted both ranges together as a single eight-drive pool, list the eight individual paths in their original order to keep that layout when upgrading.
## 🔧 Common Operations
### Cleaning Up
-54
View File
@@ -1,54 +0,0 @@
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Both-stack leg
description: Prove which S3 stack one e2e leg's servers selected and upload the leg for the summary (call with if always)
inputs:
profile:
description: The nextest profile the leg ran
required: true
stack:
description: The stack the leg requested, legacy or gateway
required: true
logs:
description: Directory or file holding the servers' output
required: true
runs:
using: composite
steps:
- name: Prove which stack the servers selected
shell: bash
env:
PROFILE: ${{ inputs.profile }}
STACK: ${{ inputs.stack }}
LOGS: ${{ inputs.logs }}
run: |
# Outside the workspace: e2e_binary.py refuses a run whose source tree changed.
mkdir -p "$RUNNER_TEMP/leg"
junit="target/nextest/${PROFILE}/junit.xml"
if [ -s "$junit" ]; then cp "$junit" "$RUNNER_TEMP/leg/junit.xml"; fi
# A gateway leg stopped by its preflight has no output; the preflight's stack.json stays.
if [ -e "$LOGS" ]; then python3 scripts/dual_stack.py stack --expect "$STACK" --out "$RUNNER_TEMP/leg/stack.json" "$LOGS"; fi
- name: Upload the leg
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dual-stack-leg-${{ inputs.profile }}-${{ inputs.stack }}
overwrite: true
path: ${{ runner.temp }}/leg
if-no-files-found: warn
retention-days: 7
@@ -1,56 +0,0 @@
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: Both-stack summary
description: Diff the legacy and gateway legs of this run (scripts/junit_diff.py) and upload one dated summary
inputs:
profiles:
description: Comma-separated nextest profiles whose two legs this run uploaded
required: true
runs:
using: composite
steps:
- name: Download the legs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: dual-stack-leg-*
path: legs
- name: Diff every profile and write the summary
id: summary
shell: bash
env:
PROFILES: ${{ inputs.profiles }}
run: |
# The shell runs with -e; the exit status is reported after the summary is written.
set +e -u -o pipefail
date="$(date -u +%F)"
echo "date=${date}" >> "$GITHUB_OUTPUT"
mkdir -p legs
python3 scripts/dual_stack.py summary --profiles "$PROFILES" --legs legs --date "$date" --out-dir dual-stack
status=$?
if [ -f dual-stack/summary.md ]; then cat dual-stack/summary.md >> "$GITHUB_STEP_SUMMARY"; fi
exit "$status"
- name: Upload the dated summary
if: always() && steps.summary.outputs.date != ''
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dual-stack-summary-${{ steps.summary.outputs.date }}
overwrite: true
path: dual-stack
if-no-files-found: error
retention-days: 30
+3 -17
View File
@@ -95,21 +95,11 @@ runs:
- name: Check s3s footprint ratchet
shell: bash
env:
# Pull requests and merge groups compare with their base revision; a
# push or scheduled run holds the tree to the baseline file exactly.
FOOTPRINT_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
run: |
if [[ -z "$FOOTPRINT_BASE" ]]; then
./scripts/check_s3s_footprint.sh
exit
fi
git cat-file -e "$FOOTPRINT_BASE^{commit}" 2>/dev/null || git fetch --no-tags --depth=1 origin "$FOOTPRINT_BASE"
./scripts/check_s3s_footprint.sh --base "$FOOTPRINT_BASE"
run: ./scripts/check_s3s_footprint.sh
- name: Check release feature boundaries
- name: Check gateway-stack release boundary
shell: bash
run: python3 scripts/check_connect_release_boundary.py --features-only
run: ./scripts/check_gateway_stack_release_boundary.sh
- name: Check per-crate test-count ratchet
shell: bash
@@ -119,10 +109,6 @@ runs:
shell: bash
run: python3 ./scripts/check_test_seams.py
- name: Check retired tests name existing replacements
shell: bash
run: python3 ./scripts/check_test_retirements.py
- name: Check cryptographic capability wording
shell: bash
run: ./scripts/check_fips_wording.sh
-6
View File
@@ -63,11 +63,6 @@ inputs:
required: false
default: "true"
outputs:
cache-hit:
description: "'true' when the Rust cache was restored from an exact key match"
value: ${{ steps.rust-cache.outputs.cache-hit }}
runs:
using: "composite"
steps:
@@ -148,7 +143,6 @@ runs:
tool: cargo-nextest@0.9.138
- name: Setup Rust cache
id: rust-cache
uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2
with:
# false is rust-cache's own default. With true, cleanup.ts returns
+4 -55
View File
@@ -360,15 +360,14 @@
"cadence": "nightly",
"timezone": "UTC",
"schedule": [
"13 1 * * *",
"17 16 * * 0"
"13 1 * * *"
],
"owner": "@overtrue",
"policy": "active",
"monitor": "workflow",
"max_age_hours": 36,
"result_owner": "build.yml",
"disposition": "Daily cron builds Linux; Sunday 16:17 UTC builds macOS and Windows. Public release writes belong to release-publish.yml.",
"disposition": "Builds artifacts only; public release writes belong to release-publish.yml.",
"failure_owner": "workflow",
"origin": "baseline",
"alert_contract": "scheduled-validation"
@@ -765,31 +764,6 @@
"monitor_reason": "Event-driven or operator-requested work has no periodic freshness obligation.",
"origin": "baseline"
},
{
"workflow": ".github/workflows/dual-stack-nightly.yml",
"name": "dual-stack-nightly",
"roles": [
"periodic",
"manual"
],
"events": [
"workflow_dispatch",
"schedule"
],
"cadence": "nightly",
"timezone": "UTC",
"schedule": [
"13 3 * * *"
],
"owner": "@overtrue",
"policy": "active",
"monitor": "workflow",
"max_age_hours": 36,
"result_owner": "dual-stack-nightly.yml",
"disposition": "both-stack JUnit diff of the two profiles only ci.yml runs; the gateway legs are report-only until T2.10",
"failure_owner": "watchdog",
"origin": "added"
},
{
"workflow": ".github/workflows/e2e-distributed.yml",
"name": "e2e-distributed",
@@ -977,31 +951,6 @@
"origin": "baseline",
"alert_contract": "scheduled-validation"
},
{
"workflow": ".github/workflows/gateway-acceptance.yml",
"name": "Gateway Acceptance",
"roles": [
"periodic",
"manual"
],
"events": [
"schedule",
"workflow_dispatch"
],
"cadence": "nightly",
"timezone": "UTC",
"schedule": [
"29 3 * * *"
],
"owner": "@overtrue",
"policy": "active",
"monitor": "workflow",
"max_age_hours": 36,
"result_owner": "gateway-acceptance.yml",
"disposition": "gateway conformance corpus (rustfs profile) and client matrix against the candidate on both S3 stacks",
"failure_owner": "watchdog",
"origin": "added"
},
{
"workflow": ".github/workflows/gateway-integration-sync.yml",
"name": "Gateway Integration Sync",
@@ -1074,7 +1023,7 @@
},
{
"workflow": ".github/workflows/manual-main-packages.yml",
"name": "Manual Linux packages",
"name": "Manual main packages",
"roles": [
"manual"
],
@@ -1088,7 +1037,7 @@
"policy": "active",
"monitor": "none",
"result_owner": "manual-main-packages.yml",
"disposition": "Packages a git tag, or the origin/main tip when the ref input is empty, as workflow artifacts only. No GitHub Release or download-channel writes.",
"disposition": "Packages the origin/main tip as workflow artifacts only. No GitHub Release or download-channel writes.",
"monitor_reason": "Operator-requested packaging has no periodic freshness obligation.",
"origin": "manual-main-packages"
},
+2 -3
View File
@@ -44,7 +44,6 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
`Swatinem/rust-cache` builds its key from the OS, architecture, toolchain and lockfile. It does not include the runner label or image, so a cache written on an `sm-standard-4` pod and one written on `ubuntu-latest` would overwrite each other under the same `shared-key`.
- Every `shared-key` has exactly one writer. `cache-warm.yml` owns `ci-dev`, `ci-feat-rio`, `ci-feat-proto` and `ci-uring`; every other `./.github/actions/setup` call states `cache-save-if` explicitly, and `scripts/security/check_cache_save_if.sh` fails the build when one does not.
- The `ci-dev` key ends in a digest of `warm-ci-dev`'s build commands. Changing what that job warms therefore needs a new key, or readers keep restoring the old artifacts: `scripts/ci_gate.py --check-workflow` prints the key the commands require and fails until the writer and every reader in `ci.yml` and `reference-parity.yml` name it.
- All readers of a key run on the same image as its writer. Keys that live on hosted runners say so in their name (`ci-uring` is warmed on `ubuntu-latest`; `ci-e2e-distributed-hosted`, `e2e-upgrade-server-hosted`, `e2e-upgrade-tests-hosted`).
- Moving a job between pools therefore means renaming its key. The new key is empty until a run from `main` (a scheduled run, or a manual dispatch of that workflow from `main`) saves it; until then that lane does a cold build. The `build.yml` aarch64 release legs append `-ubicloud-arm` for this reason. `cache-warm.yml` does not write those keys.
- `CARGO_BUILD_JOBS` and every other `CARGO*`, `CC*`, `CFLAGS*`, `CXX*`, `CMAKE*` and `RUST*` variable is hashed into the key. `ci.yml` sets `CARGO_BUILD_JOBS` at step level for that reason; promoting it to job level rotates the key for every lane that shares it.
@@ -54,7 +53,7 @@ A hosted runner is also the only choice for a few things the pods cannot do: `mo
These are the patterns that have cost real money here without validating anything more:
- An `if: always()` summary or alert job on a self-hosted label inside a `workflow_run`-triggered workflow. The upstream workflow completes many times a day, the real jobs skip, and the summary still starts a pod each time. Gate it on the upstream job having run, and keep it on `ubuntu-latest`.
- Per-push workflows on `main`. A merge triggers cache warming and a `ci.yml` run, which validates the merged tree in full unless a merge-queue run already validated that tree; `build.yml` development builds are scheduled or manual; with tens of merges a day that is as much self-hosted time as all pull requests together. Anything that does not verify the merged tree itself belongs on a schedule or behind a `paths` filter.
- Per-push workflows on `main`. A merge triggers the applicable `ci.yml` matrix and cache warming; `build.yml` development builds are scheduled or manual; with tens of merges a day that is as much self-hosted time as all pull requests together. Anything that does not verify the merged tree itself belongs on a schedule or behind a `paths` filter.
- `apt-get install` of tools the hosted image already ships. It is only needed on the minimal pod images.
- A throttled `CARGO_BUILD_JOBS` on a 14-CPU pod. The limit exists for link-storm reasons documented in `ci.yml`; measure before raising it, and measure before lowering it further.
- Tiny jobs on ARC pods. Each one pays roughly a minute of pod start-up before its first step; a compile-free job that takes ten seconds costs more to schedule than to run.
@@ -81,6 +80,6 @@ Rows using `macos-latest-xlarge` or a label starting with `sm-`, `dind-`, `ubicl
## Release stages
The [candidate release pipeline](../../docs/operations/release-pipeline.md) builds Linux first and defers paid desktop capacity until acceptance. The daily `build.yml` schedule selects Linux; the Sunday 16:17 UTC schedule selects macOS and Windows. Manual dispatch can still select any platform list. All six platforms remain required before publication. The pipeline records and promotes the exact tested bytes. Builder workflows have no public-release credentials. Keep periodic development builds, PR verification and field-test chains separate when their contracts differ; do not re-enable disabled workflows as part of a source-file migration.
The [candidate release pipeline](../../docs/operations/release-pipeline.md) builds Linux first and defers paid desktop capacity until acceptance. Daily development builds also select Linux; use manual platform selection for desktop development artifacts. All six platforms remain required before publication. The pipeline records and promotes the exact tested bytes. Builder workflows have no public-release credentials. Keep periodic development builds, PR verification and field-test chains separate when their contracts differ; do not re-enable disabled workflows as part of a source-file migration.
Larger runners remain billed for public repositories; see [GitHub runner billing](https://docs.github.com/en/actions/concepts/runners/larger-runners#billing). Actual spending comes from organization billing, not runner labels or an estimated build duration.
+8 -27
View File
@@ -12,8 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Scheduled development artifacts and reusable immutable-candidate builds.
# The daily cron builds Linux. Sunday 16:17 UTC builds macOS and Windows.
# Nightly development artifacts and reusable immutable-candidate builds.
# Public release writes live exclusively in release-publish.yml. Candidate callers
# select Linux or desktop and pin release inputs before any matrix starts.
@@ -31,8 +30,7 @@ on:
required: true
type: string
schedule:
- cron: "13 1 * * *" # Daily Linux, 01:13 UTC
- cron: "17 16 * * 0" # Weekly macOS and Windows, Sunday 16:17 UTC
- cron: "13 1 * * *" # Nightly 01:13 UTC
workflow_dispatch:
inputs:
publish:
@@ -221,33 +219,16 @@ jobs:
env:
# via env, not interpolation: a dispatch input is free-form text and
# would otherwise be pasted into the script for bash to evaluate.
# A direct schedule has no platforms input. Choose its subset from
# github.event.schedule, which is fixed when that cron fires, so a
# queued job cannot switch subsets because the runner clock moved.
# Callers that pass platforms keep that list, including a scheduled
# caller: inputs.platforms is set even though event_name stays schedule.
# Daily development builds cover Linux. Desktop remains available on
# manual dispatch and is required after candidate field acceptance.
RAW_PLATFORMS: ${{ inputs.platforms || 'linux' }}
HAS_PLATFORM_INPUT: ${{ inputs.platforms && 'true' || 'false' }}
EVENT_NAME: ${{ github.event_name }}
EVENT_SCHEDULE: ${{ github.event.schedule }}
run: |
set -euo pipefail
if [[ "${EVENT_NAME}" == "schedule" && "${HAS_PLATFORM_INPUT}" != "true" ]]; then
if [[ "${EVENT_SCHEDULE}" == "13 1 * * *" ]]; then
selected="linux"
elif [[ "${EVENT_SCHEDULE}" == "17 16 * * 0" ]]; then
selected="desktop"
else
echo "Unrecognized schedule cron: ${EVENT_SCHEDULE}" >&2
exit 1
fi
else
selected="${RAW_PLATFORMS}"
selected="$(echo "${selected}" | tr -d '[:space:]')"
if [[ -z "${selected}" ]]; then
selected="all"
fi
selected="$RAW_PLATFORMS"
selected="$(echo "${selected}" | tr -d '[:space:]')"
if [[ -z "${selected}" ]]; then
selected="all"
fi
# linux-aarch64-* need a real aarch64 host (placement rule: hardware)
+25 -36
View File
@@ -33,22 +33,22 @@
# cache-save-if to anything but 'false' for it; every lane in ci.yml reads.
# scripts/security/check_cache_save_if.sh keeps the declarations explicit.
#
# A reader restores only what the writer saved, and Cargo reuses a dependency
# only when it resolves the same features for it. Every package selection and
# feature set resolves differently and changes -Cmetadata, so each job builds
# its readers' own invocations, not a superset (rustfs/backlog#1600).
#
# Rust-cache never replaces an entry restored from an exact key match: the key
# hashes the lockfile, toolchain and compiler environment, not the commit. A
# job whose restore was an exact hit therefore skips its builds; they could
# only produce what is already stored.
# The builds are supersets of what the reading lanes compile, because a reader
# restores only what the writer saved. Feature resolution matters here: a lane
# built with e2e-test-hooks resolves dependency features differently, which
# changes -Cmetadata, so the plain build does not cover it. See
# rustfs/backlog#1600.
name: Cache Warm
on:
schedule:
# Writes a key that no push created, such as one for a new stable
# toolchain; with every key present the jobs stop after the restore.
# Every six hours; the 22:41 UTC run lands before ci.yml's nightly 00:11
# UTC matrix. A reader restores the newest entry under its key and
# rebuilds whatever its own diff changed, so a cache up to six hours old
# costs a PR the handful of crates merged since. Warming after every merge
# (fourteen completed runs a day, about two sm-standard-4 hours each)
# spent far more than that to avoid it.
- cron: "41 4,10,16,22 * * *"
push:
branches: [ main ]
@@ -96,8 +96,8 @@ env:
RUST_BACKTRACE: 1
jobs:
# Readers: workspace-clippy, workspace-lint-and-build, build-rustfs-debug-binary,
# connect-feature-boundaries (nightly) and reference-parity.yml.
# Readers: workspace-lint-and-build, workspace-tests (workspace and ILM), build-rustfs-debug-binary,
# e2e-tests, e2e-full.
warm-ci-dev:
name: Warm ci-dev
runs-on: sm-standard-4
@@ -111,13 +111,10 @@ jobs:
persist-credentials: false
- name: Setup Rust environment
id: setup
uses: ./.github/actions/setup
with:
rust-version: stable
# Digest of the build steps below; scripts/ci_gate.py --check-workflow
# names the new key when they change, and every reader must follow.
cache-shared-key: ci-dev-ea7eec74fe14
cache-shared-key: ci-dev
cache-save-if: 'true'
install-build-packaging-tools: 'false'
@@ -133,7 +130,7 @@ jobs:
# 50% and 25% respectively; if linking dominates instead, the answer is
# mold/lld plus split-debuginfo, which is exactly the part sccache cannot
# touch. Off by default — this doubles the ci-dev build.
- name: Build the workspace with --timings
- name: Build ci-dev superset (with --timings)
if: inputs.emit_timings
env:
CARGO_BUILD_JOBS: "2"
@@ -148,18 +145,18 @@ jobs:
retention-days: 30
if-no-files-found: error
# The workspace and E2E test archives of workspace-lint-and-build and
# build-rustfs-debug-binary, then that job's hooks server.
- name: Build ci-dev reader selections
if: steps.setup.outputs.cache-hit != 'true'
# --all-targets covers the test binaries nextest builds, including
# e2e_test, which the workspace run excludes. The second build adds
# the e2e-test-hooks feature resolution that build-rustfs-debug-binary uses
# and that no lint lane enables.
- name: Build ci-dev superset
env:
# Same limit ci.yml puts on its archive steps: three concurrent test
# binary links saturate the self-hosted runner's overlay I/O and can
# wedge Cargo (#5394).
# Same limit ci.yml puts on its nextest step: this builds the same
# ~100 workspace test binaries, and three concurrent links saturate the
# self-hosted runner's overlay I/O and can wedge Cargo (#5394).
CARGO_BUILD_JOBS: "2"
run: |
python3 scripts/ci_test_artifact.py cargo warm --configuration workspace
python3 scripts/ci_test_artifact.py cargo warm --configuration e2e
cargo build --workspace --all-targets
cargo build -p rustfs --bins --features e2e-test-hooks
# ci.yml runs Clippy on every pull request. Clippy's check-mode artifacts
@@ -168,9 +165,8 @@ jobs:
# touches. A lint failure on main must not stop the warm-up: the cache is
# still saved (cache-on-failure), and the failure belongs to ci.yml.
- name: Warm Clippy check artifacts
if: steps.setup.outputs.cache-hit != 'true'
run: |
if ! python3 scripts/ci_test_artifact.py cargo lint --configuration workspace; then
if ! cargo clippy --all-targets -- -D warnings; then
echo "::warning::cargo clippy reports findings on this commit; check artifacts are still cached"
fi
@@ -217,7 +213,6 @@ jobs:
persist-credentials: false
- name: Setup Rust environment
id: setup
uses: ./.github/actions/setup
with:
rust-version: stable
@@ -226,7 +221,6 @@ jobs:
install-build-packaging-tools: 'false'
- name: Build ci-feat-rio superset
if: steps.setup.outputs.cache-hit != 'true'
env:
# --all-targets links the same test binaries as the reader lane.
CARGO_BUILD_JOBS: "2"
@@ -250,7 +244,6 @@ jobs:
persist-credentials: false
- name: Setup Rust environment
id: setup
uses: ./.github/actions/setup
with:
rust-version: stable
@@ -259,7 +252,6 @@ jobs:
install-build-packaging-tools: 'false'
- name: Build ci-feat-proto superset
if: steps.setup.outputs.cache-hit != 'true'
env:
# Avoid an unbounded burst of concurrent test-binary links (#5394).
CARGO_BUILD_JOBS: "2"
@@ -282,7 +274,6 @@ jobs:
persist-credentials: false
- name: Setup Rust environment
id: setup
uses: ./.github/actions/setup
with:
rust-version: stable
@@ -291,9 +282,7 @@ jobs:
install-build-packaging-tools: 'false'
- name: Install build dependencies
if: steps.setup.outputs.cache-hit != 'true'
run: sudo apt-get update && sudo apt-get install -y protobuf-compiler
- name: Build ci-uring superset
if: steps.setup.outputs.cache-hit != 'true'
run: cargo build -p rustfs-ecstore --all-targets
+124 -111
View File
@@ -46,6 +46,11 @@ on:
schedule:
- cron: "11 0 * * *" # Nightly 00:11 UTC, after the 22:41 UTC cache-warm run
workflow_dispatch:
inputs:
rehearse_workspace_shards:
description: Compare one full execution with at most two resource-safe shards of the same archive
type: boolean
default: false
permissions:
contents: read
@@ -83,16 +88,13 @@ jobs:
if: github.event_name != 'pull_request' || github.event.action != 'closed'
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
contents: read
actions: read
outputs:
mode: ${{ steps.scope.outputs.mode }}
plan: ${{ steps.scope.outputs.plan }}
connect: ${{ steps.scope.outputs.connect || 'true' }}
packages: ${{ steps.scope.outputs.packages }}
full_validation: ${{ steps.scope.outputs.full_validation || 'true' }}
gateway_smoke: ${{ steps.scope.outputs.gateway_smoke || 'true' }}
rehearse_shards: ${{ steps.scope.outputs.rehearse_shards || 'false' }}
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
@@ -103,13 +105,8 @@ jobs:
env:
CI_BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }}
CI_EVENT_ACTION: ${{ github.event.action }}
# A push whose tree a successful merge-group run already passed skips
# the validation jobs; the lookup answers null on any doubt.
PROOF_TOKEN: ${{ github.event_name == 'push' && github.token || '' }}
CI_REHEARSE_SHARDS: ${{ inputs.rehearse_workspace_shards && 'true' || 'false' }}
run: |
# Without a token (every event but push) the lookup answers null at once.
CI_MERGE_GROUP_PROOF="$(GITHUB_TOKEN="$PROOF_TOKEN" python3 -I scripts/ci_merge_group_proof.py)" || CI_MERGE_GROUP_PROOF=null
export CI_MERGE_GROUP_PROOF
if [[ "$GITHUB_EVENT_NAME" != "pull_request" ]]; then
python3 -I scripts/ci_gate.py select
elif [[ "$CI_BASE_SHA" =~ ^[0-9a-f]{40}$ ]] && git show "$CI_BASE_SHA:scripts/ci_gate.py" > "$RUNNER_TEMP/ci-gate-base.py"; then
@@ -118,8 +115,8 @@ jobs:
printf '%s\n' 'mode=full' >> "$GITHUB_OUTPUT"
echo "Base CI policy unavailable; running the full matrix."
fi
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=8$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'gateway_smoke=true' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
if ! grep -q '^plan=' "$GITHUB_OUTPUT" || ! grep -q '^plan_contract=4$' "$GITHUB_OUTPUT"; then
printf '%s\n' 'mode=full' 'connect=true' 'full_validation=true' 'rehearse_shards=false' 'packages=' 'plan=' >> "$GITHUB_OUTPUT"
echo "Base policy does not cover this workflow contract; retaining the full matrix."
fi
- name: Retain verification plan
@@ -173,67 +170,6 @@ jobs:
- name: Run shared quick checks
uses: ./.github/actions/quick-checks
# Clippy runs beside the test-binary build instead of in front of it. The two
# share no compiler output: Clippy writes check-mode metadata, the archive
# needs codegen and linked test binaries, and both restore the third-party
# artifacts of their own mode from ci-dev. As one job the lint held the
# archive, and every test that consumes it, back by its whole duration.
# scripts/ci_gate.py keeps this job in the Test and Lint aggregate and out
# of the archive consumer's dependency chain.
workspace-clippy:
name: Workspace Clippy
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
# Reads the ci-dev cache that cache-warm.yml writes on this image, and the
# all-targets check of the workspace does not fit a hosted runner's disk.
runs-on: sm-standard-4
timeout-minutes: 60
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
# Checkout otherwise writes the token into .git/config, where a PR's
# own build.rs or proc-macro could read it back out.
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
# Read-only, like every other ci-dev reader; cache-warm.yml warms the
# check artifacts of this exact invocation.
cache-shared-key: ci-dev-ea7eec74fe14
cache-save-if: 'false'
install-build-packaging-tools: 'false'
# Clippy gates every event, pull requests included. When it ran only after
# merge, a lint regression merged freely and then failed every main run
# until someone fixed it (10 of 12 main runs on 2026-10-17/18). cache-warm
# writes the same invocation's check artifacts into ci-dev, so a pull
# request re-checks only the crates its diff touches. Main keeps the step
# because pull requests can merge without being up to date with main.
- name: Run clippy lints
run: |
./scripts/ci/resource_sampler.sh start clippy
trap './scripts/ci/resource_sampler.sh stop' EXIT
python3 scripts/ci_test_artifact.py cargo lint --configuration workspace
- name: Upload lint diagnostics
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: workspace-clippy-diagnostics-${{ github.run_number }}-${{ github.run_attempt }}
path: artifacts/workspace
retention-days: 3
if-no-files-found: error
# Keeps its job ID and check name: it is the archive producer that
# workspace-tests and scripts/check_test_wiring.py bind to. The lint half
# is workspace-clippy above; this job builds the test archive, then runs
# the doctests and the log-analyzer anchor check on the same build output.
workspace-lint-and-build:
name: Workspace Lint and Build
if: contains(fromJSON('["full", "e2e"]'), needs.classify-changes.outputs.mode) && (github.event_name != 'pull_request' || github.event.action != 'closed')
@@ -258,7 +194,7 @@ jobs:
# Every lane reads its cache; cache-warm.yml remains the sole writer
# with the matching feature closure. Tests never compete to replace
# shared caches, including when a PR attempt is cancelled.
cache-shared-key: ci-dev-ea7eec74fe14
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
@@ -275,6 +211,18 @@ jobs:
echo "started_at=$(date --utc --iso-8601=seconds)"
} > artifacts/workspace/run-metadata.txt
# Clippy gates every event, pull requests included. When it ran only after
# merge, a lint regression merged freely and then failed every main run
# until someone fixed it (10 of 12 main runs on 2026-10-17/18). cache-warm
# writes the same invocation's check artifacts into ci-dev, so a pull
# request re-checks only the crates its diff touches. Main keeps the step
# because pull requests can merge without being up to date with main.
- name: Run clippy lints
run: |
./scripts/ci/resource_sampler.sh start clippy
trap './scripts/ci/resource_sampler.sh stop' EXIT
python3 scripts/ci_test_artifact.py cargo lint --configuration workspace
- name: Archive workspace test binaries
env:
# Workspace test compilation has high per-process memory use.
@@ -325,7 +273,7 @@ jobs:
# rustfs/backlog#1289: fail if a seed rule's log anchor no longer exists
# verbatim in the source tree (log message drifted without updating the
# rule). Placed here where the workspace — including the la-dump-anchors
# bin's dependencies — is already built by the test archive step above.
# bin — is already built by the clippy/test steps above.
- name: Check log-analyzer rule anchors
run: ./scripts/check_log_analyzer_rules.sh
@@ -336,6 +284,8 @@ jobs:
# requires the same runner image until the hosted-runtime pilot passes.
runs-on: sm-standard-4
timeout-minutes: 90
outputs:
workspace_shards: ${{ steps.shards.outputs.workspace_shards || '[]' }}
env:
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
steps:
@@ -424,6 +374,89 @@ jobs:
retention-days: 3
if-no-files-found: error
- name: Prepare bounded shards from this candidate's observed durations
id: shards
if: needs.classify-changes.outputs.rehearse_shards == 'true'
run: |
python3 scripts/ci_test_shards.py prepare \
--listing artifacts/workspace/core-test-listing.json \
--junit target/nextest/ci/junit.xml --output artifacts/workspace-shard-plan
- name: Upload same-run shard plan
if: needs.classify-changes.outputs.rehearse_shards == 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: workspace-shard-plan-${{ github.run_attempt }}
path: artifacts/workspace-shard-plan
retention-days: 3
if-no-files-found: error
workspace-shards:
name: Workspace shards (${{ matrix.shard }})
if: needs.classify-changes.outputs.rehearse_shards == 'true'
needs: [classify-changes, workspace-tests]
# Reuse the producer image and checkout path for the full RustFS archive.
runs-on: sm-standard-4
timeout-minutes: 90
strategy:
fail-fast: false
max-parallel: 2
matrix:
shard: ${{ fromJSON(needs.workspace-tests.outputs.workspace_shards || '[]') }}
env:
CI_PACKAGES: ${{ needs.classify-changes.outputs.packages }}
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: ./.github/actions/test-runtime
- name: Protect Connect test home
run: chmod go-w "$(realpath "$HOME")"
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: workspace-test-binaries-${{ github.run_attempt }}
path: ${{ runner.temp }}
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: workspace-shard-plan-${{ github.run_attempt }}
path: artifacts/workspace-shard-plan
- name: Verify and execute this shard without compilation
env:
SHARD_INDEX: ${{ matrix.shard }}
run: |
python3 scripts/ci_test_artifact.py verify --configuration workspace --archive "$RUNNER_TEMP/workspace-tests.tar.zst"
python3 scripts/ci_test_shards.py execute --plan artifacts/workspace-shard-plan \
--archive "$RUNNER_TEMP/workspace-tests.tar.zst" --index "$SHARD_INDEX" --output artifacts/workspace-shard-result
- name: Retain shard evidence including failures
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: workspace-shard-result-${{ matrix.shard }}-${{ github.run_attempt }}
path: artifacts/workspace-shard-result
retention-days: 3
if-no-files-found: error
workspace-shard-evidence:
name: Workspace shard completeness
if: always() && needs.classify-changes.outputs.rehearse_shards == 'true'
needs: [classify-changes, workspace-tests, workspace-shards]
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: workspace-shard-plan-${{ github.run_attempt }}
path: artifacts/workspace-shard-plan
- uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: workspace-shard-result-*-${{ github.run_attempt }}
path: artifacts/workspace-shards
- name: Require the exact union of successful shard executions
run: |
python3 scripts/ci_test_shards.py verify --plan artifacts/workspace-shard-plan --evidence artifacts/workspace-shards
./scripts/check_migration_gate_count.sh evidence artifacts/workspace-shard-plan/listing.json artifacts/workspace-shards/combined.xml
connect-feature-boundaries:
name: Connect Feature Boundaries
@@ -443,7 +476,7 @@ jobs:
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev-ea7eec74fe14
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
install-test-tools: 'false'
@@ -469,7 +502,7 @@ jobs:
# hold. `cargo check --release` is metadata-only and needs no
# link-concurrency limit; it is cold because ci-dev holds no release
# artifacts.
- name: Reject debug-only features in release builds
- name: Reject short credentials in release builds
run: python3 scripts/check_connect_release_boundary.py
- name: Check offline enrollment E2E root boundary
@@ -615,10 +648,6 @@ jobs:
flags: "--features sftp"
- name: http3
flags: "--features http3,swift"
- name: internode-transport
flags: "--features rustfs/internode-transport"
packages: "-p rustfs-ecstore -p rustfs-rio"
tests: "--no-tests fail -E 'test(peer_transport_)'"
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
@@ -644,7 +673,7 @@ jobs:
run: |
./scripts/ci/resource_sampler.sh start clippy
trap './scripts/ci/resource_sampler.sh stop' EXIT
cargo clippy -p rustfs ${{ matrix.features.packages || '-p rustfs-protocols' }} --all-targets ${{ matrix.features.flags }} -- -D warnings
cargo clippy -p rustfs -p rustfs-protocols --all-targets ${{ matrix.features.flags }} -- -D warnings
- name: Run tests with ${{ matrix.features.name }}
id: protocol-tests
@@ -661,7 +690,7 @@ jobs:
mkdir -p artifacts/protocol-tests
./scripts/ci/resource_sampler.sh start nextest
trap './scripts/ci/resource_sampler.sh stop' EXIT
cargo nextest run --profile ci -p rustfs ${{ matrix.features.packages || '-p rustfs-protocols' }} ${{ matrix.features.flags }} ${{ matrix.features.tests }} \
cargo nextest run --profile ci -p rustfs -p rustfs-protocols ${{ matrix.features.flags }} \
2>&1 | tee artifacts/protocol-tests/nextest.log
- name: Upload protocol test reports and diagnostics
@@ -681,9 +710,9 @@ jobs:
# The gateway S3 stack (rustfs/backlog#2734) sits behind the rustfs
# `gateway-stack` feature, which no other pull-request lane lints or tests.
# Unlike the dormant feature lanes above, this one runs for every code
# change: each operation slice puts its handler tests and its refusal-matrix
# row behind the feature, and a lane that ran only on demand would let them
# rot behind a green pull request.
# change: each operation slice puts its handler tests, its authorization
# census and its end-to-end refusal test behind the feature, and a lane that
# ran only on demand would let them rot behind a green pull request.
if: needs.classify-changes.outputs.mode == 'full' && (github.event_name != 'pull_request' || github.event.action != 'closed')
needs: [ quick-checks, classify-changes ]
# Self-hosted for build scale and cache ownership: a workspace-scale Cargo
@@ -702,7 +731,7 @@ jobs:
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev-ea7eec74fe14
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
@@ -711,10 +740,9 @@ jobs:
- name: Run clippy with gateway-stack
run: cargo clippy --locked -p rustfs --all-targets --features gateway-stack -- -D warnings
# The gateway stack's own tests: the assembly, the operation table and
# its completeness, the shared handler sequence, the stack selection, and
# the embedded server driven through the whole middleware stack. The
# filter names modules, never operations.
# The gateway stack's own tests: the assembly, the authorizer table and
# its census, the handler template, the migrated use cases, and the
# embedded server driven through the whole middleware stack.
- name: Run tests with gateway-stack
env:
# The same bounded link concurrency as the other feature lanes.
@@ -722,7 +750,7 @@ jobs:
run: |
cargo nextest run --profile ci --locked -p rustfs --features gateway-stack --no-tests fail \
--status-level all --final-status-level all \
-E 'test(/gateway_stack|gateway_handlers|server::stack::/) | binary(embedded_gateway_stack_test)'
-E 'test(/gateway_stack|gateway_handlers|server::stack::|list_buckets|legacy_stack_is_always_selectable/) | binary(embedded_gateway_stack_test)'
build-rustfs-debug-binary:
name: Build RustFS Debug Binary
@@ -742,7 +770,7 @@ jobs:
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev-ea7eec74fe14
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
@@ -1113,24 +1141,9 @@ jobs:
# Test and Lint aggregate and becomes required at T2.10. The test-binary
# download, identity, membership, run and completeness steps must stay
# identical to e2e-tests; scripts/ci_gate.py enforces that.
#
# When it runs: every push to main or gateway/integration, merge group,
# nightly schedule and manual dispatch, and every pull request that changes a
# gateway-stack input (GATEWAY_SMOKE_PATHS and the rustfs-gateway pins in
# scripts/ci_gate.py, published by Select CI scope as gateway_smoke).
# Why it is skipped on every other pull request: until T2.10 the gateway
# stack does not serve the whole smoke suite, so the lane fails on any
# revision. A pull request that touches none of its inputs can neither cause
# nor fix that failure, and a red check there says nothing about the change.
# Skipped is the honest state; a failure must never be rendered green, so no
# step here may carry continue-on-error. A selection that is missing or
# unreadable runs the lane. At T2.10, when the lane becomes required, it
# must run on every pull request again: delete the selection, do not extend
# it.
e2e-tests-gateway:
name: e2e-smoke (gateway, report-only until T2.10)
needs: [ classify-changes, build-rustfs-debug-binary ]
if: github.event_name != 'pull_request' || needs.classify-changes.outputs.gateway_smoke != 'false'
needs: [ build-rustfs-debug-binary ]
runs-on: sm-standard-2
timeout-minutes: 30
continue-on-error: true
@@ -1604,9 +1617,10 @@ jobs:
- classify-changes
- typos
- quick-checks
- workspace-clippy
- workspace-lint-and-build
- workspace-tests
- workspace-shards
- workspace-shard-evidence
- core-contract
- connect-feature-boundaries
- fips-provider
@@ -1642,7 +1656,6 @@ jobs:
- required-checks
- typos
- quick-checks
- workspace-clippy
- workspace-lint-and-build
- workspace-tests
- core-contract
-194
View File
@@ -1,194 +0,0 @@
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Nightly both-stack diff of the two e2e profiles only ci.yml runs, e2e-smoke
# and e2e-full (rustfs/backlog#2772, T3.1). The other profiles carry a `stack`
# dimension in their own workflows, where the existing lane is the legacy leg.
#
# One build serves every leg. One preflight starts that binary with
# RUSTFS_S3_STACK=gateway (scripts/dual_stack.py preflight); without a proven
# gateway start the gateway legs do not run and the summary says `skipped:
# gateway stack refuses startup: <why>`. Which stack a leg ran is read from the
# servers' own start-up line. The gateway legs are report-only until T2.10.
#
# Left out of both stacks: `dual_stack_differential::` (servers with an explicit
# stack), `fake_s3_target::`, the upgrade write-readiness cases and one deadline
# case (no server), and `distributed_startup_regression_test::` (needs ci.yml's
# hooks-binary manifest). e2e-full runs without the mc, Vault and iptables
# tools ci.yml installs: their cases fail on both stacks.
name: dual-stack-nightly
on:
workflow_dispatch:
schedule:
# 03:13 UTC nightly, clear of e2e-nightly (04:29), the ODM lane (05:23) and
# e2e-distributed (05:53).
- cron: "13 3 * * *"
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: ${{ github.event_name != 'schedule' }}
permissions:
contents: read
defaults:
run:
shell: bash
jobs:
build:
name: Build once and probe the gateway stack
if: github.repository == 'rustfs/rustfs'
# The runner and cache of ci.yml's build-rustfs-debug-binary.
runs-on: sm-standard-4
timeout-minutes: 60
outputs:
stacks: ${{ steps.preflight.outputs.stacks }}
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Build the rustfs binary
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
- name: Upload the binary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dual-stack-binary
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
if-no-files-found: error
retention-days: 1
# Without a proven gateway start there is no gateway leg to run.
- name: Gateway preflight
id: preflight
run: |
if python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/preflight/stack.json"; then
echo 'stacks=["legacy","gateway"]' >> "$GITHUB_OUTPUT"
else
echo 'stacks=["legacy"]' >> "$GITHUB_OUTPUT"
fi
# The evidence stands in for a gateway leg that is skipped; a leg that runs overwrites it.
- name: Upload the preflight evidence for e2e-smoke
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dual-stack-leg-e2e-smoke-gateway
path: ${{ runner.temp }}/preflight
if-no-files-found: warn
retention-days: 7
- name: Upload the preflight evidence for e2e-full
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: dual-stack-leg-e2e-full-gateway
path: ${{ runner.temp }}/preflight
if-no-files-found: warn
retention-days: 7
leg:
name: ${{ matrix.profile }} (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
needs: [ build ]
runs-on: sm-standard-4
timeout-minutes: 120
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy:
fail-fast: false
matrix:
profile: [ e2e-smoke, e2e-full ]
stack: ${{ fromJSON(needs.build.outputs.stacks) }}
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Download the binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: dual-stack-binary
path: target/debug
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install awscurl
run: |
python3 -m pip install --user --upgrade pip "awscurl==0.44"
echo "AWSCURL_PATH=$HOME/.local/bin/awscurl" >> "$GITHUB_ENV"
test -x "$HOME/.local/bin/awscurl"
- name: Run the profile on the ${{ matrix.stack }} stack
env:
PROFILE: ${{ matrix.profile }}
RUSTFS_S3_STACK: ${{ matrix.stack }}
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/server-logs
run: |
chmod +x target/debug/rustfs
filter='not test(/^(dual_stack_differential|fake_s3_target|distributed_startup_regression_test)::/)'
filter="$filter & not test(/^upgrade_compatibility_test::upgrade_write_readiness_tests::/)"
filter="$filter & not test(=list_objects_v2_pagination_test::tests::test_list_objects_v2_deadline_rejects_unresponsive_peer)"
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile "$PROFILE" -p e2e_test -E "$filter"
- name: Prove the stack and upload the leg
if: always()
uses: ./.github/actions/dual-stack-leg
with: { profile: "${{ matrix.profile }}", stack: "${{ matrix.stack }}", logs: "${{ runner.temp }}/server-logs" }
summary:
name: Both-stack diff summary
needs: [ build, leg ]
if: github.repository == 'rustfs/rustfs' && always()
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: ./.github/actions/dual-stack-summary
with:
profiles: e2e-smoke,e2e-full
+2 -33
View File
@@ -53,14 +53,11 @@ concurrency:
jobs:
distributed:
if: github.repository == 'rustfs/rustfs'
name: Distributed 4-node 4-disk e2e (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
name: Distributed 4-node 4-disk e2e
# GitHub-hosted VM: loop and tmpfs mounts work here. sm-standard-4 is an
# ARC pod and rejects both (`mount -o loop` ENOENT, tmpfs "read-only").
runs-on: ubuntu-latest
timeout-minutes: 180
# The gateway leg is report-only until T2.10.
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NO_PROXY: 127.0.0.1,localhost
@@ -132,11 +129,6 @@ jobs:
run: |
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
# A gateway binary that refuses to start skips the suite; the summary says why.
- name: Gateway preflight
if: matrix.stack == 'gateway'
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
- name: Verify distributed e2e membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-distributed-list.json
@@ -148,7 +140,6 @@ jobs:
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-distributed-logs
FILTER: ${{ inputs.filter }}
RUSTFS_S3_STACK: ${{ matrix.stack }}
run: |
set -euo pipefail
rm -f target/nextest/e2e-distributed/junit.xml
@@ -159,19 +150,11 @@ jobs:
fi
test -s target/nextest/e2e-distributed/junit.xml
- name: Prove the stack and upload the leg
if: always()
uses: ./.github/actions/dual-stack-leg
with:
profile: e2e-distributed
stack: ${{ matrix.stack }}
logs: ${{ runner.temp }}/rustfs-e2e-distributed-logs
- name: Upload distributed e2e diagnostics
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-distributed-${{ github.run_number }}-${{ matrix.stack }}
name: e2e-distributed-${{ github.run_number }}
path: |
target/nextest/e2e-distributed/junit.xml
${{ runner.temp }}/rustfs-e2e-distributed-list.json
@@ -191,20 +174,6 @@ jobs:
fi
done
dual-stack-summary:
name: Both-stack diff summary
needs: [distributed]
if: github.repository == 'rustfs/rustfs' && always()
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: ./.github/actions/dual-stack-summary
with:
profiles: e2e-distributed
alert-on-failure:
name: Alert on scheduled failure
needs: [distributed]
+8 -78
View File
@@ -54,15 +54,12 @@ concurrency:
jobs:
repl-nightly:
if: github.repository == 'rustfs/rustfs'
name: Replication e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
name: Replication e2e (nightly)
# dual-node tests spawn two full rustfs servers each; the extra headroom
# over the per-PR sm-standard-2 e2e job keeps the parallel process fan-out
# from starving the runner. Mirrors the ILM serial lane's runner class.
runs-on: sm-standard-4
timeout-minutes: 45
# The gateway leg is report-only until T2.10.
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
@@ -98,11 +95,6 @@ jobs:
run: |
python3 scripts/e2e_binary.py build --bins
# A gateway binary that refuses to start skips the suite; the summary says why.
- name: Gateway preflight
if: matrix.stack == 'gateway'
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
- name: Verify replication e2e membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
@@ -113,22 +105,13 @@ jobs:
- name: Run replication e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
RUSTFS_S3_STACK: ${{ matrix.stack }}
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-repl-nightly -p e2e_test
- name: Prove the stack and upload the leg
if: always()
uses: ./.github/actions/dual-stack-leg
with:
profile: e2e-repl-nightly
stack: ${{ matrix.stack }}
logs: ${{ runner.temp }}/rustfs-e2e-repl-nightly-logs
- name: Upload nextest junit report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-replication-nightly-junit-${{ github.run_number }}-${{ matrix.stack }}
name: e2e-replication-nightly-junit-${{ github.run_number }}
path: |
target/nextest/e2e-repl-nightly/junit.xml
${{ runner.temp }}/rustfs-e2e-repl-nightly-list.json
@@ -138,12 +121,9 @@ jobs:
cluster-nightly:
if: github.repository == 'rustfs/rustfs'
name: Cluster fault e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
name: Cluster fault e2e (nightly)
runs-on: sm-standard-4
timeout-minutes: 90
# The gateway leg is report-only until T2.10.
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
@@ -164,11 +144,6 @@ jobs:
run: |
python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
# A gateway binary that refuses to start skips the suite; the summary says why.
- name: Gateway preflight
if: matrix.stack == 'gateway'
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
- name: Verify cluster fault e2e membership
env:
NEXTEST_LISTING: ${{ runner.temp }}/rustfs-e2e-nightly-list.json
@@ -180,22 +155,13 @@ jobs:
env:
CARGO_BIN_EXE_rustfs: ${{ github.workspace }}/target/debug/rustfs
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-nightly-logs
RUSTFS_S3_STACK: ${{ matrix.stack }}
run: python3 scripts/e2e_binary.py run --features e2e-test-hooks -- cargo nextest run --profile e2e-nightly -p e2e_test
- name: Prove the stack and upload the leg
if: always()
uses: ./.github/actions/dual-stack-leg
with:
profile: e2e-nightly
stack: ${{ matrix.stack }}
logs: ${{ runner.temp }}/rustfs-e2e-nightly-logs
- name: Upload cluster fault diagnostics
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-cluster-nightly-${{ github.run_number }}-${{ matrix.stack }}
name: e2e-cluster-nightly-${{ github.run_number }}
path: |
target/nextest/e2e-nightly/junit.xml
${{ runner.temp }}/rustfs-e2e-nightly-list.json
@@ -205,12 +171,9 @@ jobs:
protocols-nightly:
if: github.repository == 'rustfs/rustfs'
name: Protocol e2e (nightly, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
name: Protocol e2e (nightly)
runs-on: sm-standard-4
timeout-minutes: 90
# The gateway leg is report-only until T2.10.
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
RUSTFS_BUILD_FEATURES: ftps,webdav,sftp
@@ -237,11 +200,6 @@ jobs:
- name: Build protocol server
run: python3 scripts/e2e_binary.py build --features "$RUSTFS_BUILD_FEATURES"
# A gateway binary that refuses to start skips the suite; the summary says why.
- name: Gateway preflight
if: matrix.stack == 'gateway'
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
# The suite owns fixed protocol ports and serializes its internal cases.
- name: Verify protocol e2e membership
env:
@@ -253,28 +211,14 @@ jobs:
- name: Run protocol e2e nightly suite
env:
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-protocol-e2e-logs
RUSTFS_S3_STACK: ${{ matrix.stack }}
# These servers set no log level of their own; the stack proof reads their start-up line.
RUST_LOG: rustfs=info
# The servers inherit nextest's stdout (--no-capture): keep it where the stack proof reads it.
run: |
set -o pipefail
mkdir -p "$RUSTFS_E2E_LOG_DIR"
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture 2>&1 | tee "$RUSTFS_E2E_LOG_DIR/nextest.log"
- name: Prove the stack and upload the leg
if: always()
uses: ./.github/actions/dual-stack-leg
with:
profile: e2e-protocols
stack: ${{ matrix.stack }}
logs: ${{ runner.temp }}/rustfs-protocol-e2e-logs
run: >-
python3 scripts/e2e_binary.py run --features "$RUSTFS_BUILD_FEATURES" -- cargo nextest run -j 1 --profile e2e-protocols -p e2e_test --no-capture
- name: Upload protocol diagnostics
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: e2e-protocol-nightly-${{ github.run_number }}-${{ matrix.stack }}
name: e2e-protocol-nightly-${{ github.run_number }}
path: |
target/nextest/e2e-protocols/junit.xml
${{ runner.temp }}/rustfs-e2e-protocols-list.json
@@ -282,20 +226,6 @@ jobs:
retention-days: 7
if-no-files-found: warn
dual-stack-summary:
name: Both-stack diff summary
needs: [repl-nightly, cluster-nightly, protocols-nightly]
if: github.repository == 'rustfs/rustfs' && always()
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: ./.github/actions/dual-stack-summary
with:
profiles: e2e-repl-nightly,e2e-nightly,e2e-protocols
alert-on-failure:
name: Alert on scheduled failure
needs: [repl-nightly, cluster-nightly, protocols-nightly]
-395
View File
@@ -1,395 +0,0 @@
# Copyright 2026 RustFS Team
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Gateway black-box acceptance lanes (rustfs/backlog#2775 T3.4, #2776 T3.5).
#
# Runs two suites of the rustfs/gateway repository against the candidate binary,
# once per S3 stack (RUSTFS_S3_STACK=legacy and =gateway), one server per stack:
#
# - the conformance corpus, `--profile rustfs`. That profile describes the
# gateway edge, so scripts/check_conformance_rulings.py judges the gateway
# stack against .config/conformance-rulings.toml (red: an unruled failure, a
# ruled case failing differently, a stale ruling, a skip on the server's own
# answer) and publishes the legacy run as a reference that is never judged:
# it only has to serve, stay up and complete;
# - the client matrix, ci/compat/run_matrix.sh --external, judged per stack by
# scripts/compat_matrix_report.py (no `fail`, no `sut-unregistered`, every
# client ran; a client that could not be installed is reported as skipped).
# The legacy matrix is the baseline: every cell that passes on legacy must
# pass on gateway (`compare`).
#
# The suites are checked out at the rustfs/gateway rev Cargo.toml pins, so the
# suite and the framework under test never drift apart. The verdict of each
# stack is computed from what the run left on disk (startup record, the stack
# the server's own log names, runner exit code, liveness after the corpus,
# report), never from the exit status of an earlier step. Needs
# scripts/dual_stack.py and the `s3_stack` startup field of rustfs/rustfs#9367.
#
# The gateway stack is report-only until T2.10: its job may fail without failing
# the workflow, and its summary says what it observed (did not start, or how many
# cases ran, passed, failed and were answered 501). The stack comparison is
# report-only for the same reason; its conformance half is information only.
#
# Not a pull-request gate. Scheduled failures are owned by the scheduled-validation
# watchdog (.github/scheduled-validations.json).
name: Gateway Acceptance
on:
schedule:
- cron: "29 3 * * *"
workflow_dispatch:
# Matches ci.yml: these variables are hashed into the rust-cache key, so the
# candidate build reads the same ci-dev cache as build-rustfs-debug-binary.
env:
CARGO_TERM_COLOR: always
RUST_BACKTRACE: 1
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
defaults:
run:
shell: bash
jobs:
plan:
name: Read the gateway pin
runs-on: ubuntu-latest
timeout-minutes: 5
outputs:
gateway-rev: ${{ steps.pin.outputs.rev }}
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Read the rustfs/gateway rev from Cargo.toml
id: pin
run: |
set -euo pipefail
python3 - <<'PY' >> "$GITHUB_OUTPUT"
import re, sys, tomllib
dependencies = tomllib.load(open("Cargo.toml", "rb"))["workspace"]["dependencies"]
revs = {spec.get("rev") for spec in dependencies.values()
if isinstance(spec, dict) and spec.get("git") == "https://github.com/rustfs/gateway.git"}
if len(revs) != 1 or not re.fullmatch(r"[0-9a-f]{40}", str(next(iter(revs)))):
sys.exit(f"Cargo.toml must pin rustfs/gateway at exactly one full rev, found {sorted(map(str, revs))}")
print(f"rev={revs.pop()}")
PY
cat "$GITHUB_OUTPUT"
build-candidate:
name: Build candidate binary
# The same debug build as ci.yml's build-rustfs-debug-binary, reading the
# ci-dev cache written on sm-standard-4.
runs-on: sm-standard-4
timeout-minutes: 60
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Build debug binary
run: python3 scripts/e2e_binary.py build --bins --features e2e-test-hooks
- name: Upload candidate binary
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: gateway-acceptance-candidate-binary
overwrite: true
path: |
target/debug/rustfs
target/debug/rustfs.e2e.json
if-no-files-found: error
retention-days: 1
build-gateway-tools:
name: Build gateway conformance runner and compat-sut
needs: [ plan ]
# The gateway workspace builds with its own rust-toolchain.toml, which rustup
# installs on first use; nothing here names a toolchain.
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Checkout rustfs/gateway at the pinned rev
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/gateway
ref: ${{ needs.plan.outputs.gateway-rev }}
path: gateway
persist-credentials: false
- name: Build
working-directory: gateway
run: |
cargo build --locked --release -p rustfs-gateway-conformance -p rustfs-gateway-compat-sut \
--bin rustfs-gateway-conformance --bin compat-sut
- name: Upload tools
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: gateway-acceptance-tools
overwrite: true
path: |
gateway/target/release/rustfs-gateway-conformance
gateway/target/release/compat-sut
if-no-files-found: error
retention-days: 1
acceptance:
name: ${{ matrix.stack == 'gateway' && 'Acceptance (gateway, report-only until T2.10)' || 'Acceptance (legacy)' }}
needs: [ plan, build-candidate, build-gateway-tools ]
# GitHub-hosted like the s3-tests targets of reference-parity.yml: the job
# only runs prebuilt binaries and the pinned clients, none of which needs Docker.
runs-on: ubuntu-latest
# About 3x the 15 minutes the legacy job took in run 43325101234 (client install about
# 8, matrix about 3, corpus about 3), so a hung server costs 45 runner minutes, not more.
timeout-minutes: 45
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy:
fail-fast: false
matrix:
stack: [ legacy, gateway ]
env:
STACK: ${{ matrix.stack }}
ENDPOINT: http://127.0.0.1:9000
# The conformance suite signs with these fixed fixture credentials (the AWS
# documentation example pair), so the candidate is started with them and the
# client matrix is handed the same pair.
ACCESS_KEY: AKIAIOSFODNN7EXAMPLE
SECRET_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Checkout rustfs/gateway at the pinned rev
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
repository: rustfs/gateway
ref: ${{ needs.plan.outputs.gateway-rev }}
path: gateway
persist-credentials: false
- name: Download candidate binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: gateway-acceptance-candidate-binary
path: target/debug
- name: Download gateway tools
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: gateway-acceptance-tools
path: tools
# Records `ready`, or what the process said when it exited, in lane/startup.txt. `ready`
# also needs the server's own `server_starting` log event to name the requested stack
# (its `s3_stack` field comes from rustfs/rustfs#9367; scripts/dual_stack.py reads it).
- name: Start candidate (${{ matrix.stack }} stack)
run: |
set -uo pipefail
chmod +x target/debug/rustfs tools/*
mkdir -p lane "$RUNNER_TEMP"/data/rustfs{0..3}
target/debug/rustfs --version | tee lane/candidate-version.txt
# The four directories share the runner's one disk; the bypass is the CI case the
# physical-disk guard sanctions (as in mint.yml).
RUSTFS_S3_STACK="$STACK" RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true RUST_LOG=info target/debug/rustfs --address 127.0.0.1:9000 \
--access-key "$ACCESS_KEY" --secret-key "$SECRET_KEY" "$RUNNER_TEMP"/data/rustfs{0..3} \
> lane/rustfs.log 2>&1 &
pid=$!
echo "$pid" > lane/rustfs.pid
# Liveness first: a ready answer counts only while this process is still running.
for _ in $(seq 120); do
if ! kill -0 "$pid" 2>/dev/null; then
status=0
wait "$pid" || status=$?
echo "did not start: exit ${status}: $(grep -v '^[[:space:]]*$' lane/rustfs.log | tail -n 1)" > lane/startup.txt
cat lane/startup.txt
exit 1
fi
if curl -sf "$ENDPOINT/health/ready" >/dev/null; then
if python3 scripts/dual_stack.py stack --expect "$STACK" --out lane/stack.json lane/rustfs.log > /dev/null; then
echo ready > lane/startup.txt
exit 0
fi
echo "did not start: stack not observed" > lane/startup.txt
cat lane/startup.txt
exit 1
fi
sleep 1
done
echo "did not start: /health/ready did not answer within 120 seconds" > lane/startup.txt
exit 1
# The exit code is data for the judge step, not this step's result.
- name: Run gateway conformance (rustfs profile)
run: |
set +e
tools/rustfs-gateway-conformance run --root gateway/conformance --profile rustfs \
--endpoint "$ENDPOINT" --allow-external-fixtures \
--json lane/report.json > lane/conformance.log 2>&1
echo "$?" > lane/runner-exit.txt
if kill -0 "$(cat lane/rustfs.pid)" 2>/dev/null; then echo alive; else echo exited; fi > lane/alive-after-conformance.txt
tail -n 20 lane/conformance.log
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
# Toolchains for the matrix's SDK driver programs, as the gateway's own
# client-matrix.yml sets them up; the client versions come from
# gateway/compat/versions.toml.
- uses: actions/setup-go@41dfa10bad2bb2ae585af6ee5bb4d7d973ad74ed # v5.1.0
with:
go-version: "1.26"
cache: false
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "22"
- uses: actions/setup-dotnet@67a3573c9a986a3f9c594539f4ab511d57bb3ce9 # v4.3.1
with:
dotnet-version: "8.0.x"
- uses: actions/setup-java@cf277c60eb25467037889841efdb72551f06f6c3 # v4.9.1
with:
distribution: temurin
java-version: "21"
- name: Install Maven
env:
MAVEN_VERSION: "3.9.16"
MAVEN_SHA512: "831a8591fe20c8243b1dbe7d71e3244f31d1665b0804b2e825e38cbbe5ce0cafb8338851f90780735568773e0a6cd07bbec107cda0b896b008b861075358b6f6"
run: |
set -euo pipefail
archive="$RUNNER_TEMP/apache-maven.tar.gz"
curl -sSfL -o "$archive" "https://archive.apache.org/dist/maven/maven-3/${MAVEN_VERSION}/binaries/apache-maven-${MAVEN_VERSION}-bin.tar.gz"
echo "${MAVEN_SHA512} ${archive}" | sha512sum -c -
tar -xzf "$archive" -C "$RUNNER_TEMP"
echo "$RUNNER_TEMP/apache-maven-${MAVEN_VERSION}/bin" >> "$GITHUB_PATH"
# One client at a time, so a client that cannot be installed is reported
# as skipped with its reason instead of stopping the others.
- name: Install pinned clients
working-directory: gateway
run: |
set -uo pipefail
python3 -m pip install --disable-pip-version-check "pyyaml==6.0.2"
installed=()
for client in $(python3 -c 'import tomllib; print(" ".join(sorted(tomllib.load(open("compat/versions.toml", "rb"))["clients"])))'); do
if bash ci/compat/install_clients.sh --clients "$client" > "../lane/install-${client}.log" 2>&1; then
installed+=("$client")
else
echo "${client}: skipped: $(tail -n 1 "../lane/install-${client}.log")" >> ../lane/skipped-clients.txt
fi
done
echo "COMPAT_CLIENTS=$(IFS=,; echo "${installed[*]}")" >> "$GITHUB_ENV"
cat ../lane/skipped-clients.txt 2>/dev/null || echo "every pinned client installed"
# The exit code is data for the judge step, not this step's result.
- name: Run client matrix (external)
working-directory: gateway
env:
GATEWAY_COMPAT_SUT_BIN: ${{ github.workspace }}/tools/compat-sut
GATEWAY_COMPAT_RUN_DIR: ${{ github.workspace }}/lane/compat
GATEWAY_COMPAT_MATRIX_OUT: ${{ github.workspace }}/lane/compat/matrix.json
run: |
set +e
export GATEWAY_COMPAT_ACCESS_KEY="$ACCESS_KEY" GATEWAY_COMPAT_SECRET_KEY="$SECRET_KEY"
PATH="$(go env GOPATH)/bin:$PATH"
if [[ -f ../lane/skipped-clients.txt ]]; then
bash ci/compat/run_matrix.sh --external "$ENDPOINT" --product rustfs --clients "$COMPAT_CLIENTS"
else
bash ci/compat/run_matrix.sh --external "$ENDPOINT" --product rustfs
fi
echo "$?" > ../lane/matrix-exit.txt
- name: Judge and summarize
if: always()
run: |
set -uo pipefail
status=0
python3 scripts/check_conformance_rulings.py --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
python3 scripts/compat_matrix_report.py check --stack "$STACK" --run-dir lane | tee -a "$GITHUB_STEP_SUMMARY" || status=1
exit "$status"
- name: Upload lane results
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: gateway-acceptance-lane-${{ matrix.stack }}
overwrite: true
path: |
lane/*.txt
lane/*.log
lane/report.json
lane/stack.json
lane/compat/matrix.json
lane/compat/probe.jsonl
lane/compat/sut.log
lane/compat/results/**
if-no-files-found: error
retention-days: 14
compare:
name: Compare stacks (report-only until T2.10)
needs: [ acceptance ]
if: always()
runs-on: ubuntu-latest
timeout-minutes: 10
continue-on-error: true
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download lane results
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: gateway-acceptance-lane-*
path: lanes
- name: Diff legacy and gateway
run: |
set -uo pipefail
status=0
{
python3 scripts/check_conformance_rulings.py \
--diff lanes/gateway-acceptance-lane-legacy lanes/gateway-acceptance-lane-gateway || status=1
echo
python3 scripts/compat_matrix_report.py diff \
--legacy lanes/gateway-acceptance-lane-legacy --gateway lanes/gateway-acceptance-lane-gateway || status=1
} > diff.md
cat diff.md
cat diff.md >> "$GITHUB_STEP_SUMMARY"
exit "$status"
+37 -85
View File
@@ -1,13 +1,11 @@
# Copyright 2026 RustFS Team
# SPDX-License-Identifier: Apache-2.0
# Manual Linux zip/deb/rpm packaging. Workflow artifacts only.
# Dispatch uses this file from the selected branch. An empty ref packages
# the tip of refs/heads/main. A non-empty ref must already exist; the
# resolver fetches that tag, branch, or commit so checkout is not limited
# to the default branch. Nothing is published.
# Manual packaging of the current origin/main tip. Workflow artifacts only.
# Dispatch uses this file from the selected branch. The build fetches
# refs/heads/main and checks out that commit, not the dispatch ref or a tag.
name: Manual Linux packages
name: Manual main packages
on:
workflow_dispatch:
@@ -21,16 +19,10 @@ on:
- arm
- all
default: all
ref:
description: "Git tag to package. Leave empty to package the tip of origin/main. A branch name or full commit SHA is also accepted."
required: false
type: string
default: ""
version:
description: "Package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v. Required for origin/main. For a tag, leave empty to derive it from the tag; an explicit value is used as entered."
required: false
description: "Binary and package version: MAJOR.MINOR.PATCH, or MAJOR.MINOR.PATCH-(alpha|beta|rc).N. No leading v."
required: true
type: string
default: ""
permissions:
contents: read
@@ -42,43 +34,48 @@ env:
jobs:
prepare:
name: Resolve source and version
name: Resolve main tip and version
if: github.repository == 'rustfs/rustfs'
runs-on: ubuntu-latest
timeout-minutes: 15
outputs:
source_sha: ${{ steps.source.outputs.source_sha }}
source_kind: ${{ steps.source.outputs.source_kind }}
source_name: ${{ steps.source.outputs.source_name }}
version: ${{ steps.source.outputs.version }}
packaging: ${{ steps.source.outputs.packaging }}
tool_sha: ${{ steps.source.outputs.tool_sha }}
confirm_from: ${{ steps.source.outputs.confirm_from }}
source_sha: ${{ steps.pin.outputs.source_sha }}
version: ${{ steps.version.outputs.version }}
toolchain: ${{ steps.inputs.outputs.toolchain }}
console: ${{ steps.inputs.outputs.console }}
matrix: ${{ steps.matrix.outputs.matrix }}
steps:
# Tooling checkout. An empty ref then detaches this tree to the tip of
# origin/main. A tag or other ref is fetched and pinned without
# replacing this tree, because console resolution needs candidate.py.
- name: Checkout repository
- name: Checkout main
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: main
fetch-depth: 0
persist-credentials: false
- name: Resolve source and version
id: source
- name: Pin the tip of origin/main
id: pin
shell: bash
run: |
set -euo pipefail
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
sha=$(git rev-parse FETCH_HEAD)
git checkout --detach "$sha"
test "$(git rev-parse HEAD)" = "$sha"
printf 'source_sha=%s\n' "$sha" >> "$GITHUB_OUTPUT"
printf 'Pinned origin/main %s\n' "$sha"
- name: Validate package version
id: version
shell: bash
env:
SOURCE_REF: ${{ inputs.ref }}
PACKAGE_VERSION: ${{ inputs.version }}
run: |
set -euo pipefail
bash scripts/release/package_manual_artifacts.sh resolve-source \
--source "$SOURCE_REF" \
--version "$PACKAGE_VERSION" | tee -a "$GITHUB_OUTPUT"
bash scripts/release/package_manual_artifacts.sh validate \
--version "$PACKAGE_VERSION" \
--arch x86_64 \
--libc gnu | tee "$RUNNER_TEMP/package-version.txt"
printf 'version=%s\n' "$PACKAGE_VERSION" >> "$GITHUB_OUTPUT"
- name: Install stable Rust
uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable
@@ -137,45 +134,22 @@ jobs:
fail-fast: false
matrix: ${{ fromJson(needs.prepare.outputs.matrix) }}
steps:
- name: Checkout the pinned source commit
- name: Checkout the pinned origin/main commit
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: ${{ needs.prepare.outputs.source_sha }}
fetch-depth: 0
persist-credentials: false
# Tags cut before the manual packaging script still compile from their
# own tree. The official zip/deb/rpm recipe, and confirm-source when the
# tagged script does not implement it, come from the prepare checkout.
- name: Checkout packaging recipe
if: ${{ needs.prepare.outputs.packaging == 'tool' || needs.prepare.outputs.confirm_from == 'tool' }}
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
ref: ${{ needs.prepare.outputs.tool_sha }}
path: packaging-tools
fetch-depth: 1
persist-credentials: false
- name: Confirm the pinned source
- name: Confirm the checkout is that main commit
shell: bash
env:
SOURCE_KIND: ${{ needs.prepare.outputs.source_kind }}
SOURCE_NAME: ${{ needs.prepare.outputs.source_name }}
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
run: |
set -euo pipefail
helper=scripts/release/package_manual_artifacts.sh
if [[ ! -f "$helper" ]] || ! grep -q 'confirm-source)' "$helper"; then
helper=packaging-tools/scripts/release/package_manual_artifacts.sh
fi
if [[ ! -f "$helper" ]] || ! grep -q 'confirm-source)' "$helper"; then
printf 'no packaging script implements confirm-source\n' >&2
exit 1
fi
bash "$helper" confirm-source \
--kind "$SOURCE_KIND" \
--name "$SOURCE_NAME" \
--sha "$SOURCE_SHA"
test "$(git rev-parse HEAD)" = "$SOURCE_SHA"
GIT_TERMINAL_PROMPT=0 git fetch --no-tags origin refs/heads/main
git merge-base --is-ancestor "$SOURCE_SHA" FETCH_HEAD
- name: Setup Rust environment
uses: ./.github/actions/setup
@@ -273,15 +247,7 @@ jobs:
exit 1
fi
if [[ -n "$BUILD_GLIBC" ]]; then
glibc_check=scripts/release/check_glibc.py
if [[ ! -f "$glibc_check" ]]; then
glibc_check=packaging-tools/scripts/release/check_glibc.py
fi
if [[ ! -f "$glibc_check" ]]; then
printf 'glibc check script not found\n' >&2
exit 1
fi
python3 "$glibc_check" "$BUILD_GLIBC" "$bin"
python3 scripts/release/check_glibc.py "$BUILD_GLIBC" "$bin"
fi
- name: Generate the CPU symbol catalog
@@ -319,23 +285,9 @@ jobs:
LIBC: ${{ matrix.libc }}
BUILD_TARGET: ${{ matrix.target }}
BUILD_FEATURES: ${{ matrix.features }}
SOURCE_KIND: ${{ needs.prepare.outputs.source_kind }}
SOURCE_SHA: ${{ needs.prepare.outputs.source_sha }}
PACKAGING: ${{ needs.prepare.outputs.packaging }}
run: |
set -euo pipefail
case "$PACKAGING" in
source) script=scripts/release/package_manual_artifacts.sh ;;
tool) script=packaging-tools/scripts/release/package_manual_artifacts.sh ;;
*)
printf 'unexpected packaging recipe: %q\n' "$PACKAGING" >&2
exit 1
;;
esac
if [[ ! -f "$script" ]]; then
printf 'packaging script not found: %s\n' "$script" >&2
exit 1
fi
binary="target/${BUILD_TARGET}/release/rustfs"
extra=()
if [[ "$BUILD_TARGET" == *-linux-gnu && "$BUILD_FEATURES" == pyroscope ]]; then
@@ -344,14 +296,14 @@ jobs:
--zip-extra "target/${BUILD_TARGET}/release/rustfs.cpu-symbol-catalog.sha256"
)
fi
bash "$script" package \
bash scripts/release/package_manual_artifacts.sh package \
--version "$PACKAGE_VERSION" \
--arch "$ARCH" \
--libc "$LIBC" \
--binary "$binary" \
--output dist \
"${extra[@]}"
printf 'Packaged %s from %s %s\n' "$PACKAGE_VERSION" "$SOURCE_KIND" "$SOURCE_SHA"
printf 'Packaged %s from origin/main %s\n' "$PACKAGE_VERSION" "$SOURCE_SHA"
- name: Verify the packaged binary
shell: bash
+10 -89
View File
@@ -44,13 +44,6 @@
# well inside the 120-minute budget and buys deterministic infra. The
# docker-capable self-hosted `dind-sm-standard-2` label was the alternative but
# has fewer cores and reintroduces fleet-state risk for no reliability gain.
#
# Stack dimension (rustfs/backlog#2774, T3.3): one image, run with RUSTFS_S3_STACK=legacy and
# =gateway. `compare` (scripts/mint_stack_diff.py) holds the gate that every case legacy passes
# passes on gateway with no new failure. The gateway leg and `compare` are report-only until
# T2.10. A gateway server that refuses to start stops its leg at "Wait for RustFS ready", before
# mint runs; the summary says `skipped: gateway stack refuses startup: <why>`. The stack is read
# from the server's own start-up line (scripts/dual_stack.py stack).
name: mint
@@ -115,8 +108,14 @@ defaults:
shell: bash
jobs:
image:
name: Build the RustFS image once
mint:
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
# label was abandoned (no docker.sock on the pod its only run landed on).
# sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
# buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
# TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
# run) is added by the ci-8 composite action; do not implement it here.
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
@@ -135,48 +134,6 @@ jobs:
--cache-to "type=gha,mode=max,scope=${BUILDX_CACHE_SCOPE}" \
-t rustfs-ci \
-f Dockerfile.source .
docker save rustfs-ci --output "${RUNNER_TEMP}/rustfs-ci.tar"
- name: Upload the image
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: mint-image
path: ${{ runner.temp }}/rustfs-ci.tar
retention-days: 1
mint:
name: mint (${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
needs: [image]
# Report-only until T2.10: the gateway stack cannot serve the suites yet.
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy:
fail-fast: false
matrix:
stack: [legacy, gateway]
env:
STACK: ${{ matrix.stack }}
# GitHub-hosted: reliably provides a running Docker daemon + buildx +
# python3. See the header note (ci-2) for why the self-hosted sm-standard-4
# label was abandoned (no docker.sock on the pod its only run landed on).
# sm-standard-2 has no docker.sock either (weekly runs failed at "Enable
# buildx" from 2026-09-27 to 2026-11-08), so keep this job on ubuntu-latest.
# TODO(ci-8): scheduled-failure alerting (auto-open issue on a red weekly
# run) is added by the ci-8 composite action; do not implement it here.
runs-on: ubuntu-latest
timeout-minutes: 120
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download the image
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: mint-image
path: ${{ runner.temp }}
- name: Load the image
run: docker load --input "${RUNNER_TEMP}/rustfs-ci.tar"
- name: Start RustFS
run: |
@@ -194,8 +151,6 @@ jobs:
-e RUSTFS_SECRET_KEY="${S3_SECRET_KEY}" \
-e RUSTFS_VOLUMES="/data/rustfs{0...3}" \
-e RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true \
-e RUSTFS_S3_STACK="${STACK}" \
-e RUST_LOG=rustfs=info \
-v /tmp/rustfs-mint:/data \
rustfs-ci
@@ -291,50 +246,16 @@ jobs:
mkdir -p artifacts/rustfs-mint
docker logs rustfs-mint > artifacts/rustfs-mint/rustfs.log 2>&1 || true
- name: Prove which stack the server selected
if: always()
run: python3 scripts/dual_stack.py stack --expect "${STACK}" --out artifacts/stack.json artifacts/rustfs-mint
- name: Upload artifacts
if: always() && env.ACT != 'true'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: mint-${{ matrix.stack }}
name: mint
path: artifacts/**
compare:
name: Compare stacks (report-only until T2.10)
needs: [mint]
if: always()
continue-on-error: true
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Download both legs
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: mint-{legacy,gateway}
path: legs
# A leg that never uploaded leaves no directory: create both so that the
# comparison reports "no result" for it instead of failing to find it.
- name: Compare the stacks
run: |
# The shell runs with -e; the exit status is reported after the summary is written.
set +e -u -o pipefail
mkdir -p legs/mint-legacy legs/mint-gateway
python3 scripts/mint_stack_diff.py legs/mint-legacy legs/mint-gateway | tee report.txt
status="${PIPESTATUS[0]}"
{ echo '## Mint: gateway versus legacy'; echo; echo '```'; cat report.txt; echo '```'; } >> "$GITHUB_STEP_SUMMARY"
exit "$status"
alert-on-failure:
name: Alert on scheduled failure
needs: [image, mint]
needs: [mint]
# `always()` is required: without it this job is skipped when a needed
# job fails. Alerts only for scheduled runs (backlog#1149 ci-8) — manual
# dispatch failures are already watched by a human.
@@ -68,14 +68,11 @@ env:
jobs:
minio-source:
name: MinIO source (read-through, list-through, backfill, ${{ matrix.stack }}${{ matrix.stack == 'gateway' && ', report-only until T2.10' || '' }})
name: MinIO source (read-through, list-through, backfill)
# Keep this scheduled interoperability lane scoped to the upstream repo.
if: github.repository == 'rustfs/rustfs'
runs-on: ubuntu-latest
timeout-minutes: 90
# The gateway leg is report-only until T2.10.
continue-on-error: ${{ matrix.stack == 'gateway' }}
strategy: { fail-fast: false, matrix: { stack: [legacy, gateway] } }
env:
NO_PROXY: 127.0.0.1,localhost
# Fixed credentials of the container this job starts and throws away;
@@ -140,11 +137,6 @@ jobs:
- name: Build the RustFS binary under test
run: python3 scripts/e2e_binary.py build --bins
# A gateway binary that refuses to start skips the suite; the summary says why.
- name: Gateway preflight
if: matrix.stack == 'gateway'
run: python3 scripts/dual_stack.py preflight --binary target/debug/rustfs --out "$RUNNER_TEMP/leg/stack.json"
# The lane selects tests by module, so a rename would quietly shrink it.
# The committed digest in .config/e2e-odm-interop-selection.txt fails
# closed on that.
@@ -154,19 +146,8 @@ jobs:
python3 ./scripts/check_test_wiring.py --check-profile e2e-odm-interop "${NEXTEST_LISTING}"
- name: Run the interop cases against MinIO
env:
RUSTFS_S3_STACK: ${{ matrix.stack }}
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-e2e-logs
run: python3 scripts/e2e_binary.py run -- cargo nextest run --profile e2e-odm-interop -p e2e_test --no-tests=fail
- name: Prove the stack and upload the leg
if: always()
uses: ./.github/actions/dual-stack-leg
with:
profile: e2e-odm-interop
stack: ${{ matrix.stack }}
logs: ${{ runner.temp }}/rustfs-e2e-logs
- name: Build the MinIO interop report
if: always()
uses: ./.github/actions/odm-interop-report
@@ -190,7 +171,7 @@ jobs:
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}-${{ matrix.stack }}
name: odm-interop-minio-${{ github.run_number }}-${{ github.run_attempt }}
path: |
artifacts/odm-interop/minio
target/nextest/e2e-odm-interop/junit.xml
@@ -316,20 +297,6 @@ jobs:
target/nextest/e2e-odm-interop/junit.xml
retention-days: 14
dual-stack-summary:
name: Both-stack diff summary
needs: [minio-source]
if: github.repository == 'rustfs/rustfs' && always()
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- uses: ./.github/actions/dual-stack-summary
with:
profiles: e2e-odm-interop
alert-on-failure:
name: Alert on scheduled failure
needs: [minio-source, cloud-source]
+1 -137
View File
@@ -30,16 +30,6 @@
# result and honours .config/reference-parity-tolerated.txt, whose entries must
# each match a real difference.
#
# The `differential` job adds the 73-operation dual-stack differential
# (crates/e2e_test/src/dual_stack_differential, rustfs/backlog#2740) against the
# same pinned reference: side A is the released binary (RUSTFS_E2E_REFERENCE_BINARY,
# started without RUSTFS_S3_STACK), side B is the candidate built above on its
# legacy stack. It reuses the candidate build artifact, never builds the candidate
# twice, and prints reference_side, version_a and version_b from the report the test
# wrote; it fails when side A was not the reference or both sides report the same
# version. Every (op, field, a, b) difference goes to the job summary. Report-only
# until the first result has been reviewed.
#
# Report-only: not part of the Test and Lint aggregate. Scheduled failures are
# owned by the scheduled-validation watchdog (.github/scheduled-validations.json).
@@ -55,7 +45,6 @@ on:
- ".config/reference-parity-tolerated.txt"
- "scripts/s3-tests/reference_parity.py"
- "scripts/junit_diff.py"
- "crates/e2e_test/src/dual_stack_differential/**"
schedule:
- cron: "37 1 * * *"
workflow_dispatch:
@@ -116,7 +105,7 @@ jobs:
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev-ea7eec74fe14
cache-shared-key: ci-dev
cache-save-if: 'false'
install-build-packaging-tools: 'false'
@@ -264,131 +253,6 @@ jobs:
if-no-files-found: error
retention-days: 14
differential:
name: Dual-stack differential (reference vs candidate)
needs: [ build-candidate ]
# Compiles the e2e_test harness against the ci-dev cache like the other
# sm-standard-4 jobs; the candidate server itself is the downloaded artifact.
runs-on: sm-standard-4
timeout-minutes: 90
env:
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: "true"
NO_PROXY: 127.0.0.1,localhost,::1
steps:
- name: Checkout repository
uses: actions/checkout@f548e57e544e1ff5a4c46bf1e1b8685f8e4a348a # v7
with:
persist-credentials: false
- name: Setup Rust environment
uses: ./.github/actions/setup
with:
rust-version: stable
cache-shared-key: ci-dev-ea7eec74fe14
cache-save-if: 'false'
install-build-packaging-tools: 'false'
- name: Fetch pinned reference binary
id: reference
uses: ./.github/actions/reference-binary
- name: Download candidate binary
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: reference-parity-candidate-binary
path: target/debug
- name: Run dual-stack differential
id: differential
continue-on-error: true
env:
REFERENCE_PATH: ${{ steps.reference.outputs.path }}
RUSTFS_E2E_LOG_DIR: ${{ runner.temp }}/rustfs-differential-logs
run: |
set -euo pipefail
export no_proxy="$NO_PROXY"
chmod +x target/debug/rustfs
test -x "$REFERENCE_PATH"
mkdir -p "$RUSTFS_E2E_LOG_DIR"
# Side A: the released binary. Side B: the candidate on its legacy stack.
export RUSTFS_E2E_REFERENCE_BINARY="$REFERENCE_PATH"
test_name='dual_stack_differential::tests::both_stacks_answer_every_operation_identically'
python3 scripts/e2e_binary.py run --features e2e-test-hooks -- \
cargo test --locked -p e2e_test --lib "$test_name" -- --exact --nocapture \
2>&1 | tee "$RUNNER_TEMP/differential-test.log"
- name: Summarize report and verify the reference side
if: always()
run: |
set -euo pipefail
python3 - <<'PY'
import glob, json, os, sys
summary = os.environ["GITHUB_STEP_SUMMARY"]
reports = sorted(glob.glob("target/dual-stack/self-comparison-*.json"))
lines = ["## Dual-stack differential: reference vs candidate", ""]
failures = []
if len(reports) != 1:
failures.append(f"expected exactly one self-comparison report, found {len(reports)}: {reports}")
else:
report = json.load(open(reports[0]))
side = report.get("reference_side")
version_a = report.get("version_a")
version_b = report.get("version_b")
lines += [
f"- report: `{reports[0]}`",
f"- reference_side: `{side}`",
f"- version_a: `{(version_a or '').splitlines()[0] if version_a else version_a}`",
f"- version_b: `{(version_b or '').splitlines()[0] if version_b else version_b}`",
f"- stack_a: `{report.get('stack_a')}`, stack_b: `{report.get('stack_b')}`",
f"- compared_ops: {report.get('compared_ops')}, compared_cases: {report.get('compared_cases')}",
"",
]
print(f"reference_side={side}\nversion_a={version_a}\nversion_b={version_b}")
lines += ["```", "version_a:", version_a or "", "", "version_b:", version_b or "", "```", ""]
if side != "a":
failures.append(f"reference_side is {side!r}, expected 'a'")
if not version_a or not version_b:
failures.append("version_a or version_b is missing")
elif version_a == version_b:
failures.append("version_a equals version_b: the reference was replaced by the candidate")
diffs = report.get("differences") or []
lines.append(f"### Differences: {len(diffs)}")
by_op = {}
for d in diffs:
by_op.setdefault(d["op"], []).append(d)
for op in sorted(by_op):
lines += ["", f"#### {op}", "", "| case | field | a (reference) | b (candidate) |", "| --- | --- | --- | --- |"]
for d in by_op[op]:
cell = lambda v: "`" + str(v).replace("|", "\\|").replace("`", "'") + "`"
lines.append(f"| {d['case']} | {d['field']} | {cell(d['a'])} | {cell(d['b'])} |")
if failures:
lines += ["", "### Observation check failed", ""] + [f"- {f}" for f in failures]
open(summary, "a").write("\n".join(lines) + "\n")
print("\n".join(lines))
for f in failures:
print(f"::error::{f}")
sys.exit(1 if failures else 0)
PY
- name: Upload differential reports
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: reference-differential-report
overwrite: true
path: |
target/dual-stack/*.json
${{ runner.temp }}/differential-test.log
if-no-files-found: warn
retention-days: 14
- name: Fail when the differential found differences
if: steps.differential.outcome == 'failure'
run: |
echo "::error::the dual-stack differential failed; see the job summary for every (op, field, a, b)"
exit 1
compare:
name: Compare targets
needs: [ plan, s3tests ]
-12
View File
@@ -13,21 +13,15 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
### Deployment
- Before upgrading, verify that the configured root access key is at least 3 bytes and the root secret key is at least 8 bytes. RustFS now rejects shorter values at startup and identifies the setting to update. Version 1.0.5 accepted these settings at startup and returned HTTP 500 when root requested STS credentials. Coordinate root credential changes across all nodes and clients. See [root credential rotation considerations](docs/operations/two-factor-auth.md#root-credentials-cannot-be-changed-at-runtime).
- Existing volumes created by older root-running container images must be writable by the non-root runtime identity before upgrading. The default container UID/GID is `10001:10001`; stop the affected instances, verify host mount paths, and for ordinary bind mounts run `sudo chown -R 10001:10001 /actual/data /actual/logs` on each RustFS data/log mount. Account for ACLs, NFS, Kubernetes, and rootless UID mappings. Image-build ownership changes do not migrate mounted data. See the Docker Quick Start in [README.md](README.md).
- Local disk initialization now checks file creation, writing, and deletion in the data root and fixed internal metadata directories. Unwritable disks are rejected using existing startup/quorum rules, with filesystem errors and Unix runtime/owner IDs logged for permission failures, instead of admitting those disks and discovering the mismatch on a later write. Existing object trees are not scanned and volume ownership is never changed automatically.
### Security
- **Key-scoped KMS policy statements now hold for every spelling of an AWS key**: a statement such as `Deny kms:Decrypt` on `arn:aws:kms:::key/<key-id>` was matched against the key identifier exactly as the request spelled it, so naming the same key by its AWS ARN (`arn:aws:kms:<region>:<account>:key/<key-id>`) was evaluated as a different key and the `Deny` did not apply. With `RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true` and the AWS KMS backend, an identity allowed on all keys but denied one could still write under that key, and read objects stored under its ARN spelling; the per-key admin endpoints were affected the same way. A key ARN is now evaluated as its key id, on the SSE-KMS data path and the admin endpoints alike. A request that names a key by alias, which only the backend can resolve, now fails closed against key-scoped statements: every key-scoped `Deny` applies and only an `Allow` on every key covers it. The KMS deletion impact report recognizes the default key and bucket default-encryption references across the id and ARN spellings as well. Deployments without key-scoped KMS statements, and the other KMS backends, whose key ids have a single spelling, are unaffected.
- **Header-signed SigV4 requests honour only signed headers** (GHSA-xm99-m3gq-83g8): a request authenticated with a SigV4 `Authorization` header that carries an `x-amz-*` request header not listed in its `SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3 and the presigned rule from GHSA-g8w9-qw9q-fghr. Previously anyone holding one header-signed `PutObject` request could add an unsigned `x-amz-copy-source` and turn it into a `CopyObject` that ran with the signer's permissions, copying any object the signer could read into the target. An `Authorization` header whose algorithm token is not `AWS4-HMAC-SHA256` is now rejected instead of being verified as SigV4. The request-envelope headers `x-amz-content-sha256`, `x-amz-decoded-content-length`, `x-amz-trailer` and `x-amz-checksum-algorithm` (the same set the upstream `s3s` fix exempts) and `x-amz-cf-id` (CloudFront) remain tolerated unsigned; AWS SDKs and RustFS's own signers already sign every other `x-amz-*` header. SigV2, JWT and anonymous requests are unchanged.
### Replication
- Site replication now seals an SSE-KMS replica under the peer's key of the same name as the source object's key. The replica write used to carry only the `aws:kms` intent, so an object written under a key named in the request was sealed under the peer's bucket default or site default instead, and with `RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true` a caller scoped to the named key could read the object on the site that took the write but got `AccessDenied` for its replica. This changes the default outbound request to site-replication peers: every key name in use, including each site's default key name, must exist on every site, or the affected SSE-KMS objects stay `FAILED` until the key is created. Existing replicas are not rewritten. Bucket replication through a remote target still sends the intent only.
- Site replication now replicates SSE-KMS objects when the receiving site runs with `RUSTFS_KMS_ENFORCE_SSE_KEY_POLICY=true`. The receiving site authorizes each replica write against the KMS key it resolves to, and the managed `site-replicator-0` account held no `kms` grant, so every SSE-KMS object stayed `FAILED` with `AccessDenied` while SSE-S3 and SSE-C objects replicated. The account's embedded policy now grants `kms:GenerateDataKey` on every key; existing accounts are refreshed by the reconcile tick and the failed objects catch up afterwards. It is deliberately not granted `kms:Decrypt`, so reads proxied to a peer for an SSE-KMS object are still refused under enforcement. Bucket replication through a remote target needs the same grant on the identity whose credentials the target uses. Sites that do not enable enforcement are unaffected.
- Object Lock replication PUTs now carry a required integrity header, fixing target rejection introduced by the plain-payload default ([#7097](https://github.com/rustfs/rustfs/pull/7097)). This changes the default outbound request for locked objects but adds no persisted format.
- Multipart source objects stay on the multipart transport even when their checksum record is a whole-object checksum, so objects above the single-PUT limit remain replicable ([#7047](https://github.com/rustfs/rustfs/pull/7047)).
- Targets that mint their own version IDs now use a per-target version ledger for tag, retention, legal-hold, and permanent-delete mutations; ambiguous pre-ledger matches fail with backoff instead of guessing ([#7368](https://github.com/rustfs/rustfs/pull/7368)). This adds dual-prefixed internal metadata keys that older readers ignore.
@@ -43,11 +37,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **Presigned URLs honour only signed headers** (GHSA-g8w9-qw9q-fghr): a SigV4 presigned request that carries an `x-amz-*` request header not listed in `X-Amz-SignedHeaders` is now rejected with `403 AccessDenied` ("There were headers present in the request which were not signed"), matching AWS S3. Previously the holder of a presigned `PutObject` URL could add unsigned `x-amz-tagging`, `x-amz-storage-class`, `x-amz-website-redirect-location`, ACL, metadata, Object Lock or SSE headers and have them applied. Presigners that intend a property must set it before signing so the SDK lists the header in `SignedHeaders`; `x-amz-cf-id` (CloudFront) remains tolerated unsigned. Header-signed SigV4 and SigV2 requests are unchanged.
### Fixed
- **KMS keys could not be described by ARN through the path route**: `GET /rustfs/admin/v3/kms/keys/{key_id}` read the path segment verbatim, so a key id that has to be percent-encoded to fit in one segment — an AWS key ARN, with its `:` and `/` — was looked up, authorized and audited under its encoded spelling and answered `404`. The segment is now percent-decoded first; a segment that is not valid UTF-8 after decoding is refused with `400 InvalidArgument`. The `describe-key?keyId=` form was unaffected.
- **SSE-KMS objects written through the AWS KMS backend could not be read**: `GetObject`, `CopyObject` and every other read of an `aws:kms` object failed with `400 InvalidRequest` ("SSE-S3 requires RUSTFS_SSE_S3_MASTER_KEY to be set ...") on a deployment using `RUSTFS_KMS_BACKEND=aws`, although the write and `HeadObject` succeeded. The AWS backend stored the `GenerateDataKey` blob as it came from AWS, the read path took those unrecognized bytes for a locally wrapped key, and AWS was never asked to decrypt. The backend now stores the blob inside the same data-key envelope the other backends use, and a blob stored without one is routed to AWS when the AWS backend is active. Objects written by 1.0.4 and 1.0.5 are intact and become readable after the upgrade, with no rewrite. Setting `RUSTFS_SSE_S3_MASTER_KEY` was never a workaround. An object written by this release is not readable by an older one; the same was already true of every object the older release wrote itself.
- **Immediate KMS key deletion no longer reports success without destroying the key** (behavior change): with `RUSTFS_KMS_ALLOW_IMMEDIATE_DELETION=true`, a `DELETE /rustfs/admin/v3/kms/keys/delete` request with `force_immediate` against an `Enabled` or `Disabled` key on the Vault KV2 or Vault Transit backend returned success but only moved the key to `PendingDeletion`; the key material stayed in Vault and objects encrypted under it remained readable, while the Local backend destroyed the key in that one request. Key destruction is now the same two explicit steps on every backend: schedule the deletion, then either let the waiting window elapse or send `force_immediate` to destroy the pending key early. `force_immediate` against a key that is not `PendingDeletion` is refused with `400` and changes nothing, so a success response always means the key is gone and no single request can destroy a key in use. Automation that destroyed Local keys with one `force_immediate` request must schedule the deletion first. On Vault Transit, a read under a key that no longer exists in Vault now fails with `KMS.NotFoundException` instead of `503 ServiceUnavailable`. No persisted format changes. See `docs/operations/kms-key-destruction.md`.
- **`mc admin info` and `mc admin replicate resync` work again**: since the cluster snapshot view (#3803), `GET /v3/info` and `GET /v3/storageinfo` returned the madmin `InfoMessage` / `StorageInfo` nested under an `info` key next to `admin_discovery` (and, for `/info`, `bitrotSelftest`). madmin decodes those bodies directly, so `mc admin info` printed empty values and `mc admin replicate resync start/status` rejected every alias with "alias provided is not part of cluster replication" because the local `deploymentID` read as empty. Requests on the MinIO-compatible `/minio/admin/v3` prefix now receive the bare madmin body again; `/rustfs/admin/v3`, which rc and the console use, keeps the envelope unchanged.
- **Replication status writebacks no longer invalidate scanner cycles**: every replication status writeback (`persist_replication_state_if_current` in `Update` mode, including the heals the scanner queues during its own walk) marked the bucket dirty on the namespace path, advanced the dirty-usage generation and woke the scanner, so a site-replication resync or any steadily replicated bucket ended every cycle `superseded`, the authoritative usage baseline was never certified, and the console reported usage freshness as stale for as long as replication ran (#8375). Those writebacks are now status class: they do not touch the dirty-usage generation, the activity snapshot, the dirty-usage wake or the durable dirty journal, and only the process-local producer coverage is recorded. They need no rescan queue, because a deployment with an active replication rule already walks every bucket on every cycle and that walk refreshes `replication_stats`. Tier-expiration cleanup, which removes data, stays on the namespace path. No persisted format, protocol, admin API or configuration changes. Design: `docs/architecture/scanner-per-bucket-convergence.md`.
- **A bucket written during a scanner walk no longer supersedes the whole cycle**: the post-walk cluster activity proof required the entire activity snapshot and the dirty-usage generation to be unchanged, so a single-node deployment with one bucket written every few seconds never reached a `Complete` cycle, `usage_snapshot_converged` never became true and the authoritative usage baseline was never certified (#8494). Dirty usage is now certified per bucket: a structurally complete walk that the guardrail did not veto is `Complete`, buckets whose dirty generation still matches the start snapshot are cleared, and buckets written during the walk stay dirty and are published as the pending set, so `usage_snapshot_converged` means "no bucket pending" and the next cycle is scoped to the pending buckets. Such a cycle writes the authoritative usage object; the observed object is now selected by an explicit observational mark that vetoed cycles carry. The guardrail itself vetoes only on the fields that change which namespace the walk described (peer set, process instance, protocol version, topology, maintenance generation, data movement, movement generation, publication blocked) and reports the field and host in the scanner log and in the new `usage_freshness` fields `guardrail_veto_total`, `last_guardrail_veto_unix_secs` and `last_guardrail_veto_reason`. In a distributed deployment a remote peer whose dirty usage moved during the walk still supersedes the cycle until scoped acknowledgements can name its buckets. `.usage.v2.json` gains the additive, default-when-absent fields `usage_snapshot_pending_buckets`, `usage_snapshot_pending_bucket_count` and `usage_snapshot_observational`; older readers ignore them and treat the snapshot as an authoritative but unconverged publication. Quota is unchanged: an uncertified bucket keeps its last usage value as the floor and live writes do not advance it. No protocol or configuration changes. Design: `docs/architecture/scanner-per-bucket-convergence.md`.
@@ -75,7 +64,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- **Per-pool erasure parity**: Erasure parity (STANDARD and reduced-redundancy) is now resolved independently for every pool instead of reusing the first pool's value. A heterogeneous topology — for example a 4-drive pool plus a 2-drive pool created during expansion — previously inherited the first pool's parity and could resolve to zero data shards in the smaller pool, panicking Reed-Solomon construction on write. Automatic parity now resolves per pool (for example `2+2` in the 4-drive pool and `1+1` in the 2-drive pool). Fixes #4801.
### Added
- **Multiple bucket regions**: `RUSTFS_REGION` / `--region` accepts a comma-separated list. The first entry stays the server's own region (request signing and `GetBucketLocation` are unchanged); every entry is accepted as a `CreateBucket` `LocationConstraint`, so one deployment can stand in for buckets that live in several AWS regions (local development and test rigs) now that location constraints are validated (#8739). Regions outside the list are still rejected with `InvalidLocationConstraint`. A single region behaves exactly as before; a comma was never a valid region, so no existing value changes meaning.
- **On-Demand Migration**: Lazy, pull-style migration of an existing S3-compatible bucket into RustFS. A local bucket is attached to an external source bucket; a GET for a key that does not exist locally fetches it from the source, streams it to the client, and stores it locally in the same pass, so every later read is served locally. The module is on by default; set `RUSTFS_ON_DEMAND_MIGRATION_ENABLED=false` on every node to turn it off. A bucket with no source configured behaves exactly as before — the runtime never intervenes on its reads and makes no outbound call. Operator guide at `docs/operations/on-demand-migration.md`.
- Per-bucket configuration persisted as `on-demand-migration.json` in the bucket metadata: source provider (`s3`, `aws`, `minio`, `rustfs`, `r2`, `gcs`), endpoint, region, addressing style, credentials and TLS material, an optional key-prefix filter and source-prefix rewrite, and a policy block covering the inline size threshold, multipart part size, concurrency, queue capacity, timeouts, bandwidth limit and negative-cache TTL
- Admin routes under `/rustfs/admin/v3/on-demand-migration/{bucket}`: `PUT` (with `?dry-run=true` to validate and probe the source without saving), `GET`, `DELETE`, `GET .../status`, plus `POST .../backfill?op=start|cancel` and `GET .../backfill` for the background full-backfill job with its resumable checkpoint. Authorized by the new `admin:GetBucketOnDemandMigration` and `admin:SetBucketOnDemandMigration` actions; every response redacts `secret_key` and `session_token`
Generated
+256 -267
View File
File diff suppressed because it is too large Load Diff
+30 -30
View File
@@ -149,14 +149,14 @@ rustfs-zip = { path = "./crates/zip", version = "1.0.5" }
# engine crates move onto from s3s (rustfs/backlog#2745). The one pin of the
# gateway in this workspace; no crate may enable a `compat-s3s*` feature except
# the s3s-facing edge of the rustfs binary.
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.40.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.2.0", default-features = false }
rustfs-gateway-types = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.32.0", default-features = false }
rustfs-gateway-stream = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.2.0", default-features = false }
# The gateway framework itself, consumed for its extension traits by the ring-2
# adapter. `default-features = false` keeps the `server` feature, and with it the
# server crate, out of the build.
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", version = "0.62.0", default-features = false }
rustfs-gateway = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", version = "0.57.0", default-features = false }
# Test-only: builds the `EffectiveHost` a host-resolution test queries with.
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "7915d06950f88611d883f6ab3d95efc90c65dec0", default-features = false }
rustfs-gateway-http = { git = "https://github.com/rustfs/gateway.git", rev = "21283e874c2381bab611e9347acbc5b92e9e2b3c", default-features = false }
# Async Runtime and Networking
async-channel = "2.5.0"
@@ -211,7 +211,7 @@ bytesize = "2.7.0"
byteorder = "1.5.0"
flatbuffers = "25.12.19"
form_urlencoded = "1.2.2"
prost = "0.14.5"
prost = "0.14.4"
quick-xml = "0.42.0"
rmp = { version = "0.8.15" }
rmp-serde = { version = "1.3.1" }
@@ -240,7 +240,7 @@ p256 = { version = "0.14.0", features = ["ecdsa", "pkcs8"] }
rsa = { version = "=0.10.0-rc.18" }
rustls = { default-features = false, version = "0.23.46" }
rustls-native-certs = "0.8"
rustls-pki-types = "1.16.0"
rustls-pki-types = "1.15.1"
x509-parser = "0.18.1"
sha1 = "0.11.0"
sha2 = "0.11.1"
@@ -251,7 +251,7 @@ proptest = "1"
# Time and Date
chrono = { version = "0.4.45" }
humantime = "2.4.0"
jiff = { version = "0.2.43" }
jiff = { version = "0.2.39" }
time = { version = "0.3.55" }
# Database
@@ -261,22 +261,22 @@ tokio-postgres-rustls = "0.14.0"
# Utilities and Tools
anyhow = "1.0.104"
arc-swap = "1.9.5"
arc-swap = "1.9.3"
rustfs-tokio-tar = { version = "0.7.1" }
# Candidate Snowball parser versions exercised by rustfs-zip compatibility fixtures.
tar-codec = "0.0.16"
tar-framing = "0.0.16"
atoi = "3.1.1"
atomic_enum = "0.3.0"
aws-config = { version = "1.13.2" }
aws-config = { version = "1.13.0" }
aws-credential-types = { version = "1.3.1" }
aws-sdk-kms = { default-features = false, version = "1.131.0" }
aws-sdk-s3 = { default-features = false, version = "1.159.0" }
aws-sdk-sts = { default-features = false, version = "1.126.0" }
aws-sdk-kms = { default-features = false, version = "1.129.0" }
aws-sdk-s3 = { default-features = false, version = "1.157.0" }
aws-sdk-sts = { default-features = false, version = "1.124.0" }
aws-smithy-async = { version = "1.3.1" }
aws-smithy-http-client = { default-features = false, version = "1.5.2" }
aws-smithy-runtime-api = { version = "1.21.0" }
aws-smithy-types = { version = "1.9.4" }
aws-smithy-runtime-api = { version = "1.19.2" }
aws-smithy-types = { version = "1.9.1" }
base64-simd = "0.8.0"
brotli = "9.1.1"
clap = { version = "4.6.7" }
@@ -292,10 +292,10 @@ derive_builder = "0.20.2"
ed25519-dalek = "3.0.0"
enumset = "1.1.15"
faster-hex = "1.0.0"
flate2 = "1.1.11"
flate2 = "1.1.10"
glob = "0.3.4"
google-cloud-storage = "1.22.0"
google-cloud-auth = "1.19.0"
google-cloud-storage = "1.21.0"
google-cloud-auth = "1.18.0"
hashbrown = { version = "0.17.1" }
# Base32 for RFC 6238 TOTP shared secrets (RFC 4648 unpadded, the alphabet
# every authenticator app expects). Already in the graph transitively.
@@ -319,7 +319,7 @@ moka = { version = "0.12.16" }
netif = "0.1.6"
num_cpus = { version = "1.17.0" }
nvml-wrapper = "0.13.0"
object = { version = "=0.41.0", default-features = false }
object = { version = "=0.40.0", default-features = false }
parking_lot = "0.13.0"
same-file = "1.0.6"
path-absolutize = "4.0.1"
@@ -340,7 +340,7 @@ rumqttc = { package = "rumqttc-next", version = "0.34.0" }
redis = { version = "1.8.1" }
rustify = { version = "0.7", default-features = false }
rustix = { version = "1.1.6" }
rust-embed = { version = "8.14.0" }
rust-embed = { version = "8.13.0" }
rustc-hash = { version = "2.1.3" }
s3s = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0", features = ["minio"] }
s3s-sigv4 = { git = "https://github.com/s3s-project/s3s", rev = "5761ddfe4c509fcac87e3b99fd5bd9e176b442c0" }
@@ -351,13 +351,13 @@ legacy-s3s = { package = "s3s", git = "https://github.com/s3s-project/s3s", rev
serial_test = "4.0.1"
shadow-rs = { default-features = false, version = "2.0.0" }
siphasher = "1.0.4"
smallvec = { version = "1.16.5" }
smallvec = { version = "1.16.4" }
compact_str = "0.10.0"
snap = "1.1.2"
starshard = { version = "2.4.1" }
starshard = { version = "2.4.0" }
strum = { version = "0.28.0" }
sync_wrapper = "1.0.2"
sysinfo = "0.40.3"
sysinfo = "0.40.1"
temp-env = "0.3.6"
tempfile = "3.27.0"
test-case = "3.4.1"
@@ -371,7 +371,7 @@ tracing-subscriber = { version = "0.3.23" }
transform-stream = "0.3.1"
url = "2.5.8"
urlencoding = "2.1.3"
uuid = { version = "1.29.2" }
uuid = { version = "1.29.0" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.0"
@@ -379,13 +379,13 @@ winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.19" }
zip = "9.5.1"
zstd = "0.14.4"
zip = "9.3.0"
zstd = "0.14.3"
# Observability and Metrics
metrics = "0.24.6"
metrics-util = "0.20"
dial9 = { version = "0.5.8", default-features = false }
dial9 = { version = "0.5.7", default-features = false }
opentelemetry = { version = "0.33.1" }
opentelemetry-appender-tracing = { version = "0.33.1" }
opentelemetry-otlp = { version = "0.33.1" }
@@ -397,19 +397,19 @@ pyroscope = { version = "2.1.5" }
# FTP and SFTP
libunftp = { version = "0.24.0" }
unftp-core = "0.1.1"
unftp-core = "0.1.0"
suppaftp = { version = "12.2.1" }
rcgen = { version = "0.14.10", default-features = false, features = ["aws_lc_rs", "crypto", "pem"] }
russh = { version = "0.65.1" }
russh-sftp = "3.0.1"
# WebDAV
dav-server = "0.12.0"
dav-server = "0.11.0"
# Performance Analysis and Memory Profiling
rustfs-mimalloc = { version = "0.7.4" }
rustfs-mimalloc = { version = "0.7.3" }
# Preserve Unicode focus filters until rustfs/backlog#2302 is resolved.
hotpath = { version = "0.31.2", default-features = false }
hotpath = { version = "0.31.1", default-features = false }
# Snapshot testing for output format regression detection
insta = { version = "1.49" }
+73 -53
View File
@@ -34,6 +34,9 @@ async fn test_audit_system_startup_performance() {
println!("Audit system startup took: {elapsed:?}");
// Should complete within 5 seconds
assert!(elapsed < Duration::from_secs(5), "Startup took too long: {elapsed:?}");
// Clean up
let _ = system.close().await;
}
@@ -65,6 +68,9 @@ async fn test_concurrent_target_creation() {
println!("Concurrent target creation took: {elapsed:?}");
// Should complete quickly even with multiple targets
assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {elapsed:?}");
// Verify it fails with expected error (server not initialized)
match result {
Err(AuditError::StorageNotAvailable(_)) => {
@@ -160,6 +166,9 @@ async fn test_audit_log_dispatch_performance() {
println!("Audit log dispatch took: {elapsed:?}");
// Should be very fast (sub-millisecond for no targets)
assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
assert!(
matches!(result, Err(AuditError::NotInitialized(_))),
"Dispatch on a stopped system should return NotInitialized, got: {result:?}"
@@ -198,12 +207,11 @@ async fn test_system_state_transitions() {
}
#[test]
fn test_event_name_mask_matches_expansion() {
fn test_event_name_mask_performance() {
use rustfs_targets::EventName;
// A compound event's mask is exactly the union of its leaves' masks, and a
// leaf owns exactly one bit. Timing is printed for information only.
let events = [
// Test that event name mask calculation is efficient
let events = vec![
EventName::ObjectCreatedPut,
EventName::ObjectAccessedGet,
EventName::ObjectRemovedDelete,
@@ -212,59 +220,47 @@ fn test_event_name_mask_matches_expansion() {
];
let start = Instant::now();
for event in events {
let union = event.expand().iter().fold(0u64, |acc, leaf| acc | leaf.mask());
assert_eq!(event.mask(), union, "{event:?} mask must equal the union of its expansion");
}
println!("Event mask check took: {:?}", start.elapsed());
assert_eq!(EventName::ObjectCreatedPut.mask().count_ones(), 1);
assert_eq!(EventName::ObjectCreatedAll.mask().count_ones(), 4);
// Calculate masks for many events
for _ in 0..1000 {
for event in &events {
let _mask = event.mask();
}
}
let elapsed = start.elapsed();
println!("Event mask calculation (5000 ops) took: {elapsed:?}");
// Should be very fast
assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {elapsed:?}");
}
#[test]
fn test_event_name_expansion_performance() {
use rustfs_targets::EventName;
use std::collections::HashSet;
// Expansion is a pure lookup. Wall-clock bounds flake on shared runners, so
// this asserts what the time was a proxy for: each compound event yields a
// fixed-size set of distinct leaf events and nothing else (no duplicates,
// no compound left to expand again). Timing is printed for information only.
let cases = [
(EventName::ObjectCreatedAll, 4),
(EventName::ObjectAccessedAll, 5),
(EventName::ObjectRemovedAll, 3),
// Test that event name expansion is efficient
let compound_events = vec![
EventName::ObjectCreatedAll,
EventName::ObjectAccessedAll,
EventName::ObjectRemovedAll,
EventName::Everything,
];
let start = Instant::now();
for (event, expected_len) in cases {
let expanded = event.expand();
assert_eq!(expanded.len(), expected_len, "{event:?} expansion size");
assert_eq!(
expanded.iter().collect::<HashSet<_>>().len(),
expanded.len(),
"{event:?} expansion has duplicates"
);
assert_eq!(expanded, event.expand(), "{event:?} expansion must be deterministic");
for leaf in &expanded {
assert_eq!(leaf.expand(), vec![*leaf], "{event:?} expanded to non-leaf {leaf:?}");
// Expand events many times
for _ in 0..1000 {
for event in &compound_events {
let _expanded = event.expand();
}
}
let everything = EventName::Everything.expand();
assert!(everything.len() > 12, "Everything must cover at least every compound's leaves");
assert_eq!(
everything.iter().collect::<HashSet<_>>().len(),
everything.len(),
"Everything has duplicates"
);
for (event, _) in cases {
for leaf in event.expand() {
assert!(everything.contains(&leaf), "Everything is missing {leaf:?}");
}
}
println!("Event expansion check took: {:?}", start.elapsed());
let elapsed = start.elapsed();
println!("Event expansion (4000 ops) took: {elapsed:?}");
// Should be very fast
assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {elapsed:?}");
}
#[tokio::test]
@@ -282,22 +278,46 @@ async fn test_registry_operations_performance() {
let elapsed = start.elapsed();
println!("Registry operations (2000 ops) took: {elapsed:?}");
// Should be very fast for empty registry
assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {elapsed:?}");
}
// The former 3k EPS / latency assertions measured the CI host, not the code
// (the loop only formats strings and reads the clock), so they are replaced by
// a count of what the loop produced. Throughput is printed for information.
// Performance requirements validation
#[test]
fn test_performance_requirements() {
use std::collections::HashSet;
// According to requirements: ≥ 3k EPS/node; P99 < 30ms (default)
// These are synthetic tests since we can't actually achieve 3k EPS
// without real server storage and network targets, but we can validate
// that our core algorithms are efficient enough
let start = Instant::now();
let ids: HashSet<String> = (0..3000).map(|i| format!("s3:ObjectCreated:Put_{i}")).collect();
let timestamps = (0..3000).filter(|_| !jiff::Timestamp::now().to_string().is_empty()).count();
// Simulate processing 3000 events worth of operations
for i in 0..3000 {
// Simulate event name parsing and processing
let _event_id = format!("s3:ObjectCreated:Put_{i}");
let _timestamp = jiff::Timestamp::now().to_string();
// Simulate basic audit entry creation overhead
let _entry_size = 512; // bytes
let _processing_time = std::time::Duration::from_nanos(100); // simulated
}
let elapsed = start.elapsed();
let eps = 3000.0 / elapsed.as_secs_f64();
println!("Simulated 3000 events in {elapsed:?}");
println!("Simulated 3000 events in {elapsed:?} ({eps:.0} EPS)");
assert_eq!(ids.len(), 3000, "every simulated event id must be distinct");
assert_eq!(timestamps, 3000, "every simulated event must get a timestamp");
// Our core processing should easily handle 3k EPS worth of CPU overhead
// The actual EPS limit will be determined by network I/O to targets
assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {eps} EPS");
// P99 latency requirement: < 30ms
// For core processing, we should be much faster than this
let avg_latency = elapsed / 3000;
println!("Average processing latency: {avg_latency:?}");
assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {avg_latency:?}");
}
+6 -16
View File
@@ -50,8 +50,10 @@ const _: () = assert!(DEFAULT_ADMIN_PEER_PROBE_TIMEOUT_SECS <= MAX_ADMIN_PEER_PR
// ── Client-side internode gRPC channel tuning (P0) ──
// These mirror the server-side HTTP/2 transport tuning in `rustfs/src/server/http.rs`
// on the *client* `tonic` `Endpoint` used for internode RPCs. Explicit values make
// the operator's receive-credit policy independent of changes to library defaults.
// on the *client* `tonic` `Endpoint` used for internode control-plane RPCs. Prior to
// this, the client channel only set timeouts/keepalive, leaving Nagle enabled and the
// default 64KiB HTTP/2 window in place — hurting small lock-RPC latency and large
// metadata-response throughput respectively.
/// Disable Nagle's algorithm on internode gRPC client sockets.
///
@@ -67,8 +69,8 @@ const _: () = assert!(DEFAULT_INTERNODE_RPC_TCP_NODELAY);
/// HTTP/2 initial stream window size (bytes) for internode gRPC client channels.
///
/// Size this for the bandwidth-delay product and concurrent response budget of
/// unary responses such as `ReadMultiple` and `BatchReadVersion`. Set to 0 to use the
/// The library default (64KiB) throttles larger unary responses (e.g. `ReadMultiple`,
/// `BatchReadVersion`) by the bandwidth-delay product. Set to 0 to fall back to the
/// `tonic`/`hyper` default. Defaults to 1MiB.
pub const ENV_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE";
pub const DEFAULT_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE: u32 = 1024 * 1024;
@@ -282,18 +284,6 @@ const _: () = assert!(!DEFAULT_INTERNODE_CHANNEL_ISOLATION);
pub const ENV_INTERNODE_BULK_CHANNELS: &str = "RUSTFS_INTERNODE_BULK_CHANNELS";
pub const DEFAULT_INTERNODE_BULK_CHANNELS: usize = 2;
/// Optional receive-window overrides for isolated bulk channels. When unset, inherit the
/// corresponding `RUSTFS_INTERNODE_RPC_HTTP2_*` setting. Zero selects the library default.
pub const ENV_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE";
pub const ENV_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE: &str = "RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE";
/// Transport admission and pending-buffer limits for isolated bulk channels. These limits
/// apply to the tonic service; complete unary bodies are observed separately by RPC metrics.
pub const ENV_INTERNODE_BULK_CONCURRENCY_LIMIT: &str = "RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT";
pub const DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT: usize = 64;
pub const ENV_INTERNODE_BULK_BUFFER_SIZE: &str = "RUSTFS_INTERNODE_BULK_BUFFER_SIZE";
pub const DEFAULT_INTERNODE_BULK_BUFFER_SIZE: usize = 1024;
/// Profile selector for conservative internode HTTP data-plane client tuning.
pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
-55
View File
@@ -117,30 +117,6 @@ pub const ENV_OBJECT_GET_SKIP_BITROT_VERIFY: &str = "RUSTFS_OBJECT_GET_SKIP_BITR
/// Default: bitrot verification is enabled on GetObject reads (do not skip).
pub const DEFAULT_OBJECT_GET_SKIP_BITROT_VERIFY: bool = false;
/// Comma-separated buckets that hold a content-addressed cache (an LLM KV
/// cache) and serve PutObject, GetObject and HeadObject without distributed
/// locks or authoritative bucket-metadata reads. See the ecstore
/// `bucket::lockless_cache` module for what such a bucket gives up.
///
/// Default is empty: every bucket keeps the locked paths.
pub const ENV_LOCKLESS_CACHE_BUCKETS: &str = "RUSTFS_LOCKLESS_CACHE_BUCKETS";
/// Smallest declared body, in bytes, whose ETag MD5 is computed on a
/// blocking-pool thread while the body streams into the erasure encoder,
/// instead of inline on the request's task. The ETag and Content-MD5
/// checks are unchanged; `0` keeps every MD5 inline.
///
/// Each offloaded upload holds one blocking-pool thread until its body ends,
/// and that pool also serves disk I/O, so size the threshold against the
/// expected number of concurrent large uploads before enabling it.
pub const ENV_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES: &str = "RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES";
/// Default: disabled; every ETag MD5 is computed inline. Set
/// `RUSTFS_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES` (for example `8388608`) to opt in.
pub const DEFAULT_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES: u64 = 0;
const _: () = assert!(DEFAULT_OBJECT_ETAG_MD5_OFFLOAD_MIN_BYTES == 0);
/// Create independent shard commitments for new writes after the fleet is upgraded.
/// Existing protected objects and multipart uploads retain their protection.
pub const ENV_SHARD_INTEGRITY_WRITE: &str = "RUSTFS_SHARD_INTEGRITY_WRITE";
@@ -322,37 +298,6 @@ pub const ENV_OBJECT_DISK_READ_TIMEOUT: &str = "RUSTFS_OBJECT_DISK_READ_TIMEOUT"
/// Default disk read timeout in seconds.
pub const DEFAULT_OBJECT_DISK_READ_TIMEOUT: u64 = 10;
/// Environment variable for the smallest bitrot frame (hash + shard bytes)
/// that is verified on a blocking thread instead of inline on the decode task
/// (`0` disables off-thread verification).
///
/// The shard reads of one stripe complete on one task, so with large shards
/// their hashes were computed one after another on one core: at 4 MiB shards
/// a 12+4 stripe spent about 10 ms hashing serially. Frames at or above this
/// size are hashed on the blocking pool so the shards of a stripe verify in
/// parallel; small frames stay inline because a thread hop would cost more
/// than the hash.
pub const ENV_BITROT_VERIFY_OFFLOAD_MIN_BYTES: &str = "RUSTFS_BITROT_VERIFY_OFFLOAD_MIN_BYTES";
/// Default smallest off-thread bitrot frame: 2 MiB (frames of the legacy 1 MiB
/// block, and 1 MiB shards, stay inline).
pub const DEFAULT_BITROT_VERIFY_OFFLOAD_MIN_BYTES: usize = 2 * 1024 * 1024;
/// Environment variable for the erasure shard size, in bytes, of newly written
/// objects (`0`, the default, keeps the historical 1 MiB erasure block).
///
/// When set, the erasure block of a new object is `data shards x shard bytes`,
/// so every shard read and write is exactly this large and aligned: 4 MiB on a
/// 12+4 layout gives a 48 MiB block, 48x fewer stripes per object than today.
/// Existing objects keep the block recorded in their `xl.meta`. System
/// metadata under `.rustfs.sys` always uses the 1 MiB block. The value must be
/// a multiple of 4 KiB and at most `MAX_ERASURE_SHARD_BYTES`; invalid values
/// are ignored with a warning.
pub const ENV_ERASURE_SHARD_BYTES: &str = "RUSTFS_ERASURE_SHARD_BYTES";
/// Default erasure shard size override (`0` = derive from the 1 MiB block).
pub const DEFAULT_ERASURE_SHARD_BYTES: usize = 0;
/// Largest accepted erasure shard size override (64 MiB).
pub const MAX_ERASURE_SHARD_BYTES: usize = 64 * 1024 * 1024;
/// Environment variable for the per-shard erasure write stall timeout (seconds).
///
/// A single shard write (or shard-writer shutdown) that makes no forward
-15
View File
@@ -114,14 +114,6 @@ pub const ENV_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: &str = "RUSTFS_OBJECT_FILE_
pub const ENV_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: &str = "RUSTFS_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD";
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_WRITE_ENABLE: bool = false;
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_READ_ENABLE: bool = true;
/// Bytes of page-cache read-ahead a ranged file stream keeps queued in front
/// of its read position (`posix_fadvise(WILLNEED)`, Linux only; `0` disables).
/// A shard stream served to a peer reads 4 MiB chunks one after another; the
/// kernel's own read-ahead window ramps up slowly for large reads, so without
/// this each chunk waits for its own device round trips.
pub const ENV_OBJECT_STREAM_READ_AHEAD_BYTES: &str = "RUSTFS_OBJECT_STREAM_READ_AHEAD_BYTES";
pub const DEFAULT_OBJECT_STREAM_READ_AHEAD_BYTES: usize = 16 * 1024 * 1024;
pub const DEFAULT_OBJECT_FILE_CACHE_RECLAIM_THRESHOLD: usize = 4 * 1024 * 1024;
/// Threshold for small object seek support in bytes.
@@ -166,10 +158,3 @@ pub const ENV_OBJECT_DATA_CACHE_MIN_FREE_MEMORY_PERCENT: &str = "RUSTFS_OBJECT_D
pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_PER_CPU";
pub const ENV_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_CONCURRENCY_MAX";
pub const ENV_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX: &str = "RUSTFS_OBJECT_DATA_CACHE_IDENTITY_KEYS_MAX";
/// Fill the object data cache from the PUT body as it streams in, so the first
/// GET after a write is a hit (write-once KV-cache chunks). Only objects that
/// fit the cache entry limit and take the plain streaming PUT path (no
/// compression, no SSE) are filled; the body is kept in memory once more
/// during the PUT.
pub const ENV_OBJECT_DATA_CACHE_FILL_ON_PUT: &str = "RUSTFS_OBJECT_DATA_CACHE_FILL_ON_PUT";
pub const DEFAULT_OBJECT_DATA_CACHE_FILL_ON_PUT: bool = false;
-4
View File
@@ -55,9 +55,6 @@ pub const ENV_OBS_PUT_STAGE_METRICS_ENABLED: &str = "RUSTFS_OBS_PUT_STAGE_METRIC
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_ENABLED";
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_SAMPLE_RATE";
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS";
/// Path of the sampled GetObject stage summary file (defaults to `/var/log/rustfs/get-stage-summary.log`).
pub const ENV_OBS_GET_STAGE_LOCAL_SUMMARY_FILE: &str = "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_FILE";
pub const DEFAULT_OBS_GET_STAGE_LOCAL_SUMMARY_FILE: &str = "/var/log/rustfs/get-stage-summary.log";
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED";
@@ -181,7 +178,6 @@ mod tests {
ENV_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS,
"RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_INTERVAL_SECS"
);
assert_eq!(ENV_OBS_GET_STAGE_LOCAL_SUMMARY_FILE, "RUSTFS_OBS_GET_STAGE_LOCAL_SUMMARY_FILE");
// Test log cleanup related env keys
assert_eq!(ENV_OBS_LOG_MAX_TOTAL_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES");
assert_eq!(ENV_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES, "RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES");
+6 -1
View File
@@ -1764,7 +1764,12 @@ impl DataUsageCache {
let mut found = HashSet::new();
found.insert(top);
mark(self, &top_e, &mut found);
self.cache.retain(|k, _| found.contains(k));
self.cache.retain(|k, _| {
if !found.contains(k) {
return false;
}
true
});
}
pub fn reduce_children_of(&mut self, path: &DataUsageHash, limit: usize, compact_self: bool) {
@@ -135,72 +135,6 @@ mod tests {
Ok(())
}
/// A comma-separated region list widens the regions `CreateBucket`
/// accepts without changing the server's own region (the first entry): a
/// listed region creates the bucket, an unlisted one is still rejected,
/// and `GetBucketLocation` keeps reporting the site region.
#[tokio::test]
async fn test_create_bucket_accepts_listed_regions() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec!["--region", "us-east-1,ap-southeast-2,us-west-2"])
.await?;
for region in ["us-east-1", "ap-southeast-2", "us-west-2"] {
let client = Client::from_conf(
env.create_s3_client()
.config()
.to_builder()
.region(Region::new(region))
.build(),
);
let bucket = format!("additional-region-{region}");
client
.create_bucket()
.bucket(&bucket)
.create_bucket_configuration(
CreateBucketConfiguration::builder()
.location_constraint(BucketLocationConstraint::from(region))
.build(),
)
.send()
.await
.expect("a listed region must create the bucket");
let location = client.get_bucket_location().bucket(&bucket).send().await?;
assert_eq!(
location.location_constraint().map(|value| value.as_str()),
Some(""),
"GetBucketLocation keeps reporting the site region (us-east-1)"
);
}
let client = env.create_s3_client();
let error = client
.create_bucket()
.bucket("unlisted-region")
.create_bucket_configuration(
CreateBucketConfiguration::builder()
.location_constraint(BucketLocationConstraint::from("eu-west-1"))
.build(),
)
.send()
.await
.expect_err("a region missing from the region list must be rejected");
assert_eq!(error.raw_response().map(|response| response.status().as_u16()), Some(400));
assert_eq!(
error.as_service_error().and_then(ProvideErrorMetadata::code),
Some("InvalidLocationConstraint")
);
let missing = client
.head_bucket()
.bucket("unlisted-region")
.send()
.await
.expect_err("rejected bucket must not exist");
assert_eq!(missing.raw_response().map(|response| response.status().as_u16()), Some(404));
Ok(())
}
#[tokio::test]
async fn test_create_bucket_uses_configured_region() -> Result<(), Box<dyn Error + Send + Sync>> {
init_logging();
@@ -766,10 +766,9 @@ impl Run {
|| async {
let mut ready = true;
let mut observations = Vec::with_capacity(4);
assert!(self.dist.cluster.nodes.len() >= 4);
for (node, cluster_node) in self.dist.cluster.nodes.iter().enumerate().take(4) {
for node in 0..4 {
match http
.get(format!("{}/health/ready", cluster_node.url))
.get(format!("{}/health/ready", self.dist.cluster.nodes[node].url))
.timeout(Duration::from_secs(3))
.send()
.await
@@ -13,7 +13,7 @@
// limitations under the License.
use super::harness::{
DistCluster, DistLayout, TestResult, cluster_admin, enable_versioning, put_bucket_replication, put_object,
DistCluster, DistLayout, TestResult, cluster_admin_ok, enable_versioning, put_bucket_replication, put_object,
quota_stats_current_usage, quota_stats_on_every_node, retrying_put, set_bucket_quota, set_remote_target, unique_bucket,
wait_for_ready, wait_for_replicated_bytes, wait_until,
};
@@ -119,7 +119,6 @@ async fn four_node_quota_changes_refresh_peers_before_response() -> TestResult {
let native_path = format!("/rustfs/admin/v3/quota/{bucket}");
let compat_set = format!("/rustfs/admin/v3/set-bucket-quota?bucket={bucket}");
let compat_get = format!("/rustfs/admin/v3/get-bucket-quota?bucket={bucket}");
let complete_reload = serde_json::json!({"status": "complete", "unconfirmed_peers": []});
// Warm every node's no-quota cache. The compatibility GET does not
// depend on a scanner usage snapshot for this newly created bucket.
@@ -134,12 +133,7 @@ async fn four_node_quota_changes_refresh_peers_before_response() -> TestResult {
)
.await?;
assert_eq!(status, StatusCode::OK, "{body}");
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(response["quota"], 0);
assert!(
response.get("metadata_reload").is_none(),
"GET must not report a mutation outcome: {body}"
);
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["quota"], 0);
}
wait_until(
Duration::from_secs(30),
@@ -157,8 +151,6 @@ async fn four_node_quota_changes_refresh_peers_before_response() -> TestResult {
return Ok(false);
}
assert_eq!(status, StatusCode::OK, "{body}");
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(response["metadata_reload"], complete_reload, "{body}");
Ok(true)
},
"quota fleet capability ready",
@@ -188,14 +180,7 @@ async fn four_node_quota_changes_refresh_peers_before_response() -> TestResult {
),
] {
if method != Method::GET {
let (status, body) = cluster_admin(&dist.cluster, method, path, payload.map(|value| value.to_string())).await?;
assert_eq!(status, StatusCode::OK, "{body}");
if path == compat_set {
assert!(body.is_empty(), "compatible quota mutations must preserve their empty response: {body}");
} else {
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(response["metadata_reload"], complete_reload, "{body}");
}
cluster_admin_ok(&dist.cluster, method, path, payload.map(|value| value.to_string())).await?;
}
// No polling: a successful mutation waits for healthy peers to reload.
for (index, node) in dist.cluster.nodes.iter().enumerate() {
@@ -209,11 +194,10 @@ async fn four_node_quota_changes_refresh_peers_before_response() -> TestResult {
)
.await?;
assert_eq!(status, StatusCode::OK, "node {index}: {body}");
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(response["quota"], expected.unwrap_or(0), "node {index} kept stale quota metadata",);
assert!(
response.get("metadata_reload").is_none(),
"GET must not report a mutation outcome: {body}"
assert_eq!(
serde_json::from_str::<serde_json::Value>(&body)?["quota"],
expected.unwrap_or(0),
"node {index} kept stale quota metadata",
);
if expected.is_some() {
let (status, body) = admin_request(
@@ -226,37 +210,21 @@ async fn four_node_quota_changes_refresh_peers_before_response() -> TestResult {
)
.await?;
assert_eq!(status, StatusCode::OK, "node {index}: {body}");
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(response["quota"].as_u64(), expected);
assert!(
response.get("metadata_reload").is_none(),
"GET must not report a mutation outcome: {body}"
);
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["quota"].as_u64(), expected);
}
}
}
let (status, body) = cluster_admin(
cluster_admin_ok(
&dist.cluster,
Method::PUT,
&native_path,
Some(serde_json::json!({"quota": 8192, "quota_type": "HARD"}).to_string()),
)
.await?;
assert_eq!(status, StatusCode::OK, "{body}");
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(response["metadata_reload"], complete_reload, "{body}");
let offline_host = dist.cluster.nodes[3].address.clone();
dist.cluster.stop_node(3)?;
// An unreachable peer cannot turn an already committed clear into a failure.
let (status, body) = cluster_admin(&dist.cluster, Method::DELETE, &native_path, None).await?;
assert_eq!(status, StatusCode::OK, "{body}");
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(
response["metadata_reload"],
serde_json::json!({"status": "incomplete", "unconfirmed_peers": [offline_host]}),
"the response must identify exactly the peer that did not acknowledge the reload: {body}",
);
cluster_admin_ok(&dist.cluster, Method::DELETE, &native_path, None).await?;
for node in dist.cluster.nodes.iter().take(3) {
let (status, body) = admin_request(
&node.url,
@@ -268,12 +236,7 @@ async fn four_node_quota_changes_refresh_peers_before_response() -> TestResult {
)
.await?;
assert_eq!(status, StatusCode::OK, "{body}");
let response: serde_json::Value = serde_json::from_str(&body)?;
assert_eq!(response["quota"], 0);
assert!(
response.get("metadata_reload").is_none(),
"GET must not report a mutation outcome: {body}"
);
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["quota"], 0);
}
Ok(())
}
@@ -1332,15 +1332,14 @@ mod tests {
.collect();
assert_eq!(confirmed.len(), 1);
let mut receivers = Vec::new();
let receiver_endpoints = phase.disks.get(1).map_or(&[][..], |disks| &phase.endpoints[1][..disks.len()]);
for endpoint in receiver_endpoints {
for drive in 0..phase.disks.get(1).map_or(0, Vec::len) {
for cas in [prepares[1], commits[1]] {
let matching: Vec<_> = events[1]
.iter()
.copied()
.filter(|event| {
event["kind"] == "receiver"
&& event["disk"] == *endpoint
&& event["disk"] == phase.endpoints[1][drive]
&& event["dst_volume"] == ".rustfs.sys"
&& event["dst_path"] == "pool.bin"
&& event["etag"] == cas["etag"]
@@ -20,8 +20,6 @@
//! sends is left out of the comparison. Values minted per process (upload IDs, timestamps) are
//! never recorded, and the comparator reduces a version ID to whether it is `null`.
mod extra;
use super::compare::Observation;
use crate::common::{RustFSTestEnvironment, build_test_s3_config};
use aws_sdk_s3::Client;
@@ -365,7 +363,6 @@ pub(crate) async fn run(env: &RustFSTestEnvironment) -> Vec<Observation> {
Box::pin(multipart(&mut s, &c)).await;
Box::pin(object_extras(&mut s, &c)).await;
Box::pin(deletes_and_listings(&mut s, &c)).await;
Box::pin(extra::cases(&mut s, &c)).await;
s.observations
}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `bucket_access` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `bucket_configs` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `bucket_core` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `bucket_replication_lifecycle` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `listing_multipart` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,46 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Cases the operation batches of rustfs/backlog#2765 add to the script, one file per batch.
//!
//! They run after the base script in `super`, in the fixed order below, so the base script's
//! ordered `(op, case)` list never moves. A batch adds its cases to its own file only, on buckets
//! and keys of its own; it does not edit this file or the base script.
use super::Recorder;
use aws_sdk_s3::Client;
mod bucket_access;
mod bucket_configs;
mod bucket_core;
mod bucket_replication_lifecycle;
mod listing_multipart;
mod object_commit;
mod object_meta;
mod object_misc;
mod object_read;
mod object_write;
pub(super) async fn cases(s: &mut Recorder, c: &Client) {
Box::pin(bucket_core::cases(s, c)).await;
Box::pin(bucket_configs::cases(s, c)).await;
Box::pin(bucket_access::cases(s, c)).await;
Box::pin(bucket_replication_lifecycle::cases(s, c)).await;
Box::pin(object_meta::cases(s, c)).await;
Box::pin(listing_multipart::cases(s, c)).await;
Box::pin(object_read::cases(s, c)).await;
Box::pin(object_write::cases(s, c)).await;
Box::pin(object_commit::cases(s, c)).await;
Box::pin(object_misc::cases(s, c)).await;
}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `object_commit` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `object_meta` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `object_misc` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `object_read` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -1,20 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Differential cases of the `object_write` batch (see `super`).
use super::super::Recorder;
use aws_sdk_s3::Client;
pub(super) async fn cases(_s: &mut Recorder, _c: &Client) {}
@@ -586,18 +586,7 @@ async fn test_reads_under_a_deleted_kms_key_report_key_not_found() -> Result<(),
.await?;
assert_eq!(put.ssekms_key_id(), Some(doomed_key_id.as_str()));
// Destruction is two requests: schedule the deletion, then destroy the
// pending key without waiting out its window.
info!("🗑️ scheduling {doomed_key_id} for deletion, then destroying it");
kms_admin_request(
&base.url,
http::Method::DELETE,
"/rustfs/admin/v3/kms/keys/delete",
Some(&serde_json::json!({ "key_id": doomed_key_id }).to_string()),
&base.access_key,
&base.secret_key,
)
.await?;
info!("🗑️ deleting {doomed_key_id} immediately");
kms_admin_request(
&base.url,
http::Method::DELETE,
+5
View File
@@ -301,6 +301,11 @@ mod bucket_logging_test;
#[cfg(test)]
mod website_hosting_test;
// Website hosting and the metadata listings on the gateway stack, compared with the legacy
// stack on one data directory (rustfs/backlog#2734 T2.11; R-064).
#[cfg(test)]
mod website_stack_interim_test;
// Multipart control API auth regression tests
#[cfg(test)]
mod multipart_auth_test;
@@ -395,64 +395,11 @@ impl Drop for OpaMock {
}
}
#[tokio::test]
async fn test_sts_session_policy_id_limits_object_access() -> TestResult {
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server(vec![]).await?;
let admin = env.create_s3_client();
let bucket = "session-policy-id";
admin.create_bucket().bucket(bucket).send().await?;
for key in ["allowed", "denied"] {
admin
.put_object()
.bucket(bucket)
.key(key)
.body(Bytes::from_static(b"data").into())
.send()
.await?;
}
let policy = serde_json::json!({
"Id": "read-one-object",
"Version": "2012-10-17",
"Statement": [{"Effect": "Allow", "Action": "s3:GetObject", "Resource": format!("arn:aws:s3:::{bucket}/allowed")}]
});
let response = sts_client(&env.url, &env.access_key, &env.secret_key, None)
.assume_role()
.role_arn("arn:aws:iam::123456789012:role/test")
.role_session_name("session-policy-id")
.policy(policy.to_string())
.send()
.await?;
let credentials = response.credentials().ok_or("AssumeRole returned no credentials")?;
let session = aws_sdk_s3::Client::from_conf(build_test_s3_config(
&env.url,
credentials.access_key_id(),
credentials.secret_access_key(),
Some(credentials.session_token()),
"e2e-session-policy-id",
));
let allowed = session.get_object().bucket(bucket).key("allowed").send().await?;
assert_eq!(allowed.body.collect().await?.into_bytes().as_ref(), b"data");
let error = session
.get_object()
.bucket(bucket)
.key("denied")
.send()
.await
.expect_err("object outside session policy");
assert_eq!(error.as_service_error().and_then(ProvideErrorMetadata::code), Some("AccessDenied"));
Ok(())
}
#[tokio::test]
async fn test_sts_query_responses_are_aws_sdk_compatible() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
// Exercise the minimum root secret length accepted at startup.
env.secret_key.truncate(8);
assert_eq!(env.secret_key.len(), 8);
env.start_rustfs_server(vec![]).await?;
let assumed = sts_client(&env.url, &env.access_key, &env.secret_key, None)
@@ -0,0 +1,475 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Website hosting and the `?metadata=true` listings on the gateway stack (rustfs/backlog#2734
//! T2.11, PR "W"; release blocker R-064).
//!
//! Responsible for: proving that a server on `RUSTFS_S3_STACK=gateway` answers website-host
//! requests and the two metadata listings exactly as the legacy stack does. One data directory,
//! one binary, two processes in turn: the legacy stack seeds the buckets (the gateway stack cannot
//! write them yet) and answers a fixed set of requests, then the same directory is restarted on
//! the gateway stack and answers the same requests. Each answer is compared on status, `Location`,
//! `Content-Type`, `Vary`, the CORS headers and the body bytes. The legacy answers are also held
//! to what a website must answer, so that two stacks failing the same way cannot pass, and one
//! request the gateway stack must NOT hand to legacy is held to differ, so that a restart that
//! silently kept the legacy stack cannot pass either.
//! NOT responsible for: the comparison against the released 1.0.5 binary
//! (`.config/reference-binary.toml`); this harness has no reference side yet, so what is proved
//! here is gateway == legacy of the candidate, not candidate == 1.0.5.
//! Upstream: the routes under test are `rustfs/src/server/stack/{website,metadata}.rs`.
#[cfg(test)]
mod tests {
use crate::common::{RustFSTestEnvironment, admin_ok, init_logging};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{CorsConfiguration, CorsRule, ErrorDocument, IndexDocument, WebsiteConfiguration};
use http::Method;
use http::header::HOST;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use s3s::Body;
use std::time::Duration;
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
const WEBSITE_DOMAIN: &str = "website.test";
const PUBLIC_BUCKET: &str = "interim-site-public";
const PRIVATE_BUCKET: &str = "interim-site-private";
const NO_LIST_USER: &str = "interimsitenolist";
const NO_LIST_SECRET: &str = "interim-site-no-list-secret";
const ORIGIN: &str = "https://app.example";
/// The response headers compared besides status, `Location` and body.
const COMPARED_HEADERS: [&str; 8] = [
"content-type",
"vary",
"access-control-allow-origin",
"access-control-allow-methods",
"access-control-allow-headers",
"access-control-expose-headers",
"access-control-allow-credentials",
"access-control-max-age",
];
/// What one request was answered with, reduced to the compared fields.
#[derive(Debug, PartialEq, Eq)]
struct Answer {
status: u16,
location: Option<String>,
headers: Vec<(&'static str, String)>,
body: Vec<u8>,
}
impl Answer {
async fn of(response: reqwest::Response) -> TestResult<Self> {
let status = response.status().as_u16();
let location = header_text(response.headers(), "location");
let headers = COMPARED_HEADERS
.iter()
.filter_map(|name| header_text(response.headers(), name).map(|value| (*name, value)))
.collect();
let body = response.bytes().await?.to_vec();
Ok(Self {
status,
location,
headers,
body,
})
}
fn body_text(&self) -> String {
String::from_utf8_lossy(&self.body).into_owned()
}
}
/// Every value of `name`, joined, so a second `Vary` line is a difference too.
fn header_text(headers: &reqwest::header::HeaderMap, name: &str) -> Option<String> {
let values: Vec<&str> = headers
.get_all(name)
.iter()
.map(|value| value.to_str().unwrap_or("<not UTF-8>"))
.collect();
(!values.is_empty()).then(|| values.join(" | "))
}
/// A local client that reports a redirect instead of following it: the 302 and its
/// `Location` are what is compared.
fn no_redirect_client() -> reqwest::Client {
reqwest::Client::builder()
.no_proxy()
.redirect(reqwest::redirect::Policy::none())
.build()
.expect("local http client")
}
fn website_host(bucket: &str) -> String {
format!("{bucket}.{WEBSITE_DOMAIN}")
}
/// An unsigned request to `target` on this server, addressed to `host`.
async fn anonymous(
env: &RustFSTestEnvironment,
method: Method,
host: &str,
target: &str,
extra: &[(&str, &str)],
) -> TestResult<Answer> {
let mut request = no_redirect_client()
.request(method, format!("{}{target}", env.url))
.header(HOST, host)
.header("origin", ORIGIN);
for (name, value) in extra {
request = request.header(*name, *value);
}
Answer::of(request.send().await?).await
}
/// A SigV4 request to `target` on this server, addressed and signed for `host`.
async fn signed(
env: &RustFSTestEnvironment,
host: &str,
target: &str,
access_key: &str,
secret_key: &str,
) -> TestResult<Answer> {
let url = format!("{}{target}", env.url);
let request = http::Request::builder()
.method(Method::GET)
.uri(url.as_str())
.header(HOST, host)
.header("origin", ORIGIN)
.header("x-amz-content-sha256", UNSIGNED_PAYLOAD)
.body(Body::empty())?;
let signed = sign_v4(request, 0, access_key, secret_key, "", "us-east-1");
let mut request = no_redirect_client().get(url);
for (name, value) in signed.headers() {
request = request.header(name, value);
}
Answer::of(request.send().await?).await
}
/// The server's own address: a path-style request, not a website one.
fn server_host(env: &RustFSTestEnvironment) -> &str {
&env.address
}
/// Writes everything the comparison reads, through the legacy stack.
async fn seed(env: &RustFSTestEnvironment) -> TestResult {
let s3 = env.create_s3_client();
for bucket in [PUBLIC_BUCKET, PRIVATE_BUCKET] {
s3.create_bucket().bucket(bucket).send().await?;
for (key, body) in [
("index.html", "home"),
("404.html", "not here"),
("section/index.html", "section page"),
] {
s3.put_object()
.bucket(bucket)
.key(key)
.content_type("text/html")
.metadata("colour", "blue")
.body(ByteStream::from(body.as_bytes().to_vec()))
.send()
.await?;
}
s3.put_bucket_website()
.bucket(bucket)
.website_configuration(
WebsiteConfiguration::builder()
.index_document(IndexDocument::builder().suffix("index.html").build()?)
.error_document(ErrorDocument::builder().key("404.html").build()?)
.build(),
)
.send()
.await?;
}
// Only the public bucket is readable anonymously, and only by GetObject: no listing.
let policy = serde_json::json!({
"Version": "2012-10-17",
"Statement": [{
"Effect": "Allow",
"Principal": "*",
"Action": "s3:GetObject",
"Resource": format!("arn:aws:s3:::{PUBLIC_BUCKET}/*")
}]
});
s3.put_bucket_policy()
.bucket(PUBLIC_BUCKET)
.policy(policy.to_string())
.send()
.await?;
s3.put_bucket_cors()
.bucket(PUBLIC_BUCKET)
.cors_configuration(
CorsConfiguration::builder()
.cors_rules(
CorsRule::builder()
.allowed_origins(ORIGIN)
.allowed_methods("GET")
.allowed_methods("HEAD")
.expose_headers("ETag")
.build()?,
)
.build()?,
)
.send()
.await?;
// A real user whose policy grants nothing, so it may not list either bucket.
admin_ok(
env,
Method::PUT,
&format!("/rustfs/admin/v3/add-user?accessKey={NO_LIST_USER}"),
Some(serde_json::json!({ "secretKey": NO_LIST_SECRET, "status": "enabled" }).to_string()),
)
.await?;
Ok(())
}
/// IAM loads its users in the background after a start, so a user unknown for a moment is
/// answered `InvalidAccessKeyId`; wait until the user without a policy is known again.
async fn wait_until_the_user_is_loaded(env: &RustFSTestEnvironment) -> TestResult {
let target = format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true");
let deadline = tokio::time::Instant::now() + Duration::from_secs(30);
loop {
let answer = signed(env, server_host(env), &target, NO_LIST_USER, NO_LIST_SECRET).await?;
if !answer.body_text().contains("InvalidAccessKeyId") {
return Ok(());
}
if tokio::time::Instant::now() >= deadline {
return Err(format!("{NO_LIST_USER} is still unknown: {answer:?}").into());
}
tokio::time::sleep(Duration::from_millis(250)).await;
}
}
/// The fixed requests, in order, each with the name the comparison reports it under.
async fn observe(env: &RustFSTestEnvironment) -> TestResult<Vec<(&'static str, Answer)>> {
let public = website_host(PUBLIC_BUCKET);
let private = website_host(PRIVATE_BUCKET);
let mut answers = Vec::new();
wait_until_the_user_is_loaded(env).await?;
let head = anonymous(env, Method::HEAD, &public, "/", &[]).await?;
let etag = no_redirect_client()
.head(format!("{}/", env.url))
.header(HOST, &public)
.send()
.await?
.headers()
.get("etag")
.and_then(|value| value.to_str().ok())
.map(str::to_owned)
.ok_or("the website index document has no ETag")?;
answers.push(("website GET /", anonymous(env, Method::GET, &public, "/", &[]).await?));
answers.push(("website HEAD /", head));
answers.push(("website GET directory", anonymous(env, Method::GET, &public, "/section", &[]).await?));
answers.push(("website GET directory/", anonymous(env, Method::GET, &public, "/section/", &[]).await?));
answers.push((
"website GET missing key",
anonymous(env, Method::GET, &public, "/missing.html", &[]).await?,
));
answers.push((
"website GET Range",
anonymous(env, Method::GET, &public, "/index.html", &[("range", "bytes=0-1")]).await?,
));
answers.push((
"website GET If-None-Match",
anonymous(env, Method::GET, &public, "/", &[("if-none-match", etag.as_str())]).await?,
));
answers.push(("website POST", anonymous(env, Method::POST, &public, "/", &[]).await?));
answers.push(("website GET bare domain", anonymous(env, Method::GET, WEBSITE_DOMAIN, "/", &[]).await?));
answers.push(("website GET private bucket", anonymous(env, Method::GET, &private, "/", &[]).await?));
answers.push((
"website GET private bucket, owner-signed",
signed(env, &private, "/", &env.access_key, &env.secret_key).await?,
));
answers.push((
"website GET public bucket, owner-signed",
signed(env, &public, "/", &env.access_key, &env.secret_key).await?,
));
let host = server_host(env);
answers.push((
"ListObjectsV2 metadata=true",
signed(
env,
host,
&format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true"),
&env.access_key,
&env.secret_key,
)
.await?,
));
answers.push((
"ListObjectVersions metadata=true",
signed(
env,
host,
&format!("/{PUBLIC_BUCKET}?versions&metadata=true"),
&env.access_key,
&env.secret_key,
)
.await?,
));
answers.push((
"ListObjectsV2 metadata=true, no list permission",
signed(
env,
host,
&format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true"),
NO_LIST_USER,
NO_LIST_SECRET,
)
.await?,
));
answers.push((
"ListObjectsV2 metadata=true, anonymous",
anonymous(env, Method::GET, host, &format!("/{PUBLIC_BUCKET}?list-type=2&metadata=true"), &[]).await?,
));
Ok(answers)
}
/// A plain ListObjectsV2: not a metadata listing, so the gateway stack must answer it itself.
async fn plain_listing(env: &RustFSTestEnvironment) -> TestResult<Answer> {
signed(
env,
server_host(env),
&format!("/{PUBLIC_BUCKET}?list-type=2"),
&env.access_key,
&env.secret_key,
)
.await
}
fn answer<'a>(answers: &'a [(&'static str, Answer)], name: &str) -> &'a Answer {
&answers
.iter()
.find(|(named, _)| *named == name)
.unwrap_or_else(|| panic!("no answer named {name}"))
.1
}
/// What a website and the metadata listings must answer, whichever stack serves them.
fn assert_is_a_website(answers: &[(&'static str, Answer)]) {
let get = answer(answers, "website GET /");
assert_eq!((get.status, get.body_text().as_str()), (200, "home"), "{get:?}");
let head = answer(answers, "website HEAD /");
assert_eq!((head.status, head.body.len()), (200, 0), "{head:?}");
let directory = answer(answers, "website GET directory");
assert_eq!(
(directory.status, directory.location.as_deref()),
(302, Some("/section/")),
"{directory:?}"
);
let section = answer(answers, "website GET directory/");
assert_eq!((section.status, section.body_text().as_str()), (200, "section page"), "{section:?}");
let missing = answer(answers, "website GET missing key");
assert_eq!((missing.status, missing.body_text().as_str()), (404, "not here"), "{missing:?}");
let range = answer(answers, "website GET Range");
assert_eq!((range.status, range.body_text().as_str()), (206, "ho"), "{range:?}");
let not_modified = answer(answers, "website GET If-None-Match");
assert_eq!((not_modified.status, not_modified.body.len()), (304, 0), "{not_modified:?}");
assert_eq!(answer(answers, "website POST").status, 405);
assert_eq!(answer(answers, "website GET bare domain").status, 404);
for name in ["website GET private bucket", "website GET private bucket, owner-signed"] {
let refused = answer(answers, name);
assert_eq!(refused.status, 403, "{name}: {refused:?}");
assert!(!refused.body_text().contains("home"), "{name} leaked the index: {refused:?}");
}
let signed_public = answer(answers, "website GET public bucket, owner-signed");
assert_eq!((signed_public.status, signed_public.body_text().as_str()), (200, "home"));
for name in ["ListObjectsV2 metadata=true", "ListObjectVersions metadata=true"] {
let listing = answer(answers, name);
let body = listing.body_text();
assert_eq!(listing.status, 200, "{name}: {body}");
assert!(body.contains("<UserMetadata>"), "{name} lost the extension: {body}");
assert!(body.contains("blue"), "{name} lost the user metadata value: {body}");
}
for name in [
"ListObjectsV2 metadata=true, no list permission",
"ListObjectsV2 metadata=true, anonymous",
] {
let refused = answer(answers, name);
assert_eq!(refused.status, 403, "{name}: {refused:?}");
assert!(refused.body_text().contains("<Code>AccessDenied</Code>"), "{name}: {refused:?}");
}
}
#[tokio::test]
async fn the_gateway_stack_answers_website_and_metadata_listing_requests_as_the_legacy_stack_does() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
let legacy_env = [("RUSTFS_S3_STACK", "legacy"), ("RUSTFS_WEBSITE_DOMAINS", WEBSITE_DOMAIN)];
let gateway_env = [("RUSTFS_S3_STACK", "gateway"), ("RUSTFS_WEBSITE_DOMAINS", WEBSITE_DOMAIN)];
env.start_rustfs_server_with_env(vec![], &legacy_env).await?;
seed(&env).await?;
let legacy = observe(&env).await?;
let legacy_plain = plain_listing(&env).await?;
env.restart_server_preserving_data(vec![], &gateway_env).await?;
let gateway = observe(&env).await?;
let gateway_plain = plain_listing(&env).await?;
env.stop_server();
// The control: a request that is neither family is the gateway's own. Legacy lists the
// bucket; the gateway stack does not serve ListObjectsV2 yet. Equal answers here would
// mean the restart never reached the gateway stack, and every comparison below is void.
assert_eq!(legacy_plain.status, 200, "{legacy_plain:?}");
assert_eq!(gateway_plain.status, 501, "{gateway_plain:?}");
assert_is_a_website(&legacy);
// Every difference, not the first one: a family that regressed alone must name itself.
let differences: Vec<String> = legacy
.iter()
.zip(&gateway)
.filter(|((_, on_legacy), (_, on_gateway))| on_legacy != on_gateway)
.map(|((name, on_legacy), (_, on_gateway))| format!("{name}:\n legacy: {on_legacy:?}\n gateway: {on_gateway:?}"))
.collect();
assert!(
differences.is_empty(),
"the gateway stack answered {} of {} requests differently from the legacy stack:\n{}",
differences.len(),
legacy.len(),
differences.join("\n")
);
assert_eq!(gateway.len(), legacy.len());
Ok(())
}
/// The metadata listing on a bucket that does not exist: answered by the legacy route on both
/// stacks, so `NoSuchBucket`, never the gateway's `NotImplemented`.
#[tokio::test]
async fn a_metadata_listing_of_a_missing_bucket_is_refused_by_legacy_on_the_gateway_stack() -> TestResult {
init_logging();
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_with_env(vec![], &[("RUSTFS_S3_STACK", "gateway")])
.await?;
let host = env.address.clone();
let missing = signed(
&env,
&host,
"/interim-no-such-bucket?list-type=2&metadata=true",
&env.access_key,
&env.secret_key,
)
.await?;
let plain = signed(&env, &host, "/interim-no-such-bucket?list-type=2", &env.access_key, &env.secret_key).await?;
env.stop_server();
assert_eq!(missing.status, 404, "{missing:?}");
assert!(missing.body_text().contains("<Code>NoSuchBucket</Code>"), "{missing:?}");
assert_eq!(plain.status, 501, "{plain:?}");
assert!(plain.body_text().contains("<Code>NotImplemented</Code>"), "{plain:?}");
Ok(())
}
}
-5
View File
@@ -29,12 +29,7 @@ documentation = "https://docs.rs/rustfs-ecstore/latest/rustfs_ecstore/"
[lints]
workspace = true
[[example]]
name = "internode_rpc_bench"
test = true
[features]
internode-transport = []
default = []
fips = ["rustfs-s3-client/fips", "rustfs-utils/fips", "rustfs-tls-runtime/fips"]
diagnostic-read-version-caller = ["rustfs-io-metrics/diagnostic-read-version-caller"]
@@ -1,866 +0,0 @@
// Copyright 2026 RustFS Team
// SPDX-License-Identifier: Apache-2.0
//! Read-only authenticated ReadAll/Ping probe against explicitly prepared test disks.
//! Usage: internode_rpc_bench manifest.json output.json
//! Fixtures live under each test disk's .rustfs.sys/tmp/<fixture_id>/ directory.
use rustfs_ecstore::api::{
disk::{DiskAPI, DiskOption, DiskStore, endpoint::Endpoint, new_disk},
rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client},
};
use rustfs_protos::proto_gen::node_service::PingRequest;
use rustfs_tls_runtime::{OutboundTlsMaterial, TlsGeneration, ensure_crypto_provider, publish_global_outbound_tls_state};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use std::collections::BTreeSet;
use std::io::{Error, ErrorKind, Result, Write};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
use tokio::io::AsyncReadExt;
use tokio::sync::Semaphore;
use tokio::task::JoinSet;
use tokio::time::{Instant, sleep, sleep_until, timeout};
use uuid::Uuid;
const MAX_PAYLOAD: usize = 8 * 1024 * 1024;
const MAX_BULK_ROWS: usize = 4_000_000;
const MAX_CONTROL_ROWS: u32 = 1_000_000;
#[derive(Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct Config {
namespace: String,
targets: Vec<String>,
disk_ids: Option<Vec<Uuid>>,
fixture_id: Uuid,
fixture_file: PathBuf,
duration_secs: u32,
bulk_workers: usize,
control_rps: u32,
rpc_timeout_ms: u32,
tls_ca: Option<PathBuf>,
start_marker: Option<PathBuf>,
end_marker: Option<PathBuf>,
}
impl Config {
fn endpoints(&self) -> Result<Vec<Endpoint>> {
let valid_namespace = self.namespace.starts_with("grpc-bench-")
&& (12..=64).contains(&self.namespace.len())
&& self
.namespace
.bytes()
.all(|byte| byte.is_ascii_alphanumeric() || byte == b'-');
if !valid_namespace || self.fixture_id.is_nil() || !(1..=64).contains(&self.targets.len()) {
return Err(Error::new(ErrorKind::InvalidInput, "invalid test namespace, fixture ID or target count"));
}
if self
.disk_ids
.as_ref()
.is_some_and(|ids| ids.len() != self.targets.len() || ids.iter().any(Uuid::is_nil))
{
return Err(Error::new(
ErrorKind::InvalidInput,
"disk IDs must match every declared target and be non-nil",
));
}
if !(1..=1800).contains(&self.duration_secs)
|| !(1..=128).contains(&self.bulk_workers)
|| !(1..=10_000).contains(&self.control_rps)
|| self
.duration_secs
.checked_mul(self.control_rps)
.is_none_or(|count| count > MAX_CONTROL_ROWS)
|| !(1..=30_000).contains(&self.rpc_timeout_ms)
{
return Err(Error::new(ErrorKind::InvalidInput, "workload exceeds the probe's resource bounds"));
}
if self.start_marker.is_some() != self.end_marker.is_some()
|| self.start_marker == self.end_marker && self.start_marker.is_some()
{
return Err(Error::new(ErrorKind::InvalidInput, "provide distinct start/end markers or neither"));
}
let mut seen = BTreeSet::new();
self.targets
.iter()
.map(|target| {
let mut endpoint = Endpoint::try_from(target.as_str())
.map_err(|_| Error::new(ErrorKind::InvalidInput, "invalid test endpoint URL"))?;
let url = &endpoint.url;
if target.len() > 1024
|| !matches!(url.scheme(), "http" | "https")
|| url.host_str().is_none()
|| url.port().is_none_or(|port| port < 1024)
|| !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
|| url.path().contains('%')
|| !url.path().ends_with(&format!("/{}", self.namespace))
|| !seen.insert(url.to_string())
|| url.scheme() == "https" && self.tls_ca.is_none()
{
return Err(Error::new(ErrorKind::InvalidInput, "target must name an explicit test-disk URL"));
}
// This probe never constructs a local disk or invokes its startup cleanup.
endpoint.is_local = false;
Ok(endpoint)
})
.collect()
}
}
#[derive(Deserialize)]
#[serde(deny_unknown_fields)]
struct Marker {
time_ns: u64,
}
#[derive(Serialize)]
struct Sample {
operation: &'static str,
peer: usize,
scheduled_ns: u64,
dispatched_ns: Option<u64>,
completed_ns: u64,
outcome: &'static str,
bytes: usize,
}
#[derive(Serialize)]
struct Report {
complete: bool,
errors: usize,
manifest_sha256: String,
fixture_sha256: String,
start_unix_ns: String,
observation_end_unix_ns: String,
offered_duration_secs: u32,
control_rps: u32,
bulk_bytes_completed_during_offering: u64,
bulk_coverage_verified: bool,
bulk_workers: usize,
control_expected: u32,
targets: usize,
peers: usize,
samples: Vec<Sample>,
external_markers: bool,
scope: &'static str,
}
fn now_ns() -> Result<u64> {
u64::try_from(SystemTime::now().duration_since(UNIX_EPOCH).map_err(Error::other)?.as_nanos()).map_err(Error::other)
}
fn elapsed_ns(origin: Instant) -> Result<u64> {
u64::try_from(origin.elapsed().as_nanos()).map_err(Error::other)
}
fn measurement_origin(start_ns: u64, current_ns: u64, monotonic_now: Instant) -> Result<Instant> {
let lag = current_ns
.checked_sub(start_ns)
.filter(|lag| *lag <= 1_000_000_000)
.ok_or_else(|| Error::other("measurement start must be within the preceding second"))?;
monotonic_now
.checked_sub(Duration::from_nanos(lag))
.ok_or_else(|| Error::other("invalid monotonic origin"))
}
fn validate_observation_end(start_ns: u64, end_ns: u64, finished_ns: u64, current_ns: u64, elapsed: u64) -> Result<()> {
let clock_end = start_ns
.checked_add(elapsed)
.ok_or_else(|| Error::other("observation window overflow"))?;
if end_ns < finished_ns || end_ns > current_ns || clock_end.abs_diff(current_ns) > 100_000_000 {
return Err(Error::other("invalid measurement-end acknowledgement or clock drift"));
}
Ok(())
}
async fn bounded_file(path: &Path, limit: usize) -> Result<Vec<u8>> {
let file = tokio::fs::File::open(path).await?;
if !file.metadata().await?.is_file() {
return Err(Error::new(ErrorKind::InvalidInput, "probe inputs must be regular files"));
}
let mut data = Vec::new();
file.take(u64::try_from(limit).map_err(Error::other)? + 1)
.read_to_end(&mut data)
.await?;
if data.len() > limit {
return Err(Error::new(ErrorKind::InvalidData, "probe input exceeds its size bound"));
}
Ok(data)
}
async fn marker(path: &Path) -> Result<u64> {
timeout(Duration::from_secs(30), async {
loop {
match bounded_file(path, 512).await {
Ok(data) => {
return serde_json::from_slice::<Marker>(&data)
.map(|value| value.time_ns)
.map_err(Error::other);
}
Err(error) if error.kind() == ErrorKind::NotFound => sleep(Duration::from_millis(5)).await,
Err(error) => return Err(error),
}
}
})
.await
.map_err(|_| Error::new(ErrorKind::TimedOut, "measurement marker did not arrive"))?
}
fn phase(name: &str, time_ns: u64) -> Result<()> {
println!(
"{}",
serde_json::json!({"phase":name,"time_ns":time_ns.to_string(),"pid":std::process::id()})
);
std::io::stdout().flush()
}
async fn ping(peer: &str) -> Result<(u64, bytes::Bytes)> {
let mut client = node_service_time_out_client(peer, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
.await
.map_err(|_| Error::other("authenticated control client acquisition failed"))?
.max_decoding_message_size(4096);
let response = client
.ping(PingRequest {
version: 1,
body: bytes::Bytes::new(),
})
.await
.map_err(|_| Error::other("authenticated Ping failed"))?
.into_inner();
Ok((response.version, response.body))
}
struct Context {
config: Config,
disks: Vec<DiskStore>,
peers: Vec<String>,
echoes: Vec<bytes::Bytes>,
payload: Vec<u8>,
path: String,
}
async fn bulk(context: Arc<Context>, worker: usize, origin: Instant, end: Instant) -> Result<Vec<Sample>> {
let mut rows = Vec::with_capacity(256);
let limit = MAX_BULK_ROWS / context.config.bulk_workers;
for sequence in 0..limit {
if Instant::now() >= end {
return Ok(rows);
}
let peer = (worker + sequence) % context.disks.len();
let dispatched = elapsed_ns(origin)?;
let result = timeout(
Duration::from_millis(u64::from(context.config.rpc_timeout_ms)),
context.disks[peer].read_all(".rustfs.sys", &context.path),
)
.await;
let completed = elapsed_ns(origin)?;
let outcome = match result {
Ok(Ok(data)) if data.as_ref() == context.payload.as_slice() => "ok",
Ok(Ok(_)) => "payload_mismatch",
Ok(Err(_)) => "rpc_error",
Err(_) => "deadline",
};
rows.push(Sample {
operation: "ReadAll",
peer,
scheduled_ns: dispatched,
dispatched_ns: Some(dispatched),
completed_ns: completed,
outcome,
bytes: if outcome == "ok" { context.payload.len() } else { 0 },
});
}
if Instant::now() >= end {
Ok(rows)
} else {
Err(Error::other("bulk raw-sample limit reached; attribution is invalid"))
}
}
async fn control(context: Arc<Context>, origin: Instant) -> Result<Vec<Sample>> {
let count = context.config.duration_secs * context.config.control_rps;
let capacity = usize::try_from(count).map_err(Error::other)?;
let mut rows = Vec::with_capacity(capacity);
let permits = Arc::new(Semaphore::new(128));
let end = origin + Duration::from_secs(u64::from(context.config.duration_secs));
let mut tasks = JoinSet::new();
for sequence in 0..count {
// Configuration bounds count to one million, so this product fits u64.
let scheduled = u64::from(sequence) * 1_000_000_000 / u64::from(context.config.control_rps);
sleep_until(origin + Duration::from_nanos(scheduled)).await;
while let Some(result) = tasks.try_join_next() {
rows.push(result.map_err(Error::other)??);
}
let peer = usize::try_from(sequence).map_err(Error::other)? % context.peers.len();
let permit = (Instant::now() < end)
.then(|| permits.clone().try_acquire_owned())
.and_then(std::result::Result::ok);
let Some(permit) = permit else {
rows.push(Sample {
operation: "Ping",
peer,
scheduled_ns: scheduled,
dispatched_ns: None,
completed_ns: elapsed_ns(origin)?,
outcome: "schedule_drop",
bytes: 0,
});
continue;
};
let context = context.clone();
tasks.spawn(async move {
let _permit = permit;
let dispatched = elapsed_ns(origin)?;
let result = timeout(
Duration::from_millis(u64::from(context.config.rpc_timeout_ms)),
ping(&context.peers[peer]),
)
.await;
let completed = elapsed_ns(origin)?;
let outcome = match result {
Ok(Ok((1, body))) if body == context.echoes[peer] => "ok",
Ok(Ok(_)) => "payload_mismatch",
Ok(Err(_)) => "rpc_error",
Err(_) => "deadline",
};
Ok::<_, Error>(Sample {
operation: "Ping",
peer,
scheduled_ns: scheduled,
dispatched_ns: Some(dispatched),
completed_ns: completed,
outcome,
bytes: 0,
})
});
}
while let Some(result) = tasks.join_next().await {
rows.push(result.map_err(Error::other)??);
}
if rows.len() != capacity {
return Err(Error::other("control arrival accounting is incomplete"));
}
Ok(rows)
}
async fn run(config_bytes: &[u8], base: &Path, secret: String) -> Result<Report> {
let mut config: Config = serde_json::from_slice(config_bytes).map_err(|_| Error::other("invalid probe manifest"))?;
if config.fixture_file.is_relative() {
config.fixture_file = base.join(&config.fixture_file);
}
for path in [&mut config.tls_ca, &mut config.start_marker, &mut config.end_marker]
.into_iter()
.flatten()
{
if path.is_relative() {
*path = base.join(&*path);
}
}
let endpoints = config.endpoints()?;
for path in [&config.start_marker, &config.end_marker].into_iter().flatten() {
if tokio::fs::try_exists(path).await? {
return Err(Error::other("measurement markers must be fresh"));
}
}
// Require the test deployment's explicit secret; never use a default credential.
if secret.trim().is_empty() {
return Err(Error::other("RUSTFS_RPC_SECRET is required"));
}
rustfs_credentials::set_global_rpc_secret(secret).map_err(|_| Error::other("RPC secret already initialized"))?;
ensure_crypto_provider().map_err(Error::other)?;
if let Some(path) = &config.tls_ca {
let ca = bounded_file(path, 1024 * 1024).await?;
if ca.is_empty() {
return Err(Error::other("the explicit test CA must not be empty"));
}
publish_global_outbound_tls_state(
TlsGeneration(1),
&OutboundTlsMaterial {
root_ca_pem: ca,
mtls_identity: None,
},
)
.await;
}
let payload = bounded_file(&config.fixture_file, MAX_PAYLOAD).await?;
if payload.is_empty()
|| payload
.len()
.checked_mul(config.bulk_workers)
.is_none_or(|bytes| bytes > 512 * 1024 * 1024)
{
return Err(Error::other("fixture and worker count exceed the payload memory budget"));
}
let path = format!("tmp/{}/grpc-bench-payload.bin", config.fixture_id);
let mut disks = Vec::with_capacity(endpoints.len());
let mut peers = BTreeSet::new();
for (index, endpoint) in endpoints.into_iter().enumerate() {
peers.insert(endpoint.grid_host());
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.map_err(Error::other)?;
if let Some(ids) = &config.disk_ids {
disk.set_disk_id(Some(ids[index])).await.map_err(Error::other)?;
}
let data = timeout(Duration::from_secs(30), disk.read_all(".rustfs.sys", &path))
.await
.map_err(Error::other)?
.map_err(|error| Error::other(format!("ReadAll fixture preflight failed: disk_error_code={}", error.to_u32())))?;
if data.as_ref() != payload.as_slice() {
return Err(Error::other("ReadAll fixture preflight mismatch"));
}
disks.push(disk);
}
let peers: Vec<_> = peers.into_iter().collect();
let mut echoes = Vec::with_capacity(peers.len());
for peer in &peers {
let (version, body) = timeout(Duration::from_secs(30), ping(peer)).await.map_err(Error::other)??;
if version != 1 || body.is_empty() {
return Err(Error::other("Ping preflight mismatch"));
}
echoes.push(body);
}
let context = Arc::new(Context {
config,
disks,
peers,
echoes,
payload,
path,
});
phase("ready", now_ns()?)?;
let start_ns = match &context.config.start_marker {
Some(path) => marker(path).await?,
None => now_ns()?,
};
let current_ns = now_ns()?;
let origin = measurement_origin(start_ns, current_ns, Instant::now())?;
let offered = Duration::from_secs(u64::from(context.config.duration_secs));
let end = origin + offered;
phase("measure_start", start_ns)?;
let mut tasks = JoinSet::new();
for worker in 0..context.config.bulk_workers {
tasks.spawn(bulk(context.clone(), worker, origin, end));
}
tasks.spawn(control(context.clone(), origin));
let mut samples = Vec::new();
while let Some(result) = tasks.join_next().await {
samples.extend(result.map_err(Error::other)??);
}
let finished_ns = now_ns()?;
phase("load_finished", finished_ns)?;
let end_ns = match &context.config.end_marker {
Some(path) => marker(path).await?,
None => now_ns()?,
};
let current_ns = now_ns()?;
validate_observation_end(start_ns, end_ns, finished_ns, current_ns, elapsed_ns(origin)?)?;
let offered_ns = u64::try_from(offered.as_nanos()).map_err(Error::other)?;
let errors = samples.iter().filter(|sample| sample.outcome != "ok").count();
let measured_bytes = samples
.iter()
.filter(|sample| sample.operation == "ReadAll" && sample.outcome == "ok" && sample.completed_ns <= offered_ns)
.try_fold(0_u64, |total, sample| {
total
.checked_add(u64::try_from(sample.bytes).map_err(Error::other)?)
.ok_or_else(|| Error::other("measured payload byte counter overflow"))
})?;
let report = Report {
complete: true,
errors,
manifest_sha256: hex_simd::encode_to_string(Sha256::digest(config_bytes), hex_simd::AsciiCase::Lower),
fixture_sha256: hex_simd::encode_to_string(Sha256::digest(&context.payload), hex_simd::AsciiCase::Lower),
start_unix_ns: start_ns.to_string(),
observation_end_unix_ns: end_ns.to_string(),
offered_duration_secs: context.config.duration_secs,
control_rps: context.config.control_rps,
bulk_bytes_completed_during_offering: measured_bytes,
bulk_coverage_verified: measured_bytes > 0,
bulk_workers: context.config.bulk_workers,
control_expected: context.config.duration_secs * context.config.control_rps,
targets: context.disks.len(),
peers: context.peers.len(),
samples,
external_markers: context.config.start_marker.is_some(),
scope: "Authenticated production RPC clients and ReadAll temporary fixtures; client telemetry export is disabled. Storage S3/durability, mTLS/rotation and ABBA qualification remain separate.",
};
for disk in &context.disks {
disk.close().await.map_err(Error::other)?;
}
phase("measure_end", end_ns)?;
Ok(report)
}
#[tokio::main(worker_threads = 8)]
async fn main() -> Result<()> {
let mut args = std::env::args_os().skip(1);
let config = args
.next()
.map(PathBuf::from)
.ok_or_else(|| Error::other("usage: internode_rpc_bench manifest.json output.json"))?;
let output = args
.next()
.map(PathBuf::from)
.ok_or_else(|| Error::other("output path is required"))?;
if args.next().is_some() {
return Err(Error::other("unexpected probe arguments"));
}
let input = bounded_file(&config, 64 * 1024).await?;
let file = tokio::fs::OpenOptions::new()
.write(true)
.create_new(true)
.open(output)
.await?
.into_std()
.await;
let secret = std::env::var("RUSTFS_RPC_SECRET").map_err(|_| Error::other("RUSTFS_RPC_SECRET is required"))?;
let report = run(&input, config.parent().unwrap_or_else(|| Path::new(".")), secret).await?;
let success = report.errors == 0 && report.bulk_coverage_verified;
tokio::task::spawn_blocking(move || {
let mut output = std::io::BufWriter::new(file);
serde_json::to_writer(&mut output, &report).map_err(Error::other)?;
output.flush()
})
.await
.map_err(Error::other)??;
if success {
Ok(())
} else {
Err(Error::other(
"RPC failures or missing bulk coverage invalidate attribution; see raw samples",
))
}
}
#[cfg(test)]
mod tests {
use super::*;
use rustfs_protos::proto_gen::node_service::{PingResponse, ReadAllRequest, ReadAllResponse};
use std::convert::Infallible;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use std::task::{Context as TaskContext, Poll};
use tonic::codegen::{Body, BoxFuture, Service, StdError, http};
fn config() -> Config {
Config {
namespace: "grpc-bench-test".into(),
targets: vec!["http://127.0.0.1:19000/data/grpc-bench-test".into()],
disk_ids: None,
fixture_id: Uuid::from_u128(1),
fixture_file: PathBuf::from("fixture.bin"),
duration_secs: 1,
bulk_workers: 2,
control_rps: 20,
rpc_timeout_ms: 1000,
tls_ca: None,
start_marker: None,
end_marker: None,
}
}
#[test]
fn probe_rejects_non_test_and_credential_bearing_targets() {
for target in [
"file:///data/grpc-bench-test",
"http://user:private@127.0.0.1:19000/data/grpc-bench-test",
"http://127.0.0.1:19000/data/mnmd",
"http://127.0.0.1:80/data/grpc-bench-test",
"http://127.0.0.1:19000/data/grpc-bench-test?token=private",
"http://127.0.0.1:19000/data/%67rpc-bench-test",
"https://127.0.0.1:19000/data/grpc-bench-test",
] {
let mut config = config();
config.targets = vec![target.into()];
assert!(config.endpoints().is_err(), "unsafe target was accepted");
}
}
#[test]
fn probe_forces_remote_client_and_rejects_duplicate_disks() {
let mut config = config();
assert!(!config.endpoints().expect("valid loopback test URL")[0].is_local);
config.targets.push(config.targets[0].clone());
assert!(config.endpoints().is_err(), "duplicate target must not inflate the inventory");
}
#[test]
fn probe_requires_a_complete_non_nil_disk_id_mapping() {
let mut config = config();
config.disk_ids = Some(vec![]);
assert!(config.endpoints().is_err());
config.disk_ids = Some(vec![Uuid::nil()]);
assert!(config.endpoints().is_err());
config.disk_ids = Some(vec![Uuid::from_u128(42)]);
assert!(config.endpoints().is_ok());
}
#[test]
fn probe_bounds_work_and_requires_distinct_measurement_markers() {
for (seconds, workers, rate, deadline) in [
(0, 2, 20, 1000),
(1, 0, 20, 1000),
(1, 129, 20, 1000),
(1800, 2, 10_000, 1000),
(1, 2, 0, 1000),
(1, 2, 20, 0),
] {
let mut config = config();
config.duration_secs = seconds;
config.bulk_workers = workers;
config.control_rps = rate;
config.rpc_timeout_ms = deadline;
assert!(config.endpoints().is_err());
}
let mut config = config();
config.start_marker = Some("marker.json".into());
assert!(config.endpoints().is_err());
config.end_marker = config.start_marker.clone();
assert!(config.endpoints().is_err());
}
#[test]
fn probe_rejects_stale_future_or_truncated_measurement_markers() {
let now = Instant::now();
assert_eq!(
measurement_origin(10_000_000_000, 11_000_000_000, now).expect("one-second lag bound"),
now - Duration::from_secs(1)
);
assert!(measurement_origin(10_000_000_000, 11_000_000_001, now).is_err());
assert!(measurement_origin(11_000_000_001, 11_000_000_000, now).is_err());
assert!(validate_observation_end(100, 200, 200, 250, 150).is_ok());
assert!(validate_observation_end(100, 199, 200, 250, 150).is_err());
assert!(validate_observation_end(100, 251, 200, 250, 150).is_err());
assert!(validate_observation_end(100, 200, 200, 250_000_000, 150).is_err());
}
#[tokio::test]
async fn probe_marker_input_has_an_explicit_schema_and_size_bound() {
let directory = tempfile::tempdir().expect("marker directory");
let path = directory.path().join("marker.json");
tokio::fs::write(&path, br#"{"time_ns":100}"#).await.expect("marker");
assert_eq!(marker(&path).await.expect("valid marker"), 100);
tokio::fs::write(&path, br#"{"time_ns":100,"unknown":true}"#)
.await
.expect("invalid marker");
assert!(marker(&path).await.is_err());
tokio::fs::write(&path, vec![b' '; 513]).await.expect("oversized marker");
assert!(marker(&path).await.is_err());
}
#[tokio::test]
async fn probe_bounds_fixture_reads_and_rejects_directories() {
let directory = tempfile::tempdir().expect("temporary fixture directory");
let path = directory.path().join("fixture.bin");
tokio::fs::write(&path, [7_u8; 32]).await.expect("fixture write");
assert_eq!(bounded_file(&path, 32).await.expect("exact-size fixture"), vec![7_u8; 32]);
tokio::fs::write(&path, [7_u8; 33]).await.expect("oversized fixture write");
assert!(bounded_file(&path, 32).await.is_err());
assert!(bounded_file(directory.path(), 32).await.is_err());
}
#[tokio::test]
async fn probe_accounts_for_expired_control_arrivals_without_sending_them() {
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("owned control socket");
let context = Arc::new(super::Context {
config: config(),
disks: vec![],
peers: vec![format!("http://{}", listener.local_addr().expect("owned address"))],
echoes: vec![bytes::Bytes::from_static(b"hello")],
payload: vec![],
path: String::new(),
});
let rows = control(context, Instant::now() - Duration::from_secs(2))
.await
.expect("arrival accounting");
assert_eq!(rows.len(), 20);
assert!(
rows.iter()
.all(|row| row.outcome == "schedule_drop" && row.dispatched_ns.is_none())
);
}
#[derive(Clone)]
struct ReadFixture(bytes::Bytes, Arc<AtomicBool>);
impl tonic::server::UnaryService<ReadAllRequest> for ReadFixture {
type Response = ReadAllResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, request: tonic::Request<ReadAllRequest>) -> Self::Future {
assert_eq!(request.get_ref().volume, ".rustfs.sys");
assert_eq!(request.get_ref().disk, Uuid::from_u128(42).to_string());
assert!(request.get_ref().path.ends_with("/grpc-bench-payload.bin"));
let data = if self.1.load(Ordering::SeqCst) {
bytes::Bytes::from_static(b"corrupt fixture")
} else {
self.0.clone()
};
Box::pin(async move {
Ok(tonic::Response::new(ReadAllResponse {
success: true,
data,
error: None,
}))
})
}
}
#[derive(Clone)]
struct ReplyPing;
impl tonic::server::UnaryService<PingRequest> for ReplyPing {
type Response = PingResponse;
type Future = BoxFuture<tonic::Response<Self::Response>, tonic::Status>;
fn call(&mut self, _: tonic::Request<PingRequest>) -> Self::Future {
Box::pin(async {
Ok(tonic::Response::new(PingResponse {
version: 1,
body: bytes::Bytes::from_static(b"hello, caller"),
}))
})
}
}
#[derive(Clone)]
struct FixtureService {
data: bytes::Bytes,
signed: Arc<AtomicUsize>,
corrupt: Arc<AtomicBool>,
}
impl tonic::server::NamedService for FixtureService {
const NAME: &'static str = "node_service.NodeService";
}
impl<B> Service<http::Request<B>> for FixtureService
where
B: Body + Send + 'static,
B::Error: Into<StdError> + Send + 'static,
{
type Response = http::Response<tonic::body::Body>;
type Error = Infallible;
type Future = BoxFuture<Self::Response, Self::Error>;
fn poll_ready(&mut self, _: &mut TaskContext<'_>) -> Poll<std::result::Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, request: http::Request<B>) -> Self::Future {
assert_eq!(
request
.headers()
.get("x-rustfs-rpc-auth-version")
.expect("signed production client"),
"2"
);
assert!(request.headers().contains_key("x-rustfs-rpc-signature-v2"));
self.signed.fetch_add(1, Ordering::Relaxed);
let data = self.data.clone();
let corrupt = self.corrupt.clone();
Box::pin(async move {
if request.uri().path().ends_with("/ReadAll") {
Ok(tonic::server::Grpc::new(tonic_prost::ProstCodec::default())
.unary(ReadFixture(data, corrupt), request)
.await)
} else {
assert!(request.uri().path().ends_with("/Ping"));
Ok(tonic::server::Grpc::new(tonic_prost::ProstCodec::default())
.unary(ReplyPing, request)
.await)
}
})
}
}
#[tokio::test]
async fn probe_exercises_production_readall_and_fixed_rate_ping_clients() {
let directory = tempfile::tempdir().expect("probe fixtures");
let payload = vec![17_u8; 4096];
let file = directory.path().join("fixture.bin");
tokio::fs::write(&file, &payload).await.expect("fixture bytes");
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
.await
.expect("owned RPC listener");
let address = listener.local_addr().expect("owned RPC address");
let incoming = futures_util::stream::unfold(listener, |listener| async move {
Some((listener.accept().await.map(|(stream, _)| stream), listener))
});
let (stop, stopped) = tokio::sync::oneshot::channel();
let signed = Arc::new(AtomicUsize::new(0));
let corrupt = Arc::new(AtomicBool::new(false));
let service = FixtureService {
data: payload.into(),
signed: signed.clone(),
corrupt: corrupt.clone(),
};
let mut server = tokio::spawn(
tonic::transport::Server::builder()
.add_service(service)
.serve_with_incoming_shutdown(Box::pin(incoming), async {
let _ = stopped.await;
}),
);
let mut config = config();
config.targets = vec![format!("http://{address}/data/grpc-bench-test")];
config.disk_ids = Some(vec![Uuid::from_u128(42)]);
config.fixture_file = file;
let input = serde_json::to_vec(&config).expect("probe manifest");
let result = run(&input, directory.path(), "test-bench-rpc-secret".into()).await;
let negative = async {
let endpoint = config.endpoints()?.remove(0);
let disk = new_disk(
&endpoint,
&DiskOption {
cleanup: false,
health_check: true,
},
)
.await
.map_err(Error::other)?;
disk.set_disk_id(Some(Uuid::from_u128(42))).await.map_err(Error::other)?;
corrupt.store(true, Ordering::SeqCst);
let context = Arc::new(super::Context {
path: format!("tmp/{}/grpc-bench-payload.bin", config.fixture_id),
config: config.clone(),
disks: vec![disk.clone()],
peers: vec![],
echoes: vec![],
payload: tokio::fs::read(&config.fixture_file).await?,
});
let origin = Instant::now();
let rows = bulk(context, 0, origin, origin + Duration::from_millis(50)).await?;
disk.close().await.map_err(Error::other)?;
Ok::<_, Error>(rows)
}
.await;
let _ = stop.send(());
let retirement = timeout(Duration::from_secs(5), &mut server).await;
if retirement.is_err() {
server.abort();
let _ = server.await;
}
assert!(matches!(retirement, Ok(Ok(Ok(())))), "owned RPC fixture must retire successfully");
let result = result.expect("production client probe");
assert!(result.bulk_coverage_verified);
let report = serde_json::to_value(result).expect("raw sample report");
assert_eq!(report["errors"], 0);
assert_eq!(report["control_expected"], 20);
let rows = report["samples"].as_array().expect("raw rows");
assert_eq!(rows.iter().filter(|row| row["operation"] == "Ping").count(), 20);
assert!(rows.iter().any(|row| row["operation"] == "ReadAll"));
assert!(signed.load(Ordering::Relaxed) > 20);
let rejected = negative.expect("corrupted-response probe");
assert!(!rejected.is_empty());
assert!(rejected.iter().all(|row| row.outcome == "payload_mismatch" && row.bytes == 0));
}
}
+5 -29
View File
@@ -194,8 +194,8 @@ pub mod bucket {
table_catalog_path_hash, unreadable_config_refusal,
};
pub use crate::bucket::metadata::{
BUCKET_DURABILITY_CONFIG, BUCKET_ETAG_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT,
MAX_SWIFT_METADATA_VALUE_SIZE, SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
BUCKET_DURABILITY_CONFIG, BUCKET_ON_DEMAND_MIGRATION_CONFIG, MAX_SWIFT_METADATA_COUNT, MAX_SWIFT_METADATA_VALUE_SIZE,
SWIFT_ACCOUNT_META_TAG_PREFIX, SWIFT_CONTAINER_META_TAG_PREFIX, SWIFT_TAGGING_MARKER,
};
pub use crate::bucket::metadata::{MAX_BUCKET_TAGS, is_swift_bucket_metadata_tag_key, validate_tag_set_entries};
}
@@ -212,16 +212,6 @@ pub mod bucket {
};
}
pub mod etag {
pub use crate::bucket::etag::{
BUCKET_ETAG_ALGORITHM_MD5, BUCKET_ETAG_ALGORITHM_XXH3, BucketEtagConfig, bucket_uses_xxh3_etag,
};
}
pub mod lockless_cache {
pub use crate::bucket::lockless_cache::is_lockless_cache_bucket;
}
pub mod metadata_sys {
pub use crate::bucket::metadata_sys::{
BUCKET_CONFIG_PUBLISH_HOOK, BucketConfigPublishHook, BucketMetadataMutationGuard, BucketMetadataSys,
@@ -230,7 +220,7 @@ pub mod bucket {
acquire_scanner_bucket_incarnation_read_fence, capture_bucket_metadata_incarnation, delete, delete_if_incarnation,
delete_if_incarnation_at, delete_under_transaction_lock, get, get_accelerate_config, get_bucket_policy,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_etag_config, get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_object_lock_config_state, get_on_demand_migration_config,
get_on_demand_migration_config_in, get_public_access_block_config, get_quota_config, get_replication_config,
get_request_payment_config, get_s3_tagging_config, get_sse_config, get_tagging_config, get_versioning_config,
@@ -604,14 +594,9 @@ pub mod integrity {
}
pub mod object {
pub use crate::erasure::codec::shard_buf::{
ArenaBlock, RegisteredSlabArena, ShardArena, ShardBuf, install_shard_arena, shard_arena,
};
pub use crate::erasure::codec::shard_emit::ShardEmitter;
pub use crate::object_api::{
BLOCK_SIZE_V2, ERASURE_ALGORITHM, EncryptionResolutionError, EncryptionResolutionErrorKind, GetObjectBodyCacheHook,
GetObjectBodyCacheHookLookup, GetObjectBodySource, GetObjectReadCondition, GetObjectReadMetadata, GetObjectReader,
GetObjectSink, GetObjectSinkSegmentFactory, GetObjectSinkSegments, GetObjectSinkStream, GetObjectSinkWriter,
NamespaceLockFence, ObjectEncryptionResolver, ObjectInfo, ObjectLockConfigSnapshot, ObjectMutationHook, ObjectOptions,
PutObjReader, QuotaAdmission, RangedDecompressReader, ReadEncryptionMaterial, ReadEncryptionMode, ReadEncryptionRequest,
SCANNER_PUBLICATION_LEASE_FENCE_METADATA_KEY, ScannerPublicationCommitScope, ScannerPublicationCommitStartError,
@@ -647,20 +632,13 @@ pub mod rebalance {
pub mod rio {
pub use crate::io_support::rio::{
DecryptReader, DynReader, EncryptReader, EtagAlgorithm, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption,
WritePlan, boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
DecryptReader, DynReader, EncryptReader, HardLimitReader, HashReader, ReadStream, Reader, WriteEncryption, WritePlan,
boxed_reader, compression_metadata_value, validate_compression_scheme, wrap_reader,
};
}
pub mod rpc {
#[cfg(feature = "internode-transport")]
pub use crate::cluster::rpc::http_auth::ExternalStreamAuth;
pub use crate::cluster::rpc::http_auth::with_rpc_background_request;
#[cfg(feature = "internode-transport")]
pub use crate::cluster::rpc::internode_data_transport::{
InternodeDataTransport, InternodeDataTransportCapabilities, NsScannerCapabilityRequest, NsScannerStreamRequest,
ReadStreamRequest, TcpHttpInternodeDataTransport, WalkDirStreamRequest, WriteStreamRequest,
};
pub use crate::cluster::rpc::{
AuthenticatedChannel, KMS_SIGNAL_SUBSYSTEM, LocalPeerS3Client, MAX_NETWORK_PROBE_BYTES, MAX_NETWORK_PROBE_DURATION,
NetworkPeerProbeClient, NetworkPeerProbeError, NetworkPeerProbeMeasurement, NetworkPeerTarget, PEER_RESTDRY_RUN,
@@ -721,8 +699,6 @@ pub mod store_list {
pub mod storage {
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
pub use crate::store::HealWalkVersion;
#[cfg(feature = "internode-transport")]
pub use crate::store::external_rpc_disk;
pub use crate::store::{
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
@@ -1189,7 +1189,6 @@ impl BucketTargetSys {
disable_proxy: target.disable_proxy,
arn: target.arn.clone(),
reset_id: target.reset_id.clone(),
site_peer_deployment_id: target.deployment_id.trim().to_string(),
secure: target.secure,
health_check_duration: target.health_check_duration,
replicate_sync: target.replication_sync,
@@ -2012,9 +2011,6 @@ pub struct TargetClient {
pub disable_proxy: bool,
pub arn: String,
pub reset_id: String,
/// Deployment id of the site-replication peer this target was derived
/// for; empty for an operator-configured bucket-replication target.
pub site_peer_deployment_id: String,
pub secure: bool,
pub health_check_duration: Duration,
pub replicate_sync: bool,
@@ -2023,13 +2019,6 @@ pub struct TargetClient {
}
impl TargetClient {
/// Whether the target is a site-replication peer. Peers share bucket
/// configuration and IAM, and are required to hold KMS keys under the
/// same names; an operator-configured target promises neither.
pub fn is_site_replication_peer(&self) -> bool {
!self.site_peer_deployment_id.is_empty()
}
fn credentials_expired_at(&self, now: jiff::Timestamp) -> bool {
self.credentials
.as_ref()
@@ -3061,7 +3050,6 @@ mod tests {
disable_proxy: false,
arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
reset_id: String::new(),
site_peer_deployment_id: String::new(),
secure: true,
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
@@ -3605,7 +3593,6 @@ mod tests {
disable_proxy: false,
arn: "arn:rustfs:replication:us-east-1:target:bucket".to_string(),
reset_id: String::new(),
site_peer_deployment_id: String::new(),
secure: true,
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
@@ -3919,7 +3906,6 @@ mod tests {
disable_proxy: false,
arn: arn.to_string(),
reset_id: String::new(),
site_peer_deployment_id: String::new(),
secure: true,
health_check_duration: Duration::from_secs(5),
replicate_sync: false,
-150
View File
@@ -1,150 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Per-bucket ETag algorithm.
//!
//! A bucket can derive the ETags of objects written without Content-MD5 from
//! XXH3-128 instead of MD5, for clients that never treat the ETag as an MD5
//! (an LLM KV cache). The choice is stored as a RustFS extension entry
//! (`etag.json`) in the bucket metadata file and published to an in-memory
//! registry whenever the metadata is installed, so the write path looks it up
//! without I/O. Buckets without the entry keep MD5 ETags.
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{OnceLock, PoisonError, RwLock};
pub const BUCKET_ETAG_ALGORITHM_MD5: &str = "md5";
pub const BUCKET_ETAG_ALGORITHM_XXH3: &str = "xxh3";
/// JSON payload stored under the `etag.json` bucket metadata entry. An absent
/// entry, an empty payload or an invalid algorithm all mean MD5 ETags.
#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)]
pub struct BucketEtagConfig {
/// `md5` | `xxh3` (case-insensitive, whitespace-tolerant).
#[serde(default)]
pub algorithm: String,
}
impl BucketEtagConfig {
pub fn new(algorithm: &str) -> Self {
Self {
algorithm: algorithm.to_string(),
}
}
/// Whether `algorithm` names a supported ETag algorithm.
pub fn is_valid_algorithm(algorithm: &str) -> bool {
matches!(
algorithm.trim().to_ascii_lowercase().as_str(),
BUCKET_ETAG_ALGORITHM_MD5 | BUCKET_ETAG_ALGORITHM_XXH3
)
}
/// The canonical (trimmed, lowercase) algorithm name, or `None` when the
/// config does not carry a valid one.
pub fn normalized_algorithm(&self) -> Option<String> {
let algorithm = self.algorithm.trim().to_ascii_lowercase();
if Self::is_valid_algorithm(&algorithm) {
Some(algorithm)
} else {
None
}
}
/// Whether this config selects XXH3 ETags.
pub fn uses_xxh3(&self) -> bool {
self.normalized_algorithm().as_deref() == Some(BUCKET_ETAG_ALGORITHM_XXH3)
}
}
/// Buckets whose ETags use XXH3, kept in step with the installed bucket
/// metadata. `ANY` lets the write path skip the lock while no bucket opts in.
static XXH3_BUCKETS: OnceLock<RwLock<HashSet<String>>> = OnceLock::new();
static ANY: AtomicBool = AtomicBool::new(false);
fn xxh3_buckets() -> &'static RwLock<HashSet<String>> {
XXH3_BUCKETS.get_or_init(|| RwLock::new(HashSet::new()))
}
/// Records whether `bucket` uses XXH3 ETags.
pub fn set_bucket_xxh3_etag(bucket: &str, xxh3: bool) {
if bucket.is_empty() {
return;
}
let mut buckets = xxh3_buckets().write().unwrap_or_else(PoisonError::into_inner);
if xxh3 {
buckets.insert(bucket.to_string());
} else {
buckets.remove(bucket);
}
ANY.store(!buckets.is_empty(), Ordering::Release);
}
/// Whether objects written to `bucket` without Content-MD5 get XXH3 ETags.
pub fn bucket_uses_xxh3_etag(bucket: &str) -> bool {
if !ANY.load(Ordering::Acquire) {
return false;
}
xxh3_buckets().read().unwrap_or_else(PoisonError::into_inner).contains(bucket)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn valid_algorithms_are_recognized() {
assert!(BucketEtagConfig::is_valid_algorithm("md5"));
assert!(BucketEtagConfig::is_valid_algorithm(" XXH3 "));
assert!(!BucketEtagConfig::is_valid_algorithm(""));
assert!(!BucketEtagConfig::is_valid_algorithm("sha256"));
}
#[test]
fn normalized_algorithm_canonicalizes_or_rejects() {
assert_eq!(BucketEtagConfig::new(" Xxh3 ").normalized_algorithm().as_deref(), Some("xxh3"));
assert_eq!(BucketEtagConfig::new("MD5").normalized_algorithm().as_deref(), Some("md5"));
assert_eq!(BucketEtagConfig::new("crc32").normalized_algorithm(), None);
assert!(BucketEtagConfig::new("xxh3").uses_xxh3());
assert!(!BucketEtagConfig::new("md5").uses_xxh3());
assert!(!BucketEtagConfig::default().uses_xxh3());
}
#[test]
fn json_round_trip() {
let config = BucketEtagConfig::new("xxh3");
let json = serde_json::to_vec(&config).expect("serialize");
assert_eq!(json, br#"{"algorithm":"xxh3"}"#);
assert_eq!(serde_json::from_slice::<BucketEtagConfig>(&json).expect("parse"), config);
assert_eq!(
serde_json::from_slice::<BucketEtagConfig>(b"{}").expect("parse"),
BucketEtagConfig::default()
);
}
#[test]
fn registry_tracks_xxh3_buckets() {
let bucket = "etag-registry-test-bucket";
assert!(!bucket_uses_xxh3_etag(bucket));
set_bucket_xxh3_etag(bucket, true);
assert!(bucket_uses_xxh3_etag(bucket));
assert!(!bucket_uses_xxh3_etag("etag-registry-test-other"));
set_bucket_xxh3_etag(bucket, false);
assert!(!bucket_uses_xxh3_etag(bucket));
set_bucket_xxh3_etag("", true);
assert!(!bucket_uses_xxh3_etag(""));
}
}
@@ -1,76 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Lockless cache buckets.
//!
//! A bucket named in `RUSTFS_LOCKLESS_CACHE_BUCKETS` holds a content-addressed
//! cache (an LLM KV cache): a key always names the same bytes, and a lost or
//! unreadable object only costs a recompute. PutObject, GetObject and HeadObject
//! on such a bucket take no distributed locks and read no bucket metadata from
//! disk, so concurrent writers into one bucket no longer meet on bucket-wide
//! lock keys.
//!
//! What a listed bucket gives up, by design:
//! - Writes of one key are not serialized. Racing commits of the same bytes can
//! leave the object without read quorum until it is rewritten or healed,
//! which a cache treats as a miss.
//! - Reads are not fenced against a concurrent overwrite or delete.
//! - Writes are not fenced against bucket deletion, Object Lock, quota,
//! versioning, replication or table-catalog changes made while they run. A
//! PUT falls back to the locked path whenever the cached bucket state shows
//! any of those features, but listed buckets must not enable them.
//!
//! The list is read once per process; RustFS system buckets are never lockless.
use super::utils::is_meta_bucketname;
use std::collections::HashSet;
use std::sync::OnceLock;
/// Buckets named in a comma-separated list, without blanks or system buckets.
fn parse_lockless_cache_buckets(value: &str) -> HashSet<String> {
value
.split(',')
.map(str::trim)
.filter(|bucket| !bucket.is_empty() && !is_meta_bucketname(bucket))
.map(str::to_owned)
.collect()
}
/// Whether `bucket` is configured as a lockless cache bucket.
pub fn is_lockless_cache_bucket(bucket: &str) -> bool {
static BUCKETS: OnceLock<HashSet<String>> = OnceLock::new();
BUCKETS
.get_or_init(|| {
rustfs_utils::get_env_opt_str(rustfs_config::ENV_LOCKLESS_CACHE_BUCKETS)
.map(|value| parse_lockless_cache_buckets(&value))
.unwrap_or_default()
})
.contains(bucket)
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_keeps_named_user_buckets_only() {
let buckets = parse_lockless_cache_buckets(" kvcache-a,,kvcache-b , .rustfs.sys ,");
assert_eq!(
buckets,
HashSet::from(["kvcache-a".to_owned(), "kvcache-b".to_owned()]),
"blanks and system buckets must be dropped"
);
assert!(parse_lockless_cache_buckets("").is_empty());
}
}
+2 -64
View File
@@ -390,7 +390,6 @@ pub const BUCKET_PUBLIC_ACCESS_BLOCK_CONFIG: &str = "public-access-block.xml";
pub const BUCKET_ACL_CONFIG: &str = "bucket-acl.json";
pub const BUCKET_TABLE_CONFIG: &str = "table-bucket.json";
pub const BUCKET_DURABILITY_CONFIG: &str = "durability.json";
pub const BUCKET_ETAG_CONFIG: &str = "etag.json";
pub const BUCKET_ON_DEMAND_MIGRATION_CONFIG: &str = "on-demand-migration.json";
pub const BUCKET_TABLE_RESERVED_PREFIX: &str = ".rustfs-table";
pub const BUCKET_TABLE_CATALOG_META_PREFIX: &str = "s3tables/catalog";
@@ -582,7 +581,6 @@ pub struct BucketMetadata {
pub table_bucket_config_json: Vec<u8>,
pub durability_config_json: Vec<u8>,
pub on_demand_migration_config_json: Vec<u8>,
pub etag_config_json: Vec<u8>,
pub policy_config_updated_at: OffsetDateTime,
pub object_lock_config_updated_at: OffsetDateTime,
@@ -605,7 +603,6 @@ pub struct BucketMetadata {
pub table_bucket_config_updated_at: OffsetDateTime,
pub durability_config_updated_at: OffsetDateTime,
pub on_demand_migration_config_updated_at: OffsetDateTime,
pub etag_config_updated_at: OffsetDateTime,
pub new_field_updated_at: OffsetDateTime,
@@ -660,7 +657,6 @@ impl Default for BucketMetadata {
table_bucket_config_json: Default::default(),
durability_config_json: Default::default(),
on_demand_migration_config_json: Default::default(),
etag_config_json: Default::default(),
policy_config_updated_at: OffsetDateTime::UNIX_EPOCH,
object_lock_config_updated_at: OffsetDateTime::UNIX_EPOCH,
encryption_config_updated_at: OffsetDateTime::UNIX_EPOCH,
@@ -682,7 +678,6 @@ impl Default for BucketMetadata {
table_bucket_config_updated_at: OffsetDateTime::UNIX_EPOCH,
durability_config_updated_at: OffsetDateTime::UNIX_EPOCH,
on_demand_migration_config_updated_at: OffsetDateTime::UNIX_EPOCH,
etag_config_updated_at: OffsetDateTime::UNIX_EPOCH,
new_field_updated_at: OffsetDateTime::UNIX_EPOCH,
policy_config: Default::default(),
notification_config: Default::default(),
@@ -809,29 +804,6 @@ impl BucketMetadata {
}
}
/// Parsed per-bucket ETag algorithm, if one is stored. Invalid payloads
/// keep MD5 ETags after logging a parse failure.
pub fn etag_config(&self) -> Option<super::etag::BucketEtagConfig> {
if self.etag_config_json.is_empty() {
return None;
}
match serde_json::from_slice(&self.etag_config_json) {
Ok(cfg) => Some(cfg),
Err(e) => {
tracing::warn!(
event = "bucket_metadata_parse_failed",
component = "ecstore",
subsystem = "bucket_metadata",
bucket = %self.name,
config = "etag",
error = %e,
"Failed to parse bucket metadata config"
);
None
}
}
}
/// Decode from msgp bytes. Field order follows MinIO BucketMetadata for compatibility.
pub fn decode_from<R: Read>(&mut self, rd: &mut R) -> Result<()> {
let mut fields = rmp::decode::read_map_len(rd)?;
@@ -892,7 +864,6 @@ impl BucketMetadata {
"OnDemandMigrationConfigJSON" | "OnDemandMigrationConfigJson" => {
self.on_demand_migration_config_json = read_msgp_bin(rd)?
}
"EtagConfigJSON" | "EtagConfigJson" => self.etag_config_json = read_msgp_bin(rd)?,
"CorsConfigUpdatedAt" => self.cors_config_updated_at = read_msgp_time_value(rd)?,
"LoggingConfigUpdatedAt" => self.logging_config_updated_at = read_msgp_time_value(rd)?,
"WebsiteConfigUpdatedAt" => self.website_config_updated_at = read_msgp_time_value(rd)?,
@@ -903,7 +874,6 @@ impl BucketMetadata {
"TableBucketConfigUpdatedAt" => self.table_bucket_config_updated_at = read_msgp_time_value(rd)?,
"DurabilityConfigUpdatedAt" => self.durability_config_updated_at = read_msgp_time_value(rd)?,
"OnDemandMigrationConfigUpdatedAt" => self.on_demand_migration_config_updated_at = read_msgp_time_value(rd)?,
"EtagConfigUpdatedAt" => self.etag_config_updated_at = read_msgp_time_value(rd)?,
"BucketCreationCommitted" => {
self.bucket_creation_committed = read_msgp_bool(rd)?;
self.bucket_creation_commit_record_present = true;
@@ -922,8 +892,8 @@ impl BucketMetadata {
pub fn encode_to<W: Write>(&self, wr: &mut W) -> Result<()> {
// Preserve the absence of a legacy commit record across config rewrites.
let commit_record_present = self.bucket_creation_commit_record_present || self.bucket_creation_committed;
// Map size: MinIO fields (25) + RustFS extensions (23) + optional commit record.
let map_len: u32 = 48 + u32::from(commit_record_present);
// Map size: MinIO fields (25) + RustFS extensions (21) + optional commit record.
let map_len: u32 = 46 + u32::from(commit_record_present);
rmp::encode::write_map_len(wr, map_len)?;
// MinIO field order (same as Go struct)
@@ -984,7 +954,6 @@ impl BucketMetadata {
write_bin_field(wr, "TableBucketConfigJSON", &self.table_bucket_config_json)?;
write_bin_field(wr, "DurabilityConfigJSON", &self.durability_config_json)?;
write_bin_field(wr, "OnDemandMigrationConfigJSON", &self.on_demand_migration_config_json)?;
write_bin_field(wr, "EtagConfigJSON", &self.etag_config_json)?;
rmp::encode::write_str(wr, "CorsConfigUpdatedAt")?;
write_msgp_time(wr, self.cors_config_updated_at)?;
rmp::encode::write_str(wr, "LoggingConfigUpdatedAt")?;
@@ -1005,8 +974,6 @@ impl BucketMetadata {
write_msgp_time(wr, self.durability_config_updated_at)?;
rmp::encode::write_str(wr, "OnDemandMigrationConfigUpdatedAt")?;
write_msgp_time(wr, self.on_demand_migration_config_updated_at)?;
rmp::encode::write_str(wr, "EtagConfigUpdatedAt")?;
write_msgp_time(wr, self.etag_config_updated_at)?;
if commit_record_present {
rmp::encode::write_str(wr, "BucketCreationCommitted")?;
rmp::encode::write_bool(wr, self.bucket_creation_committed)?;
@@ -1115,9 +1082,6 @@ impl BucketMetadata {
if self.on_demand_migration_config_updated_at == OffsetDateTime::UNIX_EPOCH {
self.on_demand_migration_config_updated_at = self.created
}
if self.etag_config_updated_at == OffsetDateTime::UNIX_EPOCH {
self.etag_config_updated_at = self.created
}
}
/// Replace one config payload and stamp its `*_config_updated_at` with the
@@ -1264,10 +1228,6 @@ impl BucketMetadata {
self.on_demand_migration_config_json = data;
self.on_demand_migration_config_updated_at = updated;
}
BUCKET_ETAG_CONFIG => {
self.etag_config_json = data;
self.etag_config_updated_at = updated;
}
_ => return Err(Error::other(format!("config file not found : {config_file}"))),
}
@@ -2760,28 +2720,6 @@ mod test {
/// HP-5b (rustfs/backlog#938): the durability override is a RustFS
/// extension entry and must survive an encode/decode round trip.
#[test]
fn etag_config_round_trips_and_tracks_updates() {
let mut bm = BucketMetadata::new("etag-bucket");
assert!(bm.etag_config().is_none());
bm.update_config(BUCKET_ETAG_CONFIG, br#"{"algorithm":"xxh3"}"#.to_vec())
.expect("set etag config");
assert_ne!(bm.etag_config_updated_at, OffsetDateTime::UNIX_EPOCH);
assert!(bm.etag_config().expect("parsed").uses_xxh3());
let back = BucketMetadata::unmarshal(&bm.marshal_msg().expect("marshal")).expect("unmarshal");
assert_eq!(back.etag_config_json, bm.etag_config_json);
assert_eq!(back.etag_config_updated_at.unix_timestamp(), bm.etag_config_updated_at.unix_timestamp());
// The durability entry is independent of the ETag entry.
assert!(back.durability_config_json.is_empty());
bm.update_config(BUCKET_ETAG_CONFIG, Vec::new()).expect("clear etag config");
assert!(bm.etag_config().is_none());
bm.etag_config_json = b"not-json".to_vec();
assert!(bm.etag_config().is_none(), "garbage must keep MD5 ETags");
}
#[test]
fn durability_config_round_trips_and_tracks_updates() {
let mut bm = BucketMetadata::new("durability-bucket");
+8 -72
View File
@@ -440,25 +440,6 @@ fn clear_bucket_durability(bucket: &str) {
crate::disk::local::bucket_durability::set(bucket, None);
}
/// Publish the bucket's ETag algorithm to the registry the write path consults.
fn sync_bucket_etag(bucket: &str, bm: &BucketMetadata) {
let xxh3 = bm.etag_config().is_some_and(|cfg| cfg.uses_xxh3());
crate::bucket::etag::set_bucket_xxh3_etag(bucket, xxh3);
}
/// Publish every per-bucket override kept outside the metadata cache
/// (durability tier, ETag algorithm) on each install path.
fn sync_bucket_overrides(bucket: &str, bm: &BucketMetadata) {
sync_bucket_durability(bucket, bm);
sync_bucket_etag(bucket, bm);
}
/// Drop every per-bucket override when the bucket's metadata leaves the cache.
fn clear_bucket_overrides(bucket: &str) {
clear_bucket_durability(bucket);
crate::bucket::etag::set_bucket_xxh3_etag(bucket, false);
}
/// Publish application-owned bytes on every cache install path.
fn sync_on_demand_migration(bucket: &str, bm: &BucketMetadata) {
if let Some(hook) = BUCKET_CONFIG_PUBLISH_HOOK.get() {
@@ -1606,16 +1587,6 @@ pub async fn get_durability_config(
Ok((bm.durability_config(), bm.durability_config_updated_at))
}
/// The bucket's ETag algorithm config with its update time. `Ok((None, ..))`
/// means the bucket keeps MD5 ETags.
pub async fn get_etag_config(bucket: &str) -> Result<(Option<crate::bucket::etag::BucketEtagConfig>, OffsetDateTime)> {
let bucket_meta_sys_lock = get_bucket_metadata_sys()?;
let bucket_meta_sys = bucket_meta_sys_lock.read().await;
let (bm, _) = bucket_meta_sys.get_config(bucket).await?;
Ok((bm.etag_config(), bm.etag_config_updated_at))
}
/// The bucket's on-demand migration config with its update time, or
/// `Ok(None)` when the bucket has none. Bytes are opaque to the metadata owner.
pub async fn get_on_demand_migration_config(bucket: &str) -> Result<Option<(Vec<u8>, OffsetDateTime)>> {
@@ -2480,7 +2451,7 @@ impl BucketMetadataSys {
self.missing_buckets.insert(bucket.to_string(), ()).await;
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_bucket_durability(bucket);
clear_on_demand_migration(bucket);
}
}
@@ -2518,7 +2489,7 @@ impl BucketMetadataSys {
self.fabricated_metadata.write().await.remove(bucket);
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &bm).await;
sync_bucket_overrides(bucket, &bm);
sync_bucket_durability(bucket, &bm);
sync_on_demand_migration(bucket, &bm);
}
MetadataLoadMode::Initial => {
@@ -2566,7 +2537,7 @@ impl BucketMetadataSys {
self.missing_buckets.invalidate(bucket).await;
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_bucket_durability(bucket);
clear_on_demand_migration(bucket);
}
return Ok(());
@@ -2589,7 +2560,7 @@ impl BucketMetadataSys {
self.fabricated_metadata.write().await.remove(bucket);
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &metadata).await;
sync_bucket_overrides(bucket, &metadata);
sync_bucket_durability(bucket, &metadata);
sync_on_demand_migration(bucket, &metadata);
Ok(())
}
@@ -2619,7 +2590,7 @@ impl BucketMetadataSys {
self.fabricated_metadata.write().await.remove(&bucket);
self.missing_buckets.invalidate(&bucket).await;
sync_bucket_target_sys(&bucket, &bm).await;
sync_bucket_overrides(&bucket, &bm);
sync_bucket_durability(&bucket, &bm);
sync_on_demand_migration(&bucket, &bm);
}
}
@@ -2643,7 +2614,7 @@ impl BucketMetadataSys {
super::config_parse_mode::forget_bucket_config_parse_state(bucket);
if removed {
BucketTargetSys::get().delete(bucket).await;
clear_bucket_overrides(bucket);
clear_bucket_durability(bucket);
clear_on_demand_migration(bucket);
}
removed || removed_fabricated
@@ -2986,7 +2957,7 @@ impl BucketMetadataSys {
self.fabricated_metadata.write().await.remove(bucket);
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &bm).await;
sync_bucket_overrides(bucket, &bm);
sync_bucket_durability(bucket, &bm);
sync_on_demand_migration(bucket, &bm);
return Ok((bm, true));
} else {
@@ -3437,7 +3408,7 @@ impl BucketMetadataSys {
self.fabricated_metadata.write().await.remove(bucket);
self.missing_buckets.invalidate(bucket).await;
sync_bucket_target_sys(bucket, &metadata).await;
sync_bucket_overrides(bucket, &metadata);
sync_bucket_durability(bucket, &metadata);
sync_on_demand_migration(bucket, &metadata);
Ok(BucketMetadataAuthority::Authoritative(metadata))
}
@@ -6715,41 +6686,6 @@ mod tests {
target_sys.delete(bucket).await;
}
/// Installing bucket metadata publishes the bucket's ETag algorithm to the
/// write-path registry; metadata without it, an invalid payload or
/// clearing the bucket's overrides withdraws XXH3.
#[test]
fn metadata_sync_publishes_and_clears_etag_algorithm() {
use crate::bucket::etag::bucket_uses_xxh3_etag;
let bucket = "metadata-sync-etag";
let mut bm = BucketMetadata::new(bucket);
bm.etag_config_json = br#"{"algorithm":"xxh3"}"#.to_vec();
sync_bucket_overrides(bucket, &bm);
assert!(bucket_uses_xxh3_etag(bucket));
sync_bucket_overrides(bucket, &BucketMetadata::new(bucket));
assert!(!bucket_uses_xxh3_etag(bucket));
let mut bm = BucketMetadata::new(bucket);
bm.etag_config_json = br#"{"algorithm":"md5"}"#.to_vec();
sync_bucket_overrides(bucket, &bm);
assert!(!bucket_uses_xxh3_etag(bucket));
let mut bm = BucketMetadata::new(bucket);
bm.etag_config_json = b"not-json".to_vec();
sync_bucket_overrides(bucket, &bm);
assert!(!bucket_uses_xxh3_etag(bucket));
let mut bm = BucketMetadata::new(bucket);
bm.etag_config_json = br#"{"algorithm":"XXH3"}"#.to_vec();
sync_bucket_overrides(bucket, &bm);
assert!(bucket_uses_xxh3_etag(bucket));
clear_bucket_overrides(bucket);
assert!(!bucket_uses_xxh3_etag(bucket));
}
/// HP-5b (rustfs/backlog#938): installing bucket metadata publishes the
/// durability override to the disk-layer registry, and clearing the
/// config (or an invalid payload) withdraws it.
+1 -3
View File
@@ -579,10 +579,8 @@ mod tests {
] {
let first: Error = make_error("buckets/first/.metadata.bin".into()).into();
let second: Error = make_error("buckets/second/.metadata.bin".into()).into();
let cloned = first.clone();
assert_eq!(first, second, "record paths must not fragment error grouping");
assert_eq!(cloned, second, "cloning must preserve error grouping");
assert_eq!(std::io::Error::from(cloned).to_string(), message);
assert_eq!(first.clone(), second, "cloning must preserve error grouping");
let io_error = std::io::Error::from(first);
let detail = io_error
-2
View File
@@ -19,9 +19,7 @@ pub mod bucket_target_sys;
pub mod config_parse_mode;
pub mod durability;
pub mod error;
pub mod etag;
pub mod lifecycle;
pub mod lockless_cache;
pub mod metadata;
pub mod metadata_sys;
#[cfg(test)]
+6 -11
View File
@@ -826,17 +826,12 @@ pub(crate) async fn begin(
) -> Result<QuotaContext> {
let snapshot_admission = opts.quota_admission;
let data_movement = opts.data_movement;
// A lockless cache PUT is only taken when the bucket has no quota, so it
// reserves nothing and needs no metadata-transaction lock.
let lockless_cache = opts.is_lockless_cache_write() && snapshot_admission.is_none();
if lockless_cache || crate::bucket::utils::is_meta_bucketname(bucket) {
if !lockless_cache {
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
meta_bucket_fast_path_started,
);
}
if crate::bucket::utils::is_meta_bucketname(bucket) {
let meta_bucket_fast_path_started = rustfs_io_metrics::put_stage_timer();
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_PUT_OBJECT_QUOTA_BEGIN_META_BUCKET_FAST_PATH,
meta_bucket_fast_path_started,
);
return Ok(QuotaContext {
store: None,
bucket: bucket.to_string(),
@@ -66,9 +66,9 @@ use super::replication_target_boundary::{
VersionIdentityCapability, is_replication_target_offline_error, replication_action_for_target_head,
replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options,
replication_force_delete_remove_options, replication_object_is_ssec_encrypted, replication_put_object_header_size,
replication_put_object_options_for_target, replication_source_etag_is_comparable,
replication_target_head_is_newer_null_version, resolve_read_api_version_id, ssec_passthrough_evidence_present,
ssec_passthrough_gate, version_identity_capability_from_put, version_identity_drifted,
replication_put_object_options, replication_source_etag_is_comparable, replication_target_head_is_newer_null_version,
resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_capability_from_put,
version_identity_drifted,
};
use super::replication_versioning_boundary::ReplicationVersioningStore;
use super::runtime_boundary as runtime_sources;
@@ -4813,7 +4813,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
rinfo.size = size;
rinfo.replication_action = replication_action;
let (put_opts, is_multipart) = match replication_put_object_options_for_target(&tgt_client, &object_info) {
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
Ok((put_opts, is_mp)) => (put_opts, is_mp),
Err(e) => {
// Unsupported source metadata (e.g. managed SSE) is a fail-closed
@@ -5046,7 +5046,7 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
// The target client has no metadata-only operation. Reuse the existing
// object transport so metadata changes carry tags and object-lock state
// atomically with the source version.
let (put_opts, is_multipart) = match replication_put_object_options_for_target(&tgt_client, &object_info) {
let (put_opts, is_multipart) = match replication_put_object_options(&tgt_client.storage_class, &object_info) {
Ok((put_opts, is_mp)) => (put_opts, is_mp),
Err(e) => {
fail_replicate_all_put_options(&mut rinfo, &tgt_client, &bucket, object_info, &e, start_time);
@@ -6577,7 +6577,7 @@ mod tests {
assert_eq!(persisted.targets_map["arn:same"].resync_status, ResyncStatusType::ResyncCompleted);
}
use super::super::replication_target_boundary::{BucketTarget, BucketTargets, replication_put_object_options};
use super::super::replication_target_boundary::{BucketTarget, BucketTargets};
use super::*;
use rustfs_gateway_types::dto::Status;
use rustfs_gateway_types::persistence::PersistedVersioningConfiguration as VersioningConfiguration;
@@ -6588,39 +6588,6 @@ mod tests {
use time::OffsetDateTime;
use uuid::Uuid;
/// Only a site-replication peer is told the source object's KMS key
/// name; an operator-configured target gets the `aws:kms` intent alone.
#[test]
fn replica_write_names_the_kms_key_for_site_replication_peers_only() {
use rustfs_utils::http::{AMZ_SERVER_SIDE_ENCRYPTION, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID};
let object_info = ObjectInfo {
user_defined: HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "finance-key".to_string()),
])
.into(),
..Default::default()
};
let mut operator_target = test_target_client("http://127.0.0.1:1".to_string());
let (options, _) =
replication_put_object_options_for_target(&operator_target, &object_info).expect("put options must build");
assert_eq!(options.user_metadata.get(AMZ_SERVER_SIDE_ENCRYPTION), Some(&"aws:kms".to_string()));
assert!(!options.user_metadata.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID));
Arc::get_mut(&mut operator_target)
.expect("test target is uniquely owned")
.site_peer_deployment_id = "peer-deployment".to_string();
let site_peer = operator_target;
let (options, _) = replication_put_object_options_for_target(&site_peer, &object_info).expect("put options must build");
assert_eq!(options.user_metadata.get(AMZ_SERVER_SIDE_ENCRYPTION), Some(&"aws:kms".to_string()));
assert_eq!(
options.user_metadata.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID),
Some(&"finance-key".to_string())
);
}
fn test_target_client(endpoint: String) -> Arc<TargetClient> {
let config = aws_sdk_s3::Config::builder()
.endpoint_url(endpoint.clone())
@@ -6639,7 +6606,6 @@ mod tests {
disable_proxy: false,
arn: format!("arn:rustfs:replication:us-east-1:target:{}", Uuid::new_v4()),
reset_id: String::new(),
site_peer_deployment_id: String::new(),
secure: false,
health_check_duration: std::time::Duration::from_secs(5),
replicate_sync: false,
@@ -225,50 +225,6 @@ impl ReplicationTargetStore {
}
}
/// Name the source object's KMS key on a replica write to a site-replication
/// peer, so the peer seals the replica under its key of the same name.
///
/// Sites share bucket default-encryption key names and key-scoped IAM
/// policies verbatim and are required to hold every key under the same name.
/// With only the `aws:kms` intent the peer falls back to its bucket or site
/// default, and an object written under an explicitly named key ends up
/// sealed under a different key on each site: a caller scoped to the named
/// key can then read the object on one site and not on the other. Only the
/// name travels; the envelope and the encryption context stay on the source.
///
/// Must not be applied to an operator-configured target, whose KMS has no
/// reason to hold a key of that name.
fn name_source_kms_key_for_site_peer(options: &mut PutObjectOptions, object_info: &ObjectInfo) {
if !matches!(
classify_replication_source_encryption(&object_info.user_defined),
ReplicationSourceEncryption::SseKms
) {
return;
}
if let Some(key_id) = metadata_value(&object_info.user_defined, AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID)
.map(str::trim)
.filter(|key_id| !key_id.is_empty())
{
options
.user_metadata
.insert(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), key_id.to_string());
}
}
/// Put options for replicating `object_info` to `target`: the transport
/// options every target gets, plus the source KMS key name when the target is
/// a site-replication peer.
pub(crate) fn replication_put_object_options_for_target(
target: &TargetClient,
object_info: &ObjectInfo,
) -> Result<(PutObjectOptions, bool)> {
let (mut options, is_multipart) = replication_put_object_options(&target.storage_class, object_info)?;
if target.is_site_replication_peer() {
name_source_kms_key_for_site_peer(&mut options, object_info);
}
Ok((options, is_multipart))
}
pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo) -> Result<(PutObjectOptions, bool)> {
use base64_simd::STANDARD as BASE64_STANDARD;
use rustfs_utils::http::{AMZ_CHECKSUM_TYPE, AMZ_CHECKSUM_TYPE_FULL_OBJECT};
@@ -325,9 +281,8 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
// Managed SSE replicates as plaintext (the replication reader decrypts via
// the object-encryption resolver) and re-encrypts on the target with the
// target's own KMS. Send only the encryption intent: a key id means
// nothing to an arbitrary target's KMS. Site-replication peers are the
// exception and get the key name from `name_source_kms_key_for_site_peer`.
// target's own KMS. Send only the encryption intent — never the source
// key id, whose meaning is local to the source site's KMS.
if matches!(source_encryption, ReplicationSourceEncryption::SseS3) {
meta.insert(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string());
} else if matches!(source_encryption, ReplicationSourceEncryption::SseKms) {
@@ -1635,80 +1590,6 @@ mod tests {
);
}
#[test]
fn site_peer_replica_write_names_the_source_kms_key_only() {
use rustfs_utils::http::object_encryption_keys::{
INTERNAL_ENCRYPTION_KEY_HEADER, MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER,
};
let object_info = ObjectInfo {
user_defined: Arc::new(HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), "finance-key".to_string()),
(INTERNAL_ENCRYPTION_KEY_HEADER.to_string(), "sealed-envelope".to_string()),
(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER.to_string(), "ctx".to_string()),
])),
..Default::default()
};
let (mut options, _) = replication_put_object_options("", &object_info).expect("managed SSE-KMS must build put options");
name_source_kms_key_for_site_peer(&mut options, &object_info);
assert_eq!(options.user_metadata.get(AMZ_SERVER_SIDE_ENCRYPTION), Some(&"aws:kms".to_string()));
assert_eq!(
options.user_metadata.get(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID),
Some(&"finance-key".to_string())
);
// The name is all that travels: no envelope, no encryption context.
assert!(!options.user_metadata.contains_key(INTERNAL_ENCRYPTION_KEY_HEADER));
assert!(
!options
.user_metadata
.contains_key(MINIO_INTERNAL_ENCRYPTION_KMS_CONTEXT_HEADER)
);
assert!(!options.user_metadata.values().any(|value| value.contains("sealed-envelope")));
}
#[test]
fn site_peer_replica_write_names_no_key_for_other_encryption() {
use rustfs_utils::http::object_encryption_keys::INTERNAL_ENCRYPTION_KEY_ID_HEADER;
for (label, user_defined) in [
("plaintext", HashMap::from([("x-user-meta".to_string(), "value".to_string())])),
(
// SSE-S3 wraps its data key under the site default key; that
// key is internal and is not a name the peer was asked to use.
"sse-s3",
HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "AES256".to_string()),
(INTERNAL_ENCRYPTION_KEY_ID_HEADER.to_string(), "site-default".to_string()),
]),
),
(
"sse-kms without a recorded key id",
HashMap::from([(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string())]),
),
(
"sse-kms with a blank key id",
HashMap::from([
(AMZ_SERVER_SIDE_ENCRYPTION.to_string(), "aws:kms".to_string()),
(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID.to_string(), " ".to_string()),
]),
),
] {
let object_info = ObjectInfo {
user_defined: Arc::new(user_defined),
..Default::default()
};
let (mut options, _) = replication_put_object_options("", &object_info).expect("put options must build");
name_source_kms_key_for_site_peer(&mut options, &object_info);
assert!(
!options.user_metadata.contains_key(AMZ_SERVER_SIDE_ENCRYPTION_KMS_ID),
"{label}: no key id may be sent"
);
}
}
#[test]
fn replication_put_options_rejects_unknown_encryption_without_echoing_metadata() {
let secret_like_value = "opaque-context-that-must-not-be-logged";
@@ -1399,73 +1399,6 @@ pub fn build_auth_headers(url: &str, method: &Method, headers: &mut HeaderMap) -
Ok(())
}
/// Signs only external stream controls targeting members of this instance's topology.
#[cfg(feature = "internode-transport")]
#[derive(Clone)]
pub struct ExternalStreamAuth {
peers: std::sync::Arc<std::collections::HashSet<url::Origin>>,
}
#[cfg(feature = "internode-transport")]
impl ExternalStreamAuth {
/// The authenticated HTTP route used by peer stream controls.
pub const PATH: &'static str = "/rustfs/rpc/external_stream";
/// Capture the topology already configured for this host; a missing topology has no peers.
pub fn new(instance: &crate::runtime::instance::InstanceContext) -> Self {
let mut peers = std::collections::HashSet::new();
if let Some(topology) = instance.endpoints() {
for pool in topology.0 {
for endpoint in pool.endpoints.as_ref() {
if matches!(endpoint.url.scheme(), "http" | "https") {
peers.insert(endpoint.url.origin());
}
}
}
}
Self {
peers: std::sync::Arc::new(peers),
}
}
/// Sign a fresh POST for one configured origin, request body digest and replay nonce.
/// The caller must send it to the returned URL without redirects or resubmission.
pub fn sign_control(&self, endpoint: &str, digest: &[u8; 32]) -> std::io::Result<(String, HeaderMap)> {
let mut url = url::Url::parse(endpoint).map_err(|error| std::io::Error::new(std::io::ErrorKind::InvalidInput, error))?;
if !url.username().is_empty()
|| url.password().is_some()
|| !self.peers.contains(&url.origin())
|| url.path() != "/"
|| url.query().is_some()
|| url.fragment().is_some()
{
return Err(std::io::Error::new(
std::io::ErrorKind::PermissionDenied,
"invalid internode control endpoint",
));
}
url.set_path(Self::PATH);
url.query_pairs_mut()
.append_pair("body_sha256", &hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower))
.append_pair("request_nonce", &Uuid::new_v4().to_string());
let url = url.to_string();
let mut headers = HeaderMap::new();
headers.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/octet-stream"));
build_auth_headers(&url, &Method::POST, &mut headers)?;
Ok((url, headers))
}
/// Derive the operation key for this stream protocol and exact request digest.
/// The shared RPC signing credential is never exposed to the stream provider.
pub fn transfer_key(&self, request_digest: &[u8; 32]) -> std::io::Result<[u8; 32]> {
let secret = get_shared_secret()?;
let mut mac = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(std::io::Error::other)?;
mac.update(b"rustfs-external-stream-key-v1\0");
mac.update(request_digest);
Ok(mac.finalize().into_bytes().into())
}
}
pub fn gen_signature_headers(url: &str, method: &Method) -> std::io::Result<HeaderMap> {
let secret = get_shared_secret()?;
let timestamp = OffsetDateTime::now_utc().unix_timestamp();
@@ -220,11 +220,6 @@ impl InternodeDataTransportCapabilities {
}
#[derive(Debug, Clone)]
#[cfg_attr(
feature = "internode-transport",
derive(serde::Serialize, serde::Deserialize),
serde(deny_unknown_fields)
)]
pub struct ReadStreamRequest {
pub endpoint: String,
pub disk: String,
@@ -236,11 +231,6 @@ pub struct ReadStreamRequest {
}
#[derive(Debug, Clone)]
#[cfg_attr(
feature = "internode-transport",
derive(serde::Serialize, serde::Deserialize),
serde(deny_unknown_fields)
)]
pub struct WriteStreamRequest {
pub endpoint: String,
pub disk: String,
@@ -1253,38 +1243,6 @@ mod tests {
}
}
#[cfg(feature = "internode-transport")]
#[test]
fn peer_transport_is_selected_once_per_instance() {
use crate::runtime::instance::InstanceContext;
let first = InstanceContext::new();
let second = InstanceContext::new();
let provider: Arc<dyn InternodeDataTransport> = Arc::new(LegacyTestTransport);
first
.select_internode_data_transport(provider.clone(), Some(" LEGACY-TEST "))
.expect("select external transport");
let selected = first.internode_data_transport().expect("first instance selection");
assert!(Arc::ptr_eq(selected, &provider));
assert!(second.internode_data_transport().is_none());
assert_eq!(
first
.select_internode_data_transport(provider.clone(), Some("tcp"))
.expect_err("already configured")
.kind(),
io::ErrorKind::AlreadyExists
);
second
.select_internode_data_transport(provider.clone(), None)
.expect("ordinary TCP");
assert_eq!(
second.internode_data_transport().expect("ordinary selection").name(),
DEFAULT_INTERNODE_DATA_TRANSPORT
);
let invalid = InstanceContext::new();
assert!(invalid.select_internode_data_transport(provider, Some("unknown")).is_err());
assert!(invalid.internode_data_transport().is_none());
}
#[tokio::test]
async fn legacy_transport_defaults_namespace_scanner_to_unsupported() {
let transport = LegacyTestTransport;
+19 -117
View File
@@ -54,10 +54,6 @@ use bytes::Bytes;
use futures::lock::Mutex;
use metrics::counter;
use rustfs_filemeta::{FileInfo, ObjectPartInfo, RawFileInfo};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_OPERATION_GRPC_WRITE_ALL, measure_client_rpc,
};
use rustfs_io_metrics::internode_metrics::{
INTERNODE_STAGE_BATCH_READ_VERSION_REQUEST_ENCODE, INTERNODE_STAGE_BATCH_READ_VERSION_RESPONSE_DECODE,
INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, INTERNODE_STAGE_READ_VERSION_REQUEST_ENCODE,
@@ -1001,8 +997,6 @@ pub struct RemoteDisk {
#[cfg(test)]
recovery_monitor_teardown_hook: Arc<tokio::sync::Mutex<Option<Arc<RecoveryMonitorTeardownHook>>>>,
data_transport: Arc<dyn InternodeDataTransport>,
#[cfg(feature = "internode-transport")]
external_streams: bool,
/// Bound the per-operation failure warnings of this peer. An unreachable
/// peer fails every in-flight shard operation; the counters keep the exact
/// numbers per operation. Per handle, so one peer cannot hide another's.
@@ -1444,8 +1438,6 @@ impl RemoteDisk {
#[cfg(test)]
recovery_monitor_teardown_hook: Arc::new(tokio::sync::Mutex::new(None)),
data_transport,
#[cfg(feature = "internode-transport")]
external_streams: false,
timeout_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
network_error_logs: rustfs_utils::LogThrottle::new(REMOTE_DISK_RPC_FAILURE_LOG_INTERVAL_MS),
};
@@ -1460,14 +1452,6 @@ impl RemoteDisk {
Ok(disk)
}
#[cfg(feature = "internode-transport")]
pub(crate) fn with_instance_transport(mut self) -> Self {
// Built-in names are reserved by the instance registration API. A host provider
// owns its open, data verification and cancellation as one operation.
self.external_streams = !rustfs_config::KNOWN_INTERNODE_DATA_TRANSPORT_BACKENDS.contains(&self.data_transport.name());
self
}
pub fn runtime_state(&self) -> RuntimeDriveHealthState {
self.health.runtime_state()
}
@@ -1491,10 +1475,6 @@ impl RemoteDisk {
}
async fn open_write_with_retry(&self, request: WriteStreamRequest) -> Result<FileWriter> {
#[cfg(feature = "internode-transport")]
if self.external_streams {
return self.data_transport.open_write(request).await;
}
let mut attempt = 1;
let mut last_retry_classification = None;
loop {
@@ -3741,17 +3721,13 @@ impl DiskAPI for RemoteDisk {
}
Err(err) => {
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(
crate::cluster::rpc::runtime_sources::ReadVersionErrorStage::RpcStatus,
);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
return Err(err.into());
}
};
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(
crate::cluster::rpc::runtime_sources::ReadVersionErrorStage::RemoteResponse,
);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
return Err(response.error.unwrap_or_default().into());
}
@@ -3770,9 +3746,7 @@ impl DiskAPI for RemoteDisk {
}
Err(err) => {
record_read_version_stage(INTERNODE_STAGE_READ_VERSION_RESPONSE_DECODE, decode_started);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error(
crate::cluster::rpc::runtime_sources::ReadVersionErrorStage::ResponseDecode,
);
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_version_error();
return Err(err);
}
};
@@ -3834,8 +3808,12 @@ impl DiskAPI for RemoteDisk {
);
}
let rpc_started = read_version_stage_timer(batch_read_version_attribution_enabled);
let response_result =
measure_client_rpc(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, client.batch_read_version(request)).await;
let response_result = client.batch_read_version(request).await;
if let Some(rpc_started) = rpc_started {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_batch_read_version_duration(
rpc_started.elapsed(),
);
}
let response = match response_result {
Ok(response) => {
record_batch_read_version_stage(INTERNODE_STAGE_BATCH_READ_VERSION_RPC_ROUNDTRIP, rpc_started);
@@ -4228,10 +4206,6 @@ impl DiskAPI for RemoteDisk {
length,
stall_timeout: timeouts.body_stall,
};
#[cfg(feature = "internode-transport")]
if self.external_streams {
return self.data_transport.open_read(request).await;
}
let reader = self.open_read_with_retry(request.clone()).await?;
Ok(Box::new(RetryingRemoteReader::new_with_timeouts(
reader,
@@ -4263,10 +4237,6 @@ impl DiskAPI for RemoteDisk {
length,
stall_timeout: timeouts.body_stall,
};
#[cfg(feature = "internode-transport")]
if self.external_streams {
return self.data_transport.open_read_chunks(request).await;
}
let reader = self.open_read_chunks_with_retry(request.clone()).await?;
Ok(reader.map(|reader| {
Box::new(RetryingRemoteChunkReader::new_with_timeouts(
@@ -4861,7 +4831,9 @@ impl DiskAPI for RemoteDisk {
read_multiple_req_bin: read_multiple_req_bin.into(),
});
let response = measure_client_rpc(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, client.read_multiple(request)).await;
let rpc_started = Instant::now();
let response = client.read_multiple(request).await;
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_multiple_duration(rpc_started.elapsed());
let response = response?.into_inner();
if !response.success {
@@ -4913,7 +4885,8 @@ impl DiskAPI for RemoteDisk {
attach_mutation_body_digest(&mut request, canonical_body, "write_all")?;
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_request();
let response = match measure_client_rpc(INTERNODE_OPERATION_GRPC_WRITE_ALL, client.write_all(request)).await {
let rpc_started = Instant::now();
let response = match client.write_all(request).await {
Ok(response) => response.into_inner(),
Err(err) => {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_error();
@@ -4921,6 +4894,7 @@ impl DiskAPI for RemoteDisk {
}
};
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_duration(rpc_started.elapsed());
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_write_all_sent_bytes(data_len);
if !response.success {
@@ -5018,7 +4992,8 @@ impl DiskAPI for RemoteDisk {
});
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_request();
let response = match measure_client_rpc(INTERNODE_OPERATION_GRPC_READ_ALL, client.read_all(request)).await {
let rpc_started = Instant::now();
let response = match client.read_all(request).await {
Ok(response) => response.into_inner(),
Err(err) => {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
@@ -5026,6 +5001,8 @@ impl DiskAPI for RemoteDisk {
}
};
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_duration(rpc_started.elapsed());
if !response.success {
crate::cluster::rpc::runtime_sources::record_remote_disk_grpc_read_all_error();
return Err(response.error.unwrap_or_default().into());
@@ -10417,81 +10394,6 @@ mod tests {
}
}
#[cfg(feature = "internode-transport")]
#[tokio::test]
async fn peer_transport_open_returns_the_original_error_after_one_attempt() {
let failure = || {
OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
rustfs_rio::InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::SERVICE_UNAVAILABLE),
)))
};
let writer = RetryingOpenWriteInternodeDataTransport::with_steps(vec![failure(), OpenWriteTestStep::Success]);
let disk = new_remote_disk_with_transport(Arc::new(writer.clone()))
.await
.with_instance_transport();
let error = disk
.create_file("", "bucket", "shard", 3)
.await
.err()
.expect("provider write failure");
assert_eq!(
error.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::SERVICE_UNAVAILABLE))
);
assert_eq!(writer.calls().len(), 1);
let reader = RetryingOpenReadInternodeDataTransport::with_steps(vec![failure(), OpenWriteTestStep::Success]);
let disk = new_remote_disk_with_transport(Arc::new(reader.clone()))
.await
.with_instance_transport();
let error = disk
.read_file_stream("bucket", "shard", 0, 3)
.await
.err()
.expect("provider read failure");
assert_eq!(
error.internode_http_error_kind(),
Some(rustfs_rio::InternodeHttpErrorKind::HttpStatus(reqwest::StatusCode::SERVICE_UNAVAILABLE))
);
assert_eq!(reader.calls().len(), 1);
}
#[cfg(feature = "internode-transport")]
#[tokio::test]
async fn peer_transport_body_failures_never_open_a_fresh_stream() {
let provider = Arc::new(ResumeTransport::with_read_steps(vec![ResumeReadStep::PartialThenReset(b"x".to_vec())]));
let disk = new_remote_disk_with_transport(provider.clone())
.await
.with_instance_transport();
let mut reader = disk.read_file_stream("bucket", "shard", 0, 3).await.expect("reader");
let mut bytes = Vec::new();
let error = tokio::io::AsyncReadExt::read_to_end(&mut reader, &mut bytes)
.await
.expect_err("stream reset");
assert_eq!(error.kind(), std_io::ErrorKind::ConnectionReset);
assert_eq!(bytes, b"x");
assert_eq!(provider.read_requests.lock().expect("recorded transport calls").len(), 1);
assert!(provider.fresh_read_requests.lock().expect("recorded fresh calls").is_empty());
let provider = Arc::new(ResumeTransport::with_chunk_steps(vec![ResumeReadStep::PartialThenReset(b"x".to_vec())]));
let disk = new_remote_disk_with_transport(provider.clone())
.await
.with_instance_transport();
let mut reader = disk
.read_file_stream_chunks("bucket", "shard", 0, 3)
.await
.expect("chunked stream")
.expect("supported");
let chunk = std::future::poll_fn(|cx| Pin::new(reader.as_mut()).poll_read_chunk(cx, 16))
.await
.expect("first chunk");
assert_eq!(chunk, Some(Bytes::from_static(b"x")));
let error = std::future::poll_fn(|cx| Pin::new(reader.as_mut()).poll_read_chunk(cx, 16))
.await
.expect_err("chunk stream reset");
assert_eq!(error.kind(), std_io::ErrorKind::ConnectionReset);
assert_eq!(provider.chunk_requests.lock().expect("recorded transport calls").len(), 1);
assert!(provider.fresh_chunk_requests.lock().expect("recorded fresh calls").is_empty());
}
#[tokio::test]
#[serial(internode_metrics)]
async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
@@ -481,6 +481,14 @@ pub(crate) fn record_remote_disk_grpc_write_all_request() {
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_write_all_duration(duration: Duration) {
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_write_all_sent_bytes(bytes: usize) {
global_internode_metrics().record_sent_bytes_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_WRITE_ALL,
@@ -521,6 +529,14 @@ pub(crate) fn record_remote_disk_grpc_read_all_request() {
.record_outgoing_request_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_read_all_duration(duration: Duration) {
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
#[cfg(feature = "diagnostic-read-version-caller")]
fn read_version_request_mode(opts: &ReadOptions) -> ReadVersionRequestMode {
match (opts.read_data, opts.healing, opts.incl_free_versions) {
@@ -598,6 +614,17 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_stage(stage: &'static s
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_duration(duration: Duration) {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
pub(crate) fn record_remote_disk_grpc_batch_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
@@ -629,34 +656,12 @@ pub(crate) fn record_remote_disk_grpc_batch_read_version_recv_bytes(bytes: usize
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_BATCH_READ_VERSION, bytes);
}
#[derive(Clone, Copy)]
pub(crate) enum ReadVersionErrorStage {
RpcStatus,
RemoteResponse,
ResponseDecode,
}
impl ReadVersionErrorStage {
fn classification(self) -> &'static str {
match self {
Self::RpcStatus => "grpc_status",
Self::RemoteResponse => "remote_response",
Self::ResponseDecode => "response_decode",
}
}
}
pub(crate) fn record_remote_disk_grpc_read_version_error(stage: ReadVersionErrorStage) {
pub(crate) fn record_remote_disk_grpc_read_version_error() {
if !rustfs_io_metrics::get_stage_metrics_enabled() {
return;
}
let metrics = global_internode_metrics();
metrics.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
metrics.record_classified_error_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_VERSION,
INTERNODE_TRANSPORT_BACKEND_GRPC,
stage.classification(),
);
global_internode_metrics()
.record_error_for_operation_and_backend(INTERNODE_OPERATION_GRPC_READ_VERSION, INTERNODE_TRANSPORT_BACKEND_GRPC);
}
pub(crate) fn record_remote_disk_grpc_read_version_sent_bytes(bytes: usize) {
@@ -712,6 +717,14 @@ pub(crate) fn record_remote_disk_grpc_read_multiple_recv_bytes(bytes: usize) {
record_grpc_payload_size(INTERNODE_OPERATION_GRPC_READ_MULTIPLE, bytes);
}
pub(crate) fn record_remote_disk_grpc_read_multiple_duration(duration: Duration) {
global_internode_metrics().record_duration_for_operation_and_backend(
INTERNODE_OPERATION_GRPC_READ_MULTIPLE,
INTERNODE_TRANSPORT_BACKEND_GRPC,
duration,
);
}
/// Payload-size threshold (bytes) above which a unary internode gRPC response is counted as a
/// "large payload" for alerting. Env-overridable via `RUSTFS_INTERNODE_RPC_LARGE_PAYLOAD_WARN_BYTES`.
fn internode_rpc_large_payload_warn_bytes() -> usize {
@@ -791,53 +804,6 @@ mod tests {
#[cfg(feature = "diagnostic-read-version-caller")]
use super::*;
#[test]
fn read_version_errors_preserve_totals_and_classify_failure_boundaries() {
use super::{ReadVersionErrorStage, record_remote_disk_grpc_read_version_error};
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
rustfs_io_metrics::set_get_stage_metrics_enabled(true);
metrics::with_local_recorder(&recorder, || {
for stage in [
ReadVersionErrorStage::RpcStatus,
ReadVersionErrorStage::RemoteResponse,
ReadVersionErrorStage::ResponseDecode,
] {
record_remote_disk_grpc_read_version_error(stage);
}
});
rustfs_io_metrics::set_get_stage_metrics_enabled(false);
let entries = snapshotter.snapshot().into_vec();
let total = entries
.iter()
.find(|entry| {
entry.0.key().name() == "rustfs_system_network_internode_operation_errors_total"
&& entry
.0
.key()
.labels()
.any(|label| label.key() == "operation" && label.value() == "grpc_read_version")
})
.expect("existing operation error counter");
assert!(matches!(total.3, DebugValue::Counter(3)));
for classification in ["grpc_status", "remote_response", "response_decode"] {
let count = entries
.iter()
.find(|entry| {
entry.0.key().name() == "rustfs_system_network_internode_operation_classified_errors_total"
&& entry
.0
.key()
.labels()
.any(|label| label.key() == "classification" && label.value() == classification)
})
.expect("bounded failure classification");
assert!(matches!(count.3, DebugValue::Counter(1)));
}
}
#[cfg(feature = "diagnostic-read-version-caller")]
fn diagnostic_operation(operation: rustfs_s3_ops::S3Operation) -> S3HttpOperationContext {
let mut request = rustfs_io_metrics::s3_http_metrics::S3HttpRequestGuard::new("GET");
+4 -4
View File
@@ -20945,12 +20945,12 @@ impl ECStore {
let locator = parse_decommission_durable_ilm_receipt_path(&receipt_path)?;
let mut proof = None::<DecommissionDurableIlmReceipt>;
let mut nonterminal_receipt_found = false;
for (pool_idx, pool) in self.pools.iter().enumerate() {
for pool_idx in 0..self.pools.len() {
if pool_idx == source_pool_idx {
continue;
}
let data = match read_config_limited_preserve_empty(
pool.clone(),
self.pools[pool_idx].clone(),
&receipt_path,
DECOMMISSION_DURABLE_ILM_RECEIPT_MAX_SIZE,
)
@@ -28413,7 +28413,7 @@ mod tests {
for wrapped in [
err.clone(),
data_movement::data_movement_stage_error_for_test("decommission_object", "put_object", "b", "o", err.clone()),
with_decommission_entry_context("migrate_object", "b", "o", err),
with_decommission_entry_context("migrate_object", "b", "o", err.clone()),
] {
assert_eq!(
classify_decommission_capacity_failure(&wrapped),
@@ -40281,7 +40281,7 @@ mod pools_tests {
)
.expect("finalize records the completion");
let mut adopted = meta;
let mut adopted = meta.clone();
let plan = decommission_capacity_intent_adoption_plan(&adopted, 0, generation)
.expect("plan adoption")
.expect("the interrupted completion should be planned");
+6 -89
View File
@@ -14,8 +14,6 @@
// limitations under the License.
use crate::disk::error_reduce::count_errs;
#[cfg(any(test, feature = "test-util"))]
use crate::disk::new_disk;
use crate::error::{Error, Result, is_all_volume_not_found, is_err_object_not_found, is_err_strict_volume_not_found};
use crate::layout::set_heal::{formats_to_drives_info, new_heal_format_sets};
use crate::multipart_listing::paginate_multipart_listing;
@@ -30,6 +28,7 @@ use crate::{
DiskAPI, DiskOption, DiskStore,
error::DiskError,
format::{DistributionAlgoVersion, FormatV3},
new_disk,
},
error::StorageError,
layout::endpoints::{Endpoints, PoolEndpoints},
@@ -270,18 +269,15 @@ impl Sets {
if disk.as_ref().unwrap().is_local() && instance_ctx.is_dist_erasure().await {
let local_disk = runtime_sources::local_disk_set_drive(&instance_ctx, pool_idx, i, j).await;
let Some(local_disk) = local_disk else {
if local_disk.is_none() {
warn!("sets new set_drive {}-{} local_disk is none", i, j);
set_drive.push(None);
continue;
};
if let Some(previous) = disk.as_ref()
&& !Arc::ptr_eq(previous, &local_disk)
{
let _ = previous.close().await;
}
disk = Some(local_disk);
let _ = disk.as_ref().unwrap().close().await;
disk = local_disk;
}
let has_disk_id = disk.as_ref().unwrap().get_disk_id().await.unwrap_or_else(|err| {
@@ -1109,7 +1105,6 @@ impl Sets {
F: FnMut() -> bool + Send,
{
let (disks, init_errs) = init_storage_disks_with_errors(
&self.ctx,
&self.endpoints.endpoints,
&DiskOption {
cleanup: false,
@@ -1315,7 +1310,6 @@ async fn _close_storage_disks(disks: &[Option<DiskStore>]) {
}
async fn init_storage_disks_with_errors(
instance: &InstanceContext,
endpoints: &Endpoints,
opts: &DiskOption,
) -> (Vec<Option<DiskStore>>, Vec<Option<DiskError>>) {
@@ -1324,7 +1318,7 @@ async fn init_storage_disks_with_errors(
// let errs = Arc::new(RwLock::new(vec![None; endpoints.as_ref().len()]));
let mut futures = Vec::with_capacity(endpoints.as_ref().len());
for endpoint in endpoints.as_ref().iter() {
futures.push(crate::disk::new_disk_for_instance(instance, endpoint, opts));
futures.push(new_disk(endpoint, opts));
// let ep = endpoint.clone();
// let opt = opts.clone();
@@ -2391,83 +2385,6 @@ mod tests {
);
}
#[tokio::test]
async fn distributed_sets_preserve_the_admitted_disk_health_monitor() {
use crate::layout::endpoints::EndpointServerPools;
for reuse_registered in [true, false] {
let directory = tempfile::tempdir().expect("distributed set root");
let metadata = directory.path().join(crate::disk::RUSTFS_META_BUCKET);
tokio::fs::create_dir_all(&metadata).await.expect("metadata directory");
let mut format = FormatV3::new(1, 1);
format.erasure.this = format.erasure.sets[0][0];
tokio::fs::write(
metadata.join(crate::disk::FORMAT_CONFIG_FILE),
format.to_json().expect("disk format bytes"),
)
.await
.expect("persist distributed set format");
let mut endpoint =
Endpoint::try_from(directory.path().to_str().expect("UTF-8 disk root")).expect("distributed endpoint");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(0);
let pool = PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 1,
endpoints: vec![endpoint.clone()].into(),
cmd_line: "distributed-health-monitor".to_string(),
platform: "test".to_string(),
};
let context = Arc::new(InstanceContext::new());
context.update_erasure_type(SetupType::DistErasure).await;
let options = DiskOption {
cleanup: false,
health_check: true,
};
runtime_sources::initialize_local_disk_maps(&context, EndpointServerPools(vec![pool.clone()]), &options)
.await
.expect("register admitted disk");
let admitted = runtime_sources::local_disk_set_drive(&context, 0, 0, 0)
.await
.expect("registered physical handle");
let input = if reuse_registered {
admitted.clone()
} else {
new_disk(&endpoint, &options).await.expect("duplicate startup handle")
};
if !reuse_registered {
input.enable_health_check();
}
admitted.enable_health_check();
let sets = Sets::new_with_instance_ctx(vec![Some(input.clone())], &pool, &format, 0, 0, context)
.await
.expect("construct distributed set");
let active = sets.disk_set[0].disk_inventory().await[0]
.clone()
.expect("admitted runtime handle");
assert!(Arc::ptr_eq(&active, &admitted));
let crate::disk::Disk::Local(active_wrapper) = active.as_ref() else {
panic!("fixture must retain a local handle");
};
assert!(active_wrapper.health_check_enabled_for_test());
assert!(!active_wrapper.health_monitor_cancelled_for_test());
let crate::disk::Disk::Local(input_wrapper) = input.as_ref() else {
panic!("fixture input must be local");
};
assert_eq!(input_wrapper.health_monitor_cancelled_for_test(), !reuse_registered);
assert_eq!(active.get_disk_id().await.expect("active handle identity"), Some(format.erasure.this));
active
.disk_info(&crate::disk::DiskInfoOptions::default())
.await
.expect("admitted handle remains usable");
active.close().await.expect("stop test health monitor");
}
}
// The legacy constructor keeps single-instance behavior byte-for-byte: it
// still adopts the process bootstrap context.
#[tokio::test]
+1 -1
View File
@@ -4242,7 +4242,7 @@ mod tests {
"compatible aliases for {suffix} should match"
);
let mut source_without_marker = source;
let mut source_without_marker = source.clone();
rustfs_utils::http::remove_str(Arc::make_mut(&mut source_without_marker.user_defined), suffix);
let mut target_with_generated_marker = source_without_marker.clone();
rustfs_utils::http::insert_str(
+2 -2
View File
@@ -1512,7 +1512,7 @@ async fn normalize_loaded_data_usage(
&& !cfg.role.is_empty()
{
data_usage_info.replication_info.insert(
cfg.role,
cfg.role.clone(),
BucketTargetUsageInfo {
replication_failed_size: bui.replication_failed_size_v1,
replication_failed_count: bui.replication_failed_count_v1,
@@ -1725,7 +1725,7 @@ fn merge_snapshot(aggregated: &mut DataUsageInfo, mut snapshot: LocalUsageSnapsh
match aggregated.buckets_usage.entry(bucket.clone()) {
Entry::Occupied(mut entry) => entry.get_mut().merge(&usage),
Entry::Vacant(entry) => {
entry.insert(usage);
entry.insert(usage.clone());
}
}
@@ -521,7 +521,7 @@ impl EndpointTopology {
}
};
topology.aliases.entry(host_port).or_insert(member_index);
topology.aliases.entry(host_port.clone()).or_insert(member_index);
topology.aliases.entry(ep.to_string()).or_insert(member_index);
if let Some(display_endpoint) = ep.url.host_str() {
match display_aliases.entry(display_endpoint.to_owned()) {
+1 -10
View File
@@ -124,7 +124,7 @@ where
Box::pin(operation(io_permit))
}
pub(crate) struct SlowDiskWriter {
struct SlowDiskWriter {
inner: crate::disk::FileWriter,
endpoint: Endpoint,
pending: Option<(&'static str, time::Instant)>,
@@ -143,10 +143,6 @@ impl SlowDiskWriter {
}
}
pub(crate) fn inner_mut(&mut self) -> &mut dyn crate::disk::DiskFileWriter {
&mut *self.inner
}
fn observe<T>(&mut self, cx: &mut Context<'_>, op: &'static str, result: Poll<T>) -> Poll<T> {
if result.is_ready() {
self.pending = None;
@@ -1950,11 +1946,6 @@ impl LocalDiskWrapper {
self.health_check
}
#[cfg(test)]
pub(crate) fn health_monitor_cancelled_for_test(&self) -> bool {
self.cancel_token.is_cancelled()
}
#[cfg(test)]
pub(crate) fn override_operation_timeout_for_test(&self, timeout: Duration) -> ScopedDiskOperationTimeout {
assert!(!timeout.is_zero(), "fault injection needs a positive timeout");
+1 -1
View File
@@ -1626,7 +1626,7 @@ mod tests {
for disk_error in disk_errors {
let original_message = disk_error.to_string();
let io_error: std::io::Error = disk_error.into();
let io_error: std::io::Error = disk_error.clone().into();
// The io::Error should contain the original error message
assert!(io_error.to_string().contains(&original_message));
File diff suppressed because it is too large Load Diff
+2 -9
View File
@@ -1106,7 +1106,7 @@ impl LocalDisk {
if let Err(err) = os::rename_all_with_precondition_and_parent_sync_std(
staged_backup,
&backup_path,
&skip_parent,
&dst_volume_dir_owned,
&publication_root_owned,
None,
sync_commit_metadata,
@@ -1464,17 +1464,11 @@ impl LocalDisk {
}
if has_dst_buf_owned.is_none() {
let mut ancestor = dst_file_path_owned.parent().and_then(|object_dir| object_dir.parent());
let mut ancestor_step = 0usize;
while let Some(dir) = ancestor {
if !dir.starts_with(&dst_volume_dir_owned) {
break;
}
if let Err(err) = os::fsync_commit_ancestor_directory_limited_std(
dir,
&dst_volume_dir_owned,
ancestor_step,
&directory_guard_owned,
) {
if let Err(err) = os::fsync_dir_limited_std(dir) {
return Ok(MultipartFusedCommitOutcome::RestoreTargetAfterError(
to_file_error(err).into(),
true,
@@ -1484,7 +1478,6 @@ impl LocalDisk {
break;
}
ancestor = dir.parent();
ancestor_step += 1;
}
}
}
+111 -457
View File
@@ -358,26 +358,14 @@ pub(in crate::disk) fn generation(mut canonical: FileInfo) -> Result<crate::disk
_ => {}
}
}
struct Sha256Writer(Sha256);
impl std::io::Write for Sha256Writer {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.update(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
let mut json_value = serde_json::to_value(&canonical)?;
sort_json_in_place(&mut json_value);
let mut hasher = Sha256Writer(Sha256::new());
serde_json::to_writer(&mut hasher, &json_value)?;
Ok(crate::disk::DeleteRecoveryVersionIdentity {
version_id: canonical.version_id,
mod_time: canonical.mod_time,
data_dir: canonical.data_dir,
deleted: canonical.deleted,
metadata_digest: hasher.0.finalize().into(),
metadata_digest: digest(&serde_json::to_vec(&json_value)?),
})
}
@@ -1186,7 +1174,6 @@ struct PreparedDeleteReceiptEntry {
sole_transaction: bool,
had_old_data_dir: bool,
rollback_dir_fd: Option<Arc<std::fs::File>>,
file_path_dir: Option<Arc<std::fs::File>>,
}
#[derive(Clone)]
@@ -1198,183 +1185,6 @@ struct PublishedDeleteReceiptEntry {
journal_raw: Vec<u8>,
had_old_data_dir: bool,
rollback_dir_fd: Option<Arc<std::fs::File>>,
file_path_dir: Option<Arc<std::fs::File>>,
}
#[cfg(unix)]
fn read_child_regular_bytes(
parent_fd: Option<&std::fs::File>,
child_name: &str,
fallback_path: &Path,
) -> std::io::Result<Vec<u8>> {
if let Some(dir) = parent_fd {
use rustix::fs::{Mode, OFlags, openat};
let fd = openat(dir, child_name, OFlags::RDONLY | OFlags::CLOEXEC, Mode::empty()).map_err(std::io::Error::from)?;
let mut file = std::fs::File::from(fd);
let mut buf = Vec::new();
std::io::Read::read_to_end(&mut file, &mut buf)?;
return Ok(buf);
}
std::fs::read(fallback_path)
}
#[cfg(not(unix))]
fn read_child_regular_bytes(
_parent_fd: Option<&std::fs::File>,
_child_name: &str,
fallback_path: &Path,
) -> std::io::Result<Vec<u8>> {
std::fs::read(fallback_path)
}
#[cfg(unix)]
fn create_and_write_child_file(
parent_fd: Option<&std::fs::File>,
child_name: &str,
fallback_path: &Path,
bytes: &[u8],
) -> std::io::Result<std::fs::File> {
let mut file = if let Some(dir) = parent_fd {
use rustix::fs::{Mode, OFlags, openat};
let fd = openat(
dir,
child_name,
OFlags::WRONLY | OFlags::CREATE | OFlags::TRUNC | OFlags::CLOEXEC,
Mode::from_raw_mode(0o666),
)
.map_err(std::io::Error::from)?;
std::fs::File::from(fd)
} else {
std::fs::File::create(fallback_path)?
};
std::io::Write::write_all(&mut file, bytes)?;
Ok(file)
}
#[cfg(not(unix))]
fn create_and_write_child_file(
_parent_fd: Option<&std::fs::File>,
_child_name: &str,
fallback_path: &Path,
bytes: &[u8],
) -> std::io::Result<std::fs::File> {
let mut file = std::fs::File::create(fallback_path)?;
std::io::Write::write_all(&mut file, bytes)?;
Ok(file)
}
#[cfg(unix)]
fn link_child_files(
src_dir_fd: Option<&std::fs::File>,
src_name: &str,
src_fallback: &Path,
dst_dir_fd: Option<&std::fs::File>,
dst_name: &str,
dst_fallback: &Path,
) -> std::io::Result<()> {
if let (Some(s_fd), Some(d_fd)) = (src_dir_fd, dst_dir_fd) {
return rustix::fs::linkat(s_fd, src_name, d_fd, dst_name, rustix::fs::AtFlags::empty()).map_err(std::io::Error::from);
}
std::fs::hard_link(src_fallback, dst_fallback)
}
#[cfg(not(unix))]
fn link_child_files(
_src_dir_fd: Option<&std::fs::File>,
_src_name: &str,
src_fallback: &Path,
_dst_dir_fd: Option<&std::fs::File>,
_dst_name: &str,
dst_fallback: &Path,
) -> std::io::Result<()> {
std::fs::hard_link(src_fallback, dst_fallback)
}
#[cfg(unix)]
fn link_from_dir_fd(
src_dir_fd: Option<&std::fs::File>,
src_name: &str,
src_fallback: &Path,
dst_path: &Path,
) -> std::io::Result<()> {
if let Some(s_fd) = src_dir_fd {
return rustix::fs::linkat(s_fd, src_name, rustix::fs::CWD, dst_path, rustix::fs::AtFlags::empty())
.map_err(std::io::Error::from);
}
std::fs::hard_link(src_fallback, dst_path)
}
#[cfg(not(unix))]
fn link_from_dir_fd(
_src_dir_fd: Option<&std::fs::File>,
_src_name: &str,
src_fallback: &Path,
dst_path: &Path,
) -> std::io::Result<()> {
std::fs::hard_link(src_fallback, dst_path)
}
#[cfg(unix)]
fn rename_into_dir_fd(
src_path: &Path,
dst_dir_fd: Option<&std::fs::File>,
dst_name: &str,
dst_fallback: &Path,
) -> std::io::Result<()> {
if let Some(d_fd) = dst_dir_fd {
return rustix::fs::renameat(rustix::fs::CWD, src_path, d_fd, dst_name).map_err(std::io::Error::from);
}
std::fs::rename(src_path, dst_fallback)
}
#[cfg(not(unix))]
fn rename_into_dir_fd(
src_path: &Path,
_dst_dir_fd: Option<&std::fs::File>,
_dst_name: &str,
dst_fallback: &Path,
) -> std::io::Result<()> {
std::fs::rename(src_path, dst_fallback)
}
#[cfg(unix)]
fn child_regular_file_exists(dir_fd: Option<&std::fs::File>, child_name: &str, fallback_path: &Path) -> bool {
if let Some(d_fd) = dir_fd {
return rustix::fs::statat(d_fd, child_name, rustix::fs::AtFlags::SYMLINK_NOFOLLOW)
.is_ok_and(|st| (st.st_mode & libc::S_IFMT) == libc::S_IFREG);
}
fallback_path.is_file()
}
#[cfg(not(unix))]
fn child_regular_file_exists(_dir_fd: Option<&std::fs::File>, _child_name: &str, fallback_path: &Path) -> bool {
fallback_path.is_file()
}
#[cfg(unix)]
fn unlink_child_file(dir_fd: Option<&std::fs::File>, child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
if let Some(d_fd) = dir_fd {
return rustix::fs::unlinkat(d_fd, child_name, rustix::fs::AtFlags::empty()).map_err(std::io::Error::from);
}
std::fs::remove_file(fallback_path)
}
#[cfg(not(unix))]
fn unlink_child_file(_dir_fd: Option<&std::fs::File>, _child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
std::fs::remove_file(fallback_path)
}
#[cfg(unix)]
fn rmdir_child_dir(dir_fd: Option<&std::fs::File>, child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
if let Some(d_fd) = dir_fd {
return rustix::fs::unlinkat(d_fd, child_name, rustix::fs::AtFlags::REMOVEDIR).map_err(std::io::Error::from);
}
std::fs::remove_dir(fallback_path)
}
#[cfg(not(unix))]
fn rmdir_child_dir(_dir_fd: Option<&std::fs::File>, _child_name: &str, fallback_path: &Path) -> std::io::Result<()> {
std::fs::remove_dir(fallback_path)
}
const PREPARED_DELETE_RECEIPTS_PER_SHARD: usize = 64;
@@ -1401,71 +1211,6 @@ fn take_published_delete_receipt(rollback: &Path) -> Option<PublishedDeleteRecei
PUBLISHED_DELETE_RECEIPTS.remove(rollback)
}
pub(super) fn clear_delete_receipts_under(io_root: &Path, root: &Path) {
for prefix in [io_root, root] {
PREPARED_DELETE_RECEIPTS.retain_outside_prefix(prefix);
PUBLISHED_DELETE_RECEIPTS.retain_outside_prefix(prefix);
}
}
#[cfg(unix)]
fn open_child_dir_fd(parent_fd: Option<&std::fs::File>, child_name: &str, fallback_path: &Path) -> Option<std::fs::File> {
if let Some(p_fd) = parent_fd {
use rustix::fs::{Mode, OFlags, openat};
return openat(
p_fd,
child_name,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)
.ok()
.map(std::fs::File::from);
}
open_dir_fd(fallback_path).ok()
}
#[cfg(not(unix))]
fn open_child_dir_fd(_parent_fd: Option<&std::fs::File>, _child_name: &str, _fallback_path: &Path) -> Option<std::fs::File> {
None
}
#[cfg(unix)]
fn rename_in_same_dir_fd(
dir_fd: Option<&std::fs::File>,
src_name: &str,
src_fallback: &Path,
dst_name: &str,
dst_fallback: &Path,
) -> std::io::Result<()> {
if let Some(d_fd) = dir_fd {
return rustix::fs::renameat(d_fd, src_name, d_fd, dst_name).map_err(std::io::Error::from);
}
std::fs::rename(src_fallback, dst_fallback)
}
#[cfg(not(unix))]
fn rename_in_same_dir_fd(
_dir_fd: Option<&std::fs::File>,
_src_name: &str,
src_fallback: &Path,
_dst_name: &str,
dst_fallback: &Path,
) -> std::io::Result<()> {
std::fs::rename(src_fallback, dst_fallback)
}
#[cfg(unix)]
fn open_dir_fd(path: &Path) -> std::io::Result<std::fs::File> {
use rustix::fs::{Mode, OFlags, open};
open(
path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)
.map(std::fs::File::from)
.map_err(std::io::Error::from)
}
fn format_commit_receipt_bytes(journal_raw: &[u8], finished: bool) -> Vec<u8> {
let suffix: &[u8] = if finished {
b",\"finished\":true}"
@@ -2797,22 +2542,11 @@ impl LocalDisk {
let object_str = object.to_owned();
let recovery_scope = recovery_operation_scope();
type FusedStage1Outcome = Option<(Option<Uuid>, PathBuf, PreparedDeleteReceiptEntry)>;
let res = os::run_blocking_namespace_operation_with_sync(
lease,
sync,
move || -> std::io::Result<Result<FusedStage1Outcome>> {
#[cfg(unix)]
let file_path_dir = match open_dir_fd(&file_path) {
Ok(dir) => Some(Arc::new(dir)),
Err(err) if matches!(err.kind(), ErrorKind::NotFound | ErrorKind::NotADirectory) => {
return Ok(Ok(None));
}
Err(_) => None,
};
#[cfg(not(unix))]
let file_path_dir: Option<Arc<std::fs::File>> = None;
let buf = match read_child_regular_bytes(file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path) {
move || -> std::io::Result<Result<Option<Option<Uuid>>>> {
let buf = match std::fs::read(&xl_path) {
Ok(buf) => buf,
Err(err) if err.kind() == ErrorKind::NotFound => return Ok(Ok(None)),
Err(err) => return Err(err),
@@ -2970,23 +2704,10 @@ impl LocalDisk {
Err(err) => return Err(err),
};
let staged_dir = staged.path().to_path_buf();
#[cfg(unix)]
let rollback_dir_fd = open_dir_fd(&staged_dir).ok().map(Arc::new);
#[cfg(not(unix))]
let rollback_dir_fd: Option<Arc<std::fs::File>> = None;
let backup_path = staged_dir.join(STORAGE_FORMAT_FILE_BACKUP);
if link_child_files(
file_path_dir.as_deref(),
STORAGE_FORMAT_FILE,
&xl_path,
rollback_dir_fd.as_deref(),
STORAGE_FORMAT_FILE_BACKUP,
&backup_path,
)
.is_err()
{
let file =
create_and_write_child_file(rollback_dir_fd.as_deref(), STORAGE_FORMAT_FILE_BACKUP, &backup_path, &buf)?;
if std::fs::hard_link(&xl_path, &backup_path).is_err() {
let mut file = std::fs::File::create(&backup_path)?;
std::io::Write::write_all(&mut file, &buf)?;
if sync && !buf.is_empty() {
file.sync_data()?;
}
@@ -2996,7 +2717,8 @@ impl LocalDisk {
let Some(bytes) = bytes else {
continue;
};
let file = create_and_write_child_file(rollback_dir_fd.as_deref(), name, &staged_dir.join(name), bytes)?;
let mut file = std::fs::File::create(staged_dir.join(name))?;
std::io::Write::write_all(&mut file, bytes)?;
if sync && !bytes.is_empty() {
written_files.push(file);
}
@@ -3004,20 +2726,21 @@ impl LocalDisk {
for file in written_files {
file.sync_data()?;
}
#[cfg(unix)]
let rollback_dir_fd = sync.then(|| std::fs::File::open(&staged_dir).ok().map(Arc::new)).flatten();
#[cfg(not(unix))]
let rollback_dir_fd: Option<Arc<std::fs::File>> = None;
if sync {
os::fsync_directory_handle_std(&staged_dir, rollback_dir_fd.as_deref())?;
}
let tx_name = transaction.to_string();
if !destination_missing {
match std::fs::symlink_metadata(&destination) {
Ok(metadata) if metadata.is_dir() => {
match rmdir_child_dir(file_path_dir.as_deref(), &tx_name, &destination) {
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
Err(err) => return Err(err),
}
}
Ok(metadata) if metadata.is_dir() => match std::fs::remove_dir(&destination) {
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) if is_dir_not_empty_error(&err) => return Ok(Err(DiskError::OutdatedXLMeta)),
Err(err) => return Err(err),
},
Ok(_) => return Ok(Err(DiskError::OutdatedXLMeta)),
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
@@ -3025,26 +2748,32 @@ impl LocalDisk {
}
let staged_dir = staged.keep();
os::record_native_publication(&destination);
if let Err(err) = rename_into_dir_fd(&staged_dir, file_path_dir.as_deref(), &tx_name, &destination) {
if let Err(err) = std::fs::rename(&staged_dir, &destination) {
let _ = std::fs::remove_dir_all(&staged_dir);
return Err(err);
}
let old_data_path = old_dir
.map(|uuid| (uuid.to_string(), file_path.join(uuid.to_string())))
.filter(|(_, p)| old_data_dir_state.unwrap_or_else(|| p.is_dir()));
if sync && let Err(err) = os::fsync_directory_handle_std(&file_path, file_path_dir.as_deref()) {
.map(|uuid| file_path.join(uuid.to_string()))
.filter(|p| old_data_dir_state.unwrap_or_else(|| p.is_dir()));
#[cfg(unix)]
let file_path_dir = sync.then(|| std::fs::File::open(&file_path).ok()).flatten();
#[cfg(not(unix))]
let file_path_dir: Option<std::fs::File> = None;
if sync && let Err(err) = os::fsync_directory_handle_std(&file_path, file_path_dir.as_ref()) {
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
#[cfg(all(test, unix))]
fused_prepare_checkpoint::run(&file_path);
let mut reserved_data_path = None;
if let Some((old_dir_name, old_path)) = old_data_path {
let reserved_name = format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
let reserved_marker = old_path.join(&reserved_name);
let old_dir_file = open_child_dir_fd(file_path_dir.as_deref(), &old_dir_name, &old_path);
if let Some(old_path) = old_data_path {
let reserved_marker = old_path.join(format!("{RESERVED_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
#[cfg(unix)]
let old_dir_file = sync.then(|| std::fs::File::open(&old_path).ok()).flatten();
#[cfg(not(unix))]
let old_dir_file: Option<std::fs::File> = None;
let reserve_res = (|| -> std::io::Result<()> {
let marker = create_and_write_child_file(old_dir_file.as_ref(), &reserved_name, &reserved_marker, &[])?;
let marker = std::fs::File::create(&reserved_marker)?;
if sync {
marker.sync_all()?;
os::fsync_directory_handle_std(&old_path, old_dir_file.as_ref())?;
@@ -3052,27 +2781,27 @@ impl LocalDisk {
Ok(())
})();
if let Err(err) = reserve_res {
let _ = unlink_child_file(old_dir_file.as_ref(), &reserved_name, &reserved_marker);
let _ = std::fs::remove_file(&reserved_marker);
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
reserved_data_path = Some((old_path, reserved_name, reserved_marker, old_dir_file));
reserved_data_path = Some((old_path, reserved_marker, old_dir_file));
}
if let Some(ref target_bytes) = target {
// A provisional head must stay outside the published recovery transaction.
let target_in_rollback = destination.join(TARGET);
let linked_xl = staging_root.join(format!("delete-xl-{transaction}"));
os::record_native_publication(&xl_path);
let linked = match link_from_dir_fd(rollback_dir_fd.as_deref(), TARGET, &target_in_rollback, &linked_xl) {
let linked = match std::fs::hard_link(&target_in_rollback, &linked_xl) {
Ok(()) => true,
Err(err) if err.kind() == ErrorKind::AlreadyExists => {
let _ = std::fs::remove_file(&linked_xl);
link_from_dir_fd(rollback_dir_fd.as_deref(), TARGET, &target_in_rollback, &linked_xl).is_ok()
std::fs::hard_link(&target_in_rollback, &linked_xl).is_ok()
}
Err(_) => false,
};
let write_res = if linked {
let rename_res = rename_into_dir_fd(&linked_xl, file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path);
let rename_res = std::fs::rename(&linked_xl, &xl_path);
if rename_res.is_err() {
let _ = std::fs::remove_file(&linked_xl);
}
@@ -3084,47 +2813,39 @@ impl LocalDisk {
if sync && !target_bytes.is_empty() {
staged_xl.as_file().sync_data()?;
}
rename_into_dir_fd(staged_xl.path(), file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path)
std::fs::rename(staged_xl.path(), &xl_path)
})()
};
if let Err(err) = write_res {
if let Some((_, reserved_name, reserved_marker, old_dir_file)) = &reserved_data_path {
let _ = unlink_child_file(old_dir_file.as_ref(), reserved_name, reserved_marker);
if let Some((_, reserved_marker, _)) = &reserved_data_path {
let _ = std::fs::remove_file(reserved_marker);
}
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
} else if let Err(err) = unlink_child_file(file_path_dir.as_deref(), STORAGE_FORMAT_FILE, &xl_path)
} else if let Err(err) = std::fs::remove_file(&xl_path)
&& err.kind() != ErrorKind::NotFound
{
if let Some((_, reserved_name, reserved_marker, old_dir_file)) = &reserved_data_path {
let _ = unlink_child_file(old_dir_file.as_ref(), reserved_name, reserved_marker);
if let Some((_, reserved_marker, _)) = &reserved_data_path {
let _ = std::fs::remove_file(reserved_marker);
}
let _ = std::fs::remove_dir_all(&destination);
return Err(err);
}
if sync {
os::fsync_directory_handle_std(&file_path, file_path_dir.as_deref())?;
os::fsync_directory_handle_std(&file_path, file_path_dir.as_ref())?;
}
let had_old_data_dir = reserved_data_path.is_some();
if let Some((old_path, reserved_name, reserved_marker, old_dir_file)) = reserved_data_path {
let recovery_name = format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}");
let recovery_marker = old_path.join(&recovery_name);
match rename_in_same_dir_fd(
old_dir_file.as_ref(),
&reserved_name,
&reserved_marker,
&recovery_name,
&recovery_marker,
) {
if let Some((old_path, reserved_marker, old_dir_file)) = reserved_data_path {
let recovery_marker = old_path.join(format!("{RECOVERY_DELETE_DATA_DIR_MARKER_PREFIX}{transaction}"));
match std::fs::rename(&reserved_marker, &recovery_marker) {
Ok(()) if sync => os::fsync_directory_handle_std(&old_path, old_dir_file.as_ref())?,
Ok(()) => {}
Err(err) if err.kind() == ErrorKind::NotFound => {}
Err(err) => return Err(err),
}
}
Ok(Ok(Some((
old_dir.filter(|_| had_old_data_dir),
put_prepared_delete_receipt(
destination,
PreparedDeleteReceiptEntry {
object_path,
@@ -3136,18 +2857,17 @@ impl LocalDisk {
sole_transaction,
had_old_data_dir,
rollback_dir_fd,
file_path_dir,
},
))))
);
Ok(Ok(Some(old_dir.filter(|_| had_old_data_dir))))
},
)
.await
.map_err(to_file_error)??;
let Some((old_dir, destination, receipt_entry)) = res else {
let Some(old_dir) = res else {
return Ok(None);
};
put_prepared_delete_receipt(destination, receipt_entry);
if let Some(uuid) = old_dir {
self.io_backend
.invalidate_cached_fds_under(volume, &format!("{object}/{uuid}"));
@@ -3198,7 +2918,6 @@ impl LocalDisk {
entry.sole_transaction,
entry.had_old_data_dir,
entry.rollback_dir_fd,
entry.file_path_dir,
)),
),
None => {
@@ -3226,14 +2945,12 @@ impl LocalDisk {
let volume_str = volume.to_owned();
let object_str = object.to_owned();
type PublishDeleteFastReturn = (DeleteRecoveryJournal, bool, bool, Option<(PathBuf, PublishedDeleteReceiptEntry)>);
let res = os::run_blocking_namespace_operation_with_sync(
os::run_blocking_namespace_operation_with_sync(
lease,
sync,
move || -> std::io::Result<Result<Option<PublishDeleteFastReturn>>> {
if let Some((journal_raw, journal, sole_transaction, had_old_data_dir, rollback_dir_fd, file_path_dir)) =
cached_receipt
&& child_regular_file_exists(rollback_dir_fd.as_deref(), PREPARED, &rollback.join(PREPARED))
move || -> std::io::Result<Result<Option<(DeleteRecoveryJournal, bool, bool)>>> {
if let Some((journal_raw, journal, sole_transaction, had_old_data_dir, rollback_dir_fd)) = cached_receipt
&& rollback.join(PREPARED).is_file()
{
let tmp_receipt = staging_root.join(format!("receipt-{transaction}.tmp"));
let receipt_bytes = format_commit_receipt_bytes(&journal_raw, false);
@@ -3244,12 +2961,7 @@ impl LocalDisk {
file.sync_data()?;
}
os::record_native_publication(&receipt_path);
rename_into_dir_fd(
&tmp_receipt,
rollback_dir_fd.as_deref(),
DELETE_QUORUM_COMMITTED_FILE,
&receipt_path,
)?;
std::fs::rename(&tmp_receipt, &receipt_path)?;
if sync {
os::fsync_directory_handle_std(&rollback, rollback_dir_fd.as_deref())?;
}
@@ -3259,8 +2971,8 @@ impl LocalDisk {
let _ = std::fs::remove_file(&tmp_receipt);
return Err(err);
}
let published_entry = bucket_path.map(|bucket_path| {
(
if let Some(bucket_path) = bucket_path {
put_published_delete_receipt(
rollback,
PublishedDeleteReceiptEntry {
object_path,
@@ -3270,11 +2982,10 @@ impl LocalDisk {
journal_raw,
had_old_data_dir,
rollback_dir_fd,
file_path_dir,
},
)
});
return Ok(Ok(Some((journal, sole_transaction, true, published_entry))));
);
}
return Ok(Ok(Some((journal, sole_transaction, true))));
}
let prepared = match regular_bytes(&rollback.join(PREPARED)) {
Ok(Some(bytes)) => bytes,
@@ -3353,19 +3064,11 @@ impl LocalDisk {
if sync {
os::fsync_dir_std(&rollback)?;
}
Ok(Ok(Some((journal, sole_transaction, target_satisfied, None))))
Ok(Ok(Some((journal, sole_transaction, target_satisfied))))
},
)
.await
.map_err(to_file_error)?;
let Some((journal, sole_transaction, target_satisfied, published_entry)) = res? else {
return Ok(None);
};
if let Some((rollback, entry)) = published_entry {
put_published_delete_receipt(rollback, entry);
}
Ok(Some((journal, sole_transaction, target_satisfied)))
.map_err(to_file_error)?
}
#[cfg(test)]
@@ -3415,49 +3118,34 @@ impl LocalDisk {
None
};
let cached_published = rollback_open_path.as_deref().and_then(take_published_delete_receipt);
if cached_published.is_none()
&& let Some(ref tx_open) = rollback_open_path
{
let _ = take_prepared_delete_receipt(tx_open);
}
let (
object_path,
bucket_path,
staging_root,
root,
cached_journal_raw,
cached_had_old_data_dir,
mut rollback_dir_fd,
mut file_path_dir,
) = match cached_published {
Some(entry) => (
entry.object_path,
entry.bucket_path,
entry.staging_root,
entry.file_path,
Some(entry.journal_raw),
Some(entry.had_old_data_dir),
entry.rollback_dir_fd,
entry.file_path_dir,
),
None => {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) {
return Err(DiskError::InvalidPath);
let (object_path, bucket_path, staging_root, root, cached_journal_raw, cached_had_old_data_dir, mut rollback_dir_fd) =
match cached_published {
Some(entry) => (
entry.object_path,
entry.bucket_path,
entry.staging_root,
entry.file_path,
Some(entry.journal_raw),
Some(entry.had_old_data_dir),
entry.rollback_dir_fd,
),
None => {
self.reject_multipart_delete_recovery_mutation(volume, object)?;
if !safe_path(object, false) {
return Err(DiskError::InvalidPath);
}
let obj_path = self.get_object_path(volume, object)?;
(
obj_path,
self.io_get_bucket_path(volume)?,
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
self.io_get_object_path(volume, object)?,
None,
None,
None,
)
}
let obj_path = self.get_object_path(volume, object)?;
(
obj_path,
self.io_get_bucket_path(volume)?,
self.io_get_bucket_path(RUSTFS_META_TMP_BUCKET)?,
self.io_get_object_path(volume, object)?,
None,
None,
None,
None,
)
}
};
};
let metadata_owner = os::acquire_metadata_mutation_lease(&object_path, namespace_owner).await;
let volume_guard = os::disk_volume_mutation_lock(&self.root, volume).read_owned().await;
let owner: Option<Arc<dyn Send + Sync>> = Some(Arc::new((metadata_owner, volume_guard)));
@@ -3535,15 +3223,11 @@ impl LocalDisk {
if cached_had_old_data_dir != Some(false) {
let bases: &[&Path] = if fast_cleanup { &[&root] } else { &[&root, &rollback] };
for data_dir in &data_dirs {
let data_dir_str = data_dir.to_string();
for base in bases {
let dir_path = base.join(&data_dir_str);
let dir_path = base.join(data_dir.to_string());
match std::fs::read_dir(&dir_path) {
Ok(entries) => {
had_physical_data_dirs = true;
let data_dir_fd = (*base == root)
.then(|| open_child_dir_fd(file_path_dir.as_deref(), &data_dir_str, &dir_path))
.flatten();
let mut removable = Vec::new();
let mut preserve_dir = false;
for entry in recovery_entries(entries, work_budget.as_deref()) {
@@ -3560,33 +3244,25 @@ impl LocalDisk {
|| name == reserved
|| is_delete_transaction_marker(name, RESERVED_DELETE_DATA_DIR_MARKER_PREFIX))
{
removable.push((name.to_owned(), entry.path()));
removable.push(entry.path());
} else {
preserve_dir = true;
}
}
for (name, path) in removable {
ignore_not_found(unlink_child_file(data_dir_fd.as_ref(), &name, &path))?;
for path in removable {
ignore_not_found(std::fs::remove_file(path))?;
}
let mut dir_removed = false;
if !preserve_dir {
drop(data_dir_fd);
let rmdir_res = if *base == root {
rmdir_child_dir(file_path_dir.as_deref(), &data_dir_str, &dir_path)
} else {
std::fs::remove_dir(&dir_path)
};
match rmdir_res {
match std::fs::remove_dir(&dir_path) {
Ok(()) => dir_removed = true,
Err(err) if err.kind() == ErrorKind::NotFound => dir_removed = true,
Err(err) if is_dir_not_empty_error(&err) => {}
Err(err) => return Err(err),
}
if !dir_removed && sync {
os::fsync_dir_std(&dir_path)?;
}
} else if sync {
os::fsync_directory_handle_std(&dir_path, data_dir_fd.as_ref())?;
}
if !dir_removed && sync {
os::fsync_dir_std(&dir_path)?;
}
}
Err(err) if err.kind() == ErrorKind::NotFound => {}
@@ -3596,7 +3272,7 @@ impl LocalDisk {
}
}
if had_physical_data_dirs && sync && root.is_dir() {
os::fsync_directory_handle_std(&root, file_path_dir.as_deref())?;
os::fsync_dir_std(&root)?;
}
if fast_cleanup {
let backup_or_marker = if has_original {
@@ -3607,10 +3283,8 @@ impl LocalDisk {
None
};
if let Some(name) = backup_or_marker
&& let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &rollback.join(name)))
&& let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name)))
{
drop(rollback_dir_fd.take());
drop(file_path_dir.take());
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
@@ -3620,9 +3294,7 @@ impl LocalDisk {
let path = rollback.join(name);
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
if let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &path)) {
drop(rollback_dir_fd.take());
drop(file_path_dir.take());
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
@@ -3631,10 +3303,7 @@ impl LocalDisk {
}
} else {
for name in [STORAGE_FORMAT_FILE_BACKUP, DELETE_MARKER_ROLLBACK_FILE] {
if let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &rollback.join(name)))
{
drop(rollback_dir_fd.take());
drop(file_path_dir.take());
if let Err(err) = ignore_not_found(std::fs::remove_file(rollback.join(name))) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
@@ -3645,9 +3314,7 @@ impl LocalDisk {
let path = rollback.join(name);
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &path);
if let Err(err) = ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), name, &path)) {
drop(rollback_dir_fd.take());
drop(file_path_dir.take());
if let Err(err) = ignore_not_found(std::fs::remove_file(path)) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
@@ -3655,39 +3322,27 @@ impl LocalDisk {
}
}
}
if sync && let Err(err) = os::fsync_directory_handle_std(&rollback, rollback_dir_fd.as_deref()) {
drop(rollback_dir_fd.take());
drop(file_path_dir.take());
let rollback_sync = rollback_dir_fd.take();
if sync && let Err(err) = os::fsync_directory_handle_std(&rollback, rollback_sync.as_deref()) {
drop(rollback_sync);
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
return Err(err);
}
drop(rollback_sync);
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, &receipt_path);
let receipt_unlink_res =
ignore_not_found(unlink_child_file(rollback_dir_fd.as_deref(), DELETE_QUORUM_COMMITTED_FILE, &receipt_path));
drop(rollback_dir_fd.take());
if let Err(err) = receipt_unlink_res {
drop(file_path_dir.take());
if let Err(err) = ignore_not_found(std::fs::remove_file(&receipt_path)) {
if has_target || had_physical_data_dirs {
let _ = persist_receipt(true);
}
return Err(err);
}
let tx_name = transaction.to_string();
for (idx, dir_path) in prune_paths.iter().enumerate() {
for dir_path in &prune_paths {
#[cfg(all(any(test, feature = "test-util"), not(windows)))]
os::prepared_publication_test_hooks::run(os::prepared_publication_test_hooks::Stage::Remove, dir_path);
let rmdir_res = if idx == 0 && *dir_path == rollback {
let r = rmdir_child_dir(file_path_dir.as_deref(), &tx_name, dir_path);
drop(file_path_dir.take());
r
} else {
drop(file_path_dir.take());
std::fs::remove_dir(dir_path)
};
if let Err(err) = rmdir_res {
if let Err(err) = std::fs::remove_dir(dir_path) {
if is_dir_not_empty_error(&err) {
break;
}
@@ -6179,7 +5834,6 @@ mod tests {
drop(disk);
let endpoint = Endpoint::try_from(temp.path().to_str().expect("UTF-8 disk path")).expect("endpoint");
let disk = LocalDisk::new(&endpoint, false).await.expect("reopen after cleanup");
let root = disk.io_get_object_path("recovery", "object").expect("reopened object path");
let observed = disk
.inspect_delete_transaction("recovery", "object", transaction)
.await
@@ -40,15 +40,16 @@ static OWNERS: LazyLock<parking_lot::Mutex<HashMap<PhysicalRoot, Weak<IndexOwner
LazyLock::new(|| parking_lot::Mutex::new(HashMap::new()));
pub(super) struct IndexOwner {
// Revalidate the configured path against this owner's physical identity.
// Only the path: holding the disk's pinned root handle here would keep the
// mount alive past the last disk owner, which the disk lifecycle forbids.
root_path: Arc<PathBuf>,
physical: PhysicalRoot,
#[cfg(unix)]
_identity_pin: std::fs::File,
// Linux names _identity_pin so retiring a caller's root cannot retarget it.
base: PathBuf,
epoch: Uuid,
// Retain this physical owner's queue across bootstrap and disk-handle gaps.
// Keep the original pinned-fd alias registered while a detached bootstrap
// or reconnect still owns this physical disk's index.
blocking_queue: Arc<os::DiskBlockingQueue>,
// No persisted readiness is trusted: an old binary may have written while
// this process was stopped. Same-process handles share the continuous owner.
@@ -263,22 +264,14 @@ impl IndexOwner {
}
pin
};
#[cfg(target_os = "linux")]
let base = {
use std::os::fd::AsRawFd;
PathBuf::from(format!("/proc/self/fd/{}/.", identity_pin.as_raw_fd()))
};
#[cfg(not(target_os = "linux"))]
let base = base.to_owned();
let blocking_queue = os::register_disk_blocking_queue(logical_root, &base);
let owner = Arc::new(Self {
root_path: Arc::new(root.path().to_path_buf()),
physical: key,
#[cfg(unix)]
_identity_pin: identity_pin,
base,
base: base.to_owned(),
epoch: Uuid::new_v4(),
blocking_queue,
blocking_queue: os::register_disk_blocking_queue(logical_root, base),
initialized: OnceLock::new(),
driving: AtomicBool::new(false),
disabled: AtomicBool::new(false),
@@ -1641,97 +1634,6 @@ mod tests {
assert!(Arc::ptr_eq(&new.blocking_queue_for_test(), &reconnected.blocking_queue));
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn multipart_index_owner_queue_survives_reassigned_caller_alias() {
const CHILD: &str = "RUSTFS_TEST_INDEX_OWNER_QUEUE_ALIAS_CHILD";
if std::env::var_os(CHILD).is_none() {
// Alias reassignment mutates a process-wide registry. Isolate it
// from concurrent tests that may reuse the retired fd number.
let mut child = std::process::Command::new(std::env::current_exe().expect("current test binary"))
.arg("--exact")
.arg("disk::local::multipart_index::tests::multipart_index_owner_queue_survives_reassigned_caller_alias")
.arg("--nocapture")
.env(CHILD, "1")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.expect("start isolated queue-alias regression");
let completed = tokio::time::timeout(std::time::Duration::from_secs(10), async {
let mut poll = tokio::time::interval(std::time::Duration::from_millis(10));
loop {
poll.tick().await;
if child.try_wait().expect("poll queue-alias regression").is_some() {
break;
}
}
})
.await;
if completed.is_err() {
child.kill().expect("stop blocked queue-alias regression");
}
let output = child.wait_with_output().expect("collect queue-alias regression");
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
assert!(completed.is_ok(), "queue-alias regression must finish within its deadline");
assert!(output.status.success(), "queue-alias regression failed: {stdout} {stderr}");
assert!(stdout.contains("1 passed"), "the child must execute the queue-alias regression: {stdout}");
return;
}
let root = tempfile::tempdir().expect("original disk root");
let (path, info) = upload("bucket", "legacy");
seed_legacy(root.path(), &path, info);
let publication = os::PublicationRoot::new(root.path()).expect("original caller root");
let caller_alias = publication.io_path().to_path_buf();
let owner = IndexOwner::acquire(&publication, root.path(), &caller_alias).expect("original physical owner");
let original_queue = owner.blocking_queue.clone();
let other_root = tempfile::tempdir().expect("other disk root");
let other_publication = os::PublicationRoot::new(other_root.path()).expect("other pinned root");
drop(publication);
// A retired caller's fd number can be reused by another disk. Assign
// its alias explicitly so the test does not depend on fd allocation.
let other_queue = os::register_disk_blocking_queue(other_root.path(), &caller_alias);
let other_attached = os::register_disk_blocking_queue(other_root.path(), other_publication.io_path());
assert!(Arc::ptr_eq(&other_queue, &other_attached));
assert!(!Arc::ptr_eq(&original_queue, &other_queue));
tokio::time::timeout(std::time::Duration::from_secs(5), owner.initialize())
.await
.expect("bootstrap completes after the caller alias is reassigned");
assert!(Arc::ptr_eq(
owner.observed_rebuild_queue.lock().as_ref().expect("actual rebuild queue"),
&original_queue,
));
assert_eq!(
owner
.page("bucket", None, Uuid::new_v4())
.await
.expect("original disk index coverage")
.roots
.len(),
1,
);
let reconnect = os::PublicationRoot::new(root.path()).expect("reconnected caller root");
let attached = owner.attach_blocking_queue(root.path(), reconnect.io_path());
assert!(Arc::ptr_eq(&attached, &original_queue));
for (path, expected_queue) in [
(owner.base.join(INDEX), &original_queue),
(reconnect.io_path().join(INDEX), &original_queue),
(root.path().join(INDEX), &original_queue),
(other_publication.io_path().join(INDEX), &other_queue),
(other_root.path().join(INDEX), &other_queue),
] {
let lease = os::acquire_namespace_mutation_lease_with_owner(&path, None).await;
assert!(
Arc::ptr_eq(&lease.blocking_queue_for_test(), expected_queue),
"queue routing for {path:?}"
);
}
}
#[test]
fn multipart_index_rebuild_counts_empty_and_invalid_entries_toward_job_budget() {
let root = tempfile::tempdir().expect("disk root");
@@ -295,11 +295,17 @@ pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: By
}
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
&& !(cfg!(test)
&& (source.join(STATE).exists()
|| source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|| source.join(".mpu-completions").exists()))
if FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) != Some(expected_head_stamp) {
return false;
}
#[cfg(test)]
if source.join(STATE).exists()
|| source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|| source.join(".mpu-completions").exists()
{
return false;
}
true
}
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
@@ -770,20 +776,15 @@ impl SourceStore {
} else {
original_path(source, owner)
};
#[cfg(not(target_os = "linux"))]
let _pin = if source_already_pinned {
None
} else {
Some(os::pin_multipart_source_directory(source, base, root)?)
};
#[cfg(target_os = "linux")]
let _ = source_already_pinned;
let parent = path
.parent()
.ok_or_else(|| std::io::Error::other("source backup has no parent"))?;
os::existing_base_directory_suffix(source, base)?;
#[cfg_attr(not(unix), allow(unused_mut))]
let mut parents = os::mkdir_all_below_existing_base_std(parent, source, root)?;
let _parents = os::mkdir_all_below_existing_base_std(parent, base, root)?;
let existed = if fresh_owner {
false
} else if target {
@@ -804,31 +805,10 @@ impl SourceStore {
false
};
if !existed {
#[cfg(target_os = "linux")]
let leaf_fd = parents.pop();
#[cfg(all(unix, not(target_os = "linux")))]
let leaf_fd = None;
os::replace_multipart_source_metadata_with_leaf_fd(
&path,
bytes,
base,
root,
#[cfg(unix)]
leaf_fd,
)?;
#[cfg(all(unix, not(target_os = "linux")))]
let _ = parents.pop();
os::replace_multipart_source_metadata(&path, bytes, base, root)?;
}
if sync_parents {
os::sync_multipart_owned_record_parents_options(
source,
parent,
existed,
sync_source_root,
base,
root,
Some(parents),
)?;
os::sync_multipart_owned_record_parents_options(source, parent, existed, sync_source_root, base, root)?;
}
Ok(())
}
@@ -947,12 +927,9 @@ impl SourceStore {
let base = self.base.clone();
let root = self.publication_root.clone();
os::run_blocking_namespace_operation(self.lease.clone(), move || {
#[cfg(not(target_os = "linux"))]
let _pin = os::pin_multipart_source_directory(&source, &base, &root)?;
let parent = path.parent().ok_or_else(|| IoError::other("certificate has no parent"))?;
os::existing_base_directory_suffix(&source, &base)?;
#[cfg_attr(not(unix), allow(unused_mut))]
let mut parents = os::mkdir_all_below_existing_base_std(parent, &source, &root)?;
let _parents = os::mkdir_all_below_existing_base_std(parent, &base, &root)?;
let existed = if let Some(previous) = read_optional(&path, &base, &root).map_err(IoError::other)? {
if previous.as_ref() != raw.as_slice() {
return Err(IoError::new(std::io::ErrorKind::AlreadyExists, "immutable recovery certificate changed"));
@@ -962,22 +939,9 @@ impl SourceStore {
false
};
if !existed {
#[cfg(target_os = "linux")]
let leaf_fd = parents.pop();
#[cfg(all(unix, not(target_os = "linux")))]
let leaf_fd = None;
os::replace_multipart_source_metadata_with_leaf_fd(
&path,
&raw,
&base,
&root,
#[cfg(unix)]
leaf_fd,
)?;
#[cfg(all(unix, not(target_os = "linux")))]
let _ = parents.pop();
os::replace_multipart_source_metadata(&path, &raw, &base, &root)?;
}
os::sync_multipart_owned_record_parents_options(&source, parent, existed, true, &base, &root, Some(parents))
os::sync_multipart_owned_record_parents(&source, parent, existed, &base, &root)
})
.await?;
Ok(digest)
@@ -1941,14 +1905,8 @@ impl LocalDisk {
os::run_blocking_namespace_operation_with_sync(staging.lease.clone(), true, move || {
SourceStore::write_owned_backup_std(&source_dir, &base_dir, &pub_root, owner, false, &original_bytes, false)?;
SourceStore::persist_state_std(&source_dir, &base_dir, &pub_root, &state_raw, terminal.as_deref(), false)?;
let staging_base = directory.parent().filter(|p| p.starts_with(&base_dir)).unwrap_or(&base_dir);
os::replace_multipart_source_metadata_creating_parent(
&directory.join(RETIREMENT_ANCHOR),
&anchor_bytes,
staging_base,
&base_dir,
&pub_root,
)?;
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base_dir, &pub_root)?;
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base_dir, &pub_root)?;
#[cfg(target_os = "linux")]
if can_mark_presynced {
super::mark_presynced_tmp_data_dir(directory);
@@ -2159,14 +2117,8 @@ impl NativeMultipartPublication {
false,
Some(directory_guard),
)?;
let staging_base = directory.parent().filter(|p| p.starts_with(base)).unwrap_or(base);
os::replace_multipart_source_metadata_creating_parent(
&directory.join(RETIREMENT_ANCHOR),
&anchor_bytes,
staging_base,
base,
root,
)?;
let _parent = os::mkdir_all_below_existing_base_std(&directory, base, root)?;
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, base, root)?;
#[cfg(target_os = "linux")]
if can_mark_presynced {
super::mark_presynced_tmp_data_dir(directory);
@@ -2293,14 +2245,8 @@ impl NativeMultipartPublication {
SourceStore::write_owned_backup_std(&source, &base, &root, backup_owner, true, previous, false)?;
}
SourceStore::persist_state_std(&source, &base, &root, &next_state_raw, terminal.as_deref(), false)?;
let staging_base = directory.parent().filter(|p| p.starts_with(&base)).unwrap_or(&base);
os::replace_multipart_source_metadata_creating_parent(
&directory.join(RETIREMENT_ANCHOR),
&anchor_bytes,
staging_base,
&base,
&root,
)?;
let _parent = os::mkdir_all_below_existing_base_std(&directory, &base, &root)?;
os::replace_multipart_source_metadata(&directory.join(RETIREMENT_ANCHOR), &anchor_bytes, &base, &root)?;
#[cfg(target_os = "linux")]
if can_mark_presynced {
super::mark_presynced_tmp_data_dir(directory);
@@ -444,6 +444,7 @@ impl LocalDisk {
{
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "multipart source layout changed"));
}
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
if incoming.data.is_none() && !incoming.is_remote() {
let source_data_dir = source_object.join(source_dir.to_string());
let mut pinned_source_data = None;
@@ -508,8 +509,6 @@ impl LocalDisk {
sync_directory,
cancelled,
)?;
} else {
let _attempt = os::mkdir_all_below_existing_base_std(&private_object, &base, &publication_root)?;
}
let has_state = !from_active_cache
&& (source_object.join(".mpu-source-state").exists()

Some files were not shown because too many files have changed in this diff Show More