mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-08 06:13:14 +00:00
b0b5ffb8e2
test(e2e): add drivesPerNode and 2-pool cluster harness topology (backlog #1325) Extend the e2e cluster harness so tests can declare per-node multiple drives (drivesPerNode) and a two-pool topology, alongside a smoke suite that boots such a cluster and round-trips a PUT/GET. A new `ClusterTopology` describes node_count, drives_per_node, and pool membership, and `RustFSTestClusterEnvironment::with_topology` builds the matching on-disk drive directories and `RUSTFS_VOLUMES` string. `new(node_count)` now delegates to a single-pool single-drive topology, so existing single-drive cluster tests are byte-for-byte unchanged. `ClusterNode` gains `data_dirs` (all drives, `data_dirs[0] == data_dir`) and `pool_idx`; multi-drive/multi-pool clusters automatically add `RUSTFS_UNSAFE_BYPASS_DISK_CHECK=true` because their drives share the same temp filesystem. The `RUSTFS_VOLUMES` assembly matches the two forms the server parser accepts on a single localhost machine (verified against ecstore `DisksLayout::from_volumes`): a single pool is the explicit enumeration of every `(node, drive)` endpoint (no ellipses, one legacy DistErasure pool), while multiple pools each contribute one ellipses argument `http://<addr><node-base>/drive{0...N-1}`. A pool argument is a single URL template, so it cannot enumerate multiple distinct-port hosts; a pool striped across several localhost nodes would need a host ellipses that forces a shared on-disk path and collides physically. The topology validator therefore requires every pool in a multi-pool layout to own exactly one node and requires drives_per_node >= 2 (the parser rejects a single-drive `drive{0...0}` ellipses pool). Genuine multi-node pools need real multi-host infrastructure and are deferred to the nightly cluster lane (backlog #1313/#1314). Unit tests assert the volumes-string layout for single-pool single-drive (backward compatible), single-pool multi-drive (8 explicit endpoints), and two-pool (one ellipses arg per pool), plus the validator rejections. The smoke suite `cluster_multidrive_pool_test` boots a 4-node x 2-drive single pool and a 2-node/2-pool cluster, asserts the volumes layout, and round-trips a PUT/GET using the harness readiness handshake (no fixed sleeps). It joins the six existing RustFSTestClusterEnvironment suites excluded from the merge-gate `e2e-full` profile so it runs in the nightly 4-node lane. Network fault injection (toxiproxy / socket proxy) and 5GiB large-object budgets remain out of scope for this block.
259 lines
6.9 KiB
Rust
259 lines
6.9 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
mod reliant;
|
|
mod storage_api;
|
|
|
|
// Common utilities for all E2E tests
|
|
#[cfg(test)]
|
|
pub mod common;
|
|
|
|
// In-process fault-injection primitives (disk offline/replacement, shard corruption)
|
|
#[cfg(test)]
|
|
pub mod chaos;
|
|
|
|
// Programmable S3 target for replication failure-path tests (backlog#1147 repl-8).
|
|
#[cfg(test)]
|
|
pub mod fake_s3_target;
|
|
|
|
// Reliability tests built on the fault-injection harness
|
|
#[cfg(test)]
|
|
mod reliability_disk_fault_test;
|
|
|
|
// dist-13 (backlog#1150/#1155): e2e regression net proving a large-object
|
|
// degraded EC read never returns a silently truncated body (rustfs#4594/#4560/#4585).
|
|
#[cfg(test)]
|
|
mod degraded_read_eof_regression_test;
|
|
|
|
// backlog#1183: GET codec-streaming fast path must be byte/header identical to
|
|
// the legacy duplex path before its rollout gates can be flipped on by default.
|
|
#[cfg(test)]
|
|
mod get_codec_streaming_compat_test;
|
|
|
|
#[cfg(test)]
|
|
mod version_id_regression_test;
|
|
|
|
// Data usage regression tests
|
|
#[cfg(test)]
|
|
mod data_usage_test;
|
|
|
|
// KMS-specific test modules
|
|
#[cfg(test)]
|
|
mod kms;
|
|
|
|
// Regression test for issue #1797
|
|
#[cfg(test)]
|
|
mod list_objects_duplicates_test;
|
|
|
|
// Quota tests
|
|
#[cfg(test)]
|
|
mod quota_test;
|
|
|
|
// Harness regression tests: console port isolation + fail-fast startup
|
|
#[cfg(test)]
|
|
mod server_startup_failfast_test;
|
|
|
|
#[cfg(test)]
|
|
mod bucket_policy_check_test;
|
|
|
|
// Security boundary tests: DoS limits, SSRF prevention, concurrent-write integrity
|
|
#[cfg(test)]
|
|
mod security_boundary_test;
|
|
|
|
// Opt-in per-client S3 API rate limiting (backlog#1191)
|
|
#[cfg(test)]
|
|
mod api_rate_limit_test;
|
|
|
|
// Admin authorization gate: non-admin denial + root-credential lifecycle (backlog#1151 sec-4)
|
|
#[cfg(test)]
|
|
mod admin_auth_test;
|
|
|
|
/// IAM / bucket / STS session policy with `s3:ExistingObjectTag` conditions (E2E).
|
|
#[cfg(test)]
|
|
mod existing_object_tag_policy_test;
|
|
|
|
// Regression tests for Issue #2036: anonymous access with PublicAccessBlock
|
|
#[cfg(test)]
|
|
mod anonymous_access_test;
|
|
|
|
// Special characters in path test modules
|
|
#[cfg(test)]
|
|
mod special_chars_test;
|
|
|
|
// Leading/duplicate slash key normalization tests (Issue #2427)
|
|
#[cfg(test)]
|
|
mod leading_slash_key_test;
|
|
|
|
// Content-Encoding header preservation test
|
|
#[cfg(test)]
|
|
mod content_encoding_test;
|
|
|
|
#[cfg(test)]
|
|
mod archive_download_integrity_test;
|
|
|
|
// ListObjectsV2 pagination test (Issue #1596)
|
|
#[cfg(test)]
|
|
mod list_objects_v2_pagination_test;
|
|
|
|
// Regression test for Issue #3107: mc mirror small-bucket listing must not time out.
|
|
#[cfg(test)]
|
|
mod mc_mirror_small_bucket_test;
|
|
|
|
// Policy variables tests
|
|
#[cfg(test)]
|
|
mod policy;
|
|
|
|
#[cfg(test)]
|
|
mod compression_test;
|
|
|
|
// Regression test for Issue #1878: DeleteMarkers not visible immediately after delete_objects
|
|
#[cfg(test)]
|
|
mod delete_objects_versioning_test;
|
|
|
|
// Regression test for signed DELETE Object?versionId requests without Content-Length.
|
|
#[cfg(test)]
|
|
mod delete_object_no_content_length_test;
|
|
|
|
// Delete-marker visibility baseline for data-movement migration proof.
|
|
#[cfg(test)]
|
|
mod delete_marker_migration_semantics_test;
|
|
|
|
// Regression test for Issue #2252: ListObjectVersions misses newest version after put -> delete -> put
|
|
#[cfg(test)]
|
|
mod list_object_versions_regression_test;
|
|
|
|
// versions&metadata=true extension regression test
|
|
#[cfg(test)]
|
|
mod list_object_versions_metadata_extension_test;
|
|
|
|
// list-type=2&metadata=true extension regression test
|
|
#[cfg(test)]
|
|
mod list_objects_v2_metadata_extension_test;
|
|
|
|
#[cfg(test)]
|
|
mod protocols;
|
|
|
|
// Object Lock tests
|
|
#[cfg(test)]
|
|
mod object_lock;
|
|
|
|
#[cfg(test)]
|
|
mod cluster_concurrency_test;
|
|
|
|
// Multi-drive (drivesPerNode) and 2-pool cluster harness smoke tests
|
|
#[cfg(test)]
|
|
mod cluster_multidrive_pool_test;
|
|
|
|
// PutObject / MultipartUpload with checksum (Content-MD5, x-amz-checksum-*)
|
|
#[cfg(test)]
|
|
mod checksum_upload_test;
|
|
|
|
// Group deletion tests
|
|
#[cfg(test)]
|
|
mod group_delete_test;
|
|
|
|
#[cfg(test)]
|
|
mod head_object_range_test;
|
|
|
|
#[cfg(test)]
|
|
mod head_object_consistency_test;
|
|
|
|
#[cfg(test)]
|
|
mod heal_erasure_disk_rebuild_test;
|
|
|
|
#[cfg(test)]
|
|
mod copy_object_metadata_test;
|
|
|
|
#[cfg(test)]
|
|
mod copy_object_version_restore_test;
|
|
|
|
// S3 dummy-compat bucket API tests
|
|
#[cfg(test)]
|
|
mod bucket_logging_test;
|
|
|
|
// Multipart control API auth regression tests
|
|
#[cfg(test)]
|
|
mod multipart_auth_test;
|
|
|
|
// Negative presigned-URL (query-string SigV4) regression suite (backlog#1151
|
|
// sec-2): expired, tampered signature, wrong secret, tampered target.
|
|
#[cfg(test)]
|
|
mod presigned_negative_test;
|
|
|
|
// Negative header-SigV4 regression suite (backlog#1151 sec-1): tampered
|
|
// signature, wrong secret, skewed date, malformed Authorization.
|
|
#[cfg(test)]
|
|
mod negative_sigv4_test;
|
|
|
|
#[cfg(test)]
|
|
mod stale_multipart_cleanup_cluster_test;
|
|
|
|
// Object lambda end-to-end regression tests
|
|
#[cfg(test)]
|
|
mod object_lambda_test;
|
|
|
|
// S3 event-notification webhook delivery end-to-end (backlog#1154 peri-1):
|
|
// configure webhook target -> PutBucketNotificationConfiguration -> object
|
|
// operation -> event delivered, plus filter negatives and store-queue redelivery.
|
|
#[cfg(test)]
|
|
mod notification_webhook_test;
|
|
|
|
// TLS certificate hot-reload live-listener e2e (backlog#1154 peri-5): swap
|
|
// certificates without a restart, existing sessions survive, bad material is
|
|
// fail-safe (old certificate keeps serving, failure is logged).
|
|
#[cfg(test)]
|
|
mod tls_hot_reload_test;
|
|
|
|
// Console listener over-the-wire smoke (backlog#1154 peri-4): public console
|
|
// endpoints answer without credentials or leaks, the SPA prefix never falls
|
|
// through to the S3 API, and the protected surface stays authenticated.
|
|
#[cfg(test)]
|
|
mod console_smoke_test;
|
|
|
|
// Admin IAM management CRUD e2e (backlog#1154 peri-2): user / canned-policy /
|
|
// service-account lifecycle over signed HTTP with data-plane effect assertions,
|
|
// plus non-admin 403 probes per endpoint (sec-4 pattern).
|
|
#[cfg(test)]
|
|
mod admin_iam_crud_test;
|
|
|
|
// Replication extension end-to-end regression tests
|
|
#[cfg(test)]
|
|
mod replication_extension_test;
|
|
|
|
#[cfg(test)]
|
|
mod snowball_auto_extract_test;
|
|
|
|
#[cfg(test)]
|
|
mod namespace_lock_quorum_test;
|
|
|
|
#[cfg(test)]
|
|
mod admin_timeout_regression_test;
|
|
|
|
#[cfg(test)]
|
|
mod overwrite_cleanup_regression_test;
|
|
|
|
// Regression test for backlog#601: `GET //` ListBuckets browser compatibility.
|
|
#[cfg(test)]
|
|
mod list_buckets_double_slash_test;
|
|
|
|
// Regression test for backlog#629(b): region-aware CreateBucket SigV4.
|
|
#[cfg(test)]
|
|
mod create_bucket_region_test;
|
|
|
|
// Regression coverage for backlog#618 item 8: copy-source invalid-date header.
|
|
#[cfg(test)]
|
|
mod copy_source_invalid_date_test;
|
|
|
|
pub mod tls_gen;
|