mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
ci(ilm): add gated s3-tests lane for lifecycle expiration cases (#4715)
* ci(ilm): move first batch of 5 s3-tests lifecycle expiration cases into a gated behavior lane (backlog#1148 ilm-10)
The 20 lifecycle cases in excluded_tests.txt were labeled "vendor-specific"
but the real blocker was the absence of a Ceph lc_debug_interval equivalent.
RUSTFS_ILM_DEBUG_DAY_SECS (ilm-5) now provides that, so Days>=1 expiration
behavior is testable in seconds.
These cases assert that objects/versions/uploads are actually removed by the
background scanner and the stale-multipart cleanup loop. They cannot join the
default single-server s3-implemented-tests gate: it disables the scanner, and a
global RUSTFS_ILM_DEBUG_DAY_SECS also shrinks the x-amz-expiration header that
the already-passing test_lifecycle_expiration_header_* cases assert on. So this
adds an isolated lane instead of putting them in implemented_tests.txt.
First batch (5), each verified against the pinned upstream source and the
RustFS evaluator/scanner/stale-multipart execution paths:
- test_lifecycle_expiration (prefix Days=1/Days=5, scanner delete)
- test_lifecyclev2_expiration (same via ListObjectsV2)
- test_lifecycle_noncur_expiration (NoncurrentVersionExpiration)
- test_lifecycle_deletemarker_expiration (ExpiredObjectDeleteMarker cascade)
- test_lifecycle_multipart_expiration (AbortIncompleteMultipartUpload)
Dropped from the batch, kept excluded with reason:
- test_lifecycle_expiration_days0: RustFS accepts Expiration{Days:0} (returns
200; only Days<0 is rejected in crates/lifecycle/src/core.rs validate()),
while AWS/this test expect InvalidArgument. Real validation gap.
Changes:
- scripts/s3-tests/lifecycle_behavior_tests.txt: new exact-node-id run set.
- scripts/s3-tests/run.sh: honor an IMPLEMENTED_TESTS_FILE override so a lane
can point at an alternate whitelist.
- .github/workflows/ci.yml: new s3-lifecycle-behavior-tests PR-gate job that
starts rustfs with RUSTFS_ILM_DEBUG_DAY_SECS=10, scanner enabled (cycle 2s,
no start delay) and a 2s stale-multipart cleanup interval, running the new
list serially.
- scripts/s3-tests/report_compat.py: recognize the behavior list so the weekly
scope=all sweep (plain server) does not misclassify expected failures.
- scripts/s3-tests/excluded_tests.txt: remove the 5 enabled cases; re-annotate
the remaining 15 lifecycle exclusions with real reasons + batch-2 plan.
- docs/architecture/s3-compatibility-matrix.md: sync counts (implemented 451,
excluded 274, behavior 5) and document the lane.
Refs backlog#1148 (ilm-10), master plan backlog#1155.
* fix(lifecycle): reject zero-day expiration/noncurrent/abort rules (backlog#1148 ilm-10) (#4722)
This commit is contained in:
@@ -495,3 +495,61 @@ jobs:
|
||||
path: artifacts/s3tests-single/**
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
# Dedicated lifecycle behavior lane (backlog#1148 ilm-10, master plan #1155).
|
||||
#
|
||||
# These ceph/s3-tests cases assert that objects/versions/uploads are actually
|
||||
# removed by the background scanner and the stale-multipart cleanup loop, so
|
||||
# they need a debug-accelerated day plus an enabled scanner. That configuration
|
||||
# cannot be applied to the s3-implemented-tests lane above because a global
|
||||
# RUSTFS_ILM_DEBUG_DAY_SECS also shrinks the x-amz-expiration header those tests
|
||||
# assert on (test_lifecycle_expiration_header_*). Hence a separate server here.
|
||||
s3-lifecycle-behavior-tests:
|
||||
name: S3 Lifecycle Behavior Tests
|
||||
needs: [ build-rustfs-debug-binary ]
|
||||
runs-on: sm-standard-4
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||
|
||||
- name: Download debug binary
|
||||
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||
with:
|
||||
name: rustfs-debug-binary
|
||||
path: target/debug
|
||||
|
||||
- name: Make binary executable
|
||||
run: chmod +x ./target/debug/rustfs
|
||||
|
||||
- name: Run lifecycle behavior s3-tests
|
||||
run: |
|
||||
RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
|
||||
mkdir -p "${RUN_ROOT}"
|
||||
S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
|
||||
# Serial (XDIST=0) for timing determinism: the cases sleep on multiples
|
||||
# of the debug day and assert scanner-driven deletion.
|
||||
DEPLOY_MODE=binary \
|
||||
RUSTFS_BINARY=./target/debug/rustfs \
|
||||
TEST_MODE=single \
|
||||
MAXFAIL=0 \
|
||||
XDIST=0 \
|
||||
IMPLEMENTED_TESTS_FILE=scripts/s3-tests/lifecycle_behavior_tests.txt \
|
||||
RUSTFS_ILM_DEBUG_DAY_SECS=10 \
|
||||
RUSTFS_SCANNER_ENABLED=true \
|
||||
RUSTFS_SCANNER_CYCLE=2 \
|
||||
RUSTFS_SCANNER_START_DELAY_SECS=0 \
|
||||
RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s \
|
||||
S3_PORT="${S3_PORT}" \
|
||||
DATA_ROOT="${RUN_ROOT}" \
|
||||
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
|
||||
./scripts/s3-tests/run.sh
|
||||
|
||||
- name: Upload s3 test artifacts
|
||||
if: always()
|
||||
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||
with:
|
||||
name: s3tests-lifecycle-behavior-${{ github.run_number }}
|
||||
path: artifacts/s3tests-single/**
|
||||
if-no-files-found: ignore
|
||||
retention-days: 3
|
||||
|
||||
@@ -160,12 +160,16 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
#[serial]
|
||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||
async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
||||
async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
||||
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
|
||||
|
||||
let client = create_aws_s3_client().await?;
|
||||
setup_test_bucket(&client).await?;
|
||||
|
||||
// S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and the
|
||||
// ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with an
|
||||
// InvalidArgument (HTTP 400). See crates/lifecycle validate() + the
|
||||
// PutBucketLifecycleConfiguration handler which maps it to s3_error!(InvalidArgument).
|
||||
let expiration = LifecycleExpiration::builder().days(0).build();
|
||||
let filter = LifecycleRuleFilter::builder().prefix("zero-days/").build();
|
||||
let rule = LifecycleRule::builder()
|
||||
@@ -176,12 +180,20 @@ async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::er
|
||||
.build()?;
|
||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||
|
||||
client
|
||||
let err = client
|
||||
.put_bucket_lifecycle_configuration()
|
||||
.bucket(BUCKET)
|
||||
.lifecycle_configuration(lifecycle)
|
||||
.send()
|
||||
.await?;
|
||||
.await
|
||||
.expect_err("zero-day expiration lifecycle rule should be rejected");
|
||||
|
||||
let service_error = err.as_service_error().expect("expected an S3 service error");
|
||||
assert_eq!(
|
||||
service_error.meta().code(),
|
||||
Some("InvalidArgument"),
|
||||
"expected InvalidArgument for zero-day expiration, got: {err:?}"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -40,10 +40,11 @@ const _ERR_XML_NOT_WELL_FORMED: &str =
|
||||
const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
|
||||
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
|
||||
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
|
||||
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must not be negative";
|
||||
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str = "Lifecycle noncurrent expiration days must not be negative";
|
||||
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
|
||||
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
|
||||
"'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
|
||||
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
|
||||
"DaysAfterInitiation must be 0 or greater when used with AbortIncompleteMultipartUpload";
|
||||
"'DaysAfterInitiation' for AbortIncompleteMultipartUpload action must be a positive integer";
|
||||
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
|
||||
const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER: &str =
|
||||
"ExpiredObjectDeleteMarker cannot be specified with Days or Date";
|
||||
@@ -325,21 +326,29 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT));
|
||||
}
|
||||
}
|
||||
// S3 requires Expiration.Days to be a positive integer (>= 1). AWS and the
|
||||
// ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with
|
||||
// InvalidArgument; only Date-based rules may omit Days entirely.
|
||||
if let Some(days) = expiration.days
|
||||
&& days < 0
|
||||
&& days < 1
|
||||
{
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS));
|
||||
}
|
||||
}
|
||||
// S3 requires NoncurrentVersionExpiration.NoncurrentDays to be a positive
|
||||
// integer (>= 1); AWS rejects 0 with InvalidArgument.
|
||||
if let Some(noncurrent_version_expiration) = &r.noncurrent_version_expiration
|
||||
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
|
||||
&& noncurrent_days < 0
|
||||
&& noncurrent_days < 1
|
||||
{
|
||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS));
|
||||
}
|
||||
// S3 requires AbortIncompleteMultipartUpload.DaysAfterInitiation to be present
|
||||
// and a positive integer (>= 1); AWS rejects 0 (and a missing value) with
|
||||
// InvalidArgument.
|
||||
if let Some(abort_incomplete_multipart_upload) = &r.abort_incomplete_multipart_upload {
|
||||
match abort_incomplete_multipart_upload.days_after_initiation {
|
||||
Some(days) if days >= 0 => {}
|
||||
Some(days) if days >= 1 => {}
|
||||
_ => return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS)),
|
||||
}
|
||||
}
|
||||
@@ -1066,7 +1075,11 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validate_accepts_zero_expiration_days() {
|
||||
async fn validate_rejects_zero_expiration_days() {
|
||||
// S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and
|
||||
// the ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with
|
||||
// InvalidArgument. The PutBucketLifecycleConfiguration path maps this io::Error
|
||||
// to s3_error!(InvalidArgument) (see execute_put_bucket_lifecycle_configuration).
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
@@ -1086,9 +1099,12 @@ mod tests {
|
||||
}],
|
||||
};
|
||||
|
||||
lc.validate(&ObjectLockConfiguration::default())
|
||||
let err = lc
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect("zero-day expiration should be accepted");
|
||||
.expect_err("zero-day expiration should be rejected");
|
||||
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1148,6 +1164,41 @@ mod tests {
|
||||
.expect("expected validation to pass");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validate_accepts_one_day_boundary_values() {
|
||||
// Pin the exact >= 1 boundary: a value of 1 is the smallest legal positive
|
||||
// integer and must be accepted for every day-count field tightened for S3
|
||||
// compatibility. This catches an off-by-one that would reject Days == 1.
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(1),
|
||||
..Default::default()
|
||||
}),
|
||||
abort_incomplete_multipart_upload: Some(s3s::dto::AbortIncompleteMultipartUpload {
|
||||
days_after_initiation: Some(1),
|
||||
}),
|
||||
del_marker_expiration: None,
|
||||
filter: None,
|
||||
id: Some("one-day-boundary".to_string()),
|
||||
noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration {
|
||||
noncurrent_days: Some(1),
|
||||
newer_noncurrent_versions: None,
|
||||
}),
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: Some("boundary/".to_string()),
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
|
||||
lc.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect("one-day boundary values should be accepted");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn has_active_rules_accepts_zero_day_expiration() {
|
||||
@@ -1175,7 +1226,9 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validate_accepts_zero_noncurrent_expiration_days() {
|
||||
async fn validate_rejects_zero_noncurrent_expiration_days() {
|
||||
// S3 compatibility: NoncurrentVersionExpiration.NoncurrentDays must be a positive
|
||||
// integer (>= 1); AWS rejects 0 with InvalidArgument.
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
@@ -1195,9 +1248,12 @@ mod tests {
|
||||
}],
|
||||
};
|
||||
|
||||
lc.validate(&ObjectLockConfiguration::default())
|
||||
let err = lc
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect("zero-day noncurrent expiration should be accepted");
|
||||
.expect_err("zero-day noncurrent expiration should be rejected");
|
||||
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1258,7 +1314,9 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn validate_accepts_zero_abort_incomplete_multipart_upload_days() {
|
||||
async fn validate_rejects_zero_abort_incomplete_multipart_upload_days() {
|
||||
// S3 compatibility: AbortIncompleteMultipartUpload.DaysAfterInitiation must be a
|
||||
// positive integer (>= 1); AWS rejects 0 with InvalidArgument.
|
||||
let lc = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
@@ -1277,9 +1335,12 @@ mod tests {
|
||||
}],
|
||||
};
|
||||
|
||||
lc.validate(&ObjectLockConfiguration::default())
|
||||
let err = lc
|
||||
.validate(&ObjectLockConfiguration::default())
|
||||
.await
|
||||
.expect("zero-day abort incomplete multipart upload should be accepted");
|
||||
.expect_err("zero-day abort incomplete multipart upload should be rejected");
|
||||
|
||||
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -15,12 +15,21 @@ features are covered by the compatibility matrix and test lists.
|
||||
|
||||
| List | Purpose | Current count | Source |
|
||||
|---|---:|---:|---|
|
||||
| Implemented tests | Standard S3 tests expected to pass and used by the default local s3tests run. | 449 | `scripts/s3-tests/implemented_tests.txt` |
|
||||
| Implemented tests | Standard S3 tests expected to pass and used by the default local s3tests run. | 452 | `scripts/s3-tests/implemented_tests.txt` |
|
||||
| Lifecycle behavior tests | Expiration behavior cases gated by the dedicated `s3-lifecycle-behavior-tests` lane (debug-accelerated day + scanner enabled). | 5 | `scripts/s3-tests/lifecycle_behavior_tests.txt` |
|
||||
| Unimplemented tests | Standard S3 features planned but not yet implemented. | 17 | `scripts/s3-tests/unimplemented_tests.txt` |
|
||||
| Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS compatibility gating. | 281 | `scripts/s3-tests/excluded_tests.txt` |
|
||||
| Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS compatibility gating. | 273 | `scripts/s3-tests/excluded_tests.txt` |
|
||||
|
||||
Counts ignore blank lines and comments.
|
||||
|
||||
The lifecycle behavior lane runs real Days-based expiration cases that need
|
||||
`RUSTFS_ILM_DEBUG_DAY_SECS` (Ceph `lc_debug_interval` equivalent) and an enabled
|
||||
background scanner; it cannot share the default single-server gate because a
|
||||
global debug day would also shrink the `x-amz-expiration` header asserted by the
|
||||
`test_lifecycle_expiration_header_*` cases. See `scripts/s3-tests/run.sh`
|
||||
(`IMPLEMENTED_TESTS_FILE` override) and the `s3-lifecycle-behavior-tests` job in
|
||||
`.github/workflows/ci.yml`.
|
||||
|
||||
## Supported Coverage
|
||||
|
||||
The implemented test list currently covers the common object-storage surface:
|
||||
|
||||
@@ -1567,3 +1567,60 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
|
||||
"unexpected error message: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||
#[serial]
|
||||
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||
async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
|
||||
// S3 compatibility (ceph s3-tests `test_lifecycle_expiration_days0`): a rule with
|
||||
// Expiration{Days:0} must be rejected with InvalidArgument through the real
|
||||
// PutBucketLifecycleConfiguration handler, not merely at the crates/lifecycle
|
||||
// io::Error layer. No object lock required.
|
||||
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||
let usecase = DefaultBucketUsecase::from_global();
|
||||
|
||||
let bucket = format!("test-zero-day-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||
|
||||
let lifecycle = BucketLifecycleConfiguration {
|
||||
expiry_updated_at: None,
|
||||
rules: vec![LifecycleRule {
|
||||
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||
abort_incomplete_multipart_upload: None,
|
||||
del_marker_expiration: None,
|
||||
expiration: Some(LifecycleExpiration {
|
||||
days: Some(0),
|
||||
..Default::default()
|
||||
}),
|
||||
filter: Some(LifecycleRuleFilter {
|
||||
prefix: Some("days0/".to_string()),
|
||||
..Default::default()
|
||||
}),
|
||||
id: Some("expire-zero-days".to_string()),
|
||||
noncurrent_version_expiration: None,
|
||||
noncurrent_version_transitions: None,
|
||||
prefix: None,
|
||||
transitions: None,
|
||||
}],
|
||||
};
|
||||
|
||||
let req = build_request(
|
||||
PutBucketLifecycleConfigurationInput::builder()
|
||||
.bucket(bucket.clone())
|
||||
.lifecycle_configuration(Some(lifecycle))
|
||||
.build()
|
||||
.unwrap(),
|
||||
Method::PUT,
|
||||
);
|
||||
|
||||
let err = usecase
|
||||
.execute_put_bucket_lifecycle_configuration(req)
|
||||
.await
|
||||
.expect_err("zero-day expiration must be rejected with InvalidArgument");
|
||||
assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
|
||||
let message = err.message().unwrap_or_default();
|
||||
assert!(
|
||||
message.contains("'Days' for Expiration action must be a positive integer"),
|
||||
"unexpected error message: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -157,26 +157,47 @@ test_encryption_sse_c_multipart_invalid_chunks_1
|
||||
test_encryption_sse_c_multipart_invalid_chunks_2
|
||||
test_get_multipart_checksum_object_attributes
|
||||
test_head_bucket_usage
|
||||
test_lifecycle_cloud_multiple_transition
|
||||
test_lifecycle_cloud_transition
|
||||
test_lifecycle_cloud_transition_large_obj
|
||||
test_lifecycle_deletemarker_expiration
|
||||
# -----------------------------------------------------------------------------
|
||||
# Lifecycle behavior (backlog#1148 ilm-10, master plan #1155)
|
||||
#
|
||||
# The upstream "Vendor-specific" label on these was inaccurate: the real blocker
|
||||
# was the absence of a Ceph `lc_debug_interval` equivalent, so Days>=1 expiration
|
||||
# behavior could not be exercised in seconds. RUSTFS_ILM_DEBUG_DAY_SECS (ilm-5)
|
||||
# now provides that. The first batch of expiration tests has been moved into the
|
||||
# dedicated behavior lane: scripts/s3-tests/lifecycle_behavior_tests.txt
|
||||
# (run by the ci.yml job `s3-lifecycle-behavior-tests`). The entries below are
|
||||
# NOT vendor-specific — they stay excluded from the DEFAULT single-server gate
|
||||
# for the specific reasons noted, and are queued for later batches.
|
||||
#
|
||||
# --- Batch 2 candidates: expiration with tag / size filters + noncurrent counts.
|
||||
# These need the same debug/scanner lane as batch 1 plus tag/size-filter and
|
||||
# NewerNoncurrentVersions coverage (see ilm-9). Queued for ilm-10 batch 2.
|
||||
test_lifecycle_deletemarker_expiration_with_days_tag
|
||||
test_lifecycle_expiration
|
||||
test_lifecycle_expiration_days0
|
||||
test_lifecycle_expiration_newer_noncurrent
|
||||
test_lifecycle_expiration_noncur_tags1
|
||||
test_lifecycle_expiration_size_gt
|
||||
test_lifecycle_expiration_size_lt
|
||||
test_lifecycle_expiration_tags2
|
||||
test_lifecycle_expiration_versioned_tags2
|
||||
test_lifecycle_multipart_expiration
|
||||
test_lifecycle_noncur_cloud_transition
|
||||
test_lifecycle_noncur_expiration
|
||||
#
|
||||
# --- Storage-class transition to a remote/warm tier. Needs a real tier backend
|
||||
# (the hermetic second-RustFS cold layer from ilm-7) which is not wired into
|
||||
# the s3-tests topology. Queued behind ilm-7.
|
||||
test_lifecycle_noncur_transition
|
||||
test_lifecycle_transition
|
||||
#
|
||||
# --- Single-rule multi-level Transition (hot->warm->cold). RustFS only honors
|
||||
# transitions.first() (core.rs:676); multi-level semantics are undecided and
|
||||
# tracked as a product item (ex-ilm-20). Do not enable until that lands.
|
||||
test_lifecycle_transition_single_rule_multi_trans
|
||||
test_lifecyclev2_expiration
|
||||
#
|
||||
# --- Cloud (S3/remote) transition + restore. Requires the [s3 cloud] endpoint
|
||||
# the RustFS SNSD harness does not provision; depends on ilm-7 tier carrier.
|
||||
test_lifecycle_cloud_multiple_transition
|
||||
test_lifecycle_cloud_transition
|
||||
test_lifecycle_cloud_transition_large_obj
|
||||
test_lifecycle_noncur_cloud_transition
|
||||
# -----------------------------------------------------------------------------
|
||||
test_list_buckets_anonymous
|
||||
test_list_buckets_paginated
|
||||
test_list_multipart_upload
|
||||
|
||||
@@ -225,6 +225,8 @@ test_delete_bucket_encryption_kms
|
||||
test_delete_bucket_encryption_s3
|
||||
# Lifecycle tests
|
||||
test_lifecycle_delete
|
||||
# Negative validation: Expiration{Days:0} must return InvalidArgument (backlog#1148 ilm-10).
|
||||
test_lifecycle_expiration_days0
|
||||
test_lifecycle_get
|
||||
test_lifecycle_get_no_id
|
||||
test_lifecycle_id_too_long
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# Lifecycle behavior tests (ceph/s3-tests) — backlog#1148 ilm-10, master plan #1155
|
||||
# =================================================================================
|
||||
#
|
||||
# These upstream lifecycle tests exercise *real expiration behavior*: they PUT a
|
||||
# lifecycle rule with Days/NoncurrentDays/DaysAfterInitiation, sleep a multiple of
|
||||
# the debug interval, then assert objects/versions/uploads were actually removed by
|
||||
# the background scanner and the stale-multipart cleanup loop.
|
||||
#
|
||||
# They CANNOT run in the default single-server gate (implemented_tests.txt) because:
|
||||
# 1. The default harness disables the background scanner (run.sh sets
|
||||
# RUSTFS_SCANNER_ENABLED=false) — nothing expires.
|
||||
# 2. A real day is 86400s; without acceleration a Days=1 rule takes 24h.
|
||||
# 3. Enabling RUSTFS_ILM_DEBUG_DAY_SECS *globally* would break the already-passing
|
||||
# x-amz-expiration header tests (test_lifecycle_expiration_header_*), which
|
||||
# assert `(expiry-date - now).days == Days` — the header is computed from the
|
||||
# same debug-scaled expected_expiry_time().
|
||||
#
|
||||
# So this list is run by a DEDICATED lane (ci.yml job `s3-lifecycle-behavior-tests`)
|
||||
# against a server started with:
|
||||
# RUSTFS_ILM_DEBUG_DAY_SECS=10 (1 "day" = 10s, matches s3-tests lc_debug_interval)
|
||||
# RUSTFS_SCANNER_ENABLED=true
|
||||
# RUSTFS_SCANNER_CYCLE=2 (scan every 2s, well under a debug-day)
|
||||
# RUSTFS_SCANNER_START_DELAY_SECS=0
|
||||
# RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s (multipart abort lane)
|
||||
#
|
||||
# Point run.sh at this file with IMPLEMENTED_TESTS_FILE=<this path>.
|
||||
#
|
||||
# First batch (ilm-10). Each entry was verified against the pinned upstream source
|
||||
# and the RustFS evaluator/scanner/stale-multipart execution paths.
|
||||
|
||||
# Basic prefix expiration: Days=1 / Days=5 rules, scanner-driven deletion.
|
||||
test_lifecycle_expiration
|
||||
# Same as above but validates ListObjectsV2 sees the expirations.
|
||||
test_lifecyclev2_expiration
|
||||
# Versioned NoncurrentVersionExpiration (NoncurrentDays): noncurrent versions removed.
|
||||
test_lifecycle_noncur_expiration
|
||||
# ExpiredObjectDeleteMarker + NoncurrentVersionExpiration cascade on a versioned bucket.
|
||||
test_lifecycle_deletemarker_expiration
|
||||
# AbortIncompleteMultipartUpload (DaysAfterInitiation): stale-multipart cleanup loop.
|
||||
test_lifecycle_multipart_expiration
|
||||
@@ -36,6 +36,12 @@ LIST_FILES = {
|
||||
"implemented": "implemented_tests.txt",
|
||||
"unimplemented": "unimplemented_tests.txt",
|
||||
"excluded": "excluded_tests.txt",
|
||||
# Lifecycle behavior lane (backlog#1148 ilm-10): gated by its own ci.yml job
|
||||
# against a debug-accelerated server. In the full `scope=all` sweep these run
|
||||
# against the plain server (no scanner / no RUSTFS_ILM_DEBUG_DAY_SECS) and are
|
||||
# EXPECTED to fail there, so a failure here is neither a regression nor an
|
||||
# unclassified failure.
|
||||
"behavior": "lifecycle_behavior_tests.txt",
|
||||
}
|
||||
|
||||
|
||||
@@ -122,6 +128,7 @@ def main() -> int:
|
||||
promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
|
||||
unclassified_passed: list[str] = []
|
||||
unclassified_failed: list[str] = []
|
||||
behavior_passed: list[str] = []
|
||||
counts = {"passed": 0, "failed": 0, "error": 0, "skipped": 0}
|
||||
|
||||
for name, status in results.items():
|
||||
@@ -129,13 +136,19 @@ def main() -> int:
|
||||
if status in ("failed", "error"):
|
||||
if name in lists["implemented"]:
|
||||
regressions.append(name)
|
||||
elif name not in lists["unimplemented"] and name not in lists["excluded"]:
|
||||
elif (
|
||||
name not in lists["unimplemented"]
|
||||
and name not in lists["excluded"]
|
||||
and name not in lists["behavior"]
|
||||
):
|
||||
unclassified_failed.append(name)
|
||||
elif status == "passed":
|
||||
if name in lists["unimplemented"]:
|
||||
promotions["unimplemented"].append(name)
|
||||
elif name in lists["excluded"]:
|
||||
promotions["excluded"].append(name)
|
||||
elif name in lists["behavior"]:
|
||||
behavior_passed.append(name)
|
||||
elif name not in lists["implemented"]:
|
||||
unclassified_passed.append(name)
|
||||
|
||||
@@ -162,6 +175,12 @@ def main() -> int:
|
||||
promotions["excluded"],
|
||||
"Passing despite being excluded — re-evaluate the exclusion.",
|
||||
)
|
||||
lines += render_section(
|
||||
"Lifecycle behavior lane (passing in this run)",
|
||||
behavior_passed,
|
||||
"Gated by the dedicated `s3-lifecycle-behavior-tests` lane; expected to "
|
||||
"fail in the plain `scope=all` sweep (no debug day / scanner).",
|
||||
)
|
||||
lines += render_section(
|
||||
"Unclassified passes",
|
||||
unclassified_passed,
|
||||
|
||||
@@ -159,7 +159,10 @@ PY
|
||||
|
||||
# Test list files location
|
||||
TEST_LISTS_DIR="${SCRIPT_DIR}"
|
||||
IMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/implemented_tests.txt"
|
||||
# IMPLEMENTED_TESTS_FILE may be overridden to run an alternate exact-node-id
|
||||
# whitelist (e.g. lifecycle_behavior_tests.txt for the dedicated lifecycle
|
||||
# behavior lane, which needs RUSTFS_ILM_DEBUG_DAY_SECS + the scanner enabled).
|
||||
IMPLEMENTED_TESTS_FILE="${IMPLEMENTED_TESTS_FILE:-${TEST_LISTS_DIR}/implemented_tests.txt}"
|
||||
UNIMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/unimplemented_tests.txt"
|
||||
EXCLUDED_TESTS_FILE="${TEST_LISTS_DIR}/excluded_tests.txt"
|
||||
S3_TEST_FILE="s3tests/functional/test_s3.py"
|
||||
|
||||
Reference in New Issue
Block a user