ci(ilm): add gated s3-tests lane for lifecycle expiration cases (#4715)

* ci(ilm): move first batch of 5 s3-tests lifecycle expiration cases into a gated behavior lane (backlog#1148 ilm-10)

The 20 lifecycle cases in excluded_tests.txt were labeled "vendor-specific"
but the real blocker was the absence of a Ceph lc_debug_interval equivalent.
RUSTFS_ILM_DEBUG_DAY_SECS (ilm-5) now provides that, so Days>=1 expiration
behavior is testable in seconds.

These cases assert that objects/versions/uploads are actually removed by the
background scanner and the stale-multipart cleanup loop. They cannot join the
default single-server s3-implemented-tests gate: it disables the scanner, and a
global RUSTFS_ILM_DEBUG_DAY_SECS also shrinks the x-amz-expiration header that
the already-passing test_lifecycle_expiration_header_* cases assert on. So this
adds an isolated lane instead of putting them in implemented_tests.txt.

First batch (5), each verified against the pinned upstream source and the
RustFS evaluator/scanner/stale-multipart execution paths:
  - test_lifecycle_expiration          (prefix Days=1/Days=5, scanner delete)
  - test_lifecyclev2_expiration        (same via ListObjectsV2)
  - test_lifecycle_noncur_expiration   (NoncurrentVersionExpiration)
  - test_lifecycle_deletemarker_expiration (ExpiredObjectDeleteMarker cascade)
  - test_lifecycle_multipart_expiration    (AbortIncompleteMultipartUpload)

Dropped from the batch, kept excluded with reason:
  - test_lifecycle_expiration_days0: RustFS accepts Expiration{Days:0} (returns
    200; only Days<0 is rejected in crates/lifecycle/src/core.rs validate()),
    while AWS/this test expect InvalidArgument. Real validation gap.

Changes:
  - scripts/s3-tests/lifecycle_behavior_tests.txt: new exact-node-id run set.
  - scripts/s3-tests/run.sh: honor an IMPLEMENTED_TESTS_FILE override so a lane
    can point at an alternate whitelist.
  - .github/workflows/ci.yml: new s3-lifecycle-behavior-tests PR-gate job that
    starts rustfs with RUSTFS_ILM_DEBUG_DAY_SECS=10, scanner enabled (cycle 2s,
    no start delay) and a 2s stale-multipart cleanup interval, running the new
    list serially.
  - scripts/s3-tests/report_compat.py: recognize the behavior list so the weekly
    scope=all sweep (plain server) does not misclassify expected failures.
  - scripts/s3-tests/excluded_tests.txt: remove the 5 enabled cases; re-annotate
    the remaining 15 lifecycle exclusions with real reasons + batch-2 plan.
  - docs/architecture/s3-compatibility-matrix.md: sync counts (implemented 451,
    excluded 274, behavior 5) and document the lane.

Refs backlog#1148 (ilm-10), master plan backlog#1155.

* fix(lifecycle): reject zero-day expiration/noncurrent/abort rules (backlog#1148 ilm-10) (#4722)
This commit is contained in:
Zhengchao An
2026-07-11 13:16:55 +08:00
committed by GitHub
parent 4de73c0653
commit 4b83efaf36
10 changed files with 314 additions and 32 deletions
+58
View File
@@ -495,3 +495,61 @@ jobs:
path: artifacts/s3tests-single/** path: artifacts/s3tests-single/**
if-no-files-found: ignore if-no-files-found: ignore
retention-days: 3 retention-days: 3
# Dedicated lifecycle behavior lane (backlog#1148 ilm-10, master plan #1155).
#
# These ceph/s3-tests cases assert that objects/versions/uploads are actually
# removed by the background scanner and the stale-multipart cleanup loop, so
# they need a debug-accelerated day plus an enabled scanner. That configuration
# cannot be applied to the s3-implemented-tests lane above because a global
# RUSTFS_ILM_DEBUG_DAY_SECS also shrinks the x-amz-expiration header those tests
# assert on (test_lifecycle_expiration_header_*). Hence a separate server here.
s3-lifecycle-behavior-tests:
name: S3 Lifecycle Behavior Tests
needs: [ build-rustfs-debug-binary ]
runs-on: sm-standard-4
timeout-minutes: 30
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
- name: Download debug binary
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
with:
name: rustfs-debug-binary
path: target/debug
- name: Make binary executable
run: chmod +x ./target/debug/rustfs
- name: Run lifecycle behavior s3-tests
run: |
RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
mkdir -p "${RUN_ROOT}"
S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
# Serial (XDIST=0) for timing determinism: the cases sleep on multiples
# of the debug day and assert scanner-driven deletion.
DEPLOY_MODE=binary \
RUSTFS_BINARY=./target/debug/rustfs \
TEST_MODE=single \
MAXFAIL=0 \
XDIST=0 \
IMPLEMENTED_TESTS_FILE=scripts/s3-tests/lifecycle_behavior_tests.txt \
RUSTFS_ILM_DEBUG_DAY_SECS=10 \
RUSTFS_SCANNER_ENABLED=true \
RUSTFS_SCANNER_CYCLE=2 \
RUSTFS_SCANNER_START_DELAY_SECS=0 \
RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s \
S3_PORT="${S3_PORT}" \
DATA_ROOT="${RUN_ROOT}" \
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
./scripts/s3-tests/run.sh
- name: Upload s3 test artifacts
if: always()
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
with:
name: s3tests-lifecycle-behavior-${{ github.run_number }}
path: artifacts/s3tests-single/**
if-no-files-found: ignore
retention-days: 3
+15 -3
View File
@@ -160,12 +160,16 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
#[tokio::test(flavor = "multi_thread", worker_threads = 2)] #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
#[serial] #[serial]
#[ignore = "requires running RustFS server at localhost:9000"] #[ignore = "requires running RustFS server at localhost:9000"]
async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> { async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter}; use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
let client = create_aws_s3_client().await?; let client = create_aws_s3_client().await?;
setup_test_bucket(&client).await?; setup_test_bucket(&client).await?;
// S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and the
// ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with an
// InvalidArgument (HTTP 400). See crates/lifecycle validate() + the
// PutBucketLifecycleConfiguration handler which maps it to s3_error!(InvalidArgument).
let expiration = LifecycleExpiration::builder().days(0).build(); let expiration = LifecycleExpiration::builder().days(0).build();
let filter = LifecycleRuleFilter::builder().prefix("zero-days/").build(); let filter = LifecycleRuleFilter::builder().prefix("zero-days/").build();
let rule = LifecycleRule::builder() let rule = LifecycleRule::builder()
@@ -176,12 +180,20 @@ async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::er
.build()?; .build()?;
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?; let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
client let err = client
.put_bucket_lifecycle_configuration() .put_bucket_lifecycle_configuration()
.bucket(BUCKET) .bucket(BUCKET)
.lifecycle_configuration(lifecycle) .lifecycle_configuration(lifecycle)
.send() .send()
.await?; .await
.expect_err("zero-day expiration lifecycle rule should be rejected");
let service_error = err.as_service_error().expect("expected an S3 service error");
assert_eq!(
service_error.meta().code(),
Some("InvalidArgument"),
"expected InvalidArgument for zero-day expiration, got: {err:?}"
);
Ok(()) Ok(())
} }
+76 -15
View File
@@ -40,10 +40,11 @@ const _ERR_XML_NOT_WELL_FORMED: &str =
const ERR_LIFECYCLE_BUCKET_LOCKED: &str = const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket"; "ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules"; const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must not be negative"; const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str = "Lifecycle noncurrent expiration days must not be negative"; const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
"'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str = const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
"DaysAfterInitiation must be 0 or greater when used with AbortIncompleteMultipartUpload"; "'DaysAfterInitiation' for AbortIncompleteMultipartUpload action must be a positive integer";
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC"; const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER: &str = const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER: &str =
"ExpiredObjectDeleteMarker cannot be specified with Days or Date"; "ExpiredObjectDeleteMarker cannot be specified with Days or Date";
@@ -325,21 +326,29 @@ impl Lifecycle for BucketLifecycleConfiguration {
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT)); return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT));
} }
} }
// S3 requires Expiration.Days to be a positive integer (>= 1). AWS and the
// ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with
// InvalidArgument; only Date-based rules may omit Days entirely.
if let Some(days) = expiration.days if let Some(days) = expiration.days
&& days < 0 && days < 1
{ {
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS)); return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS));
} }
} }
// S3 requires NoncurrentVersionExpiration.NoncurrentDays to be a positive
// integer (>= 1); AWS rejects 0 with InvalidArgument.
if let Some(noncurrent_version_expiration) = &r.noncurrent_version_expiration if let Some(noncurrent_version_expiration) = &r.noncurrent_version_expiration
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days && let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
&& noncurrent_days < 0 && noncurrent_days < 1
{ {
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS)); return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS));
} }
// S3 requires AbortIncompleteMultipartUpload.DaysAfterInitiation to be present
// and a positive integer (>= 1); AWS rejects 0 (and a missing value) with
// InvalidArgument.
if let Some(abort_incomplete_multipart_upload) = &r.abort_incomplete_multipart_upload { if let Some(abort_incomplete_multipart_upload) = &r.abort_incomplete_multipart_upload {
match abort_incomplete_multipart_upload.days_after_initiation { match abort_incomplete_multipart_upload.days_after_initiation {
Some(days) if days >= 0 => {} Some(days) if days >= 1 => {}
_ => return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS)), _ => return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS)),
} }
} }
@@ -1066,7 +1075,11 @@ mod tests {
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn validate_accepts_zero_expiration_days() { async fn validate_rejects_zero_expiration_days() {
// S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and
// the ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with
// InvalidArgument. The PutBucketLifecycleConfiguration path maps this io::Error
// to s3_error!(InvalidArgument) (see execute_put_bucket_lifecycle_configuration).
let lc = BucketLifecycleConfiguration { let lc = BucketLifecycleConfiguration {
expiry_updated_at: None, expiry_updated_at: None,
rules: vec![LifecycleRule { rules: vec![LifecycleRule {
@@ -1086,9 +1099,12 @@ mod tests {
}], }],
}; };
lc.validate(&ObjectLockConfiguration::default()) let err = lc
.validate(&ObjectLockConfiguration::default())
.await .await
.expect("zero-day expiration should be accepted"); .expect_err("zero-day expiration should be rejected");
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS);
} }
#[tokio::test] #[tokio::test]
@@ -1148,6 +1164,41 @@ mod tests {
.expect("expected validation to pass"); .expect("expected validation to pass");
} }
#[tokio::test]
#[serial]
async fn validate_accepts_one_day_boundary_values() {
// Pin the exact >= 1 boundary: a value of 1 is the smallest legal positive
// integer and must be accepted for every day-count field tightened for S3
// compatibility. This catches an off-by-one that would reject Days == 1.
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
expiration: Some(LifecycleExpiration {
days: Some(1),
..Default::default()
}),
abort_incomplete_multipart_upload: Some(s3s::dto::AbortIncompleteMultipartUpload {
days_after_initiation: Some(1),
}),
del_marker_expiration: None,
filter: None,
id: Some("one-day-boundary".to_string()),
noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration {
noncurrent_days: Some(1),
newer_noncurrent_versions: None,
}),
noncurrent_version_transitions: None,
prefix: Some("boundary/".to_string()),
transitions: None,
}],
};
lc.validate(&ObjectLockConfiguration::default())
.await
.expect("one-day boundary values should be accepted");
}
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn has_active_rules_accepts_zero_day_expiration() { async fn has_active_rules_accepts_zero_day_expiration() {
@@ -1175,7 +1226,9 @@ mod tests {
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn validate_accepts_zero_noncurrent_expiration_days() { async fn validate_rejects_zero_noncurrent_expiration_days() {
// S3 compatibility: NoncurrentVersionExpiration.NoncurrentDays must be a positive
// integer (>= 1); AWS rejects 0 with InvalidArgument.
let lc = BucketLifecycleConfiguration { let lc = BucketLifecycleConfiguration {
expiry_updated_at: None, expiry_updated_at: None,
rules: vec![LifecycleRule { rules: vec![LifecycleRule {
@@ -1195,9 +1248,12 @@ mod tests {
}], }],
}; };
lc.validate(&ObjectLockConfiguration::default()) let err = lc
.validate(&ObjectLockConfiguration::default())
.await .await
.expect("zero-day noncurrent expiration should be accepted"); .expect_err("zero-day noncurrent expiration should be rejected");
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS);
} }
#[tokio::test] #[tokio::test]
@@ -1258,7 +1314,9 @@ mod tests {
#[tokio::test] #[tokio::test]
#[serial] #[serial]
async fn validate_accepts_zero_abort_incomplete_multipart_upload_days() { async fn validate_rejects_zero_abort_incomplete_multipart_upload_days() {
// S3 compatibility: AbortIncompleteMultipartUpload.DaysAfterInitiation must be a
// positive integer (>= 1); AWS rejects 0 with InvalidArgument.
let lc = BucketLifecycleConfiguration { let lc = BucketLifecycleConfiguration {
expiry_updated_at: None, expiry_updated_at: None,
rules: vec![LifecycleRule { rules: vec![LifecycleRule {
@@ -1277,9 +1335,12 @@ mod tests {
}], }],
}; };
lc.validate(&ObjectLockConfiguration::default()) let err = lc
.validate(&ObjectLockConfiguration::default())
.await .await
.expect("zero-day abort incomplete multipart upload should be accepted"); .expect_err("zero-day abort incomplete multipart upload should be rejected");
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS);
} }
#[tokio::test] #[tokio::test]
+11 -2
View File
@@ -15,12 +15,21 @@ features are covered by the compatibility matrix and test lists.
| List | Purpose | Current count | Source | | List | Purpose | Current count | Source |
|---|---:|---:|---| |---|---:|---:|---|
| Implemented tests | Standard S3 tests expected to pass and used by the default local s3tests run. | 449 | `scripts/s3-tests/implemented_tests.txt` | | Implemented tests | Standard S3 tests expected to pass and used by the default local s3tests run. | 452 | `scripts/s3-tests/implemented_tests.txt` |
| Lifecycle behavior tests | Expiration behavior cases gated by the dedicated `s3-lifecycle-behavior-tests` lane (debug-accelerated day + scanner enabled). | 5 | `scripts/s3-tests/lifecycle_behavior_tests.txt` |
| Unimplemented tests | Standard S3 features planned but not yet implemented. | 17 | `scripts/s3-tests/unimplemented_tests.txt` | | Unimplemented tests | Standard S3 features planned but not yet implemented. | 17 | `scripts/s3-tests/unimplemented_tests.txt` |
| Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS compatibility gating. | 281 | `scripts/s3-tests/excluded_tests.txt` | | Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS compatibility gating. | 273 | `scripts/s3-tests/excluded_tests.txt` |
Counts ignore blank lines and comments. Counts ignore blank lines and comments.
The lifecycle behavior lane runs real Days-based expiration cases that need
`RUSTFS_ILM_DEBUG_DAY_SECS` (Ceph `lc_debug_interval` equivalent) and an enabled
background scanner; it cannot share the default single-server gate because a
global debug day would also shrink the `x-amz-expiration` header asserted by the
`test_lifecycle_expiration_header_*` cases. See `scripts/s3-tests/run.sh`
(`IMPLEMENTED_TESTS_FILE` override) and the `s3-lifecycle-behavior-tests` job in
`.github/workflows/ci.yml`.
## Supported Coverage ## Supported Coverage
The implemented test list currently covers the common object-storage surface: The implemented test list currently covers the common object-storage surface:
@@ -1567,3 +1567,60 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
"unexpected error message: {message}" "unexpected error message: {message}"
); );
} }
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
#[serial]
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
// S3 compatibility (ceph s3-tests `test_lifecycle_expiration_days0`): a rule with
// Expiration{Days:0} must be rejected with InvalidArgument through the real
// PutBucketLifecycleConfiguration handler, not merely at the crates/lifecycle
// io::Error layer. No object lock required.
let (_disk_paths, ecstore) = setup_test_env().await;
let usecase = DefaultBucketUsecase::from_global();
let bucket = format!("test-zero-day-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
create_test_bucket(&ecstore, bucket.as_str()).await;
let lifecycle = BucketLifecycleConfiguration {
expiry_updated_at: None,
rules: vec![LifecycleRule {
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
abort_incomplete_multipart_upload: None,
del_marker_expiration: None,
expiration: Some(LifecycleExpiration {
days: Some(0),
..Default::default()
}),
filter: Some(LifecycleRuleFilter {
prefix: Some("days0/".to_string()),
..Default::default()
}),
id: Some("expire-zero-days".to_string()),
noncurrent_version_expiration: None,
noncurrent_version_transitions: None,
prefix: None,
transitions: None,
}],
};
let req = build_request(
PutBucketLifecycleConfigurationInput::builder()
.bucket(bucket.clone())
.lifecycle_configuration(Some(lifecycle))
.build()
.unwrap(),
Method::PUT,
);
let err = usecase
.execute_put_bucket_lifecycle_configuration(req)
.await
.expect_err("zero-day expiration must be rejected with InvalidArgument");
assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
let message = err.message().unwrap_or_default();
assert!(
message.contains("'Days' for Expiration action must be a positive integer"),
"unexpected error message: {message}"
);
}
+31 -10
View File
@@ -157,26 +157,47 @@ test_encryption_sse_c_multipart_invalid_chunks_1
test_encryption_sse_c_multipart_invalid_chunks_2 test_encryption_sse_c_multipart_invalid_chunks_2
test_get_multipart_checksum_object_attributes test_get_multipart_checksum_object_attributes
test_head_bucket_usage test_head_bucket_usage
test_lifecycle_cloud_multiple_transition # -----------------------------------------------------------------------------
test_lifecycle_cloud_transition # Lifecycle behavior (backlog#1148 ilm-10, master plan #1155)
test_lifecycle_cloud_transition_large_obj #
test_lifecycle_deletemarker_expiration # The upstream "Vendor-specific" label on these was inaccurate: the real blocker
# was the absence of a Ceph `lc_debug_interval` equivalent, so Days>=1 expiration
# behavior could not be exercised in seconds. RUSTFS_ILM_DEBUG_DAY_SECS (ilm-5)
# now provides that. The first batch of expiration tests has been moved into the
# dedicated behavior lane: scripts/s3-tests/lifecycle_behavior_tests.txt
# (run by the ci.yml job `s3-lifecycle-behavior-tests`). The entries below are
# NOT vendor-specific — they stay excluded from the DEFAULT single-server gate
# for the specific reasons noted, and are queued for later batches.
#
# --- Batch 2 candidates: expiration with tag / size filters + noncurrent counts.
# These need the same debug/scanner lane as batch 1 plus tag/size-filter and
# NewerNoncurrentVersions coverage (see ilm-9). Queued for ilm-10 batch 2.
test_lifecycle_deletemarker_expiration_with_days_tag test_lifecycle_deletemarker_expiration_with_days_tag
test_lifecycle_expiration
test_lifecycle_expiration_days0
test_lifecycle_expiration_newer_noncurrent test_lifecycle_expiration_newer_noncurrent
test_lifecycle_expiration_noncur_tags1 test_lifecycle_expiration_noncur_tags1
test_lifecycle_expiration_size_gt test_lifecycle_expiration_size_gt
test_lifecycle_expiration_size_lt test_lifecycle_expiration_size_lt
test_lifecycle_expiration_tags2 test_lifecycle_expiration_tags2
test_lifecycle_expiration_versioned_tags2 test_lifecycle_expiration_versioned_tags2
test_lifecycle_multipart_expiration #
test_lifecycle_noncur_cloud_transition # --- Storage-class transition to a remote/warm tier. Needs a real tier backend
test_lifecycle_noncur_expiration # (the hermetic second-RustFS cold layer from ilm-7) which is not wired into
# the s3-tests topology. Queued behind ilm-7.
test_lifecycle_noncur_transition test_lifecycle_noncur_transition
test_lifecycle_transition test_lifecycle_transition
#
# --- Single-rule multi-level Transition (hot->warm->cold). RustFS only honors
# transitions.first() (core.rs:676); multi-level semantics are undecided and
# tracked as a product item (ex-ilm-20). Do not enable until that lands.
test_lifecycle_transition_single_rule_multi_trans test_lifecycle_transition_single_rule_multi_trans
test_lifecyclev2_expiration #
# --- Cloud (S3/remote) transition + restore. Requires the [s3 cloud] endpoint
# the RustFS SNSD harness does not provision; depends on ilm-7 tier carrier.
test_lifecycle_cloud_multiple_transition
test_lifecycle_cloud_transition
test_lifecycle_cloud_transition_large_obj
test_lifecycle_noncur_cloud_transition
# -----------------------------------------------------------------------------
test_list_buckets_anonymous test_list_buckets_anonymous
test_list_buckets_paginated test_list_buckets_paginated
test_list_multipart_upload test_list_multipart_upload
+2
View File
@@ -225,6 +225,8 @@ test_delete_bucket_encryption_kms
test_delete_bucket_encryption_s3 test_delete_bucket_encryption_s3
# Lifecycle tests # Lifecycle tests
test_lifecycle_delete test_lifecycle_delete
# Negative validation: Expiration{Days:0} must return InvalidArgument (backlog#1148 ilm-10).
test_lifecycle_expiration_days0
test_lifecycle_get test_lifecycle_get
test_lifecycle_get_no_id test_lifecycle_get_no_id
test_lifecycle_id_too_long test_lifecycle_id_too_long
@@ -0,0 +1,40 @@
# Lifecycle behavior tests (ceph/s3-tests) — backlog#1148 ilm-10, master plan #1155
# =================================================================================
#
# These upstream lifecycle tests exercise *real expiration behavior*: they PUT a
# lifecycle rule with Days/NoncurrentDays/DaysAfterInitiation, sleep a multiple of
# the debug interval, then assert objects/versions/uploads were actually removed by
# the background scanner and the stale-multipart cleanup loop.
#
# They CANNOT run in the default single-server gate (implemented_tests.txt) because:
# 1. The default harness disables the background scanner (run.sh sets
# RUSTFS_SCANNER_ENABLED=false) — nothing expires.
# 2. A real day is 86400s; without acceleration a Days=1 rule takes 24h.
# 3. Enabling RUSTFS_ILM_DEBUG_DAY_SECS *globally* would break the already-passing
# x-amz-expiration header tests (test_lifecycle_expiration_header_*), which
# assert `(expiry-date - now).days == Days` — the header is computed from the
# same debug-scaled expected_expiry_time().
#
# So this list is run by a DEDICATED lane (ci.yml job `s3-lifecycle-behavior-tests`)
# against a server started with:
# RUSTFS_ILM_DEBUG_DAY_SECS=10 (1 "day" = 10s, matches s3-tests lc_debug_interval)
# RUSTFS_SCANNER_ENABLED=true
# RUSTFS_SCANNER_CYCLE=2 (scan every 2s, well under a debug-day)
# RUSTFS_SCANNER_START_DELAY_SECS=0
# RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s (multipart abort lane)
#
# Point run.sh at this file with IMPLEMENTED_TESTS_FILE=<this path>.
#
# First batch (ilm-10). Each entry was verified against the pinned upstream source
# and the RustFS evaluator/scanner/stale-multipart execution paths.
# Basic prefix expiration: Days=1 / Days=5 rules, scanner-driven deletion.
test_lifecycle_expiration
# Same as above but validates ListObjectsV2 sees the expirations.
test_lifecyclev2_expiration
# Versioned NoncurrentVersionExpiration (NoncurrentDays): noncurrent versions removed.
test_lifecycle_noncur_expiration
# ExpiredObjectDeleteMarker + NoncurrentVersionExpiration cascade on a versioned bucket.
test_lifecycle_deletemarker_expiration
# AbortIncompleteMultipartUpload (DaysAfterInitiation): stale-multipart cleanup loop.
test_lifecycle_multipart_expiration
+20 -1
View File
@@ -36,6 +36,12 @@ LIST_FILES = {
"implemented": "implemented_tests.txt", "implemented": "implemented_tests.txt",
"unimplemented": "unimplemented_tests.txt", "unimplemented": "unimplemented_tests.txt",
"excluded": "excluded_tests.txt", "excluded": "excluded_tests.txt",
# Lifecycle behavior lane (backlog#1148 ilm-10): gated by its own ci.yml job
# against a debug-accelerated server. In the full `scope=all` sweep these run
# against the plain server (no scanner / no RUSTFS_ILM_DEBUG_DAY_SECS) and are
# EXPECTED to fail there, so a failure here is neither a regression nor an
# unclassified failure.
"behavior": "lifecycle_behavior_tests.txt",
} }
@@ -122,6 +128,7 @@ def main() -> int:
promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []} promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
unclassified_passed: list[str] = [] unclassified_passed: list[str] = []
unclassified_failed: list[str] = [] unclassified_failed: list[str] = []
behavior_passed: list[str] = []
counts = {"passed": 0, "failed": 0, "error": 0, "skipped": 0} counts = {"passed": 0, "failed": 0, "error": 0, "skipped": 0}
for name, status in results.items(): for name, status in results.items():
@@ -129,13 +136,19 @@ def main() -> int:
if status in ("failed", "error"): if status in ("failed", "error"):
if name in lists["implemented"]: if name in lists["implemented"]:
regressions.append(name) regressions.append(name)
elif name not in lists["unimplemented"] and name not in lists["excluded"]: elif (
name not in lists["unimplemented"]
and name not in lists["excluded"]
and name not in lists["behavior"]
):
unclassified_failed.append(name) unclassified_failed.append(name)
elif status == "passed": elif status == "passed":
if name in lists["unimplemented"]: if name in lists["unimplemented"]:
promotions["unimplemented"].append(name) promotions["unimplemented"].append(name)
elif name in lists["excluded"]: elif name in lists["excluded"]:
promotions["excluded"].append(name) promotions["excluded"].append(name)
elif name in lists["behavior"]:
behavior_passed.append(name)
elif name not in lists["implemented"]: elif name not in lists["implemented"]:
unclassified_passed.append(name) unclassified_passed.append(name)
@@ -162,6 +175,12 @@ def main() -> int:
promotions["excluded"], promotions["excluded"],
"Passing despite being excluded — re-evaluate the exclusion.", "Passing despite being excluded — re-evaluate the exclusion.",
) )
lines += render_section(
"Lifecycle behavior lane (passing in this run)",
behavior_passed,
"Gated by the dedicated `s3-lifecycle-behavior-tests` lane; expected to "
"fail in the plain `scope=all` sweep (no debug day / scanner).",
)
lines += render_section( lines += render_section(
"Unclassified passes", "Unclassified passes",
unclassified_passed, unclassified_passed,
+4 -1
View File
@@ -159,7 +159,10 @@ PY
# Test list files location # Test list files location
TEST_LISTS_DIR="${SCRIPT_DIR}" TEST_LISTS_DIR="${SCRIPT_DIR}"
IMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/implemented_tests.txt" # IMPLEMENTED_TESTS_FILE may be overridden to run an alternate exact-node-id
# whitelist (e.g. lifecycle_behavior_tests.txt for the dedicated lifecycle
# behavior lane, which needs RUSTFS_ILM_DEBUG_DAY_SECS + the scanner enabled).
IMPLEMENTED_TESTS_FILE="${IMPLEMENTED_TESTS_FILE:-${TEST_LISTS_DIR}/implemented_tests.txt}"
UNIMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/unimplemented_tests.txt" UNIMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/unimplemented_tests.txt"
EXCLUDED_TESTS_FILE="${TEST_LISTS_DIR}/excluded_tests.txt" EXCLUDED_TESTS_FILE="${TEST_LISTS_DIR}/excluded_tests.txt"
S3_TEST_FILE="s3tests/functional/test_s3.py" S3_TEST_FILE="s3tests/functional/test_s3.py"