mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-26 08:18:18 +00:00
ci(ilm): add gated s3-tests lane for lifecycle expiration cases (#4715)
* ci(ilm): move first batch of 5 s3-tests lifecycle expiration cases into a gated behavior lane (backlog#1148 ilm-10)
The 20 lifecycle cases in excluded_tests.txt were labeled "vendor-specific"
but the real blocker was the absence of a Ceph lc_debug_interval equivalent.
RUSTFS_ILM_DEBUG_DAY_SECS (ilm-5) now provides that, so Days>=1 expiration
behavior is testable in seconds.
These cases assert that objects/versions/uploads are actually removed by the
background scanner and the stale-multipart cleanup loop. They cannot join the
default single-server s3-implemented-tests gate: it disables the scanner, and a
global RUSTFS_ILM_DEBUG_DAY_SECS also shrinks the x-amz-expiration header that
the already-passing test_lifecycle_expiration_header_* cases assert on. So this
adds an isolated lane instead of putting them in implemented_tests.txt.
First batch (5), each verified against the pinned upstream source and the
RustFS evaluator/scanner/stale-multipart execution paths:
- test_lifecycle_expiration (prefix Days=1/Days=5, scanner delete)
- test_lifecyclev2_expiration (same via ListObjectsV2)
- test_lifecycle_noncur_expiration (NoncurrentVersionExpiration)
- test_lifecycle_deletemarker_expiration (ExpiredObjectDeleteMarker cascade)
- test_lifecycle_multipart_expiration (AbortIncompleteMultipartUpload)
Dropped from the batch, kept excluded with reason:
- test_lifecycle_expiration_days0: RustFS accepts Expiration{Days:0} (returns
200; only Days<0 is rejected in crates/lifecycle/src/core.rs validate()),
while AWS/this test expect InvalidArgument. Real validation gap.
Changes:
- scripts/s3-tests/lifecycle_behavior_tests.txt: new exact-node-id run set.
- scripts/s3-tests/run.sh: honor an IMPLEMENTED_TESTS_FILE override so a lane
can point at an alternate whitelist.
- .github/workflows/ci.yml: new s3-lifecycle-behavior-tests PR-gate job that
starts rustfs with RUSTFS_ILM_DEBUG_DAY_SECS=10, scanner enabled (cycle 2s,
no start delay) and a 2s stale-multipart cleanup interval, running the new
list serially.
- scripts/s3-tests/report_compat.py: recognize the behavior list so the weekly
scope=all sweep (plain server) does not misclassify expected failures.
- scripts/s3-tests/excluded_tests.txt: remove the 5 enabled cases; re-annotate
the remaining 15 lifecycle exclusions with real reasons + batch-2 plan.
- docs/architecture/s3-compatibility-matrix.md: sync counts (implemented 451,
excluded 274, behavior 5) and document the lane.
Refs backlog#1148 (ilm-10), master plan backlog#1155.
* fix(lifecycle): reject zero-day expiration/noncurrent/abort rules (backlog#1148 ilm-10) (#4722)
This commit is contained in:
@@ -495,3 +495,61 @@ jobs:
|
|||||||
path: artifacts/s3tests-single/**
|
path: artifacts/s3tests-single/**
|
||||||
if-no-files-found: ignore
|
if-no-files-found: ignore
|
||||||
retention-days: 3
|
retention-days: 3
|
||||||
|
|
||||||
|
# Dedicated lifecycle behavior lane (backlog#1148 ilm-10, master plan #1155).
|
||||||
|
#
|
||||||
|
# These ceph/s3-tests cases assert that objects/versions/uploads are actually
|
||||||
|
# removed by the background scanner and the stale-multipart cleanup loop, so
|
||||||
|
# they need a debug-accelerated day plus an enabled scanner. That configuration
|
||||||
|
# cannot be applied to the s3-implemented-tests lane above because a global
|
||||||
|
# RUSTFS_ILM_DEBUG_DAY_SECS also shrinks the x-amz-expiration header those tests
|
||||||
|
# assert on (test_lifecycle_expiration_header_*). Hence a separate server here.
|
||||||
|
s3-lifecycle-behavior-tests:
|
||||||
|
name: S3 Lifecycle Behavior Tests
|
||||||
|
needs: [ build-rustfs-debug-binary ]
|
||||||
|
runs-on: sm-standard-4
|
||||||
|
timeout-minutes: 30
|
||||||
|
steps:
|
||||||
|
- name: Checkout repository
|
||||||
|
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
|
||||||
|
|
||||||
|
- name: Download debug binary
|
||||||
|
uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7
|
||||||
|
with:
|
||||||
|
name: rustfs-debug-binary
|
||||||
|
path: target/debug
|
||||||
|
|
||||||
|
- name: Make binary executable
|
||||||
|
run: chmod +x ./target/debug/rustfs
|
||||||
|
|
||||||
|
- name: Run lifecycle behavior s3-tests
|
||||||
|
run: |
|
||||||
|
RUN_ROOT="${RUNNER_TEMP}/rustfs-s3tests-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${GITHUB_JOB}"
|
||||||
|
mkdir -p "${RUN_ROOT}"
|
||||||
|
S3_PORT="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1", 0)); print(s.getsockname()[1]); s.close()')"
|
||||||
|
# Serial (XDIST=0) for timing determinism: the cases sleep on multiples
|
||||||
|
# of the debug day and assert scanner-driven deletion.
|
||||||
|
DEPLOY_MODE=binary \
|
||||||
|
RUSTFS_BINARY=./target/debug/rustfs \
|
||||||
|
TEST_MODE=single \
|
||||||
|
MAXFAIL=0 \
|
||||||
|
XDIST=0 \
|
||||||
|
IMPLEMENTED_TESTS_FILE=scripts/s3-tests/lifecycle_behavior_tests.txt \
|
||||||
|
RUSTFS_ILM_DEBUG_DAY_SECS=10 \
|
||||||
|
RUSTFS_SCANNER_ENABLED=true \
|
||||||
|
RUSTFS_SCANNER_CYCLE=2 \
|
||||||
|
RUSTFS_SCANNER_START_DELAY_SECS=0 \
|
||||||
|
RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s \
|
||||||
|
S3_PORT="${S3_PORT}" \
|
||||||
|
DATA_ROOT="${RUN_ROOT}" \
|
||||||
|
S3TESTS_CONF=artifacts/s3tests-single/s3tests.conf \
|
||||||
|
./scripts/s3-tests/run.sh
|
||||||
|
|
||||||
|
- name: Upload s3 test artifacts
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6
|
||||||
|
with:
|
||||||
|
name: s3tests-lifecycle-behavior-${{ github.run_number }}
|
||||||
|
path: artifacts/s3tests-single/**
|
||||||
|
if-no-files-found: ignore
|
||||||
|
retention-days: 3
|
||||||
|
|||||||
@@ -160,12 +160,16 @@ async fn test_bucket_lifecycle_configuration() -> Result<(), Box<dyn std::error:
|
|||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
#[serial]
|
#[serial]
|
||||||
#[ignore = "requires running RustFS server at localhost:9000"]
|
#[ignore = "requires running RustFS server at localhost:9000"]
|
||||||
async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
async fn test_bucket_lifecycle_rejects_zero_days() -> Result<(), Box<dyn std::error::Error>> {
|
||||||
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
|
use aws_sdk_s3::types::{BucketLifecycleConfiguration, LifecycleExpiration, LifecycleRule, LifecycleRuleFilter};
|
||||||
|
|
||||||
let client = create_aws_s3_client().await?;
|
let client = create_aws_s3_client().await?;
|
||||||
setup_test_bucket(&client).await?;
|
setup_test_bucket(&client).await?;
|
||||||
|
|
||||||
|
// S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and the
|
||||||
|
// ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with an
|
||||||
|
// InvalidArgument (HTTP 400). See crates/lifecycle validate() + the
|
||||||
|
// PutBucketLifecycleConfiguration handler which maps it to s3_error!(InvalidArgument).
|
||||||
let expiration = LifecycleExpiration::builder().days(0).build();
|
let expiration = LifecycleExpiration::builder().days(0).build();
|
||||||
let filter = LifecycleRuleFilter::builder().prefix("zero-days/").build();
|
let filter = LifecycleRuleFilter::builder().prefix("zero-days/").build();
|
||||||
let rule = LifecycleRule::builder()
|
let rule = LifecycleRule::builder()
|
||||||
@@ -176,12 +180,20 @@ async fn test_bucket_lifecycle_accepts_zero_days() -> Result<(), Box<dyn std::er
|
|||||||
.build()?;
|
.build()?;
|
||||||
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
let lifecycle = BucketLifecycleConfiguration::builder().rules(rule).build()?;
|
||||||
|
|
||||||
client
|
let err = client
|
||||||
.put_bucket_lifecycle_configuration()
|
.put_bucket_lifecycle_configuration()
|
||||||
.bucket(BUCKET)
|
.bucket(BUCKET)
|
||||||
.lifecycle_configuration(lifecycle)
|
.lifecycle_configuration(lifecycle)
|
||||||
.send()
|
.send()
|
||||||
.await?;
|
.await
|
||||||
|
.expect_err("zero-day expiration lifecycle rule should be rejected");
|
||||||
|
|
||||||
|
let service_error = err.as_service_error().expect("expected an S3 service error");
|
||||||
|
assert_eq!(
|
||||||
|
service_error.meta().code(),
|
||||||
|
Some("InvalidArgument"),
|
||||||
|
"expected InvalidArgument for zero-day expiration, got: {err:?}"
|
||||||
|
);
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -40,10 +40,11 @@ const _ERR_XML_NOT_WELL_FORMED: &str =
|
|||||||
const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
|
const ERR_LIFECYCLE_BUCKET_LOCKED: &str =
|
||||||
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
|
"ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket";
|
||||||
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
|
const ERR_LIFECYCLE_TOO_MANY_RULES: &str = "Lifecycle configuration should have at most 1000 rules";
|
||||||
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "Lifecycle expiration days must not be negative";
|
const ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS: &str = "'Days' for Expiration action must be a positive integer";
|
||||||
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str = "Lifecycle noncurrent expiration days must not be negative";
|
const ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS: &str =
|
||||||
|
"'NoncurrentDays' for NoncurrentVersionExpiration action must be a positive integer";
|
||||||
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
|
const ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS: &str =
|
||||||
"DaysAfterInitiation must be 0 or greater when used with AbortIncompleteMultipartUpload";
|
"'DaysAfterInitiation' for AbortIncompleteMultipartUpload action must be a positive integer";
|
||||||
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
|
const ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT: &str = "Expiration.Date must be at midnight UTC";
|
||||||
const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER: &str =
|
const ERR_LIFECYCLE_INVALID_EXPIRED_OBJECT_DELETE_MARKER: &str =
|
||||||
"ExpiredObjectDeleteMarker cannot be specified with Days or Date";
|
"ExpiredObjectDeleteMarker cannot be specified with Days or Date";
|
||||||
@@ -325,21 +326,29 @@ impl Lifecycle for BucketLifecycleConfiguration {
|
|||||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT));
|
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DATE_NOT_MIDNIGHT));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// S3 requires Expiration.Days to be a positive integer (>= 1). AWS and the
|
||||||
|
// ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with
|
||||||
|
// InvalidArgument; only Date-based rules may omit Days entirely.
|
||||||
if let Some(days) = expiration.days
|
if let Some(days) = expiration.days
|
||||||
&& days < 0
|
&& days < 1
|
||||||
{
|
{
|
||||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS));
|
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// S3 requires NoncurrentVersionExpiration.NoncurrentDays to be a positive
|
||||||
|
// integer (>= 1); AWS rejects 0 with InvalidArgument.
|
||||||
if let Some(noncurrent_version_expiration) = &r.noncurrent_version_expiration
|
if let Some(noncurrent_version_expiration) = &r.noncurrent_version_expiration
|
||||||
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
|
&& let Some(noncurrent_days) = noncurrent_version_expiration.noncurrent_days
|
||||||
&& noncurrent_days < 0
|
&& noncurrent_days < 1
|
||||||
{
|
{
|
||||||
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS));
|
return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS));
|
||||||
}
|
}
|
||||||
|
// S3 requires AbortIncompleteMultipartUpload.DaysAfterInitiation to be present
|
||||||
|
// and a positive integer (>= 1); AWS rejects 0 (and a missing value) with
|
||||||
|
// InvalidArgument.
|
||||||
if let Some(abort_incomplete_multipart_upload) = &r.abort_incomplete_multipart_upload {
|
if let Some(abort_incomplete_multipart_upload) = &r.abort_incomplete_multipart_upload {
|
||||||
match abort_incomplete_multipart_upload.days_after_initiation {
|
match abort_incomplete_multipart_upload.days_after_initiation {
|
||||||
Some(days) if days >= 0 => {}
|
Some(days) if days >= 1 => {}
|
||||||
_ => return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS)),
|
_ => return Err(std::io::Error::other(ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1066,7 +1075,11 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn validate_accepts_zero_expiration_days() {
|
async fn validate_rejects_zero_expiration_days() {
|
||||||
|
// S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and
|
||||||
|
// the ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with
|
||||||
|
// InvalidArgument. The PutBucketLifecycleConfiguration path maps this io::Error
|
||||||
|
// to s3_error!(InvalidArgument) (see execute_put_bucket_lifecycle_configuration).
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
expiry_updated_at: None,
|
expiry_updated_at: None,
|
||||||
rules: vec![LifecycleRule {
|
rules: vec![LifecycleRule {
|
||||||
@@ -1086,9 +1099,12 @@ mod tests {
|
|||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
|
|
||||||
lc.validate(&ObjectLockConfiguration::default())
|
let err = lc
|
||||||
|
.validate(&ObjectLockConfiguration::default())
|
||||||
.await
|
.await
|
||||||
.expect("zero-day expiration should be accepted");
|
.expect_err("zero-day expiration should be rejected");
|
||||||
|
|
||||||
|
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_EXPIRATION_DAYS);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1148,6 +1164,41 @@ mod tests {
|
|||||||
.expect("expected validation to pass");
|
.expect("expected validation to pass");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn validate_accepts_one_day_boundary_values() {
|
||||||
|
// Pin the exact >= 1 boundary: a value of 1 is the smallest legal positive
|
||||||
|
// integer and must be accepted for every day-count field tightened for S3
|
||||||
|
// compatibility. This catches an off-by-one that would reject Days == 1.
|
||||||
|
let lc = BucketLifecycleConfiguration {
|
||||||
|
expiry_updated_at: None,
|
||||||
|
rules: vec![LifecycleRule {
|
||||||
|
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||||
|
expiration: Some(LifecycleExpiration {
|
||||||
|
days: Some(1),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
abort_incomplete_multipart_upload: Some(s3s::dto::AbortIncompleteMultipartUpload {
|
||||||
|
days_after_initiation: Some(1),
|
||||||
|
}),
|
||||||
|
del_marker_expiration: None,
|
||||||
|
filter: None,
|
||||||
|
id: Some("one-day-boundary".to_string()),
|
||||||
|
noncurrent_version_expiration: Some(s3s::dto::NoncurrentVersionExpiration {
|
||||||
|
noncurrent_days: Some(1),
|
||||||
|
newer_noncurrent_versions: None,
|
||||||
|
}),
|
||||||
|
noncurrent_version_transitions: None,
|
||||||
|
prefix: Some("boundary/".to_string()),
|
||||||
|
transitions: None,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
lc.validate(&ObjectLockConfiguration::default())
|
||||||
|
.await
|
||||||
|
.expect("one-day boundary values should be accepted");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn has_active_rules_accepts_zero_day_expiration() {
|
async fn has_active_rules_accepts_zero_day_expiration() {
|
||||||
@@ -1175,7 +1226,9 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn validate_accepts_zero_noncurrent_expiration_days() {
|
async fn validate_rejects_zero_noncurrent_expiration_days() {
|
||||||
|
// S3 compatibility: NoncurrentVersionExpiration.NoncurrentDays must be a positive
|
||||||
|
// integer (>= 1); AWS rejects 0 with InvalidArgument.
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
expiry_updated_at: None,
|
expiry_updated_at: None,
|
||||||
rules: vec![LifecycleRule {
|
rules: vec![LifecycleRule {
|
||||||
@@ -1195,9 +1248,12 @@ mod tests {
|
|||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
|
|
||||||
lc.validate(&ObjectLockConfiguration::default())
|
let err = lc
|
||||||
|
.validate(&ObjectLockConfiguration::default())
|
||||||
.await
|
.await
|
||||||
.expect("zero-day noncurrent expiration should be accepted");
|
.expect_err("zero-day noncurrent expiration should be rejected");
|
||||||
|
|
||||||
|
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_NONCURRENT_EXPIRATION_DAYS);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1258,7 +1314,9 @@ mod tests {
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn validate_accepts_zero_abort_incomplete_multipart_upload_days() {
|
async fn validate_rejects_zero_abort_incomplete_multipart_upload_days() {
|
||||||
|
// S3 compatibility: AbortIncompleteMultipartUpload.DaysAfterInitiation must be a
|
||||||
|
// positive integer (>= 1); AWS rejects 0 with InvalidArgument.
|
||||||
let lc = BucketLifecycleConfiguration {
|
let lc = BucketLifecycleConfiguration {
|
||||||
expiry_updated_at: None,
|
expiry_updated_at: None,
|
||||||
rules: vec![LifecycleRule {
|
rules: vec![LifecycleRule {
|
||||||
@@ -1277,9 +1335,12 @@ mod tests {
|
|||||||
}],
|
}],
|
||||||
};
|
};
|
||||||
|
|
||||||
lc.validate(&ObjectLockConfiguration::default())
|
let err = lc
|
||||||
|
.validate(&ObjectLockConfiguration::default())
|
||||||
.await
|
.await
|
||||||
.expect("zero-day abort incomplete multipart upload should be accepted");
|
.expect_err("zero-day abort incomplete multipart upload should be rejected");
|
||||||
|
|
||||||
|
assert_eq!(err.to_string(), ERR_LIFECYCLE_INVALID_ABORT_INCOMPLETE_MPU_DAYS);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -15,12 +15,21 @@ features are covered by the compatibility matrix and test lists.
|
|||||||
|
|
||||||
| List | Purpose | Current count | Source |
|
| List | Purpose | Current count | Source |
|
||||||
|---|---:|---:|---|
|
|---|---:|---:|---|
|
||||||
| Implemented tests | Standard S3 tests expected to pass and used by the default local s3tests run. | 449 | `scripts/s3-tests/implemented_tests.txt` |
|
| Implemented tests | Standard S3 tests expected to pass and used by the default local s3tests run. | 452 | `scripts/s3-tests/implemented_tests.txt` |
|
||||||
|
| Lifecycle behavior tests | Expiration behavior cases gated by the dedicated `s3-lifecycle-behavior-tests` lane (debug-accelerated day + scanner enabled). | 5 | `scripts/s3-tests/lifecycle_behavior_tests.txt` |
|
||||||
| Unimplemented tests | Standard S3 features planned but not yet implemented. | 17 | `scripts/s3-tests/unimplemented_tests.txt` |
|
| Unimplemented tests | Standard S3 features planned but not yet implemented. | 17 | `scripts/s3-tests/unimplemented_tests.txt` |
|
||||||
| Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS compatibility gating. | 281 | `scripts/s3-tests/excluded_tests.txt` |
|
| Excluded tests | Vendor-specific or intentionally unsupported behavior excluded from RustFS compatibility gating. | 273 | `scripts/s3-tests/excluded_tests.txt` |
|
||||||
|
|
||||||
Counts ignore blank lines and comments.
|
Counts ignore blank lines and comments.
|
||||||
|
|
||||||
|
The lifecycle behavior lane runs real Days-based expiration cases that need
|
||||||
|
`RUSTFS_ILM_DEBUG_DAY_SECS` (Ceph `lc_debug_interval` equivalent) and an enabled
|
||||||
|
background scanner; it cannot share the default single-server gate because a
|
||||||
|
global debug day would also shrink the `x-amz-expiration` header asserted by the
|
||||||
|
`test_lifecycle_expiration_header_*` cases. See `scripts/s3-tests/run.sh`
|
||||||
|
(`IMPLEMENTED_TESTS_FILE` override) and the `s3-lifecycle-behavior-tests` job in
|
||||||
|
`.github/workflows/ci.yml`.
|
||||||
|
|
||||||
## Supported Coverage
|
## Supported Coverage
|
||||||
|
|
||||||
The implemented test list currently covers the common object-storage surface:
|
The implemented test list currently covers the common object-storage surface:
|
||||||
|
|||||||
@@ -1567,3 +1567,60 @@ async fn put_bucket_lifecycle_configuration_rejects_zero_day_del_marker_expirati
|
|||||||
"unexpected error message: {message}"
|
"unexpected error message: {message}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 1)]
|
||||||
|
#[serial]
|
||||||
|
#[ignore = "global-state ILM integration test: runs serialized in the CI ILM Integration (serial) lane, see ci.yml test-ilm-integration-serial and rustfs/backlog#1148 (ilm-1)"]
|
||||||
|
async fn put_bucket_lifecycle_configuration_rejects_zero_day_expiration() {
|
||||||
|
// S3 compatibility (ceph s3-tests `test_lifecycle_expiration_days0`): a rule with
|
||||||
|
// Expiration{Days:0} must be rejected with InvalidArgument through the real
|
||||||
|
// PutBucketLifecycleConfiguration handler, not merely at the crates/lifecycle
|
||||||
|
// io::Error layer. No object lock required.
|
||||||
|
let (_disk_paths, ecstore) = setup_test_env().await;
|
||||||
|
let usecase = DefaultBucketUsecase::from_global();
|
||||||
|
|
||||||
|
let bucket = format!("test-zero-day-exp-{}", &Uuid::new_v4().simple().to_string()[..8]);
|
||||||
|
create_test_bucket(&ecstore, bucket.as_str()).await;
|
||||||
|
|
||||||
|
let lifecycle = BucketLifecycleConfiguration {
|
||||||
|
expiry_updated_at: None,
|
||||||
|
rules: vec![LifecycleRule {
|
||||||
|
status: ExpirationStatus::from_static(ExpirationStatus::ENABLED),
|
||||||
|
abort_incomplete_multipart_upload: None,
|
||||||
|
del_marker_expiration: None,
|
||||||
|
expiration: Some(LifecycleExpiration {
|
||||||
|
days: Some(0),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
filter: Some(LifecycleRuleFilter {
|
||||||
|
prefix: Some("days0/".to_string()),
|
||||||
|
..Default::default()
|
||||||
|
}),
|
||||||
|
id: Some("expire-zero-days".to_string()),
|
||||||
|
noncurrent_version_expiration: None,
|
||||||
|
noncurrent_version_transitions: None,
|
||||||
|
prefix: None,
|
||||||
|
transitions: None,
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
|
||||||
|
let req = build_request(
|
||||||
|
PutBucketLifecycleConfigurationInput::builder()
|
||||||
|
.bucket(bucket.clone())
|
||||||
|
.lifecycle_configuration(Some(lifecycle))
|
||||||
|
.build()
|
||||||
|
.unwrap(),
|
||||||
|
Method::PUT,
|
||||||
|
);
|
||||||
|
|
||||||
|
let err = usecase
|
||||||
|
.execute_put_bucket_lifecycle_configuration(req)
|
||||||
|
.await
|
||||||
|
.expect_err("zero-day expiration must be rejected with InvalidArgument");
|
||||||
|
assert_eq!(err.code(), &s3s::S3ErrorCode::InvalidArgument);
|
||||||
|
let message = err.message().unwrap_or_default();
|
||||||
|
assert!(
|
||||||
|
message.contains("'Days' for Expiration action must be a positive integer"),
|
||||||
|
"unexpected error message: {message}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|||||||
@@ -157,26 +157,47 @@ test_encryption_sse_c_multipart_invalid_chunks_1
|
|||||||
test_encryption_sse_c_multipart_invalid_chunks_2
|
test_encryption_sse_c_multipart_invalid_chunks_2
|
||||||
test_get_multipart_checksum_object_attributes
|
test_get_multipart_checksum_object_attributes
|
||||||
test_head_bucket_usage
|
test_head_bucket_usage
|
||||||
test_lifecycle_cloud_multiple_transition
|
# -----------------------------------------------------------------------------
|
||||||
test_lifecycle_cloud_transition
|
# Lifecycle behavior (backlog#1148 ilm-10, master plan #1155)
|
||||||
test_lifecycle_cloud_transition_large_obj
|
#
|
||||||
test_lifecycle_deletemarker_expiration
|
# The upstream "Vendor-specific" label on these was inaccurate: the real blocker
|
||||||
|
# was the absence of a Ceph `lc_debug_interval` equivalent, so Days>=1 expiration
|
||||||
|
# behavior could not be exercised in seconds. RUSTFS_ILM_DEBUG_DAY_SECS (ilm-5)
|
||||||
|
# now provides that. The first batch of expiration tests has been moved into the
|
||||||
|
# dedicated behavior lane: scripts/s3-tests/lifecycle_behavior_tests.txt
|
||||||
|
# (run by the ci.yml job `s3-lifecycle-behavior-tests`). The entries below are
|
||||||
|
# NOT vendor-specific — they stay excluded from the DEFAULT single-server gate
|
||||||
|
# for the specific reasons noted, and are queued for later batches.
|
||||||
|
#
|
||||||
|
# --- Batch 2 candidates: expiration with tag / size filters + noncurrent counts.
|
||||||
|
# These need the same debug/scanner lane as batch 1 plus tag/size-filter and
|
||||||
|
# NewerNoncurrentVersions coverage (see ilm-9). Queued for ilm-10 batch 2.
|
||||||
test_lifecycle_deletemarker_expiration_with_days_tag
|
test_lifecycle_deletemarker_expiration_with_days_tag
|
||||||
test_lifecycle_expiration
|
|
||||||
test_lifecycle_expiration_days0
|
|
||||||
test_lifecycle_expiration_newer_noncurrent
|
test_lifecycle_expiration_newer_noncurrent
|
||||||
test_lifecycle_expiration_noncur_tags1
|
test_lifecycle_expiration_noncur_tags1
|
||||||
test_lifecycle_expiration_size_gt
|
test_lifecycle_expiration_size_gt
|
||||||
test_lifecycle_expiration_size_lt
|
test_lifecycle_expiration_size_lt
|
||||||
test_lifecycle_expiration_tags2
|
test_lifecycle_expiration_tags2
|
||||||
test_lifecycle_expiration_versioned_tags2
|
test_lifecycle_expiration_versioned_tags2
|
||||||
test_lifecycle_multipart_expiration
|
#
|
||||||
test_lifecycle_noncur_cloud_transition
|
# --- Storage-class transition to a remote/warm tier. Needs a real tier backend
|
||||||
test_lifecycle_noncur_expiration
|
# (the hermetic second-RustFS cold layer from ilm-7) which is not wired into
|
||||||
|
# the s3-tests topology. Queued behind ilm-7.
|
||||||
test_lifecycle_noncur_transition
|
test_lifecycle_noncur_transition
|
||||||
test_lifecycle_transition
|
test_lifecycle_transition
|
||||||
|
#
|
||||||
|
# --- Single-rule multi-level Transition (hot->warm->cold). RustFS only honors
|
||||||
|
# transitions.first() (core.rs:676); multi-level semantics are undecided and
|
||||||
|
# tracked as a product item (ex-ilm-20). Do not enable until that lands.
|
||||||
test_lifecycle_transition_single_rule_multi_trans
|
test_lifecycle_transition_single_rule_multi_trans
|
||||||
test_lifecyclev2_expiration
|
#
|
||||||
|
# --- Cloud (S3/remote) transition + restore. Requires the [s3 cloud] endpoint
|
||||||
|
# the RustFS SNSD harness does not provision; depends on ilm-7 tier carrier.
|
||||||
|
test_lifecycle_cloud_multiple_transition
|
||||||
|
test_lifecycle_cloud_transition
|
||||||
|
test_lifecycle_cloud_transition_large_obj
|
||||||
|
test_lifecycle_noncur_cloud_transition
|
||||||
|
# -----------------------------------------------------------------------------
|
||||||
test_list_buckets_anonymous
|
test_list_buckets_anonymous
|
||||||
test_list_buckets_paginated
|
test_list_buckets_paginated
|
||||||
test_list_multipart_upload
|
test_list_multipart_upload
|
||||||
|
|||||||
@@ -225,6 +225,8 @@ test_delete_bucket_encryption_kms
|
|||||||
test_delete_bucket_encryption_s3
|
test_delete_bucket_encryption_s3
|
||||||
# Lifecycle tests
|
# Lifecycle tests
|
||||||
test_lifecycle_delete
|
test_lifecycle_delete
|
||||||
|
# Negative validation: Expiration{Days:0} must return InvalidArgument (backlog#1148 ilm-10).
|
||||||
|
test_lifecycle_expiration_days0
|
||||||
test_lifecycle_get
|
test_lifecycle_get
|
||||||
test_lifecycle_get_no_id
|
test_lifecycle_get_no_id
|
||||||
test_lifecycle_id_too_long
|
test_lifecycle_id_too_long
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# Lifecycle behavior tests (ceph/s3-tests) — backlog#1148 ilm-10, master plan #1155
|
||||||
|
# =================================================================================
|
||||||
|
#
|
||||||
|
# These upstream lifecycle tests exercise *real expiration behavior*: they PUT a
|
||||||
|
# lifecycle rule with Days/NoncurrentDays/DaysAfterInitiation, sleep a multiple of
|
||||||
|
# the debug interval, then assert objects/versions/uploads were actually removed by
|
||||||
|
# the background scanner and the stale-multipart cleanup loop.
|
||||||
|
#
|
||||||
|
# They CANNOT run in the default single-server gate (implemented_tests.txt) because:
|
||||||
|
# 1. The default harness disables the background scanner (run.sh sets
|
||||||
|
# RUSTFS_SCANNER_ENABLED=false) — nothing expires.
|
||||||
|
# 2. A real day is 86400s; without acceleration a Days=1 rule takes 24h.
|
||||||
|
# 3. Enabling RUSTFS_ILM_DEBUG_DAY_SECS *globally* would break the already-passing
|
||||||
|
# x-amz-expiration header tests (test_lifecycle_expiration_header_*), which
|
||||||
|
# assert `(expiry-date - now).days == Days` — the header is computed from the
|
||||||
|
# same debug-scaled expected_expiry_time().
|
||||||
|
#
|
||||||
|
# So this list is run by a DEDICATED lane (ci.yml job `s3-lifecycle-behavior-tests`)
|
||||||
|
# against a server started with:
|
||||||
|
# RUSTFS_ILM_DEBUG_DAY_SECS=10 (1 "day" = 10s, matches s3-tests lc_debug_interval)
|
||||||
|
# RUSTFS_SCANNER_ENABLED=true
|
||||||
|
# RUSTFS_SCANNER_CYCLE=2 (scan every 2s, well under a debug-day)
|
||||||
|
# RUSTFS_SCANNER_START_DELAY_SECS=0
|
||||||
|
# RUSTFS_API_STALE_UPLOADS_CLEANUP_INTERVAL=2s (multipart abort lane)
|
||||||
|
#
|
||||||
|
# Point run.sh at this file with IMPLEMENTED_TESTS_FILE=<this path>.
|
||||||
|
#
|
||||||
|
# First batch (ilm-10). Each entry was verified against the pinned upstream source
|
||||||
|
# and the RustFS evaluator/scanner/stale-multipart execution paths.
|
||||||
|
|
||||||
|
# Basic prefix expiration: Days=1 / Days=5 rules, scanner-driven deletion.
|
||||||
|
test_lifecycle_expiration
|
||||||
|
# Same as above but validates ListObjectsV2 sees the expirations.
|
||||||
|
test_lifecyclev2_expiration
|
||||||
|
# Versioned NoncurrentVersionExpiration (NoncurrentDays): noncurrent versions removed.
|
||||||
|
test_lifecycle_noncur_expiration
|
||||||
|
# ExpiredObjectDeleteMarker + NoncurrentVersionExpiration cascade on a versioned bucket.
|
||||||
|
test_lifecycle_deletemarker_expiration
|
||||||
|
# AbortIncompleteMultipartUpload (DaysAfterInitiation): stale-multipart cleanup loop.
|
||||||
|
test_lifecycle_multipart_expiration
|
||||||
@@ -36,6 +36,12 @@ LIST_FILES = {
|
|||||||
"implemented": "implemented_tests.txt",
|
"implemented": "implemented_tests.txt",
|
||||||
"unimplemented": "unimplemented_tests.txt",
|
"unimplemented": "unimplemented_tests.txt",
|
||||||
"excluded": "excluded_tests.txt",
|
"excluded": "excluded_tests.txt",
|
||||||
|
# Lifecycle behavior lane (backlog#1148 ilm-10): gated by its own ci.yml job
|
||||||
|
# against a debug-accelerated server. In the full `scope=all` sweep these run
|
||||||
|
# against the plain server (no scanner / no RUSTFS_ILM_DEBUG_DAY_SECS) and are
|
||||||
|
# EXPECTED to fail there, so a failure here is neither a regression nor an
|
||||||
|
# unclassified failure.
|
||||||
|
"behavior": "lifecycle_behavior_tests.txt",
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -122,6 +128,7 @@ def main() -> int:
|
|||||||
promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
|
promotions: dict[str, list[str]] = {"unimplemented": [], "excluded": []}
|
||||||
unclassified_passed: list[str] = []
|
unclassified_passed: list[str] = []
|
||||||
unclassified_failed: list[str] = []
|
unclassified_failed: list[str] = []
|
||||||
|
behavior_passed: list[str] = []
|
||||||
counts = {"passed": 0, "failed": 0, "error": 0, "skipped": 0}
|
counts = {"passed": 0, "failed": 0, "error": 0, "skipped": 0}
|
||||||
|
|
||||||
for name, status in results.items():
|
for name, status in results.items():
|
||||||
@@ -129,13 +136,19 @@ def main() -> int:
|
|||||||
if status in ("failed", "error"):
|
if status in ("failed", "error"):
|
||||||
if name in lists["implemented"]:
|
if name in lists["implemented"]:
|
||||||
regressions.append(name)
|
regressions.append(name)
|
||||||
elif name not in lists["unimplemented"] and name not in lists["excluded"]:
|
elif (
|
||||||
|
name not in lists["unimplemented"]
|
||||||
|
and name not in lists["excluded"]
|
||||||
|
and name not in lists["behavior"]
|
||||||
|
):
|
||||||
unclassified_failed.append(name)
|
unclassified_failed.append(name)
|
||||||
elif status == "passed":
|
elif status == "passed":
|
||||||
if name in lists["unimplemented"]:
|
if name in lists["unimplemented"]:
|
||||||
promotions["unimplemented"].append(name)
|
promotions["unimplemented"].append(name)
|
||||||
elif name in lists["excluded"]:
|
elif name in lists["excluded"]:
|
||||||
promotions["excluded"].append(name)
|
promotions["excluded"].append(name)
|
||||||
|
elif name in lists["behavior"]:
|
||||||
|
behavior_passed.append(name)
|
||||||
elif name not in lists["implemented"]:
|
elif name not in lists["implemented"]:
|
||||||
unclassified_passed.append(name)
|
unclassified_passed.append(name)
|
||||||
|
|
||||||
@@ -162,6 +175,12 @@ def main() -> int:
|
|||||||
promotions["excluded"],
|
promotions["excluded"],
|
||||||
"Passing despite being excluded — re-evaluate the exclusion.",
|
"Passing despite being excluded — re-evaluate the exclusion.",
|
||||||
)
|
)
|
||||||
|
lines += render_section(
|
||||||
|
"Lifecycle behavior lane (passing in this run)",
|
||||||
|
behavior_passed,
|
||||||
|
"Gated by the dedicated `s3-lifecycle-behavior-tests` lane; expected to "
|
||||||
|
"fail in the plain `scope=all` sweep (no debug day / scanner).",
|
||||||
|
)
|
||||||
lines += render_section(
|
lines += render_section(
|
||||||
"Unclassified passes",
|
"Unclassified passes",
|
||||||
unclassified_passed,
|
unclassified_passed,
|
||||||
|
|||||||
@@ -159,7 +159,10 @@ PY
|
|||||||
|
|
||||||
# Test list files location
|
# Test list files location
|
||||||
TEST_LISTS_DIR="${SCRIPT_DIR}"
|
TEST_LISTS_DIR="${SCRIPT_DIR}"
|
||||||
IMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/implemented_tests.txt"
|
# IMPLEMENTED_TESTS_FILE may be overridden to run an alternate exact-node-id
|
||||||
|
# whitelist (e.g. lifecycle_behavior_tests.txt for the dedicated lifecycle
|
||||||
|
# behavior lane, which needs RUSTFS_ILM_DEBUG_DAY_SECS + the scanner enabled).
|
||||||
|
IMPLEMENTED_TESTS_FILE="${IMPLEMENTED_TESTS_FILE:-${TEST_LISTS_DIR}/implemented_tests.txt}"
|
||||||
UNIMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/unimplemented_tests.txt"
|
UNIMPLEMENTED_TESTS_FILE="${TEST_LISTS_DIR}/unimplemented_tests.txt"
|
||||||
EXCLUDED_TESTS_FILE="${TEST_LISTS_DIR}/excluded_tests.txt"
|
EXCLUDED_TESTS_FILE="${TEST_LISTS_DIR}/excluded_tests.txt"
|
||||||
S3_TEST_FILE="s3tests/functional/test_s3.py"
|
S3_TEST_FILE="s3tests/functional/test_s3.py"
|
||||||
|
|||||||
Reference in New Issue
Block a user