Compare commits

..

1 Commits

Author SHA1 Message Date
overtrue abca7ddfd7 test(kms): move the Vault KV2 Transit-wrapping doc guard into check_fips_wording.sh
`test_vault_kv2_sources_do_not_claim_transit_wrapping` asserted that four
`include_str!`-pinned files never describe the Vault KV2 backend as wrapping key
material through Vault's Transit engine. The invariant is a documentation-claim
invariant with no behavioral twin by construction, and the test form was weak in
both directions: it saw only four files (the same prose in a fifth file passed
silently) and it stopped compiling — rather than reporting a violation — as soon
as one of them was renamed.

Move the four literals verbatim into `scripts/check_fips_wording.sh`, which
already guards the adjacent cryptographic over-claim class (unsupported FIPS
validation wording) and is anchored to the same policy document. The guard now
greps every file under `crates/kms` for the same four case-sensitive literals and
separately reports a moved pinned source instead of failing to build.

`check_fips_wording.sh` previously ran only in `make pre-commit` / `pre-pr`, so
wire it into the Quick Checks job of both CI workflows to keep the invariant's
failure visibility at least as strong as the deleted test's.
2026-08-18 17:30:11 +08:00
21 changed files with 2302 additions and 432 deletions
+2 -2
View File
@@ -66,8 +66,8 @@ s3s-footprint-check: ## Check the s3s dependency footprint ratchet stays frozen
./scripts/check_s3s_footprint.sh
.PHONY: fips-wording-check
fips-wording-check: ## Check outward docs do not make unsupported FIPS claims
@echo "📣 Checking FIPS wording guard..."
fips-wording-check: ## Check docs and crates/kms do not over-claim crypto capabilities
@echo "📣 Checking cryptographic capability wording guard..."
./scripts/check_fips_wording.sh
.PHONY: log-analyzer-rules-check
+3
View File
@@ -117,6 +117,9 @@ jobs:
- name: Check s3s footprint ratchet
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
run: ./scripts/check_fips_wording.sh
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
+3
View File
@@ -152,6 +152,9 @@ jobs:
- name: Check s3s footprint ratchet
run: ./scripts/check_s3s_footprint.sh
- name: Check cryptographic capability wording
run: ./scripts/check_fips_wording.sh
- name: Check no planning docs committed
run: ./scripts/check_no_planning_docs.sh
Generated
+5
View File
@@ -10489,10 +10489,15 @@ dependencies = [
name = "rustfs-zip"
version = "1.0.0-rc.2"
dependencies = [
"astral-tokio-tar",
"async-compression",
"criterion",
"hotpath",
"tempfile",
"thiserror 2.0.20",
"tokio",
"tokio-stream",
"zip",
]
[[package]]
+16 -66
View File
@@ -9264,27 +9264,17 @@ impl DiskAPI for LocalDisk {
// accept that window (documented in docs/operations/durability-modes.md).
if durability.syncs_commit_metadata()
&& let Some(parent) = dst_file_path.parent()
&& let Err(err) = os::fsync_dir(parent).await
{
let fsync_started = rustfs_io_metrics::put_stage_timer();
if let Err(err) = os::fsync_dir(parent).await {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
fsync_started,
);
rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir)
.map_err(to_file_error)?;
// The commit rename changed the dst part inodes before this fsync
// failed and rolled them back; drop any fd cached during that
// window so readers re-open the restored inode (rustfs/backlog#1177).
for part_path in &invalidate_part_paths {
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
}
return Err(to_file_error(err).into());
rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir)
.map_err(to_file_error)?;
// The commit rename changed the dst part inodes before this fsync
// failed and rolled them back; drop any fd cached during that
// window so readers re-open the restored inode (rustfs/backlog#1177).
for part_path in &invalidate_part_paths {
self.io_backend.invalidate_cached_fd(dst_volume, part_path).await;
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
fsync_started,
);
return Err(to_file_error(err).into());
}
// First PUT of an object creates its directory (and any missing prefix
@@ -9303,12 +9293,7 @@ impl DiskAPI for LocalDisk {
if !dir.starts_with(&dst_volume_dir) {
break;
}
let fsync_started = rustfs_io_metrics::put_stage_timer();
if let Err(err) = os::fsync_dir(dir).await {
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
fsync_started,
);
rollback_committed_rename_std(&dst_file_path, committed_new_data_path, rollback_data_dir)
.map_err(to_file_error)?;
// Same post-commit rollback window as above — drop cached
@@ -9319,10 +9304,6 @@ impl DiskAPI for LocalDisk {
}
return Err(to_file_error(err).into());
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
fsync_started,
);
if dir == dst_volume_dir.as_path() {
break;
}
@@ -9551,21 +9532,10 @@ impl DiskAPI for LocalDisk {
}
if let Some(admission) = file_sync_admission.as_ref()
&& let Some(backup_parent) = backup_path.parent()
{
let fsync_started = rustfs_io_metrics::put_stage_timer();
if let Err(err) =
&& let Err(err) =
os::fsync_dir_with_namespace_file_sync_limit(backup_parent, mutation_lease.clone(), admission).await
{
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC,
fsync_started,
);
return Err(DiskError::from(to_file_error(err)));
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC,
fsync_started,
);
{
return Err(DiskError::from(to_file_error(err)));
}
local_rollback_path = None;
}
@@ -9603,22 +9573,11 @@ impl DiskAPI for LocalDisk {
// Persist the commit rename's directory entry across power loss.
if let Some(admission) = file_sync_admission.as_ref()
&& let Some(dst_parent) = dst_file_path.parent()
{
let fsync_started = rustfs_io_metrics::put_stage_timer();
if let Err(err) =
&& let Err(err) =
os::fsync_dir_with_namespace_file_sync_limit(dst_parent, mutation_lease.clone(), admission).await
{
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
fsync_started,
);
rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?;
return Err(err);
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
fsync_started,
);
{
rollback_inline_metadata_commit_std(&dst_file_path, rollback_data_dir, local_rollback_path.as_deref())?;
return Err(err);
}
// Same power-loss gap as the non-inline path (rustfs/backlog#922
@@ -9636,14 +9595,9 @@ impl DiskAPI for LocalDisk {
if !ancestor_dir.starts_with(&dst_volume_dir) {
break;
}
let fsync_started = rustfs_io_metrics::put_stage_timer();
if let Err(err) =
os::fsync_dir_with_namespace_file_sync_limit(ancestor_dir, mutation_lease.clone(), admission).await
{
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
fsync_started,
);
rollback_inline_metadata_commit_std(
&dst_file_path,
rollback_data_dir,
@@ -9651,10 +9605,6 @@ impl DiskAPI for LocalDisk {
)?;
return Err(err);
}
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
fsync_started,
);
if ancestor_dir == dst_volume_dir.as_path() {
break;
}
+6 -95
View File
@@ -343,7 +343,6 @@ pub(crate) async fn acquire_rename_data_mutation_lease(
/// this order uniform prevents one slow disk from reserving global capacity
/// while it waits for its own concurrency slot.
async fn acquire_file_sync_permits(disk_permits: Arc<Semaphore>) -> io::Result<(OwnedSemaphorePermit, SemaphorePermit<'static>)> {
let wait_started = rustfs_io_metrics::put_stage_timer();
let disk_permit = disk_permits
.acquire_owned()
.await
@@ -352,10 +351,6 @@ async fn acquire_file_sync_permits(disk_permits: Arc<Semaphore>) -> io::Result<(
.acquire()
.await
.map_err(|_| io::Error::other("global file sync concurrency limiter closed"))?;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT,
wait_started,
);
Ok((disk_permit, global_permit))
}
@@ -556,19 +551,9 @@ pub(crate) fn sync_file(path: &Path) -> io::Result<()> {
file.sync_data()
}
fn sync_file_with_put_stage_metric(path: &Path) -> io::Result<()> {
let sync_started = rustfs_io_metrics::put_stage_timer();
let result = sync_file(path);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC,
sync_started,
);
result
}
fn sync_files(paths: &[PathBuf]) -> io::Result<()> {
for path in paths {
sync_file_with_put_stage_metric(path)?;
sync_file(path)?;
}
Ok(())
}
@@ -614,13 +599,7 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef<Path>, disk_perm
let files = regular_files(&scan_dir)?;
if files.len() < PARALLEL_FILE_SYNC_THRESHOLD {
sync_files(&files)?;
let fsync_started = rustfs_io_metrics::put_stage_timer();
let result = fsync_dir_std(scan_dir);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC,
fsync_started,
);
result?;
fsync_dir_std(scan_dir)?;
return Ok(None);
}
Ok::<_, io::Error>(Some(files))
@@ -633,19 +612,10 @@ pub(crate) async fn sync_dir_files_with_limiter(dir: impl AsRef<Path>, disk_perm
futures::stream::iter(files.into_iter().map(Ok::<_, io::Error>))
.try_for_each_concurrent(MAX_PARALLEL_FILE_SYNCS, |path| {
let disk_permits = disk_permits.clone();
async move { run_file_sync_blocking(disk_permits, move || sync_file_with_put_stage_metric(&path)).await }
async move { run_file_sync_blocking(disk_permits, move || sync_file(&path)).await }
})
.await?;
run_file_sync_blocking(disk_permits, move || {
let fsync_started = rustfs_io_metrics::put_stage_timer();
let result = fsync_dir_std(dir);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC,
fsync_started,
);
result
})
.await
run_file_sync_blocking(disk_permits, move || fsync_dir_std(dir)).await
}
/// Check if the given disk path is the root disk.
@@ -1204,15 +1174,10 @@ pub(crate) struct FileSyncAdmission {
}
pub(crate) async fn acquire_file_sync_admission(disk_permits: Arc<Semaphore>) -> io::Result<FileSyncAdmission> {
let wait_started = rustfs_io_metrics::put_stage_timer();
let disk_permit = disk_permits
.acquire_owned()
.await
.map_err(|_| io::Error::other("disk file sync concurrency limiter closed"))?;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT,
wait_started,
);
Ok(FileSyncAdmission {
disk_permit: Arc::new(disk_permit),
})
@@ -1235,15 +1200,10 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
global_permits: &Semaphore,
operation: impl FnOnce() -> io::Result<T> + Send + 'static,
) -> io::Result<T> {
let wait_started = rustfs_io_metrics::put_stage_timer();
let global_permit = global_permits
.acquire()
.await
.map_err(|_| io::Error::other("global file sync concurrency limiter closed"))?;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT,
wait_started,
);
let disk_permit = admission.disk_permit.clone();
let result = tokio::task::spawn_blocking(move || {
let _lease = lease;
@@ -1460,13 +1420,7 @@ fn rename_into_existing_parent(
use rustix::fs::{Mode, OFlags, open, renameat};
let Some(parent_guard) = parent_guard else {
let rename_started = rustfs_io_metrics::put_stage_timer();
let result = super::fs::rename_std(src_file_path, dst_file_path);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL,
rename_started,
);
return result;
return super::fs::rename_std(src_file_path, dst_file_path);
};
let src_parent = src_file_path
.parent()
@@ -1487,13 +1441,7 @@ fn rename_into_existing_parent(
.last()
.ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
let rename_started = rustfs_io_metrics::put_stage_timer();
let result = renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from);
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL,
rename_started,
);
result
renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
}
#[cfg(windows)]
@@ -2942,7 +2890,6 @@ pub fn is_dir_not_empty_error(err: &io::Error) -> bool {
#[cfg(test)]
mod tests {
use super::*;
use crate::test_metrics::CapturingRecorder;
use std::sync::Mutex;
use std::time::Duration;
use tempfile::tempdir;
@@ -2963,42 +2910,6 @@ mod tests {
PublicationRoot::new(&common).expect("test publication root should open")
}
#[test]
#[serial_test::serial(file_sync_metrics)]
fn sync_file_with_put_stage_metric_records_fdatasync_only_when_enabled() {
let previous_gate = rustfs_io_metrics::put_stage_metrics_enabled();
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
let dir = tempdir().expect("temp dir should be created");
let path = dir.path().join("part.1");
std::fs::write(&path, b"payload").expect("test file should be written");
let recorder = CapturingRecorder::default();
metrics::with_local_recorder(&recorder, || {
sync_file_with_put_stage_metric(&path).expect("disabled metric sync_file should succeed");
assert_eq!(
recorder.histogram_sample_count("rustfs_s3_put_object_stage_duration_ms"),
0,
"disabled PUT stage metrics must not emit fdatasync samples"
);
rustfs_io_metrics::set_put_stage_metrics_enabled(true);
sync_file_with_put_stage_metric(&path).expect("enabled metric sync_file should succeed");
rustfs_io_metrics::set_put_stage_metrics_enabled(false);
});
assert_eq!(
recorder
.histogram_values(
"rustfs_s3_put_object_stage_duration_ms",
&[("stage", rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC)]
)
.len(),
1,
"enabled PUT stage metrics must emit one fdatasync sample"
);
rustfs_io_metrics::set_put_stage_metrics_enabled(previous_gate);
}
async fn rename_all(
src_file_path: impl AsRef<Path>,
dst_file_path: impl AsRef<Path>,
@@ -3389,15 +3389,8 @@ impl SetDisks {
// A no-op immediately-ready future in production.
Self::rename_fanout_barrier(&dst_object, i, rename_fanout_barrier_phase::RENAME).await;
let disk_wait_started = rustfs_io_metrics::put_stage_timer();
let result = disk
.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
disk_wait_started,
);
result
disk.rename_data_borrowed(&src_bucket, &src_object, file_info, &dst_bucket, &dst_object)
.await
})
.catch_unwind()
});
@@ -3410,13 +3403,7 @@ impl SetDisks {
let mut cleanup_data_dirs = vec![None; disk_count];
let mut old_current_sizes = vec![None; disk_count];
let quorum_wait_started = rustfs_io_metrics::put_stage_timer();
let fanout_result = fanout.await;
rustfs_io_metrics::record_put_object_stage_duration_from(
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
quorum_wait_started,
);
let (results, mut file_infos) = fanout_result.map_err(|_| DiskError::Unexpected)?;
let (results, mut file_infos) = fanout.await.map_err(|_| DiskError::Unexpected)?;
for (idx, result) in results.iter().enumerate() {
match result {
-69
View File
@@ -109,17 +109,6 @@ pub fn put_stage_timer() -> Option<std::time::Instant> {
put_stage_metrics_enabled().then(std::time::Instant::now)
}
pub const PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT: &str = "set_disk_rename_quorum_wait";
pub const PUT_STAGE_SET_DISK_RENAME_DISK_WAIT: &str = "set_disk_rename_disk_wait";
pub const PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_file_sync_permit_wait";
pub const PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT: &str = "set_disk_rename_global_file_sync_permit_wait";
pub const PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC: &str = "set_disk_rename_file_fdatasync";
pub const PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC: &str = "set_disk_rename_src_dir_fsync";
pub const PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC: &str = "set_disk_rename_dst_dir_fsync";
pub const PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC: &str = "set_disk_rename_backup_dir_fsync";
pub const PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC: &str = "set_disk_rename_ancestor_dir_fsync";
pub const PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL: &str = "set_disk_rename_rename_syscall";
#[inline(always)]
pub fn get_stage_metrics_enabled() -> bool {
GET_STAGE_METRICS_ENABLED.load(Ordering::Relaxed)
@@ -2629,7 +2618,6 @@ mod tests {
use super::*;
use metrics_util::MetricKind;
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
use std::collections::HashSet;
use std::sync::{Arc, Barrier, Mutex};
// Serialize tests that mutate the process-global PUT_STAGE_METRICS_ENABLED flag.
@@ -2873,63 +2861,6 @@ mod tests {
set_put_stage_metrics_enabled(false);
}
#[test]
fn put_stage_sync_tail_labels_are_static_and_gated() {
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
let stages = [
PUT_STAGE_SET_DISK_RENAME_QUORUM_WAIT,
PUT_STAGE_SET_DISK_RENAME_DISK_WAIT,
PUT_STAGE_SET_DISK_RENAME_FILE_SYNC_PERMIT_WAIT,
PUT_STAGE_SET_DISK_RENAME_GLOBAL_FILE_SYNC_PERMIT_WAIT,
PUT_STAGE_SET_DISK_RENAME_FILE_FDATASYNC,
PUT_STAGE_SET_DISK_RENAME_SRC_DIR_FSYNC,
PUT_STAGE_SET_DISK_RENAME_DST_DIR_FSYNC,
PUT_STAGE_SET_DISK_RENAME_BACKUP_DIR_FSYNC,
PUT_STAGE_SET_DISK_RENAME_ANCESTOR_DIR_FSYNC,
PUT_STAGE_SET_DISK_RENAME_RENAME_SYSCALL,
];
let unique = stages.iter().copied().collect::<HashSet<_>>();
assert_eq!(unique.len(), stages.len());
assert!(
stages
.iter()
.all(|stage| stage.starts_with("set_disk_rename_") && !stage.contains('/') && !stage.contains('{'))
);
let recorder = DebuggingRecorder::new();
let snapshotter = recorder.snapshotter();
metrics::with_local_recorder(&recorder, || {
set_put_stage_metrics_enabled(false);
for stage in stages {
record_put_object_stage_duration(stage, 1.0);
}
set_put_stage_metrics_enabled(true);
for stage in stages {
record_put_object_stage_duration(stage, 1.0);
}
set_put_stage_metrics_enabled(false);
});
let recorded = snapshotter
.snapshot()
.into_vec()
.into_iter()
.filter(|(composite, _, _, _)| {
composite.kind() == MetricKind::Histogram && composite.key().name() == "rustfs_s3_put_object_stage_duration_ms"
})
.flat_map(|(composite, _, _, _)| {
composite
.key()
.labels()
.filter(|label| label.key() == "stage")
.map(|label| label.value().to_string())
.collect::<Vec<_>>()
})
.collect::<HashSet<_>>();
assert_eq!(recorded.len(), stages.len());
assert!(stages.iter().all(|stage| recorded.contains(*stage)));
}
#[test]
fn test_put_object_diagnostic_buckets() {
assert_eq!(put_object_size_bucket(0), "unknown");
+4 -24
View File
@@ -1729,30 +1729,10 @@ mod tests {
assert!(config.validate().is_ok(), "deprecated mount_path must not be required");
}
#[test]
fn test_vault_kv2_sources_do_not_claim_transit_wrapping() {
let sources = [
("config.rs", include_str!("config.rs")),
("api_types.rs", include_str!("api_types.rs")),
("backends/vault.rs", include_str!("backends/vault.rs")),
("lib.rs", include_str!("lib.rs")),
];
// Assemble the needles at runtime so this guard does not match its own source.
let needles = [
format!("wrapping via {}", "Transit"),
format!("KV v2 + {}", "Transit"),
format!("KV2+{}", "Transit"),
format!("you would use Vault's {} engine", "transit"),
];
for (name, source) in sources {
for needle in &needles {
assert!(
!source.contains(needle.as_str()),
"{name} still describes the Vault KV2 backend with `{needle}`"
);
}
}
}
// The "VaultKv2 must not claim Transit wrapping" documentation-claim
// invariant is enforced by scripts/check_fips_wording.sh, which scans every
// file in crates/kms rather than a fixed include_str! list
// (rustfs/backlog#1884).
#[test]
fn test_legacy_persisted_vault_transit_config_uses_metadata_defaults() {
+35
View File
@@ -1199,6 +1199,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_zero_expiration_days() {
// S3 compatibility: Expiration.Days must be a positive integer (>= 1). AWS and
// the ceph s3-tests `test_lifecycle_expiration_days0` case reject Days == 0 with
@@ -1232,6 +1233,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_negative_expiration_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1261,6 +1263,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_accepts_positive_expiration_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1287,6 +1290,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_accepts_one_day_boundary_values() {
// Pin the exact >= 1 boundary: a value of 1 is the smallest legal positive
// integer and must be accepted for every day-count field tightened for S3
@@ -1321,6 +1325,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn has_active_rules_accepts_zero_day_expiration() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1345,6 +1350,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_zero_noncurrent_expiration_days() {
// S3 compatibility: NoncurrentVersionExpiration.NoncurrentDays must be a positive
// integer (>= 1); AWS rejects 0 with InvalidArgument.
@@ -1376,6 +1382,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_negative_noncurrent_expiration_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1405,6 +1412,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_accepts_abort_incomplete_multipart_upload_only_rule() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1430,6 +1438,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_zero_abort_incomplete_multipart_upload_days() {
// S3 compatibility: AbortIncompleteMultipartUpload.DaysAfterInitiation must be a
// positive integer (>= 1); AWS rejects 0 with InvalidArgument.
@@ -1460,6 +1469,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_missing_abort_incomplete_multipart_upload_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1485,6 +1495,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_negative_abort_incomplete_multipart_upload_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1545,6 +1556,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_non_midnight_expiration_date() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1626,6 +1638,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_accepts_multiple_rules_without_ids() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1669,6 +1682,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_rule_id_too_long() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1695,6 +1709,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_duplicate_rule_ids() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1737,6 +1752,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_transition_without_storage_class() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1764,6 +1780,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_transition_without_date_or_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -1791,6 +1808,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_noncurrent_transition_without_days() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -2347,6 +2365,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn noncurrent_versions_expiration_limit_returns_configured_limits() {
let lc = Arc::new(BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -2437,6 +2456,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_invalid_status_case_sensitive() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -2463,6 +2483,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn filter_rules_respects_filter_prefix() {
let filter = LifecycleRuleFilter {
prefix: Some("prefix".to_string()),
@@ -2507,6 +2528,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn filter_rules_respects_filter_and_prefix() {
let and = s3s::dto::LifecycleRuleAndOperator {
prefix: Some("prefix".to_string()),
@@ -2556,6 +2578,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn filter_rules_respects_filter_tag() {
let filter = LifecycleRuleFilter {
tag: Some(s3s::dto::Tag {
@@ -2609,6 +2632,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn filter_rules_respects_filter_and_tags() {
let filter = LifecycleRuleFilter {
and: Some(s3s::dto::LifecycleRuleAndOperator {
@@ -3062,6 +3086,7 @@ mod tests {
// --- TASK-002 tests: Object Lock + ExpiredObjectDeleteMarker compatibility ---
#[tokio::test]
#[serial]
async fn validate_allows_expired_object_delete_marker_on_locked_bucket() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3093,6 +3118,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_allows_expired_object_delete_marker_on_unlocked_bucket() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3120,6 +3146,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_allows_non_delete_marker_expiration_on_locked_bucket() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3152,6 +3179,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_del_marker_expiration_on_locked_bucket() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3182,6 +3210,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_rejects_zero_day_del_marker_expiration_on_locked_bucket() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3565,6 +3594,7 @@ mod tests {
// --- TASK-007 tests: Legacy Prefix/Filter conflict ---
#[tokio::test]
#[serial]
async fn validate_rejects_prefix_and_filter_both_present() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3593,6 +3623,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_allows_prefix_without_filter() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3619,6 +3650,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_allows_filter_without_prefix() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3648,6 +3680,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_allows_empty_prefix_with_filter() {
// Empty prefix should be treated as "not set"
let lc = BucketLifecycleConfiguration {
@@ -3680,6 +3713,7 @@ mod tests {
// --- TASK-004 tests: ExpiredObjectAllVersions ---
#[tokio::test]
#[serial]
async fn validate_rejects_expired_object_all_versions_on_locked_bucket() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
@@ -3711,6 +3745,7 @@ mod tests {
}
#[tokio::test]
#[serial]
async fn validate_allows_expired_object_all_versions_on_unlocked_bucket() {
let lc = BucketLifecycleConfiguration {
expiry_updated_at: None,
+12
View File
@@ -4574,6 +4574,7 @@ mod tests {
}
#[test]
#[serial]
fn test_randomized_cycle_delay_keeps_configured_start_delay() {
// 120s with ±10% jitter should stay clearly above the historic 30s cap.
let delay = randomized_cycle_delay_for(Duration::from_secs(120));
@@ -4592,6 +4593,7 @@ mod tests {
}
#[test]
#[serial]
fn test_initial_scanner_delay_uses_configured_start_delay() {
let delay = initial_scanner_delay_for(Some(120));
assert!(delay >= Duration::from_secs(108));
@@ -4611,12 +4613,14 @@ mod tests {
}
#[test]
#[serial]
fn test_initial_scanner_delay_skips_for_cold_usage_cache_with_buckets() {
let delay = initial_scanner_delay_for_startup(Some(120), true, true, false);
assert_eq!(delay, Duration::ZERO);
}
#[test]
#[serial]
fn test_initial_scanner_delay_keeps_configured_delay_for_warm_usage_cache_no_replication() {
let delay = initial_scanner_delay_for_startup(Some(120), false, true, false);
assert!(delay >= Duration::from_secs(108));
@@ -4624,12 +4628,14 @@ mod tests {
}
#[test]
#[serial]
fn test_initial_scanner_delay_skips_for_cold_usage_cache_without_buckets() {
let delay = initial_scanner_delay_for_startup(Some(120), true, false, false);
assert_eq!(delay, Duration::ZERO);
}
#[test]
#[serial]
fn test_initial_scanner_delay_skips_for_active_replication_warm_cache() {
// Warm cache + active replication rules → skip startup delay so that FAILED-status objects
// from a crash are healed on the first cycle, not after a 27-33 min sleep.
@@ -4638,6 +4644,7 @@ mod tests {
}
#[test]
#[serial]
fn test_initial_scanner_delay_keeps_delay_for_replication_without_buckets() {
// Active replication but no buckets → no objects to scan, keep normal delay.
let delay = initial_scanner_delay_for_startup(Some(120), false, false, true);
@@ -7392,6 +7399,7 @@ mod tests {
}
#[test]
#[serial]
fn clean_idle_cap_allows_policy_max_when_bitrot_is_disabled() {
let config = ScannerRuntimeConfig {
bitrot_cycle: None,
@@ -7507,6 +7515,7 @@ mod tests {
}
#[test]
#[serial]
fn test_randomized_cycle_delay_handles_small_start_delay() {
// 0 is treated as minimum 1 second before jitter, with lower bound preserved.
let delay = randomized_cycle_delay_for(Duration::from_secs(0));
@@ -8165,6 +8174,7 @@ mod tests {
}
#[test]
#[serial]
fn test_background_heal_info_for_scan_complete_marks_deep_idle() {
let started_at = Utc::now();
let info = BackgroundHealInfo {
@@ -8182,6 +8192,7 @@ mod tests {
}
#[test]
#[serial]
fn test_background_heal_info_for_scan_complete_leaves_normal_scan_unchanged() {
let info = BackgroundHealInfo {
bitrot_start_time: Some(Utc::now()),
@@ -8193,6 +8204,7 @@ mod tests {
}
#[test]
#[serial]
fn test_background_heal_info_for_failed_scan_preserves_deep_mode() {
let info = BackgroundHealInfo {
bitrot_start_time: Some(Utc::now()),
+1 -9
View File
@@ -92,13 +92,6 @@ pub struct TestECStoreEnv {
/// `init_local_disks` + `ECStore::new` on `127.0.0.1:0` (random port keeps
/// nextest's process-per-test parallelism safe).
pub ecstore: Arc<ECStore>,
/// The single-pool, single-set topology the store was built from.
///
/// The bootstrap does **not** publish it on the instance context (server
/// startup is what calls `set_endpoints`, and that write is once-only), so
/// a test that needs `get_global_endpoints` to resolve — admin server-info
/// and other topology readers — publishes this value itself.
pub endpoint_pools: EndpointServerPools,
}
impl TestECStoreEnv {
@@ -241,7 +234,7 @@ impl TestECStoreEnvBuilder {
// Port 0 keeps ECStore-backed integration binaries parallel-safe under
// nextest: no fixed peer port is ever shared between test processes.
let server_addr: std::net::SocketAddr = "127.0.0.1:0".parse().expect("parse test addr");
let ecstore = ECStore::new(server_addr, endpoint_pools.clone(), CancellationToken::new())
let ecstore = ECStore::new(server_addr, endpoint_pools, CancellationToken::new())
.await
.expect("build test ECStore");
@@ -261,7 +254,6 @@ impl TestECStoreEnvBuilder {
temp_root,
disk_paths,
ecstore,
endpoint_pools,
}
}
}
+14 -2
View File
@@ -20,7 +20,7 @@ repository.workspace = true
rust-version.workspace = true
version.workspace = true
homepage.workspace = true
description = "Archive format detection and async stream decoders for RustFS."
description = "ZIP file handling for RustFS, providing support for reading and writing ZIP archives."
keywords = ["zip", "compression", "rustfs", "Minio"]
categories = ["web-programming", "development-tools", "compression"]
documentation = "https://docs.rs/rustfs-zip/latest/rustfs_zip/"
@@ -28,6 +28,10 @@ documentation = "https://docs.rs/rustfs-zip/latest/rustfs_zip/"
[lib]
doctest = false
[[bench]]
name = "zip_benchmark"
harness = false
[features]
default = []
hotpath = ["hotpath/hotpath", "hotpath/tokio"]
@@ -44,8 +48,16 @@ async-compression = { workspace = true, features = [
"zstd",
"xz",
] }
tokio = { workspace = true, features = ["io-util", "macros", "rt"] }
tokio = { workspace = true, features = ["fs", "io-util", "macros", "rt-multi-thread"] }
tokio-stream = { workspace = true }
astral-tokio-tar = { workspace = true }
thiserror = { workspace = true }
zip = { workspace = true }
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
tempfile = { workspace = true }
[lints]
workspace = true
+42 -12
View File
@@ -1,9 +1,9 @@
[![RustFS](https://rustfs.com/images/rustfs-github.png)](https://rustfs.com)
# RustFS Zip - Archive Format Detection And Stream Decoding
# RustFS Zip - Archive And Compression Primitives
<p align="center">
<strong>Archive format detection and async stream decoders for RustFS object storage</strong>
<strong>High-performance compression and archiving for RustFS object storage</strong>
</p>
<p align="center">
@@ -17,23 +17,53 @@
## 📖 Overview
**RustFS Zip** provides the archive primitives used by the [RustFS](https://rustfs.com) archive extract flow:
**RustFS Zip** provides archive and compression primitives for the [RustFS](https://rustfs.com) distributed object storage system. Today it is primarily used by RustFS archive extract flows to:
- identify a compression format from an archive extension
- wrap an async reader in the matching stream decoder
- carry the shared default archive guardrails
- identify archive/compression formats by extension
- stream tar and tar+compression inputs through async decoders
- provide small ZIP read/write helpers for local archive workflows
## Current Features
- `CompressionFormat::from_extension()` for extension-based format detection, including tar-family suffixes such as `tgz`, `tbz2`, `txz`, and `tzst`
- `CompressionFormat::get_decoder()` for async stream decoding of `gzip`, `bzip2`, `zlib`, `xz`, and `zstd`, plus a pass-through reader for plain `tar`
- `ArchiveLimits` with the default entry count, entry size, total unpacked size, and path length guardrails
- A clearer type model with:
- `CompressionCodec` for stream codecs
- `ArchiveKind` for container families
- `ArchiveFormat` for concrete archive/container combinations
- Async stream codecs for `gzip`, `bzip2`, `zlib`, `xz`, and `zstd`
- Tar archive iteration over async readers through `read_archive_entries()` / `extract_tar_entries()`
- Archive guardrails through `ArchiveLimits` for entry count, entry size, total unpacked size, and path length
- In-memory compression helpers for payload round-trip workflows
- Blocking ZIP create/extract helpers for local archive files
- ZIP helper metadata via `ZipEntry`, including:
- `compression_method`
- `archive_kind`
- `format`
- `unix_mode`
- ZIP helper options via `ZipWriteOptions`, including:
- `compression_level`
- `create_directory_entries`
## Compatibility
- `CompressionFormat` is retained as a compatibility layer for existing callers
- New code should prefer `ArchiveFormat`, `ArchiveKind`, and `CompressionCodec` when expressing archive semantics
## ZIP Helper Scope
The file-based ZIP helper APIs are best suited for:
- local archive import/export flows
- admin-side packaging helpers
- test fixtures and tooling
They are not intended to be a remote streaming ZIP access engine.
## Current Boundaries
- ZIP has no stream decoder: `get_decoder()` rejects `CompressionFormat::Zip`, because ZIP needs central-directory semantics that a forward-only stream cannot provide
- This crate detects formats and hands back decoders; archive iteration, entry writing, and extraction to disk belong to the caller
- `ArchiveLimits` carries the values only; enforcement and the resulting protocol error belong to the caller
- ZIP is supported via file-based helper APIs, not the tar-family async stream APIs
- Tar-family stream APIs are intended for `tar`, `tar.gz`, `tar.bz2`, `tar.xz`, `tar.zst`, and similar compressed tar flows
- Default archive guardrails are intentionally conservative and do not replace higher-level RustFS object-path validation
- This crate does not currently implement a general-purpose parallel archive engine
- Archive extraction safety policy remains the responsibility of the RustFS caller for object-store flows
## 📚 Documentation
+416
View File
@@ -0,0 +1,416 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_zip::{
ArchiveLimits, CompressionFormat, CompressionLevel, ZipWriteOptions, create_zip_with_options, extract_tar_entries,
extract_zip_to_path_with_limits, extract_zip_with_limits,
};
use std::hint::black_box;
use std::sync::Arc;
use std::sync::atomic::{AtomicUsize, Ordering};
use tempfile::tempdir;
use tokio::runtime::Builder;
use tokio_tar::{Builder as TarBuilder, Header};
use zip::ZipArchive;
fn build_runtime() -> tokio::runtime::Runtime {
Builder::new_current_thread()
.enable_all()
.build()
.expect("build tokio runtime for rustfs-zip benchmarks")
}
async fn build_tar_payload(entry_count: usize, payload_size: usize) -> Vec<u8> {
let sink = tokio::io::duplex(64 * 1024);
let (writer, mut reader) = sink;
let write_task = tokio::spawn(async move {
let mut builder = TarBuilder::new(writer);
let payload = vec![b'a'; payload_size];
for index in 0..entry_count {
let mut header = Header::new_gnu();
header.set_size(payload.len() as u64);
header.set_mode(0o644);
header.set_cksum();
builder
.append_data(&mut header, format!("entry-{index}.txt"), &payload[..])
.await
.expect("append tar benchmark entry");
}
builder.finish().await.expect("finish tar benchmark archive");
});
let mut output = Vec::new();
tokio::io::copy(&mut reader, &mut output)
.await
.expect("read tar benchmark archive");
write_task.await.expect("join tar writer task");
output
}
async fn build_compressed_tar_payload(format: CompressionFormat, entry_count: usize, payload_size: usize) -> Vec<u8> {
let tar_payload = build_tar_payload(entry_count, payload_size).await;
rustfs_zip::Compressor::new(format)
.compress(&tar_payload)
.await
.expect("compress tar benchmark payload")
}
fn bench_tar_family_extract(c: &mut Criterion) {
let runtime = build_runtime();
let mut group = c.benchmark_group("zip_tar_family_extract");
for (name, format, entry_count, payload_size) in [
("tar_gzip_small_many", CompressionFormat::Gzip, 64usize, 256usize),
("tar_zstd_medium", CompressionFormat::Zstd, 16usize, 16 * 1024usize),
] {
let payload = runtime.block_on(build_compressed_tar_payload(format, entry_count, payload_size));
group.throughput(Throughput::Bytes(payload.len() as u64));
group.bench_with_input(BenchmarkId::new(name, payload.len()), &payload, |b, payload| {
b.iter(|| {
runtime.block_on(async {
let seen = Arc::new(AtomicUsize::new(0));
let seen_ref = Arc::clone(&seen);
extract_tar_entries(std::io::Cursor::new(payload.clone()), format, move |_entry| {
let seen_ref = Arc::clone(&seen_ref);
async move {
seen_ref.fetch_add(1, Ordering::Relaxed);
Ok(())
}
})
.await
.expect("extract tar benchmark payload");
black_box(seen.load(Ordering::Relaxed));
});
});
});
}
group.finish();
}
fn bench_zip_helper_round_trip(c: &mut Criterion) {
let runtime = build_runtime();
let mut group = c.benchmark_group("zip_helper_round_trip");
let zip_matrix = [
("stored_flat_32x128", CompressionLevel::Fastest, 32usize, 128usize, "flat"),
("stored_nested_32x256", CompressionLevel::Fastest, 32usize, 256usize, "nested"),
("stored_flat_256x128", CompressionLevel::Fastest, 256usize, 128usize, "flat"),
("deflated_flat_32x1k", CompressionLevel::Best, 32usize, 1024usize, "flat"),
("deflated_nested_256x1k", CompressionLevel::Best, 256usize, 1024usize, "nested"),
("deflated_deep_1024x4k", CompressionLevel::Best, 1024usize, 4 * 1024usize, "deep"),
];
for (name, compression_level, file_count, payload_size, layout) in zip_matrix {
let files = (0..file_count)
.map(|index| {
let path = match layout {
"flat" => format!("file-{index}.txt"),
"nested" => format!("batch-{}/file-{index}.txt", index % 8),
"deep" => format!("lvl1/lvl2-{}/lvl3-{}/file-{index}.txt", index % 16, index % 32),
_ => format!("file-{index}.txt"),
};
(path, vec![b'b'; payload_size])
})
.collect::<Vec<_>>();
let total_bytes = (file_count * payload_size) as u64;
group.throughput(Throughput::Bytes(total_bytes));
group.bench_with_input(BenchmarkId::new(name, total_bytes), &files, |b, files| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
let extract_path = temp.path().join("extract");
runtime.block_on(async {
create_zip_with_options(
&zip_path,
files.clone(),
ZipWriteOptions {
compression_level,
create_directory_entries: true,
},
)
.await
.expect("create zip benchmark archive");
let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default())
.await
.expect("extract zip benchmark archive");
black_box(entries.len());
});
});
});
}
group.finish();
}
fn bench_zip_helper_hotspot_breakdown(c: &mut Criterion) {
let runtime = build_runtime();
let mut group = c.benchmark_group("zip_helper_hotspot_breakdown");
let files = (0..32)
.map(|index| (format!("batch/file-{index}.txt"), vec![b'c'; 256]))
.collect::<Vec<_>>();
let total_bytes = (32 * 256) as u64;
group.throughput(Throughput::Bytes(total_bytes));
group.bench_function("fs_setup_cleanup_only", |b| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
let extract_path = temp.path().join("extract");
black_box((zip_path, extract_path));
});
});
group.bench_function("zip_create_only_stored_small", |b| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
runtime.block_on(async {
create_zip_with_options(
&zip_path,
files.clone(),
ZipWriteOptions {
compression_level: CompressionLevel::Fastest,
create_directory_entries: true,
},
)
.await
.expect("create zip benchmark archive");
});
});
});
let payload_for_extract = {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
runtime.block_on(async {
create_zip_with_options(
&zip_path,
files.clone(),
ZipWriteOptions {
compression_level: CompressionLevel::Fastest,
create_directory_entries: true,
},
)
.await
.expect("prepare zip benchmark extract payload");
});
std::fs::read(&zip_path).expect("read benchmark zip payload")
};
group.bench_function("zip_extract_only_stored_small", |b| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
let extract_path = temp.path().join("extract");
std::fs::write(&zip_path, &payload_for_extract).expect("write benchmark zip payload");
runtime.block_on(async {
let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default())
.await
.expect("extract zip benchmark archive");
black_box(entries.len());
});
});
});
group.bench_function("zip_extract_only_stored_small_summary_only", |b| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
let extract_path = temp.path().join("extract");
std::fs::write(&zip_path, &payload_for_extract).expect("write benchmark zip payload");
runtime.block_on(async {
let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default())
.await
.expect("extract zip benchmark summary path");
black_box(summary.entry_count);
});
});
});
group.bench_function("zip_reader_only_stored_small", |b| {
b.iter(|| {
let cursor = std::io::Cursor::new(payload_for_extract.clone());
let mut archive = ZipArchive::new(cursor).expect("open zip archive for reader-only benchmark");
let mut total_bytes = 0usize;
for index in 0..archive.len() {
let mut zip_file = archive.by_index(index).expect("access zip entry by index");
let enclosed_name = zip_file
.enclosed_name()
.expect("resolve enclosed zip entry name")
.to_string_lossy()
.replace('\\', "/");
let size = zip_file.size();
assert!(!enclosed_name.is_empty(), "zip reader-only benchmark expects non-empty names");
assert!(
size <= ArchiveLimits::default().max_entry_size,
"zip reader-only benchmark expects small entries"
);
if !zip_file.is_dir() {
let mut sink = [0_u8; 256];
let bytes_read =
std::io::Read::read(&mut zip_file, &mut sink).expect("read zip entry payload for reader-only benchmark");
total_bytes += bytes_read;
}
}
black_box(total_bytes);
});
});
group.bench_function("file_write_only_stored_small", |b| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let extract_path = temp.path().join("extract");
std::fs::create_dir_all(&extract_path).expect("create extract dir for file-write-only benchmark");
let mut total_bytes = 0usize;
for index in 0..32 {
let path = extract_path.join(format!("file-{index}.txt"));
std::fs::write(&path, [b'c'; 256]).expect("write small file for file-write-only benchmark");
total_bytes += 256;
}
black_box(total_bytes);
});
});
group.finish();
}
fn build_object_archive_files(
metadata_count: usize,
metadata_size: usize,
payload_count: usize,
payload_size: usize,
) -> Vec<(String, Vec<u8>)> {
let mut files = Vec::with_capacity(metadata_count * 2 + payload_count);
for index in 0..metadata_count {
let key_prefix = format!(
"bucket-a/shard-{}/tenant-{}/dataset-{}/object-{index:04}",
index % 8,
index % 16,
index % 32
);
files.push((
format!("{key_prefix}/meta.json"),
format!(
"{{\"key\":\"object-{index:04}\",\"etag\":\"{:032x}\",\"size\":{},\"content_type\":\"application/octet-stream\"}}",
index,
payload_size
)
.into_bytes(),
));
files.push((format!("{key_prefix}/tags.txt"), vec![b'm'; metadata_size]));
}
for index in 0..payload_count {
let payload_prefix = format!(
"bucket-a/shard-{}/tenant-{}/dataset-{}/object-{index:04}",
index % 8,
index % 16,
index % 32
);
files.push((format!("{payload_prefix}/part-00000.bin"), vec![b'p'; payload_size]));
}
files
}
fn bench_zip_object_archive_extract(c: &mut Criterion) {
let runtime = build_runtime();
let mut group = c.benchmark_group("zip_object_archive_extract");
for (name, compression_level, metadata_count, metadata_size, payload_count, payload_size) in [
(
"stored_metadata_heavy_384m_24p",
CompressionLevel::Fastest,
384usize,
192usize,
24usize,
32 * 1024usize,
),
(
"deflated_mixed_192m_32p",
CompressionLevel::Best,
192usize,
256usize,
32usize,
64 * 1024usize,
),
] {
let files = build_object_archive_files(metadata_count, metadata_size, payload_count, payload_size);
let total_bytes = files.iter().map(|(_, payload)| payload.len() as u64).sum::<u64>();
let payload = {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("object-archive.zip");
runtime.block_on(async {
create_zip_with_options(
&zip_path,
files.clone(),
ZipWriteOptions {
compression_level,
create_directory_entries: true,
},
)
.await
.expect("create object archive benchmark payload");
});
std::fs::read(&zip_path).expect("read object archive benchmark payload")
};
group.throughput(Throughput::Bytes(total_bytes));
group.bench_function(BenchmarkId::new("extract_full", name), |b| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
let extract_path = temp.path().join("extract");
std::fs::write(&zip_path, &payload).expect("write object archive benchmark payload");
runtime.block_on(async {
let entries = extract_zip_with_limits(&zip_path, &extract_path, ArchiveLimits::default())
.await
.expect("extract object archive benchmark payload");
black_box(entries.len());
});
});
});
group.bench_function(BenchmarkId::new("extract_summary_only", name), |b| {
b.iter(|| {
let temp = tempdir().expect("create benchmark tempdir");
let zip_path = temp.path().join("archive.zip");
let extract_path = temp.path().join("extract");
std::fs::write(&zip_path, &payload).expect("write object archive benchmark payload");
runtime.block_on(async {
let summary = extract_zip_to_path_with_limits(&zip_path, &extract_path, ArchiveLimits::default())
.await
.expect("extract object archive benchmark summary");
black_box(summary.file_count);
});
});
});
}
group.finish();
}
criterion_group!(
benches,
bench_tar_family_extract,
bench_zip_helper_round_trip,
bench_zip_helper_hotspot_breakdown,
bench_zip_object_archive_extract
);
criterion_main!(benches);
+1643 -46
View File
File diff suppressed because it is too large Load Diff
@@ -53,6 +53,8 @@ Suggested boilerplate when the topic cannot be avoided:
`README.md` and `CHANGELOG.md` currently contain no FIPS-related wording; `scripts/check_fips_wording.sh` is the grep guard for that public baseline. Any future occurrence of the banned strings in either file should be treated as a defect and either removed or brought under the qualifier rule above. This document intentionally contains the terminology needed to define the policy and is not part of that narrow outward-material scan.
The same script carries a second block for the adjacent over-claim: no file under `crates/kms` may describe the Vault KV2 backend as wrapping key material through Vault's Transit engine. `KmsBackend::VaultKv2` stores RustFS-wrapped key material in Vault's KV v2 engine and never calls Transit, so that wording would tell an operator their key material is cryptographically isolated inside Vault when it is not. Use the `VaultTransit` backend when that isolation is the requirement.
## The `rustfs-crypto` `fips` feature: what it actually does
`crates/crypto/Cargo.toml` declares `default = ["crypto", "fips"]`, so the feature is on in every normal build. Its entire effect is **which algorithm the write path selects**; the implementation is RustCrypto either way.
-78
View File
@@ -1541,84 +1541,6 @@ mod tests {
assert_eq!(error.code(), &S3ErrorCode::InvalidRequest);
}
/// `ServerInfoHandler` must answer an authorized admin request with the
/// per-pool erasure-set topology (rustfs/backlog#1839). That map is only
/// filled when the server-info query is issued with pools included, so a
/// handler that stopped asking for them would still return 200 with an
/// empty `pools` object instead of failing.
#[tokio::test]
#[serial_test::serial]
async fn server_info_response_carries_pool_topology() {
use crate::admin::runtime_sources::{AppContext, publish_test_app_context};
use crate::admin::storage_api::runtime::bootstrap_ctx;
use http_body_util::BodyExt as _;
use rustfs_iam::store::{Store as _, object::IAM_CONFIG_PREFIX};
use std::sync::Arc;
const ROOT_ACCESS_KEY: &str = "SERVERINFOROOTACCESSKEY";
const ROOT_SECRET_KEY: &str = "serverInfoRootSecret123";
let _ = rustfs_credentials::init_global_action_credentials(
Some(ROOT_ACCESS_KEY.to_string()),
Some(ROOT_SECRET_KEY.to_string()),
);
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("admin_server_info_pools")
.disk_count(1)
.init_bucket_metadata(false)
.build()
.await;
// Server startup owns this write in production; the test bootstrap
// stops short of it, and without a topology the server-info query
// returns before it ever looks at drives.
bootstrap_ctx().set_endpoints(env.endpoint_pools.clone());
rustfs_iam::store::object::ObjectStore::new(Arc::clone(&env.ecstore))
.save_iam_config(serde_json::json!({"version": 1}), format!("{}/format.json", *IAM_CONFIG_PREFIX))
.await
.expect("seed IAM format");
let iam = rustfs_iam::build_iam_sys(Arc::clone(&env.ecstore))
.await
.expect("build test IAM");
publish_test_app_context(Arc::new(AppContext::with_default_interfaces(
Arc::clone(&env.ecstore),
iam,
Arc::new(rustfs_kms::KmsServiceManager::new()),
)));
let request = S3Request {
input: Body::empty(),
method: Method::GET,
uri: Uri::from_static("/rustfs/admin/v3/info"),
headers: HeaderMap::new(),
extensions: Extensions::new(),
credentials: Some(s3s::auth::Credentials {
access_key: ROOT_ACCESS_KEY.to_string(),
secret_key: s3s::auth::SecretKey::from(ROOT_SECRET_KEY.to_string()),
}),
region: None,
service: None,
trailing_headers: None,
};
let (status, body) = super::ServerInfoHandler {}
.call(request, Params::new())
.await
.expect("root admin credentials must be served server info")
.output;
assert_eq!(status, hyper::StatusCode::OK);
let bytes = body.collect().await.expect("server info body should read").to_bytes();
let payload: serde_json::Value = serde_json::from_slice(&bytes).expect("server info must be json");
let pools = payload["info"]["pools"]
.as_object()
.expect("server info must carry a pools object");
assert!(
pools.contains_key("0"),
"server info must report the erasure-set topology of pool 0, got {pools:?}"
);
}
/// Authorization denial for this exact action is pinned to AccessDenied by
/// `crate::admin::auth::tests::non_admin_credential_is_denied`.
#[test]
@@ -1412,6 +1412,23 @@ fn test_health_routes_not_registered_when_disabled_by_env() {
});
}
#[test]
fn test_phase5_admin_info_contract() {
let system_src = include_str!("handlers/system.rs");
let server_info_impl_marker = "impl Operation for ServerInfoHandler";
let server_info_impl_start = system_src
.find(server_info_impl_marker)
.expect("Expected impl Operation for ServerInfoHandler in handlers/system.rs");
let server_info_impl_block = &system_src[server_info_impl_start..];
assert!(
server_info_impl_block.contains("default_admin_usecase()")
&& server_info_impl_block.contains("execute_query_server_info(QueryServerInfoRequest { include_pools: true })"),
"admin server info path must be served through admin runtime-source DefaultAdminUsecase::execute_query_server_info"
);
}
fn extract_block_between_markers<'a>(src: &'a str, start_marker: &str, end_marker: &str) -> &'a str {
let start = src
.find(start_marker)
-4
View File
@@ -948,10 +948,6 @@ pub(crate) mod runtime {
#[cfg(test)]
pub(crate) use super::{Endpoint, Endpoints, PoolEndpoints};
/// Test-only: the process instance context, so a handler test can publish
/// the endpoint topology that server startup normally installs.
#[cfg(test)]
pub(crate) use crate::storage::storage_api::ecstore_runtime::bootstrap_ctx;
}
pub(crate) mod s3 {
+78 -9
View File
@@ -1,10 +1,30 @@
#!/usr/bin/env bash
set -euo pipefail
# Guard: outward README and CHANGELOG material must not make an unsupported
# FIPS validation or certification claim. The detailed policy and permitted
# qualifiers live in docs/operations/kms-cryptographic-compliance.md; this
# check intentionally scans only the two public project-facing documents.
# Guard: cryptographic capability wording must not over-claim what RustFS
# actually does. Two independent blocks, both anchored to the policy in
# docs/operations/kms-cryptographic-compliance.md:
#
# 1. Outward README and CHANGELOG material must not make an unsupported
# FIPS validation or certification claim. This block intentionally scans
# only the two public project-facing documents; the permitted qualifiers
# live in the policy document.
#
# 2. Nothing in crates/kms may describe the Vault KV2 backend as wrapping
# key material through Vault's Transit engine. `KmsBackend::VaultKv2`
# stores RustFS-wrapped key material in Vault's KV v2 engine and never
# calls Transit (see crates/kms/src/config.rs and
# docs/operations/kms-backend-security.md), so such prose tells operators
# their key material is cryptographically isolated inside Vault when it is
# not.
#
# Block 2 replaces the unit test `test_vault_kv2_sources_do_not_claim_transit_wrapping`
# that used to live in crates/kms/src/config.rs (rustfs/backlog#1884). The
# invariant is a documentation-claim invariant, so it has no behavioral twin by
# construction and belongs in a wording guard rather than in a test. The test
# could only see four `include_str!`-pinned files and stopped compiling —
# rather than reporting a violation — the moment one of them was renamed; this
# block scans every file in the crate and reports a rename explicitly.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ROOT_DIR="${CHECK_FIPS_WORDING_ROOT:-$(cd "${SCRIPT_DIR}/.." && pwd)}"
@@ -21,12 +41,34 @@ FORBIDDEN_PATTERNS=(
'(meets|satisfies)[[:space:]]+FIPS'
)
status=0
KMS_CRATE_DIR="crates/kms"
# The four files the retired unit test pinned with include_str!. They stay
# listed so that moving one out of crates/kms is reported here instead of
# silently shrinking the scan; the scan itself is not limited to them.
KMS_PINNED_SOURCES=(
"crates/kms/src/config.rs"
"crates/kms/src/api_types.rs"
"crates/kms/src/backends/vault.rs"
"crates/kms/src/lib.rs"
)
# Literal, case-sensitive, and byte-for-byte the needles the retired test built
# at runtime via format!("wrapping via {}", "Transit") and friends.
KMS_VAULT_KV2_FORBIDDEN=(
'wrapping via Transit'
'KV v2 + Transit'
'KV2+Transit'
"you would use Vault's transit engine"
)
fips_status=0
kms_status=0
for target in "${TARGETS[@]}"; do
if [[ ! -f "$target" ]]; then
printf 'FIPS wording guard failed: %s is missing\n' "$target" >&2
status=1
fips_status=1
continue
fi
@@ -35,14 +77,41 @@ for target in "${TARGETS[@]}"; do
if [[ -n "$matches" ]]; then
printf 'FIPS wording guard failed: forbidden pattern /%s/ in %s:\n%s\n' \
"$pattern" "$target" "$matches" >&2
status=1
fips_status=1
fi
done
done
if [[ "$status" -ne 0 ]]; then
for source in "${KMS_PINNED_SOURCES[@]}"; do
if [[ ! -f "$source" ]]; then
printf 'KMS wording guard failed: %s is missing; update KMS_PINNED_SOURCES in scripts/check_fips_wording.sh after moving it\n' \
"$source" >&2
kms_status=1
fi
done
if [[ -d "$KMS_CRATE_DIR" ]]; then
for pattern in "${KMS_VAULT_KV2_FORBIDDEN[@]}"; do
matches="$(grep -r -F -n -- "$pattern" "$KMS_CRATE_DIR" || true)"
if [[ -n "$matches" ]]; then
printf 'KMS wording guard failed: forbidden Vault KV2 claim "%s" in %s:\n%s\n' \
"$pattern" "$KMS_CRATE_DIR" "$matches" >&2
kms_status=1
fi
done
fi
if [[ "$fips_status" -ne 0 ]]; then
printf 'Remove unsupported FIPS validation wording from README.md or CHANGELOG.md.\n' >&2
exit "$status"
fi
if [[ "$kms_status" -ne 0 ]]; then
printf 'The Vault KV2 backend does not wrap key material through Vault Transit; fix the wording in crates/kms.\n' >&2
fi
if [[ "$fips_status" -ne 0 || "$kms_status" -ne 0 ]]; then
exit 1
fi
printf 'FIPS wording guard passed (README.md and CHANGELOG.md contain no forbidden claims).\n'
printf 'KMS wording guard passed (crates/kms claims no Vault KV2 Transit wrapping).\n'