mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-05 19:55:37 +00:00
Compare commits
8 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 7faa9a0e69 | |||
| 7661fc1dd1 | |||
| 533cc487f4 | |||
| a0e746feb7 | |||
| 9877ce8231 | |||
| ed026a0f1f | |||
| f25df09f61 | |||
| 99081d0e78 |
@@ -20,10 +20,9 @@
|
||||
//! journal (`count_requests`) carries the assertion in every one of them.
|
||||
|
||||
use super::common::{BoxError, OdmTestEnv, RawResponse, SeedObject, start_configured_env};
|
||||
use crate::fake_s3_target::{FaultAction, Operation};
|
||||
use crate::fake_s3_target::Operation;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, VersioningConfiguration};
|
||||
use bytes::Bytes;
|
||||
use futures::{StreamExt, TryStreamExt};
|
||||
use std::time::Duration;
|
||||
|
||||
type TestResult = Result<(), BoxError>;
|
||||
@@ -146,38 +145,14 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
.await?;
|
||||
|
||||
let body = payload(128 * 1024);
|
||||
let blocker = "queue/blocker.bin";
|
||||
env.seed_source(SOURCE_BUCKET, &[SeedObject::new(blocker, body.clone())]);
|
||||
// The one-chunk range completes immediately; its full background pull
|
||||
// occupies the only slot while the remaining requests fill the queue.
|
||||
env.source.inject_for_key(
|
||||
Operation::GetObject,
|
||||
blocker,
|
||||
FaultAction::SlowSendBody {
|
||||
chunk_bytes: 1024,
|
||||
delay: Duration::from_millis(100),
|
||||
},
|
||||
2,
|
||||
);
|
||||
let response = env
|
||||
.raw_object_request(http::Method::GET, bucket, blocker, &[("range", "bytes=0-1023")])
|
||||
.await?;
|
||||
assert_eq!(response.status, 206);
|
||||
assert_eq!(response.body, body.slice(0..1024));
|
||||
env.wait_for_status_counter(bucket, "/inflight_pulls", 1, SETTLE).await?;
|
||||
|
||||
let keys: Vec<String> = (0..REQUESTS).map(|index| format!("queue/object-{index:03}.bin")).collect();
|
||||
let seeds: Vec<SeedObject> = keys.iter().map(|key| SeedObject::new(key.clone(), body.clone())).collect();
|
||||
env.seed_source(SOURCE_BUCKET, &seeds);
|
||||
|
||||
// Bound source connections below the fixture's limit while still
|
||||
// submitting all 100 requests to the eight-slot background queue.
|
||||
let responses: Vec<RawResponse> = futures::stream::iter(
|
||||
let responses: Vec<RawResponse> = futures::future::try_join_all(
|
||||
keys.iter()
|
||||
.map(|key| env.raw_object_request(http::Method::GET, bucket, key, &[("range", "bytes=0-1023")])),
|
||||
)
|
||||
.buffered(16)
|
||||
.try_collect()
|
||||
.await?;
|
||||
for (key, response) in keys.iter().zip(&responses) {
|
||||
assert_eq!(response.status, 206, "{key}: {}", String::from_utf8_lossy(&response.body));
|
||||
@@ -193,15 +168,6 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
.wait_for_status_counter(bucket, "/counters/pull_failures_total/queue_full", 1, SETTLE)
|
||||
.await?;
|
||||
assert!(queue_full > 0, "a 100-deep burst must overflow an 8-slot queue");
|
||||
let queue_full = usize::try_from(queue_full)?;
|
||||
assert!(queue_full <= REQUESTS);
|
||||
env.wait_for_status_counter(
|
||||
bucket,
|
||||
"/counters/pulled_objects_total/background",
|
||||
u64::try_from(REQUESTS + 1 - queue_full)?,
|
||||
SETTLE,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let ranged_reads: usize = keys.iter().map(|key| source_get_count(&env, key)).sum();
|
||||
assert!(
|
||||
@@ -209,6 +175,9 @@ async fn test_odm_range_burst_overflows_the_pull_queue_without_failing_clients()
|
||||
"every reader is served from the source: {ranged_reads} GETs for {REQUESTS} readers"
|
||||
);
|
||||
let dropped = keys.iter().filter(|key| source_get_count(&env, key) == 1).count();
|
||||
assert_eq!(dropped, queue_full, "only overflowed keys remain without a background GET");
|
||||
assert!(
|
||||
dropped > 0,
|
||||
"the overflowed keys are the ones with no backfill GET, but every key got one"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -265,13 +265,16 @@ async fn list_through_rejects_a_tampered_continuation_token() -> TestResult {
|
||||
let decoded = String::from_utf8(base64_simd::STANDARD.decode_to_vec(token.as_bytes())?)?;
|
||||
assert!(decoded.contains("\"t\":\"odm-list\""), "the merged token is an envelope: {decoded}");
|
||||
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":3").as_bytes());
|
||||
assert_ne!(tampered, token, "the test must change the token version");
|
||||
let query = serde_urlencoded::to_string([("continuation-token", tampered.as_str())])?;
|
||||
let rejected = env.raw_list_objects_v2(bucket, &query).await?;
|
||||
let error_body = String::from_utf8_lossy(&rejected.body);
|
||||
assert_eq!(rejected.status, 400, "a bumped token version is a client error: {}", error_body);
|
||||
assert!(error_body.contains("<Code>InvalidArgument</Code>"), "{error_body}");
|
||||
let tampered = base64_simd::STANDARD.encode_to_string(decoded.replace("\"v\":1", "\"v\":2").as_bytes());
|
||||
let rejected = env
|
||||
.raw_list_objects_v2(bucket, &format!("continuation-token={tampered}"))
|
||||
.await?;
|
||||
assert_eq!(
|
||||
rejected.status,
|
||||
400,
|
||||
"a bumped token version is a client error: {}",
|
||||
String::from_utf8_lossy(&rejected.body)
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -1281,6 +1281,8 @@ impl FolderScanner {
|
||||
}
|
||||
Err(e) => return Err(ScannerError::Io(e)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|
||||
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
|
||||
|
||||
@@ -25,6 +25,7 @@ use std::os::unix::fs::{PermissionsExt, symlink};
|
||||
use std::sync::Mutex;
|
||||
|
||||
mod checkpoint_fixture;
|
||||
pub(super) mod enumeration_restart;
|
||||
|
||||
/// Reset the process-global alert cooldown map; test-only.
|
||||
fn reset_alert_cooldowns() {
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const MAX_CACHE_BYTES: u64 = 1024 * 1024;
|
||||
const REQUEST_ENV: &str = "RUSTFS_ENUMERATION_REQUEST";
|
||||
|
||||
struct Observation {
|
||||
root: PathBuf,
|
||||
limit: u64,
|
||||
entries: u64,
|
||||
name_bytes: u64,
|
||||
}
|
||||
|
||||
static OBSERVATION: Mutex<Option<Observation>> = Mutex::new(None);
|
||||
|
||||
// Only the selected synthetic disk is observed; concurrent unrelated scanners
|
||||
// do not consume its budget. This hook is absent from non-test builds.
|
||||
pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::OsStr, budget: &ScannerCycleBudget) {
|
||||
let mut guard = OBSERVATION.lock().expect("enumeration observation lock");
|
||||
if let Some(observation) = guard.as_mut()
|
||||
&& Path::new(dir).starts_with(&observation.root)
|
||||
{
|
||||
observation.entries += 1;
|
||||
observation.name_bytes += u64::try_from(name.as_encoded_bytes().len()).expect("bounded entry name");
|
||||
if observation.entries >= observation.limit {
|
||||
budget.cancel_for_runtime();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct ObservationGuard;
|
||||
|
||||
impl Drop for ObservationGuard {
|
||||
fn drop(&mut self) {
|
||||
*OBSERVATION.lock().expect("enumeration observation cleanup") = None;
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Request {
|
||||
workspace: PathBuf,
|
||||
objects: usize,
|
||||
raw_entry_budget: u64,
|
||||
round: u32,
|
||||
}
|
||||
|
||||
async fn read_bounded(path: &Path) -> Vec<u8> {
|
||||
let file = tokio::fs::File::open(path).await.expect("open fixture artifact");
|
||||
let mut bytes = Vec::new();
|
||||
file.take(MAX_CACHE_BYTES + 1)
|
||||
.read_to_end(&mut bytes)
|
||||
.await
|
||||
.expect("read fixture artifact");
|
||||
assert!(u64::try_from(bytes.len()).expect("artifact size") <= MAX_CACHE_BYTES);
|
||||
bytes
|
||||
}
|
||||
|
||||
async fn round(request: &Request) -> serde_json::Value {
|
||||
assert!((1..=1024).contains(&request.objects));
|
||||
assert!((1..=4096).contains(&request.raw_entry_budget));
|
||||
assert!(request.round < 64);
|
||||
let disk_root = request.workspace.join("disk");
|
||||
let cache_path = request.workspace.join("cache.bin");
|
||||
if request.round == 0 {
|
||||
tokio::fs::create_dir(&disk_root).await.expect("create fresh synthetic disk");
|
||||
for index in 0..request.objects {
|
||||
let object = format!("object-{index:04}");
|
||||
let version = Uuid::from_u128(u128::try_from(index).expect("fixture index") + 1);
|
||||
let bytes = metadata_for_object_version("bucket", &object, Some(version));
|
||||
write_test_object_metadata_bytes(&disk_root, "bucket", &object, &bytes).await;
|
||||
}
|
||||
let mut initial = DataUsageCache::default();
|
||||
initial.info.name = "bucket".to_string();
|
||||
initial.info.skip_healing = true;
|
||||
initial.info.snapshot_complete = false;
|
||||
initial.replace("bucket", "", DataUsageEntry::default());
|
||||
tokio::fs::write(&cache_path, initial.marshal_msg().expect("initial cache codec"))
|
||||
.await
|
||||
.expect("persist initial cache");
|
||||
}
|
||||
let cache = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload cache codec before scan");
|
||||
assert_eq!(cache.info.name, "bucket");
|
||||
let before = cache.checked_flatten("bucket").expect("persisted bucket root").objects;
|
||||
let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("open synthetic disk in this process");
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
|
||||
*OBSERVATION.lock().expect("install observation") = Some(Observation {
|
||||
root: disk.path(),
|
||||
limit: request.raw_entry_budget,
|
||||
entries: 0,
|
||||
name_bytes: 0,
|
||||
});
|
||||
let _observation_guard = ObservationGuard;
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
budget.clone(),
|
||||
vec![disk.clone()],
|
||||
disk,
|
||||
cache.clone(),
|
||||
None,
|
||||
HealScanMode::Normal,
|
||||
SCANNER_SLEEPER.clone(),
|
||||
)
|
||||
.await;
|
||||
let (returned, outcome) = match result {
|
||||
Ok(cache) => (cache, "complete"),
|
||||
Err(ScannerError::PartialCache(cache)) => (*cache, "partial"),
|
||||
Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => {
|
||||
(cache, "cancelled_without_cache")
|
||||
}
|
||||
Err(error) => panic!("unexpected real scanner failure: {error}"),
|
||||
};
|
||||
let encoded = returned.marshal_msg().expect("returned cache codec");
|
||||
assert!(u64::try_from(encoded.len()).expect("encoded length") <= MAX_CACHE_BYTES);
|
||||
tokio::fs::write(&cache_path, encoded).await.expect("persist returned cache");
|
||||
let reloaded = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload returned cache codec");
|
||||
let retained = reloaded.checked_flatten("bucket").expect("reloaded bucket root");
|
||||
let scanned = returned.checked_flatten("bucket").expect("returned bucket root");
|
||||
assert_eq!(
|
||||
(retained.objects, retained.versions, retained.size),
|
||||
(scanned.objects, scanned.versions, scanned.size)
|
||||
);
|
||||
assert_eq!(reloaded.info.snapshot_complete, returned.info.snapshot_complete);
|
||||
let guard = OBSERVATION.lock().expect("read observation");
|
||||
let observation = guard.as_ref().expect("installed observation");
|
||||
serde_json::json!({
|
||||
"schema": 1, "pid": std::process::id(), "round": request.round,
|
||||
"objects_expected": request.objects, "raw_entry_budget": request.raw_entry_budget,
|
||||
"raw_entries": observation.entries, "raw_name_bytes": observation.name_bytes,
|
||||
"objects_processed": budget.progress().0,
|
||||
"objects_before": before, "objects_retained": retained.objects,
|
||||
"versions_retained": retained.versions, "bytes_retained": retained.size,
|
||||
"snapshot_complete": reloaded.info.snapshot_complete, "outcome": outcome,
|
||||
})
|
||||
}
|
||||
|
||||
/// Default CI is a positive healthy control. The external driver selects the
|
||||
/// same worker in a fresh OS process per round and applies its strict oracle.
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn enumeration_restart_worker() {
|
||||
if let Some(path) = std::env::var_os(REQUEST_ENV) {
|
||||
let request: Request = serde_json::from_slice(&read_bounded(Path::new(&path)).await).expect("bounded worker request");
|
||||
let report = round(&request).await;
|
||||
tokio::fs::write(
|
||||
request.workspace.join(format!("round-{}.json", request.round)),
|
||||
serde_json::to_vec(&report).expect("report JSON"),
|
||||
)
|
||||
.await
|
||||
.expect("write worker report");
|
||||
} else {
|
||||
let temp = tempfile::tempdir().expect("healthy fixture directory");
|
||||
let report = round(&Request {
|
||||
workspace: temp.path().to_path_buf(),
|
||||
objects: 4,
|
||||
raw_entry_budget: 16,
|
||||
round: 0,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(report["outcome"], "complete");
|
||||
assert_eq!(report["snapshot_complete"], true);
|
||||
assert_eq!(report["objects_retained"], 4);
|
||||
assert_eq!(report["versions_retained"], 4);
|
||||
assert_eq!(report["bytes_retained"], 4);
|
||||
assert!(report["raw_entries"].as_u64().expect("observed entries") >= 8, "{report}");
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,38 @@
|
||||
# Scanner Checkpoint Fixture
|
||||
|
||||
## Raw Enumeration Restart Diagnostic
|
||||
|
||||
`enumeration_restart_worker` exercises the real `scan_data_folder` with a local disk and valid `xl.meta` objects. Without configuration it is a positive CI control: four one-byte objects must complete and survive a cache codec round trip. It is not an ignored test or an assertion that a known defect must persist.
|
||||
|
||||
```sh
|
||||
RUST_MIN_STACK=4194304 cargo test -p rustfs-scanner --lib enumeration_restart_worker -- --nocapture
|
||||
cargo test -p rustfs-scanner --lib --no-run --message-format=json
|
||||
python3 -m unittest discover -s scripts -p 'test_diagnose_scanner_enumeration_restart.py'
|
||||
```
|
||||
|
||||
Use the `executable` from the scanner library test `compiler-artifact` JSON record as `--test-binary` below. The driver verifies that it contains the exact worker test before doing any work; a zero-test filter cannot pass.
|
||||
|
||||
```sh
|
||||
python3 scripts/diagnose_scanner_enumeration_restart.py \
|
||||
--test-binary /path/to/compiled/scanner-libtest \
|
||||
--output /tmp/scanner-enumeration-new-run \
|
||||
--objects 128 --raw-entry-budget 8 --rounds 8
|
||||
```
|
||||
|
||||
The output directory must not exist. Each round starts a new OS test-worker process, opens the same synthetic disk, decodes the preceding cache, invokes the real scanner, encodes the returned cache, and decodes it again. When cancellation returns no useful partial cache, it preserves the previous cache. Reports identify the actual child PID, round, raw entries and name bytes observed, processed objects, retained object/version/byte counts, and completeness. No observed-name set, `readdir` offset, or assumed stable ordering is used as durable progress. Namespace creation happens only during fixture setup, before scan accounting.
|
||||
|
||||
The `cfg(test)` hook observes actual entries delivered by `read_dir` and cancels the existing cycle token at the fixed entry limit. This is a deterministic injected **raw-entry work budget**, not a wall-clock performance measurement or a claim that kernel prefetch, probes, allocations, name bytes, or cache I/O are independently budgeted. The watchdog timeout only bounds worker lifetime. The hook does not replace enumeration, classification, or recursion, and does not exist in production builds. In particular, `xl.meta` object-boundary classification is unchanged.
|
||||
|
||||
Exit 0 requires exact complete object/version/byte coverage within the same fixed budget on every executed round. Exit 1 means the strict convergence oracle remains unmet, including the current flat-directory enumeration starvation case. Exit 2 means invalid input, worker failure, or invalid evidence; it is not a successful reproduction. There is no final unbudgeted sweep. Small fixtures can pass; that does not establish the general R-E gate from [the scanner review comment](https://github.com/rustfs/backlog/issues/2240#issuecomment-5549222480). Raw entries observed are not a retained enumeration watermark. This is scanner-worker process restart plus codec evidence, **not** whole-daemon restart, EC quorum persistence, crash/fsync durability, remote RPC, or a throughput benchmark. The caller owns the bounded evidence directory and may remove it after inspection.
|
||||
|
||||
### Missing Storage Capability
|
||||
|
||||
The current `scanner_folder::FolderScanner::scan_folder` collects child folders before recursing. `LocalDisk::scan_dir` also reads the whole parent before sorting and applying `forward_to`. The persistent key-only listing index's `collect_persistent_key_only_index_objects` / `rebuild_persistent_key_only_index` collects all objects in memory before publication and excludes deleted entries. It cannot supply a restartable first-build cursor over per-disk raw entries, orphan directories, and metadata boundaries. Repeated listing from the beginning is real work, not free pagination.
|
||||
|
||||
A future storage-owner capability must expose an explicit unsupported/building/ready state and a durable snapshot/index identity bound to disk mount, bucket incarnation, and directory identity. It must budget the first build and every page, including entry count, name bytes, metadata probes, I/O and time; survive a process restart during first build; seal page data before advancing the manifest; and distinguish enumerated, classified, and fully processed frontiers. An uncommitted page may be replayed only within a bounded cost. `xl.meta` classification must finish before descendants become traversable namespace. Missing capability or invalid identities must not become fabricated progress or completeness. No such capability is implemented by this diagnostic, and ordinary local storage remains without this R-E guarantee.
|
||||
|
||||
## Completed Subtree Checkpoint Fixture
|
||||
|
||||
The `checkpoint_fixture` tests exercise a bounded namespace of 24 static objects and one repeatedly updated hot object. Each of three rounds runs the production local disk scanner with an object budget, saves the returned partial cache through the production persistence codec and revision checks to a two-file test backend, and reloads it before preparing the next round. The fixture prints static-subtree coverage at each boundary and cumulative visited entries. This is a diagnostic of retained coverage, not a throughput benchmark.
|
||||
|
||||
Run the fixture and confirm the test filter selects a nonzero number of tests:
|
||||
|
||||
@@ -133,7 +133,6 @@ impl NativeHttp {
|
||||
self.send_classified(request, error_code_header, false).await
|
||||
}
|
||||
|
||||
#[cfg(feature = "gcs")]
|
||||
pub(super) async fn send_object(
|
||||
&self,
|
||||
request: reqwest::Request,
|
||||
@@ -211,7 +210,6 @@ pub(super) async fn read_text(response: reqwest::Response, max_bytes: usize) ->
|
||||
/// Base64 digest (`Content-MD5`, `md5Hash`, `x-goog-hash`) as lowercase hex.
|
||||
/// `None` when the value is not a 16-byte digest, so a CRC32C never passes as
|
||||
/// an MD5.
|
||||
#[cfg(any(test, feature = "gcs"))]
|
||||
pub(super) fn base64_md5_to_hex(value: &str) -> Option<String> {
|
||||
let raw = base64_simd::STANDARD.decode_to_vec(value.trim().as_bytes()).ok()?;
|
||||
(raw.len() == 16).then(|| faster_hex::hex_string(&raw))
|
||||
|
||||
@@ -4034,7 +4034,7 @@ mod tests {
|
||||
let enabled_before = sys.is_module_enabled();
|
||||
sys.set_module_enabled(true);
|
||||
let bucket = format!("odm-capture-failure-{}", uuid::Uuid::new_v4());
|
||||
let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","region":"us-east-1","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
|
||||
let mut config: crate::on_demand_migration::OnDemandMigrationConfig = serde_json::from_str(r#"{"source":{"provider":"minio","endpoint":"https://source.example.com","bucket":"source","credentials":{"access_key":"test","secret_key":"test"}}}"#).expect("source config");
|
||||
config.policy.list_through = true;
|
||||
sys.apply_for_incarnation(&bucket, uuid::Uuid::new_v4(), Some(&config)).await;
|
||||
let mut get = build_request(
|
||||
|
||||
@@ -46,6 +46,8 @@ their issue closes.
|
||||
|
||||
| Entry | Status | Purpose | Wiring / docs |
|
||||
|---|---|---|---|
|
||||
| `diagnose_scanner_enumeration_restart.py` | dev-tool | Strict fixed raw-entry-budget scanner-worker restart diagnostic | [Checkpoint fixture](../docs/testing/scanner-checkpoint-fixture.md) |
|
||||
| `test_diagnose_scanner_enumeration_restart.py` | dev-tool | Driver report validation and positive convergence oracle tests | Python unittest; same guide |
|
||||
| `e2e-run.sh` | ci-gate | Boots a rustfs server and runs the `s3s-e2e` black-box conformance tool against it | ci.yml `e2e-tests` jobs; `docs/testing/README.md` |
|
||||
| `run_ecstore_validation_suite.sh` | dev-tool | ecstore black-box validation suite (`quick`/`full`/`destructive`/`fuzz` profiles) | `docs/testing/README.md`, `docs/testing/ecstore-validation-suite-design.md` |
|
||||
| `run_e2e_tests.sh` | dev-tool | Local `e2e_test` crate runner (starts a server, applies filters, cleans up) | `crates/e2e_test/README.md` |
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Strict restart diagnostic using the real scanner libtest worker, not a walker model."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
WORKER = "scanner_folder::tests::enumeration_restart::enumeration_restart_worker"
|
||||
MAX_REPORT_BYTES = 16384
|
||||
|
||||
|
||||
def bounded_int(low, high):
|
||||
def parse(value):
|
||||
number = int(value)
|
||||
if not low <= number <= high:
|
||||
raise argparse.ArgumentTypeError(f"must be between {low} and {high}")
|
||||
return number
|
||||
return parse
|
||||
|
||||
|
||||
def validate_report(report, *, round_number, pid, objects, budget):
|
||||
if not isinstance(report, dict):
|
||||
raise ValueError("worker report must be an object")
|
||||
expected = {"schema": 1, "round": round_number, "pid": pid,
|
||||
"objects_expected": objects, "raw_entry_budget": budget}
|
||||
for key, value in expected.items():
|
||||
if type(report.get(key)) is not int or report[key] != value:
|
||||
raise ValueError(f"worker report mismatch: {key}")
|
||||
for key in ("raw_entries", "raw_name_bytes", "objects_before", "objects_retained",
|
||||
"versions_retained", "bytes_retained", "objects_processed"):
|
||||
if type(report.get(key)) is not int or not 0 <= report[key] <= 1048576:
|
||||
raise ValueError(f"invalid bounded counter: {key}")
|
||||
if report["raw_entries"] == 0:
|
||||
raise ValueError("nonempty fixture must observe raw entries; budget hook may not have run")
|
||||
if report["raw_entries"] > budget:
|
||||
raise ValueError("raw-entry budget exceeded; no unbudgeted tail is permitted")
|
||||
if type(report.get("snapshot_complete")) is not bool:
|
||||
raise ValueError("missing explicit completeness")
|
||||
if report.get("outcome") not in ("complete", "partial", "cancelled_without_cache"):
|
||||
raise ValueError("unexpected scanner outcome")
|
||||
|
||||
|
||||
def converged(report, objects):
|
||||
return (report["snapshot_complete"] and report["outcome"] == "complete"
|
||||
and all(report[key] == objects for key in
|
||||
("objects_retained", "versions_retained", "bytes_retained")))
|
||||
|
||||
|
||||
def run(args):
|
||||
binary = args.test_binary.resolve(strict=True)
|
||||
listed = subprocess.run([str(binary), WORKER, "--exact", "--list"],
|
||||
check=True, capture_output=True, text=True, timeout=30)
|
||||
if f"{WORKER}: test" not in listed.stdout.splitlines():
|
||||
raise ValueError("binary does not contain the exact scanner worker test")
|
||||
workspace = args.output.resolve()
|
||||
workspace.mkdir() # Refuse reuse/overwrite of previous evidence or customer data.
|
||||
reports = []
|
||||
for round_number in range(args.rounds):
|
||||
request = {"workspace": str(workspace), "objects": args.objects,
|
||||
"raw_entry_budget": args.raw_entry_budget, "round": round_number}
|
||||
request_path = workspace / "request.json"
|
||||
request_path.write_text(json.dumps(request), encoding="utf-8")
|
||||
env = dict(os.environ, RUSTFS_ENUMERATION_REQUEST=str(request_path),
|
||||
RUST_MIN_STACK="4194304", NO_PROXY="localhost,127.0.0.1,::1",
|
||||
no_proxy="localhost,127.0.0.1,::1")
|
||||
with subprocess.Popen([str(binary), WORKER, "--exact", "--test-threads=1"],
|
||||
env=env, stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL) as worker:
|
||||
try:
|
||||
status = worker.wait(timeout=args.timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
worker.kill()
|
||||
worker.wait()
|
||||
raise ValueError(f"worker round {round_number} timed out") from None
|
||||
if status:
|
||||
raise ValueError(f"real scanner worker round {round_number} exited {status}")
|
||||
report_path = workspace / f"round-{round_number}.json"
|
||||
with report_path.open("rb") as handle:
|
||||
raw = handle.read(MAX_REPORT_BYTES + 1)
|
||||
if len(raw) > MAX_REPORT_BYTES:
|
||||
raise ValueError("oversized worker report")
|
||||
report = json.loads(raw)
|
||||
validate_report(report, round_number=round_number, pid=worker.pid,
|
||||
objects=args.objects, budget=args.raw_entry_budget)
|
||||
if reports and report["objects_before"] != reports[-1]["objects_retained"]:
|
||||
raise ValueError("cache coverage did not survive the process boundary")
|
||||
reports.append(report)
|
||||
print(json.dumps(report, sort_keys=True), flush=True)
|
||||
if converged(report, args.objects):
|
||||
print("PASS: bounded scanner-worker restart convergence for this fixture only")
|
||||
return 0
|
||||
print("FAIL: fixed-budget restart convergence not established; R-E gate remains unmet",
|
||||
file=sys.stderr)
|
||||
return 1
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--test-binary", type=Path, required=True,
|
||||
help="compiled rustfs-scanner libtest executable")
|
||||
parser.add_argument("--output", type=Path, required=True, help="new evidence directory (must not exist)")
|
||||
parser.add_argument("--objects", type=bounded_int(1, 1024), default=128)
|
||||
parser.add_argument("--raw-entry-budget", type=bounded_int(1, 4096), default=8)
|
||||
parser.add_argument("--rounds", type=bounded_int(1, 64), default=8)
|
||||
parser.add_argument("--timeout", type=bounded_int(1, 120), default=60,
|
||||
help="per-worker watchdog seconds, not the scan work budget")
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
return run(args)
|
||||
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
||||
print(f"ERROR: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Driver contract tests; these do not replace the real scanner diagnostic."""
|
||||
|
||||
import unittest
|
||||
|
||||
from diagnose_scanner_enumeration_restart import converged, validate_report
|
||||
|
||||
|
||||
class ReportTests(unittest.TestCase):
|
||||
def report(self):
|
||||
return dict(schema=1, round=0, pid=123, objects_expected=4, raw_entry_budget=16,
|
||||
raw_entries=8, raw_name_bytes=64, objects_before=0, objects_retained=4,
|
||||
versions_retained=4, bytes_retained=4, objects_processed=4,
|
||||
snapshot_complete=True, outcome="complete")
|
||||
|
||||
def validate(self, report):
|
||||
validate_report(report, round_number=0, pid=123, objects=4, budget=16)
|
||||
|
||||
def test_complete_exact_coverage_satisfies_oracle(self):
|
||||
report = self.report()
|
||||
self.validate(report)
|
||||
self.assertTrue(converged(report, 4))
|
||||
|
||||
def test_incomplete_or_inexact_coverage_cannot_pass(self):
|
||||
for key, value in (("snapshot_complete", False), ("objects_retained", 3),
|
||||
("versions_retained", 3), ("bytes_retained", 3), ("outcome", "partial")):
|
||||
with self.subTest(key=key):
|
||||
report = self.report()
|
||||
report[key] = value
|
||||
self.assertFalse(converged(report, 4))
|
||||
|
||||
def test_wrong_process_or_round_rejected(self):
|
||||
for key in ("pid", "round", "schema", "raw_entry_budget", "objects_expected"):
|
||||
with self.subTest(key=key):
|
||||
report = self.report()
|
||||
report[key] += 1
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
def test_unbudgeted_tail_rejected(self):
|
||||
report = self.report()
|
||||
report["raw_entries"] = 17
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
def test_complete_coverage_without_entry_observation_rejected(self):
|
||||
report = self.report()
|
||||
report["raw_entries"] = 0
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
def test_missing_wrong_type_and_negative_counter_rejected(self):
|
||||
for value in (None, True, -1, "8", 1048577):
|
||||
with self.subTest(value=value):
|
||||
report = self.report()
|
||||
report["raw_entries"] = value
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
def test_missing_completeness_or_unknown_outcome_rejected(self):
|
||||
for key in ("snapshot_complete", "outcome"):
|
||||
report = self.report()
|
||||
del report[key]
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
def test_non_object_report_rejected(self):
|
||||
for report in (None, [], "report"):
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user