mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 12:35:54 +00:00
Compare commits
18 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| cc91483fac | |||
| c8fc50ada2 | |||
| 9badf3939c | |||
| c71c686e2a | |||
| 45f57ca57a | |||
| 265f429358 | |||
| 47f640f23e | |||
| 237c96dd3b | |||
| 647fe1a292 | |||
| 0c6314babc | |||
| fc76099280 | |||
| 899f81f3ad | |||
| 37bda24e1c | |||
| 4c2a0cdf9a | |||
| b370e746a0 | |||
| 95e6c89c1c | |||
| aafa7e2b7f | |||
| 5211b56277 |
@@ -47,6 +47,7 @@ script-tests: ## Run shell script tests
|
||||
bash -n ./scripts/validate_object_data_cache_cold_stampede.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/check_object_data_cache_follower_samples.py --self-test
|
||||
./scripts/validate_object_data_cache_cold_stampede.sh --self-test
|
||||
./scripts/run_scanner_heal_evidence_case.sh --self-test
|
||||
|
||||
.PHONY: test
|
||||
test: core-deps script-tests ## Run all tests (needs cargo-nextest; RUSTFS_ALLOW_CARGO_TEST_FALLBACK=1 to override)
|
||||
|
||||
@@ -8,10 +8,26 @@
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_restart",
|
||||
"oracle": "background-target-restart.json",
|
||||
"evidence": "process-restart",
|
||||
"unclean_shutdown_marker": false,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process restart, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
},
|
||||
"background-target-crash": {
|
||||
"gate": "G14",
|
||||
"task": "W21",
|
||||
"lane": "e2e-nightly",
|
||||
"suite": "e2e_test",
|
||||
"name": "heal_erasure_disk_rebuild_test::tests::test_cluster_root_heal_recovers_remote_shards_after_background_target_crash",
|
||||
"oracle": "background-target-crash.json",
|
||||
"evidence": "process-crash-restart",
|
||||
"unclean_shutdown_marker": true,
|
||||
"min_objects": 9,
|
||||
"max_objects": 65,
|
||||
"topology": {"nodes": 4, "drives_per_node": 1},
|
||||
"scope": "Target process killed during partial background rebuild, real unclean-shutdown marker, exact unversioned S3 bodies and replacement-disk shards; not power loss or EC8+4."
|
||||
}
|
||||
},
|
||||
"release_pending": {
|
||||
|
||||
@@ -109,6 +109,15 @@ Star RustFS on GitHub and be instantly notified of new releases.
|
||||
|
||||
## Quickstart
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Pool expansion notice:**
|
||||
>
|
||||
> - A single-node single-drive (SNSD) deployment is supported only as a standalone local path. It cannot expand in place or be added as a Pool. To move to a multi-drive topology, create a new deployment and migrate data through S3.
|
||||
> - Keep an existing multi-drive Pool's endpoints and Erasure Set width unchanged; expand by appending a new Pool. With ellipsis-based expansion, every Pool argument must contain an ellipsis expression and expand to at least two drive endpoints.
|
||||
> - Single-node multi-drive Pools and multi-node Pools with one drive per node are allowed, subject to valid Erasure Set geometry and EC settings; acceptance does not guarantee host-failure tolerance.
|
||||
>
|
||||
> These topology rules follow MinIO, but automatic parity selection differs between the projects. See the [Pool layout compatibility and regression tests](docs/testing/pool-layout-compatibility.md) before expanding a deployment.
|
||||
|
||||
To get started with RustFS, follow these steps:
|
||||
|
||||
### 1. One-click Installation (Option 1)
|
||||
|
||||
@@ -89,6 +89,15 @@ RustFS 是一个基于 Rust 构建的高性能分布式对象存储系统。Rust
|
||||
|
||||
## 快速开始
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **Pool 扩容 Notice:**
|
||||
>
|
||||
> - 单节点单盘(SNSD)部署仅支持使用本地路径独立运行,不支持原地扩容,也不能作为 Pool 加入集群。如需改为多盘拓扑,请创建新部署并通过 S3 迁移数据。
|
||||
> - 已有多盘 Pool 的端点和 Erasure Set 宽度应保持不变,扩容应追加新的 Pool。使用省略号表达式扩容时,每个 Pool 参数都必须包含省略号表达式,并展开为至少两个磁盘端点。
|
||||
> - 允许单节点多盘 Pool,也允许多节点、每节点一盘的 Pool,但必须满足 Erasure Set 布局和 EC 配置要求;配置合法不代表能够容忍整台主机故障。
|
||||
>
|
||||
> 这些拓扑规则与 MinIO 一致,但两者的默认 parity 选择方式存在差异。扩容前请阅读 [Pool 布局兼容性与回归测试说明](docs/testing/pool-layout-compatibility.md)。
|
||||
|
||||
请按照以下步骤快速上手 RustFS:
|
||||
|
||||
### 1. 一键安装脚本 (选项 1)
|
||||
|
||||
@@ -57,6 +57,8 @@ const RUSTFS_FULL_FEATURE: &str = "full";
|
||||
const TEST_PORT_MIN: u16 = 20_000;
|
||||
// Keep allocator ports below the ephemeral range used by bind(..., 0) test helpers.
|
||||
const TEST_PORT_RANGE: u16 = 10_000;
|
||||
const TEST_PORT_MIN_ENV: &str = "RUSTFS_E2E_TEST_PORT_MIN";
|
||||
const TEST_PORT_RANGE_ENV: &str = "RUSTFS_E2E_TEST_PORT_RANGE";
|
||||
const TEST_PORT_COUNTER_PATH: &str = "/tmp/rustfs_e2e_next_port";
|
||||
const TEST_PORT_LOCK_DIR: &str = "/tmp/rustfs_e2e_port_allocator.lock";
|
||||
const TEST_PORT_LOCK_STALE_AFTER: Duration = Duration::from_secs(30);
|
||||
@@ -99,22 +101,74 @@ impl Drop for PortAllocatorGuard {
|
||||
}
|
||||
}
|
||||
|
||||
fn advance_test_port(port: u16) -> u16 {
|
||||
let offset = (port - TEST_PORT_MIN + 1) % TEST_PORT_RANGE;
|
||||
TEST_PORT_MIN + offset
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
struct TestPortAllocatorConfig {
|
||||
min: u16,
|
||||
range: u16,
|
||||
}
|
||||
|
||||
fn seeded_test_port() -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(TEST_PORT_RANGE)) as u16;
|
||||
TEST_PORT_MIN + offset
|
||||
impl TestPortAllocatorConfig {
|
||||
fn max_exclusive(self) -> u32 {
|
||||
u32::from(self.min) + u32::from(self.range)
|
||||
}
|
||||
|
||||
fn contains(self, port: &u16) -> bool {
|
||||
(u32::from(self.min)..self.max_exclusive()).contains(&u32::from(*port))
|
||||
}
|
||||
}
|
||||
|
||||
fn read_next_test_port() -> u16 {
|
||||
fn parse_test_port_allocator_config(
|
||||
min_override: Option<&str>,
|
||||
range_override: Option<&str>,
|
||||
) -> Result<TestPortAllocatorConfig, Box<dyn std::error::Error + Send + Sync>> {
|
||||
let min = match min_override {
|
||||
Some(value) => value
|
||||
.parse::<u16>()
|
||||
.map_err(|err| format!("{TEST_PORT_MIN_ENV} must be a valid u16: {err}"))?,
|
||||
None => TEST_PORT_MIN,
|
||||
};
|
||||
let range = match range_override {
|
||||
Some(value) => value
|
||||
.parse::<u16>()
|
||||
.map_err(|err| format!("{TEST_PORT_RANGE_ENV} must be a valid u16: {err}"))?,
|
||||
None => TEST_PORT_RANGE,
|
||||
};
|
||||
if range == 0 {
|
||||
return Err(format!("{TEST_PORT_RANGE_ENV} must be greater than zero").into());
|
||||
}
|
||||
if min < 1024 {
|
||||
return Err(format!("{TEST_PORT_MIN_ENV} must be at least 1024").into());
|
||||
}
|
||||
let max_exclusive = u32::from(min) + u32::from(range);
|
||||
if max_exclusive > u32::from(u16::MAX) + 1 {
|
||||
return Err(format!("{TEST_PORT_MIN_ENV} + {TEST_PORT_RANGE_ENV} exceeds u16 port space").into());
|
||||
}
|
||||
Ok(TestPortAllocatorConfig { min, range })
|
||||
}
|
||||
|
||||
fn test_port_allocator_config() -> Result<TestPortAllocatorConfig, Box<dyn std::error::Error + Send + Sync>> {
|
||||
parse_test_port_allocator_config(
|
||||
std::env::var(TEST_PORT_MIN_ENV).ok().as_deref(),
|
||||
std::env::var(TEST_PORT_RANGE_ENV).ok().as_deref(),
|
||||
)
|
||||
}
|
||||
|
||||
fn advance_test_port(port: u16, config: TestPortAllocatorConfig) -> u16 {
|
||||
let offset = (port - config.min + 1) % config.range;
|
||||
config.min + offset
|
||||
}
|
||||
|
||||
fn seeded_test_port(config: TestPortAllocatorConfig) -> u16 {
|
||||
let offset = (Uuid::new_v4().as_u128() % u128::from(config.range)) as u16;
|
||||
config.min + offset
|
||||
}
|
||||
|
||||
fn read_next_test_port(config: TestPortAllocatorConfig) -> u16 {
|
||||
stdfs::read_to_string(TEST_PORT_COUNTER_PATH)
|
||||
.ok()
|
||||
.and_then(|value| value.trim().parse::<u16>().ok())
|
||||
.filter(|port| (TEST_PORT_MIN..TEST_PORT_MIN + TEST_PORT_RANGE).contains(port))
|
||||
.unwrap_or_else(seeded_test_port)
|
||||
.filter(|port| config.contains(port))
|
||||
.unwrap_or_else(|| seeded_test_port(config))
|
||||
}
|
||||
|
||||
fn remove_stale_port_allocator_lock() {
|
||||
@@ -629,11 +683,12 @@ impl RustFSTestEnvironment {
|
||||
pub async fn find_available_port() -> Result<u16, Box<dyn std::error::Error + Send + Sync>> {
|
||||
use std::net::TcpListener;
|
||||
let _guard = PortAllocatorGuard::acquire().await?;
|
||||
let mut next_port = read_next_test_port();
|
||||
let config = test_port_allocator_config()?;
|
||||
let mut next_port = read_next_test_port(config);
|
||||
|
||||
for _ in 0..TEST_PORT_RANGE {
|
||||
for _ in 0..config.range {
|
||||
let port = next_port;
|
||||
next_port = advance_test_port(next_port);
|
||||
next_port = advance_test_port(next_port, config);
|
||||
write_next_test_port(next_port)?;
|
||||
|
||||
if let Ok(listener) = TcpListener::bind(("127.0.0.1", port)) {
|
||||
@@ -2108,6 +2163,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn e2e_port_allocator_uses_default_range() {
|
||||
assert_eq!(
|
||||
parse_test_port_allocator_config(None, None).expect("default port allocator config"),
|
||||
TestPortAllocatorConfig {
|
||||
min: TEST_PORT_MIN,
|
||||
range: TEST_PORT_RANGE
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn e2e_port_allocator_accepts_explicit_test_range() {
|
||||
let config = parse_test_port_allocator_config(Some("31000"), Some("128")).expect("explicit port range");
|
||||
|
||||
assert_eq!(advance_test_port(31127, config), 31000);
|
||||
assert!(config.contains(&31000));
|
||||
assert!(config.contains(&31127));
|
||||
assert!(!config.contains(&31128));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn e2e_port_allocator_rejects_invalid_override() {
|
||||
assert!(parse_test_port_allocator_config(Some("1023"), Some("1")).is_err());
|
||||
assert!(parse_test_port_allocator_config(Some("65000"), Some("1000")).is_err());
|
||||
assert!(parse_test_port_allocator_config(Some("31000"), Some("0")).is_err());
|
||||
assert!(parse_test_port_allocator_config(Some("not-a-port"), Some("128")).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolves_rustfs_binary_in_configured_cargo_target_directory() {
|
||||
let workspace = Path::new("workspace");
|
||||
|
||||
@@ -630,6 +630,10 @@ struct MultipartPart {
|
||||
body: Bytes,
|
||||
e_tag: String,
|
||||
digest: [u8; 16],
|
||||
/// Plaintext length declared by an SSE-C passthrough sender
|
||||
/// (`x-rustfs-replication-part-actual-size`); RustFS validates the 5 MiB
|
||||
/// minimum against it rather than against the stored bytes.
|
||||
actual_size: Option<usize>,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -2823,6 +2827,11 @@ impl S3 for FakeBackend {
|
||||
|
||||
async fn upload_part(&self, req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
let fault = request_fault(&req);
|
||||
let declared_actual_size = req
|
||||
.headers
|
||||
.get("x-rustfs-replication-part-actual-size")
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.and_then(|value| value.parse::<usize>().ok());
|
||||
let _body_permit = timeout(MAX_FAULT_DURATION, Arc::clone(&self.body_limit).acquire_owned())
|
||||
.await
|
||||
.map_err(|_| s3s::s3_error!(RequestTimeout, "fake target body limiter wait exceeded 30 seconds"))?
|
||||
@@ -2864,6 +2873,7 @@ impl S3 for FakeBackend {
|
||||
body,
|
||||
e_tag: e_tag.clone(),
|
||||
digest,
|
||||
actual_size: declared_actual_size,
|
||||
},
|
||||
);
|
||||
Ok(apply_response_fault(
|
||||
@@ -2933,7 +2943,9 @@ impl S3 for FakeBackend {
|
||||
if requested_etag != &stored.e_tag {
|
||||
return Err(s3s::s3_error!(InvalidPart, "part ETag does not match"));
|
||||
}
|
||||
if index + 1 != requested_parts.len() && stored.body.len() < MIN_MULTIPART_PART_BYTES {
|
||||
if index + 1 != requested_parts.len()
|
||||
&& stored.actual_size.unwrap_or(stored.body.len()) < MIN_MULTIPART_PART_BYTES
|
||||
{
|
||||
return Err(s3s::s3_error!(EntityTooSmall, "non-final multipart part is smaller than 5 MiB"));
|
||||
}
|
||||
selected.push((*number, stored.clone()));
|
||||
|
||||
@@ -58,11 +58,22 @@ mod tests {
|
||||
struct ScannerHealEvidenceCase {
|
||||
id: &'static str,
|
||||
oracle: &'static str,
|
||||
evidence: &'static str,
|
||||
unclean_shutdown_marker: bool,
|
||||
}
|
||||
|
||||
const BACKGROUND_TARGET_RESTART_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-restart",
|
||||
oracle: "background-target-restart.json",
|
||||
evidence: "process-restart",
|
||||
unclean_shutdown_marker: false,
|
||||
};
|
||||
|
||||
const BACKGROUND_TARGET_CRASH_EVIDENCE: ScannerHealEvidenceCase = ScannerHealEvidenceCase {
|
||||
id: "background-target-crash",
|
||||
oracle: "background-target-crash.json",
|
||||
evidence: "process-crash-restart",
|
||||
unclean_shutdown_marker: true,
|
||||
};
|
||||
|
||||
struct RestartEvidenceContext {
|
||||
@@ -98,6 +109,8 @@ mod tests {
|
||||
|| case.oracle.contains('/')
|
||||
|| case.oracle.contains('\\')
|
||||
|| case.oracle.contains("..")
|
||||
|| !matches!(case.evidence, "process-restart" | "process-crash-restart")
|
||||
|| (case.evidence == "process-crash-restart") != case.unclean_shutdown_marker
|
||||
{
|
||||
return Err("invalid scanner/heal evidence case".into());
|
||||
}
|
||||
@@ -950,6 +963,16 @@ mod tests {
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_background_target_crash()
|
||||
-> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
timeout(
|
||||
Duration::from_secs(420),
|
||||
run_cluster_root_heal_interruption(InterruptionScenario::BackgroundTargetCrash),
|
||||
)
|
||||
.await?
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_cluster_root_heal_recovers_remote_shards_after_coordinator_restart() -> Result<(), Box<dyn Error + Send + Sync>>
|
||||
{
|
||||
@@ -986,21 +1009,27 @@ mod tests {
|
||||
enum InterruptionScenario {
|
||||
IsolatedTargetRestart,
|
||||
BackgroundTargetRestart,
|
||||
BackgroundTargetCrash,
|
||||
BackgroundCoordinatorRestart,
|
||||
TargetEndpointBlackhole,
|
||||
}
|
||||
|
||||
async fn run_cluster_root_heal_interruption(scenario: InterruptionScenario) -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
let server_binary = rustfs_binary_path();
|
||||
let evidence_run = if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
|
||||
} else {
|
||||
None
|
||||
let evidence_run = match scenario {
|
||||
InterruptionScenario::BackgroundTargetRestart => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_RESTART_EVIDENCE)?
|
||||
}
|
||||
InterruptionScenario::BackgroundTargetCrash => {
|
||||
restart_evidence_run(&server_binary, BACKGROUND_TARGET_CRASH_EVIDENCE)?
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let mut evidence_objects = Vec::new();
|
||||
let (background_enabled, interruption_node, interruption_kind) = match scenario {
|
||||
InterruptionScenario::IsolatedTargetRestart => (false, 1, "target_restart"),
|
||||
InterruptionScenario::BackgroundTargetRestart => (true, 1, "background_target_restart"),
|
||||
InterruptionScenario::BackgroundTargetCrash => (true, 1, "background_target_crash"),
|
||||
InterruptionScenario::BackgroundCoordinatorRestart => (true, 0, "coordinator_restart"),
|
||||
InterruptionScenario::TargetEndpointBlackhole => (false, 1, "target_endpoint_blackhole"),
|
||||
};
|
||||
@@ -1067,6 +1096,7 @@ mod tests {
|
||||
.unwrap_or(4 * 1024 * 1024)
|
||||
.clamp(1024 * 1024, 16 * 1024 * 1024);
|
||||
let mut expected_manifests = Vec::with_capacity(online_object_count);
|
||||
let mut unclean_shutdown_marker_observed = None;
|
||||
for index in 0..online_object_count {
|
||||
let key = format!("cluster/online/object-{index:04}.bin");
|
||||
let payload_seed = u8::try_from(index + 1).expect("clamped object count must fit in u8");
|
||||
@@ -1433,7 +1463,11 @@ mod tests {
|
||||
"Restored target endpoint forwarding"
|
||||
);
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
if scenario == InterruptionScenario::BackgroundTargetRestart {
|
||||
cluster.stop_node_gracefully(interruption_node).await?;
|
||||
} else {
|
||||
cluster.stop_node(interruption_node)?;
|
||||
}
|
||||
let stopped_count = metadata_count(&replaced_disk, bucket, &expected_manifests);
|
||||
assert!(
|
||||
stopped_count > 0 && stopped_count < expected_manifests.len(),
|
||||
@@ -1449,9 +1483,12 @@ mod tests {
|
||||
.join(".rustfs.sys")
|
||||
.join("unclean-shutdown");
|
||||
if background_enabled {
|
||||
let marker_exists = unclean_shutdown_marker.is_file();
|
||||
unclean_shutdown_marker_observed = Some(marker_exists);
|
||||
let expected_marker = !matches!(scenario, InterruptionScenario::BackgroundTargetRestart);
|
||||
assert!(
|
||||
unclean_shutdown_marker.is_file(),
|
||||
"background restart must retain the real unclean-shutdown marker"
|
||||
marker_exists == expected_marker,
|
||||
"background restart/crash lane observed unexpected unclean-shutdown marker state"
|
||||
);
|
||||
} else {
|
||||
match std::fs::remove_file(&unclean_shutdown_marker) {
|
||||
@@ -1651,13 +1688,14 @@ mod tests {
|
||||
"server build changed during restart"
|
||||
);
|
||||
let evidence = serde_json::json!({
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": "process-restart",
|
||||
"schema": 1, "case": evidence_context.case.id, "evidence": evidence_context.case.evidence,
|
||||
"run_id": evidence_context.run.run_id, "source_revision": evidence_context.run.source_revision,
|
||||
"test_build": compiled_test_identity(),
|
||||
"binary_sha256": evidence_context.run.binary.sha256,
|
||||
"test_binary_sha256": evidence_context.run.test_binary.sha256,
|
||||
"topology": {"nodes": cluster.nodes.len(), "drives_per_node": cluster.nodes[0].data_dirs.len()},
|
||||
"pid_before": target_pid, "pid_after": restarted_pid,
|
||||
"unclean_shutdown_marker": unclean_shutdown_marker_observed.unwrap_or(false),
|
||||
"objects": evidence_objects, "node_listings": node_listings,
|
||||
});
|
||||
let data = serde_json::to_vec(&evidence)?;
|
||||
|
||||
@@ -627,7 +627,7 @@ async fn put_bucket_replication_rules(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn delete_bucket_replication(
|
||||
pub(crate) async fn delete_bucket_replication(
|
||||
env: &RustFSTestEnvironment,
|
||||
bucket: &str,
|
||||
) -> Result<reqwest::Response, Box<dyn Error + Send + Sync>> {
|
||||
@@ -9055,9 +9055,11 @@ async fn test_replication_check_flags_multipart_only_version_minting_target() ->
|
||||
.is_some_and(|error| error.contains("CreateMultipartUpload")),
|
||||
"the failure must name the multipart path: {payload}"
|
||||
);
|
||||
// The PutObject leg mirrored, so it is the multipart probe that failed.
|
||||
// The PutObject leg mirrored, so it is the multipart probe that failed;
|
||||
// the mutation phases address the id the PUT reported and still run.
|
||||
assert_eq!(target_report["Phases"]["Put"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "SKIPPED", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["DeleteMarker"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["VersionDelete"]["Status"], "OK", "{payload}");
|
||||
assert_eq!(target_report["Phases"]["Cleanup"]["Status"], "OK", "{payload}");
|
||||
|
||||
let probe_key = target
|
||||
@@ -9245,6 +9247,9 @@ async fn test_replication_check_flags_version_minting_target() -> TestResult {
|
||||
let target_bucket = "version-fidelity-dst";
|
||||
target.create_bucket(target_bucket);
|
||||
target.assign_own_version_ids(true);
|
||||
// Wasabi shape: the probe version the VersionDelete phase removed answers
|
||||
// NoSuchVersion to cleanup's second DELETE, which must count as clean.
|
||||
target.reject_unknown_version_deletes(true);
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut env_vars = replication_fast_env();
|
||||
@@ -10185,3 +10190,199 @@ async fn test_get_object_tagging_proxies_unreplicated_object_to_replication_targ
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// backlog#2363
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Wait until the source reports a terminal replication status for `key`.
|
||||
async fn wait_terminal_replication_status(
|
||||
client: &Client,
|
||||
bucket: &str,
|
||||
key: &str,
|
||||
ssec: bool,
|
||||
timeout: Duration,
|
||||
) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let deadline = tokio::time::Instant::now() + timeout;
|
||||
loop {
|
||||
let request = client.head_object().bucket(bucket).key(key);
|
||||
let head = if ssec {
|
||||
request
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?
|
||||
} else {
|
||||
request.send().await?
|
||||
};
|
||||
let status = head.replication_status().map(|status| status.as_str().to_string());
|
||||
if matches!(status.as_deref(), Some("COMPLETED") | Some("FAILED")) {
|
||||
return Ok(status.unwrap_or_default());
|
||||
}
|
||||
if tokio::time::Instant::now() >= deadline {
|
||||
return Err(format!("{bucket}/{key}: replication never reached a terminal status; last {status:?}").into());
|
||||
}
|
||||
sleep(Duration::from_millis(250)).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// backlog#2363: SSE-C ciphertext passthrough of objects the source stored
|
||||
/// compressed. The replica on a RustFS target must decrypt to the original
|
||||
/// bytes for a single PUT and for a multipart upload.
|
||||
#[tokio::test]
|
||||
async fn test_bucket_replication_sse_c_compressed_passthrough() -> TestResult {
|
||||
init_logging();
|
||||
const PART_SIZE: usize = 5 * 1024 * 1024;
|
||||
|
||||
let mut source_env = RustFSTestEnvironment::new().await?;
|
||||
let mut target_env = RustFSTestEnvironment::new().await?;
|
||||
let mut source_process_env = replication_fast_env();
|
||||
source_process_env.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
source_process_env.extend_from_slice(FAST_SCANNER_ENV);
|
||||
source_process_env.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
("RUSTFS_COMPRESSION_ENABLED", "true"),
|
||||
("RUSTFS_COMPRESSION_MULTIPART_ENABLED", "true"),
|
||||
]);
|
||||
source_env.start_rustfs_server_with_env(vec![], &source_process_env).await?;
|
||||
target_env
|
||||
.start_rustfs_server_without_cleanup_with_env(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
])
|
||||
.await?;
|
||||
|
||||
let source_bucket = "ssec-compressed-src";
|
||||
let target_bucket = "ssec-compressed-dst";
|
||||
let source_client = source_env.create_s3_client();
|
||||
let target_client = target_env.create_s3_client();
|
||||
source_client.create_bucket().bucket(source_bucket).send().await?;
|
||||
target_client.create_bucket().bucket(target_bucket).send().await?;
|
||||
enable_bucket_versioning(&source_env, source_bucket).await?;
|
||||
enable_bucket_versioning(&target_env, target_bucket).await?;
|
||||
let target_arn = set_replication_target(&source_env, source_bucket, &target_env, target_bucket).await?;
|
||||
put_bucket_replication(&source_env, source_bucket, &target_arn).await?;
|
||||
|
||||
let customer_key = BASE64_STANDARD.encode_to_string(REPL17_SSEC_KEY);
|
||||
let customer_key_md5 = sse_customer_key_md5_base64(REPL17_SSEC_KEY);
|
||||
let text = |len: usize, seed: u32| -> Vec<u8> {
|
||||
let mut out = Vec::with_capacity(len + 64);
|
||||
let mut line = 0u64;
|
||||
while out.len() < len {
|
||||
out.extend_from_slice(format!("ssec compressed passthrough seed={seed} line={line} lorem ipsum dolor\n").as_bytes());
|
||||
line += 1;
|
||||
}
|
||||
out.truncate(len);
|
||||
out
|
||||
};
|
||||
|
||||
let single_key = "ssec-compressed-single.txt";
|
||||
let single_body = text(1024 * 1024 + 17, 1);
|
||||
source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(single_key)
|
||||
.content_type("text/plain")
|
||||
.body(ByteStream::from(single_body.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let multipart_key = "ssec-compressed-multipart.txt";
|
||||
let multipart_parts = [text(PART_SIZE, 2), text(1024 * 1024 + 4096, 3)];
|
||||
let multipart_body: Vec<u8> = multipart_parts.concat();
|
||||
let created = source_client
|
||||
.create_multipart_upload()
|
||||
.bucket(source_bucket)
|
||||
.key(multipart_key)
|
||||
.content_type("text/plain")
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
let upload_id = created.upload_id().ok_or("missing multipart upload id")?.to_string();
|
||||
let mut completed = Vec::new();
|
||||
for (index, part) in multipart_parts.iter().enumerate() {
|
||||
let part_number = i32::try_from(index + 1)?;
|
||||
let uploaded = source_client
|
||||
.upload_part()
|
||||
.bucket(source_bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(&upload_id)
|
||||
.part_number(part_number)
|
||||
.body(ByteStream::from(part.clone()))
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
completed.push(
|
||||
CompletedPart::builder()
|
||||
.part_number(part_number)
|
||||
.set_e_tag(uploaded.e_tag().map(str::to_string))
|
||||
.build(),
|
||||
);
|
||||
}
|
||||
source_client
|
||||
.complete_multipart_upload()
|
||||
.bucket(source_bucket)
|
||||
.key(multipart_key)
|
||||
.upload_id(&upload_id)
|
||||
.multipart_upload(CompletedMultipartUpload::builder().set_parts(Some(completed)).build())
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
let mut failures = Vec::new();
|
||||
for (key, body) in [(single_key, &single_body), (multipart_key, &multipart_body)] {
|
||||
let status = wait_terminal_replication_status(&source_client, source_bucket, key, true, Duration::from_secs(120)).await?;
|
||||
if status != "COMPLETED" {
|
||||
failures.push(format!("{key}: source reports {status}"));
|
||||
continue;
|
||||
}
|
||||
let replica = target_client
|
||||
.get_object()
|
||||
.bucket(target_bucket)
|
||||
.key(key)
|
||||
.sse_customer_algorithm("AES256")
|
||||
.sse_customer_key(&customer_key)
|
||||
.sse_customer_key_md5(&customer_key_md5)
|
||||
.send()
|
||||
.await;
|
||||
match replica {
|
||||
Ok(replica) => {
|
||||
let content_length = replica.content_length();
|
||||
match replica.body.collect().await {
|
||||
Ok(collected) => {
|
||||
let bytes = collected.into_bytes();
|
||||
if bytes.as_ref() != body.as_slice() {
|
||||
failures.push(format!(
|
||||
"{key}: replica bytes differ (content_length={content_length:?}, got {} bytes, want {})",
|
||||
bytes.len(),
|
||||
body.len()
|
||||
));
|
||||
}
|
||||
}
|
||||
Err(err) => failures.push(format!("{key}: replica body read failed: {err}")),
|
||||
}
|
||||
}
|
||||
Err(err) => failures.push(format!("{key}: replica GET failed: {err}")),
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
failures.is_empty(),
|
||||
"SSE-C compressed passthrough replicas must decrypt to the source bytes: {failures:?}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -36,11 +36,12 @@ use crate::fake_s3_target::{BucketMode, FAKE_ACCESS_KEY, FAKE_SECRET_KEY};
|
||||
use crate::fake_s3_target::{FakeS3Target, FaultAction as FakeTargetFault, Operation as FakeTargetOperation, RequestRecord};
|
||||
use crate::on_demand_migration::common::{OdmEnvOptions, OdmTestEnv, fake_source_client};
|
||||
use crate::replication_extension_test::{
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, enable_bucket_versioning, get_replication_reset_status,
|
||||
put_bucket_replication, put_bucket_replication_with_delete_statuses, set_replication_target_with_options,
|
||||
start_bucket_replication_reset,
|
||||
LOOPBACK_REPLICATION_TARGET_ENV, ReplicationTargetOptions, delete_bucket_replication, enable_bucket_versioning,
|
||||
get_replication_reset_status, put_bucket_replication, put_bucket_replication_with_delete_statuses,
|
||||
set_replication_target_with_options, start_bucket_replication_reset,
|
||||
};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::primitives::{ByteStream, DateTime};
|
||||
use aws_sdk_s3::types::{
|
||||
Checksum, ChecksumAlgorithm, CompletedMultipartUpload, CompletedPart, ObjectAttributes, ObjectLockLegalHold,
|
||||
@@ -640,6 +641,141 @@ async fn matrix_mint_own_version_ids_addresses_mutations_through_the_ledger() ->
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// rustfs/backlog#2340 (pending purge lifecycle): a permanent delete whose
|
||||
/// replication keeps failing leaves the version in xl.meta as a PENDING purge,
|
||||
/// hidden from listings. Once the bucket's replication configuration is
|
||||
/// removed nothing can ever confirm that purge remotely, so the delete worker
|
||||
/// must settle it locally (abandoned, with the replica left on the former
|
||||
/// target) — otherwise the bucket stays `BucketNotEmpty` forever with a
|
||||
/// residue the client cannot see.
|
||||
#[tokio::test]
|
||||
async fn matrix_removed_replication_config_abandons_pending_purge() -> TestResult {
|
||||
init_logging();
|
||||
|
||||
let target = FakeS3Target::start().await?;
|
||||
let target_bucket = "matrix-abandoned-purge-dst".to_string();
|
||||
target.create_bucket_with_object_lock(target_bucket.clone());
|
||||
TargetMode::MintOwnVersionIds.apply(&target);
|
||||
|
||||
let mut env_vars = replication_fast_env();
|
||||
env_vars.extend_from_slice(LOOPBACK_REPLICATION_TARGET_ENV);
|
||||
env_vars.extend_from_slice(&[
|
||||
("NO_PROXY", "127.0.0.1,localhost"),
|
||||
("HTTP_PROXY", ""),
|
||||
("HTTPS_PROXY", ""),
|
||||
// The scanner heal pass is what revisits a pending purge.
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "1"),
|
||||
]);
|
||||
let env = OdmTestEnv::start_with(OdmEnvOptions {
|
||||
env: env_vars,
|
||||
..OdmEnvOptions::default()
|
||||
})
|
||||
.await?;
|
||||
let source_env = &env.rustfs;
|
||||
|
||||
let source_bucket = "matrix-abandoned-purge-src";
|
||||
let source_client = source_env.create_s3_client();
|
||||
source_client
|
||||
.create_bucket()
|
||||
.bucket(source_bucket)
|
||||
.object_lock_enabled_for_bucket(true)
|
||||
.send()
|
||||
.await?;
|
||||
enable_bucket_versioning(source_env, source_bucket).await?;
|
||||
let target_arn = set_replication_target_with_options(
|
||||
source_env,
|
||||
source_bucket,
|
||||
ReplicationTargetOptions {
|
||||
endpoint: &target.address(),
|
||||
access_key: FAKE_ACCESS_KEY,
|
||||
secret_key: FAKE_SECRET_KEY,
|
||||
target_bucket: &target_bucket,
|
||||
secure: false,
|
||||
skip_tls_verify: false,
|
||||
ca_cert_pem: None,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
put_bucket_replication_with_delete_statuses(source_env, source_bucket, &target_arn, "Enabled", Some("Enabled")).await?;
|
||||
|
||||
let key = "purge/orphaned.bin";
|
||||
let put = source_client
|
||||
.put_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.body(ByteStream::from(payload(4 * 1024, 0x07)))
|
||||
.send()
|
||||
.await?;
|
||||
let source_version = put.version_id().ok_or("source PUT returned no version id")?.to_string();
|
||||
assert_eq!(
|
||||
wait_for_terminal_replication_status(&source_client, source_bucket, key).await?,
|
||||
"COMPLETED"
|
||||
);
|
||||
let replica = single_target_version(&target, &target_bucket, key)?;
|
||||
|
||||
// The target refuses every purge: the version stays a pending purge.
|
||||
// More refusals than any scanner cycle can consume within the test.
|
||||
target.inject_for_key(FakeTargetOperation::DeleteObject, key, FakeTargetFault::ResponseStatus(503), 4_000);
|
||||
source_client
|
||||
.delete_object()
|
||||
.bucket(source_bucket)
|
||||
.key(key)
|
||||
.version_id(&source_version)
|
||||
.send()
|
||||
.await?;
|
||||
wait_until("the refused purge to reach the target at least once", || async {
|
||||
Ok(target.count_requests(FakeTargetOperation::DeleteObject, key) >= 1)
|
||||
})
|
||||
.await?;
|
||||
let listed = source_client.list_object_versions().bucket(source_bucket).send().await?;
|
||||
assert!(
|
||||
listed.versions().is_empty() && listed.delete_markers().is_empty(),
|
||||
"a pending purge is hidden from listings: {listed:?}"
|
||||
);
|
||||
let blocked = source_client.delete_bucket().bucket(source_bucket).send().await;
|
||||
assert!(
|
||||
blocked
|
||||
.as_ref()
|
||||
.err()
|
||||
.and_then(|err| err.as_service_error())
|
||||
.is_some_and(|err| err.code() == Some("BucketNotEmpty")),
|
||||
"the hidden pending purge must block DeleteBucket while the target is still configured: {blocked:?}"
|
||||
);
|
||||
|
||||
// Removing the replication configuration orphans the purge; the scanner
|
||||
// heal pass must settle it locally so the bucket becomes deletable.
|
||||
let response = delete_bucket_replication(source_env, source_bucket).await?;
|
||||
assert!(response.status().is_success(), "DeleteBucketReplication: {}", response.status());
|
||||
wait_until("DeleteBucket to succeed once the orphaned purge is abandoned", || async {
|
||||
match source_client.delete_bucket().bucket(source_bucket).send().await {
|
||||
Ok(_) => Ok(true),
|
||||
Err(err) if err.as_service_error().is_some_and(|err| err.code() == Some("BucketNotEmpty")) => Ok(false),
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
// Abandoned means abandoned: the replica stays on the former target and,
|
||||
// once the attempts in flight at removal time have drained, no further
|
||||
// purge attempts are sent to it.
|
||||
assert_eq!(
|
||||
single_target_version(&target, &target_bucket, key)?,
|
||||
replica,
|
||||
"an abandoned purge must not touch the replica on the former target"
|
||||
);
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
let settled = target.count_requests(FakeTargetOperation::DeleteObject, key);
|
||||
sleep(Duration::from_secs(3)).await;
|
||||
assert_eq!(
|
||||
target.count_requests(FakeTargetOperation::DeleteObject, key),
|
||||
settled,
|
||||
"purge attempts must stop once the target is no longer configured"
|
||||
);
|
||||
|
||||
target.shutdown().await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn single_target_version(target: &FakeS3Target, target_bucket: &str, key: &str) -> Result<String, Box<dyn Error + Send + Sync>> {
|
||||
let versions = target.stored_versions(target_bucket, key);
|
||||
match versions.as_slice() {
|
||||
|
||||
@@ -1473,20 +1473,17 @@ fn layout_cases() -> Vec<LayoutCase> {
|
||||
true,
|
||||
),
|
||||
// SSE-C passthrough replicates the stored ciphertext part by part; a
|
||||
// compressible first part is stored well below 5 MiB and a standard
|
||||
// target rejects it with EntityTooSmall. rc.5 fails the same way (see
|
||||
// `rc5_baseline_replicates_multipart_layouts`), so the outcome is
|
||||
// recorded rather than asserted here; tracked as rustfs/backlog#2363.
|
||||
LayoutCase {
|
||||
assert_replication: false,
|
||||
..case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 7),
|
||||
true,
|
||||
)
|
||||
},
|
||||
// compressible first part is stored well below 5 MiB, so the sender
|
||||
// declares each part's plaintext length and the target validates the
|
||||
// 5 MiB minimum against it (rustfs/backlog#2363). rc.5 as the sender
|
||||
// still fails this layout (see `rc5_baseline_replicates_multipart_layouts`).
|
||||
case(
|
||||
LAYOUT_PLAIN_BUCKET,
|
||||
"plain/ssec-compressed-multipart-2.txt",
|
||||
two.clone(),
|
||||
layout_text(total(&two), 7),
|
||||
true,
|
||||
),
|
||||
case(
|
||||
LAYOUT_ENCRYPTED_BUCKET,
|
||||
"encrypted/single.bin",
|
||||
@@ -1838,27 +1835,20 @@ async fn direct_upgrade_from_rc5_preserves_multipart_layouts() -> TestResult {
|
||||
transport.uploaded_parts, expected,
|
||||
"{label}: stored parts must replicate as the same multipart layout"
|
||||
);
|
||||
// The current build can drive an existing object twice (two
|
||||
// full CreateMultipartUpload/UploadPart/Complete rounds with
|
||||
// distinct upload ids) while its status is still PENDING; the
|
||||
// rc.5 baseline drives once. That is a scheduling difference,
|
||||
// not a layout one, tracked as rustfs/backlog#2362.
|
||||
assert!(transport.completes >= 1, "{label}: at least one CompleteMultipartUpload");
|
||||
if transport.completes > 1 {
|
||||
tracing::warn!(
|
||||
target: "e2e_test::upgrade_compatibility_test",
|
||||
object = %label,
|
||||
completes = transport.completes,
|
||||
journal = ?transport.journal,
|
||||
"existing-object replication drove the same object more than once (rustfs/backlog#2362)"
|
||||
);
|
||||
}
|
||||
// An object still PENDING when the next scanner cycle arrives is
|
||||
// not driven a second time (rustfs/backlog#2362); the journal is
|
||||
// logged so a duplicate round is visible if this ever regresses.
|
||||
assert_eq!(
|
||||
transport.completes, 1,
|
||||
"{label}: exactly one CompleteMultipartUpload; journal {:?}",
|
||||
transport.journal
|
||||
);
|
||||
assert_eq!(
|
||||
transport.single_puts, 0,
|
||||
"{label}: a multipart layout must not go out as a single PutObject"
|
||||
);
|
||||
} else {
|
||||
assert!(transport.single_puts >= 1, "{label}: a single PUT replicates as PutObject");
|
||||
assert_eq!(transport.single_puts, 1, "{label}: a single PUT replicates as exactly one PutObject");
|
||||
assert!(transport.uploaded_parts.is_empty(), "{label}: a single PUT must not go out as multipart");
|
||||
}
|
||||
if !case.ssec {
|
||||
@@ -1928,3 +1918,59 @@ async fn rc5_baseline_replicates_multipart_layouts() -> TestResult {
|
||||
tracing::info!(target: "e2e_test::upgrade_compatibility_test", ?summary, "rc.5 baseline replication outcomes");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// backlog#2362 under the same conditions that reproduced it with the rc.5
|
||||
/// writer, but with the workspace build on both sides so it runs in the
|
||||
/// ordinary lane: every pre-existing layout is driven through exactly one
|
||||
/// upload round even though the scanner re-scans it every second while the
|
||||
/// first round is still in flight.
|
||||
#[tokio::test]
|
||||
async fn existing_object_replication_drives_each_layout_once() -> TestResult {
|
||||
init_logging();
|
||||
let server_env = layout_server_env();
|
||||
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server_with_env(vec![], &server_env).await?;
|
||||
let writer = env.create_s3_client();
|
||||
env.create_test_bucket(LAYOUT_PLAIN_BUCKET).await?;
|
||||
env.create_test_bucket(LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
enable_versioning(&writer, LAYOUT_PLAIN_BUCKET).await?;
|
||||
enable_versioning(&writer, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
put_default_sse_s3_encryption(&writer, LAYOUT_ENCRYPTED_BUCKET).await?;
|
||||
|
||||
let mut cases = layout_cases();
|
||||
for case in cases.iter_mut() {
|
||||
layout_write(&writer, case).await?;
|
||||
let head = layout_head(&writer, case).await?;
|
||||
case.rc5_etag = head
|
||||
.e_tag()
|
||||
.ok_or_else(|| format!("{}: HEAD omitted the ETag", case.label()))?
|
||||
.trim_matches('"')
|
||||
.to_string();
|
||||
}
|
||||
|
||||
// The objects come from an earlier process lifetime: the scanner starts
|
||||
// cold and every object is a candidate at once.
|
||||
env.restart_server_preserving_data(vec![], &server_env).await?;
|
||||
let client = env.create_s3_client();
|
||||
let (_target, transports) = replicate_layouts(&env, &client, &cases).await?;
|
||||
let mut duplicates = Vec::new();
|
||||
for (case, transport) in cases.iter().zip(&transports) {
|
||||
assert_eq!(
|
||||
transport.status,
|
||||
"COMPLETED",
|
||||
"{}: existing-object replication must complete",
|
||||
case.label()
|
||||
);
|
||||
let rounds = if case.is_multipart_layout() {
|
||||
transport.completes
|
||||
} else {
|
||||
transport.single_puts
|
||||
};
|
||||
if rounds != 1 {
|
||||
duplicates.push(format!("{}: {rounds} upload rounds; journal {:?}", case.label(), transport.journal));
|
||||
}
|
||||
}
|
||||
assert!(duplicates.is_empty(), "each existing object must be driven exactly once: {duplicates:?}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -33,6 +33,7 @@ pub mod bucket {
|
||||
pub use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetError, BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError,
|
||||
SsecPassthroughCapability, TargetClient, VersionIdentityCapability, append_version_id_query,
|
||||
resolve_delete_api_version_id,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -1425,7 +1425,12 @@ fn build_remove_object_headers(version_id: Option<&str>, opts: &RemoveObjectOpti
|
||||
/// and silently creates a delete marker instead of removing the version, while
|
||||
/// the source stamps `VersionPurgeStatus=Complete` (backlog#799 B8 / #857).
|
||||
/// Non-replication callers always pass the version through unchanged.
|
||||
fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObjectOptions) -> Option<String> {
|
||||
/// The `versionId` a replicated DELETE puts on the wire: none for a
|
||||
/// delete-marker creation (the target mints the marker; the source version
|
||||
/// travels in the internal headers for RustFS peers), the addressed version
|
||||
/// otherwise. A generic S3 target given the version id on a marker-creation
|
||||
/// DELETE would permanently delete that version instead.
|
||||
pub fn resolve_delete_api_version_id(version_id: Option<String>, opts: &RemoveObjectOptions) -> Option<String> {
|
||||
if opts.replication_request && opts.replication_delete_marker {
|
||||
None
|
||||
} else {
|
||||
|
||||
@@ -75,6 +75,7 @@ use tracing::{debug, info, instrument, warn};
|
||||
const EVENT_REPLICATION_WORKER_RESIZE_SKIPPED: &str = "replication_worker_resize_skipped";
|
||||
const EVENT_REPLICATION_WORKER_RESIZED: &str = "replication_worker_resized";
|
||||
const EVENT_REPLICATION_BACKPRESSURE: &str = "replication_backpressure";
|
||||
const EVENT_REPLICATION_IN_FLIGHT_SKIPPED: &str = "replication_in_flight_skipped";
|
||||
const EVENT_REPLICATION_RESYNC_LOAD_SKIPPED: &str = "replication_resync_load_skipped";
|
||||
const EVENT_REPLICATION_RESYNC_RECOVERED: &str = "replication_resync_recovered";
|
||||
const EVENT_REPLICATION_MRF_QUEUE_UNAVAILABLE: &str = "replication_mrf_queue_unavailable";
|
||||
@@ -1089,6 +1090,9 @@ pub struct ReplicationPool<S: ReplicationStorage> {
|
||||
workers: RwLock<Vec<Sender<ReplicationOperation>>>,
|
||||
lrg_workers: RwLock<Vec<Sender<ReplicationOperation>>>,
|
||||
|
||||
/// Object versions queued or being replicated right now (backlog#2362).
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
|
||||
// MRF (Most Recent Failures) channels
|
||||
mrf_replica_tx: Sender<ReplicationOperation>,
|
||||
// Shared among N MRF workers; Arc allows spawning more than one worker.
|
||||
@@ -1147,6 +1151,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
storage,
|
||||
workers: RwLock::new(Vec::new()),
|
||||
lrg_workers: RwLock::new(Vec::new()),
|
||||
in_flight: Arc::new(ReplicationInFlight::default()),
|
||||
mrf_replica_tx,
|
||||
mrf_replica_rx: Arc::new(Mutex::new(mrf_replica_rx)),
|
||||
mrf_save_tx,
|
||||
@@ -1202,12 +1207,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let active_counter = self.active_lrg_workers.clone();
|
||||
let storage = self.storage.clone();
|
||||
let stats = self.stats.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = rx;
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), in_flight.clone()).await;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1261,12 +1267,13 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let active_counter = self.active_workers.clone();
|
||||
let stats = self.stats.clone();
|
||||
let storage = self.storage.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
let mut rx = rx;
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), in_flight.clone()).await;
|
||||
}
|
||||
});
|
||||
|
||||
@@ -1305,6 +1312,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let active_counter = self.active_mrf_workers.clone();
|
||||
let stats = self.stats.clone();
|
||||
let storage = self.storage.clone();
|
||||
let in_flight = self.in_flight.clone();
|
||||
let mrf_rx = Arc::clone(&self.mrf_replica_rx);
|
||||
|
||||
let handle = tokio::spawn(async move {
|
||||
@@ -1324,7 +1332,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let Some(operation) = operation else { break };
|
||||
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), in_flight.clone()).await;
|
||||
}
|
||||
});
|
||||
self.task_handles.lock().await.push(handle);
|
||||
@@ -1454,6 +1462,24 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
|
||||
/// Queues a replica task
|
||||
pub async fn queue_replica_task(&self, ri: ReplicateObjectInfo) -> ReplicationQueueAdmission {
|
||||
// A version that is already queued or being uploaded is not driven a
|
||||
// second time: the scanner heal pass sees it as PENDING until the
|
||||
// first upload lands and would otherwise re-queue it every cycle
|
||||
// (backlog#2362). The key is released when the worker finishes, or
|
||||
// below when no worker accepts the task.
|
||||
if !self.in_flight.try_begin(&ri) {
|
||||
debug!(
|
||||
event = EVENT_REPLICATION_IN_FLIGHT_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION,
|
||||
bucket = %ri.bucket,
|
||||
object = %ri.name,
|
||||
version_id = ?ri.version_id,
|
||||
op_type = ?ri.op_type,
|
||||
"Replication task already in flight; not queued again"
|
||||
);
|
||||
return ReplicationQueueAdmission::Skipped;
|
||||
}
|
||||
let target_arns = ri.dsc.replicate_target_arns();
|
||||
// If object is large, queue it to a static set of large workers
|
||||
if should_queue_large_object(ri.size) {
|
||||
@@ -1484,7 +1510,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let resize = large_worker_backpressure_resize(existing, self.active_lrg_workers(), max_l_workers);
|
||||
drop(lrg_workers);
|
||||
|
||||
// Queue to MRF if worker is busy.
|
||||
// Queue to MRF if worker is busy. The MRF replay re-enters
|
||||
// this function, so the version is no longer in flight.
|
||||
self.in_flight.finish(&ri);
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "large_object").await;
|
||||
|
||||
if let Some(resize) = resize {
|
||||
@@ -1493,6 +1521,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
return admission;
|
||||
}
|
||||
}
|
||||
self.in_flight.finish(&ri);
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
}
|
||||
|
||||
@@ -1501,6 +1530,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
let ch = self.worker_queue_channel(&ri.op_type, &ri.bucket, &ri.name, ri.size).await;
|
||||
|
||||
let Some(channel) = ch else {
|
||||
self.in_flight.finish(&ri);
|
||||
return ReplicationQueueAdmission::Missed;
|
||||
};
|
||||
|
||||
@@ -1512,7 +1542,9 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
self.stats.dec_q(&ri.bucket, ri.size, ri.delete_marker, ri.op_type);
|
||||
self.stats.dec_target_q(&ri.bucket, &target_arns, ri.size);
|
||||
|
||||
// Queue to MRF if all workers are busy.
|
||||
// Queue to MRF if all workers are busy. The MRF replay re-enters this
|
||||
// function, so the version is no longer in flight.
|
||||
self.in_flight.finish(&ri);
|
||||
let admission = self.queue_mrf_save_admission(ri.to_mrf_entry(), "object").await;
|
||||
|
||||
// Try to scale up workers based on priority
|
||||
@@ -1811,7 +1843,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone(), self.in_flight.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1829,7 +1861,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), storage.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), storage.clone(), self.in_flight.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1846,7 +1878,7 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
) {
|
||||
while let Some(operation) = rx.recv().await {
|
||||
let _active = ActiveWorkerGuard::new(active_counter.clone());
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone()).await;
|
||||
process_replication_operation(operation, stats.clone(), self.storage.clone(), self.in_flight.clone()).await;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2281,6 +2313,64 @@ impl<S: ReplicationStorage> ReplicationPool<S> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Object versions currently queued or being uploaded, keyed by bucket,
|
||||
/// object name and version. `queue_replica_task` admits a version only once
|
||||
/// while it is in flight; the scanner heal pass and MRF replays that arrive
|
||||
/// in the meantime are `Skipped` instead of driving a second complete upload
|
||||
/// (backlog#2362). Entries are removed when the worker finishes the task or
|
||||
/// when no worker accepted it.
|
||||
#[derive(Debug, Default)]
|
||||
pub(crate) struct ReplicationInFlight {
|
||||
keys: std::sync::Mutex<std::collections::HashSet<(String, String, Option<uuid::Uuid>)>>,
|
||||
}
|
||||
|
||||
impl ReplicationInFlight {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, std::collections::HashSet<(String, String, Option<uuid::Uuid>)>> {
|
||||
self.keys.lock().unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
/// Claim `ri`; `false` when the same version is already in flight.
|
||||
fn try_begin(&self, ri: &ReplicateObjectInfo) -> bool {
|
||||
self.lock().insert((ri.bucket.clone(), ri.name.clone(), ri.version_id))
|
||||
}
|
||||
|
||||
fn finish(&self, ri: &ReplicateObjectInfo) {
|
||||
self.lock().remove(&(ri.bucket.clone(), ri.name.clone(), ri.version_id));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn len(&self) -> usize {
|
||||
self.lock().len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Releases the in-flight claim when the worker is done with the task,
|
||||
/// including when replication panics.
|
||||
struct ReplicationInFlightGuard {
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
key: ReplicateObjectInfo,
|
||||
}
|
||||
|
||||
impl ReplicationInFlightGuard {
|
||||
fn new(in_flight: Arc<ReplicationInFlight>, ri: &ReplicateObjectInfo) -> Self {
|
||||
Self {
|
||||
in_flight,
|
||||
key: ReplicateObjectInfo {
|
||||
bucket: ri.bucket.clone(),
|
||||
name: ri.name.clone(),
|
||||
version_id: ri.version_id,
|
||||
..Default::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ReplicationInFlightGuard {
|
||||
fn drop(&mut self) {
|
||||
self.in_flight.finish(&self.key);
|
||||
}
|
||||
}
|
||||
|
||||
struct ActiveWorkerGuard {
|
||||
counter: Arc<AtomicI32>,
|
||||
}
|
||||
@@ -2342,10 +2432,12 @@ async fn process_replication_operation<S: ReplicationStorage>(
|
||||
operation: ReplicationOperation,
|
||||
stats: Arc<ReplicationStats>,
|
||||
storage: Arc<S>,
|
||||
in_flight: Arc<ReplicationInFlight>,
|
||||
) {
|
||||
match operation {
|
||||
ReplicationOperation::Object(obj_info) => {
|
||||
let _backlog = ReplicationBacklogGuard::for_object(stats, obj_info.as_ref());
|
||||
let _in_flight = ReplicationInFlightGuard::new(in_flight, obj_info.as_ref());
|
||||
replicate_object(*obj_info, storage).await;
|
||||
}
|
||||
ReplicationOperation::Delete(del_info) => {
|
||||
@@ -3079,7 +3171,11 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
}
|
||||
|
||||
let rcfg = match ReplicationMetadataStore::optional_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(Some(config)) => Some(config),
|
||||
// A bucket without a configuration still owes its pending purges an
|
||||
// answer: the delete worker finishes them locally as abandoned, which
|
||||
// is what makes the bucket deletable again (rustfs/backlog#2340).
|
||||
Ok(None) if owes_version_purge(&oi) => None,
|
||||
Ok(None) => return ReplicationQueueAdmission::Skipped,
|
||||
Err(err) => {
|
||||
debug!(
|
||||
@@ -3129,7 +3225,7 @@ pub async fn queue_replication_heal(bucket: &str, oi: ObjectInfo, retry_count: u
|
||||
}
|
||||
};
|
||||
|
||||
let rcfg_wrapper = ReplicationConfig::new(Some(rcfg), tgts);
|
||||
let rcfg_wrapper = ReplicationConfig::new(rcfg, tgts);
|
||||
queue_replication_heal_internal(bucket, oi, rcfg_wrapper, retry_count)
|
||||
.await
|
||||
.admission
|
||||
@@ -3157,6 +3253,17 @@ pub async fn queue_replication_metadata(bucket: &str, oi: ObjectInfo, retry_coun
|
||||
}
|
||||
}
|
||||
|
||||
/// A version purge the persisted state still owes to named targets. Without
|
||||
/// the target list nothing can be settled, so such a version keeps the
|
||||
/// ordinary "no configuration, nothing to heal" skip.
|
||||
fn owes_version_purge(oi: &ObjectInfo) -> bool {
|
||||
!oi.version_purge_status.is_empty()
|
||||
&& oi
|
||||
.version_purge_status_internal
|
||||
.as_deref()
|
||||
.is_some_and(|statuses| !statuses.trim().is_empty())
|
||||
}
|
||||
|
||||
/// queue_replication_heal_internal enqueues objects that failed replication OR eligible for resyncing through
|
||||
/// an ongoing resync operation or via existing objects replication configuration setting.
|
||||
pub(crate) async fn queue_replication_heal_internal(
|
||||
@@ -3175,7 +3282,11 @@ pub(crate) async fn queue_replication_heal_internal(
|
||||
};
|
||||
}
|
||||
|
||||
if rcfg.config.is_none() || rcfg.remotes.is_none() {
|
||||
// Without a configuration or targets there is nothing to replicate —
|
||||
// except a version purge the bucket still owes: its stored decision names
|
||||
// the targets, and the delete worker settles the ones no longer
|
||||
// configured as abandoned (rustfs/backlog#2340).
|
||||
if (rcfg.config.is_none() || rcfg.remotes.is_none()) && !owes_version_purge(&oi) {
|
||||
return ReplicationHealQueueResult {
|
||||
object_info: roi,
|
||||
admission: ReplicationQueueAdmission::Skipped,
|
||||
@@ -3710,6 +3821,7 @@ mod tests {
|
||||
stats: Arc::new(ReplicationStats::new()),
|
||||
workers: RwLock::new(Vec::new()),
|
||||
lrg_workers: RwLock::new(Vec::new()),
|
||||
in_flight: Arc::new(ReplicationInFlight::default()),
|
||||
mrf_replica_tx,
|
||||
mrf_replica_rx: Arc::new(Mutex::new(mrf_replica_rx)),
|
||||
mrf_save_tx,
|
||||
@@ -3776,6 +3888,90 @@ mod tests {
|
||||
assert_eq!(current_queue(&pool, "admission-bucket").await, (1, 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_admits_a_version_once_while_it_is_in_flight() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let (tx, _rx) = mpsc::channel(4);
|
||||
pool.workers.write().await.push(tx);
|
||||
let ri = ReplicateObjectInfo {
|
||||
bucket: "in-flight-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
// backlog#2362: the scanner heal pass sees the version as PENDING
|
||||
// until the worker lands it; a second request must not drive it again.
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Skipped);
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (1, 4096));
|
||||
|
||||
// Another version of the same key is independent work.
|
||||
let newer = ReplicateObjectInfo {
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..ri.clone()
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(newer).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.in_flight.len(), 2);
|
||||
|
||||
// Once the worker finishes, the same version may be queued again
|
||||
// (for example after a FAILED status).
|
||||
pool.in_flight.finish(&ri);
|
||||
assert_eq!(pool.queue_replica_task(ri).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(current_queue(&pool, "in-flight-bucket").await, (3, 3 * 4096));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn queue_replica_task_releases_the_version_when_no_worker_accepts_it() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
let ri = ReplicateObjectInfo {
|
||||
bucket: "no-worker-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
size: 4096,
|
||||
op_type: ReplicationType::Object,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// No worker channel: the task is missed and must not stay claimed.
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Missed);
|
||||
assert_eq!(pool.in_flight.len(), 0);
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Missed);
|
||||
|
||||
// A full worker channel hands the task to the MRF save path; the MRF
|
||||
// replay re-enters the queue, so the claim is released here too.
|
||||
let (tx, _rx) = mpsc::channel(1);
|
||||
pool.workers.write().await.push(tx);
|
||||
assert_eq!(pool.queue_replica_task(ri.clone()).await, ReplicationQueueAdmission::Queued);
|
||||
let overflow = ReplicateObjectInfo {
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..ri
|
||||
};
|
||||
assert_eq!(pool.queue_replica_task(overflow).await, ReplicationQueueAdmission::Queued);
|
||||
assert_eq!(pool.in_flight.len(), 1, "only the version held by the worker channel stays in flight");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn in_flight_guard_releases_the_version_on_drop() {
|
||||
let in_flight = Arc::new(ReplicationInFlight::default());
|
||||
let ri = ReplicateObjectInfo {
|
||||
bucket: "guard-bucket".to_string(),
|
||||
name: "object".to_string(),
|
||||
version_id: Some(uuid::Uuid::new_v4()),
|
||||
..Default::default()
|
||||
};
|
||||
assert!(in_flight.try_begin(&ri));
|
||||
assert!(!in_flight.try_begin(&ri));
|
||||
{
|
||||
let _guard = ReplicationInFlightGuard::new(in_flight.clone(), &ri);
|
||||
assert_eq!(in_flight.len(), 1);
|
||||
}
|
||||
assert_eq!(in_flight.len(), 0);
|
||||
assert!(in_flight.try_begin(&ri));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn regular_worker_admission_counts_target_backlog_before_receive() {
|
||||
let pool = new_test_replication_pool(Arc::new(LoadResyncNodeStore::new("node-a", empty_resync_shared_state()))).await;
|
||||
|
||||
@@ -22,7 +22,8 @@ use super::replication_filemeta_boundary::ReplicationGenerationSnapshot;
|
||||
use super::replication_filemeta_boundary::{
|
||||
REPLICATE_EXISTING, ReplicateDecision, ReplicateObjectInfo, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction,
|
||||
ReplicationState, ReplicationStatusType, ReplicationType, VersionPurgeStatusType, get_replication_state,
|
||||
parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
parse_replicate_decision, replicate_decision_for_admitted_targets, replication_statuses_map, target_reset_header,
|
||||
version_purge_statuses_map,
|
||||
};
|
||||
use super::replication_lock_boundary::ReplicationLockTiming;
|
||||
use super::replication_logging::{EVENT_RESYNC_CONFIG_LOOKUP_SKIPPED, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_REPLICATION_RESYNC};
|
||||
@@ -130,6 +131,8 @@ const EVENT_REPLICATION_DRIFTED_REPLICA_LOCATED: &str = "replication_drifted_rep
|
||||
const EVENT_REPLICATION_OBJECT_FAILED: &str = "replication_object_failed";
|
||||
const EVENT_REPLICATION_PURGE_OBJECT_LOCK_DENIED: &str = "replication_purge_object_lock_denied";
|
||||
const EVENT_REPLICATION_PURGE_REPLICA_UNRESOLVED: &str = "replication_purge_replica_unresolved";
|
||||
const EVENT_REPLICATION_PURGE_ABANDONED: &str = "replication_purge_abandoned";
|
||||
const METRIC_VERSION_PURGE_ABANDONED_TOTAL: &str = "rustfs_replication_version_purge_abandoned_total";
|
||||
const EVENT_REPLICATION_DRIFTED_REPLICA_METADATA_SYNCED: &str = "replication_drifted_replica_metadata_synced";
|
||||
const METRIC_VERSION_PURGE_REPLICA_TOTAL: &str = "rustfs_replication_version_purge_replica_total";
|
||||
|
||||
@@ -2052,6 +2055,26 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
|
||||
let target_statuses = replication_statuses_map(&oi.replication_status_internal.clone().unwrap_or_default());
|
||||
let target_purge_statuses = version_purge_statuses_map(&oi.version_purge_status_internal.clone().unwrap_or_default());
|
||||
// A version purge is owed to the targets its purge state names, whatever
|
||||
// the configuration says now: the decision string is not persisted, so a
|
||||
// heal after restart (or after the configuration was removed or edited)
|
||||
// would otherwise never revisit the purge and the hidden version would
|
||||
// block DeleteBucket forever (rustfs/backlog#2340). The delete worker
|
||||
// settles a target the configuration no longer names as abandoned.
|
||||
let dsc = if delete_path && !dsc.replicate_any() {
|
||||
let owed: Vec<String> = target_purge_statuses
|
||||
.iter()
|
||||
.filter(|(_, status)| matches!(status, VersionPurgeStatusType::Pending | VersionPurgeStatusType::Failed))
|
||||
.map(|(arn, _)| arn.clone())
|
||||
.collect();
|
||||
if owed.is_empty() {
|
||||
dsc
|
||||
} else {
|
||||
replicate_decision_for_admitted_targets(&owed)
|
||||
}
|
||||
} else {
|
||||
dsc
|
||||
};
|
||||
let existing_obj_resync = if delete_path && !has_stored_delete_decision && !delete_state.0 && !delete_state.1 {
|
||||
Default::default()
|
||||
} else {
|
||||
@@ -2408,6 +2431,11 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
};
|
||||
|
||||
let purge_source = version_purge_source(&storage, &bucket, &dobj, &dsc).await.map(Arc::new);
|
||||
let configured_arns = if is_version_delete_replication(&dobj.delete_object) {
|
||||
configured_replication_arns(&bucket).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let mut join_set = JoinSet::new();
|
||||
|
||||
@@ -2429,6 +2457,17 @@ pub(crate) async fn replicate_delete_with_outcome<S: ReplicationStorage>(
|
||||
continue;
|
||||
}
|
||||
|
||||
// The bucket no longer replicates to this target: nothing can ever
|
||||
// confirm the purge remotely, so finish it locally as abandoned.
|
||||
if let Some(configured) = configured_arns.as_ref()
|
||||
&& !configured.contains(&tgt_entry.arn)
|
||||
{
|
||||
rinfos
|
||||
.targets
|
||||
.push(abandoned_purge_target_info(&bucket, &dobj, &tgt_entry.arn));
|
||||
continue;
|
||||
}
|
||||
|
||||
// Get the remote target client
|
||||
let Some(tgt_client) = ReplicationTargetStore::remote_target_client(&bucket, &tgt_entry.arn).await else {
|
||||
debug!(
|
||||
@@ -3199,6 +3238,56 @@ fn unavailable_delete_target_info(dobj: &DeletedObjectReplicationInfo, arn: &str
|
||||
rinfo
|
||||
}
|
||||
|
||||
/// The target ARNs the bucket's replication configuration still names, or
|
||||
/// `None` when that cannot be decided right now (unreadable/invalid
|
||||
/// configuration): a purge is only abandoned on positive evidence. No
|
||||
/// configuration at all names no target.
|
||||
async fn configured_replication_arns(bucket: &str) -> Option<HashSet<String>> {
|
||||
match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => Some(config.configured_target_arns()),
|
||||
Ok(None) => Some(HashSet::new()),
|
||||
Err(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Finish a version purge locally for a target the bucket no longer
|
||||
/// replicates to (the rule or the whole configuration was removed).
|
||||
///
|
||||
/// The source keeps a purged version in xl.meta, hidden from listings, until
|
||||
/// every target confirms the purge — and once the operator removed the
|
||||
/// target nothing ever will: the version stayed PENDING forever, blocking
|
||||
/// `DeleteBucket` with a residue the client could neither see nor remove
|
||||
/// (rustfs/backlog#2340). Reporting the purge as complete lets the normal
|
||||
/// writeback drop the version. The replica, if any, stays on the former
|
||||
/// target: that is the operator's data now, and this event is the record.
|
||||
fn abandoned_purge_target_info(bucket: &str, dobj: &DeletedObjectReplicationInfo, arn: &str) -> ReplicatedTargetInfo {
|
||||
let mut rinfo = dobj
|
||||
.delete_object
|
||||
.replication_state
|
||||
.clone()
|
||||
.unwrap_or_default()
|
||||
.target_state(arn);
|
||||
rinfo.op_type = dobj.op_type;
|
||||
if rinfo.version_purge_status == VersionPurgeStatusType::Complete {
|
||||
return rinfo;
|
||||
}
|
||||
warn!(
|
||||
event = EVENT_REPLICATION_PURGE_ABANDONED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket,
|
||||
object = dobj.delete_object.object_name,
|
||||
version_id = ?dobj.delete_object.version_id.or(dobj.delete_object.delete_marker_version_id),
|
||||
arn,
|
||||
reason = "target_not_configured",
|
||||
"Replicated version purge abandoned: the bucket no longer replicates to this target, so the version is purged locally"
|
||||
);
|
||||
counter!(METRIC_VERSION_PURGE_ABANDONED_TOTAL).increment(1);
|
||||
rinfo.version_purge_status = VersionPurgeStatusType::Complete;
|
||||
rinfo.error = None;
|
||||
rinfo
|
||||
}
|
||||
|
||||
/// What the source still knows about a data version being purged, read once
|
||||
/// per delete: the version stays in xl.meta with a PENDING purge status until
|
||||
/// every target confirms, so its ETag and target-version ledger are available
|
||||
@@ -5558,6 +5647,17 @@ async fn replicate_multipart_parts_and_complete<S: ReplicationObjectIO>(
|
||||
};
|
||||
header_size = 0;
|
||||
|
||||
// Passthrough parts are the stored bytes; the replica learns each
|
||||
// part's plaintext length from this header (backlog#2363).
|
||||
let mut part_options = PutObjectPartOptions::default();
|
||||
if obj_opts.raw_data_movement_read && part_info.actual_size > 0 {
|
||||
rustfs_utils::http::insert_header(
|
||||
&mut part_options.custom_header,
|
||||
rustfs_utils::http::SUFFIX_REPLICATION_PART_ACTUAL_SIZE,
|
||||
part_info.actual_size.to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let object_part = cli
|
||||
.put_object_part(
|
||||
dst_bucket,
|
||||
@@ -5566,7 +5666,7 @@ async fn replicate_multipart_parts_and_complete<S: ReplicationObjectIO>(
|
||||
part_plan.part_number,
|
||||
part_plan.part_size,
|
||||
byte_stream,
|
||||
&PutObjectPartOptions { ..Default::default() },
|
||||
&part_options,
|
||||
)
|
||||
.await
|
||||
.map_err(|e| std::io::Error::other(e.to_string()))?;
|
||||
@@ -6719,6 +6819,27 @@ mod tests {
|
||||
assert_eq!(server.join().expect("test HTTP server should finish").len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn abandoned_purge_completes_locally_and_keeps_a_finished_target_untouched() {
|
||||
let arn = "arn:rustfs:replication::removed-target";
|
||||
let dobj = version_purge_dobj(arn);
|
||||
let rinfo = abandoned_purge_target_info("source", &dobj, arn);
|
||||
assert_eq!(rinfo.version_purge_status, VersionPurgeStatusType::Complete);
|
||||
assert_eq!(rinfo.arn, arn);
|
||||
assert!(rinfo.error.is_none());
|
||||
|
||||
let mut finished = version_purge_dobj(arn);
|
||||
finished
|
||||
.delete_object
|
||||
.replication_state
|
||||
.as_mut()
|
||||
.expect("purge state")
|
||||
.purge_targets
|
||||
.insert(arn.to_string(), VersionPurgeStatusType::Complete);
|
||||
let rinfo = abandoned_purge_target_info("source", &finished, arn);
|
||||
assert_eq!(rinfo.version_purge_status, VersionPurgeStatusType::Complete);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn version_purge_refuses_a_corrupt_ledger() {
|
||||
let target = test_target_client("http://127.0.0.1:1".to_string());
|
||||
@@ -7392,6 +7513,42 @@ mod tests {
|
||||
assert_eq!(roi.target_purge_statuses.get(role), Some(&VersionPurgeStatusType::Pending));
|
||||
}
|
||||
|
||||
/// The decision string is not persisted: after a restart, or once the
|
||||
/// configuration is gone, a heal of a failed purge must still name the
|
||||
/// targets the purge state records — that is what lets the delete worker
|
||||
/// settle a removed target as abandoned instead of skipping forever.
|
||||
#[tokio::test]
|
||||
async fn heal_owes_a_failed_purge_to_the_targets_its_purge_state_names_without_a_configuration() {
|
||||
let bucket = format!("heal-orphaned-purge-{}", Uuid::new_v4());
|
||||
let arn = "arn:rustfs:replication:us-east-1:removed:bucket";
|
||||
ReplicationVersioningStore::install_prefix_state_test_config(
|
||||
&bucket,
|
||||
VersioningConfiguration {
|
||||
status: Some(BucketVersioningStatus::from_static(BucketVersioningStatus::ENABLED)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let oi = ObjectInfo {
|
||||
bucket,
|
||||
name: "purge/orphaned.bin".to_string(),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
mod_time: Some(OffsetDateTime::now_utc()),
|
||||
version_purge_status: VersionPurgeStatusType::Failed,
|
||||
version_purge_status_internal: Some(format!("{arn}=FAILED;")),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let mut roi = get_heal_replicate_object_info(&oi, &ReplicationConfig::new(None, None))
|
||||
.await
|
||||
.expect("a purge without a configuration must still classify");
|
||||
|
||||
assert!(roi.dsc.targets_map.get(arn).is_some_and(|target| target.replicate), "{:?}", roi.dsc);
|
||||
assert!(matches!(
|
||||
super::super::replication_queue_boundary::replication_heal_queue_action(&mut roi),
|
||||
super::super::replication_queue_boundary::ReplicationHealQueueAction::QueueDelete(_)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_pending_purge_reads_one_versioning_generation() {
|
||||
let bucket = format!("heal-versioning-snapshot-{}", Uuid::new_v4());
|
||||
@@ -8097,47 +8254,77 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_preserves_legacy_zero_actual_sizes() {
|
||||
run_transport(4096, None).await;
|
||||
run_transport(4096, None, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_uploads_an_empty_last_part_without_reading_a_range() {
|
||||
run_transport(0, None).await;
|
||||
run_transport(0, None, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_preserves_transformed_unknown_nonempty_parts() {
|
||||
for unknown_part in [(0, 0), (1, 0), (0, -1), (1, -1)] {
|
||||
run_transport(4096, Some(unknown_part)).await;
|
||||
run_transport(4096, Some(unknown_part), false).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_preserves_transformed_empty_tail() {
|
||||
run_transport(0, Some((1, 0))).await;
|
||||
run_transport(0, Some((1, 0)), false).await;
|
||||
}
|
||||
|
||||
async fn run_transport(tail_size: usize, unknown_part: Option<(usize, i64)>) {
|
||||
/// SSE-C passthrough of a compressed object: the stored bytes go out
|
||||
/// as-is and every UploadPart declares the part's plaintext length
|
||||
/// (backlog#2363).
|
||||
#[tokio::test]
|
||||
async fn multipart_transport_declares_passthrough_part_lengths() {
|
||||
run_transport(4096, None, true).await;
|
||||
}
|
||||
|
||||
async fn run_transport(tail_size: usize, unknown_part: Option<(usize, i64)>, passthrough: bool) {
|
||||
const FIRST_SIZE: usize = 5 * 1024 * 1024;
|
||||
// The stored (compressed ciphertext) bytes of a passthrough part
|
||||
// are shorter than the plaintext they represent.
|
||||
const PASSTHROUGH_PLAINTEXT_FACTOR: usize = 4;
|
||||
let body = Bytes::from([vec![0x35; FIRST_SIZE], vec![0xa7; tail_size]].concat());
|
||||
let etag = faster_hex::hex_string(rustfs_utils::hash::HashAlgorithm::Md5.hash_encode(&body).as_ref());
|
||||
let mut user_defined = if unknown_part.is_some() {
|
||||
HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])
|
||||
} else {
|
||||
HashMap::new()
|
||||
};
|
||||
if passthrough {
|
||||
user_defined.insert(rustfs_utils::http::SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string());
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
"klauspost/compress/s2".to_string(),
|
||||
);
|
||||
}
|
||||
let plaintext_len = |stored: usize| {
|
||||
i64::try_from(if passthrough {
|
||||
stored * PASSTHROUGH_PLAINTEXT_FACTOR
|
||||
} else {
|
||||
stored
|
||||
})
|
||||
.expect("plaintext size")
|
||||
};
|
||||
let source = Arc::new(Source {
|
||||
info: ObjectInfo {
|
||||
size: i64::try_from(body.len() + if unknown_part.is_some() { 16 } else { 0 }).expect("stored size"),
|
||||
actual_size: i64::try_from(body.len()).expect("body size"),
|
||||
actual_size: plaintext_len(body.len()),
|
||||
etag: Some(etag.clone()),
|
||||
version_id: Some(Uuid::new_v4()),
|
||||
user_defined: Arc::new(if unknown_part.is_some() {
|
||||
HashMap::from([("x-amz-server-side-encryption".to_string(), "AES256".to_string())])
|
||||
} else {
|
||||
HashMap::new()
|
||||
}),
|
||||
user_defined: Arc::new(user_defined),
|
||||
parts: Arc::new(vec![
|
||||
ObjectPartInfo {
|
||||
number: 1,
|
||||
size: FIRST_SIZE + if unknown_part.is_some() { 8 } else { 0 },
|
||||
actual_size: if let Some((0, size)) = unknown_part {
|
||||
size
|
||||
} else if passthrough {
|
||||
plaintext_len(FIRST_SIZE)
|
||||
} else if unknown_part.is_some() || tail_size == 0 {
|
||||
i64::try_from(FIRST_SIZE).expect("first part size")
|
||||
} else {
|
||||
@@ -8150,6 +8337,8 @@ mod tests {
|
||||
size: tail_size + if unknown_part.is_some() { 8 } else { 0 },
|
||||
actual_size: if let Some((1, size)) = unknown_part {
|
||||
size
|
||||
} else if passthrough {
|
||||
plaintext_len(tail_size)
|
||||
} else if unknown_part.is_some() {
|
||||
i64::try_from(tail_size).expect("tail logical size")
|
||||
} else {
|
||||
@@ -8228,6 +8417,7 @@ mod tests {
|
||||
let (put_opts, is_multipart) = replication_put_object_options("STANDARD", &source.info).expect("replication options");
|
||||
let opts = ObjectOptions {
|
||||
version_id: source.info.version_id.map(|id| id.to_string()),
|
||||
raw_data_movement_read: passthrough,
|
||||
..Default::default()
|
||||
};
|
||||
let reader = source
|
||||
@@ -8310,6 +8500,36 @@ mod tests {
|
||||
requests[index].headers.get("content-length").expect("part content length"),
|
||||
expected.len().to_string().as_str()
|
||||
);
|
||||
let declared = rustfs_utils::http::get_header(
|
||||
&requests[index].headers,
|
||||
rustfs_utils::http::SUFFIX_REPLICATION_PART_ACTUAL_SIZE,
|
||||
);
|
||||
if passthrough {
|
||||
assert_eq!(
|
||||
declared.as_deref(),
|
||||
Some(plaintext_len(expected.len()).to_string().as_str()),
|
||||
"passthrough parts declare their plaintext length"
|
||||
);
|
||||
} else {
|
||||
assert!(declared.is_none(), "decrypted transport carries no passthrough part length");
|
||||
}
|
||||
}
|
||||
if passthrough {
|
||||
let create = &requests[0];
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(&create.headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION)
|
||||
.as_deref(),
|
||||
Some("klauspost/compress/s2"),
|
||||
"the session carries the source's compression scheme"
|
||||
);
|
||||
assert_eq!(
|
||||
rustfs_utils::http::get_header(
|
||||
&create.headers,
|
||||
rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE
|
||||
)
|
||||
.as_deref(),
|
||||
Some(plaintext_len(body.len()).to_string().as_str())
|
||||
);
|
||||
}
|
||||
let complete = &requests[3];
|
||||
assert_eq!(complete.method, http::Method::POST);
|
||||
|
||||
@@ -247,6 +247,23 @@ pub(crate) fn replication_put_object_options(sc: &str, object_info: &ObjectInfo)
|
||||
meta.insert(key.to_string(), value.to_string());
|
||||
}
|
||||
|
||||
// A compressed SSE-C object passes through as its stored bytes. The target
|
||||
// cannot infer the compression layout from ciphertext, so the scheme and
|
||||
// the plaintext size travel as transport headers; each UploadPart carries
|
||||
// its own plaintext length (backlog#2363).
|
||||
if is_ssec && let Some(scheme) = get_str(&object_info.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION) {
|
||||
insert_header_map(&mut meta, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION, scheme);
|
||||
if let Ok(actual_size) = object_info.get_actual_size()
|
||||
&& actual_size >= 0
|
||||
{
|
||||
insert_header_map(
|
||||
&mut meta,
|
||||
rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE,
|
||||
actual_size.to_string(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Managed SSE replicates as plaintext (the replication reader decrypts via
|
||||
// the object-encryption resolver) and re-encrypts on the target with the
|
||||
// target's own KMS. Send only the encryption intent — never the source
|
||||
@@ -626,6 +643,59 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn compressed_ssec_objects_declare_their_compression_layout_on_the_wire() {
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE,
|
||||
insert_str,
|
||||
};
|
||||
|
||||
let mut ssec_compressed = HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]);
|
||||
insert_str(&mut ssec_compressed, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
|
||||
insert_str(&mut ssec_compressed, SUFFIX_ACTUAL_SIZE, "6295552".to_string());
|
||||
let object_info = ObjectInfo {
|
||||
etag: Some("0123456789abcdef0123456789abcdef-2".to_string()),
|
||||
size: 4321,
|
||||
actual_size: 6295552,
|
||||
user_defined: Arc::new(ssec_compressed),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
// SSE-C passthrough sends stored bytes: the scheme and the plaintext
|
||||
// size travel as transport headers, never as the internal key
|
||||
// (backlog#2363).
|
||||
let (options, _) = replication_put_object_options("STANDARD", &object_info).expect("ssec put options");
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION).as_deref(),
|
||||
Some("klauspost/compress/s2")
|
||||
);
|
||||
assert_eq!(
|
||||
get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE).as_deref(),
|
||||
Some("6295552")
|
||||
);
|
||||
assert!(
|
||||
!options
|
||||
.user_metadata
|
||||
.keys()
|
||||
.any(|key| rustfs_utils::http::is_internal_key(key)),
|
||||
"internal metadata never leaves the source as plain metadata: {:?}",
|
||||
options.user_metadata
|
||||
);
|
||||
|
||||
// A compressed object that is not SSE-C is decompressed by the
|
||||
// replication reader and travels as plaintext: no layout headers.
|
||||
let mut plain_compressed = HashMap::new();
|
||||
insert_str(&mut plain_compressed, SUFFIX_COMPRESSION, "klauspost/compress/s2".to_string());
|
||||
insert_str(&mut plain_compressed, SUFFIX_ACTUAL_SIZE, "6295552".to_string());
|
||||
let plain = ObjectInfo {
|
||||
user_defined: Arc::new(plain_compressed),
|
||||
..object_info
|
||||
};
|
||||
let (options, _) = replication_put_object_options("STANDARD", &plain).expect("plain put options");
|
||||
assert!(get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION).is_none());
|
||||
assert!(get_header_map(&options.user_metadata, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn legacy_transformed_single_put_parts_keep_the_previous_replication_route() {
|
||||
let [_, (_, compressed), (_, encrypted), (_, ssec)] = replication_route_metadata();
|
||||
|
||||
@@ -747,18 +747,27 @@ mod tests {
|
||||
let mut kvs = KVS::new();
|
||||
kvs.insert(CLASS_STANDARD.to_string(), "EC:2".to_string());
|
||||
|
||||
let err = lookup_config_for_pools_with_env(&kvs, &[4, 2], no_env_overrides())
|
||||
.expect_err("EC:2 must be rejected by the two-drive pool");
|
||||
assert!(
|
||||
err.to_string().contains("pool 1") && err.to_string().contains("2 drives"),
|
||||
"error must identify the rejecting pool: {err}"
|
||||
);
|
||||
for drives in [2, 3] {
|
||||
let err = lookup_config_for_pools_with_env(&kvs, &[4, drives], no_env_overrides())
|
||||
.expect_err("EC:2 must be rejected by a pool with fewer than four drives per set");
|
||||
assert!(
|
||||
err.to_string().contains("pool 1") && err.to_string().contains(&format!("{drives} drives")),
|
||||
"error must identify the rejecting pool: {err}"
|
||||
);
|
||||
}
|
||||
|
||||
let cfg =
|
||||
lookup_config_for_pools_with_env(&kvs, &[4, 4], no_env_overrides()).expect("EC:2 is valid for both four-drive pools");
|
||||
assert_eq!(cfg.parities_for_sc(STANDARD), Some(vec![2, 2]));
|
||||
|
||||
kvs.insert(CLASS_STANDARD.to_string(), "EC:1".to_string());
|
||||
let cfg = lookup_config_for_pools_with_env(&kvs, &[4, 2], no_env_overrides()).expect("EC:1 is valid for both pools");
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, 4), Some(1));
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, 2), Some(1));
|
||||
assert_eq!(cfg.get_parity_for_sc(STANDARD), Some(1));
|
||||
for drives in [2, 3, 4] {
|
||||
let cfg =
|
||||
lookup_config_for_pools_with_env(&kvs, &[4, drives], no_env_overrides()).expect("EC:1 is valid for both pools");
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, 4), Some(1));
|
||||
assert_eq!(cfg.parity_for_sc(STANDARD, drives), Some(1));
|
||||
assert_eq!(cfg.get_parity_for_sc(STANDARD), Some(1));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -4490,11 +4490,20 @@ impl PoolMetaWriteState {
|
||||
|
||||
fn observe_selection(&mut self, selection: &PoolMetaSelection) -> Result<()> {
|
||||
self.pool_meta_absent = selection.absent;
|
||||
self.validate_selection(selection)?;
|
||||
if self.cluster_epoch.is_none()
|
||||
&& let Some((_, metadata_epoch)) = selection.generation_identity
|
||||
{
|
||||
self.cluster_epoch = Some(metadata_epoch);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_selection(&self, selection: &PoolMetaSelection) -> Result<()> {
|
||||
if let Some(expected_cluster_id) = self.expected_cluster_id
|
||||
&& let Some((cluster_id, _)) = selection.generation_identity
|
||||
&& cluster_id != expected_cluster_id
|
||||
{
|
||||
self.block_writes();
|
||||
return Err(Error::other(format!(
|
||||
"pool metadata incompatible: cluster identity {cluster_id} does not match deployment {expected_cluster_id}"
|
||||
)));
|
||||
@@ -4503,17 +4512,11 @@ impl PoolMetaWriteState {
|
||||
&& let Some((_, metadata_epoch)) = selection.generation_identity
|
||||
&& metadata_epoch != identity_epoch
|
||||
{
|
||||
self.block_writes();
|
||||
return Err(Error::other(format!(
|
||||
"pool metadata recovery required: committed epoch {} does not match cluster identity epoch {identity_epoch}",
|
||||
metadata_epoch
|
||||
)));
|
||||
}
|
||||
if self.cluster_epoch.is_none()
|
||||
&& let Some((_, metadata_epoch)) = selection.generation_identity
|
||||
{
|
||||
self.cluster_epoch = Some(metadata_epoch);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -4546,26 +4549,25 @@ impl PoolMetaWriteState {
|
||||
if !self.pool_meta_absent {
|
||||
return Ok(());
|
||||
}
|
||||
let result = self.validate_missing_metadata_can_initialize();
|
||||
if result.is_err() {
|
||||
self.block_writes();
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
fn validate_missing_metadata_can_initialize(&self) -> Result<()> {
|
||||
match self.identity_initialized {
|
||||
Some(false) if self.bootstrap_identity_proven() && self.identity_fresh_bootstrap_nonce.is_some() => Ok(()),
|
||||
Some(false) => {
|
||||
self.block_writes();
|
||||
Err(Error::other(
|
||||
"pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof or legacy-adoption proof",
|
||||
))
|
||||
}
|
||||
Some(true) => {
|
||||
self.block_writes();
|
||||
Err(Error::other(
|
||||
"pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
|
||||
))
|
||||
}
|
||||
None => {
|
||||
self.block_writes();
|
||||
Err(Error::other(
|
||||
"pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
|
||||
))
|
||||
}
|
||||
Some(false) => Err(Error::other(
|
||||
"pool metadata recovery required: pending cluster identity exists but this startup has no verified fresh-bootstrap proof or legacy-adoption proof",
|
||||
)),
|
||||
Some(true) => Err(Error::other(
|
||||
"pool metadata recovery required: initialized cluster identity exists but every pool.bin replica is missing",
|
||||
)),
|
||||
None => Err(Error::other(
|
||||
"pool metadata recovery required: no durable bootstrap identity or pool.bin replica is available",
|
||||
)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5145,12 +5147,12 @@ fn select_pool_meta_replicas_for_read_probe<R>(
|
||||
where
|
||||
R: Into<PoolMetaReplicaRead>,
|
||||
{
|
||||
// Read-only planning probes must fail the current request on unsafe pool
|
||||
// metadata, but they must not permanently poison the shared writer gate.
|
||||
let mut probe_state = write_state.clone();
|
||||
let selection = select_pool_meta_replicas_observing(&mut probe_state, replicas)?;
|
||||
probe_state.observe_replicas(selection.replica_state);
|
||||
probe_state.ensure_write_safe(operation)?;
|
||||
let selection = select_pool_meta_replica_reads(replicas.into_iter().map(Into::into).collect())?;
|
||||
write_state.validate_selection(&selection)?;
|
||||
selection.replica_state.ensure_write_safe(operation)?;
|
||||
if selection.absent && (write_state.expected_cluster_id.is_some() || write_state.identity_initialized.is_some()) {
|
||||
write_state.validate_missing_metadata_can_initialize()?;
|
||||
}
|
||||
Ok(selection)
|
||||
}
|
||||
|
||||
@@ -9014,7 +9016,7 @@ impl ECStore {
|
||||
|
||||
async fn acquire_pool_meta_read_guard(
|
||||
&self,
|
||||
write_state: &mut PoolMetaWriteState,
|
||||
write_state: &PoolMetaWriteState,
|
||||
operation: &str,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, PoolMeta)> {
|
||||
write_state.ensure_write_safe(operation)?;
|
||||
@@ -9172,9 +9174,9 @@ impl ECStore {
|
||||
target_pool_indices: &[usize],
|
||||
phase: &'static str,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (pool_meta_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "target capacity admission failed")
|
||||
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
||||
.await?;
|
||||
for target_pool_index in target_pool_indices.iter().copied() {
|
||||
ensure_external_decommission_target_admission(&snapshot, target_pool_index, phase)?;
|
||||
@@ -9208,9 +9210,9 @@ impl ECStore {
|
||||
pub(crate) async fn acquire_decommission_capacity_release_fence_with_active_source(
|
||||
&self,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (pool_meta_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "capacity release fence failed")
|
||||
.acquire_pool_meta_read_guard(&save_guard, "capacity release fence failed")
|
||||
.await?;
|
||||
let has_active_source = pool_meta_has_active_decommission(&snapshot);
|
||||
drop(save_guard);
|
||||
@@ -9275,9 +9277,9 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let (reconciliations, model_version) = {
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (_read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "exact delete capacity reconciliation failed")
|
||||
.acquire_pool_meta_read_guard(&save_guard, "exact delete capacity reconciliation failed")
|
||||
.await?;
|
||||
let reconciliations = plan_exact_delete_capacity_reconciliations(&snapshot, object, exact)?;
|
||||
let model_version = active_decommission_capacity_model(&snapshot)?;
|
||||
@@ -9882,9 +9884,9 @@ impl ECStore {
|
||||
let non_growing_replacement = matches!(mode, DecommissionCapacityMutationMode::NonGrowingReplacement);
|
||||
let temporary_release = matches!(mode, DecommissionCapacityMutationMode::TemporaryRelease);
|
||||
let mut operation = Some(operation);
|
||||
let mut save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&mut save_guard, "target capacity admission failed")
|
||||
.acquire_pool_meta_read_guard(&save_guard, "target capacity admission failed")
|
||||
.await?;
|
||||
let admission_now = OffsetDateTime::now_utc();
|
||||
let admitted_owner = capacity_owner.and_then(|owner| {
|
||||
@@ -10269,6 +10271,14 @@ impl ECStore {
|
||||
self.pool_meta_save_gate.lock().await.ensure_write_safe(operation)
|
||||
}
|
||||
|
||||
/// Reports whether pool metadata side effects are currently writable.
|
||||
/// Read-only admission probes do not change this state; startup and real
|
||||
/// metadata transactions still latch it on unrecoverable conditions.
|
||||
pub async fn pool_meta_writes_ready(&self) -> bool {
|
||||
let write_state = self.pool_meta_save_gate.lock().await;
|
||||
!write_state.write_blocked && !write_state.aborted_transaction.load(Ordering::SeqCst)
|
||||
}
|
||||
|
||||
async fn load_runtime_pool_meta_observing(&self, write_state: &mut PoolMetaWriteState, operation: &str) -> Result<PoolMeta> {
|
||||
write_state.ensure_write_safe(operation)?;
|
||||
load_pool_meta_identity_observing(self.pools.clone(), write_state).await?;
|
||||
@@ -10891,9 +10901,9 @@ impl ECStore {
|
||||
// global lock, then fence the exact target cohort before taking the
|
||||
// write lock used to publish the terminal transition.
|
||||
let terminal_fence_plan = if acquire_runtime_fence {
|
||||
let mut read_save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let read_save_guard = self.pool_meta_save_gate.lock().await;
|
||||
let (read_guard, snapshot) = self
|
||||
.acquire_pool_meta_read_guard(&mut read_save_guard, "decommission cancel fence planning failed")
|
||||
.acquire_pool_meta_read_guard(&read_save_guard, "decommission cancel fence planning failed")
|
||||
.await?;
|
||||
let plan = decommission_capacity_terminal_fence_plan(&snapshot, idx)?;
|
||||
drop(read_guard);
|
||||
@@ -16889,6 +16899,87 @@ mod tests {
|
||||
assert!(err.to_string().contains("requires 60 bytes, but 59 bytes are available"));
|
||||
}
|
||||
|
||||
async fn single_pool_capacity_admission_test_store() -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let (temp_dirs, store) =
|
||||
crate::services::rebalance::test_store_with_persisted_rebalance_meta(RebalanceMeta::default()).await;
|
||||
crate::bucket::metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
(temp_dirs, store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn single_pool_public_writes_skip_decommission_capacity_admission() {
|
||||
let (_temp_dirs, store) = single_pool_capacity_admission_test_store().await;
|
||||
let bucket = format!("single-pool-capacity-skip-{}", uuid::Uuid::new_v4());
|
||||
store
|
||||
.make_bucket(&bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create single-pool bucket before blocking pool metadata writes");
|
||||
let incarnation = store.bucket_incarnation_id(&bucket).await.expect("load bucket incarnation");
|
||||
store.pool_meta_save_gate.lock().await.block_writes_after_fence_loss();
|
||||
|
||||
let object = "ordinary-put.bin";
|
||||
let mut put_data = crate::object_api::PutObjReader::from_vec(b"ordinary single-pool body".to_vec());
|
||||
store
|
||||
.put_object(&bucket, object, &mut put_data, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-pool ordinary PUT must not enter decommission capacity admission");
|
||||
store
|
||||
.get_object_info(&bucket, object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-pool ordinary PUT must remain readable");
|
||||
|
||||
let multipart_object = "ordinary-multipart.bin";
|
||||
let upload = store
|
||||
.new_multipart_upload(
|
||||
&bucket,
|
||||
multipart_object,
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("single-pool MPU creation must not enter decommission capacity admission");
|
||||
let mut part_data = crate::object_api::PutObjReader::from_vec(b"single-pool multipart body".to_vec());
|
||||
let part = store
|
||||
.put_object_part(
|
||||
&bucket,
|
||||
multipart_object,
|
||||
&upload.upload_id,
|
||||
1,
|
||||
&mut part_data,
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("single-pool UploadPart must not enter decommission capacity admission");
|
||||
store
|
||||
.clone()
|
||||
.complete_multipart_upload(
|
||||
&bucket,
|
||||
multipart_object,
|
||||
&upload.upload_id,
|
||||
vec![crate::storage_api_contracts::multipart::CompletePart {
|
||||
part_num: part.part_num,
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
}],
|
||||
&ObjectOptions {
|
||||
expected_bucket_incarnation_id: Some(incarnation),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("single-pool CompleteMultipartUpload must not enter decommission capacity admission");
|
||||
store
|
||||
.get_object_info(&bucket, multipart_object, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("single-pool completed MPU must remain readable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn multipart_mutations_locate_later_upload_before_reserved_pool_admission() {
|
||||
@@ -17961,6 +18052,67 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_read_probe_does_not_latch_writer_state() {
|
||||
let write_state = PoolMetaWriteState::default();
|
||||
select_pool_meta_replicas_for_read_probe(
|
||||
&write_state,
|
||||
vec![PoolMetaReplica::Unreadable("transient read failure".to_string())],
|
||||
"capacity probe",
|
||||
)
|
||||
.expect_err("an unreadable probe replica must fail the current admission");
|
||||
|
||||
assert!(
|
||||
write_state.ensure_write_safe("ordinary object write").is_ok(),
|
||||
"a read-only capacity probe must not permanently latch the pool metadata writer"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_read_probe_rejects_missing_runtime_metadata_without_latching() {
|
||||
let write_state = PoolMetaWriteState {
|
||||
expected_cluster_id: Some(uuid::Uuid::new_v4()),
|
||||
identity_initialized: Some(true),
|
||||
..Default::default()
|
||||
};
|
||||
select_pool_meta_replicas_for_read_probe(&write_state, vec![PoolMetaReplica::Missing], "capacity probe")
|
||||
.expect_err("runtime metadata disappearance must reject the current probe");
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a missing-metadata probe must not permanently latch the writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
|
||||
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||
let mut saved_disks = Vec::new();
|
||||
for set in &store.pools[1].disk_set {
|
||||
let mut disks = set.disks.write().await;
|
||||
let original = std::mem::take(&mut *disks);
|
||||
let disk_count = original.len();
|
||||
saved_disks.push((set.clone(), original));
|
||||
*disks = vec![None; disk_count];
|
||||
}
|
||||
|
||||
let write_state = store.pool_meta_save_gate.lock().await;
|
||||
store
|
||||
.acquire_pool_meta_read_guard(&write_state, "capacity probe")
|
||||
.await
|
||||
.expect_err("an unreadable metadata replica must reject this probe");
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a failed read-only probe must remain retryable");
|
||||
|
||||
for (set, disks) in saved_disks {
|
||||
*set.disks.write().await = disks;
|
||||
}
|
||||
store
|
||||
.acquire_pool_meta_read_guard(&write_state, "capacity probe retry")
|
||||
.await
|
||||
.expect("a read-only probe must succeed after the replica recovers");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_write_state_blocks_when_selection_has_no_valid_replica() {
|
||||
let replicas = vec![
|
||||
@@ -17980,41 +18132,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_read_probe_does_not_latch_writer_state() {
|
||||
let write_state = PoolMetaWriteState::default();
|
||||
select_pool_meta_replicas_for_read_probe(
|
||||
&write_state,
|
||||
vec![PoolMetaReplica::Unreadable("transient read failure".to_string())],
|
||||
"capacity probe",
|
||||
)
|
||||
.expect_err("an unreadable probe replica must fail the current admission");
|
||||
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a read-only capacity probe must not permanently latch the pool metadata writer");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn pool_meta_read_guard_does_not_latch_after_unreadable_replica() {
|
||||
let (_temp_dirs, store, _other_store) = crate::services::rebalance::test_two_pool_stores(None).await;
|
||||
for set in &store.pools[1].disk_set {
|
||||
let mut disks = set.disks.write().await;
|
||||
let disk_count = disks.len();
|
||||
*disks = vec![None; disk_count];
|
||||
}
|
||||
|
||||
let mut write_state = store.pool_meta_save_gate.lock().await;
|
||||
store
|
||||
.acquire_pool_meta_read_guard(&mut write_state, "capacity probe")
|
||||
.await
|
||||
.expect_err("an unreadable metadata replica must reject this probe");
|
||||
write_state
|
||||
.ensure_write_safe("ordinary object write")
|
||||
.expect("a failed read-only probe must remain retryable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_meta_write_state_blocks_on_any_recovery_required_selection() {
|
||||
fn assert_selection_blocks(replicas: Vec<PoolMetaReplica>) {
|
||||
|
||||
@@ -203,6 +203,19 @@ pub enum StorageError {
|
||||
NotFirstDisk,
|
||||
#[error("first disk wait")]
|
||||
FirstDiskWait,
|
||||
#[error(
|
||||
"unsupported pool expansion: an existing single-node single-drive (SNSD) deployment cannot be expanded in place (configured {configured_drives} drive endpoints); restart with the original single local path, or create a new multi-drive deployment and migrate data through S3"
|
||||
)]
|
||||
UnsupportedSnsdExpansion { configured_drives: usize },
|
||||
#[error(
|
||||
"pool topology mismatch: stored {stored_drives} drives with {stored_set_drive_count} drives per erasure set, configured {configured_drives} drives with {configured_set_drive_count} drives per erasure set; an existing pool's drive count and erasure set width cannot be changed in place; restore its original endpoints and RUSTFS_ERASURE_SET_DRIVE_COUNT setting; to expand a multi-drive deployment, append a new pool with at least 2 drive endpoints"
|
||||
)]
|
||||
PoolTopologyMismatch {
|
||||
stored_drives: usize,
|
||||
stored_set_drive_count: usize,
|
||||
configured_drives: usize,
|
||||
configured_set_drive_count: usize,
|
||||
},
|
||||
|
||||
// ── Operational ──────────────────────────────────────────────────
|
||||
#[error("Storage reached its minimum free drive threshold.")]
|
||||
@@ -629,6 +642,20 @@ impl Clone for StorageError {
|
||||
StorageError::ErasureWriteQuorum => StorageError::ErasureWriteQuorum,
|
||||
StorageError::NotFirstDisk => StorageError::NotFirstDisk,
|
||||
StorageError::FirstDiskWait => StorageError::FirstDiskWait,
|
||||
StorageError::UnsupportedSnsdExpansion { configured_drives } => StorageError::UnsupportedSnsdExpansion {
|
||||
configured_drives: *configured_drives,
|
||||
},
|
||||
StorageError::PoolTopologyMismatch {
|
||||
stored_drives,
|
||||
stored_set_drive_count,
|
||||
configured_drives,
|
||||
configured_set_drive_count,
|
||||
} => StorageError::PoolTopologyMismatch {
|
||||
stored_drives: *stored_drives,
|
||||
stored_set_drive_count: *stored_set_drive_count,
|
||||
configured_drives: *configured_drives,
|
||||
configured_set_drive_count: *configured_set_drive_count,
|
||||
},
|
||||
StorageError::TooManyOpenFiles => StorageError::TooManyOpenFiles,
|
||||
StorageError::NoHealRequired => StorageError::NoHealRequired,
|
||||
StorageError::Lock(e) => StorageError::Lock(e.clone()),
|
||||
@@ -735,6 +762,11 @@ impl StorageError {
|
||||
StorageError::ErasureWriteQuorum => StorageErrorCode::ErasureWriteQuorum,
|
||||
StorageError::NotFirstDisk => StorageErrorCode::NotFirstDisk,
|
||||
StorageError::FirstDiskWait => StorageErrorCode::FirstDiskWait,
|
||||
// Topology diagnostics reuse the existing wire code; they are
|
||||
// not disk errors and must retain their local identity for retry classification.
|
||||
StorageError::UnsupportedSnsdExpansion { .. } | StorageError::PoolTopologyMismatch { .. } => {
|
||||
StorageErrorCode::InvalidArgument
|
||||
}
|
||||
StorageError::ConfigNotFound => StorageErrorCode::ConfigNotFound,
|
||||
StorageError::TooManyOpenFiles => StorageErrorCode::TooManyOpenFiles,
|
||||
StorageError::NoHealRequired => StorageErrorCode::NoHealRequired,
|
||||
@@ -1215,6 +1247,29 @@ mod tests {
|
||||
use super::*;
|
||||
use std::io::{Error as IoError, ErrorKind};
|
||||
|
||||
#[test]
|
||||
fn startup_topology_errors_preserve_identity_and_guidance() {
|
||||
for error in [
|
||||
StorageError::UnsupportedSnsdExpansion { configured_drives: 4 },
|
||||
StorageError::PoolTopologyMismatch {
|
||||
stored_drives: 4,
|
||||
stored_set_drive_count: 4,
|
||||
configured_drives: 8,
|
||||
configured_set_drive_count: 8,
|
||||
},
|
||||
] {
|
||||
let io_error: IoError = error.clone().into();
|
||||
let restored = StorageError::from(io_error);
|
||||
assert_eq!(std::mem::discriminant(&restored), std::mem::discriminant(&error));
|
||||
assert_eq!(restored.to_string(), error.to_string());
|
||||
assert_eq!(restored.code(), StorageErrorCode::InvalidArgument);
|
||||
assert!(
|
||||
restored.narrow_to_disk().is_err(),
|
||||
"startup diagnostics must not become disk/quorum errors"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_preserves_erasure_construction_source_chain() {
|
||||
use crate::erasure::coding::ErasureConstructionError;
|
||||
|
||||
@@ -25,6 +25,20 @@ pub(crate) const MAX_ERASURE_SET_DRIVE_COUNT: usize = 16;
|
||||
const SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, MAX_ERASURE_SET_DRIVE_COUNT];
|
||||
const ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT: &str = "RUSTFS_ERASURE_SET_DRIVE_COUNT";
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
enum PoolDriveCountError {
|
||||
#[error(
|
||||
"Incorrect number of endpoints provided, size {size}; an erasure pool requires at least {} drive endpoints on one or more nodes; for a standalone single-drive deployment, use a single local path without ellipses",
|
||||
SET_SIZES[0]
|
||||
)]
|
||||
BelowMinimum { size: usize },
|
||||
#[error(
|
||||
"Incorrect number of endpoints provided, size {size}; {}={set_drive_count} requires at least {set_drive_count} drive endpoints per pool",
|
||||
ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT
|
||||
)]
|
||||
BelowSetWidth { size: usize, set_drive_count: usize },
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Debug, Default)]
|
||||
pub struct PoolDisksLayout {
|
||||
cmd_line: String,
|
||||
@@ -132,7 +146,7 @@ impl DisksLayout {
|
||||
for arg in args.iter() {
|
||||
if !has_ellipses(&[arg]) && args.len() > 1 {
|
||||
return Err(Error::other(
|
||||
"all args must have ellipses for pool expansion (Invalid arguments specified)",
|
||||
"all args must have ellipses for pool expansion (Invalid arguments specified); each pool must expand to at least 2 drive endpoints on one or more nodes; a single-drive pool cannot be added to a multi-pool deployment",
|
||||
));
|
||||
}
|
||||
|
||||
@@ -396,9 +410,11 @@ fn get_set_indexes<T: AsRef<str>>(
|
||||
}
|
||||
|
||||
for &size in total_sizes {
|
||||
// Check if total_sizes has minimum range upto set_size
|
||||
if size < SET_SIZES[0] || size < set_drive_count {
|
||||
return Err(Error::other(format!("Incorrect number of endpoints provided, size {size}")));
|
||||
if size < SET_SIZES[0] {
|
||||
return Err(Error::other(PoolDriveCountError::BelowMinimum { size }));
|
||||
}
|
||||
if size < set_drive_count {
|
||||
return Err(Error::other(PoolDriveCountError::BelowSetWidth { size, set_drive_count }));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -707,7 +723,7 @@ mod test {
|
||||
arg: "http://rustfs{2...3}/export/set{1...0}",
|
||||
..Default::default()
|
||||
},
|
||||
// Range cannot be smaller than 4 minimum.
|
||||
// Ranges must use three dots.
|
||||
TestCase {
|
||||
num: 4,
|
||||
arg: "/export{1..2}",
|
||||
@@ -926,11 +942,146 @@ mod test {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_accepts_single_node_multi_drive_pools() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for (volumes, drives) in [
|
||||
(["http://node1:9000/data{1...2}", "http://node2:9000/data{1...2}"], 2),
|
||||
(["http://node1:9000/data{1...4}", "http://node2:9000/data{1...4}"], 4),
|
||||
(["http://node{1...4}:9000/data", "http://node5:9000/data{1...4}"], 4),
|
||||
(["http://node5:9000/data{1...4}", "http://node{1...4}:9000/data"], 4),
|
||||
] {
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("single-node multi-drive pools are valid");
|
||||
|
||||
assert!(!layout.legacy);
|
||||
assert_eq!(layout.pools.len(), 2);
|
||||
for (index, volume) in volumes.iter().enumerate() {
|
||||
assert_eq!(layout.get_set_count(index), 1);
|
||||
assert_eq!(layout.get_drives_per_set(index), drives);
|
||||
assert_eq!(layout.get_cmd_line(index), *volume);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_accepts_multi_node_single_drive_pools() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for nodes in [2, 3, 4] {
|
||||
let volumes = [
|
||||
format!("http://pool1-node{{1...{nodes}}}:9000/data"),
|
||||
format!("http://pool2-node{{1...{nodes}}}:9000/data"),
|
||||
];
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("each node may contribute one drive to a pool");
|
||||
|
||||
assert_eq!(layout.pools.len(), 2);
|
||||
for pool in 0..2 {
|
||||
assert_eq!(layout.get_set_count(pool), 1);
|
||||
assert_eq!(layout.get_drives_per_set(pool), nodes);
|
||||
let expected = (1..=nodes)
|
||||
.map(|node| format!("http://pool{}-node{node}:9000/data", pool + 1))
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(layout.pools[pool].layout, vec![expected]);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_endpoints_without_ellipses_form_one_pool() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
let volumes = ["http://node1:9000/data", "http://node2:9000/data"];
|
||||
let layout = DisksLayout::from_volumes(&volumes).expect("explicit endpoints form one legacy pool");
|
||||
|
||||
assert!(layout.legacy);
|
||||
assert_eq!(layout.pools.len(), 1);
|
||||
assert_eq!(layout.pools[0].layout, vec![volumes.to_vec()]);
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn standalone_single_drive_path_remains_supported() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
let layout = DisksLayout::from_volumes(&["/data"]).expect("standalone single-drive deployment is valid");
|
||||
|
||||
assert!(layout.is_single_drive_layout());
|
||||
assert_eq!(layout.get_single_drive_layout(), "/data");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_rejects_plain_single_drive_pool_with_notice() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for volumes in [
|
||||
["http://node{1...2}:9000/data", "http://node3:9000/data"],
|
||||
["http://node3:9000/data", "http://node{1...2}:9000/data"],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).expect_err("a plain endpoint cannot be an expansion pool");
|
||||
let message = err.to_string();
|
||||
|
||||
assert!(message.contains("all args must have ellipses for pool expansion"), "{message}");
|
||||
assert!(message.contains("at least 2 drive endpoints"), "{message}");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn pool_expansion_rejects_singleton_ellipsis_pool_with_notice() {
|
||||
temp_env::with_var(ENV_RUSTFS_ERASURE_SET_DRIVE_COUNT, Some("0"), || {
|
||||
for singleton in ["http://node{3...3}:9000/data", "http://node3:9000/data{1...1}"] {
|
||||
for volumes in [
|
||||
vec!["http://node{1...2}:9000/data", singleton],
|
||||
vec![singleton, "http://node{1...2}:9000/data"],
|
||||
vec![singleton],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).expect_err("a singleton range still contains one drive");
|
||||
let message = err.to_string();
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
assert!(matches!(
|
||||
err.get_ref().and_then(|source| source.downcast_ref::<PoolDriveCountError>()),
|
||||
Some(PoolDriveCountError::BelowMinimum { size: 1 })
|
||||
));
|
||||
assert!(message.contains("at least 2 drive endpoints"), "{message}");
|
||||
assert!(message.contains("single local path without ellipses"), "{message}");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn explicit_set_size_counts_drives_not_nodes() {
|
||||
for volume in ["http://node1:9000/data{1...4}", "http://node{1...4}:9000/data"] {
|
||||
let sets = get_all_sets(2, true, &[volume]).expect("four endpoints can form two two-drive sets");
|
||||
assert_eq!(sets.iter().map(Vec::len).collect::<Vec<_>>(), vec![2, 2]);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn undersized_pool_error_identifies_requested_set_size() {
|
||||
let err =
|
||||
get_all_sets(4, true, &["http://node{1...2}:9000/data"]).expect_err("two endpoints cannot fill a four-drive set");
|
||||
let message = err.to_string();
|
||||
|
||||
assert_eq!(err.kind(), std::io::ErrorKind::Other);
|
||||
assert!(matches!(
|
||||
err.get_ref().and_then(|source| source.downcast_ref::<PoolDriveCountError>()),
|
||||
Some(PoolDriveCountError::BelowSetWidth {
|
||||
size: 2,
|
||||
set_drive_count: 4
|
||||
})
|
||||
));
|
||||
assert!(message.contains("size 2"), "{message}");
|
||||
assert!(message.contains("RUSTFS_ERASURE_SET_DRIVE_COUNT=4"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn layout_errors_do_not_echo_url_credentials() {
|
||||
for volumes in [
|
||||
vec!["http://:duplicate-secret@server/path", "http://:duplicate-secret@server/path"],
|
||||
vec!["http://:ellipsis...secret@server/path"],
|
||||
vec!["http://server{1...2}/data", "http://:plain-secret@server3/data"],
|
||||
vec!["http://server{1...2}/data", "http://:singleton-secret@server{3...3}/data"],
|
||||
] {
|
||||
let err = DisksLayout::from_volumes(&volumes).unwrap_err();
|
||||
assert!(!err.to_string().contains("secret"), "layout error leaked endpoint credentials: {err}");
|
||||
|
||||
@@ -2432,6 +2432,41 @@ mod test {
|
||||
assert_eq!(local_endpoints[0].pool_idx, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pool_expansion_resolves_single_node_multi_drive_and_multi_node_single_drive_pools() {
|
||||
for (additional_pool, expected_nodes) in [
|
||||
("http://rustfs-5.example.invalid:9000/data{1...4}", 5),
|
||||
("http://rustfs-{5...8}.example.invalid:9000/data", 8),
|
||||
] {
|
||||
let layout = temp_env::with_var("RUSTFS_ERASURE_SET_DRIVE_COUNT", Some("0"), || {
|
||||
DisksLayout::from_volumes(&["http://rustfs-{1...4}.example.invalid:9000/data", additional_pool])
|
||||
})
|
||||
.expect("both single-node multi-drive and multi-node single-drive pools should parse");
|
||||
|
||||
let (pools, setup_type) = EndpointServerPools::create_server_endpoints_with(
|
||||
"0.0.0.0:9000",
|
||||
&layout,
|
||||
Some(orchestrated_test_policy()),
|
||||
Some("rustfs-1.example.invalid"),
|
||||
)
|
||||
.await
|
||||
.expect("pool admission must not impose a minimum node count or drives per node");
|
||||
|
||||
assert_eq!(setup_type, SetupType::DistErasure);
|
||||
assert_eq!(pools.0.len(), 2);
|
||||
assert_eq!(pools.get_nodes().len(), expected_nodes);
|
||||
for (pool_index, pool) in (0_i32..).zip(&pools.0) {
|
||||
assert_eq!((pool.set_count, pool.drives_per_set), (1, 4));
|
||||
assert_eq!(pool.endpoints.as_ref().len(), 4);
|
||||
for (disk_index, endpoint) in (0_i32..).zip(pool.endpoints.as_ref()) {
|
||||
assert_eq!(endpoint.pool_idx, pool_index);
|
||||
assert_eq!(endpoint.set_idx, 0);
|
||||
assert_eq!(endpoint.disk_idx, disk_index);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn explicit_local_endpoint_host_fails_closed_for_invalid_context_or_zero_match() {
|
||||
let args = vec![
|
||||
|
||||
@@ -2353,12 +2353,18 @@ mod tests {
|
||||
.await
|
||||
.expect("quorum boundary heal should return a mapped result");
|
||||
*store.pools[0].disk_set[0].disks.write().await = original_quorum_disks;
|
||||
let quorum_err_text = quorum_err.as_ref().map(ToString::to_string);
|
||||
assert!(
|
||||
quorum_err.as_ref().is_some_and(|err| err
|
||||
.to_string()
|
||||
.contains("pool metadata writes remain blocked after a recovery-required replica state")),
|
||||
quorum_err_text.as_deref().is_some_and(|err| {
|
||||
err.contains("target capacity admission failed")
|
||||
&& err.contains("pool metadata update cannot overwrite an unreadable replica")
|
||||
}),
|
||||
"heal must fail closed when capacity admission cannot verify pool metadata, got {quorum_err:?}"
|
||||
);
|
||||
assert!(
|
||||
store.pool_meta_writes_ready().await,
|
||||
"read-only capacity admission failure must not latch the pool metadata writer"
|
||||
);
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
|
||||
@@ -103,7 +103,10 @@ const REBALANCE_INITIAL_RESUME_DELAY: Duration = Duration::from_secs(10);
|
||||
const REBALANCE_RESUME_RETRY_DELAY: Duration = Duration::from_secs(10);
|
||||
|
||||
fn should_retry_format_load(err: &Error) -> bool {
|
||||
!matches!(err, Error::CorruptedFormat)
|
||||
!matches!(
|
||||
err,
|
||||
Error::CorruptedFormat | Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. }
|
||||
)
|
||||
}
|
||||
|
||||
fn should_auto_start_rebalance_after_init(decommission_running: bool, rebalance_resume_required: bool) -> bool {
|
||||
@@ -1784,6 +1787,33 @@ mod tests {
|
||||
assert!(should_retry_format_load(&StorageError::FirstDiskWait));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_retry_format_load_rejects_permanent_topology_errors() {
|
||||
for error in [
|
||||
StorageError::UnsupportedSnsdExpansion { configured_drives: 4 },
|
||||
StorageError::PoolTopologyMismatch {
|
||||
stored_drives: 4,
|
||||
stored_set_drive_count: 4,
|
||||
configured_drives: 8,
|
||||
configured_set_drive_count: 8,
|
||||
},
|
||||
] {
|
||||
assert!(!should_retry_format_load(&error), "topology errors require operator action: {error}");
|
||||
}
|
||||
for error in [
|
||||
StorageError::DiskNotFound,
|
||||
StorageError::Timeout,
|
||||
StorageError::RemoteNotInitialized,
|
||||
StorageError::NotFirstDisk,
|
||||
StorageError::other(std::io::Error::from(std::io::ErrorKind::ConnectionRefused)),
|
||||
] {
|
||||
assert!(
|
||||
should_retry_format_load(&error),
|
||||
"transient failures retain their existing retry path: {error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_should_auto_start_rebalance_after_init_allows_active_rebalance_without_decommission() {
|
||||
assert!(should_auto_start_rebalance_after_init(false, true));
|
||||
|
||||
@@ -109,6 +109,21 @@ pub(crate) async fn connect_load_init_formats_with_instance_ctx(
|
||||
let fresh_bootstrap_proven = should_init_erasure_disks(&errs);
|
||||
let formats_present = formats.iter().flatten().count();
|
||||
let mut format_quorum = (formats_present > 0).then(|| select_format_erasure_in_quorum(&formats, 0));
|
||||
// A resized pool may never reach quorum under its new endpoint count.
|
||||
// Diagnose a valid, unambiguous stored layout before migration or waiting.
|
||||
// A healthy quorum still takes precedence over foreign minority formats;
|
||||
// conflicting or malformed observations retain their existing error path.
|
||||
if format_quorum.as_ref().is_some_and(Result::is_err)
|
||||
&& let Some(reference) = formats.iter().flatten().next()
|
||||
&& formats.iter().flatten().all(|format| {
|
||||
format.shared_identity() == reference.shared_identity()
|
||||
&& reference.erasure.sets.iter().flatten().any(|id| *id == format.erasure.this)
|
||||
})
|
||||
&& let Err(err @ (Error::UnsupportedSnsdExpansion { .. } | Error::PoolTopologyMismatch { .. })) =
|
||||
check_format_erasure_value_for_topology(reference, formats.len(), set_drive_count)
|
||||
{
|
||||
return Err(err);
|
||||
}
|
||||
if format_quorum.as_ref().is_none_or(Result::is_err)
|
||||
&& errs.iter().any(|error| {
|
||||
matches!(
|
||||
@@ -661,15 +676,18 @@ fn check_format_erasure_value_for_topology(format: &FormatV3, format_count: usiz
|
||||
.len()
|
||||
.checked_mul(set_drive_count_in_format)
|
||||
.ok_or_else(|| Error::other("erasure set drive count overflow"))?;
|
||||
if format_count != format_drive_count {
|
||||
return Err(Error::other(format!(
|
||||
"formats length for erasure.sets does not match: got {format_count}, expected {format_drive_count}"
|
||||
)));
|
||||
if format_drive_count == 1 && format_count > 1 {
|
||||
return Err(Error::UnsupportedSnsdExpansion {
|
||||
configured_drives: format_count,
|
||||
});
|
||||
}
|
||||
if set_drive_count_in_format != set_drive_count {
|
||||
return Err(Error::other(format!(
|
||||
"erasure set length for set_drive_count does not match: got {set_drive_count_in_format}, expected {set_drive_count}"
|
||||
)));
|
||||
if format_count != format_drive_count || set_drive_count_in_format != set_drive_count {
|
||||
return Err(Error::PoolTopologyMismatch {
|
||||
stored_drives: format_drive_count,
|
||||
stored_set_drive_count: set_drive_count_in_format,
|
||||
configured_drives: format_count,
|
||||
configured_set_drive_count: set_drive_count,
|
||||
});
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -877,6 +895,10 @@ mod tests {
|
||||
use serial_test::serial;
|
||||
|
||||
async fn local_disks(count: usize) -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
|
||||
local_disks_with_set_width(count, count).await
|
||||
}
|
||||
|
||||
async fn local_disks_with_set_width(count: usize, set_width: usize) -> (tempfile::TempDir, Vec<Option<DiskStore>>) {
|
||||
let temp_dir = tempfile::tempdir().expect("temporary disk root should be created");
|
||||
let mut endpoints = Vec::with_capacity(count);
|
||||
for disk_index in 0..count {
|
||||
@@ -887,8 +909,8 @@ mod tests {
|
||||
let mut endpoint =
|
||||
Endpoint::try_from(path.to_str().expect("temporary disk path should be UTF-8")).expect("endpoint should parse");
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(disk_index);
|
||||
endpoint.set_set_index(disk_index / set_width);
|
||||
endpoint.set_disk_index(disk_index % set_width);
|
||||
endpoints.push(endpoint);
|
||||
}
|
||||
|
||||
@@ -912,6 +934,21 @@ mod tests {
|
||||
(temp_dir, disks)
|
||||
}
|
||||
|
||||
async fn format_bytes(disks: &[Option<DiskStore>]) -> Vec<Option<Vec<u8>>> {
|
||||
let mut snapshots = Vec::with_capacity(disks.len());
|
||||
for disk in disks {
|
||||
let disk = disk.as_ref().expect("snapshot disk should exist");
|
||||
// Inspect bytes even when the disk wrapper rejects a format whose
|
||||
// stored slot differs from the attempted new endpoint geometry.
|
||||
match tokio::fs::read(disk.path().join(RUSTFS_META_BUCKET).join(FORMAT_CONFIG_FILE)).await {
|
||||
Ok(data) => snapshots.push(Some(data)),
|
||||
Err(err) if err.kind() == std::io::ErrorKind::NotFound => snapshots.push(None),
|
||||
Err(err) => panic!("format snapshot failed: {err}"),
|
||||
}
|
||||
}
|
||||
snapshots
|
||||
}
|
||||
|
||||
async fn write_legacy_format(disk: &Option<DiskStore>, format: &FormatV3) {
|
||||
write_legacy_bytes(disk, bytes::Bytes::from(format.to_json().expect("legacy format should serialize"))).await;
|
||||
}
|
||||
@@ -1116,6 +1153,212 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn single_drive_format_rejects_in_place_expansion_without_writes() {
|
||||
for configured_drives in [2, 4] {
|
||||
for first_disk in [false, true] {
|
||||
let (_temp_dir, mut disks) = local_disks(configured_drives).await;
|
||||
let mut original = FormatV3::new(1, 1);
|
||||
original.erasure.this = original.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(original))
|
||||
.await
|
||||
.expect("SNSD format should be written");
|
||||
let before = format_bytes(&disks).await;
|
||||
|
||||
let err = connect_load_init_formats(first_disk, &mut disks, 1, configured_drives, None)
|
||||
.await
|
||||
.expect_err("an existing SNSD deployment cannot expand in place");
|
||||
let message = err.to_string();
|
||||
assert!(message.contains("SNSD"), "expected a single-drive expansion error: {message}");
|
||||
assert!(message.contains("migrate data through S3"), "expected actionable guidance: {message}");
|
||||
assert_eq!(format_bytes(&disks).await, before, "neither old nor new formats may be written");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_pool_rejects_drive_count_or_set_width_changes_without_writes() {
|
||||
for (stored_sets, stored_width, configured_sets, configured_width) in
|
||||
[(1, 4, 1, 6), (1, 4, 1, 8), (1, 4, 1, 2), (1, 4, 2, 2), (2, 2, 1, 4)]
|
||||
{
|
||||
for first_disk in [false, true] {
|
||||
let (_temp_dir, mut disks) =
|
||||
local_disks_with_set_width(configured_sets * configured_width, configured_width).await;
|
||||
let original = FormatV3::new(stored_sets, stored_width);
|
||||
for (disk, disk_id) in disks.iter().zip(original.erasure.sets.iter().flatten()) {
|
||||
let mut format = original.clone();
|
||||
format.erasure.this = *disk_id;
|
||||
save_format_file(disk, &Some(format))
|
||||
.await
|
||||
.expect("existing format should be written");
|
||||
}
|
||||
let before = format_bytes(&disks).await;
|
||||
|
||||
let err = connect_load_init_formats(first_disk, &mut disks, configured_sets, configured_width, None)
|
||||
.await
|
||||
.expect_err("an existing pool's geometry is immutable");
|
||||
let message = err.to_string();
|
||||
assert!(message.contains("pool topology mismatch"), "expected a topology error: {message}");
|
||||
assert!(
|
||||
message.contains(&format!("stored 4 drives with {stored_width} drives per erasure set")),
|
||||
"expected stored geometry: {message}"
|
||||
);
|
||||
assert!(message.contains("append a new pool"), "expected expansion guidance: {message}");
|
||||
assert_eq!(format_bytes(&disks).await, before, "rejection must not rewrite any format");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subquorum_existing_layout_with_missing_drives_is_not_expansion() {
|
||||
let (_temp_dir, mut disks) = local_disks(1).await;
|
||||
let mut original = FormatV3::new(1, 4);
|
||||
original.erasure.this = original.erasure.sets[0][0];
|
||||
save_format_file(&disks[0], &Some(original))
|
||||
.await
|
||||
.expect("existing format should be written");
|
||||
disks.extend([None, None, None]);
|
||||
|
||||
for first_disk in [false, true] {
|
||||
assert!(matches!(
|
||||
connect_load_init_formats(first_disk, &mut disks, 1, 4, None).await,
|
||||
Err(Error::ErasureReadQuorum)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conflicting_layouts_without_quorum_are_not_expansion_proof() {
|
||||
let (_temp_dir, mut disks) = local_disks(2).await;
|
||||
for (index, (disk, width)) in disks.iter().zip([4, 2]).enumerate() {
|
||||
let mut format = FormatV3::new(1, width);
|
||||
format.erasure.this = format.erasure.sets[0][index];
|
||||
save_format_file(disk, &Some(format))
|
||||
.await
|
||||
.expect("existing format should be written");
|
||||
}
|
||||
disks.extend([None, None]);
|
||||
|
||||
let result = connect_load_init_formats(true, &mut disks, 1, 4, None).await;
|
||||
assert!(matches!(result, Err(Error::ErasureReadQuorum)), "conflicting layout result: {result:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_quorum_ignores_single_drive_outlier() {
|
||||
let (_temp_dir, mut disks) = local_disks(3).await;
|
||||
let majority = FormatV3::new(1, 3);
|
||||
for (index, disk) in disks.iter().enumerate() {
|
||||
// Slot zero lets the SNSD outlier pass the disk wrapper's own
|
||||
// slot check, so quorum selection must exclude the parsed format.
|
||||
let mut format = if index == 0 { FormatV3::new(1, 1) } else { majority.clone() };
|
||||
format.erasure.this = format.erasure.sets[0][index];
|
||||
save_format_file(disk, &Some(format))
|
||||
.await
|
||||
.expect("existing format should be written");
|
||||
}
|
||||
|
||||
let loaded = connect_load_init_formats(true, &mut disks, 1, 3, None)
|
||||
.await
|
||||
.expect("a foreign SNSD outlier must not block a healthy majority");
|
||||
assert_eq!(loaded.shared_identity(), majority.shared_identity());
|
||||
assert!(disks[0].is_none(), "the foreign single-drive format must be quarantined");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn multi_drive_pool_expansion_preserves_existing_format() {
|
||||
let (_original_dir, mut disks) = local_disks(4).await;
|
||||
let (_new_dir, mut new_disks) = local_disks(4).await;
|
||||
let original = connect_load_init_formats(true, &mut disks, 1, 4, None)
|
||||
.await
|
||||
.expect("original multi-drive pool should initialize");
|
||||
let before = format_bytes(&disks).await;
|
||||
|
||||
let added = connect_load_init_formats(true, &mut new_disks, 1, 4, Some(original.id))
|
||||
.await
|
||||
.expect("a new multi-drive pool should initialize with the existing deployment ID");
|
||||
assert_eq!(added.id, original.id);
|
||||
assert_ne!(added.erasure.sets, original.erasure.sets);
|
||||
assert_eq!(format_bytes(&disks).await, before);
|
||||
assert_eq!(
|
||||
connect_load_init_formats(true, &mut disks, 1, 4, Some(original.id))
|
||||
.await
|
||||
.expect("the original pool should restart with unchanged geometry"),
|
||||
original
|
||||
);
|
||||
assert_eq!(
|
||||
connect_load_init_formats(true, &mut new_disks, 1, 4, Some(original.id))
|
||||
.await
|
||||
.expect("the new pool should restart with its own format"),
|
||||
added
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn store_startup_rejects_pool_resize_before_retry_loop() {
|
||||
use crate::layout::endpoints::{EndpointServerPools, PoolEndpoints};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
for (stored_width, configured_width) in [(1, 4), (4, 8)] {
|
||||
let (_temp_dir, disks) = local_disks(configured_width).await;
|
||||
let original = FormatV3::new(1, stored_width);
|
||||
for (disk, disk_id) in disks.iter().zip(&original.erasure.sets[0]) {
|
||||
let mut format = original.clone();
|
||||
format.erasure.this = *disk_id;
|
||||
save_format_file(disk, &Some(format))
|
||||
.await
|
||||
.expect("old format should be written");
|
||||
}
|
||||
let before = format_bytes(&disks).await;
|
||||
let endpoints = disks.iter().flatten().map(|disk| disk.endpoint()).collect::<Vec<_>>();
|
||||
let pools = EndpointServerPools::from(vec![PoolEndpoints {
|
||||
legacy: true,
|
||||
set_count: 1,
|
||||
drives_per_set: configured_width,
|
||||
endpoints: Endpoints::from(endpoints),
|
||||
cmd_line: "test-pool".to_string(),
|
||||
platform: String::new(),
|
||||
}]);
|
||||
let shutdown = CancellationToken::new();
|
||||
let result = temp_env::async_with_vars(
|
||||
[
|
||||
(storageclass::STANDARD_ENV, None::<&str>),
|
||||
(storageclass::RRS_ENV, None::<&str>),
|
||||
(storageclass::OPTIMIZE_ENV, None::<&str>),
|
||||
(storageclass::INLINE_BLOCK_ENV, None::<&str>),
|
||||
],
|
||||
tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::store::ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address"),
|
||||
pools,
|
||||
shutdown.clone(),
|
||||
Arc::new(InstanceContext::new()),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await;
|
||||
shutdown.cancel();
|
||||
let err = result
|
||||
.expect("invalid topology must abort without the format retry backoff")
|
||||
.expect_err("resize must fail");
|
||||
match stored_width {
|
||||
1 => assert!(matches!(err, Error::UnsupportedSnsdExpansion { configured_drives: 4 }), "{err}"),
|
||||
_ => assert!(
|
||||
matches!(
|
||||
err,
|
||||
Error::PoolTopologyMismatch {
|
||||
stored_drives: 4,
|
||||
configured_drives: 8,
|
||||
..
|
||||
}
|
||||
),
|
||||
"{err}"
|
||||
),
|
||||
}
|
||||
assert_eq!(format_bytes(&disks).await, before, "failed store startup must not write formats");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_format_load_rejects_conflicting_formats_without_a_majority() {
|
||||
let (_temp_dir, mut disks) = two_local_disks_with_missing_third().await;
|
||||
|
||||
@@ -526,12 +526,8 @@ impl ECStore {
|
||||
if self.single_pool() {
|
||||
self.apply_decommission_target_mutation_fence(0, object, &mut opts, mutation_fence)
|
||||
.await;
|
||||
return self
|
||||
.run_decommission_capacity_admitted_mutation(0, None, None, || async {
|
||||
self.pools[0].new_multipart_upload(bucket, object, &opts).await
|
||||
})
|
||||
.await
|
||||
.map(|res| (res, 0, opts.expected_bucket_incarnation_id));
|
||||
let result = self.pools[0].new_multipart_upload(bucket, object, &opts).await?;
|
||||
return Ok((result, 0, opts.expected_bucket_incarnation_id));
|
||||
}
|
||||
|
||||
if opts.data_movement && opts.version_id.is_some() {
|
||||
@@ -658,7 +654,9 @@ impl ECStore {
|
||||
) -> Result<PartInfo> {
|
||||
check_put_object_part_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
let opts = &opts;
|
||||
|
||||
if self.single_pool() {
|
||||
@@ -982,7 +980,9 @@ impl ECStore {
|
||||
) -> Result<ObjectInfo> {
|
||||
check_complete_multipart_args(bucket, object, upload_id)?;
|
||||
let (mut opts, _bucket_lifecycle_guard) = self.guard_multipart_bucket_incarnation(bucket, opts).await?;
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
let opts = &opts;
|
||||
|
||||
if self.single_pool() {
|
||||
|
||||
@@ -3123,6 +3123,9 @@ impl ECStore {
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
let (lock_object, target_object) = objects;
|
||||
if self.single_pool() {
|
||||
return operation(opts).await;
|
||||
}
|
||||
let (capacity_guard, has_active_decommission) = if capacity_releasing {
|
||||
self.acquire_decommission_capacity_release_fence_with_active_source().await?
|
||||
} else {
|
||||
@@ -3220,6 +3223,9 @@ impl ECStore {
|
||||
F: FnOnce(HealOpts) -> Fut,
|
||||
Fut: std::future::Future<Output = Result<T>>,
|
||||
{
|
||||
if self.single_pool() {
|
||||
return operation(opts).await;
|
||||
}
|
||||
let (capacity_guard, has_active_decommission) = self
|
||||
.acquire_external_decommission_capacity_fence_with_active_source(&[target_pool_idx], "heal")
|
||||
.await?;
|
||||
@@ -4162,7 +4168,9 @@ impl ECStore {
|
||||
.select_put_object_pool_idx(bucket, object.as_str(), data.size(), &opts)
|
||||
.await?;
|
||||
let mut opts = opts;
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
if !self.single_pool() {
|
||||
opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
self.pools[idx]
|
||||
.put_object_with_old_current_size(bucket, object.as_str(), data, &opts)
|
||||
.await
|
||||
@@ -4340,8 +4348,10 @@ impl ECStore {
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
if !self.single_pool() {
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
return if let Some(reader) = src_info.put_object_reader.as_mut() {
|
||||
self.pools[pool_idx]
|
||||
.put_object(dst_bucket, &dst_object, reader, &put_opts)
|
||||
@@ -4376,8 +4386,10 @@ impl ECStore {
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
if !self.single_pool() {
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
return self.pools[pool_idx]
|
||||
.put_object(dst_bucket, &dst_object, reader, &put_opts)
|
||||
.await;
|
||||
@@ -4422,7 +4434,10 @@ impl ECStore {
|
||||
object_lock_config_snapshot: dst_opts.object_lock_config_snapshot.clone(),
|
||||
..Default::default()
|
||||
};
|
||||
put_opts.decommission_capacity_admission = crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
if !self.single_pool() {
|
||||
put_opts.decommission_capacity_admission =
|
||||
crate::bucket::metadata_sys::object_store_if_initialized_in(&self.ctx).await;
|
||||
}
|
||||
|
||||
if let Some(put_object_reader) = src_info.put_object_reader.as_mut() {
|
||||
return self.pools[pool_idx]
|
||||
@@ -5057,7 +5072,7 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
let _capacity_fence = if latest_marker_objects.iter().any(|creates_marker| *creates_marker) {
|
||||
let _capacity_fence = if !self.single_pool() && latest_marker_objects.iter().any(|creates_marker| *creates_marker) {
|
||||
let target_pool_indices = (0..self.pools.len()).collect::<Vec<_>>();
|
||||
match self
|
||||
.acquire_external_decommission_capacity_fence(&target_pool_indices, "batch_delete")
|
||||
@@ -5375,7 +5390,6 @@ impl ECStore {
|
||||
// self-deadlocked on the inner commits.
|
||||
let object_name = object.as_str();
|
||||
if self.single_pool() {
|
||||
opts.decommission_capacity_admission = Some(Arc::clone(&self));
|
||||
return self.pools[0]
|
||||
.clone()
|
||||
.restore_transitioned_object(bucket, object_name, &opts)
|
||||
|
||||
@@ -91,6 +91,29 @@ pub struct HealObjectOutcome {
|
||||
pub detail: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct HealObjectReceipt {
|
||||
pub identity: HealObjectIdentity,
|
||||
pub disposition: HealObjectDisposition,
|
||||
}
|
||||
|
||||
impl HealObjectReceipt {
|
||||
pub(crate) fn verified_for(&self, expected: &HealObjectIdentity) -> bool {
|
||||
matches!(
|
||||
self.disposition,
|
||||
HealObjectDisposition::Repaired
|
||||
| HealObjectDisposition::VerifiedHealthy
|
||||
| HealObjectDisposition::AuthoritativelyAbsent
|
||||
) && self.identity.kind == expected.kind
|
||||
&& self.identity.bucket == expected.bucket
|
||||
&& self.identity.object == expected.object
|
||||
&& self.identity.version_id == expected.version_id
|
||||
&& self.identity.pool_index == expected.pool_index
|
||||
&& self.identity.set_index == expected.set_index
|
||||
&& self.identity.bucket_incarnation_id.is_some()
|
||||
}
|
||||
}
|
||||
|
||||
impl HealObjectOutcome {
|
||||
fn retained_bytes(&self) -> usize {
|
||||
size_of::<Self>()
|
||||
@@ -470,4 +493,48 @@ mod canonical_outcome_tests {
|
||||
assert_eq!(outcome.counters.processed, u64::MAX);
|
||||
assert_eq!(outcome.coverage, HealTraversalCoverage::Partial);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn positive_receipt_requires_exact_identity_and_bucket_incarnation() {
|
||||
let expected = item(HealObjectDisposition::Unknown).identity;
|
||||
let mut receipt = HealObjectReceipt {
|
||||
identity: expected.clone(),
|
||||
disposition: HealObjectDisposition::Repaired,
|
||||
};
|
||||
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a positive storage receipt without bucket incarnation must remain untrusted"
|
||||
);
|
||||
|
||||
let incarnation = Uuid::new_v4();
|
||||
receipt.identity.bucket_incarnation_id = Some(incarnation);
|
||||
assert!(receipt.verified_for(&expected));
|
||||
|
||||
receipt.identity.version_id = Some("older-version".to_string());
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a storage receipt for a different object/version tuple must not clear the requested responsibility"
|
||||
);
|
||||
|
||||
receipt.identity = HealObjectIdentity {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
pool_index: Some(1),
|
||||
..expected.clone()
|
||||
};
|
||||
assert!(
|
||||
!receipt.verified_for(&expected),
|
||||
"a storage receipt for a different erasure location must not clear the requested responsibility"
|
||||
);
|
||||
|
||||
receipt.identity = HealObjectIdentity {
|
||||
bucket_incarnation_id: Some(incarnation),
|
||||
..expected
|
||||
};
|
||||
receipt.disposition = HealObjectDisposition::Unknown;
|
||||
assert!(
|
||||
!receipt.verified_for(&receipt.identity),
|
||||
"legacy success without a positive disposition remains unknown"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -15,12 +15,13 @@
|
||||
use crate::{Error, Result};
|
||||
use async_trait::async_trait;
|
||||
use base64_simd::URL_SAFE_NO_PAD;
|
||||
use rustfs_heal_contracts::heal_channel::{HealOpts, HealScanMode};
|
||||
use rustfs_heal_contracts::heal_channel::{DriveState, HealOpts, HealScanMode};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, error, warn};
|
||||
|
||||
use super::outcome::{HealObjectDisposition, HealObjectIdentity, HealObjectKind, HealObjectReceipt};
|
||||
use super::progress::stable_generation;
|
||||
use super::storage_api::owner::{EcstoreHealLifecycleExpiryContext, ecstore_load_admin_data_usage_from_backend_cached};
|
||||
use super::storage_api::storage::{
|
||||
@@ -67,6 +68,23 @@ impl HealLifecycleExpiryContext {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
pub struct HealStorageObjectResult {
|
||||
pub item: HealResultItem,
|
||||
pub error: Option<Error>,
|
||||
pub receipt: Option<HealObjectReceipt>,
|
||||
}
|
||||
|
||||
impl From<(HealResultItem, Option<Error>)> for HealStorageObjectResult {
|
||||
fn from((item, error): (HealResultItem, Option<Error>)) -> Self {
|
||||
Self {
|
||||
item,
|
||||
error,
|
||||
receipt: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const LOG_COMPONENT_HEAL: &str = "heal";
|
||||
const LOG_SUBSYSTEM_STORAGE: &str = "storage";
|
||||
const EVENT_HEAL_STORAGE_OBJECT_IO: &str = "heal_storage_object_io";
|
||||
@@ -374,6 +392,16 @@ pub trait HealStorageAPI: Send + Sync {
|
||||
opts: &HealOpts,
|
||||
) -> Result<(HealResultItem, Option<Error>)>;
|
||||
|
||||
async fn heal_object_with_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
self.heal_object(bucket, object, version_id, opts).await.map(Into::into)
|
||||
}
|
||||
|
||||
/// Heal bucket using ecstore
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem>;
|
||||
|
||||
@@ -1062,6 +1090,50 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
}
|
||||
|
||||
async fn heal_object_with_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
let (item, error) = self.heal_object(bucket, object, version_id, opts).await?;
|
||||
let receipt = if error.is_none() && !opts.dry_run {
|
||||
let ok_drive_state = DriveState::Ok.to_string();
|
||||
let all_after_drives_ok = item.after.drives.iter().all(|drive| drive.state == ok_drive_state);
|
||||
match (
|
||||
self.ecstore.bucket_incarnation_id(bucket).await,
|
||||
item.drives_reported(),
|
||||
item.drives_healed(),
|
||||
all_after_drives_ok,
|
||||
) {
|
||||
(Ok(bucket_incarnation_id), Some(_), Some(drives_healed), true) => {
|
||||
let disposition = if drives_healed > 0 {
|
||||
HealObjectDisposition::Repaired
|
||||
} else {
|
||||
HealObjectDisposition::VerifiedHealthy
|
||||
};
|
||||
Some(HealObjectReceipt {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: bucket.to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: version_id.map(ToOwned::to_owned),
|
||||
bucket_incarnation_id: Some(bucket_incarnation_id),
|
||||
pool_index: opts.pool,
|
||||
set_index: opts.set,
|
||||
},
|
||||
disposition,
|
||||
})
|
||||
}
|
||||
_ => None,
|
||||
}
|
||||
} else {
|
||||
None
|
||||
};
|
||||
Ok(HealStorageObjectResult { item, error, receipt })
|
||||
}
|
||||
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
debug!(
|
||||
target: "rustfs::heal::storage",
|
||||
|
||||
@@ -17,7 +17,7 @@ use crate::heal::{
|
||||
erasure_healer::target_outcomes_complete,
|
||||
outcome::{
|
||||
HealAbortReason, HealDeferredReason, HealFailureClass, HealObjectDisposition, HealObjectIdentity, HealObjectKind,
|
||||
HealObjectOutcome, HealTaskOutcome,
|
||||
HealObjectOutcome, HealObjectReceipt, HealTaskOutcome,
|
||||
},
|
||||
progress::HealProgress,
|
||||
resume::{
|
||||
@@ -592,6 +592,26 @@ impl HealTask {
|
||||
Some(self.outcome_identity(bucket, object, version, self.options.pool_index, self.options.set_index))
|
||||
}
|
||||
|
||||
pub(super) async fn record_verified_storage_receipt(
|
||||
&self,
|
||||
expected: HealObjectIdentity,
|
||||
receipt: Option<HealObjectReceipt>,
|
||||
) -> bool {
|
||||
let Some(receipt) = receipt else {
|
||||
return false;
|
||||
};
|
||||
if !receipt.verified_for(&expected) {
|
||||
return false;
|
||||
}
|
||||
let mut outcome = self.outcome.write().await;
|
||||
outcome.record(HealObjectOutcome {
|
||||
identity: receipt.identity,
|
||||
disposition: receipt.disposition,
|
||||
detail: None,
|
||||
});
|
||||
true
|
||||
}
|
||||
|
||||
async fn record_deferred_object(&self, reason: HealDeferredReason) {
|
||||
if let Some(identity) = self.single_object_identity() {
|
||||
let mut outcome = self.outcome.write().await;
|
||||
|
||||
@@ -163,7 +163,7 @@ impl HealTask {
|
||||
set: self.options.set_index,
|
||||
};
|
||||
|
||||
let heal_fut = self.storage.heal_object(bucket, object, version_id, &heal_opts);
|
||||
let heal_fut = self.storage.heal_object_with_receipt(bucket, object, version_id, &heal_opts);
|
||||
let heal_result = if self.source == HealRequestSource::ReadRepair {
|
||||
let result = heal_fut.await;
|
||||
if self.cancel_token.is_cancelled() {
|
||||
@@ -176,7 +176,9 @@ impl HealTask {
|
||||
};
|
||||
|
||||
match heal_result {
|
||||
Ok((result, error)) => {
|
||||
Ok(storage_result) => {
|
||||
let result = storage_result.item;
|
||||
let error = storage_result.error;
|
||||
if let Some(e) = error {
|
||||
if self.skip_dangling_delete_grace_error(bucket, object, &e).await {
|
||||
return Ok(());
|
||||
@@ -264,6 +266,10 @@ impl HealTask {
|
||||
let mut progress = self.progress.write().await;
|
||||
progress.update_object_progress(1, 1, 0, 0, object_size);
|
||||
}
|
||||
let expected_identity =
|
||||
self.outcome_identity(bucket, object, version_id, self.options.pool_index, self.options.set_index);
|
||||
self.record_verified_storage_receipt(expected_identity, storage_result.receipt)
|
||||
.await;
|
||||
self.record_result_item(result).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@
|
||||
|
||||
use super::super::{DiskOption, DiskStore, Endpoint, new_disk};
|
||||
use super::*;
|
||||
use crate::heal::storage::HealStorageObjectResult;
|
||||
|
||||
mod deferred_retry;
|
||||
|
||||
@@ -1048,6 +1049,7 @@ struct MockStorage {
|
||||
object_exists_by_name: Mutex<HashMap<String, MockObjectExists>>,
|
||||
heal_object_outcome: Mutex<Option<MockHealObjectOutcome>>,
|
||||
heal_object_outcomes: Mutex<HashMap<String, VecDeque<MockHealObjectOutcome>>>,
|
||||
heal_object_receipts: Mutex<HashMap<String, VecDeque<HealObjectReceipt>>>,
|
||||
format_no_heal_required: Mutex<bool>,
|
||||
format_error: Mutex<Option<Error>>,
|
||||
global_format_calls: Mutex<u32>,
|
||||
@@ -1151,6 +1153,90 @@ async fn execute_emits_heal_trace_task_state() {
|
||||
assert_eq!(trace_attr_string(&completed, "state").as_deref(), Some("completed"));
|
||||
}
|
||||
|
||||
fn object_receipt(object: &str, version_id: Option<&str>, disposition: HealObjectDisposition) -> HealObjectReceipt {
|
||||
HealObjectReceipt {
|
||||
identity: HealObjectIdentity {
|
||||
kind: HealObjectKind::Object,
|
||||
bucket: "bucket-a".to_string(),
|
||||
object: object.to_string(),
|
||||
version_id: version_id.map(ToOwned::to_owned),
|
||||
bucket_incarnation_id: Some(Uuid::new_v4()),
|
||||
pool_index: None,
|
||||
set_index: None,
|
||||
},
|
||||
disposition,
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_records_matching_positive_storage_receipt() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt("object-a", Some("version-a"), HealObjectDisposition::Repaired)]),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute().await.expect("mock object heal should complete");
|
||||
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 1);
|
||||
assert_eq!(outcome.counters.unknown, 0);
|
||||
let object = outcome.objects.front().expect("positive receipt should be recorded");
|
||||
assert_eq!(object.identity.object, "object-a");
|
||||
assert_eq!(object.identity.version_id.as_deref(), Some("version-a"));
|
||||
assert!(object.identity.bucket_incarnation_id.is_some());
|
||||
assert_eq!(object.disposition, HealObjectDisposition::Repaired);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn object_heal_rejects_mismatched_or_legacy_storage_receipts() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
heal_object_receipts: Mutex::new(HashMap::from([(
|
||||
"object-a".to_string(),
|
||||
VecDeque::from([object_receipt(
|
||||
"object-a",
|
||||
Some("old-version"),
|
||||
HealObjectDisposition::Repaired,
|
||||
)]),
|
||||
)])),
|
||||
..Default::default()
|
||||
});
|
||||
let task = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-a".to_string(), Some("version-a".to_string())),
|
||||
storage,
|
||||
);
|
||||
|
||||
task.execute()
|
||||
.await
|
||||
.expect("a mismatched receipt must not fail the legacy heal result");
|
||||
let outcome = task.get_outcome().await;
|
||||
assert_eq!(outcome.counters.healed, 0);
|
||||
assert_eq!(outcome.counters.unknown, 1);
|
||||
assert_eq!(
|
||||
outcome
|
||||
.objects
|
||||
.front()
|
||||
.expect("legacy fallback should be recorded")
|
||||
.disposition,
|
||||
HealObjectDisposition::Unknown
|
||||
);
|
||||
|
||||
let legacy = HealTask::from_request(
|
||||
HealRequest::object("bucket-a".to_string(), "object-b".to_string(), None),
|
||||
Arc::new(MockStorage::default()),
|
||||
);
|
||||
legacy.execute().await.expect("legacy mock object heal should complete");
|
||||
let legacy_outcome = legacy.get_outcome().await;
|
||||
assert_eq!(legacy_outcome.counters.healed, 0);
|
||||
assert_eq!(legacy_outcome.counters.unknown, 1);
|
||||
}
|
||||
|
||||
async fn recv_trace_task_state(trace: &mut TraceSubscription, task_id: &str, state: &str) -> TraceEvent {
|
||||
for _ in 0..32 {
|
||||
let event = tokio::time::timeout(Duration::from_secs(1), trace.recv())
|
||||
@@ -1408,6 +1494,23 @@ impl HealStorageAPI for MockStorage {
|
||||
))
|
||||
}
|
||||
|
||||
async fn heal_object_with_receipt(
|
||||
&self,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version_id: Option<&str>,
|
||||
opts: &HealOpts,
|
||||
) -> Result<HealStorageObjectResult> {
|
||||
let (item, error) = self.heal_object(bucket, object, version_id, opts).await?;
|
||||
let receipt = self
|
||||
.heal_object_receipts
|
||||
.lock()
|
||||
.unwrap()
|
||||
.get_mut(object)
|
||||
.and_then(VecDeque::pop_front);
|
||||
Ok(HealStorageObjectResult { item, error, receipt })
|
||||
}
|
||||
|
||||
async fn heal_bucket(&self, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
self.bucket_heal_calls.lock().unwrap().push(bucket.to_string());
|
||||
self.bucket_heal_opts.lock().unwrap().push(*opts);
|
||||
|
||||
@@ -29,7 +29,17 @@ use rustfs_heal::heal::{
|
||||
storage::{ECStoreHealStorage, HealStorageAPI},
|
||||
};
|
||||
use serial_test::serial;
|
||||
use std::{path::Path, process::Command, sync::Arc, time::Duration};
|
||||
#[cfg(unix)]
|
||||
use std::{
|
||||
fs::{File, OpenOptions},
|
||||
io::Write,
|
||||
};
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
process::{Command, Stdio},
|
||||
sync::Arc,
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
mod storage_api;
|
||||
|
||||
@@ -110,6 +120,48 @@ fn journal_record(kind: u8, bucket: &str, object: &str, version: Option<[u8; 16]
|
||||
body
|
||||
}
|
||||
|
||||
fn scoped_journal_record(
|
||||
kind: u8,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
version: Option<[u8; 16]>,
|
||||
attempts: u8,
|
||||
pool_index: u32,
|
||||
set_index: u32,
|
||||
) -> Vec<u8> {
|
||||
let mut body = vec![1u8, 2, kind, attempts];
|
||||
body.extend_from_slice(&1_700_000_000_000u64.to_le_bytes());
|
||||
match version {
|
||||
Some(bytes) => {
|
||||
body.push(1);
|
||||
body.extend_from_slice(&bytes);
|
||||
}
|
||||
None => body.push(0),
|
||||
}
|
||||
body.extend_from_slice(&pool_index.to_le_bytes());
|
||||
body.extend_from_slice(&set_index.to_le_bytes());
|
||||
body.extend_from_slice(
|
||||
&u32::try_from(bucket.len())
|
||||
.expect("fixture bucket length must fit journal format")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
body.extend_from_slice(
|
||||
&u32::try_from(object.len())
|
||||
.expect("fixture object length must fit journal format")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
body.extend_from_slice(bucket.as_bytes());
|
||||
body.extend_from_slice(object.as_bytes());
|
||||
let mut hasher = crc_fast::Digest::new(crc_fast::CrcAlgorithm::Crc32IsoHdlc);
|
||||
hasher.update(&body);
|
||||
body.extend_from_slice(
|
||||
&u32::try_from(hasher.finalize())
|
||||
.expect("CRC32 must fit the journal checksum field")
|
||||
.to_le_bytes(),
|
||||
);
|
||||
body
|
||||
}
|
||||
|
||||
fn write_journal_path_to_disks(disk_paths: &[std::path::PathBuf], relative_path: &str, data: &[u8]) {
|
||||
for path in disk_paths {
|
||||
let journal = path.join(META_BUCKET).join(relative_path);
|
||||
@@ -118,6 +170,27 @@ fn write_journal_path_to_disks(disk_paths: &[std::path::PathBuf], relative_path:
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn write_journal_path_to_disks_synced(disk_paths: &[std::path::PathBuf], relative_path: &str, data: &[u8]) {
|
||||
for path in disk_paths {
|
||||
let journal = path.join(META_BUCKET).join(relative_path);
|
||||
let parent = journal.parent().expect("journal parent");
|
||||
std::fs::create_dir_all(parent).expect("create journal dir");
|
||||
let mut file = OpenOptions::new()
|
||||
.create(true)
|
||||
.truncate(true)
|
||||
.write(true)
|
||||
.open(&journal)
|
||||
.expect("open synced journal fixture");
|
||||
file.write_all(data).expect("write synced journal fixture");
|
||||
file.sync_all().expect("sync journal fixture");
|
||||
File::open(parent)
|
||||
.expect("open journal parent for sync")
|
||||
.sync_all()
|
||||
.expect("sync journal parent");
|
||||
}
|
||||
}
|
||||
|
||||
fn write_journal_to_disks(disk_paths: &[std::path::PathBuf], data: &[u8]) {
|
||||
write_journal_path_to_disks(disk_paths, JOURNAL_REL, data);
|
||||
}
|
||||
@@ -128,6 +201,12 @@ fn journal_exists_on_all_disks(disk_paths: &[std::path::PathBuf], relative_path:
|
||||
.all(|path| Path::new(path).join(META_BUCKET).join(relative_path).exists())
|
||||
}
|
||||
|
||||
fn journal_matches_on_all_disks(disk_paths: &[PathBuf], relative_path: &str, expected: &[u8]) -> bool {
|
||||
disk_paths
|
||||
.iter()
|
||||
.all(|path| std::fs::read(path.join(META_BUCKET).join(relative_path)).is_ok_and(|actual| actual == expected))
|
||||
}
|
||||
|
||||
async fn wait_until<F, Fut>(deadline: Duration, mut probe: F) -> bool
|
||||
where
|
||||
F: FnMut() -> Fut,
|
||||
@@ -259,6 +338,72 @@ async fn authoritative_journal_is_not_merged_with_legacy_mirror() {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
|
||||
let scoped_v2 = scoped_journal_record(1, "scoped-v2-bucket", "scoped-v2-object", None, 0, 3, 7);
|
||||
let stale_legacy = journal_record(1, "stale-legacy-bucket", "stale-legacy-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &scoped_v2);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||
assert_eq!(
|
||||
mrf_queue::replay_journal_once(&manager).await,
|
||||
1,
|
||||
"a scoped v2 authoritative epoch must not be merged with a stale v1 legacy mirror"
|
||||
);
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queued_by_source.mrf,
|
||||
3,
|
||||
"only the three authoritative/scoped-only epochs should have reached the manager"
|
||||
);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
}
|
||||
|
||||
/// The authoritative journal carries the full replay responsibility identity.
|
||||
/// A stale legacy mirror must not collapse same-object records that differ by
|
||||
/// kind or erasure-set scope after restart.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn authoritative_journal_replay_preserves_kind_and_scope_identity() {
|
||||
let (disk_paths, storage) = heal_env().await;
|
||||
register_local_disks(&disk_paths, "mrf-authoritative-identity-test").await;
|
||||
|
||||
let mut authoritative = scoped_journal_record(3, "identity-bucket", "same-object", None, 0, 3, 7);
|
||||
authoritative.extend(scoped_journal_record(3, "identity-bucket", "same-object", None, 0, 3, 8));
|
||||
authoritative.extend(journal_record(2, "identity-bucket", "same-object", None, 0));
|
||||
authoritative.extend(journal_record(1, "identity-bucket", "same-object", Some([4u8; 16]), 0));
|
||||
let stale_legacy = journal_record(3, "identity-bucket", "stale-legacy-object", None, 0);
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &authoritative);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &stale_legacy);
|
||||
|
||||
let manager = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&manager).await;
|
||||
assert_eq!(
|
||||
replayed, 4,
|
||||
"all authoritative kind/scope identities must decode before manager admission"
|
||||
);
|
||||
|
||||
let snapshot = manager.operations_snapshot().await;
|
||||
assert_eq!(
|
||||
snapshot.queued_by_source.mrf, 4,
|
||||
"same-object MRF replay must retain distinct kind and scope responsibilities"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.queued_by_priority.normal, 2,
|
||||
"the two scoped partial-write records must remain independently queued"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.queued_by_priority.high, 1,
|
||||
"metadata corruption must not merge with object repair responsibility"
|
||||
);
|
||||
assert_eq!(
|
||||
snapshot.queued_by_priority.urgent, 1,
|
||||
"decode-failure repair must not merge with object repair responsibility"
|
||||
);
|
||||
assert!(disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}));
|
||||
}
|
||||
|
||||
/// If replay reaches a full heal-manager queue, the old journal remains the
|
||||
@@ -348,6 +493,135 @@ fn mrf_journal_child_process_fixture() {
|
||||
std::process::exit(77);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mrf_successor_flush_child_process_fixture() {
|
||||
let Ok(root) = std::env::var("RUSTFS_MRF_SUCCESSOR_FLUSH_CHILD_ROOT") else {
|
||||
return;
|
||||
};
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("child runtime should build");
|
||||
runtime.block_on(async {
|
||||
let (disk_paths, storage) = heal_env_at(Some(Path::new(&root))).await;
|
||||
register_local_disks(&disk_paths, "mrf-successor-flush-child").await;
|
||||
|
||||
let mut startup = journal_record(1, "successor-bucket", "first-object", None, 0);
|
||||
startup.extend(journal_record(1, "successor-bucket", "second-object", None, 0));
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &startup);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &startup);
|
||||
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
flushed,
|
||||
"child process must publish the pending successor snapshot before the delete phase"
|
||||
);
|
||||
});
|
||||
std::process::exit(78);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(unix)]
|
||||
fn mrf_successor_flush_waiting_child_process_fixture() {
|
||||
let Ok(root) = std::env::var("RUSTFS_MRF_SUCCESSOR_KILL_CHILD_ROOT") else {
|
||||
return;
|
||||
};
|
||||
let ready_path = std::env::var("RUSTFS_MRF_SUCCESSOR_KILL_READY")
|
||||
.map(PathBuf::from)
|
||||
.expect("ready marker path should be provided");
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("child runtime should build");
|
||||
runtime.block_on(async {
|
||||
let (disk_paths, storage) = heal_env_at(Some(Path::new(&root))).await;
|
||||
register_local_disks(&disk_paths, "mrf-successor-kill-child").await;
|
||||
|
||||
let mut startup = journal_record(1, "service-kill-bucket", "first-object", None, 0);
|
||||
startup.extend(journal_record(1, "service-kill-bucket", "second-object", None, 0));
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &startup);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &startup);
|
||||
|
||||
let manager = Arc::new(HealManager::new(
|
||||
storage,
|
||||
Some(HealConfig {
|
||||
queue_size: 1,
|
||||
heal_interval: Duration::from_secs(3600),
|
||||
enable_auto_heal: false,
|
||||
..Default::default()
|
||||
}),
|
||||
));
|
||||
mrf_queue::spawn_mrf_consumer(manager.clone());
|
||||
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
let flushed = wait_until(Duration::from_secs(10), || async {
|
||||
manager.operations_snapshot().await.queued_by_source.mrf == 1
|
||||
&& journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor)
|
||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &expected_successor)
|
||||
})
|
||||
.await;
|
||||
assert!(
|
||||
flushed,
|
||||
"child process must publish the pending successor snapshot before it can be killed"
|
||||
);
|
||||
std::fs::write(&ready_path, b"ready").expect("write ready marker");
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[cfg(unix)]
|
||||
fn mrf_authoritative_fsync_waiting_child_process_fixture() {
|
||||
let Ok(root) = std::env::var("RUSTFS_MRF_FSYNC_KILL_CHILD_ROOT") else {
|
||||
return;
|
||||
};
|
||||
let ready_path = std::env::var("RUSTFS_MRF_FSYNC_KILL_READY")
|
||||
.map(PathBuf::from)
|
||||
.expect("ready marker path should be provided");
|
||||
let runtime = tokio::runtime::Builder::new_current_thread()
|
||||
.enable_all()
|
||||
.build()
|
||||
.expect("child runtime should build");
|
||||
runtime.block_on(async {
|
||||
let (disk_paths, _storage) = heal_env_at(Some(Path::new(&root))).await;
|
||||
register_local_disks(&disk_paths, "mrf-fsync-kill-child").await;
|
||||
|
||||
let mut startup = journal_record(1, "fsync-kill-bucket", "first-object", None, 0);
|
||||
startup.extend(journal_record(1, "fsync-kill-bucket", "second-object", None, 0));
|
||||
write_journal_path_to_disks(&disk_paths, SCOPED_JOURNAL_REL, &startup);
|
||||
write_journal_path_to_disks(&disk_paths, JOURNAL_REL, &startup);
|
||||
|
||||
let successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
write_journal_path_to_disks_synced(&disk_paths, SCOPED_JOURNAL_REL, &successor);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &successor)
|
||||
&& journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &startup),
|
||||
"child process must reach the canonical-fsync/stale-legacy boundary"
|
||||
);
|
||||
std::fs::write(&ready_path, b"ready").expect("write ready marker");
|
||||
loop {
|
||||
tokio::time::sleep(Duration::from_secs(60)).await;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/// A journal published by a different OS process must remain a durable anchor
|
||||
/// when the restarted process can only admit a prefix of the replayed intents.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
@@ -390,3 +664,159 @@ async fn journal_replay_retains_child_process_anchor_when_manager_is_full() {
|
||||
"replay must retain the child-published journal until a successor snapshot can replace it"
|
||||
);
|
||||
}
|
||||
|
||||
/// If a process crashes after flushing a smaller successor snapshot but before
|
||||
/// deleting the startup anchor, the restarted process must replay the
|
||||
/// successor tail rather than losing it or merging it with stale records.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn journal_replay_survives_successor_flush_before_delete() {
|
||||
let temp_dir = tempfile::tempdir().expect("successor-flush MRF root");
|
||||
let status = Command::new(std::env::current_exe().expect("test binary path"))
|
||||
.arg("mrf_successor_flush_child_process_fixture")
|
||||
.arg("--exact")
|
||||
.arg("--nocapture")
|
||||
.env("RUSTFS_MRF_SUCCESSOR_FLUSH_CHILD_ROOT", temp_dir.path())
|
||||
.status()
|
||||
.expect("child MRF successor fixture should start");
|
||||
assert_eq!(status.code(), Some(78), "child process did not reach the successor flush boundary");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let expected_successor = journal_record(1, "successor-bucket", "second-object", None, 2);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor),
|
||||
"restarted process must see the pending successor snapshot"
|
||||
);
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 1, "restart after successor flush must replay only the still-pending tail");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the successor tail must be accepted after restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"a fully consumed successor snapshot may be deleted after restart replay"
|
||||
);
|
||||
}
|
||||
|
||||
/// A service-style hard kill after successor flush must be equivalent to a
|
||||
/// crash at the flush-before-delete boundary: restart may replay the smaller
|
||||
/// successor snapshot, but must not lose or merge stale startup records.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
async fn journal_replay_survives_service_kill_after_successor_flush() {
|
||||
let temp_dir = tempfile::tempdir().expect("successor-kill MRF root");
|
||||
let ready = temp_dir.path().join("successor-flushed.ready");
|
||||
let mut child = Command::new(std::env::current_exe().expect("test binary path"))
|
||||
.arg("mrf_successor_flush_waiting_child_process_fixture")
|
||||
.arg("--exact")
|
||||
.arg("--nocapture")
|
||||
.env("RUSTFS_MRF_SUCCESSOR_KILL_CHILD_ROOT", temp_dir.path())
|
||||
.env("RUSTFS_MRF_SUCCESSOR_KILL_READY", &ready)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.expect("child MRF successor fixture should start");
|
||||
let ready_seen = wait_until(Duration::from_secs(10), || {
|
||||
let ready = ready.clone();
|
||||
async move { ready.exists() }
|
||||
})
|
||||
.await;
|
||||
assert!(ready_seen, "child process did not reach the successor flush boundary");
|
||||
child.kill().expect("kill child fixture");
|
||||
let status = child.wait().expect("wait for killed child fixture");
|
||||
assert!(!status.success(), "child fixture must be terminated instead of exiting cleanly");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let expected_successor = journal_record(1, "service-kill-bucket", "second-object", None, 2);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor),
|
||||
"restarted process must see the successor snapshot produced before the kill"
|
||||
);
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 1, "restart after service kill must replay only the still-pending tail");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the successor tail must be accepted after service kill restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"a fully consumed successor snapshot may be deleted after service-kill restart replay"
|
||||
);
|
||||
}
|
||||
|
||||
/// A hard kill between the authoritative successor fsync and the legacy mirror
|
||||
/// rewrite must prefer the canonical successor tail over the stale legacy
|
||||
/// startup epoch. This models the mixed-version boundary conservatively: new
|
||||
/// readers must not merge epochs, while the old mirror remains crash-visible.
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
#[cfg(unix)]
|
||||
async fn journal_replay_survives_sigkill_after_authoritative_successor_fsync_before_legacy_mirror() {
|
||||
let temp_dir = tempfile::tempdir().expect("fsync-kill MRF root");
|
||||
let ready = temp_dir.path().join("authoritative-synced.ready");
|
||||
let mut child = Command::new(std::env::current_exe().expect("test binary path"))
|
||||
.arg("mrf_authoritative_fsync_waiting_child_process_fixture")
|
||||
.arg("--exact")
|
||||
.arg("--nocapture")
|
||||
.env("RUSTFS_MRF_FSYNC_KILL_CHILD_ROOT", temp_dir.path())
|
||||
.env("RUSTFS_MRF_FSYNC_KILL_READY", &ready)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.expect("child MRF fsync fixture should start");
|
||||
let ready_seen = wait_until(Duration::from_secs(10), || {
|
||||
let ready = ready.clone();
|
||||
async move { ready.exists() }
|
||||
})
|
||||
.await;
|
||||
assert!(ready_seen, "child process did not reach the authoritative fsync boundary");
|
||||
child.kill().expect("kill child fixture");
|
||||
let status = child.wait().expect("wait for killed child fixture");
|
||||
assert!(!status.success(), "child fixture must be terminated instead of exiting cleanly");
|
||||
|
||||
let (disk_paths, storage) = heal_env_at(Some(temp_dir.path())).await;
|
||||
let expected_successor = journal_record(1, "fsync-kill-bucket", "second-object", None, 2);
|
||||
let stale_startup = {
|
||||
let mut startup = journal_record(1, "fsync-kill-bucket", "first-object", None, 0);
|
||||
startup.extend(journal_record(1, "fsync-kill-bucket", "second-object", None, 0));
|
||||
startup
|
||||
};
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, SCOPED_JOURNAL_REL, &expected_successor),
|
||||
"restarted process must see the fsynced authoritative successor"
|
||||
);
|
||||
assert!(
|
||||
journal_matches_on_all_disks(&disk_paths, JOURNAL_REL, &stale_startup),
|
||||
"legacy mirror intentionally remains at the stale startup epoch"
|
||||
);
|
||||
|
||||
let restarted = make_manager(storage);
|
||||
let replayed = mrf_queue::replay_journal_once(&restarted).await;
|
||||
assert_eq!(replayed, 1, "new reader must replay only the authoritative successor tail");
|
||||
assert_eq!(
|
||||
restarted.operations_snapshot().await.queued_by_source.mrf,
|
||||
1,
|
||||
"the successor tail must be accepted after the fsync-boundary restart"
|
||||
);
|
||||
assert!(
|
||||
disk_paths.iter().all(|path| {
|
||||
!Path::new(path).join(META_BUCKET).join(JOURNAL_REL).exists()
|
||||
&& !Path::new(path).join(META_BUCKET).join(SCOPED_JOURNAL_REL).exists()
|
||||
}),
|
||||
"a fully consumed authoritative successor may clean both epochs after restart replay"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -241,10 +241,7 @@ impl AdaptiveTTL {
|
||||
// 1. Item is cold (low access count)
|
||||
// 2. Age is significant (> 50% of TTL)
|
||||
// 3. No recent accesses
|
||||
if access_count <= self.cold_threshold && age > current_ttl / 2 {
|
||||
return true;
|
||||
}
|
||||
false
|
||||
access_count <= self.cold_threshold && age > current_ttl / 2
|
||||
}
|
||||
|
||||
/// Calculate priority score for an item.
|
||||
|
||||
@@ -122,6 +122,10 @@ pub trait ReplicationConfigurationExt {
|
||||
fn has_active_rules(&self, prefix: &str, recursive: bool) -> bool;
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String>;
|
||||
fn filter_force_delete_target_arns(&self, prefix: &str) -> Vec<String>;
|
||||
/// Every target ARN the configuration still names, whatever the rule's
|
||||
/// status, prefix or filter: the set a pending replication delete may
|
||||
/// still be owed to. A target outside it was removed by the operator.
|
||||
fn configured_target_arns(&self) -> HashSet<String>;
|
||||
fn filter_target_replication_decisions(&self, obj: &ObjectOpts) -> Vec<(String, bool)> {
|
||||
self.filter_target_arns(obj)
|
||||
.into_iter()
|
||||
@@ -772,6 +776,19 @@ impl ReplicationConfigurationExt for ReplicationConfiguration {
|
||||
}
|
||||
|
||||
/// Filter target ARNs and return a slice of the distinct values in the config
|
||||
fn configured_target_arns(&self) -> HashSet<String> {
|
||||
let role = self.role.trim();
|
||||
if !role.is_empty() {
|
||||
return HashSet::from([role.to_string()]);
|
||||
}
|
||||
self.rules
|
||||
.iter()
|
||||
.map(|rule| rule.destination.bucket.trim())
|
||||
.filter(|arn| !arn.is_empty())
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn filter_target_arns(&self, obj: &ObjectOpts) -> Vec<String> {
|
||||
let role = self.role.trim();
|
||||
if !role.is_empty() {
|
||||
|
||||
@@ -38,6 +38,7 @@ use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf};
|
||||
use tokio::time::{Duration, Instant, sleep, timeout};
|
||||
use tracing::{debug, warn};
|
||||
|
||||
use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageOwnerStatus};
|
||||
use crate::storage_api::owner::HTTPPreconditions;
|
||||
use crate::{
|
||||
BUCKET_META_PREFIX, EcstoreError as Error, EcstoreResult as StorageResult, RUSTFS_META_BUCKET, ReplicationConfig,
|
||||
@@ -535,6 +536,10 @@ pub struct DataUsageEntryInfo {
|
||||
pub name: String,
|
||||
pub parent: String,
|
||||
pub entry: DataUsageEntry,
|
||||
/// Durable bucket incarnation that produced this bucket root. Missing
|
||||
/// values are legacy/unproven and must not authorize cold-bucket reuse.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub bucket_incarnation: Option<uuid::Uuid>,
|
||||
/// Registry generation used to classify this root entry. Older remote
|
||||
/// workers omit it; callers must reject that result when a frozen cycle
|
||||
/// requires generation fencing.
|
||||
@@ -601,6 +606,8 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_checkpoint: Option<DataUsageScanCheckpoint>,
|
||||
#[serde(default)]
|
||||
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
#[serde(default)]
|
||||
pub scan_identity: Option<DataUsageScanIdentity>,
|
||||
#[serde(default)]
|
||||
@@ -650,6 +657,11 @@ pub struct DataUsageCacheInfo {
|
||||
/// structural plan remains reusable across ordinary bucket writes.
|
||||
#[serde(default)]
|
||||
pub scan_execution_digest: Option<DataUsageScanPlanDigest>,
|
||||
/// Durable bucket incarnations captured for a complete set aggregate.
|
||||
/// Missing or nil entries are legacy/unproven and cannot authorize
|
||||
/// skipping an unselected bucket in a later scoped set scan.
|
||||
#[serde(default, skip_serializing_if = "HashMap::is_empty")]
|
||||
pub scan_bucket_incarnations: HashMap<String, uuid::Uuid>,
|
||||
}
|
||||
|
||||
impl Serialize for DataUsageCacheInfo {
|
||||
@@ -661,6 +673,7 @@ impl Serialize for DataUsageCacheInfo {
|
||||
// appended by newer scanner versions during rolling upgrades.
|
||||
let field_count = 16
|
||||
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
|
||||
+ usize::from(self.scan_raw_enumeration_page_index.is_some())
|
||||
+ usize::from(self.scan_identity.is_some())
|
||||
+ usize::from(self.scan_progress.is_some())
|
||||
+ usize::from(self.scan_coverage_receipt.is_some())
|
||||
@@ -672,7 +685,8 @@ impl Serialize for DataUsageCacheInfo {
|
||||
+ usize::from(self.lkg_last_update.is_some())
|
||||
+ usize::from(self.lkg_leader_epoch.is_some())
|
||||
+ usize::from(self.lkg_scan_plan_digest.is_some())
|
||||
+ usize::from(self.scan_execution_digest.is_some());
|
||||
+ usize::from(self.scan_execution_digest.is_some())
|
||||
+ usize::from(!self.scan_bucket_incarnations.is_empty());
|
||||
let mut state = serializer.serialize_map(Some(field_count))?;
|
||||
state.serialize_entry("name", &self.name)?;
|
||||
state.serialize_entry("next_cycle", &self.next_cycle)?;
|
||||
@@ -687,6 +701,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if let Some(cursor) = &self.scan_raw_enumeration_cursor {
|
||||
state.serialize_entry("scan_raw_enumeration_cursor", cursor)?;
|
||||
}
|
||||
if let Some(index) = &self.scan_raw_enumeration_page_index {
|
||||
state.serialize_entry("scan_raw_enumeration_page_index", index)?;
|
||||
}
|
||||
if let Some(identity) = self.scan_identity {
|
||||
state.serialize_entry("scan_identity", &identity)?;
|
||||
}
|
||||
@@ -729,6 +746,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if let Some(scan_execution_digest) = self.scan_execution_digest {
|
||||
state.serialize_entry("scan_execution_digest", &scan_execution_digest)?;
|
||||
}
|
||||
if !self.scan_bucket_incarnations.is_empty() {
|
||||
state.serialize_entry("scan_bucket_incarnations", &self.scan_bucket_incarnations)?;
|
||||
}
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
@@ -895,6 +915,7 @@ impl DataUsageCache {
|
||||
&& self.info.scan_progress.is_none()
|
||||
&& self.info.scan_checkpoint.is_none()
|
||||
&& self.info.scan_raw_enumeration_cursor.is_none()
|
||||
&& self.info.scan_raw_enumeration_page_index.is_none()
|
||||
&& self.info.scan_resume_after.is_none()
|
||||
&& self.info.scan_coverage_receipt.is_none()
|
||||
&& self.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
@@ -922,12 +943,17 @@ impl DataUsageCache {
|
||||
if self.validated_raw_enumeration_cursor().is_none() {
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
}
|
||||
if self.validated_raw_enumeration_page_index().is_none() {
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
}
|
||||
let cursor_is_valid = (self.info.scan_checkpoint.is_none()
|
||||
&& self.info.scan_raw_enumeration_cursor.is_none()
|
||||
&& self.info.scan_raw_enumeration_page_index.is_none()
|
||||
&& self.info.scan_resume_after.is_none()
|
||||
&& self.info.scan_coverage_receipt.is_none())
|
||||
|| self.validated_scan_frontier().is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some();
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| self.info.scan_raw_enumeration_page_index.is_some();
|
||||
if !cursor_is_valid {
|
||||
self.info.scan_progress = None;
|
||||
}
|
||||
@@ -949,6 +975,7 @@ impl DataUsageCache {
|
||||
self.info.scan_resume_after = None;
|
||||
self.info.scan_checkpoint = None;
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_coverage_receipt = None;
|
||||
}
|
||||
// Old readers do not understand coverage sweeps. An absent plan makes
|
||||
@@ -1026,6 +1053,25 @@ impl DataUsageCache {
|
||||
.then_some(cursor)
|
||||
}
|
||||
|
||||
pub(crate) fn validated_raw_enumeration_page_index(&self) -> Option<&RawEnumerationPageIndex> {
|
||||
let index = self.info.scan_raw_enumeration_page_index.as_ref()?;
|
||||
if self.info.scan_progress.is_none()
|
||||
|| !self.info.scan_identity.is_some_and(|identity| identity.is_valid())
|
||||
|| self.info.source.is_none()
|
||||
|| index.committed_entries().is_err()
|
||||
|| index.indexed_entries().is_err()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
let parent = match index.status() {
|
||||
RawEnumerationPageOwnerStatus::Unsupported => return None,
|
||||
RawEnumerationPageOwnerStatus::Building { parent, .. } | RawEnumerationPageOwnerStatus::Ready { parent, .. } => {
|
||||
parent
|
||||
}
|
||||
};
|
||||
path_is_in_bucket_scope(&self.info.name, &parent).then_some(index)
|
||||
}
|
||||
|
||||
/// Seal only the frontier supplied by completed traversal, never a restored cursor.
|
||||
pub(crate) fn seal_scan_frontier(&mut self, frontier: Option<&str>) -> Result<(), serde_json::Error> {
|
||||
if self.info.scan_progress.is_none() {
|
||||
|
||||
@@ -1179,6 +1179,7 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
|
||||
7,
|
||||
[7; 32],
|
||||
)),
|
||||
scan_raw_enumeration_page_index: Some(raw_page_index_fixture("bucket/prefix", &["entry-a"], false)),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
scan_execution_digest: Some(DataUsageScanPlanDigest([42; 32])),
|
||||
@@ -1207,6 +1208,7 @@ fn test_new_data_usage_cache_msgpack_round_trips_and_supports_old_reader() {
|
||||
.map(|cursor| cursor.last_entry.as_deref()),
|
||||
Some(Some("last-object"))
|
||||
);
|
||||
assert!(current.info.scan_raw_enumeration_page_index.is_some());
|
||||
assert!(current.info.snapshot_complete);
|
||||
assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
|
||||
assert_eq!(current.info.scan_execution_digest, Some(DataUsageScanPlanDigest([42; 32])));
|
||||
@@ -1260,6 +1262,24 @@ fn cache_with_raw_cursor(cursor: DataUsageRawEnumerationCursor) -> DataUsageCach
|
||||
}
|
||||
}
|
||||
|
||||
fn raw_page_index_fixture(parent: &str, entries: &[&str], complete: bool) -> RawEnumerationPageIndex {
|
||||
let mut index = RawEnumerationPageIndex::new(parent, 2).expect("raw page index should initialize");
|
||||
let generation = index.generation().expect("raw page index should expose generation");
|
||||
let outcome = if complete {
|
||||
index.ingest_owner_entries(entries.iter().map(|entry| (*entry).to_string()), entries.len().max(1), generation)
|
||||
} else {
|
||||
index.ingest_partial_owner_entries(entries.iter().map(|entry| (*entry).to_string()), entries.len().max(1), generation)
|
||||
}
|
||||
.expect("raw page index fixture should ingest entries");
|
||||
if outcome.ready_to_commit {
|
||||
let generation = index.generation().expect("raw page index should expose commit generation");
|
||||
index
|
||||
.commit_building_page(generation)
|
||||
.expect("raw page index fixture should commit ready page");
|
||||
}
|
||||
index
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_cursor_validation_requires_bucket_identity_and_bounded_marker() {
|
||||
let valid = DataUsageRawEnumerationCursor::new("bucket/raw".to_string(), Some("entry-001".to_string()), 1, [8; 32]);
|
||||
@@ -1346,6 +1366,44 @@ fn prepare_bucket_checkpoint_preserves_only_valid_raw_enumeration_cursor() {
|
||||
assert!(cache.info.scan_progress.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn prepare_bucket_checkpoint_preserves_only_valid_raw_page_index() {
|
||||
let identity = valid_scan_identity();
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let page_index = raw_page_index_fixture("bucket/raw", &["entry-001"], false);
|
||||
let mut cache = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "bucket".to_string(),
|
||||
leader_epoch: 1,
|
||||
source: Some(source),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
scan_identity: Some(identity),
|
||||
tier_registry_generation: Some(9),
|
||||
scan_progress: Some(DataUsageScanProgress {
|
||||
started_plan: TEST_PLAN_DIGEST,
|
||||
requested_plan: TEST_PLAN_DIGEST,
|
||||
}),
|
||||
scan_raw_enumeration_page_index: Some(page_index.clone()),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
assert_eq!(
|
||||
cache.prepare_bucket_checkpoint("bucket", 1, 1, source, TEST_PLAN_DIGEST, identity),
|
||||
DataUsageCachePrepareOutcome::Reused
|
||||
);
|
||||
assert_eq!(cache.info.scan_raw_enumeration_page_index, Some(page_index));
|
||||
|
||||
let invalid = raw_page_index_fixture("other/raw", &["entry-001"], false);
|
||||
cache.info.scan_raw_enumeration_page_index = Some(invalid);
|
||||
assert_eq!(
|
||||
cache.prepare_bucket_checkpoint("bucket", 1, 1, source, TEST_PLAN_DIGEST, identity),
|
||||
DataUsageCachePrepareOutcome::Reused
|
||||
);
|
||||
assert!(cache.info.scan_raw_enumeration_page_index.is_none());
|
||||
assert!(cache.info.scan_progress.is_some());
|
||||
}
|
||||
|
||||
/// Deterministic, fully populated cache used to pin the persisted
|
||||
/// `.usage-cache.bin` wire bytes. Every map/set holds at most one element
|
||||
/// so the map-encoded `marshal_msg` output is byte-stable.
|
||||
|
||||
@@ -60,6 +60,7 @@ use uuid::Uuid;
|
||||
pub mod data_usage_define;
|
||||
pub mod error;
|
||||
pub mod prefix_usage;
|
||||
pub mod raw_page_index;
|
||||
mod remote_scanner;
|
||||
pub mod runtime_config;
|
||||
pub mod scanner;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -195,6 +195,7 @@ fn test_usage(bucket: &str, objects: usize) -> DataUsageEntryInfo {
|
||||
name: bucket.to_string(),
|
||||
parent: crate::DATA_USAGE_ROOT.to_string(),
|
||||
entry,
|
||||
bucket_incarnation: Some(Uuid::from_u128(7)),
|
||||
tier_registry_generation: Some(0),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7609,6 +7609,81 @@ async fn scanner_cycle_confirms_lost_remote_ack_from_activity_snapshot() {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_cycle_confirms_lost_scoped_ack_only_after_same_instance_clean_activity() {
|
||||
let acknowledgement = ScannerDirtyUsageAcknowledgement {
|
||||
host: "node-2".to_string(),
|
||||
instance_id: "epoch-a".to_string(),
|
||||
kind: ScannerDirtyUsageAcknowledgementKind::Scoped {
|
||||
owner_id: Uuid::from_u128(0x11111111111111111111111111111111).to_string(),
|
||||
entries: vec![crate::storage_api::EcstoreScannerScopedDirtyUsageAckEntry {
|
||||
bucket: "photos".to_string(),
|
||||
bucket_incarnation: Uuid::from_u128(0x22222222222222222222222222222222),
|
||||
generation: 5,
|
||||
}],
|
||||
},
|
||||
};
|
||||
let attempted_send = Arc::new(AtomicBool::new(false));
|
||||
let attempted_send_for_ack = Arc::clone(&attempted_send);
|
||||
let cleared_activity = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-a", 7, 3))]);
|
||||
let response_lost = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
async move {
|
||||
attempted_send_for_ack.store(true, Ordering::SeqCst);
|
||||
Err::<bool, _>(std::io::Error::other("scoped ACK transport failed after peer send"))
|
||||
},
|
||||
|| async { Ok(cleared_activity) },
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
attempted_send.load(Ordering::SeqCst),
|
||||
"the confirmation oracle must run only after the scoped ACK send was attempted"
|
||||
);
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, response_lost),
|
||||
ScannerCycleOutcome::Completed,
|
||||
"a same-instance clean activity snapshot confirms a lost scoped ACK response"
|
||||
);
|
||||
|
||||
let restarted_activity = BTreeMap::from([("node-2".to_string(), scanner_node_activity("epoch-b", 7, 3))]);
|
||||
let peer_restarted = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
std::slice::from_ref(&acknowledgement),
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other(
|
||||
"scoped ACK transport failed before peer restart was observed",
|
||||
))),
|
||||
|| async { Ok(restarted_activity) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, peer_restarted),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a restarted peer cannot prove the scoped ACK reached the old scanner instance"
|
||||
);
|
||||
|
||||
let mut written_activity = scanner_node_activity("epoch-a", 7, 3);
|
||||
written_activity.dirty_usage_generation = 6;
|
||||
written_activity.dirty_usage_pending = true;
|
||||
let concurrent_write = remote_dirty_usage_acknowledgement_pending(
|
||||
8,
|
||||
1,
|
||||
&[acknowledgement],
|
||||
std::future::ready(Err::<bool, _>(std::io::Error::other(
|
||||
"scoped ACK transport failed before a concurrent write was observed",
|
||||
))),
|
||||
|| async { Ok(BTreeMap::from([("node-2".to_string(), written_activity)])) },
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, concurrent_write),
|
||||
ScannerCycleOutcome::CompletedWithPendingMaintenance,
|
||||
"a same-instance concurrent write after scoped ACK send keeps maintenance pending"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[serial]
|
||||
fn finalizing_an_already_durable_enum_without_proof_keeps_dirty_pending() {
|
||||
|
||||
@@ -25,6 +25,7 @@ use crate::data_usage_define::{
|
||||
PendingScannerHealKind, ScannerSizeSummaryExt, SizeReconciliationEntry, SizeSummary, hash_path,
|
||||
};
|
||||
use crate::error::ScannerError;
|
||||
use crate::raw_page_index::{RawEnumerationPageIndex, RawEnumerationPageIndexError};
|
||||
use crate::runtime_config::{
|
||||
scanner_alert_excess_folders, scanner_alert_excess_version_size, scanner_alert_excess_versions, scanner_yield_every_n_objects,
|
||||
};
|
||||
@@ -90,6 +91,8 @@ const DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS: usize = 250_000;
|
||||
const SCANNER_LIST_PATH_RAW_STALL_TIMEOUT: Duration = Duration::from_secs(60);
|
||||
const SCANNER_ENTRY_PROGRESS_BATCH: u64 = 32;
|
||||
const SCANNER_ENTRY_PROGRESS_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const SCANNER_RAW_ENUMERATION_PAGE_ENTRY_LIMIT: usize = 128;
|
||||
const SCANNER_RAW_ENUMERATION_PAGE_BUILD_BUDGET: usize = 1;
|
||||
// Erasure data directories contain direct part.N files; keep namespace probes bounded.
|
||||
const ERASURE_DATA_DIR_PROBE_ENTRY_LIMIT: usize = 64;
|
||||
const DEFAULT_HEAL_OBJECT_SELECT_PROB: u32 = 1024;
|
||||
@@ -751,17 +754,34 @@ struct RawEnumerationProgress {
|
||||
last_entry: Option<String>,
|
||||
entries_seen: u64,
|
||||
digest: Sha256,
|
||||
observed_entries: Vec<String>,
|
||||
revalidate_after_entries: usize,
|
||||
page_index: Option<RawEnumerationPageIndex>,
|
||||
}
|
||||
|
||||
impl RawEnumerationProgress {
|
||||
fn new(parent: &str) -> Self {
|
||||
fn new(parent: &str, page_index: Option<RawEnumerationPageIndex>) -> Self {
|
||||
let mut digest = Sha256::new();
|
||||
update_raw_enumeration_digest(&mut digest, b"parent", parent.as_bytes());
|
||||
let mut revalidate_after_entries = 0;
|
||||
let page_index = match page_index {
|
||||
Some(index) => match index.indexed_entries() {
|
||||
Ok(entries) => {
|
||||
revalidate_after_entries = entries.len();
|
||||
Some(index)
|
||||
}
|
||||
Err(_) => None,
|
||||
},
|
||||
None => RawEnumerationPageIndex::new(parent, SCANNER_RAW_ENUMERATION_PAGE_ENTRY_LIMIT).ok(),
|
||||
};
|
||||
Self {
|
||||
parent: parent.to_string(),
|
||||
last_entry: None,
|
||||
entries_seen: 0,
|
||||
digest,
|
||||
observed_entries: Vec::new(),
|
||||
revalidate_after_entries,
|
||||
page_index,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -769,19 +789,63 @@ impl RawEnumerationProgress {
|
||||
update_raw_enumeration_digest(&mut self.digest, b"entry", entry.as_bytes());
|
||||
self.last_entry = Some(entry.to_string());
|
||||
self.entries_seen = self.entries_seen.saturating_add(1);
|
||||
self.observed_entries.push(entry.to_string());
|
||||
if let Some(index) = &mut self.page_index {
|
||||
if self.observed_entries.len() < self.revalidate_after_entries {
|
||||
return;
|
||||
}
|
||||
let result = index
|
||||
.generation()
|
||||
.ok_or(RawEnumerationPageIndexError::Unsupported)
|
||||
.and_then(|generation| {
|
||||
index.ingest_partial_owner_entries(
|
||||
self.observed_entries.clone(),
|
||||
SCANNER_RAW_ENUMERATION_PAGE_BUILD_BUDGET,
|
||||
generation,
|
||||
)
|
||||
});
|
||||
match result {
|
||||
Ok(outcome) if outcome.ready_to_commit => {
|
||||
if let Some(generation) = index.generation()
|
||||
&& index.commit_building_page(generation).is_err()
|
||||
{
|
||||
self.page_index = None;
|
||||
}
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(_) => {
|
||||
self.page_index = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn into_cursor(self) -> Option<DataUsageRawEnumerationCursor> {
|
||||
fn cursor(&self) -> Option<DataUsageRawEnumerationCursor> {
|
||||
if self.entries_seen == 0 {
|
||||
return None;
|
||||
}
|
||||
Some(DataUsageRawEnumerationCursor::new(
|
||||
self.parent,
|
||||
self.last_entry,
|
||||
self.parent.clone(),
|
||||
self.last_entry.clone(),
|
||||
self.entries_seen,
|
||||
self.digest.finalize().into(),
|
||||
self.digest.clone().finalize().into(),
|
||||
))
|
||||
}
|
||||
|
||||
fn page_index(&self) -> Option<RawEnumerationPageIndex> {
|
||||
self.page_index.clone().and_then(|mut index| {
|
||||
if let Some(generation) = index.generation()
|
||||
&& matches!(index.status(), crate::raw_page_index::RawEnumerationPageOwnerStatus::Building { .. })
|
||||
&& index.commit_building_page(generation).is_err()
|
||||
{
|
||||
return None;
|
||||
}
|
||||
match index.indexed_entries() {
|
||||
Ok(entries) if !entries.is_empty() => Some(index),
|
||||
_ => None,
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
fn update_raw_enumeration_digest(digest: &mut Sha256, label: &[u8], value: &[u8]) {
|
||||
@@ -1049,6 +1113,19 @@ impl FolderScanner {
|
||||
if self.old_cache.info.scan_progress.is_none() {
|
||||
return;
|
||||
}
|
||||
let page_index = self
|
||||
.old_cache
|
||||
.validated_raw_enumeration_page_index()
|
||||
.filter(|index| match index.status() {
|
||||
crate::raw_page_index::RawEnumerationPageOwnerStatus::Building {
|
||||
parent: index_parent, ..
|
||||
}
|
||||
| crate::raw_page_index::RawEnumerationPageOwnerStatus::Ready {
|
||||
parent: index_parent, ..
|
||||
} => index_parent == parent,
|
||||
crate::raw_page_index::RawEnumerationPageOwnerStatus::Unsupported => false,
|
||||
})
|
||||
.cloned();
|
||||
if let Some(position) = self
|
||||
.raw_enumeration_progress
|
||||
.iter()
|
||||
@@ -1056,13 +1133,37 @@ impl FolderScanner {
|
||||
{
|
||||
self.raw_enumeration_progress.truncate(position + 1);
|
||||
} else {
|
||||
self.raw_enumeration_progress.push(RawEnumerationProgress::new(parent));
|
||||
self.raw_enumeration_progress
|
||||
.push(RawEnumerationProgress::new(parent, page_index));
|
||||
}
|
||||
if let Some(progress) = self.raw_enumeration_progress.last_mut() {
|
||||
progress.record_entry(entry);
|
||||
}
|
||||
}
|
||||
|
||||
fn raw_enumeration_committed_entry_oracle(&self, parent: &str) -> HashSet<String> {
|
||||
let Some(index) = self.old_cache.validated_raw_enumeration_page_index() else {
|
||||
return HashSet::new();
|
||||
};
|
||||
let generation_matches_parent = match index.status() {
|
||||
crate::raw_page_index::RawEnumerationPageOwnerStatus::Building {
|
||||
generation,
|
||||
parent: index_parent,
|
||||
..
|
||||
}
|
||||
| crate::raw_page_index::RawEnumerationPageOwnerStatus::Ready {
|
||||
generation,
|
||||
parent: index_parent,
|
||||
..
|
||||
} => generation > 0 && index_parent == parent,
|
||||
crate::raw_page_index::RawEnumerationPageOwnerStatus::Unsupported => false,
|
||||
};
|
||||
if !generation_matches_parent {
|
||||
return HashSet::new();
|
||||
}
|
||||
index.committed_entries().unwrap_or_default().into_iter().collect()
|
||||
}
|
||||
|
||||
fn finish_raw_enumeration_parent(&mut self, parent: &str) {
|
||||
self.raw_enumeration_progress.retain(|progress| {
|
||||
progress.parent != parent
|
||||
@@ -1073,11 +1174,11 @@ impl FolderScanner {
|
||||
});
|
||||
}
|
||||
|
||||
fn take_raw_enumeration_cursor(&mut self) -> Option<DataUsageRawEnumerationCursor> {
|
||||
self.raw_enumeration_progress
|
||||
.drain(..)
|
||||
.next()
|
||||
.and_then(RawEnumerationProgress::into_cursor)
|
||||
fn take_raw_enumeration_resume_state(&mut self) -> (Option<DataUsageRawEnumerationCursor>, Option<RawEnumerationPageIndex>) {
|
||||
match self.raw_enumeration_progress.drain(..).next() {
|
||||
Some(progress) => (progress.cursor(), progress.page_index()),
|
||||
None => (None, None),
|
||||
}
|
||||
}
|
||||
|
||||
fn carry_forward_old_children(&mut self, parent_hash: &DataUsageHash, entry: &mut DataUsageEntry) {
|
||||
@@ -1411,6 +1512,7 @@ impl FolderScanner {
|
||||
let mut pending_entry_progress = 0_u64;
|
||||
let mut last_entry_progress = Instant::now();
|
||||
let mut raw_enumeration_complete = false;
|
||||
let raw_enumeration_committed_entries = self.raw_enumeration_committed_entry_oracle(&folder.name);
|
||||
|
||||
loop {
|
||||
let entry = match dir_reader.next_entry().await {
|
||||
@@ -1449,20 +1551,23 @@ impl FolderScanner {
|
||||
}
|
||||
Err(e) => return Err(ScannerError::Io(e)),
|
||||
};
|
||||
#[cfg(test)]
|
||||
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|
||||
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
|
||||
{
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
pending_entry_progress = 0;
|
||||
last_entry_progress = Instant::now();
|
||||
}
|
||||
let file_name = entry.file_name().to_string_lossy().to_string();
|
||||
if file_name.is_empty() || file_name == "." || file_name == ".." {
|
||||
continue;
|
||||
}
|
||||
let raw_entry_consumed_by_owner_index = raw_enumeration_committed_entries.contains(&file_name);
|
||||
if !raw_entry_consumed_by_owner_index {
|
||||
#[cfg(test)]
|
||||
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
|
||||
pending_entry_progress = pending_entry_progress.saturating_add(1);
|
||||
if pending_entry_progress >= SCANNER_ENTRY_PROGRESS_BATCH
|
||||
|| last_entry_progress.elapsed() >= SCANNER_ENTRY_PROGRESS_INTERVAL
|
||||
{
|
||||
self.budget.record_entries_visited(pending_entry_progress);
|
||||
pending_entry_progress = 0;
|
||||
last_entry_progress = Instant::now();
|
||||
}
|
||||
}
|
||||
self.record_raw_enumeration_entry(&folder.name, &file_name);
|
||||
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
|
||||
|
||||
@@ -2686,6 +2791,7 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
new_cache.info.scan_resume_after = None;
|
||||
new_cache.info.scan_checkpoint = None;
|
||||
new_cache.info.scan_raw_enumeration_cursor = None;
|
||||
new_cache.info.scan_raw_enumeration_page_index = None;
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
if had_scan_checkpoint {
|
||||
global_metrics().record_scanner_checkpoint_cleared();
|
||||
@@ -2703,9 +2809,10 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
let root_hash = hash_path(&cache.info.name);
|
||||
let root_has_progress = data_usage_root_has_progress(&root);
|
||||
let pending_heals_changed = scanner.pending_heals_changed;
|
||||
let raw_enumeration_cursor = scanner.take_raw_enumeration_cursor();
|
||||
let carry_forward_cache =
|
||||
(raw_enumeration_cursor.is_some() && !root_has_progress).then(|| scanner.old_cache.cache.clone());
|
||||
let (raw_enumeration_cursor, raw_enumeration_page_index) = scanner.take_raw_enumeration_resume_state();
|
||||
let carry_forward_cache = ((raw_enumeration_cursor.is_some() || raw_enumeration_page_index.is_some())
|
||||
&& !root_has_progress)
|
||||
.then(|| scanner.old_cache.cache.clone());
|
||||
if root_has_progress {
|
||||
scanner.carry_forward_old_children(&root_hash, &mut root);
|
||||
}
|
||||
@@ -2722,8 +2829,15 @@ pub(crate) async fn scan_data_folder_scoped(
|
||||
new_cache.info.scan_resume_after = None;
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
}
|
||||
if raw_enumeration_page_index.is_some() {
|
||||
new_cache.info.scan_raw_enumeration_page_index = raw_enumeration_page_index;
|
||||
new_cache.info.scan_checkpoint = None;
|
||||
new_cache.info.scan_resume_after = None;
|
||||
new_cache.info.scan_coverage_receipt = None;
|
||||
}
|
||||
if partial_cache_is_useful(&root, pending_heals_changed)
|
||||
|| new_cache.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| new_cache.info.scan_raw_enumeration_page_index.is_some()
|
||||
|| !new_cache.info.size_reconciliation.is_empty()
|
||||
{
|
||||
if new_cache.root().is_some() {
|
||||
|
||||
@@ -1036,12 +1036,19 @@ impl ScannerItem {
|
||||
return;
|
||||
}
|
||||
|
||||
let Some(replication) = self.replication.clone() else {
|
||||
return;
|
||||
let replication = match self.replication.clone() {
|
||||
Some(replication) => (*replication).clone(),
|
||||
// No active rules or targets, but a purge the bucket still owes
|
||||
// must reach the heal path: the delete worker settles it against
|
||||
// the current configuration (abandoned when the target is gone,
|
||||
// rustfs/backlog#2340) so the hidden version stops blocking
|
||||
// DeleteBucket.
|
||||
None if !oi.version_purge_status.is_empty() => ReplicationConfig::new(None, None),
|
||||
None => return,
|
||||
};
|
||||
|
||||
let done_replication = Metrics::time(Metric::CheckReplication);
|
||||
let replication_result = queue_replication_heal(&oi.bucket, oi.clone(), (*replication).clone(), 0).await;
|
||||
let replication_result = queue_replication_heal(&oi.bucket, oi.clone(), replication, 0).await;
|
||||
done_replication();
|
||||
let roi = replication_result.object_info;
|
||||
record_scanner_replication_admission(global_metrics(), &roi, replication_result.admission);
|
||||
|
||||
@@ -2451,6 +2451,82 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
|
||||
assert_eq!((bucket.size, bucket.objects), (17, 3));
|
||||
}
|
||||
|
||||
async fn scan_hot_cold_segment_fixture(scoped: bool) -> (DataUsageEntry, Vec<String>) {
|
||||
let (mut scanner, temp_dir) = build_test_scanner().await;
|
||||
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
|
||||
write_test_object_metadata(&temp_dir, "bucket", "cold/object").await;
|
||||
write_test_object_metadata(&temp_dir, "bucket", "hot/object").await;
|
||||
|
||||
scanner.old_cache.info.name = "bucket".to_string();
|
||||
scanner.new_cache.info.name = "bucket".to_string();
|
||||
scanner.update_cache.info.name = "bucket".to_string();
|
||||
if scoped {
|
||||
scanner.old_cache.replace("bucket", "", DataUsageEntry::default());
|
||||
scanner.old_cache.replace(
|
||||
"bucket/cold",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: 0,
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
scanner.prefix_scan_scope =
|
||||
ScannerBucketPrefixScanScope::from_dirty_top_level_entries(HashSet::from(["hot".to_string()]));
|
||||
}
|
||||
|
||||
let walked = Arc::new(Mutex::new(Vec::<String>::new()));
|
||||
scanner.update_current_path = Arc::new({
|
||||
let walked = walked.clone();
|
||||
move |path: &str| {
|
||||
walked.lock().expect("lock observed scanner paths").push(path.to_string());
|
||||
Box::pin(async {})
|
||||
}
|
||||
});
|
||||
|
||||
let folder = CachedFolder {
|
||||
name: "bucket".to_string(),
|
||||
parent: None,
|
||||
object_heal_prob_div: 1,
|
||||
};
|
||||
let mut root = DataUsageEntry::default();
|
||||
scanner
|
||||
.scan_folder(CancellationToken::new(), folder, &mut root)
|
||||
.await
|
||||
.expect("segment fixture scan should finish");
|
||||
let root = scanner
|
||||
.new_cache
|
||||
.size_recursive("bucket")
|
||||
.expect("segment fixture should produce a bucket cache root");
|
||||
let walked = walked.lock().expect("read observed scanner paths").clone();
|
||||
(root, walked)
|
||||
}
|
||||
|
||||
fn walked_path_in(paths: &[String], subtree: &str) -> bool {
|
||||
paths
|
||||
.iter()
|
||||
.any(|path| path == subtree || path.strip_prefix(subtree).is_some_and(|rest| rest.starts_with('/')))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_root_scan_zero_walks_clean_cold_segment_with_full_oracle_equivalence() {
|
||||
let (full, full_walked) = scan_hot_cold_segment_fixture(false).await;
|
||||
let (scoped, scoped_walked) = scan_hot_cold_segment_fixture(true).await;
|
||||
|
||||
assert_eq!((scoped.size, scoped.objects), (full.size, full.objects));
|
||||
assert_eq!((scoped.size, scoped.objects), (0, 2));
|
||||
assert!(
|
||||
walked_path_in(&full_walked, "bucket/cold"),
|
||||
"the full oracle must prove the cold segment would be walked without scoped reuse"
|
||||
);
|
||||
assert!(walked_path_in(&scoped_walked, "bucket/hot"), "the dirty hot segment must still be walked");
|
||||
assert!(
|
||||
!walked_path_in(&scoped_walked, "bucket/cold"),
|
||||
"a clean cold segment must be copied from the durable baseline without walker callbacks"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn scoped_root_scan_preserves_erasure_health_walks() {
|
||||
@@ -2722,6 +2798,27 @@ async fn test_scan_data_folder_returns_raw_cursor_on_enumeration_cancel_without_
|
||||
assert!(raw_cursor.last_entry.is_some());
|
||||
assert_ne!(raw_cursor.page_digest, [0; 32]);
|
||||
assert_eq!(partial_cache.validated_raw_enumeration_cursor(), Some(raw_cursor));
|
||||
let page_index = partial_cache
|
||||
.validated_raw_enumeration_page_index()
|
||||
.expect("raw enumeration cancellation should persist a validated page index");
|
||||
assert_eq!(
|
||||
page_index
|
||||
.indexed_entries()
|
||||
.expect("persisted raw page index entries should validate")
|
||||
.len(),
|
||||
1
|
||||
);
|
||||
assert_eq!(
|
||||
page_index
|
||||
.committed_entries()
|
||||
.expect("checkpointed raw page should validate as committed coverage"),
|
||||
vec![
|
||||
raw_cursor
|
||||
.last_entry
|
||||
.clone()
|
||||
.expect("checkpointed page should include the observed entry")
|
||||
]
|
||||
);
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Runtime));
|
||||
}
|
||||
|
||||
@@ -3370,3 +3467,74 @@ fn test_should_log_failed_object_samples_after_initial_limit() {
|
||||
assert!(!should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY + 1));
|
||||
assert!(should_log_failed_object(SCANNER_FAILED_OBJECT_LOG_EVERY * 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_progress_waits_for_resume_index_floor_before_revalidation() {
|
||||
let mut index = RawEnumerationPageIndex::new("bucket", 2).expect("raw page index should initialize");
|
||||
let generation = index.generation().expect("raw page index should expose generation");
|
||||
index
|
||||
.ingest_partial_owner_entries(["entry-a".to_string(), "entry-b".to_string()], 2, generation)
|
||||
.expect("initial entries should build a page");
|
||||
let generation = index.generation().expect("raw page index should expose next generation");
|
||||
index.commit_building_page(generation).expect("initial page should commit");
|
||||
|
||||
let mut progress = RawEnumerationProgress::new("bucket", Some(index));
|
||||
progress.record_entry("entry-b");
|
||||
assert!(
|
||||
progress.page_index.is_some(),
|
||||
"resume index must not be dropped before the current run observes the old index floor"
|
||||
);
|
||||
|
||||
progress.record_entry("entry-a");
|
||||
assert!(
|
||||
progress.page_index.is_some(),
|
||||
"same entry identity after the observation floor should keep the resume index"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_progress_checkpoint_commits_budgeted_page_for_oracle() {
|
||||
let mut progress = RawEnumerationProgress::new("bucket", None);
|
||||
progress.record_entry("entry-b");
|
||||
|
||||
let page_index = progress
|
||||
.page_index()
|
||||
.expect("checkpointed raw progress should retain a committed owner page");
|
||||
let page_entries = page_index
|
||||
.committed_entries()
|
||||
.expect("checkpointed owner page should validate by digest");
|
||||
assert_eq!(page_entries, vec!["entry-b".to_string()]);
|
||||
assert_eq!(
|
||||
page_index
|
||||
.indexed_entries()
|
||||
.expect("checkpointed owner index should validate"),
|
||||
page_entries
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn raw_enumeration_progress_retains_resume_index_until_unordered_entries_reappear() {
|
||||
let mut index = RawEnumerationPageIndex::new("bucket", 2).expect("raw page index should initialize");
|
||||
let generation = index.generation().expect("raw page index should expose generation");
|
||||
index
|
||||
.ingest_partial_owner_entries(["entry-a".to_string(), "entry-b".to_string()], 2, generation)
|
||||
.expect("initial entries should build a page");
|
||||
let generation = index.generation().expect("raw page index should expose next generation");
|
||||
index.commit_building_page(generation).expect("initial page should commit");
|
||||
|
||||
let mut progress = RawEnumerationProgress::new("bucket", Some(index));
|
||||
progress.record_entry("entry-a");
|
||||
assert!(progress.page_index.is_some());
|
||||
|
||||
progress.record_entry("entry-c");
|
||||
assert!(
|
||||
progress.page_index.is_some(),
|
||||
"partial observations must not discard the resume index before an unordered old entry can reappear"
|
||||
);
|
||||
|
||||
progress.record_entry("entry-b");
|
||||
assert!(
|
||||
progress.page_index.is_some(),
|
||||
"same source identity should keep the resume index even when read_dir order changes"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! Fixture-only range diagnostics. No result is supplied to a scan selector.
|
||||
|
||||
use super::*;
|
||||
use crate::DATA_USAGE_CACHE_KEY_FORMAT;
|
||||
use std::collections::BTreeSet;
|
||||
|
||||
const MAX_SEGMENTS: usize = 4;
|
||||
@@ -15,6 +16,102 @@ enum ProposalError {
|
||||
InvalidKey,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum ProducerKind {
|
||||
Put,
|
||||
Delete,
|
||||
DeleteMarker,
|
||||
Multipart,
|
||||
Replication,
|
||||
Tier,
|
||||
DirectoryObject,
|
||||
}
|
||||
|
||||
impl ProducerKind {
|
||||
const REQUIRED: [Self; 7] = [
|
||||
Self::Put,
|
||||
Self::Delete,
|
||||
Self::DeleteMarker,
|
||||
Self::Multipart,
|
||||
Self::Replication,
|
||||
Self::Tier,
|
||||
Self::DirectoryObject,
|
||||
];
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
enum SegmentInvalidationDomain {
|
||||
LocalSingleSet,
|
||||
DistributedEc,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct SegmentObservationEnvelope<'a> {
|
||||
source: DataUsageCacheSource,
|
||||
bucket_incarnation: uuid::Uuid,
|
||||
key_format: u16,
|
||||
baseline_scan_plan_digest: DataUsageScanPlanDigest,
|
||||
process_epoch: &'a str,
|
||||
generation_start: u64,
|
||||
generation_end: u64,
|
||||
restart_gap: bool,
|
||||
overflow: bool,
|
||||
producers: BTreeSet<&'a str>,
|
||||
keys: &'a [&'a str],
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct SegmentObservationProof<'a> {
|
||||
source: DataUsageCacheSource,
|
||||
bucket_incarnation: uuid::Uuid,
|
||||
key_format: u16,
|
||||
baseline_scan_plan_digest: DataUsageScanPlanDigest,
|
||||
process_epoch: &'a str,
|
||||
durable_producer_identity: bool,
|
||||
invalidation_domain: SegmentInvalidationDomain,
|
||||
distributed_ec_invalidation: bool,
|
||||
cold_zero_walk_oracle: bool,
|
||||
}
|
||||
|
||||
fn producer_name(kind: ProducerKind) -> &'static str {
|
||||
match kind {
|
||||
ProducerKind::Put => "put",
|
||||
ProducerKind::Delete => "delete",
|
||||
ProducerKind::DeleteMarker => "delete_marker",
|
||||
ProducerKind::Multipart => "multipart",
|
||||
ProducerKind::Replication => "replication",
|
||||
ProducerKind::Tier => "tier",
|
||||
ProducerKind::DirectoryObject => "directory_object",
|
||||
}
|
||||
}
|
||||
|
||||
fn trusted_fixture_proposal(
|
||||
envelope: &SegmentObservationEnvelope<'_>,
|
||||
proof: &SegmentObservationProof<'_>,
|
||||
) -> Result<BTreeSet<String>, ProposalError> {
|
||||
if envelope.source != proof.source
|
||||
|| envelope.bucket_incarnation.is_nil()
|
||||
|| envelope.bucket_incarnation != proof.bucket_incarnation
|
||||
|| envelope.key_format != proof.key_format
|
||||
|| envelope.baseline_scan_plan_digest != proof.baseline_scan_plan_digest
|
||||
|| envelope.process_epoch != proof.process_epoch
|
||||
|| !proof.durable_producer_identity
|
||||
|| !proof.cold_zero_walk_oracle
|
||||
|| (proof.invalidation_domain == SegmentInvalidationDomain::DistributedEc && !proof.distributed_ec_invalidation)
|
||||
|| envelope.generation_start == 0
|
||||
|| envelope.generation_end < envelope.generation_start
|
||||
|| envelope.restart_gap
|
||||
|| envelope.overflow
|
||||
|| !ProducerKind::REQUIRED
|
||||
.iter()
|
||||
.all(|producer| envelope.producers.contains(producer_name(*producer)))
|
||||
{
|
||||
return Err(ProposalError::InvalidKey);
|
||||
}
|
||||
|
||||
fixture_proposal(envelope.keys)
|
||||
}
|
||||
|
||||
// Keys come from successful fixture writes, not a production mutation stream.
|
||||
fn fixture_proposal(keys: &[&str]) -> Result<BTreeSet<String>, ProposalError> {
|
||||
let mut segments = BTreeSet::new();
|
||||
@@ -54,6 +151,107 @@ fn segment_observation_fixture_proposal_bounds() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn segment_observation_trusted_proposal_requires_identity_and_complete_producer_coverage() {
|
||||
let source = DataUsageCacheSource::new(2, 3);
|
||||
let incarnation = uuid::Uuid::from_u128(0x12345678123456781234567812345678);
|
||||
let baseline = DataUsageScanPlanDigest([9; 32]);
|
||||
let producers = ProducerKind::REQUIRED
|
||||
.iter()
|
||||
.map(|producer| producer_name(*producer))
|
||||
.collect::<BTreeSet<_>>();
|
||||
let envelope = SegmentObservationEnvelope {
|
||||
source,
|
||||
bucket_incarnation: incarnation,
|
||||
key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
baseline_scan_plan_digest: baseline,
|
||||
process_epoch: "epoch-a",
|
||||
generation_start: 11,
|
||||
generation_end: 13,
|
||||
restart_gap: false,
|
||||
overflow: false,
|
||||
producers,
|
||||
keys: &["hot/one", "hot/two", "archive/delete-marker"],
|
||||
};
|
||||
let proof = SegmentObservationProof {
|
||||
source,
|
||||
bucket_incarnation: incarnation,
|
||||
key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
baseline_scan_plan_digest: baseline,
|
||||
process_epoch: "epoch-a",
|
||||
durable_producer_identity: true,
|
||||
invalidation_domain: SegmentInvalidationDomain::LocalSingleSet,
|
||||
distributed_ec_invalidation: false,
|
||||
cold_zero_walk_oracle: true,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
trusted_fixture_proposal(&envelope, &proof),
|
||||
Ok(BTreeSet::from(["archive".to_string(), "hot".to_string()]))
|
||||
);
|
||||
|
||||
let mut wrong_source = envelope.clone();
|
||||
wrong_source.source = DataUsageCacheSource::new(2, 4);
|
||||
assert_eq!(trusted_fixture_proposal(&wrong_source, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut missing_incarnation = envelope.clone();
|
||||
missing_incarnation.bucket_incarnation = uuid::Uuid::nil();
|
||||
assert_eq!(trusted_fixture_proposal(&missing_incarnation, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut wrong_key_format = envelope.clone();
|
||||
wrong_key_format.key_format = DATA_USAGE_CACHE_KEY_FORMAT.saturating_add(1);
|
||||
assert_eq!(trusted_fixture_proposal(&wrong_key_format, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut wrong_baseline = envelope.clone();
|
||||
wrong_baseline.baseline_scan_plan_digest = DataUsageScanPlanDigest([8; 32]);
|
||||
assert_eq!(trusted_fixture_proposal(&wrong_baseline, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut wrong_epoch = envelope.clone();
|
||||
wrong_epoch.process_epoch = "epoch-b";
|
||||
assert_eq!(trusted_fixture_proposal(&wrong_epoch, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut no_durable_identity = proof.clone();
|
||||
no_durable_identity.durable_producer_identity = false;
|
||||
assert_eq!(trusted_fixture_proposal(&envelope, &no_durable_identity), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut restart_gap = envelope.clone();
|
||||
restart_gap.restart_gap = true;
|
||||
assert_eq!(trusted_fixture_proposal(&restart_gap, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut overflow = envelope.clone();
|
||||
overflow.overflow = true;
|
||||
assert_eq!(trusted_fixture_proposal(&overflow, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut generation_gap = envelope.clone();
|
||||
generation_gap.generation_end = generation_gap.generation_start - 1;
|
||||
assert_eq!(trusted_fixture_proposal(&generation_gap, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut missing_producer = envelope.clone();
|
||||
missing_producer.producers.remove(producer_name(ProducerKind::Replication));
|
||||
assert_eq!(trusted_fixture_proposal(&missing_producer, &proof), Err(ProposalError::InvalidKey));
|
||||
|
||||
let mut missing_zero_walk_oracle = proof.clone();
|
||||
missing_zero_walk_oracle.cold_zero_walk_oracle = false;
|
||||
assert_eq!(
|
||||
trusted_fixture_proposal(&envelope, &missing_zero_walk_oracle),
|
||||
Err(ProposalError::InvalidKey)
|
||||
);
|
||||
|
||||
let mut distributed_without_invalidation = proof.clone();
|
||||
distributed_without_invalidation.invalidation_domain = SegmentInvalidationDomain::DistributedEc;
|
||||
assert_eq!(
|
||||
trusted_fixture_proposal(&envelope, &distributed_without_invalidation),
|
||||
Err(ProposalError::InvalidKey)
|
||||
);
|
||||
|
||||
let mut distributed_with_invalidation = distributed_without_invalidation;
|
||||
distributed_with_invalidation.distributed_ec_invalidation = true;
|
||||
assert_eq!(
|
||||
trusted_fixture_proposal(&envelope, &distributed_with_invalidation),
|
||||
Ok(BTreeSet::from(["archive".to_string(), "hot".to_string()]))
|
||||
);
|
||||
}
|
||||
|
||||
fn cache_value(cache: &DataUsageCache) -> serde_json::Value {
|
||||
let mut value = serde_json::to_value(cache).expect("serialize the entire cache");
|
||||
// Children are a HashSet: canonicalize only that unordered field, without
|
||||
@@ -181,10 +379,6 @@ async fn walk_and_save(observe: bool) -> (Vec<String>, serde_json::Value) {
|
||||
2,
|
||||
"the two non-proposed segments must still be walked"
|
||||
);
|
||||
eprintln!(
|
||||
"segment fixture: proposed={proposed:?}, actual_segments={walked_segments:?}, actual_walk_callbacks={}, production_producer_coverage=unverified",
|
||||
paths.len()
|
||||
);
|
||||
} else {
|
||||
assert!(proposed_walked.lock().expect("read disabled observations").is_empty());
|
||||
}
|
||||
|
||||
@@ -29,6 +29,9 @@ pub(in crate::scanner_folder) fn observe_raw_entry(dir: &str, name: &std::ffi::O
|
||||
let relative_dir = Path::new(dir)
|
||||
.strip_prefix(&observation.root)
|
||||
.unwrap_or_else(|_| Path::new(""));
|
||||
if relative_dir.components().count() != 1 {
|
||||
return;
|
||||
}
|
||||
let entry_marker = relative_dir.join(name).to_string_lossy().to_string();
|
||||
observation.first_entry.get_or_insert_with(|| entry_marker.clone());
|
||||
observation.last_entry = Some(entry_marker);
|
||||
@@ -130,7 +133,13 @@ async fn round(request: &Request) -> serde_json::Value {
|
||||
.await
|
||||
.expect("open synthetic disk in this process");
|
||||
let parent = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, Default::default());
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
crate::scanner_budget::ScannerCycleBudgetConfig {
|
||||
max_objects: Some(request.raw_entry_budget),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let _observation_guard = install_raw_entry_budget(disk.path(), request.raw_entry_budget);
|
||||
let result = scan_data_folder(
|
||||
budget.token(),
|
||||
@@ -157,6 +166,16 @@ async fn round(request: &Request) -> serde_json::Value {
|
||||
let reloaded = DataUsageCache::unmarshal(&read_bounded(&cache_path).await).expect("reload returned cache codec");
|
||||
let retained = reloaded.checked_flatten("bucket").expect("reloaded bucket root");
|
||||
let scanned = returned.checked_flatten("bucket").expect("returned bucket root");
|
||||
let raw_page_index_committed_entries = reloaded
|
||||
.validated_raw_enumeration_page_index()
|
||||
.and_then(|index| index.committed_entries().ok())
|
||||
.map(|entries| entries.len())
|
||||
.unwrap_or(0);
|
||||
let raw_page_index_indexed_entries = reloaded
|
||||
.validated_raw_enumeration_page_index()
|
||||
.and_then(|index| index.indexed_entries().ok())
|
||||
.map(|entries| entries.len())
|
||||
.unwrap_or(0);
|
||||
assert_eq!(
|
||||
(retained.objects, retained.versions, retained.size),
|
||||
(scanned.objects, scanned.versions, scanned.size)
|
||||
@@ -169,6 +188,8 @@ async fn round(request: &Request) -> serde_json::Value {
|
||||
"objects_expected": request.objects, "raw_entry_budget": request.raw_entry_budget,
|
||||
"raw_entries": observation.entries, "raw_name_bytes": observation.name_bytes,
|
||||
"raw_first_entry": observation.first_entry, "raw_last_entry": observation.last_entry,
|
||||
"raw_page_index_committed_entries": raw_page_index_committed_entries,
|
||||
"raw_page_index_indexed_entries": raw_page_index_indexed_entries,
|
||||
"objects_processed": budget.progress().0,
|
||||
"objects_before": before, "objects_retained": retained.objects,
|
||||
"versions_retained": retained.versions, "bytes_retained": retained.size,
|
||||
@@ -194,16 +215,16 @@ async fn enumeration_restart_worker() {
|
||||
let temp = tempfile::tempdir().expect("healthy fixture directory");
|
||||
let report = round(&Request {
|
||||
workspace: temp.path().to_path_buf(),
|
||||
objects: 4,
|
||||
objects: 8,
|
||||
raw_entry_budget: 16,
|
||||
round: 0,
|
||||
})
|
||||
.await;
|
||||
assert_eq!(report["outcome"], "complete");
|
||||
assert_eq!(report["snapshot_complete"], true);
|
||||
assert_eq!(report["objects_retained"], 4);
|
||||
assert_eq!(report["versions_retained"], 4);
|
||||
assert_eq!(report["bytes_retained"], 4);
|
||||
assert_eq!(report["objects_retained"], 8);
|
||||
assert_eq!(report["versions_retained"], 8);
|
||||
assert_eq!(report["bytes_retained"], 8);
|
||||
assert!(report["raw_entries"].as_u64().expect("observed entries") >= 8, "{report}");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -741,6 +741,11 @@ pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Resu
|
||||
name: cache.info.name.clone(),
|
||||
parent: DATA_USAGE_ROOT.to_string(),
|
||||
entry,
|
||||
bucket_incarnation: cache
|
||||
.info
|
||||
.scan_identity
|
||||
.map(|identity| identity.bucket_incarnation)
|
||||
.filter(|incarnation| !incarnation.is_nil()),
|
||||
tier_registry_generation: cache.info.tier_registry_generation,
|
||||
})
|
||||
}
|
||||
@@ -752,6 +757,14 @@ fn apply_bucket_result_to_cache(cache: &mut DataUsageCache, result: DataUsageEnt
|
||||
// forces the caller to re-account it under one frozen registry.
|
||||
return false;
|
||||
}
|
||||
match result.bucket_incarnation {
|
||||
Some(incarnation) if !incarnation.is_nil() => {
|
||||
cache.info.scan_bucket_incarnations.insert(result.name.clone(), incarnation);
|
||||
}
|
||||
_ => {
|
||||
cache.info.scan_bucket_incarnations.remove(&result.name);
|
||||
}
|
||||
}
|
||||
cache.replace(&result.name, &result.parent, result.entry);
|
||||
cache.info.last_update = Some(update_time);
|
||||
true
|
||||
|
||||
@@ -34,17 +34,12 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
all_buckets: &[BucketInfo],
|
||||
scope: &ScannerBucketScanScope,
|
||||
generation: ScannerSetCacheGeneration,
|
||||
current_bucket_incarnations: Option<&HashMap<String, uuid::Uuid>>,
|
||||
) -> Option<PreparedScopedSetScan> {
|
||||
let (Some(selected_buckets), Some(baseline_scan_plan_digest)) = (&scope.selected_buckets, scope.baseline_scan_plan_digest)
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
// The existing cache does not bind each bucket to a durable incarnation.
|
||||
// Listing creation times can come from volume metadata, so even Some(time)
|
||||
// cannot prove that an unselected same-name bucket is the cached bucket.
|
||||
if all_buckets.iter().any(|bucket| !selected_buckets.contains(&bucket.name)) {
|
||||
return None;
|
||||
}
|
||||
if selected_buckets.is_empty()
|
||||
|| !old_cache.info.snapshot_complete
|
||||
|| old_cache.info.last_update.is_none()
|
||||
@@ -56,6 +51,7 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
|| old_cache.info.scan_plan_digest != Some(baseline_scan_plan_digest)
|
||||
|| old_cache.info.cache_key_format != DATA_USAGE_CACHE_KEY_FORMAT
|
||||
|| !old_cache.has_complete_root_inventory(&old_cache.find(DATA_USAGE_ROOT)?.children)
|
||||
|| !unselected_bucket_incarnations_match(old_cache, all_buckets, selected_buckets, current_bucket_incarnations)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
@@ -75,6 +71,7 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
lkg_last_update: old_cache.info.last_update,
|
||||
lkg_leader_epoch: Some(old_cache.info.leader_epoch),
|
||||
lkg_scan_plan_digest: old_cache.info.scan_plan_digest,
|
||||
scan_bucket_incarnations: old_cache.info.scan_bucket_incarnations.clone(),
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
@@ -85,7 +82,15 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
if !current_bucket_names.insert(bucket.name.as_str()) {
|
||||
return None;
|
||||
}
|
||||
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
if selected_buckets.contains(&bucket.name) {
|
||||
cache.replace(&bucket.name, DATA_USAGE_ROOT, DataUsageEntry::default());
|
||||
} else {
|
||||
cache.copy_with_children(
|
||||
old_cache,
|
||||
&rustfs_data_usage::hash_path(&bucket.name),
|
||||
&Some(rustfs_data_usage::hash_path(DATA_USAGE_ROOT)),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Some(PreparedScopedSetScan {
|
||||
@@ -98,6 +103,47 @@ pub(super) fn prepare_scoped_set_scan(
|
||||
})
|
||||
}
|
||||
|
||||
fn unselected_bucket_incarnations_match(
|
||||
old_cache: &DataUsageCache,
|
||||
all_buckets: &[BucketInfo],
|
||||
selected_buckets: &HashSet<String>,
|
||||
current_bucket_incarnations: Option<&HashMap<String, uuid::Uuid>>,
|
||||
) -> bool {
|
||||
let Some(current_bucket_incarnations) = current_bucket_incarnations else {
|
||||
return all_buckets.iter().all(|bucket| selected_buckets.contains(&bucket.name));
|
||||
};
|
||||
all_buckets
|
||||
.iter()
|
||||
.filter(|bucket| !selected_buckets.contains(&bucket.name))
|
||||
.all(|bucket| {
|
||||
let Some(current) = current_bucket_incarnations
|
||||
.get(&bucket.name)
|
||||
.filter(|incarnation| !incarnation.is_nil())
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
old_cache
|
||||
.info
|
||||
.scan_bucket_incarnations
|
||||
.get(&bucket.name)
|
||||
.filter(|cached| !cached.is_nil())
|
||||
== Some(current)
|
||||
})
|
||||
}
|
||||
|
||||
async fn scanner_current_bucket_incarnations(set: &SetDisks, all_buckets: &[BucketInfo]) -> Option<HashMap<String, uuid::Uuid>> {
|
||||
let mut incarnations = HashMap::with_capacity(all_buckets.len());
|
||||
for bucket in all_buckets {
|
||||
let Ok(incarnation) = set.bucket_incarnation_id_from_disk(&bucket.name).await else {
|
||||
return None;
|
||||
};
|
||||
if incarnation.is_nil() || incarnations.insert(bucket.name.clone(), incarnation).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Some(incarnations)
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl ScannerIOCache for SetDisks {
|
||||
#[tracing::instrument(skip(self, budget, scan_plan, updates))]
|
||||
@@ -158,6 +204,7 @@ impl ScannerIOCache for SetDisks {
|
||||
None
|
||||
}
|
||||
};
|
||||
let current_bucket_incarnations = scanner_current_bucket_incarnations(self.as_ref(), &all_buckets).await;
|
||||
let scoped_scan = prepare_scoped_set_scan(
|
||||
&old_cache,
|
||||
&buckets,
|
||||
@@ -170,6 +217,7 @@ impl ScannerIOCache for SetDisks {
|
||||
source,
|
||||
scan_plan_digest,
|
||||
},
|
||||
current_bucket_incarnations.as_ref(),
|
||||
);
|
||||
let mut scoped_cache = scoped_scan.map(|mut prepared| {
|
||||
buckets = prepared.buckets;
|
||||
@@ -191,6 +239,7 @@ impl ScannerIOCache for SetDisks {
|
||||
scan_plan_digest: Some(scan_plan_digest),
|
||||
scan_coverage_digest: Some(bucket_coverage_digest),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
scan_bucket_incarnations: current_bucket_incarnations.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
@@ -486,6 +535,7 @@ impl ScannerIOCache for SetDisks {
|
||||
lkg_last_update: old_cache.info.lkg_last_update,
|
||||
lkg_leader_epoch: old_cache.info.lkg_leader_epoch,
|
||||
lkg_scan_plan_digest: old_cache.info.lkg_scan_plan_digest,
|
||||
scan_bucket_incarnations: current_bucket_incarnations.clone().unwrap_or_default(),
|
||||
..Default::default()
|
||||
},
|
||||
cache: HashMap::new(),
|
||||
@@ -574,7 +624,6 @@ impl ScannerIOCache for SetDisks {
|
||||
let budget_clone = budget.clone();
|
||||
let store_clone_clone = self.clone();
|
||||
let bucket_result_tx_clone = bucket_result_tx.clone();
|
||||
let disk_clone = disk.clone();
|
||||
let set_disk_inventory_clone = set_disk_inventory.clone();
|
||||
let disk_scan_semaphore_clone = disk_scan_semaphore.clone();
|
||||
let queued_disk_bucket_scans_clone = queued_disk_bucket_scans.clone();
|
||||
@@ -622,10 +671,7 @@ impl ScannerIOCache for SetDisks {
|
||||
BucketWorkGuard::new(remaining_bucket_work_clone.clone(), bucket_work_complete_clone.clone());
|
||||
// Prefix hints are process-local. Never hand one to a
|
||||
// remote or legacy-coordinator disk path.
|
||||
let prefix_scan_scope = disk_clone
|
||||
.is_local()
|
||||
.then(|| scope_clone.prefix_scope_for(&bucket.name))
|
||||
.flatten();
|
||||
let prefix_scan_scope = disk.is_local().then(|| scope_clone.prefix_scope_for(&bucket.name)).flatten();
|
||||
|
||||
metrics::histogram!(
|
||||
METRIC_SCANNER_DISK_SCAN_WAIT_SECONDS,
|
||||
@@ -680,7 +726,7 @@ impl ScannerIOCache for SetDisks {
|
||||
};
|
||||
remote_session_sequence = next_sequence;
|
||||
let remote_outcome = crate::remote_scanner::scan_remote_bucket(
|
||||
&disk_clone,
|
||||
&disk,
|
||||
ctx_clone.clone(),
|
||||
budget_clone.clone(),
|
||||
crate::remote_scanner::RemoteScannerScanSpec {
|
||||
@@ -813,8 +859,8 @@ impl ScannerIOCache for SetDisks {
|
||||
continue;
|
||||
}
|
||||
|
||||
let _local_admission = if disk_clone.is_local() {
|
||||
match crate::remote_scanner::try_admit_remote_scanner(&disk_clone) {
|
||||
let _local_admission = if disk.is_local() {
|
||||
match crate::remote_scanner::try_admit_remote_scanner(&disk) {
|
||||
Ok(admission) => Some(admission),
|
||||
Err(e) => {
|
||||
if requeue_bucket_work(&bucket_tx_clone, &bucket, &mut work_guard).await {
|
||||
@@ -1055,7 +1101,7 @@ impl ScannerIOCache for SetDisks {
|
||||
let before = cache.info.last_update;
|
||||
|
||||
let scan_ctx = ctx_clone.child_token();
|
||||
let scan = disk_clone.clone().nsscanner_disk(
|
||||
let scan = disk.clone().nsscanner_disk(
|
||||
scan_ctx.clone(),
|
||||
budget_clone.clone(),
|
||||
set_disk_inventory_clone.as_ref().clone(),
|
||||
|
||||
@@ -1232,6 +1232,19 @@ fn complete_set_usage_cache(buckets: &[(&str, usize)], scan_plan_digest: DataUsa
|
||||
cache
|
||||
}
|
||||
|
||||
fn test_bucket_incarnations(buckets: &[&str]) -> HashMap<String, Uuid> {
|
||||
buckets
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(index, bucket)| {
|
||||
(
|
||||
(*bucket).to_string(),
|
||||
Uuid::from_u128(u128::try_from(index).expect("test index should fit") + 1),
|
||||
)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers() {
|
||||
@@ -1651,6 +1664,82 @@ fn scoped_scan_uses_only_locally_verified_prefix_hints() {
|
||||
assert!(distributed_scope.prefix_scope_for("photos").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn remote_dirty_usage_invalidates_local_prefix_hints_until_distributed_proof_exists() {
|
||||
let source = DataUsageCacheSource::new(1, 2);
|
||||
let expected_sources = HashSet::from([source]);
|
||||
let scan_plan_digest = DataUsageScanPlanDigest([6; 32]);
|
||||
let baseline = complete_usage_baseline(source, scan_plan_digest, 7, 11);
|
||||
let expected_peers = HashMap::from([(
|
||||
"node-a:9000".to_string(),
|
||||
ScannerPeerDirtyUsageExpectation {
|
||||
instance_id: "instance-a".to_string(),
|
||||
generation: 7,
|
||||
pending: true,
|
||||
},
|
||||
)]);
|
||||
let remote_dirty_usage = verified_remote_dirty_usage(
|
||||
&expected_peers,
|
||||
vec![(
|
||||
"node-a:9000".to_string(),
|
||||
peer_dirty_usage_snapshot("instance-a", 7, true, &[("photos", 7)]),
|
||||
)],
|
||||
)
|
||||
.expect("fixture remote dirty usage should verify at bucket granularity");
|
||||
let dirty_scopes = HashMap::from([(
|
||||
"photos".to_string(),
|
||||
DirtyUsageBucketScope::TopLevelEntries(HashSet::from(["2026".to_string()])),
|
||||
)]);
|
||||
let locally_scoped = scoped_scan_scope_from_dirty_buckets(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string()]),
|
||||
Some(&dirty_scopes),
|
||||
true,
|
||||
&[bucket_info("photos")],
|
||||
ScannerCacheBaselineProof {
|
||||
authoritative_data: Some(&baseline),
|
||||
observed_candidate_data: None,
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
},
|
||||
);
|
||||
assert!(
|
||||
locally_scoped.prefix_scope_for("photos").is_some(),
|
||||
"local-only evidence may narrow to a direct child segment"
|
||||
);
|
||||
|
||||
let distributed = resolve_remote_dirty_usage_scope(
|
||||
ScannerBucketScanScope::default(),
|
||||
HashSet::from(["photos".to_string()]),
|
||||
remote_dirty_usage,
|
||||
&[bucket_info("photos")],
|
||||
ScannerCacheBaselineProof {
|
||||
authoritative_data: Some(&baseline),
|
||||
observed_candidate_data: None,
|
||||
expected_sources: &expected_sources,
|
||||
leader_epoch: 11,
|
||||
want_cycle: 8,
|
||||
scan_plan_digest,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
distributed
|
||||
.scope
|
||||
.selected_buckets
|
||||
.as_deref()
|
||||
.expect("distributed invalidation still selects the dirty bucket"),
|
||||
&HashSet::from(["photos".to_string()])
|
||||
);
|
||||
assert!(
|
||||
distributed.scope.prefix_scope_for("photos").is_none(),
|
||||
"peer dirty state is not a distributed segment invalidation proof"
|
||||
);
|
||||
assert_eq!(distributed.remote_dirty_usage_acknowledgements.len(), 1);
|
||||
}
|
||||
|
||||
fn peer_dirty_usage_snapshot(
|
||||
instance_id: &str,
|
||||
generation: u64,
|
||||
@@ -2006,6 +2095,7 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
|
||||
source: DataUsageCacheSource::new(1, 2),
|
||||
scan_plan_digest: current_digest,
|
||||
},
|
||||
None,
|
||||
)
|
||||
.expect("complete matching set cache should support a scoped scan");
|
||||
|
||||
@@ -2029,6 +2119,57 @@ fn scoped_set_scan_rebuilds_selected_buckets_and_drops_deleted_buckets() {
|
||||
assert_eq!(prepared.cache.info.lkg_scan_plan_digest, Some(baseline_digest));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_set_scan_reuses_unselected_buckets_with_matching_incarnations() {
|
||||
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
|
||||
let current_digest = DataUsageScanPlanDigest([2; 32]);
|
||||
let mut old_cache = complete_set_usage_cache(&[("stable", 10), ("dirty", 20)], baseline_digest);
|
||||
old_cache.replace(
|
||||
"stable/prefix",
|
||||
"stable",
|
||||
DataUsageEntry {
|
||||
size: 5,
|
||||
objects: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
old_cache.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "dirty"]);
|
||||
let current_incarnations = old_cache.info.scan_bucket_incarnations.clone();
|
||||
let all_buckets = vec![
|
||||
bucket_info_with_created_time("stable"),
|
||||
bucket_info_with_created_time("dirty"),
|
||||
];
|
||||
|
||||
let prepared = prepare_scoped_set_scan(
|
||||
&old_cache,
|
||||
&all_buckets,
|
||||
&all_buckets,
|
||||
&ScannerBucketScanScope {
|
||||
selected_buckets: Some(Arc::new(HashSet::from(["dirty".to_string()]))),
|
||||
selected_bucket_prefixes: None,
|
||||
baseline_scan_plan_digest: Some(baseline_digest),
|
||||
},
|
||||
ScannerSetCacheGeneration {
|
||||
want_cycle: 8,
|
||||
leader_epoch: 11,
|
||||
tier_registry_generation: 13,
|
||||
source: DataUsageCacheSource::new(1, 2),
|
||||
scan_plan_digest: current_digest,
|
||||
},
|
||||
Some(¤t_incarnations),
|
||||
)
|
||||
.expect("matching bucket incarnations should authorize cold bucket reuse");
|
||||
|
||||
assert_eq!(prepared.buckets.iter().map(|bucket| bucket.name.as_str()).collect::<Vec<_>>(), ["dirty"]);
|
||||
let stable = prepared
|
||||
.cache
|
||||
.checked_flatten("stable")
|
||||
.expect("unselected stable bucket should be copied with children");
|
||||
assert_eq!((stable.size, stable.objects), (15, 2));
|
||||
assert_eq!(prepared.cache.find("dirty").map(|entry| (entry.size, entry.objects)), Some((0, 0)));
|
||||
assert_eq!(prepared.cache.info.scan_bucket_incarnations, current_incarnations);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scoped_set_scan_rejects_unbound_bucket_incarnations() {
|
||||
let baseline_digest = DataUsageScanPlanDigest([1; 32]);
|
||||
@@ -2054,10 +2195,22 @@ fn scoped_set_scan_rejects_unbound_bucket_incarnations() {
|
||||
stable.created = created;
|
||||
let buckets = vec![stable, bucket_info_with_created_time("dirty")];
|
||||
assert!(
|
||||
prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation).is_none(),
|
||||
prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation, None).is_none(),
|
||||
"missing identity, volume timestamps and same-name recreation must all rebuild"
|
||||
);
|
||||
}
|
||||
let mut mismatched = test_bucket_incarnations(&["stable", "dirty"]);
|
||||
mismatched.insert("stable".to_string(), Uuid::from_u128(99));
|
||||
let mut old_cache = old_cache;
|
||||
old_cache.info.scan_bucket_incarnations = test_bucket_incarnations(&["stable", "dirty"]);
|
||||
let buckets = vec![
|
||||
bucket_info_with_created_time("stable"),
|
||||
bucket_info_with_created_time("dirty"),
|
||||
];
|
||||
assert!(
|
||||
prepare_scoped_set_scan(&old_cache, &buckets, &buckets, &scope, generation, Some(&mismatched)).is_none(),
|
||||
"a same-name unselected bucket with a different incarnation must rebuild"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -2083,6 +2236,7 @@ fn scoped_set_scan_falls_back_when_an_unselected_bucket_has_no_baseline() {
|
||||
source: DataUsageCacheSource::new(1, 2),
|
||||
scan_plan_digest: DataUsageScanPlanDigest([4; 32]),
|
||||
},
|
||||
Some(&test_bucket_incarnations(&["stable", "new"])),
|
||||
)
|
||||
.is_none()
|
||||
);
|
||||
@@ -2107,19 +2261,19 @@ fn scoped_set_scan_requires_an_exact_complete_baseline() {
|
||||
|
||||
let mut incomplete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
|
||||
incomplete.info.snapshot_complete = false;
|
||||
assert!(prepare_scoped_set_scan(&incomplete, &all_buckets, &all_buckets, &scope, generation).is_none());
|
||||
assert!(prepare_scoped_set_scan(&incomplete, &all_buckets, &all_buckets, &scope, generation, None).is_none());
|
||||
|
||||
let mut not_durable = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
|
||||
not_durable.info.last_update = None;
|
||||
assert!(prepare_scoped_set_scan(¬_durable, &all_buckets, &all_buckets, &scope, generation).is_none());
|
||||
assert!(prepare_scoped_set_scan(¬_durable, &all_buckets, &all_buckets, &scope, generation, None).is_none());
|
||||
|
||||
let mut unscoped_usage = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
|
||||
unscoped_usage.cache.get_mut(DATA_USAGE_ROOT).expect("set root").objects = 1;
|
||||
assert!(prepare_scoped_set_scan(&unscoped_usage, &all_buckets, &all_buckets, &scope, generation).is_none());
|
||||
assert!(prepare_scoped_set_scan(&unscoped_usage, &all_buckets, &all_buckets, &scope, generation, None).is_none());
|
||||
|
||||
let mut wrong_digest = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
|
||||
wrong_digest.info.scan_plan_digest = Some(DataUsageScanPlanDigest([7; 32]));
|
||||
assert!(prepare_scoped_set_scan(&wrong_digest, &all_buckets, &all_buckets, &scope, generation).is_none());
|
||||
assert!(prepare_scoped_set_scan(&wrong_digest, &all_buckets, &all_buckets, &scope, generation, None).is_none());
|
||||
|
||||
let empty_scope = ScannerBucketScanScope {
|
||||
selected_buckets: Some(Arc::new(HashSet::new())),
|
||||
@@ -2127,18 +2281,18 @@ fn scoped_set_scan_requires_an_exact_complete_baseline() {
|
||||
baseline_scan_plan_digest: Some(baseline_digest),
|
||||
};
|
||||
let complete = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
|
||||
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &empty_scope, generation).is_none());
|
||||
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &scope, generation).is_some());
|
||||
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &empty_scope, generation, None).is_none());
|
||||
assert!(prepare_scoped_set_scan(&complete, &all_buckets, &all_buckets, &scope, generation, None).is_some());
|
||||
|
||||
let unidentified_buckets = vec![bucket_info("dirty")];
|
||||
assert!(
|
||||
prepare_scoped_set_scan(&complete, &unidentified_buckets, &unidentified_buckets, &scope, generation).is_some(),
|
||||
prepare_scoped_set_scan(&complete, &unidentified_buckets, &unidentified_buckets, &scope, generation, None).is_some(),
|
||||
"fully selected buckets are rebuilt without reusing an unproven incarnation"
|
||||
);
|
||||
|
||||
let mut future_cache = complete_set_usage_cache(&[("dirty", 10)], baseline_digest);
|
||||
future_cache.info.next_cycle = generation.want_cycle.saturating_add(1);
|
||||
assert!(prepare_scoped_set_scan(&future_cache, &all_buckets, &all_buckets, &scope, generation).is_none());
|
||||
assert!(prepare_scoped_set_scan(&future_cache, &all_buckets, &all_buckets, &scope, generation, None).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3007,6 +3161,7 @@ fn apply_bucket_result_to_cache_updates_bucket_entry() {
|
||||
objects: 2,
|
||||
..Default::default()
|
||||
},
|
||||
bucket_incarnation: Some(Uuid::from_u128(7)),
|
||||
tier_registry_generation: None,
|
||||
},
|
||||
update_time,
|
||||
@@ -3016,6 +3171,7 @@ fn apply_bucket_result_to_cache_updates_bucket_entry() {
|
||||
let entry = cache.find("bucket").expect("bucket entry should remain present");
|
||||
assert_eq!(entry.size, 10);
|
||||
assert_eq!(entry.objects, 2);
|
||||
assert_eq!(cache.info.scan_bucket_incarnations.get("bucket"), Some(&Uuid::from_u128(7)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -3046,6 +3202,7 @@ fn apply_bucket_result_to_cache_rejects_a_different_tier_generation() {
|
||||
size: 11,
|
||||
..Default::default()
|
||||
},
|
||||
bucket_incarnation: Some(Uuid::from_u128(7)),
|
||||
tier_registry_generation: Some(8),
|
||||
},
|
||||
SystemTime::now(),
|
||||
|
||||
@@ -82,8 +82,8 @@ async fn persist_baseline(store: &Arc<ECStore>, baseline: &DataUsageInfo) {
|
||||
.expect("fixture baseline should persist");
|
||||
}
|
||||
|
||||
// Every invocation uses the production default scope. The expected walker set
|
||||
// comes from storage's per-source inventory, not the resolver's selected names.
|
||||
// Every invocation uses the production default scope. Once durable bucket
|
||||
// incarnations are present, the expected walker set follows the resolved scope.
|
||||
async fn run_entry(store: &Arc<ECStore>, cycle: u64, selected: Option<&str>, expect_walks: bool) -> DataUsageInfo {
|
||||
let drives = drive_identities(store).await;
|
||||
let inventory = store
|
||||
@@ -99,6 +99,7 @@ async fn run_entry(store: &Arc<ECStore>, cycle: u64, selected: Option<&str>, exp
|
||||
let source = DataUsageCacheSource::new(set.pool_index, set.set_index);
|
||||
set.buckets.into_iter().map(move |bucket| ((source, bucket.name), 1_u64))
|
||||
})
|
||||
.filter(|((_, bucket), _)| selected.is_none_or(|selected| bucket == selected))
|
||||
.collect::<HashMap<_, _>>()
|
||||
} else {
|
||||
HashMap::new()
|
||||
@@ -201,8 +202,8 @@ async fn scoped_entry_fallback_distinguishes_planned_scope_from_real_cold_walks(
|
||||
let baseline = run_entry(&store, 1, None, true).await;
|
||||
persist_baseline(&store, &baseline).await;
|
||||
|
||||
// A same-intent, same-cycle Current cache is a retry, not proof that a
|
||||
// later cycle may reuse unselected buckets without durable incarnation.
|
||||
// Same-cycle Current remains a retry. The later cycle may skip the cold
|
||||
// bucket only after the prior complete set cache has durable incarnations.
|
||||
run_entry(&store, 1, Some(&hot), false).await;
|
||||
let usage = run_entry(&store, 2, Some(&hot), true).await;
|
||||
assert_eq!(usage.buckets_usage[&hot].objects_count, 1);
|
||||
|
||||
@@ -2552,7 +2552,7 @@ mod serial_tests {
|
||||
.expect("Failed to upload multipart part");
|
||||
completed.push(CompletePart {
|
||||
part_num: idx + 1,
|
||||
etag: part.etag.clone(),
|
||||
etag: part.etag,
|
||||
..Default::default()
|
||||
});
|
||||
offset += part_size;
|
||||
|
||||
@@ -43,6 +43,16 @@ pub const SUFFIX_FORCE_DELETE: &str = "force-delete";
|
||||
pub const SUFFIX_INCLUDE_DELETED: &str = "include-deleted";
|
||||
pub const SUFFIX_REPLICATION_RESET_STATUS: &str = "replication-reset-status";
|
||||
pub const SUFFIX_REPLICATION_ACTUAL_OBJECT_SIZE: &str = "replication-actual-object-size";
|
||||
/// SSE-C ciphertext passthrough of an object the source stored compressed:
|
||||
/// the stored compression scheme travels under this name so the replica
|
||||
/// decompresses after decrypting (backlog#2363).
|
||||
pub const SUFFIX_REPLICATION_COMPRESSION: &str = "replication-compression";
|
||||
/// Plaintext size of a compressed passthrough object (backlog#2363).
|
||||
pub const SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE: &str = "replication-compression-actual-size";
|
||||
/// Plaintext length of one passthrough multipart part, sent on UploadPart so
|
||||
/// the replica records the logical part size and checks the 5 MiB minimum
|
||||
/// against it rather than against the stored bytes (backlog#2363).
|
||||
pub const SUFFIX_REPLICATION_PART_ACTUAL_SIZE: &str = "replication-part-actual-size";
|
||||
pub const SUFFIX_SOURCE_VERSION_ID: &str = "source-version-id";
|
||||
pub const SUFFIX_SOURCE_MTIME: &str = "source-mtime";
|
||||
pub const SUFFIX_SOURCE_ETAG: &str = "source-etag";
|
||||
|
||||
@@ -25,6 +25,8 @@ The [scanner checkpoint fixture](scanner-checkpoint-fixture.md) diagnoses retain
|
||||
|
||||
The [scanner cache cost profile](scanner-cache-cost.md) separates clone, subtree copy, encoding, and counted save costs without changing production cache behavior.
|
||||
|
||||
The [Pool layout compatibility reference](pool-layout-compatibility.md) defines the topology and EC regression matrix for single-drive, single-node multi-drive, and multi-node expansion pools.
|
||||
|
||||
## Naming conventions
|
||||
|
||||
### Reserved test-name substrings (migration gate)
|
||||
|
||||
@@ -0,0 +1,107 @@
|
||||
# Pool Layout Compatibility and Regression Tests
|
||||
|
||||
**Use this when:** configuring `RUSTFS_VOLUMES` for expansion, investigating issue #6186, or changing pool admission and its regression tests.
|
||||
**Source of truth:** `DisksLayout::from_volumes` and `get_set_indexes` in `crates/ecstore/src/layout/disks_layout.rs`, `EndpointServerPools::create_server_endpoints` in `crates/ecstore/src/layout/endpoints.rs`, startup format validation in `crates/ecstore/src/store/init_format.rs`, and `lookup_config_for_pools` in `crates/ecstore/src/config/storageclass.rs`. Geometry and parity invariants are owned by [erasure-coding.md](../architecture/erasure-coding.md).
|
||||
|
||||
## Notice: count drives, not just nodes
|
||||
|
||||
An erasure pool requires at least two drive endpoints. There is no additional admission rule requiring two nodes per pool or two drives per node. A single-node multi-drive pool and a multi-node pool with one drive per node may both be valid.
|
||||
|
||||
For command-line / `RUSTFS_VOLUMES` expansion:
|
||||
|
||||
- If any volume argument contains an ellipsis expression, each argument describes a separate pool and must contain an ellipsis expression. Each pool must expand to at least two distinct drive endpoints and form a valid set layout.
|
||||
- A singleton range such as `http://node{3...3}:9000/data` still describes only one drive. It cannot bypass the minimum drive count.
|
||||
- Without ellipses, all explicit endpoints describe one pool, not one pool per endpoint.
|
||||
- A single local path such as `/data` remains a supported standalone single-drive deployment. A single URL endpoint is not a valid standalone single-drive endpoint, and a single-drive pool cannot be appended to a multi-pool deployment.
|
||||
- An initialized single-node single-drive (SNSD) deployment cannot expand in place by adding endpoints or pools. Create a new multi-drive deployment and migrate data through S3 instead. Increasing the capacity of its underlying filesystem is not a pool-topology expansion and adds no redundancy.
|
||||
- An existing multi-drive pool's drive count and set width are immutable. Preserve its original endpoints and `RUSTFS_ERASURE_SET_DRIVE_COUNT` setting, then append a new pool. Changing `/data{1...4}` to `/data{1...8}` resizes the old pool; appending `/other-data{1...4}` creates a new one.
|
||||
- Multi-drive sets contain 2 through 16 drives. A pool may contain multiple sets; 16 is not a limit on total drives in a pool. Set divisibility, automatic layout symmetry, duplicate endpoints, endpoint locality, physical-disk validation, and storage-class validation still apply.
|
||||
- An explicit storage-class parity must fit every pool's set width: `parity <= drives_per_set / 2`, with `STANDARD parity >= RRS parity`. Do not silently lower an explicit parity to admit a smaller pool.
|
||||
|
||||
Topology acceptance is not a high-availability guarantee. Losing the only host of a single-node pool loses access to every shard in that pool. With a two-drive set at `EC:1`, losing one drive leaves read quorum but not write quorum. Plan failure domains and quorum separately from admission.
|
||||
|
||||
These are valid four-drive-per-set topology examples, subject to the remaining startup checks:
|
||||
|
||||
```text
|
||||
# Two pools, each with four nodes and one drive per node.
|
||||
RUSTFS_VOLUMES="http://node{1...4}:9000/data http://node{5...8}:9000/data"
|
||||
|
||||
# A four-node pool plus a single-node, four-drive pool.
|
||||
RUSTFS_VOLUMES="http://node{1...4}:9000/data http://node5:9000/data{1...4}"
|
||||
```
|
||||
|
||||
## Rejection and recovery
|
||||
|
||||
Invalid single-drive expansion arguments fail during layout parsing. When a syntactically valid layout tries to resize an initialized pool, startup compares the stored format with the configured drive count and set width before initializing or migrating formats for that pool:
|
||||
|
||||
- `UnsupportedSnsdExpansion` explains that SNSD cannot expand in place and directs the operator to restore the single local path or migrate through S3 to a new deployment.
|
||||
- `PoolTopologyMismatch` reports stored and configured drive counts and set widths, and directs the operator to restore the original pool and append a new pool instead.
|
||||
|
||||
These are permanent startup errors, not retryable quorum failures. Rejection does not rewrite the affected pool's old format or initialize its new drives. Do not delete `format.json` to bypass it. This is a per-pool check, not an atomic, read-only preflight across every pool in the deployment.
|
||||
|
||||
A healthy format quorum remains authoritative; a foreign or malformed minority is quarantined as before. Without a quorum, an unambiguous, valid observed layout can identify a topology mismatch before the wait/retry path. Conflicting observed layouts are not treated as proof of expansion. Missing disks and transient network failures alone do not establish a topology change and retain their existing handling.
|
||||
|
||||
## MinIO comparison boundary
|
||||
|
||||
The reference is MinIO Community source at commit `7aac2a2c5b7c882e68c1ce017d8256be2feea27f`, not an unversioned claim about all MinIO products or releases:
|
||||
|
||||
- [Endpoint expansion](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/endpoint-ellipses.go): `mergeDisksLayoutFromArgs` requires ellipses on every expansion argument, and `getSetIndexes` rejects fewer than two endpoints.
|
||||
- [Endpoint admission](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/endpoint.go): `CreatePoolEndpoints` does not require two nodes per pool; its standalone single-drive special case requires a local path.
|
||||
- [Pool initialization](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/erasure-server-pool.go): `newErasureServerPools` checks a common parity against every pool.
|
||||
- [Storage preparation](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/prepare-storage.go) and [format validation](https://github.com/minio/minio/blob/7aac2a2c5b7c882e68c1ce017d8256be2feea27f/cmd/format-erasure.go): persisted drive counts and set widths must match the configured pool; format-layout errors are not ordinary quorum-wait conditions. RustFS keeps its existing majority/minority handling rather than adopting MinIO's all-format validation order.
|
||||
|
||||
The node/drive admission rules above match this baseline. This reference does not claim complete startup or storage-class equivalence:
|
||||
|
||||
- RustFS resolves automatic parity independently for each pool's set width. For widths `[4, 2]`, automatic STANDARD parity resolves to `[2, 1]`. MinIO uses a common parity, initially selected from the first pool when no value is configured, and rejects a later pool that cannot accommodate it. RustFS's existing automatic policy is not changed by these regression tests.
|
||||
- An explicit STANDARD `EC:2` rejects a two- or three-drive set in RustFS; `EC:1` fits both. Explicit configuration is shared, not a user-configurable per-pool override.
|
||||
- RustFS also checks symmetry when `RUSTFS_ERASURE_SET_DRIVE_COUNT` is explicitly set. The MinIO baseline skips automatic symmetry selection for an explicit set width. The topology tests below do not establish equivalence for every explicit-width layout.
|
||||
|
||||
## Regression matrix
|
||||
|
||||
Layout tests use symbolic endpoints and fixed set-count inputs. Startup tests use temporary local drives and the production format-loading path, comparing format bytes before and after rejection. They do not require production disks, DNS records, or a running MinIO server. Storage-class tests inject configuration directly rather than mutating the process environment.
|
||||
|
||||
| Scenario | Expected result | Regression guard |
|
||||
|---|---|---|
|
||||
| Standalone `/data` | One single-drive layout | `standalone_single_drive_path_remains_supported` |
|
||||
| Standalone single URL endpoint | Reject; single-drive mode requires a local path | `test_create_pool_endpoints` |
|
||||
| Two explicit URLs, no ellipses | One pool containing both drives | `explicit_endpoints_without_ellipses_form_one_pool` |
|
||||
| Two single-node pools, each with 2 or 4 drives | Two valid pools | `pool_expansion_accepts_single_node_multi_drive_pools` |
|
||||
| Four-node, one-drive-per-node pool mixed with a single-node, four-drive pool, in either order | Both pool boundaries and set widths preserved | `pool_expansion_accepts_single_node_multi_drive_pools` |
|
||||
| Two pools with 2, 3, or 4 nodes per pool and one drive per node | One set per pool; every drive retained in its pool | `pool_expansion_accepts_multi_node_single_drive_pools` |
|
||||
| Ellipsis pool mixed with a plain single-drive endpoint, in either order | Reject with the ellipsis requirement and minimum-drive notice | `pool_expansion_rejects_plain_single_drive_pool_with_notice` |
|
||||
| Singleton host or drive range, alone or before/after another pool | Reject with the minimum-drive notice and standalone-path guidance | `pool_expansion_rejects_singleton_ellipsis_pool_with_notice` |
|
||||
| Four drives on one node or four nodes, explicit set width 2 | Two two-drive sets | `explicit_set_size_counts_drives_not_nodes` |
|
||||
| Two-drive pool, explicit set width 4 | Reject and identify the requested set width | `undersized_pool_error_identifies_requested_set_size` |
|
||||
| Credentials in rejected plain or singleton pool endpoints | Errors do not echo secrets | `layout_errors_do_not_echo_url_credentials` |
|
||||
| Mixed single-node multi-drive / multi-node single-drive pools through endpoint resolution | Distributed setup, correct node count and pool/set/disk indices | `pool_expansion_resolves_single_node_multi_drive_and_multi_node_single_drive_pools` |
|
||||
| Additional set width 2 or 3, explicit STANDARD `EC:2` | Reject and identify the incompatible pool | `explicit_standard_parity_is_validated_against_every_pool` |
|
||||
| Set widths `[4, 4]` with `EC:2`, or `[4, 2/3/4]` with `EC:1` | Shared explicit parity accepted | `explicit_standard_parity_is_validated_against_every_pool` |
|
||||
| Explicit environment STANDARD `EC:2`, widths `[4, 2]` | Reject; do not clamp parity | `explicit_environment_standard_parity_is_not_clamped` |
|
||||
| Automatic parity, widths `[4, 2]` | Preserve RustFS's existing per-pool `[2, 1]` policy | `automatic_parity_is_resolved_per_pool` |
|
||||
| Existing SNSD plus new drives, on first/non-first server | Reject with SNSD migration guidance; old format unchanged and new drives unformatted | `single_drive_format_rejects_in_place_expansion_without_writes` |
|
||||
| Existing four-drive pool resized to 2, 6, or 8 drives, or regrouped between one four-drive set and two two-drive sets | Reject with stored/configured geometry and append-pool guidance; no format writes | `existing_pool_rejects_drive_count_or_set_width_changes_without_writes` |
|
||||
| Existing four-drive pool with only one drive reachable | Retain quorum failure, not an expansion error | `subquorum_existing_layout_with_missing_drives_is_not_expansion` |
|
||||
| Conflicting four-drive and two-drive formats without a quorum | Retain quorum failure; do not infer the original topology | `conflicting_layouts_without_quorum_are_not_expansion_proof` |
|
||||
| Healthy three-drive majority with a foreign SNSD minority | Start with the majority and quarantine the outlier | `existing_format_quorum_ignores_single_drive_outlier` |
|
||||
| New four-drive pool alongside an initialized four-drive pool | Preserve the deployment ID and original format; original pool restarts | `multi_drive_pool_expansion_preserves_existing_format` |
|
||||
| Typed SNSD/topology errors versus missing-disk, network, and quorum errors | Only permanent topology/corruption errors bypass the format retry loop | `test_should_retry_format_load_rejects_permanent_topology_errors` |
|
||||
| Full store startup, SNSD to four drives or four-drive pool to eight | Return the typed topology error before retry backoff; no format writes | `store_startup_rejects_pool_resize_before_retry_loop` |
|
||||
| Startup topology error cloning and I/O wrapping | Retain error type and guidance; do not narrow into a disk/quorum error | `startup_topology_errors_preserve_identity_and_guidance` |
|
||||
|
||||
Layout and endpoint guards live in the layout source files above; parity guards live in the storage-class module. The existing `test_get_set_indexes` and `test_into_endpoint_set` tables cover larger, multi-set layouts and malformed ranges.
|
||||
|
||||
Run the focused crate tests:
|
||||
|
||||
```bash
|
||||
cargo nextest run -p rustfs-ecstore --lib \
|
||||
-E 'test(layout::disks_layout::) | test(layout::endpoints::) | test(config::storageclass::) | test(store::init_format::) | test(test_should_retry_format_load) | test(error::)'
|
||||
```
|
||||
|
||||
## Runtime coverage
|
||||
|
||||
Keep the existing single-node multi-drive pool scenarios. They are valid topologies, not exceptions that need a node-count bypass:
|
||||
|
||||
- `cluster_two_pool_smoke` in `crates/e2e_test/src/cluster_multidrive_pool_test.rs` exercises real S3 traffic against two pools.
|
||||
- `four_node_pool_expand_preserves_objects_then_rebalance` in `crates/e2e_test/src/distributed/expand_decommission_rebalance_test.rs` appends pools, verifies existing objects, restarts, and exercises rebalance.
|
||||
|
||||
The localhost harness uses separate processes and ports; it does not prove independent physical-host failure tolerance. See [distributed-e2e.md](distributed-e2e.md) for the binary, filesystem, and execution requirements before running expansion tests. Parser and endpoint unit tests establish admission, not persistent-data migration safety or production availability.
|
||||
@@ -656,6 +656,7 @@ fn summarize_storage_readiness(snapshot: &ClusterReadOnlySnapshot) -> Capability
|
||||
.iter()
|
||||
.filter_map(|reason| match reason {
|
||||
ReadinessDegradedReason::StorageQuorumUnavailable
|
||||
| ReadinessDegradedReason::PoolMetaWriteBlocked
|
||||
| ReadinessDegradedReason::StorageAndIamUnavailable
|
||||
| ReadinessDegradedReason::StorageAndLockUnavailable
|
||||
| ReadinessDegradedReason::StorageIamAndLockUnavailable => Some(reason.as_str()),
|
||||
@@ -1282,6 +1283,41 @@ mod tests {
|
||||
assert_eq!(view.components.usage.last_usage_save_result, "skipped_stale");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_snapshot_storage_summary_reports_pool_meta_write_blocked() {
|
||||
let snapshot = ClusterReadOnlySnapshot {
|
||||
topology: TopologySnapshot::default(),
|
||||
membership: ClusterMembershipSnapshot::default(),
|
||||
pool_state: ClusterPoolStateSnapshot::default(),
|
||||
local_storage: ClusterLocalNodeStorageSnapshot::default(),
|
||||
peer_health: ClusterPeerHealthSnapshot::default(),
|
||||
rpc_boundary: sample_rpc_boundary_snapshot(),
|
||||
observability: ObservabilitySnapshot::default(),
|
||||
workload_admission: WorkloadAdmissionRegistrySnapshot::new(Vec::new()),
|
||||
runtime_status: ClusterRuntimeStatusSnapshot {
|
||||
readiness: DependencyReadiness {
|
||||
storage_ready: false,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
peer_health_ready: true,
|
||||
},
|
||||
state: ClusterRuntimeReadinessState::Degraded,
|
||||
degraded_reasons: vec![ReadinessDegradedReason::PoolMetaWriteBlocked],
|
||||
},
|
||||
usage_freshness: ClusterUsageFreshnessSnapshot::default(),
|
||||
listing_diagnostics: ClusterListingDiagnosticsSnapshot::default(),
|
||||
};
|
||||
|
||||
let view = ClusterSnapshotView::from(snapshot);
|
||||
|
||||
assert_eq!(view.components.storage.status.state, CapabilityState::Unknown);
|
||||
assert_eq!(
|
||||
view.components.storage.status.reason.as_deref(),
|
||||
Some("storage readiness degraded: pool_meta_write_blocked")
|
||||
);
|
||||
assert_eq!(view.components.storage.condition, "degraded");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_snapshot_listing_component_keeps_historical_stalls_as_evidence() {
|
||||
let snapshot = ClusterReadOnlySnapshot {
|
||||
|
||||
@@ -18,7 +18,7 @@ use super::storage_api::bucket::replication::{self, BucketReplicationResyncStatu
|
||||
use super::storage_api::bucket::target::{BucketTarget, BucketTargetType, BucketTargets};
|
||||
use super::storage_api::bucket::target_sys::{
|
||||
BucketTargetSys, PutObjectOptions, RemoveObjectOptions, S3ClientError, SsecPassthroughCapability, TargetClient,
|
||||
VersionIdentityCapability, append_version_id_query,
|
||||
VersionIdentityCapability, append_version_id_query, resolve_delete_api_version_id,
|
||||
};
|
||||
use super::storage_api::bucket::versioning_sys::BucketVersioningSys;
|
||||
use super::storage_api::bucket::{AdminReplicationConfigExt as _, AdminVersioningConfigExt as _};
|
||||
@@ -2745,12 +2745,22 @@ async fn delete_replication_probe_object(
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_CHECK, "true");
|
||||
}
|
||||
|
||||
if let Some(version_id) = version_id {
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_VERSION_ID, version_id);
|
||||
}
|
||||
|
||||
// Same wire shape as live delete replication: a marker creation carries
|
||||
// no `versionId` (the target mints the marker), a version delete does. A
|
||||
// generic S3 target handed the version id on the marker step would
|
||||
// permanently delete the probe version instead, and the VersionDelete
|
||||
// phase would then find nothing (seen on Wasabi).
|
||||
let api_version_id = resolve_delete_api_version_id(version_id.map(ToOwned::to_owned), &options);
|
||||
target_client
|
||||
.client
|
||||
.delete_object()
|
||||
.bucket(target_bucket)
|
||||
.key(probe_key)
|
||||
.set_version_id(version_id.map(ToOwned::to_owned))
|
||||
.set_version_id(api_version_id)
|
||||
.customize()
|
||||
.map_request(move |mut req| {
|
||||
for (key, value) in headers.clone() {
|
||||
@@ -2782,6 +2792,14 @@ async fn delete_replication_probe_version(
|
||||
.map_err(S3ClientError::from)
|
||||
}
|
||||
|
||||
/// The VersionDelete phase already removed the probe version the cleanup is
|
||||
/// handed, and a strict S3 target (Wasabi) answers a second DELETE of that
|
||||
/// id with `NoSuchVersion` where RustFS/MinIO answer 204: the goal is met
|
||||
/// either way.
|
||||
fn probe_version_already_gone(err: &S3ClientError) -> bool {
|
||||
matches!(err.code.as_deref(), Some("NoSuchKey" | "NoSuchVersion"))
|
||||
}
|
||||
|
||||
async fn cleanup_replication_probe<'a>(
|
||||
target_client: &TargetClient,
|
||||
target_bucket: &str,
|
||||
@@ -2793,6 +2811,7 @@ async fn cleanup_replication_probe<'a>(
|
||||
for version_id in known_version_ids.into_iter().flatten() {
|
||||
if deleted_ids.insert(version_id.to_string())
|
||||
&& let Err(err) = delete_replication_probe_version(target_client, target_bucket, probe_key, version_id).await
|
||||
&& !probe_version_already_gone(&err)
|
||||
{
|
||||
errors.push(format_replication_check_client_error(
|
||||
&err,
|
||||
@@ -2838,6 +2857,7 @@ async fn cleanup_replication_probe<'a>(
|
||||
for version_id in discovered_ids {
|
||||
if deleted_ids.insert(version_id.clone())
|
||||
&& let Err(err) = delete_replication_probe_version(target_client, target_bucket, probe_key, &version_id).await
|
||||
&& !probe_version_already_gone(&err)
|
||||
{
|
||||
errors.push(format_replication_check_client_error(
|
||||
&err,
|
||||
|
||||
@@ -204,6 +204,7 @@ pub(crate) mod bandwidth {
|
||||
|
||||
pub(crate) mod bucket_target_sys {
|
||||
pub(crate) use super::ecstore_bucket::bucket_target_sys::append_version_id_query;
|
||||
pub(crate) use super::ecstore_bucket::bucket_target_sys::resolve_delete_api_version_id;
|
||||
pub(crate) type AdvancedPutOptions = super::ecstore_bucket::bucket_target_sys::AdvancedPutOptions;
|
||||
pub(crate) type BucketTargetError = super::ecstore_bucket::bucket_target_sys::BucketTargetError;
|
||||
pub(crate) type BucketTargetSys = super::ecstore_bucket::bucket_target_sys::BucketTargetSys;
|
||||
|
||||
@@ -1247,9 +1247,19 @@ impl DefaultMultipartUsecase {
|
||||
StreamReader::new(body_stream.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
);
|
||||
|
||||
let is_disk_compressed = rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
|
||||
// An SSE-C passthrough session stores ciphertext parts verbatim: a
|
||||
// compression key restored on the session describes those stored
|
||||
// bytes and must not add a second compression layer, and each part's
|
||||
// plaintext length comes from the sender (backlog#2363).
|
||||
let preserve_ciphertext = contains_key_str(&fi.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||
let is_disk_compressed = !preserve_ciphertext
|
||||
&& rustfs_utils::http::contains_key_str(&fi.user_defined, rustfs_utils::http::SUFFIX_COMPRESSION);
|
||||
|
||||
let actual_size = size;
|
||||
let actual_size = if preserve_ciphertext {
|
||||
passthrough_part_actual_size(&req.headers).unwrap_or(size)
|
||||
} else {
|
||||
size
|
||||
};
|
||||
|
||||
let mut md5hex = if let Some(base64_md5) = input.content_md5 {
|
||||
let md5 = base64_simd::STANDARD
|
||||
@@ -1286,7 +1296,6 @@ impl DefaultMultipartUsecase {
|
||||
|
||||
// An SSE-C passthrough session stores ciphertext parts verbatim: no
|
||||
// material recovery, no validation against the (absent) customer key.
|
||||
let preserve_ciphertext = contains_key_str(&fi.user_defined, SUFFIX_REPLICATION_PRESERVE_CIPHERTEXT);
|
||||
let has_ssec = !preserve_ciphertext
|
||||
&& fi
|
||||
.user_defined
|
||||
@@ -1903,6 +1912,15 @@ impl DefaultMultipartUsecase {
|
||||
}
|
||||
}
|
||||
|
||||
/// Plaintext length of one SSE-C passthrough part, declared by the sender on
|
||||
/// UploadPart (backlog#2363).
|
||||
fn passthrough_part_actual_size(headers: &HeaderMap) -> Option<i64> {
|
||||
get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_PART_ACTUAL_SIZE)?
|
||||
.parse::<i64>()
|
||||
.ok()
|
||||
.filter(|size| *size > 0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1681,7 +1681,16 @@ impl DefaultObjectUsecase {
|
||||
};
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from("app_prelookup", prelookup_stage_start);
|
||||
|
||||
let actual_size = size;
|
||||
// A compressed SSE-C passthrough body is the source's stored bytes:
|
||||
// its logical length is the plaintext size restored from the
|
||||
// transport headers (backlog#2363). The body itself is still read
|
||||
// at its wire size.
|
||||
let body_size = size;
|
||||
let actual_size = if ciphertext_passthrough {
|
||||
passthrough_compressed_actual_size(&opts.user_defined).unwrap_or(size)
|
||||
} else {
|
||||
size
|
||||
};
|
||||
if !ciphertext_passthrough && let Some(quota_check) = quota_check.as_ref() {
|
||||
ensure_object_size_within_quota(
|
||||
quota_check,
|
||||
@@ -1733,17 +1742,17 @@ impl DefaultObjectUsecase {
|
||||
} else {
|
||||
if use_zero_copy_eager_put_path {
|
||||
let zero_copy_start = std::time::Instant::now();
|
||||
let eager_body = read_zero_copy_put_body_exact(body, actual_size as usize).await?;
|
||||
rustfs_io_metrics::record_zero_copy_write(actual_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
|
||||
let eager_body = read_zero_copy_put_body_exact(body, body_size as usize).await?;
|
||||
rustfs_io_metrics::record_zero_copy_write(body_size as usize, zero_copy_start.elapsed().as_secs_f64() * 1000.0);
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
} else if use_empty_or_small_eager_put_path {
|
||||
if (actual_size as usize) <= POOL_BYPASS_MAX_SIZE {
|
||||
if (body_size as usize) <= POOL_BYPASS_MAX_SIZE {
|
||||
// Bypass BytesPool for very small objects to avoid Small-tier
|
||||
// Mutex contention under high concurrency. Direct allocation
|
||||
// for ≤4KiB is negligible cost.
|
||||
let eager_body = read_small_put_body_exact_direct(
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
actual_size as usize,
|
||||
body_size as usize,
|
||||
)
|
||||
.await?;
|
||||
HashReader::from_stream(eager_body, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
@@ -1751,11 +1760,11 @@ impl DefaultObjectUsecase {
|
||||
let pool = get_concurrency_manager().bytes_pool();
|
||||
let eager_body = read_small_put_body_exact_pooled(
|
||||
StreamReader::new(body.map(|f| f.map_err(s3s_body_error_to_io))),
|
||||
actual_size as usize,
|
||||
body_size as usize,
|
||||
pool.as_ref(),
|
||||
)
|
||||
.await?;
|
||||
let eager_reader = PooledBufferReader::new(eager_body, actual_size as usize);
|
||||
let eager_reader = PooledBufferReader::new(eager_body, body_size as usize);
|
||||
HashReader::from_stream(eager_reader, size, actual_size, md5hex, sha256hex, false).map_err(ApiError::from)?
|
||||
}
|
||||
} else {
|
||||
@@ -2105,6 +2114,18 @@ pub(super) fn previous_current_size_from_backfill(backfill: Option<OldCurrentSiz
|
||||
})
|
||||
}
|
||||
|
||||
/// Plaintext size of a compressed SSE-C passthrough body, restored from the
|
||||
/// replication transport headers into the object metadata (backlog#2363).
|
||||
fn passthrough_compressed_actual_size(user_defined: &HashMap<String, String>) -> Option<i64> {
|
||||
if !rustfs_utils::http::contains_key_str(user_defined, SUFFIX_COMPRESSION) {
|
||||
return None;
|
||||
}
|
||||
rustfs_utils::http::get_str(user_defined, SUFFIX_ACTUAL_SIZE)?
|
||||
.parse::<i64>()
|
||||
.ok()
|
||||
.filter(|size| *size >= 0)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
+109
-17
@@ -256,7 +256,13 @@ pub async fn publish_ready_when_runtime_ready(
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct StorageReadinessCacheEntry {
|
||||
captured_at: Instant,
|
||||
storage_ready: bool,
|
||||
status: StorageWriteReadinessStatus,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
struct StorageWriteReadinessStatus {
|
||||
ready: bool,
|
||||
pool_meta_write_blocked: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
@@ -356,7 +362,7 @@ async fn reset_cluster_health_report_caches() {
|
||||
*cluster_read_health_report_cache().lock().await = None;
|
||||
}
|
||||
|
||||
async fn load_cached_storage_readiness() -> Option<bool> {
|
||||
async fn load_cached_storage_readiness() -> Option<StorageWriteReadinessStatus> {
|
||||
let ttl = health_readiness_cache_ttl();
|
||||
if ttl.is_zero() {
|
||||
return None;
|
||||
@@ -365,7 +371,7 @@ async fn load_cached_storage_readiness() -> Option<bool> {
|
||||
let cache = storage_readiness_cache().lock().await;
|
||||
let entry = cache.as_ref()?;
|
||||
if entry.captured_at.elapsed() <= ttl {
|
||||
return Some(entry.storage_ready);
|
||||
return Some(entry.status);
|
||||
}
|
||||
|
||||
None
|
||||
@@ -398,7 +404,7 @@ async fn update_cluster_health_report_cache(kind: ClusterHealthProbeKind, report
|
||||
});
|
||||
}
|
||||
|
||||
async fn update_storage_readiness_cache(storage_ready: bool) {
|
||||
async fn update_storage_readiness_cache(status: StorageWriteReadinessStatus) {
|
||||
if health_readiness_cache_ttl().is_zero() {
|
||||
return;
|
||||
}
|
||||
@@ -406,7 +412,7 @@ async fn update_storage_readiness_cache(storage_ready: bool) {
|
||||
let mut cache = storage_readiness_cache().lock().await;
|
||||
*cache = Some(StorageReadinessCacheEntry {
|
||||
captured_at: Instant::now(),
|
||||
storage_ready,
|
||||
status,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -608,7 +614,11 @@ where
|
||||
}
|
||||
|
||||
fn storage_ready_from_runtime_state(info: &StorageInfo) -> bool {
|
||||
storage_ready_from_runtime_state_with_quorum(info, pool_write_quorum)
|
||||
storage_ready_from_runtime_state_with_pool_meta(info, true)
|
||||
}
|
||||
|
||||
fn storage_ready_from_runtime_state_with_pool_meta(info: &StorageInfo, pool_meta_ready: bool) -> bool {
|
||||
pool_meta_ready && storage_ready_from_runtime_state_with_quorum(info, pool_write_quorum)
|
||||
}
|
||||
|
||||
fn storage_read_ready_from_runtime_state(info: &StorageInfo) -> bool {
|
||||
@@ -665,6 +675,20 @@ fn degraded_reasons(readiness: DependencyReadiness) -> Vec<ReadinessDegradedReas
|
||||
reasons
|
||||
}
|
||||
|
||||
fn degraded_reasons_with_pool_meta_status(
|
||||
readiness: DependencyReadiness,
|
||||
pool_meta_write_blocked: bool,
|
||||
) -> Vec<ReadinessDegradedReason> {
|
||||
let mut reasons = degraded_reasons(readiness);
|
||||
if pool_meta_write_blocked {
|
||||
reasons.retain(|reason| *reason != ReadinessDegradedReason::StorageQuorumUnavailable);
|
||||
if !reasons.contains(&ReadinessDegradedReason::PoolMetaWriteBlocked) {
|
||||
reasons.insert(0, ReadinessDegradedReason::PoolMetaWriteBlocked);
|
||||
}
|
||||
}
|
||||
reasons
|
||||
}
|
||||
|
||||
fn record_readiness_report(report: &DependencyReadinessReport) {
|
||||
let ready = report.readiness.storage_ready
|
||||
&& report.readiness.iam_ready
|
||||
@@ -688,24 +712,34 @@ fn dependency_readiness_report_from_readiness(readiness: DependencyReadiness) ->
|
||||
}
|
||||
}
|
||||
|
||||
fn dependency_readiness_report_from_write_status(
|
||||
readiness: DependencyReadiness,
|
||||
storage: StorageWriteReadinessStatus,
|
||||
) -> DependencyReadinessReport {
|
||||
DependencyReadinessReport {
|
||||
degraded_reasons: degraded_reasons_with_pool_meta_status(readiness, storage.pool_meta_write_blocked),
|
||||
readiness,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn collect_dependency_readiness_report() -> DependencyReadinessReport {
|
||||
let iam_ready_raw = runtime_sources::current_iam_ready();
|
||||
let storage_ready = if let Some(cached) = load_cached_storage_readiness().await {
|
||||
let storage = if let Some(cached) = load_cached_storage_readiness().await {
|
||||
cached
|
||||
} else {
|
||||
let computed = collect_storage_readiness_uncached().await;
|
||||
let computed = collect_storage_write_readiness_uncached().await;
|
||||
update_storage_readiness_cache(computed).await;
|
||||
computed
|
||||
};
|
||||
let lock_quorum_status = collect_lock_quorum_status().await;
|
||||
|
||||
let readiness = DependencyReadiness {
|
||||
storage_ready,
|
||||
storage_ready: storage.ready,
|
||||
iam_ready: iam_ready_raw,
|
||||
lock_quorum_ready: lock_quorum_status.ready,
|
||||
peer_health_ready: collect_peer_health_readiness(),
|
||||
};
|
||||
let report = dependency_readiness_report_from_readiness(readiness);
|
||||
let report = dependency_readiness_report_from_write_status(readiness, storage);
|
||||
record_readiness_report(&report);
|
||||
report
|
||||
}
|
||||
@@ -719,13 +753,14 @@ pub async fn collect_cluster_read_health_report() -> DependencyReadinessReport {
|
||||
}
|
||||
|
||||
pub async fn collect_node_readiness_report() -> DependencyReadinessReport {
|
||||
let storage = node_pool_meta_write_readiness().await;
|
||||
let readiness = DependencyReadiness {
|
||||
storage_ready: runtime_sources::current_object_store_handle().is_some(),
|
||||
storage_ready: storage.ready,
|
||||
iam_ready: runtime_sources::current_iam_ready(),
|
||||
lock_quorum_ready: collect_lock_quorum_status().await.ready,
|
||||
peer_health_ready: collect_peer_health_readiness(),
|
||||
};
|
||||
let report = dependency_readiness_report_from_readiness(readiness);
|
||||
let report = dependency_readiness_report_from_write_status(readiness, storage);
|
||||
record_readiness_report(&report);
|
||||
report
|
||||
}
|
||||
@@ -788,14 +823,15 @@ pub async fn collect_cluster_read_dependency_readiness_report() -> DependencyRea
|
||||
}
|
||||
|
||||
pub(crate) async fn snapshot_dependency_readiness_report() -> DependencyReadinessReport {
|
||||
let storage = collect_storage_write_readiness_uncached().await;
|
||||
let readiness = DependencyReadiness {
|
||||
storage_ready: collect_storage_readiness_uncached().await,
|
||||
storage_ready: storage.ready,
|
||||
iam_ready: runtime_sources::current_iam_ready(),
|
||||
lock_quorum_ready: collect_lock_quorum_status_uncached().await.ready,
|
||||
peer_health_ready: collect_peer_health_readiness(),
|
||||
};
|
||||
|
||||
dependency_readiness_report_from_readiness(readiness)
|
||||
dependency_readiness_report_from_write_status(readiness, storage)
|
||||
}
|
||||
|
||||
async fn collect_lock_quorum_status() -> LockQuorumStatus {
|
||||
@@ -808,12 +844,33 @@ async fn collect_lock_quorum_status() -> LockQuorumStatus {
|
||||
}
|
||||
}
|
||||
|
||||
async fn collect_storage_readiness_uncached() -> bool {
|
||||
async fn node_pool_meta_write_readiness() -> StorageWriteReadinessStatus {
|
||||
if let Some(store) = runtime_sources::current_object_store_handle() {
|
||||
let ready = store.pool_meta_writes_ready().await;
|
||||
return StorageWriteReadinessStatus {
|
||||
ready,
|
||||
pool_meta_write_blocked: !ready,
|
||||
};
|
||||
}
|
||||
|
||||
StorageWriteReadinessStatus::default()
|
||||
}
|
||||
|
||||
async fn collect_storage_write_readiness_uncached() -> StorageWriteReadinessStatus {
|
||||
if let Some(store) = runtime_sources::current_object_store_handle() {
|
||||
if !store.pool_meta_writes_ready().await {
|
||||
return StorageWriteReadinessStatus {
|
||||
ready: false,
|
||||
pool_meta_write_blocked: true,
|
||||
};
|
||||
}
|
||||
let storage_info = StorageAdminApi::storage_info(store.as_ref()).await;
|
||||
storage_ready_from_runtime_state(&storage_info)
|
||||
StorageWriteReadinessStatus {
|
||||
ready: storage_ready_from_runtime_state(&storage_info),
|
||||
pool_meta_write_blocked: false,
|
||||
}
|
||||
} else {
|
||||
false
|
||||
StorageWriteReadinessStatus::default()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1555,6 +1612,7 @@ mod tests {
|
||||
};
|
||||
|
||||
assert!(storage_ready_from_runtime_state(&info));
|
||||
assert!(!storage_ready_from_runtime_state_with_pool_meta(&info, false));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1791,6 +1849,40 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_reasons_report_pool_meta_write_blocked() {
|
||||
let readiness = DependencyReadiness {
|
||||
storage_ready: false,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: true,
|
||||
peer_health_ready: true,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
degraded_reasons_with_pool_meta_status(readiness, true),
|
||||
vec![ReadinessDegradedReason::PoolMetaWriteBlocked]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_reasons_keep_pool_meta_source_with_other_failures() {
|
||||
let readiness = DependencyReadiness {
|
||||
storage_ready: false,
|
||||
iam_ready: true,
|
||||
lock_quorum_ready: false,
|
||||
peer_health_ready: false,
|
||||
};
|
||||
|
||||
assert_eq!(
|
||||
degraded_reasons_with_pool_meta_status(readiness, true),
|
||||
vec![
|
||||
ReadinessDegradedReason::PoolMetaWriteBlocked,
|
||||
ReadinessDegradedReason::StorageAndLockUnavailable,
|
||||
ReadinessDegradedReason::PeerHealthUnavailable,
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn degraded_reasons_append_peer_health_gate_failures() {
|
||||
let readiness = DependencyReadiness {
|
||||
|
||||
@@ -43,6 +43,7 @@ pub enum ReadinessDegradedReason {
|
||||
KmsNotReady,
|
||||
ObjectReadStalled,
|
||||
ObjectWriteStalled,
|
||||
PoolMetaWriteBlocked,
|
||||
ClusterHealthTimeout,
|
||||
PeerHealthUnavailable,
|
||||
StartupFinalizationPending,
|
||||
@@ -61,6 +62,7 @@ impl ReadinessDegradedReason {
|
||||
ReadinessDegradedReason::KmsNotReady => "kms_not_ready",
|
||||
ReadinessDegradedReason::ObjectReadStalled => "object_read_stalled",
|
||||
ReadinessDegradedReason::ObjectWriteStalled => "object_write_stalled",
|
||||
ReadinessDegradedReason::PoolMetaWriteBlocked => "pool_meta_write_blocked",
|
||||
ReadinessDegradedReason::ClusterHealthTimeout => "cluster_health_timeout",
|
||||
ReadinessDegradedReason::PeerHealthUnavailable => "peer_health_unavailable",
|
||||
ReadinessDegradedReason::StartupFinalizationPending => "startup_finalization_pending",
|
||||
|
||||
@@ -500,6 +500,25 @@ pub fn put_opts_from_headers_with_replication_authorization(
|
||||
if let Some(restored) = rustfs_utils::http::ssec_transport_to_stored_metadata(headers) {
|
||||
opts.user_defined.extend(restored);
|
||||
opts.preserve_ciphertext = true;
|
||||
// A compressed passthrough object restores its compression
|
||||
// layout as well, so the replica decompresses after decrypting
|
||||
// (backlog#2363). The value is validated when the object is read.
|
||||
if let Some(scheme) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION) {
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut opts.user_defined,
|
||||
rustfs_utils::http::SUFFIX_COMPRESSION,
|
||||
scheme.into_owned(),
|
||||
);
|
||||
if let Some(actual_size) = get_header(headers, rustfs_utils::http::SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE)
|
||||
&& actual_size.parse::<i64>().is_ok_and(|size| size >= 0)
|
||||
{
|
||||
rustfs_utils::http::insert_str(
|
||||
&mut opts.user_defined,
|
||||
rustfs_utils::http::SUFFIX_ACTUAL_SIZE,
|
||||
actual_size.into_owned(),
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(crc) = get_header(headers, SUFFIX_REPLICATION_SSEC_CRC) {
|
||||
insert_header_map(&mut opts.user_defined, SUFFIX_REPLICATION_SSEC_CRC, crc.into_owned());
|
||||
@@ -1588,6 +1607,59 @@ mod tests {
|
||||
assert!(!has_replication_retention_update(&missing_request, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn put_opts_from_headers_restores_the_compression_layout_only_for_ssec_passthrough() {
|
||||
use rustfs_utils::http::object_encryption_keys::REPLICATION_SSEC_ALGORITHM_HEADER;
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_ACTUAL_SIZE, SUFFIX_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE,
|
||||
get_str,
|
||||
};
|
||||
|
||||
let mut headers = HeaderMap::new();
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_REPLICATION_REQUEST, "true");
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION, "klauspost/compress/s2");
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "6295552");
|
||||
|
||||
// Without SSE-C transport headers the request is not a passthrough:
|
||||
// the target compresses (or not) by its own policy and must not adopt
|
||||
// a layout the body does not have.
|
||||
let plain = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("authorized replication request should parse");
|
||||
assert!(!plain.preserve_ciphertext);
|
||||
assert!(get_str(&plain.user_defined, SUFFIX_COMPRESSION).is_none());
|
||||
assert!(get_str(&plain.user_defined, SUFFIX_ACTUAL_SIZE).is_none());
|
||||
|
||||
headers.insert(
|
||||
REPLICATION_SSEC_ALGORITHM_HEADER.parse::<http::HeaderName>().unwrap(),
|
||||
HeaderValue::from_static("AES256"),
|
||||
);
|
||||
|
||||
// Unauthorized: inert, like the SSE-C transport itself.
|
||||
let untrusted = put_opts_from_headers(&headers, HashMap::new()).expect("ordinary PUT options should be created");
|
||||
assert!(!untrusted.preserve_ciphertext);
|
||||
assert!(get_str(&untrusted.user_defined, SUFFIX_COMPRESSION).is_none());
|
||||
|
||||
// Authorized passthrough: the stored bytes are compressed ciphertext,
|
||||
// so the replica records the scheme and the plaintext size
|
||||
// (backlog#2363).
|
||||
let trusted = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("authorized replication request should parse");
|
||||
assert!(trusted.preserve_ciphertext);
|
||||
assert_eq!(
|
||||
get_str(&trusted.user_defined, SUFFIX_COMPRESSION).as_deref(),
|
||||
Some("klauspost/compress/s2")
|
||||
);
|
||||
assert_eq!(get_str(&trusted.user_defined, SUFFIX_ACTUAL_SIZE).as_deref(), Some("6295552"));
|
||||
|
||||
// A malformed plaintext size is dropped; the scheme alone still lets
|
||||
// the read path derive the size from the parts.
|
||||
insert_header(&mut headers, SUFFIX_REPLICATION_COMPRESSION_ACTUAL_SIZE, "-5");
|
||||
let malformed = put_opts_from_headers_with_replication_authorization(&headers, HashMap::new(), true)
|
||||
.expect("authorized replication request should parse");
|
||||
assert!(get_str(&malformed.user_defined, SUFFIX_COMPRESSION).is_some());
|
||||
assert!(get_str(&malformed.user_defined, SUFFIX_ACTUAL_SIZE).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_put_opts_from_headers_gates_ssec_passthrough_on_authorization() {
|
||||
use rustfs_utils::http::object_encryption_keys::{
|
||||
|
||||
@@ -2750,12 +2750,12 @@ mod tests {
|
||||
GetAllBucketStatsRequest, GetBucketInfoRequest, GetBucketStatsDataRequest, GetCpusRequest, GetMemInfoRequest,
|
||||
GetMetacacheListingRequest, GetMetricsRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest,
|
||||
GetProcInfoRequest, GetSeLinuxInfoRequest, GetSrMetricsDataRequest, GetSysConfigRequest, GetSysErrorsRequest,
|
||||
HealBucketRequest, HealControlRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, LoadBucketMetadataRequest,
|
||||
LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, LoadServiceAccountRequest,
|
||||
LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest,
|
||||
MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest, ReadMultipleRequest,
|
||||
ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest,
|
||||
RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest,
|
||||
HealBucketRequest, HealControlRequest, HealControlResponse, ListBucketRequest, ListDirRequest, ListVolumesRequest,
|
||||
LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest,
|
||||
LoadServiceAccountRequest, LoadTransitionTierConfigRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest,
|
||||
MakeVolumeRequest, MakeVolumesRequest, Mss, PingRequest, PreparePartTransactionRequest, ReadAllRequest, ReadAtRequest,
|
||||
ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest,
|
||||
RenameDataRequest, RenameFileRequest, RenamePartRequest, ScannerActivityRequest, ScannerDirtyUsageSnapshotRequest,
|
||||
ScannerPublicationLeaseReleaseRequest, ScannerPublicationLeaseRequest, ServerInfoRequest, SettlePartTransactionRequest,
|
||||
SignalServiceRequest, SnapshotLeaseReleaseRequest, SnapshotLeaseRenewRequest, SnapshotLeaseRequest,
|
||||
StartDecommissionRequest, StartProfilingRequest, StatVolumeRequest, StopRebalanceRequest, TierMutationAbortRequest,
|
||||
@@ -2992,6 +2992,172 @@ mod tests {
|
||||
(manager, request, metadata)
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum HealControlTransportFault {
|
||||
None,
|
||||
DropBeforeAdmission,
|
||||
DropAfterAdmission,
|
||||
}
|
||||
|
||||
struct HealControlTransportFaultService {
|
||||
manager: Arc<HealManager>,
|
||||
fingerprint: String,
|
||||
coordinator_epoch: u64,
|
||||
fault: HealControlTransportFault,
|
||||
}
|
||||
|
||||
#[tonic::async_trait]
|
||||
impl rustfs_protos::proto_gen::node_service::heal_control_service_server::HealControlService
|
||||
for HealControlTransportFaultService
|
||||
{
|
||||
async fn heal_control(&self, request: Request<HealControlRequest>) -> Result<Response<HealControlResponse>, Status> {
|
||||
let command = request.get_ref().command.to_vec();
|
||||
let body = rustfs_protos::canonical_heal_control_request_body(
|
||||
request.get_ref().version,
|
||||
&request.get_ref().topology_fingerprint,
|
||||
&request.get_ref().command,
|
||||
)
|
||||
.map_err(|_| Status::invalid_argument("heal control request length cannot be represented"))?;
|
||||
crate::storage::storage_api::verify_tonic_canonical_body_digest(&request, &body)
|
||||
.map_err(|err| Status::permission_denied(format!("heal control authentication failed: {err}")))?;
|
||||
if request.get_ref().version != rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION {
|
||||
return Err(Status::failed_precondition("unsupported heal control protocol version"));
|
||||
}
|
||||
if request.get_ref().topology_fingerprint != self.fingerprint {
|
||||
return Err(Status::failed_precondition("heal control topology does not match"));
|
||||
}
|
||||
if matches!(self.fault, HealControlTransportFault::DropBeforeAdmission) {
|
||||
return Err(Status::unavailable("transport failed before heal admission"));
|
||||
}
|
||||
|
||||
let envelope = rustfs_protos::heal_control::decode_envelope(&command).map_err(Status::invalid_argument)?;
|
||||
let result =
|
||||
execute_heal_control_envelope_with_manager(envelope, self.coordinator_epoch, Some(Arc::clone(&self.manager)))
|
||||
.await?;
|
||||
if matches!(self.fault, HealControlTransportFault::DropAfterAdmission) {
|
||||
return Err(Status::unavailable("transport failed after heal admission"));
|
||||
}
|
||||
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
request.get_ref().version,
|
||||
&self.fingerprint,
|
||||
&command,
|
||||
&result,
|
||||
)
|
||||
.map_err(|_| Status::internal("heal control response length cannot be represented"))?;
|
||||
let response_proof = crate::storage::storage_api::sign_tonic_rpc_response_proof(&canonical_response)
|
||||
.map_err(|_| Status::internal("heal control response proof is unavailable"))?;
|
||||
Ok(Response::new(HealControlResponse {
|
||||
success: true,
|
||||
result: result.into(),
|
||||
error_info: None,
|
||||
response_proof: response_proof.into(),
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
async fn connect_faulty_heal_control_client(
|
||||
manager: Arc<HealManager>,
|
||||
fingerprint: &str,
|
||||
coordinator_epoch: u64,
|
||||
fault: HealControlTransportFault,
|
||||
) -> Option<HealControlServiceClient<tonic::transport::Channel>> {
|
||||
let listener = match TcpListener::bind("127.0.0.1:0").await {
|
||||
Ok(listener) => listener,
|
||||
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return None,
|
||||
Err(err) => panic!("test listener should bind: {err}"),
|
||||
};
|
||||
let addr = listener.local_addr().expect("listener local address should be available");
|
||||
let service = HealControlTransportFaultService {
|
||||
manager,
|
||||
fingerprint: fingerprint.to_string(),
|
||||
coordinator_epoch,
|
||||
fault,
|
||||
};
|
||||
|
||||
tokio::spawn(async move {
|
||||
tonic::transport::Server::builder()
|
||||
.add_service(
|
||||
HealControlServiceServer::new(service)
|
||||
.max_decoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE)
|
||||
.max_encoding_message_size(rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE),
|
||||
)
|
||||
.serve_with_incoming(TcpListenerStream::new(listener))
|
||||
.await
|
||||
.expect("faulty heal control transport server should run");
|
||||
});
|
||||
|
||||
Some(
|
||||
HealControlServiceClient::connect(format!("http://{addr}"))
|
||||
.await
|
||||
.expect("faulty heal control test client should connect"),
|
||||
)
|
||||
}
|
||||
|
||||
fn signed_heal_control_request(fingerprint: &str, command: Vec<u8>) -> Request<HealControlRequest> {
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version: rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
topology_fingerprint: fingerprint.to_string(),
|
||||
command: command.into(),
|
||||
});
|
||||
request.set_timeout(rustfs_protos::heal_control_execution_timeout());
|
||||
let body = rustfs_protos::canonical_heal_control_request_body(
|
||||
request.get_ref().version,
|
||||
&request.get_ref().topology_fingerprint,
|
||||
&request.get_ref().command,
|
||||
)
|
||||
.expect("heal control transport request should encode");
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut request);
|
||||
request
|
||||
}
|
||||
|
||||
async fn call_heal_control_transport(
|
||||
client: &mut HealControlServiceClient<tonic::transport::Channel>,
|
||||
fingerprint: &str,
|
||||
command: Vec<u8>,
|
||||
) -> Result<Vec<u8>, Status> {
|
||||
let response = client
|
||||
.heal_control(signed_heal_control_request(fingerprint, command.clone()))
|
||||
.await?
|
||||
.into_inner();
|
||||
if !response.success {
|
||||
return Err(Status::unknown(
|
||||
response
|
||||
.error_info
|
||||
.unwrap_or_else(|| "peer heal control failed without an error".to_string()),
|
||||
));
|
||||
}
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
fingerprint,
|
||||
&command,
|
||||
&response.result,
|
||||
)
|
||||
.map_err(|_| Status::internal("heal control response length cannot be represented"))?;
|
||||
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
|
||||
.map_err(|err| Status::permission_denied(format!("heal control response proof failed: {err}")))?;
|
||||
Ok(response.result.to_vec())
|
||||
}
|
||||
|
||||
fn encode_transport_start(
|
||||
request: rustfs_heal_contracts::heal_channel::HealChannelRequest,
|
||||
metadata: rustfs_protos::heal_control::RequestMetadata,
|
||||
) -> Vec<u8> {
|
||||
let envelope = rustfs_protos::heal_control::Envelope::start(request, metadata).expect("valid start envelope");
|
||||
rustfs_protos::heal_control::encode_envelope(&envelope).expect("valid start command should encode")
|
||||
}
|
||||
|
||||
fn decode_transport_start_outcome(
|
||||
result: &[u8],
|
||||
request_id: &str,
|
||||
coordinator_epoch: u64,
|
||||
) -> rustfs_protos::heal_control::Outcome {
|
||||
rustfs_protos::heal_control::decode_result(result)
|
||||
.and_then(|result| result.into_outcome(request_id, coordinator_epoch))
|
||||
.expect("heal-control start response should carry a matching canonical receipt")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_start_retry_exact_forced_envelope_returns_cached_admission() {
|
||||
let (manager, request, metadata) = heal_start_retry_fixture();
|
||||
@@ -3103,6 +3269,273 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_transport_pre_admission_loss_retries_original_deadline_envelope() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("heal-control-transport-fault-test-secret".to_string());
|
||||
let (manager, request, metadata) = heal_start_retry_fixture();
|
||||
let fingerprint = "transport-pre-admission-fingerprint";
|
||||
let command = encode_transport_start(request.clone(), metadata);
|
||||
let mut lost_before_admission = match connect_faulty_heal_control_client(
|
||||
Arc::clone(&manager),
|
||||
fingerprint,
|
||||
metadata.coordinator_epoch,
|
||||
HealControlTransportFault::DropBeforeAdmission,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Some(client) => client,
|
||||
None => return,
|
||||
};
|
||||
|
||||
let lost = call_heal_control_transport(&mut lost_before_admission, fingerprint, command.clone())
|
||||
.await
|
||||
.expect_err("transport loss before admission must be visible to the caller");
|
||||
assert_eq!(lost.code(), tonic::Code::Unavailable);
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queue_length,
|
||||
0,
|
||||
"pre-admission transport loss must not create a canonical task"
|
||||
);
|
||||
assert!(matches!(
|
||||
manager.get_task_status(&request.id).await,
|
||||
Err(rustfs_heal::Error::TaskNotFound { .. })
|
||||
));
|
||||
|
||||
let mut retry = connect_faulty_heal_control_client(
|
||||
Arc::clone(&manager),
|
||||
fingerprint,
|
||||
metadata.coordinator_epoch,
|
||||
HealControlTransportFault::None,
|
||||
)
|
||||
.await
|
||||
.expect("retry listener should bind");
|
||||
let accepted = call_heal_control_transport(&mut retry, fingerprint, command)
|
||||
.await
|
||||
.expect("original envelope should remain usable within its deadline");
|
||||
let outcome = rustfs_protos::heal_control::decode_result(&accepted)
|
||||
.and_then(|result| result.into_outcome(&request.id, metadata.coordinator_epoch))
|
||||
.expect("accepted retry should carry a canonical receipt");
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
rustfs_protos::heal_control::Outcome::Start {
|
||||
task_id,
|
||||
admission: rustfs_protos::heal_control::Admission::Accepted,
|
||||
} if task_id == request.id
|
||||
));
|
||||
assert_eq!(manager.operations_snapshot().await.queue_length, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_transport_post_admission_loss_replays_receipt_but_fresh_force_start_is_distinct() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("heal-control-transport-fault-test-secret".to_string());
|
||||
let (manager, request, metadata) = heal_start_retry_fixture();
|
||||
let fingerprint = "transport-post-admission-fingerprint";
|
||||
let first_id = request.id.clone();
|
||||
let first_command = encode_transport_start(request.clone(), metadata);
|
||||
let mut lost_after_admission = match connect_faulty_heal_control_client(
|
||||
Arc::clone(&manager),
|
||||
fingerprint,
|
||||
metadata.coordinator_epoch,
|
||||
HealControlTransportFault::DropAfterAdmission,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Some(client) => client,
|
||||
None => return,
|
||||
};
|
||||
|
||||
let lost = call_heal_control_transport(&mut lost_after_admission, fingerprint, first_command.clone())
|
||||
.await
|
||||
.expect_err("post-admission response loss must be visible to the caller");
|
||||
assert_eq!(lost.code(), tonic::Code::Unavailable);
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queue_length,
|
||||
1,
|
||||
"post-admission response loss must leave exactly one canonical task"
|
||||
);
|
||||
|
||||
let mut retry = connect_faulty_heal_control_client(
|
||||
Arc::clone(&manager),
|
||||
fingerprint,
|
||||
metadata.coordinator_epoch,
|
||||
HealControlTransportFault::None,
|
||||
)
|
||||
.await
|
||||
.expect("retry listener should bind");
|
||||
let replayed = call_heal_control_transport(&mut retry, fingerprint, first_command)
|
||||
.await
|
||||
.expect("exact transport retry should replay the original receipt");
|
||||
let replayed = rustfs_protos::heal_control::decode_result(&replayed)
|
||||
.and_then(|result| result.into_outcome(&first_id, metadata.coordinator_epoch))
|
||||
.expect("replayed retry should carry a canonical receipt");
|
||||
assert!(matches!(
|
||||
replayed,
|
||||
rustfs_protos::heal_control::Outcome::Start {
|
||||
task_id,
|
||||
admission: rustfs_protos::heal_control::Admission::Accepted,
|
||||
} if task_id == first_id
|
||||
));
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queue_length,
|
||||
1,
|
||||
"exact replay must not duplicate a destructive forced start"
|
||||
);
|
||||
|
||||
let mut duplicate_request = request.clone();
|
||||
duplicate_request.id = Uuid::new_v4().to_string();
|
||||
duplicate_request.force_start = false;
|
||||
let duplicate_id = duplicate_request.id.clone();
|
||||
let duplicate_metadata = rustfs_protos::heal_control::RequestMetadata {
|
||||
nonce: *Uuid::new_v4().as_bytes(),
|
||||
..metadata
|
||||
};
|
||||
let duplicate_command = encode_transport_start(duplicate_request, duplicate_metadata);
|
||||
let duplicate = call_heal_control_transport(&mut retry, fingerprint, duplicate_command)
|
||||
.await
|
||||
.expect("same-target duplicate producer should receive a canonical receipt");
|
||||
let duplicate = rustfs_protos::heal_control::decode_result(&duplicate)
|
||||
.and_then(|result| result.into_outcome(&duplicate_id, metadata.coordinator_epoch))
|
||||
.expect("duplicate producer should carry a canonical receipt");
|
||||
assert!(matches!(
|
||||
duplicate,
|
||||
rustfs_protos::heal_control::Outcome::Start {
|
||||
task_id,
|
||||
admission: rustfs_protos::heal_control::Admission::Merged,
|
||||
} if task_id == first_id
|
||||
));
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queue_length,
|
||||
1,
|
||||
"a duplicate producer after lost response must not create a second task"
|
||||
);
|
||||
|
||||
let mut fresh_request = request;
|
||||
fresh_request.id = Uuid::new_v4().to_string();
|
||||
let fresh_id = fresh_request.id.clone();
|
||||
let fresh_metadata = rustfs_protos::heal_control::RequestMetadata {
|
||||
nonce: *Uuid::new_v4().as_bytes(),
|
||||
..metadata
|
||||
};
|
||||
let fresh_command = encode_transport_start(fresh_request, fresh_metadata);
|
||||
let fresh = call_heal_control_transport(&mut retry, fingerprint, fresh_command)
|
||||
.await
|
||||
.expect("fresh forceStart should keep explicit new-start semantics");
|
||||
let fresh = rustfs_protos::heal_control::decode_result(&fresh)
|
||||
.and_then(|result| result.into_outcome(&fresh_id, metadata.coordinator_epoch))
|
||||
.expect("fresh forceStart should carry its own receipt");
|
||||
assert!(matches!(
|
||||
fresh,
|
||||
rustfs_protos::heal_control::Outcome::Start {
|
||||
task_id,
|
||||
admission: rustfs_protos::heal_control::Admission::Accepted,
|
||||
} if task_id == fresh_id && task_id != first_id
|
||||
));
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queue_length,
|
||||
2,
|
||||
"a new forceStart request must be counted as a distinct canonical task"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_transport_peer_restart_replays_observed_receipt_without_readmission() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("heal-control-transport-fault-test-secret".to_string());
|
||||
let (manager, request, metadata) = heal_start_retry_fixture();
|
||||
let fingerprint = "transport-peer-restart-fingerprint";
|
||||
let first_id = request.id.clone();
|
||||
let first_command = encode_transport_start(request.clone(), metadata);
|
||||
let mut lost_after_admission = match connect_faulty_heal_control_client(
|
||||
Arc::clone(&manager),
|
||||
fingerprint,
|
||||
metadata.coordinator_epoch,
|
||||
HealControlTransportFault::DropAfterAdmission,
|
||||
)
|
||||
.await
|
||||
{
|
||||
Some(client) => client,
|
||||
None => return,
|
||||
};
|
||||
|
||||
let lost = call_heal_control_transport(&mut lost_after_admission, fingerprint, first_command.clone())
|
||||
.await
|
||||
.expect_err("post-admission response loss must be visible before receipt replay");
|
||||
assert_eq!(lost.code(), tonic::Code::Unavailable);
|
||||
assert_eq!(
|
||||
manager.operations_snapshot().await.queue_length,
|
||||
1,
|
||||
"the lost response path must still admit one canonical task"
|
||||
);
|
||||
|
||||
let mut retry = connect_faulty_heal_control_client(
|
||||
Arc::clone(&manager),
|
||||
fingerprint,
|
||||
metadata.coordinator_epoch,
|
||||
HealControlTransportFault::None,
|
||||
)
|
||||
.await
|
||||
.expect("retry listener should bind");
|
||||
let replayed = call_heal_control_transport(&mut retry, fingerprint, first_command.clone())
|
||||
.await
|
||||
.expect("exact retry should return the canonical receipt before restart");
|
||||
assert!(matches!(
|
||||
decode_transport_start_outcome(&replayed, &first_id, metadata.coordinator_epoch),
|
||||
rustfs_protos::heal_control::Outcome::Start {
|
||||
task_id,
|
||||
admission: rustfs_protos::heal_control::Admission::Accepted,
|
||||
} if task_id == first_id
|
||||
));
|
||||
|
||||
let restarted_manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None));
|
||||
let mut restarted_peer = connect_faulty_heal_control_client(
|
||||
Arc::clone(&restarted_manager),
|
||||
fingerprint,
|
||||
metadata.coordinator_epoch,
|
||||
HealControlTransportFault::None,
|
||||
)
|
||||
.await
|
||||
.expect("restarted peer listener should bind");
|
||||
let replayed_after_restart = call_heal_control_transport(&mut restarted_peer, fingerprint, first_command)
|
||||
.await
|
||||
.expect("restarted peer should replay an observed receipt for the exact envelope");
|
||||
assert_eq!(
|
||||
replayed_after_restart, replayed,
|
||||
"restart after receipt replay must return the same canonical receipt bytes"
|
||||
);
|
||||
assert_eq!(
|
||||
restarted_manager.operations_snapshot().await.queue_length,
|
||||
0,
|
||||
"exact replay to a restarted peer must not re-admit the destructive start"
|
||||
);
|
||||
assert!(matches!(
|
||||
restarted_manager.get_task_status(&first_id).await,
|
||||
Err(rustfs_heal::Error::TaskNotFound { .. })
|
||||
));
|
||||
|
||||
let mut fresh_request = request;
|
||||
fresh_request.id = Uuid::new_v4().to_string();
|
||||
let fresh_id = fresh_request.id.clone();
|
||||
let fresh_metadata = rustfs_protos::heal_control::RequestMetadata {
|
||||
nonce: *Uuid::new_v4().as_bytes(),
|
||||
..metadata
|
||||
};
|
||||
let fresh_command = encode_transport_start(fresh_request, fresh_metadata);
|
||||
let fresh = call_heal_control_transport(&mut restarted_peer, fingerprint, fresh_command)
|
||||
.await
|
||||
.expect("fresh forceStart after peer restart remains an explicit new start");
|
||||
assert!(matches!(
|
||||
decode_transport_start_outcome(&fresh, &fresh_id, metadata.coordinator_epoch),
|
||||
rustfs_protos::heal_control::Outcome::Start {
|
||||
task_id,
|
||||
admission: rustfs_protos::heal_control::Admission::Accepted,
|
||||
} if task_id == fresh_id && task_id != first_id
|
||||
));
|
||||
assert_eq!(
|
||||
restarted_manager.operations_snapshot().await.queue_length,
|
||||
1,
|
||||
"fresh forceStart after restart must be counted separately from receipt replay"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_executor_preserves_canonical_token_and_drops_query_results() {
|
||||
let manager = Arc::new(HealManager::new(Arc::new(HealControlMockStorage), None));
|
||||
|
||||
@@ -55,6 +55,7 @@ their issue closes.
|
||||
| `run.ps1` | dev-tool | Windows counterpart of `run.sh` | — |
|
||||
| `probe.sh` | dev-tool | Probe-style e2e run | `make probe-e2e` |
|
||||
| `run_scanner_validation_harness.sh` | dev-tool | Scanner validation harness | `docs/operations/scanner-benchmark-runbook.md` |
|
||||
| `run_scanner_heal_evidence_case.sh` | dev-tool | Runs one Scanner/Heal release-evidence registry case and checks the produced receipt/oracle | `.config/scanner-heal-required-tests.json`; `check_test_wiring.py --check-scanner-heal` |
|
||||
| `test_scanner_validation_harness.sh` | dev-tool | Self-test for the scanner validation harness | — |
|
||||
| `scanner_abba.py` | dev-tool | Scanner/heal ABBA orchestration and evidence gates via `run_scanner_validation_harness.sh --abba` | `docs/operations/scanner-benchmark-runbook.md` |
|
||||
| `test_scanner_abba.py` | dev-tool | Synthetic ABBA adapter and failure-path tests | `test_scanner_validation_harness.sh` |
|
||||
|
||||
@@ -10,7 +10,6 @@ import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import tomllib
|
||||
import unittest
|
||||
import uuid
|
||||
import xml.etree.ElementTree as ET
|
||||
@@ -19,6 +18,11 @@ from unittest import mock
|
||||
from pathlib import Path
|
||||
from zoneinfo import ZoneInfo, ZoneInfoNotFoundError
|
||||
|
||||
try:
|
||||
import tomllib
|
||||
except ModuleNotFoundError:
|
||||
import tomli as tomllib
|
||||
|
||||
from scanner_abba import MAX_JSON_BYTES, digest, number, read_json, require, sha, write_json
|
||||
|
||||
|
||||
@@ -894,6 +898,10 @@ def scanner_heal_oracle_names(root: Path) -> tuple[str, ...]:
|
||||
require(isinstance(oracle, str) and oracle.endswith(".json"), f"invalid oracle for {case_id}")
|
||||
path = Path(oracle)
|
||||
require(not path.is_absolute() and ".." not in path.parts, f"oracle path escapes run directory for {case_id}")
|
||||
require(requirement.get("evidence") in ("process-restart", "process-crash-restart"),
|
||||
f"invalid evidence for {case_id}")
|
||||
require(type(requirement.get("unclean_shutdown_marker")) is bool,
|
||||
f"invalid unclean-shutdown marker expectation for {case_id}")
|
||||
names.add(oracle)
|
||||
return tuple(sorted(names))
|
||||
|
||||
@@ -1024,9 +1032,11 @@ def check_scanner_heal_evidence(root: Path, directory: Path, case_id: str) -> li
|
||||
require(digest(path) == execution["artifacts"][requirement["oracle"]], "oracle hash mismatch")
|
||||
oracle = read_json(path)
|
||||
evidence_integer(oracle.get("schema"), "oracle schema", 1, 1)
|
||||
require(oracle.get("evidence") == "process-restart", "not real process-restart evidence")
|
||||
require(oracle.get("evidence") == requirement["evidence"], f"not real {requirement['evidence']} evidence")
|
||||
require(oracle.get("case") == name and oracle.get("run_id") == run["run_id"], "oracle belongs to another case/run")
|
||||
require(oracle.get("source_revision") == run["source_revision"], "oracle source mismatch")
|
||||
require(oracle.get("unclean_shutdown_marker") is requirement["unclean_shutdown_marker"],
|
||||
"unclean-shutdown marker evidence mismatch")
|
||||
built = oracle["test_build"]
|
||||
for key in ("source_revision", "dirty", "lock_blob", "features"):
|
||||
require(built[key] == expected_build[key], f"compiled test {key} mismatch")
|
||||
@@ -1240,7 +1250,7 @@ class SelfTests(unittest.TestCase):
|
||||
run_dir.mkdir()
|
||||
registry = read_json(ROOT / ".config/scanner-heal-required-tests.json")
|
||||
write_json(root / ".config/scanner-heal-required-tests.json", registry)
|
||||
requirement = registry["cases"]["background-target-restart"]
|
||||
requirements = registry["cases"]
|
||||
binary = directory / "fake-binary"
|
||||
binary.write_bytes(b"parser fixture, not a real build")
|
||||
binary.chmod(0o700)
|
||||
@@ -1251,14 +1261,23 @@ class SelfTests(unittest.TestCase):
|
||||
"lock_blob": "c" * 40, "features": "default"},
|
||||
"started_at": datetime.now(timezone.utc).timestamp() - 1,
|
||||
"binary": build, "test_binary": build})
|
||||
write_json(run_dir / "listing.json", {"rust-suites": {requirement["suite"]: {
|
||||
"binary-id": requirement["suite"], "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target",
|
||||
suite = "e2e_test"
|
||||
write_json(run_dir / "listing.json", {"rust-suites": {suite: {
|
||||
"binary-id": suite, "binary-path": str(binary), "package-name": "e2e_test", "build-platform": "target",
|
||||
"testcases": {
|
||||
requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}}
|
||||
}}}})
|
||||
requirement["name"]: {"ignored": False, "filter-match": {"status": "matches"}}
|
||||
for requirement in requirements.values()
|
||||
}
|
||||
}}})
|
||||
(run_dir / "junit.xml").write_text(
|
||||
f'<testsuites><testsuite><testcase name="{requirement["name"]}" classname="{requirement["suite"]}" '
|
||||
f'timestamp="{datetime.now(timezone.utc).isoformat(timespec="milliseconds")}"/></testsuite></testsuites>')
|
||||
"<testsuites><testsuite>"
|
||||
+ "".join(
|
||||
f'<testcase name="{requirement["name"]}" classname="{requirement["suite"]}" '
|
||||
f'timestamp="{datetime.now(timezone.utc).isoformat(timespec="milliseconds")}"/>'
|
||||
for requirement in requirements.values()
|
||||
)
|
||||
+ "</testsuite></testsuites>"
|
||||
)
|
||||
physical = {"has_xl_meta": True, "version_id": None, "data_dir": "data-generation",
|
||||
"erasure_index": 1, "data_blocks": 2, "parity_blocks": 2, "expected_part_numbers": [1],
|
||||
"present_part_fingerprints": {"1": {"size": 12, "sha256": "c" * 64}},
|
||||
@@ -1268,15 +1287,17 @@ class SelfTests(unittest.TestCase):
|
||||
"expected_physical": physical, "physical": physical}
|
||||
objects = [dict(obj, key=f"object-{index}") for index in range(9)]
|
||||
objects[-1] = dict(objects[-1], expected_physical=None)
|
||||
write_json(run_dir / "background-target-restart.json", {
|
||||
"schema": 1, "evidence": "process-restart", "case": "background-target-restart",
|
||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
"test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40,
|
||||
"features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""},
|
||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||
"topology": {"nodes": 4, "drives_per_node": 1}, "pid_before": 10, "pid_after": 11,
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
||||
})
|
||||
for case_id, requirement in requirements.items():
|
||||
write_json(run_dir / requirement["oracle"], {
|
||||
"schema": 1, "evidence": requirement["evidence"], "case": case_id,
|
||||
"run_id": "a" * 32, "source_revision": "b" * 40,
|
||||
"test_build": {"source_revision": "b" * 40, "dirty": False, "lock_blob": "c" * 40,
|
||||
"features": "default", "target": "aarch64-apple-darwin", "profile": "debug", "rustflags_hex": ""},
|
||||
"binary_sha256": build["sha256"], "test_binary_sha256": build["sha256"],
|
||||
"topology": requirement["topology"], "pid_before": 10, "pid_after": 11,
|
||||
"unclean_shutdown_marker": requirement["unclean_shutdown_marker"],
|
||||
"objects": objects, "node_listings": [[item["key"] for item in objects]] * 4,
|
||||
})
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
return root, run_dir
|
||||
|
||||
@@ -1332,6 +1353,19 @@ class SelfTests(unittest.TestCase):
|
||||
self.assertEqual(len(errors), 21)
|
||||
self.assertTrue(all(error.startswith("pending ") for error in errors))
|
||||
|
||||
def test_scanner_heal_crash_case_rejects_restart_oracle(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root, run_dir = self.scanner_heal_fixture(Path(tmp))
|
||||
path = run_dir / "background-target-crash.json"
|
||||
oracle = read_json(path)
|
||||
oracle["evidence"] = "process-restart"
|
||||
oracle["unclean_shutdown_marker"] = False
|
||||
write_json(path, oracle)
|
||||
(run_dir / "execution.json").unlink()
|
||||
finish_scanner_heal_receipt(run_dir, 0, root)
|
||||
|
||||
self.assertTrue(check_scanner_heal_evidence(root, run_dir, "background-target-crash"))
|
||||
|
||||
def test_scanner_heal_rejects_broken_execution_and_artifacts(self) -> None:
|
||||
for fault in ("exit", "missing", "zero", "skipped", "failed", "retry", "filtered", "ignored", "stale",
|
||||
"hash", "binary", "synthetic", "wrong-run", "same-pid", "body", "parts", "listing", "topology"):
|
||||
|
||||
@@ -30,15 +30,21 @@ def validate_report(report, *, round_number, pid, objects, budget):
|
||||
if type(report.get(key)) is not int or report[key] != value:
|
||||
raise ValueError(f"worker report mismatch: {key}")
|
||||
for key in ("raw_entries", "raw_name_bytes", "objects_before", "objects_retained",
|
||||
"versions_retained", "bytes_retained", "objects_processed"):
|
||||
"versions_retained", "bytes_retained", "objects_processed",
|
||||
"raw_page_index_committed_entries", "raw_page_index_indexed_entries"):
|
||||
if type(report.get(key)) is not int or not 0 <= report[key] <= 1048576:
|
||||
raise ValueError(f"invalid bounded counter: {key}")
|
||||
if report["raw_entries"] == 0:
|
||||
made_budgeted_object_progress = report["objects_processed"] > 0
|
||||
if report["raw_entries"] == 0 and not made_budgeted_object_progress:
|
||||
raise ValueError("nonempty fixture must observe raw entries; budget hook may not have run")
|
||||
if report["raw_entries"] > budget:
|
||||
raise ValueError("raw-entry budget exceeded; no unbudgeted tail is permitted")
|
||||
if report["objects_processed"] > budget:
|
||||
raise ValueError("object budget exceeded; no unbudgeted scan tail is permitted")
|
||||
for key in ("raw_first_entry", "raw_last_entry"):
|
||||
value = report.get(key)
|
||||
if report["raw_entries"] == 0 and made_budgeted_object_progress and value is None:
|
||||
continue
|
||||
if type(value) is not str or not 0 < len(value.encode("utf-8")) <= 512:
|
||||
raise ValueError(f"invalid raw entry marker: {key}")
|
||||
if type(report.get("snapshot_complete")) is not bool:
|
||||
|
||||
@@ -35,7 +35,7 @@
|
||||
1|crates/ecstore/src/erasure/coding/decode_reader.rs
|
||||
10|crates/ecstore/src/erasure/coding/encode.rs
|
||||
25|crates/ecstore/src/erasure/coding/erasure.rs
|
||||
4|crates/ecstore/src/layout/disks_layout.rs
|
||||
3|crates/ecstore/src/layout/disks_layout.rs
|
||||
2|crates/ecstore/src/layout/endpoint.rs
|
||||
17|crates/ecstore/src/layout/endpoints.rs
|
||||
1|crates/ecstore/src/layout/format.rs
|
||||
@@ -67,7 +67,6 @@
|
||||
5|crates/ecstore/src/store/bucket.rs
|
||||
1|crates/ecstore/src/store/heal_walk.rs
|
||||
12|crates/ecstore/src/store/init.rs
|
||||
2|crates/ecstore/src/store/init_format.rs
|
||||
3|crates/ecstore/src/store/multipart.rs
|
||||
6|crates/ecstore/src/store/object.rs
|
||||
5|crates/ecstore/src/store/rebalance/support.rs
|
||||
|
||||
Executable
+234
@@ -0,0 +1,234 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}"
|
||||
PROFILE="e2e-nightly"
|
||||
CASE_ID="background-target-crash"
|
||||
RUN_DIR=""
|
||||
PLAN_ONLY=0
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage: scripts/run_scanner_heal_evidence_case.sh [OPTIONS]
|
||||
|
||||
Run one Scanner/Heal release-evidence case through the real e2e test binary,
|
||||
then validate the produced receipt, nextest listing, JUnit, and case oracle.
|
||||
|
||||
Options:
|
||||
--case CASE Registry case to run (default: background-target-crash)
|
||||
--profile PROFILE Nextest profile to use (default: e2e-nightly)
|
||||
--run-dir DIR New evidence directory (default: target/scanner-heal-evidence/CASE-TIMESTAMP)
|
||||
--plan-only Validate registry selection and print the exact filter without running cargo
|
||||
--self-test Run lightweight CLI/registry checks without building Rust
|
||||
-h, --help Show this help
|
||||
|
||||
The script intentionally runs a single case, not the release pseudo-case. After
|
||||
a successful case run it verifies that the release gate still remains blocked.
|
||||
Set RUSTFS_E2E_TEST_PORT_MIN and RUSTFS_E2E_TEST_PORT_RANGE to move the e2e
|
||||
port allocator when the default 20000..30000 test range is unavailable.
|
||||
USAGE
|
||||
}
|
||||
|
||||
case_field() {
|
||||
local case_id="$1"
|
||||
local field="$2"
|
||||
"$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" "$field" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
registry = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
case = registry["cases"][sys.argv[2]]
|
||||
value = case[sys.argv[3]]
|
||||
if not isinstance(value, str):
|
||||
raise SystemExit(f"{sys.argv[3]} is not a string")
|
||||
print(value)
|
||||
PY
|
||||
}
|
||||
|
||||
test_filter_for() {
|
||||
local case_id="$1"
|
||||
"$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$case_id" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import sys
|
||||
|
||||
registry = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
case = registry["cases"][sys.argv[2]]
|
||||
print("test(/^" + re.escape(case["name"]) + "$/)")
|
||||
PY
|
||||
}
|
||||
|
||||
test_binary_from_listing() {
|
||||
local listing="$1"
|
||||
local case_id="$2"
|
||||
"$PYTHON_BIN" - "$ROOT/.config/scanner-heal-required-tests.json" "$listing" "$case_id" <<'PY'
|
||||
import json
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
registry = json.loads(pathlib.Path(sys.argv[1]).read_text())
|
||||
listing = json.loads(pathlib.Path(sys.argv[2]).read_text())
|
||||
case = registry["cases"][sys.argv[3]]
|
||||
suite = listing["rust-suites"][case["suite"]]
|
||||
testcase = suite["testcases"][case["name"]]
|
||||
if testcase.get("ignored") is not False or testcase.get("filter-match", {}).get("status") != "matches":
|
||||
raise SystemExit("selected case is not matched by the nextest listing")
|
||||
matches = 0
|
||||
for listed_suite in listing.get("rust-suites", {}).values():
|
||||
for listed in listed_suite.get("testcases", {}).values():
|
||||
if listed.get("filter-match", {}).get("status") == "matches":
|
||||
matches += 1
|
||||
if matches != 1:
|
||||
raise SystemExit(f"expected exactly one selected case, got {matches}")
|
||||
print(suite["binary-path"])
|
||||
PY
|
||||
}
|
||||
|
||||
release_gate_must_remain_blocked() {
|
||||
local run_dir="$1"
|
||||
local output="$run_dir/release-check.txt"
|
||||
if "$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$run_dir" release >"$output" 2>&1; then
|
||||
echo "release gate unexpectedly approved a single Scanner/Heal evidence run" >&2
|
||||
return 1
|
||||
fi
|
||||
if ! grep -Eq 'required test not selected:|pending [A-Z0-9-]+:' "$output"; then
|
||||
echo "release gate did not explain why the Scanner/Heal release remains blocked" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
run_self_test() {
|
||||
local filter
|
||||
filter="$(test_filter_for background-target-crash)"
|
||||
case "$filter" in
|
||||
*background_target_crash*) ;;
|
||||
*)
|
||||
echo "self-test failed: crash case filter missing" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
if "$0" --case release --plan-only >/dev/null 2>&1; then
|
||||
echo "self-test failed: release pseudo-case must not be runnable" >&2
|
||||
return 1
|
||||
fi
|
||||
"$0" --case background-target-crash --plan-only >/dev/null
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--case)
|
||||
CASE_ID="$2"
|
||||
shift 2
|
||||
;;
|
||||
--profile)
|
||||
PROFILE="$2"
|
||||
shift 2
|
||||
;;
|
||||
--run-dir)
|
||||
RUN_DIR="$2"
|
||||
shift 2
|
||||
;;
|
||||
--plan-only)
|
||||
PLAN_ONLY=1
|
||||
shift
|
||||
;;
|
||||
--self-test)
|
||||
run_self_test
|
||||
exit $?
|
||||
;;
|
||||
-h|--help)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
echo "unknown option: $1" >&2
|
||||
usage >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [[ "$CASE_ID" == "release" ]]; then
|
||||
echo "release is a checker-only pseudo-case; run a concrete registry case" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
case_field "$CASE_ID" name >/dev/null
|
||||
TEST_FILTER="$(test_filter_for "$CASE_ID")"
|
||||
if [[ -z "$RUN_DIR" ]]; then
|
||||
RUN_DIR="$ROOT/target/scanner-heal-evidence/${CASE_ID}-$(date -u +%Y%m%dT%H%M%SZ)"
|
||||
elif [[ "$RUN_DIR" != /* ]]; then
|
||||
RUN_DIR="$ROOT/$RUN_DIR"
|
||||
fi
|
||||
|
||||
if [[ "$PLAN_ONLY" == 1 ]]; then
|
||||
echo "case=$CASE_ID"
|
||||
echo "profile=$PROFILE"
|
||||
echo "filter=$TEST_FILTER"
|
||||
echo "run_dir=$RUN_DIR"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -e "$RUN_DIR" ]]; then
|
||||
echo "evidence run directory already exists: $RUN_DIR" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
cd "$ROOT"
|
||||
if [[ -n "$(git status --porcelain --untracked-files=no)" ]]; then
|
||||
echo "commit tracked source changes before creating evidence" >&2
|
||||
exit 1
|
||||
fi
|
||||
mkdir -p "$(dirname "$RUN_DIR")"
|
||||
TMP_DIR="$(mktemp -d "${TMPDIR:-/tmp}/rustfs-scanner-heal-evidence.XXXXXX")"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
|
||||
BUILD_FEATURES="${RUSTFS_BUILD_FEATURES:-}"
|
||||
cargo clean -p rustfs
|
||||
if [[ -n "$BUILD_FEATURES" ]]; then
|
||||
cargo build --locked -p rustfs --bins --features "$BUILD_FEATURES"
|
||||
else
|
||||
cargo build --locked -p rustfs --bins
|
||||
fi
|
||||
printf '%s' "$BUILD_FEATURES" >"$ROOT/target/debug/rustfs.features"
|
||||
|
||||
LISTING_TMP="$TMP_DIR/listing.json"
|
||||
cargo nextest list --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --message-format json >"$LISTING_TMP"
|
||||
TEST_BINARY="$(test_binary_from_listing "$LISTING_TMP" "$CASE_ID")"
|
||||
|
||||
export RUSTFS_E2E_EXPECTED_FEATURES="${RUSTFS_E2E_EXPECTED_FEATURES:-default}"
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --begin-scanner-heal "$RUN_DIR" "$ROOT/target/debug/rustfs" "$TEST_BINARY"
|
||||
cp "$LISTING_TMP" "$RUN_DIR/listing.json"
|
||||
export RUSTFS_E2E_LOG_DIR="${RUSTFS_E2E_LOG_DIR:-$RUN_DIR/e2e-logs}"
|
||||
mkdir -p "$RUSTFS_E2E_LOG_DIR"
|
||||
|
||||
JUNIT_PATH="$ROOT/target/nextest/$PROFILE/junit.xml"
|
||||
rm -f "$JUNIT_PATH"
|
||||
set +e
|
||||
NO_PROXY="${NO_PROXY:-127.0.0.1,localhost}" \
|
||||
HTTP_PROXY= \
|
||||
HTTPS_PROXY= \
|
||||
RUSTFS_SCANNER_HEAL_RUN_DIR="$RUN_DIR" \
|
||||
cargo nextest run --profile "$PROFILE" -p e2e_test -E "$TEST_FILTER" --no-tests=fail
|
||||
STATUS=$?
|
||||
set -e
|
||||
|
||||
if [[ -f "$JUNIT_PATH" ]]; then
|
||||
cp "$JUNIT_PATH" "$RUN_DIR/junit.xml"
|
||||
fi
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --finish-scanner-heal "$RUN_DIR" "$STATUS"
|
||||
|
||||
if [[ "$STATUS" -ne 0 ]]; then
|
||||
echo "Scanner/Heal evidence case failed: $CASE_ID (exit $STATUS); receipt kept at $RUN_DIR" >&2
|
||||
exit "$STATUS"
|
||||
fi
|
||||
|
||||
"$PYTHON_BIN" "$ROOT/scripts/check_test_wiring.py" --check-scanner-heal "$RUN_DIR" "$CASE_ID"
|
||||
release_gate_must_remain_blocked "$RUN_DIR"
|
||||
echo "Scanner/Heal evidence case verified: $CASE_ID"
|
||||
echo "Release gate remains blocked; details: $RUN_DIR/release-check.txt"
|
||||
echo "Evidence directory: $RUN_DIR"
|
||||
+45
-2
@@ -22,6 +22,9 @@ METRICS = (
|
||||
"p99_ms", "throughput_ops", "rss_bytes", "cpu_seconds", "iops", "rpc_count",
|
||||
"cache_clone_bytes", "encode_bytes", "save_bytes", "oldest_age_seconds",
|
||||
"walk_objects", "cold_walk_objects", "healed_objects", "errors", "requests",
|
||||
"foreground_pressure_samples", "foreground_pressure_high_samples",
|
||||
"heal_lock_wait_p99_ms", "heal_attempts", "heal_attempt_failures",
|
||||
"heal_retry_attempts",
|
||||
)
|
||||
REPEATABILITY_LIMIT = Decimal("0.05")
|
||||
P2_WORK_MULTIPLE_LIMIT = Decimal("1.2")
|
||||
@@ -72,7 +75,12 @@ def report_number(value):
|
||||
|
||||
def digest(path):
|
||||
with Path(path).open("rb") as stream:
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
if hasattr(hashlib, "file_digest"):
|
||||
return hashlib.file_digest(stream, "sha256").hexdigest()
|
||||
hasher = hashlib.sha256()
|
||||
while chunk := stream.read(1024 * 1024):
|
||||
hasher.update(chunk)
|
||||
return hasher.hexdigest()
|
||||
|
||||
|
||||
def read_json(path):
|
||||
@@ -247,6 +255,11 @@ def validate_result(result, request, expected):
|
||||
number(metrics.get(key), key)
|
||||
for key in ("requests", "p99_ms", "throughput_ops"):
|
||||
require(metrics[key] > 0, f"zero {key}")
|
||||
require(metrics["foreground_pressure_samples"] > 0, "zero foreground pressure samples")
|
||||
require(metrics["foreground_pressure_high_samples"] <= metrics["foreground_pressure_samples"],
|
||||
"foreground pressure high samples exceed samples")
|
||||
require(metrics["heal_attempt_failures"] <= metrics["heal_attempts"], "heal failures exceed attempts")
|
||||
require(metrics["heal_retry_attempts"] <= metrics["heal_attempts"], "heal retries exceed attempts")
|
||||
require(metrics["errors"] == 0, "workload request errors")
|
||||
require(metrics["cold_walk_objects"] <= metrics["walk_objects"], "cold walk exceeds total walk")
|
||||
require(result.get("oracle") == expected, "object/version/byte oracle mismatch")
|
||||
@@ -258,6 +271,18 @@ def validate_result(result, request, expected):
|
||||
return result
|
||||
|
||||
|
||||
def attempt_cost(metrics):
|
||||
healed = decimal_number(metrics["healed_objects"], "healed_objects")
|
||||
if healed == 0:
|
||||
return None
|
||||
return ratio(metrics["heal_attempts"], healed, "heal attempt cost")
|
||||
|
||||
|
||||
def pressure_high_ratio(metrics):
|
||||
return ratio(metrics["foreground_pressure_high_samples"],
|
||||
metrics["foreground_pressure_samples"], "foreground pressure high samples")
|
||||
|
||||
|
||||
def convergence(result):
|
||||
window = result.get("convergence")
|
||||
if not window or window.get("writes_stopped") is not True or window.get("last_mutation_observed") is not True or window.get("first_complete_publication") is not True:
|
||||
@@ -313,6 +338,10 @@ def evaluate(cells):
|
||||
p2_pending = any(value is None for value in candidate_p2)
|
||||
passed &= all(ratio(value, 1, "p2 work multiple") <= P2_WORK_MULTIPLE_LIMIT for value in candidate_p2 if value is not None)
|
||||
p2_report = [None if value is None else float(value) for value in p2]
|
||||
attempt_costs = [attempt_cost(cell["result"]["metrics"]) if cell["background"] == "on" else None for cell in group]
|
||||
candidate_attempt_costs = [
|
||||
value for cell, value in zip(group, attempt_costs) if cell["leg"].startswith("B") and value is not None
|
||||
]
|
||||
inconclusive |= noise or p2_pending
|
||||
if not noise and not passed:
|
||||
failed = True
|
||||
@@ -322,7 +351,21 @@ def evaluate(cells):
|
||||
"p99_regression": float(p99), "throughput_change": float(throughput),
|
||||
"thresholds": {key: float(value) for key, value in thresholds.items()},
|
||||
"p1": p1, "p2_max_work_multiple": float(P2_WORK_MULTIPLE_LIMIT),
|
||||
"p2_post_stop_work_multiples": p2_report})
|
||||
"p2_post_stop_work_multiples": p2_report,
|
||||
"w10_w11": {
|
||||
"foreground_pressure_high_sample_ratios": [
|
||||
float(pressure_high_ratio(cell["result"]["metrics"])) for cell in group
|
||||
],
|
||||
"heal_lock_wait_p99_ms": [
|
||||
cell["result"]["metrics"]["heal_lock_wait_p99_ms"] for cell in group
|
||||
],
|
||||
"attempt_cost_per_healed_object": [
|
||||
None if value is None else float(value) for value in attempt_costs
|
||||
],
|
||||
"candidate_attempt_cost_per_healed_object": (
|
||||
None if not candidate_attempt_costs else float(max(candidate_attempt_costs))
|
||||
),
|
||||
}})
|
||||
return ("fail" if failed else "inconclusive" if inconclusive else "pass"), comparisons
|
||||
|
||||
|
||||
|
||||
@@ -10,6 +10,8 @@ class ReportTests(unittest.TestCase):
|
||||
return dict(schema=1, round=0, pid=123, objects_expected=4, raw_entry_budget=16,
|
||||
raw_entries=8, raw_name_bytes=64, objects_before=0, objects_retained=4,
|
||||
versions_retained=4, bytes_retained=4, objects_processed=4,
|
||||
raw_page_index_committed_entries=4,
|
||||
raw_page_index_indexed_entries=4,
|
||||
raw_first_entry="bucket/object-0000",
|
||||
raw_last_entry="bucket/object-0003/xl.meta",
|
||||
snapshot_complete=True, outcome="complete")
|
||||
@@ -44,6 +46,11 @@ class ReportTests(unittest.TestCase):
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
report = self.report()
|
||||
report["objects_processed"] = 17
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
def test_missing_or_oversized_raw_marker_rejected(self):
|
||||
for value in (None, True, "", "x" * 513):
|
||||
with self.subTest(value=value):
|
||||
@@ -55,9 +62,19 @@ class ReportTests(unittest.TestCase):
|
||||
def test_complete_coverage_without_entry_observation_rejected(self):
|
||||
report = self.report()
|
||||
report["raw_entries"] = 0
|
||||
report["objects_processed"] = 0
|
||||
report["raw_page_index_committed_entries"] = 3
|
||||
with self.assertRaises(ValueError):
|
||||
self.validate(report)
|
||||
|
||||
def test_budgeted_object_progress_can_consume_all_raw_entries(self):
|
||||
report = self.report()
|
||||
report["raw_entries"] = 0
|
||||
report["raw_first_entry"] = None
|
||||
report["raw_last_entry"] = None
|
||||
report["objects_processed"] = 1
|
||||
self.validate(report)
|
||||
|
||||
def test_missing_wrong_type_and_negative_counter_rejected(self):
|
||||
for value in (None, True, -1, "8", 1048577):
|
||||
with self.subTest(value=value):
|
||||
|
||||
@@ -96,6 +96,12 @@ def fake_adapter():
|
||||
del result["metrics"]["save_bytes"]
|
||||
elif fault == "incomplete-repair":
|
||||
result["metrics"]["healed_objects"] = 0
|
||||
elif fault == "zero-pressure-samples":
|
||||
result["metrics"]["foreground_pressure_samples"] = 0
|
||||
elif fault == "pressure-sample-order":
|
||||
result["metrics"]["foreground_pressure_high_samples"] = result["metrics"]["foreground_pressure_samples"] + 1
|
||||
elif fault == "attempt-accounting":
|
||||
result["metrics"]["heal_attempt_failures"] = result["metrics"]["heal_attempts"] + 1
|
||||
harness.write_json(Path(output_path), result)
|
||||
return 0
|
||||
|
||||
@@ -271,10 +277,18 @@ class ScannerAbbaTest(unittest.TestCase):
|
||||
legs = [r for r in requests if (r["scenario"], r["comparison"], r["round"]) == (scenario, comparison, round_id)]
|
||||
self.assertEqual({r["leg"] for r in legs}, set(harness.LEGS))
|
||||
self.assertTrue(all(c["p2_max_work_multiple"] == 1.2 for c in report["comparisons"]))
|
||||
for comparison in report["comparisons"]:
|
||||
w10_w11 = comparison["w10_w11"]
|
||||
self.assertEqual(w10_w11["foreground_pressure_high_sample_ratios"], [1.0, 1.0, 1.0, 1.0])
|
||||
self.assertEqual(w10_w11["heal_lock_wait_p99_ms"], [10, 10, 10, 10])
|
||||
expected_attempt_cost = [None, 1.0, 1.0, None] if comparison["comparison"] == "background" else [1.0, 1.0, 1.0, 1.0]
|
||||
self.assertEqual(w10_w11["attempt_cost_per_healed_object"], expected_attempt_cost)
|
||||
self.assertEqual(w10_w11["candidate_attempt_cost_per_healed_object"], 1.0)
|
||||
|
||||
def test_fail_closed_adapter_and_data_errors(self):
|
||||
for fault in ("measure-exit", "oracle-exit", "missing-oracle", "oracle-mismatch", "zero-samples",
|
||||
"zero-requests", "request-errors", "load-drift", "missing-metric", "incomplete-repair"):
|
||||
"zero-requests", "request-errors", "load-drift", "missing-metric", "incomplete-repair",
|
||||
"zero-pressure-samples", "pressure-sample-order", "attempt-accounting"):
|
||||
with self.subTest(fault=fault), tempfile.TemporaryDirectory() as directory:
|
||||
self.root = Path(directory)
|
||||
with self.assertRaises((ValueError, OSError, subprocess.SubprocessError)):
|
||||
|
||||
Reference in New Issue
Block a user