mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-21 07:26:37 +00:00
Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 149f5700b4 | |||
| 38aa318fb2 | |||
| e7c5d7c198 |
@@ -129,12 +129,6 @@ when restoring `/rustfs/console` for a custom OEM build.
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_SET_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_MAX_CONCURRENT_DISK_SCANS` (canonical)
|
||||
- `RUSTFS_SCANNER_WALKERS_PER_DISK` (canonical)
|
||||
- `RUSTFS_SCANNER_READ_ONLY_HANDOFF` (canonical, default `false`)
|
||||
|
||||
Read-only restart continuity:
|
||||
|
||||
- `RUSTFS_SCANNER_READ_ONLY_HANDOFF=true` enables reuse of unpublished scanner progress across verified read-only process restarts. Unset, false or invalid values disable both identity restoration and receipt creation. Existing receipts are still durably consumed before recovery starts, even when the feature is disabled.
|
||||
- Deploy receipt-aware readers to every possible writer before enabling this flag. Before rolling back to an older binary, disable it on every node, start each node with all its drives present, wait for readiness, and shut down cleanly. Do not start the older binary until every node completes this preparation. Direct rollback after enabling receipt writes is unsupported: older startup/recovery paths can write without durably invalidating the receipt. See [the receipt and rollback contract](../../docs/architecture/scanner-usage-publication.md#read-only-restart-continuity).
|
||||
|
||||
Scanner cycle budget controls:
|
||||
|
||||
|
||||
@@ -21,11 +21,6 @@ pub const APP_NAME: &str = "RustFS";
|
||||
/// Cargo package version for this workspace member.
|
||||
pub const VERSION: &str = env!("CARGO_PKG_VERSION");
|
||||
|
||||
/// Opt in to scanner checkpoint continuity across verified read-only restarts.
|
||||
/// Defaults to false. Enable only after all possible writers support receipt
|
||||
/// consumption; disable and restart before rolling back to an older writer.
|
||||
pub const ENV_SCANNER_READ_ONLY_HANDOFF: &str = "RUSTFS_SCANNER_READ_ONLY_HANDOFF";
|
||||
|
||||
/// Default configuration logger level
|
||||
/// Default value: error
|
||||
/// Environment variable: RUSTFS_OBS_LOGGER_LEVEL
|
||||
|
||||
@@ -34,7 +34,10 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_s3_types::Body;
|
||||
@@ -396,6 +399,54 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1758,129 +1758,6 @@ mod upgrade_write_readiness_tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn stop_scanner_handoff_server(env: &mut RustFSTestEnvironment) -> TestResult {
|
||||
let process = env.process.as_mut().ok_or("handoff server is not running")?;
|
||||
signal_process(process.id(), "TERM")?;
|
||||
tokio::time::timeout(Duration::from_secs(90), async {
|
||||
loop {
|
||||
if let Some(status) = process.try_wait()? {
|
||||
assert!(status.success(), "handoff shutdown failed: {status}");
|
||||
return Ok::<(), BoxError>(());
|
||||
}
|
||||
sleep(Duration::from_millis(50)).await;
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
env.process.take();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn scanner_handoff_identity(env: &RustFSTestEnvironment) -> Result<String, BoxError> {
|
||||
let receipt: Value = serde_json::from_slice(&std::fs::read(Path::new(&env.temp_dir).join(".rustfs.sys/unclean-shutdown"))?)?;
|
||||
assert_eq!(receipt["schema"], 1);
|
||||
assert_eq!(receipt["owner"], env.address);
|
||||
Ok(receipt["checkpoint_instance_id"]
|
||||
.as_str()
|
||||
.ok_or("handoff receipt omitted checkpoint identity")?
|
||||
.to_owned())
|
||||
}
|
||||
|
||||
async fn assert_scanner_handoff_consumed(marker: &Path) -> TestResult {
|
||||
let marker = tokio::fs::read_to_string(marker).await?;
|
||||
// Heal may replace the startup marker with its ordinary timestamp before
|
||||
// readiness. Neither form carries the prior checkpoint identity.
|
||||
assert!(
|
||||
marker == "running" || marker.parse::<u64>().is_ok(),
|
||||
"readiness must expose an ordinary unclean marker, never a handoff receipt"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_read_only_handoff_requires_explicit_opt_in() -> TestResult {
|
||||
const HANDOFF: &str = "RUSTFS_SCANNER_READ_ONLY_HANDOFF";
|
||||
assert!(std::env::var_os(HANDOFF).is_none(), "this scenario tests the unset default");
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let marker = Path::new(&env.temp_dir).join(".rustfs.sys/unclean-shutdown");
|
||||
env.start_rustfs_server_without_cleanup(vec![]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert!(!marker.exists(), "the default must not create a receipt");
|
||||
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
let previous = scanner_handoff_identity(&env)?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
assert_scanner_handoff_consumed(&marker).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert_eq!(scanner_handoff_identity(&env)?, previous, "opted-in read-only boots retain identity");
|
||||
|
||||
for disabled in ["invalid", "false"] {
|
||||
let before_disabled = scanner_handoff_identity(&env)?;
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, disabled)])
|
||||
.await?;
|
||||
assert_scanner_handoff_consumed(&marker).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert!(!marker.exists(), "disabled shutdown must not recreate the receipt");
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert_ne!(scanner_handoff_identity(&env)?, before_disabled, "opt-out breaks the prior lineage");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Exercise the receipt protocol in the existing pinned-release rollback lane.
|
||||
async fn assert_scanner_handoff_prepares_legacy_rollback(previous_binary: &Path) -> TestResult {
|
||||
const HANDOFF: &str = "RUSTFS_SCANNER_READ_ONLY_HANDOFF";
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let marker = Path::new(&env.temp_dir).join(".rustfs.sys/unclean-shutdown");
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
let previous = scanner_handoff_identity(&env)?;
|
||||
|
||||
// Complete the current reader's rollback preparation before any old writer
|
||||
// opens storage, including an old writer with all background services off.
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "false")])
|
||||
.await?;
|
||||
assert_scanner_handoff_consumed(&marker).await?;
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert!(!marker.exists());
|
||||
env.start_rustfs_server_from_binary_without_cleanup(
|
||||
previous_binary,
|
||||
vec![],
|
||||
&[
|
||||
("RUSTFS_SCANNER_ENABLED", "false"),
|
||||
("RUSTFS_HEAL_ENABLED", "false"),
|
||||
("RUSTFS_HEAL_AUTO_HEAL_ENABLE", "false"),
|
||||
("RUSTFS_HEAL_MRF_ENABLE", "false"),
|
||||
],
|
||||
)
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
env.create_test_bucket("handoff-rollback").await?;
|
||||
client
|
||||
.put_object()
|
||||
.bucket("handoff-rollback")
|
||||
.key("legacy-write")
|
||||
.body(ByteStream::from_static(b"legacy writer survived rollback preparation"))
|
||||
.send()
|
||||
.await?;
|
||||
env.stop_server();
|
||||
assert!(!marker.exists(), "the legacy crash cannot resurrect a consumed receipt");
|
||||
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[(HANDOFF, "true")]).await?;
|
||||
assert_eq!(
|
||||
read_object(&client, "handoff-rollback", "legacy-write", None).await?.1,
|
||||
b"legacy writer survived rollback preparation"
|
||||
);
|
||||
stop_scanner_handoff_server(&mut env).await?;
|
||||
assert_ne!(
|
||||
scanner_handoff_identity(&env)?,
|
||||
previous,
|
||||
"a legacy write must not retain the old lineage"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Pins the published old writer's limitation and the supported recovery
|
||||
/// procedure. This is not a promise that mixed-version ODM is supported.
|
||||
/// Replace the loss assertion when ODM gains independent persistence;
|
||||
@@ -3585,7 +3462,6 @@ async fn rollback_to_previous_release_reads_current_bucket_metadata() -> TestRes
|
||||
init_logging();
|
||||
let previous_binary = source_binary()?;
|
||||
let evidence_context = g09_evidence_context(&previous_binary).await?;
|
||||
assert_scanner_handoff_prepares_legacy_rollback(&previous_binary).await?;
|
||||
|
||||
let replication_target = FakeS3Target::start().await?;
|
||||
replication_target.create_bucket(ROLLBACK_REPLICA_BUCKET);
|
||||
|
||||
@@ -503,7 +503,7 @@ pub mod disk {
|
||||
PoolMetaNativeInspection, PoolMetaNativeObject, PoolMetaNativeRecoveryRequest, PoolMetaNativeRecoveryResponse,
|
||||
QUOTA_MUTATION_FENCE_METADATA_SUFFIX, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, RUSTFS_META_TMP_BUCKET,
|
||||
ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE, SnapshotLeaseToken,
|
||||
UNCLEAN_SHUTDOWN_MARKER_PATH, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
UpdateMetadataOpts, VolumeInfo, WalkDirOptions, is_quota_mutation_fence_path, new_disk,
|
||||
validate_batch_read_version_item_count, validate_format_control_path,
|
||||
};
|
||||
pub use crate::disk::{
|
||||
@@ -769,8 +769,8 @@ pub mod storage {
|
||||
pub use crate::store::external_rpc_disk;
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, find_local_disk_by_ref_for_scanner_rpc,
|
||||
init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
|
||||
init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
pub use crate::store::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
|
||||
@@ -5082,14 +5082,7 @@ mod tests {
|
||||
let cert = generate_simple_self_signed(vec!["localhost".to_string()]).expect("certificate should generate");
|
||||
let (port, server) = spawn_https_server(&cert, 2);
|
||||
let endpoint = format!("https://localhost:{port}");
|
||||
let strict_client = s3_client_for_test(port, None);
|
||||
// A failed trust check must consume just one of the two fixture connections.
|
||||
let strict_config = strict_client
|
||||
.config()
|
||||
.to_builder()
|
||||
.retry_config(aws_sdk_s3::config::retry::RetryConfig::disabled())
|
||||
.build();
|
||||
let strict = target_client_for_test("arn:strict", endpoint.clone(), S3Client::from_conf(strict_config));
|
||||
let strict = target_client_for_test("arn:strict", endpoint.clone(), s3_client_for_test(port, None));
|
||||
let insecure = target_client_for_test(
|
||||
"arn:insecure",
|
||||
endpoint,
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_bo
|
||||
use crate::error::{Error, Result};
|
||||
use crate::storage_api_contracts::internode::{
|
||||
SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
};
|
||||
use crate::{
|
||||
@@ -192,7 +192,6 @@ fn validate_signal_service_protocol(sig: u64, sub_sys: &str, protocol_version: u
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct ScannerPeerActivity {
|
||||
pub instance_id: String,
|
||||
pub checkpoint_instance_id: Option<String>,
|
||||
pub namespace_generation: u64,
|
||||
pub maintenance_generation: u64,
|
||||
pub protocol_version: u32,
|
||||
@@ -355,9 +354,6 @@ fn decode_scanner_activity_with_verifier(
|
||||
challenge: &[u8; 16],
|
||||
verify_proof: impl FnOnce(&[u8], &[u8]) -> Result<()>,
|
||||
) -> Result<ScannerPeerActivity> {
|
||||
if response.protocol_version != SCANNER_ACTIVITY_PROTOCOL_VERSION && response.checkpoint_instance_id.is_some() {
|
||||
return Err(Error::other("legacy scanner activity peer returned unauthenticated checkpoint identity"));
|
||||
}
|
||||
let instance_id = &response.instance_id;
|
||||
if !scanner_instance_id_is_valid(instance_id) {
|
||||
return Err(Error::other("peer returned an invalid scanner activity instance ID"));
|
||||
@@ -427,7 +423,7 @@ fn decode_scanner_activity_with_verifier(
|
||||
None,
|
||||
)
|
||||
}
|
||||
SCANNER_ACTIVITY_V7_PROTOCOL_VERSION | SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => {
|
||||
if response.dirty_usage_pending && response.dirty_usage_generation == 0 {
|
||||
return Err(Error::other("scanner activity peer returned pending dirty usage without a generation"));
|
||||
}
|
||||
@@ -440,20 +436,8 @@ fn decode_scanner_activity_with_verifier(
|
||||
if movement_generation == u64::MAX {
|
||||
return Err(Error::other("scanner activity peer exhausted its movement generation"));
|
||||
}
|
||||
let canonical = if response.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION {
|
||||
let instance = response
|
||||
.checkpoint_instance_id
|
||||
.as_deref()
|
||||
.filter(|instance| scanner_instance_id_is_valid(instance))
|
||||
.ok_or_else(|| Error::other("scanner activity peer omitted its checkpoint identity"))?;
|
||||
if instance.bytes().all(|byte| byte == b'0') {
|
||||
return Err(Error::other("scanner activity peer returned an empty checkpoint identity"));
|
||||
}
|
||||
rustfs_protos::canonical_scanner_activity_v8_response_body(challenge, &response)
|
||||
} else {
|
||||
rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
|
||||
}
|
||||
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
|
||||
let canonical = rustfs_protos::canonical_scanner_activity_v7_response_body(challenge, &response)
|
||||
.map_err(|_| Error::other("scanner activity peer response is too large to authenticate"))?;
|
||||
verify_proof(&canonical, &response.response_proof)?;
|
||||
(
|
||||
Some(
|
||||
@@ -474,7 +458,6 @@ fn decode_scanner_activity_with_verifier(
|
||||
};
|
||||
Ok(ScannerPeerActivity {
|
||||
instance_id: response.instance_id,
|
||||
checkpoint_instance_id: response.checkpoint_instance_id,
|
||||
namespace_generation: response.namespace_generation,
|
||||
maintenance_generation: response.maintenance_generation,
|
||||
protocol_version: response.protocol_version,
|
||||
@@ -2469,15 +2452,11 @@ impl PeerRestClient {
|
||||
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
let previous = self
|
||||
.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if previous.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
previous
|
||||
}
|
||||
// A v6 peer cannot parse the v7 marker. Its authenticated
|
||||
// response is still decoded as untrusted terminal state, so the
|
||||
// scanner will defer publication until every peer is v7.
|
||||
self.scanner_activity_request_with_protocol(String::new(), 0, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
result
|
||||
}
|
||||
@@ -2744,15 +2723,8 @@ impl PeerRestClient {
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if result.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
let previous = self
|
||||
.scanner_activity_request_with_protocol(instance_id.clone(), generation, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION)
|
||||
.await;
|
||||
if previous.as_ref().err().is_some_and(scanner_activity_protocol_unsupported) {
|
||||
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
previous
|
||||
}
|
||||
self.scanner_activity_request_with_protocol(instance_id, generation, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION)
|
||||
.await
|
||||
} else {
|
||||
result
|
||||
}
|
||||
@@ -3708,7 +3680,6 @@ mod tests {
|
||||
#[test]
|
||||
fn scanner_scoped_dirty_usage_ack_reconciliation_requires_same_clean_instance() {
|
||||
let activity = |instance_id: &str, generation, pending| ScannerPeerActivity {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: instance_id.to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
@@ -3990,44 +3961,9 @@ mod tests {
|
||||
.await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_checkpoint_identity_requires_v8_proof_and_preserves_rpc_epoch() {
|
||||
let response = ScannerActivityResponse {
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
instance_id: "1".repeat(32),
|
||||
checkpoint_instance_id: Some("2".repeat(32)),
|
||||
topology_digest: vec![7; 32].into(),
|
||||
movement_generation: Some(0),
|
||||
publication_blocked: Some(false),
|
||||
response_proof: b"proof".to_vec().into(),
|
||||
..Default::default()
|
||||
};
|
||||
let decoded = decode_test_scanner_activity(response.clone()).expect("authenticated v8");
|
||||
assert_eq!(decoded.instance_id, "1".repeat(32));
|
||||
assert_eq!(decoded.checkpoint_instance_id, Some("2".repeat(32)));
|
||||
for checkpoint_instance_id in [None, Some(String::new()), Some("0".repeat(32)), Some("A".repeat(32))] {
|
||||
let malformed = ScannerActivityResponse {
|
||||
checkpoint_instance_id,
|
||||
..response.clone()
|
||||
};
|
||||
assert!(decode_test_scanner_activity(malformed).is_err(), "invalid continuity must fail closed");
|
||||
}
|
||||
let v7 = ScannerActivityResponse {
|
||||
protocol_version: SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
..response
|
||||
};
|
||||
assert!(decode_test_scanner_activity(v7.clone()).is_err(), "legacy proof does not bind continuity");
|
||||
let v7 = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
..v7
|
||||
};
|
||||
assert!(decode_test_scanner_activity(v7).is_ok(), "v7 without continuity remains supported");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_requires_restart_safe_peer_identity() {
|
||||
let legacy = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4044,7 +3980,6 @@ mod tests {
|
||||
assert_eq!(
|
||||
legacy,
|
||||
ScannerPeerActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4059,7 +3994,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let previous = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4076,7 +4010,6 @@ mod tests {
|
||||
assert_eq!(
|
||||
previous,
|
||||
ScannerPeerActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4091,7 +4024,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let v6 = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4110,7 +4042,6 @@ mod tests {
|
||||
assert_eq!(v6.dirty_usage_generation, Some(11));
|
||||
|
||||
let malformed_topology = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4131,7 +4062,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let missing_instance = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: String::new(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4152,7 +4082,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let malformed_instance = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "ABCDEF0123456789ABCDEF0123456789".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4173,7 +4102,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let activity = decode_test_scanner_activity(ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4190,7 +4118,6 @@ mod tests {
|
||||
assert_eq!(
|
||||
activity,
|
||||
ScannerPeerActivity {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4205,7 +4132,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let missing_movement_generation = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4226,7 +4152,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let pending_without_generation = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4247,7 +4172,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let previous_with_dirty_usage = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4268,7 +4192,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let legacy_with_topology = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4289,7 +4212,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let unsupported_protocol = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -4310,7 +4232,6 @@ mod tests {
|
||||
);
|
||||
|
||||
let missing_proof = ScannerActivityResponse {
|
||||
checkpoint_instance_id: Some("1123456789abcdef0123456789abcdef".to_owned()),
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
use crate::disk::cleanup_runtime;
|
||||
use crate::disk::io_admission::{
|
||||
AdmissionReader, BoxedChunkReader, DiskIoAdmission, DiskIoClass, DiskIoContext, current_disk_io_context,
|
||||
current_resource_unit, with_detached_disk_io_context, with_disk_io_context, with_disk_io_permit, with_optional_resource_unit,
|
||||
current_resource_unit, with_disk_io_context, with_disk_io_permit, with_optional_resource_unit,
|
||||
};
|
||||
use crate::disk::{
|
||||
CheckPartsResp, DataDirDeleteStatus, DeleteOptions, DiskAPI, DiskError, DiskInfo, DiskInfoOptions, DiskLocation, Endpoint,
|
||||
@@ -479,13 +479,10 @@ where
|
||||
if external_guard.is_none() {
|
||||
return operation().await;
|
||||
}
|
||||
tokio::spawn(with_optional_resource_unit(
|
||||
current_resource_unit(),
|
||||
with_detached_disk_io_context(async move {
|
||||
let _external_guard = external_guard;
|
||||
operation().await
|
||||
}),
|
||||
))
|
||||
tokio::spawn(with_optional_resource_unit(current_resource_unit(), async move {
|
||||
let _external_guard = external_guard;
|
||||
operation().await
|
||||
}))
|
||||
.await
|
||||
.map_err(|_| Error::other("owned mutation task failed"))?
|
||||
}
|
||||
@@ -1831,7 +1828,6 @@ impl LocalDiskWrapper {
|
||||
current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: current_resource_unit(),
|
||||
@@ -1874,10 +1870,10 @@ impl LocalDiskWrapper {
|
||||
|
||||
async fn acquire_disk_io_permit(&self) -> Result<Option<crate::disk::io_admission::DiskIoPermit>> {
|
||||
if let Some(context) = current_disk_io_context() {
|
||||
if let Some(permit) = context.permit.as_ref()
|
||||
if let Some(permit) = context.permit
|
||||
&& permit.is_for(&self.io_admission)
|
||||
{
|
||||
return Ok(Some(permit.clone()));
|
||||
return Ok(Some(permit));
|
||||
}
|
||||
if context
|
||||
.admission
|
||||
@@ -1887,8 +1883,9 @@ impl LocalDiskWrapper {
|
||||
return Ok(None);
|
||||
}
|
||||
if context.class == DiskIoClass::Background {
|
||||
return context
|
||||
.background_permit(&self.io_admission)
|
||||
return self
|
||||
.io_admission
|
||||
.background(&context.cancellation)
|
||||
.await
|
||||
.map(Some)
|
||||
.map_err(DiskError::Io);
|
||||
@@ -4412,29 +4409,6 @@ mod tests {
|
||||
assert!(matches!(wrapper.read_all(volume, &body_path).await, Err(DiskError::FileNotFound)));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_retains_background_class_without_waiter_cancellation() {
|
||||
let cancellation = CancellationToken::new();
|
||||
let caller = cancellation.clone();
|
||||
let guard: Arc<dyn Send + Sync> = Arc::new(());
|
||||
crate::disk::with_background_disk_io(
|
||||
cancellation,
|
||||
run_owned_mutation(Some(guard), move || async move {
|
||||
caller.cancel();
|
||||
let context = current_disk_io_context().expect("publication owner retains background class");
|
||||
assert_eq!(context.class, DiskIoClass::Background);
|
||||
assert!(!context.cancellation.is_cancelled(), "waiter cancellation must not interrupt publication");
|
||||
assert!(
|
||||
context.admission_deadline.is_some(),
|
||||
"queued publication must retain a bounded admission lifetime"
|
||||
);
|
||||
Ok(())
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.expect("owned publication completes");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn owned_mutation_keeps_publication_guard_after_waiter_cancellation() {
|
||||
use rustfs_concurrency::adaptive::AdaptiveAdmission;
|
||||
|
||||
@@ -161,24 +161,11 @@ pub(crate) enum DiskIoClass {
|
||||
pub(crate) struct DiskIoContext {
|
||||
pub(crate) class: DiskIoClass,
|
||||
pub(crate) cancellation: CancellationToken,
|
||||
pub(crate) admission_deadline: Option<Instant>,
|
||||
pub(crate) admission: Option<Arc<DiskIoAdmission>>,
|
||||
pub(crate) permit: Option<DiskIoPermit>,
|
||||
pub(crate) resource_unit: Option<ResourceIoUnit>,
|
||||
}
|
||||
|
||||
impl DiskIoContext {
|
||||
pub(crate) async fn background_permit(&self, admission: &Arc<DiskIoAdmission>) -> io::Result<DiskIoPermit> {
|
||||
let acquire = admission.background_with_resource(&self.cancellation, self.resource_unit.clone());
|
||||
match self.admission_deadline {
|
||||
Some(deadline) => tokio::time::timeout_at(deadline, acquire)
|
||||
.await
|
||||
.map_err(|_| io::Error::new(io::ErrorKind::TimedOut, "owned background disk admission timed out"))?,
|
||||
None => acquire.await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Mark disk I/O issued by `future` as background work for local-disk admission.
|
||||
#[doc(hidden)]
|
||||
pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken, future: F) -> F::Output {
|
||||
@@ -186,7 +173,6 @@ pub async fn with_background_disk_io<F: Future>(cancellation: CancellationToken,
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Background,
|
||||
cancellation,
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: current_resource_unit(),
|
||||
@@ -259,7 +245,6 @@ pub(crate) async fn with_disk_io_permit<F: Future>(permit: Option<DiskIoPermit>,
|
||||
let mut context = current_disk_io_context().unwrap_or_else(|| DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: None,
|
||||
@@ -294,34 +279,6 @@ pub(crate) fn with_optional_disk_io_context<F: Future>(
|
||||
}
|
||||
}
|
||||
|
||||
/// An owned commit outlives its waiter. Retain an outer I/O class and physical
|
||||
/// ownership, but give admission its own cancellation lifetime. Leave per-disk
|
||||
/// scopes to the caller's existing ownership handoff: inheriting their class
|
||||
/// without a lease could wait behind the parent awaiting this commit.
|
||||
/// The drive timeout bounds only queued admission; an admitted mutation still
|
||||
/// drains to completion. Nested owners retain the original admission deadline.
|
||||
pub(crate) fn with_detached_disk_io_context<F: Future>(future: F) -> impl Future<Output = F::Output> {
|
||||
let context = current_disk_io_context()
|
||||
.filter(|context| context.admission.is_none() && context.permit.is_none())
|
||||
.map(|mut context| {
|
||||
context.cancellation = CancellationToken::new();
|
||||
let timeout = super::disk_store::get_max_timeout_duration();
|
||||
if context.class == DiskIoClass::Background
|
||||
&& !timeout.is_zero()
|
||||
&& let Some(deadline) = Instant::now().checked_add(timeout)
|
||||
{
|
||||
context.admission_deadline = Some(context.admission_deadline.map_or(deadline, |old| old.min(deadline)));
|
||||
}
|
||||
context
|
||||
});
|
||||
async move {
|
||||
match context {
|
||||
Some(context) => with_disk_io_context(context, future).await,
|
||||
None => future.await,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn current_disk_io_context() -> Option<DiskIoContext> {
|
||||
DISK_IO_CONTEXT
|
||||
.try_with(|context| {
|
||||
@@ -340,17 +297,17 @@ pub(crate) async fn acquire_context_disk_io_permit() -> io::Result<Option<DiskIo
|
||||
let Some(context) = current_disk_io_context() else {
|
||||
return Ok(None);
|
||||
};
|
||||
let Some(admission) = context.admission.as_ref() else {
|
||||
let Some(admission) = context.admission else {
|
||||
return Ok(None);
|
||||
};
|
||||
if let Some(permit) = context.permit.as_ref()
|
||||
&& permit.is_for(admission)
|
||||
if let Some(permit) = context.permit
|
||||
&& permit.is_for(&admission)
|
||||
{
|
||||
return Ok(Some(permit.clone()));
|
||||
return Ok(Some(permit));
|
||||
}
|
||||
match context.class {
|
||||
DiskIoClass::Foreground => Ok(Some(admission.foreground())),
|
||||
DiskIoClass::Background => Ok(Some(context.background_permit(admission).await?)),
|
||||
DiskIoClass::Background => Ok(Some(admission.background(&context.cancellation).await?)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -424,7 +381,6 @@ impl DiskIoAdmission {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) async fn background(self: &Arc<Self>, cancellation: &CancellationToken) -> std::io::Result<DiskIoPermit> {
|
||||
self.background_with_resource(cancellation, current_resource_unit()).await
|
||||
}
|
||||
@@ -707,8 +663,11 @@ impl<R> AdmissionReader<R> {
|
||||
let mut wait = self.wait.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if wait.is_none() {
|
||||
let admission = self.admission.clone();
|
||||
let context = self.context.clone();
|
||||
*wait = Some(Box::pin(async move { context.background_permit(&admission).await }));
|
||||
let cancellation = self.context.cancellation.clone();
|
||||
let resource_unit = self.context.resource_unit.clone();
|
||||
*wait = Some(Box::pin(
|
||||
async move { admission.background_with_resource(&cancellation, resource_unit).await },
|
||||
));
|
||||
}
|
||||
let Some(wait_future) = wait.as_mut() else {
|
||||
return Poll::Ready(Err(io::Error::other("background disk I/O admission waiter is missing")));
|
||||
@@ -950,7 +909,6 @@ mod tests {
|
||||
let existing = DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: cancellation.clone(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: None,
|
||||
@@ -996,94 +954,6 @@ mod tests {
|
||||
assert_eq!(gate.snapshot().active, 0);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn detached_commit_keeps_background_admission_after_waiter_cancellation() {
|
||||
let admission = DiskIoAdmission::new("detached-commit".to_string());
|
||||
let foreground = admission.foreground();
|
||||
let cancellation = CancellationToken::new();
|
||||
let worker_admission = admission.clone();
|
||||
with_background_disk_io(cancellation.clone(), async {
|
||||
let worker = tokio::spawn(with_detached_disk_io_context(async move {
|
||||
let context = current_disk_io_context().expect("owned commit retains its class");
|
||||
assert_eq!(context.class, DiskIoClass::Background);
|
||||
context.background_permit(&worker_admission).await
|
||||
}));
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
cancellation.cancel();
|
||||
tokio::task::yield_now().await;
|
||||
assert!(!worker.is_finished(), "caller cancellation must not abandon a queued commit");
|
||||
drop(foreground);
|
||||
let permit = tokio::time::timeout(Duration::from_secs(1), worker)
|
||||
.await
|
||||
.expect("commit admission should finish after foreground work")
|
||||
.expect("owned worker joins")
|
||||
.expect("owned commit survives waiter cancellation");
|
||||
assert_eq!(admission.background_admissions_for_tests(), 1);
|
||||
drop(permit);
|
||||
})
|
||||
.await;
|
||||
assert!(!admission.background_active_for_tests());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn detached_commit_does_not_inherit_a_parent_disk_lease() {
|
||||
let admission = DiskIoAdmission::new("parent-lease".to_string());
|
||||
let cancellation = CancellationToken::new();
|
||||
let permit = admission.background(&cancellation).await.expect("parent disk admission");
|
||||
with_background_disk_io(
|
||||
cancellation,
|
||||
with_disk_io_permit(Some(permit), async {
|
||||
tokio::spawn(with_detached_disk_io_context(async {
|
||||
assert!(current_disk_io_context().is_none(), "do not queue behind the parent's disk lease");
|
||||
}))
|
||||
.await
|
||||
.expect("resource-only commit joins while parent keeps its disk lease");
|
||||
assert!(admission.background_active_for_tests());
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
assert!(!admission.background_active_for_tests());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn detached_commit_bounds_queued_admission_without_extending_an_existing_deadline() {
|
||||
let admission = DiskIoAdmission::new("stalled-background-disk");
|
||||
let held = admission
|
||||
.background(&CancellationToken::new())
|
||||
.await
|
||||
.expect("existing native I/O");
|
||||
let deadline = Instant::now() + Duration::from_secs(2);
|
||||
let worker_admission = admission.clone();
|
||||
let error = with_background_disk_io(CancellationToken::new(), async {
|
||||
let mut context = current_disk_io_context().expect("background scope");
|
||||
context.admission_deadline = Some(deadline);
|
||||
with_disk_io_context(context, async {
|
||||
let worker = tokio::spawn(with_detached_disk_io_context(async move {
|
||||
let context = current_disk_io_context().expect("owned scope");
|
||||
assert_eq!(
|
||||
context.admission_deadline,
|
||||
Some(deadline),
|
||||
"nested ownership must not reset admission time"
|
||||
);
|
||||
context.background_permit(&worker_admission).await
|
||||
}));
|
||||
wait_for_background_waiters(&admission, 1).await;
|
||||
tokio::time::advance(Duration::from_secs(2)).await;
|
||||
worker
|
||||
.await
|
||||
.expect("owned admission worker")
|
||||
.expect_err("stalled disk must not retain queued commit")
|
||||
})
|
||||
.await
|
||||
})
|
||||
.await;
|
||||
assert_eq!(error.kind(), io::ErrorKind::TimedOut);
|
||||
assert_eq!(admission.background_waiters.load(Ordering::Acquire), 0);
|
||||
assert!(admission.background_active_for_tests(), "timeout must not interrupt admitted native I/O");
|
||||
drop(held);
|
||||
assert!(!admission.background_active_for_tests());
|
||||
}
|
||||
|
||||
struct BlockingWorker(tokio::task::JoinHandle<()>);
|
||||
|
||||
async fn wait_for_background_waiters(admission: &DiskIoAdmission, expected: usize) {
|
||||
@@ -1202,7 +1072,6 @@ mod tests {
|
||||
DiskIoContext {
|
||||
class: DiskIoClass::Foreground,
|
||||
cancellation: CancellationToken::new(),
|
||||
admission_deadline: None,
|
||||
admission: None,
|
||||
permit: None,
|
||||
resource_unit: None,
|
||||
|
||||
@@ -79,21 +79,12 @@ impl BackgroundMemoryBudget {
|
||||
pub(crate) fn reserve(self: &Arc<Self>, bytes: usize) -> io::Result<Reservation> {
|
||||
let bytes = u64::try_from(bytes).map_err(|_| io::Error::from(io::ErrorKind::OutOfMemory))?;
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let expired = Instant::now() >= state.valid_until;
|
||||
let exhausted = || {
|
||||
io::Error::new(
|
||||
io::ErrorKind::WouldBlock,
|
||||
BackgroundMemoryExhausted {
|
||||
requested: bytes,
|
||||
used: state.used,
|
||||
limit: state.limit,
|
||||
expired,
|
||||
},
|
||||
)
|
||||
};
|
||||
let used = state.used.checked_add(bytes).ok_or_else(exhausted)?;
|
||||
if bytes != 0 && (expired || used > state.limit) {
|
||||
return Err(exhausted());
|
||||
let used = state
|
||||
.used
|
||||
.checked_add(bytes)
|
||||
.ok_or_else(|| io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted))?;
|
||||
if bytes != 0 && (Instant::now() >= state.valid_until || used > state.limit) {
|
||||
return Err(io::Error::new(io::ErrorKind::WouldBlock, BackgroundMemoryExhausted));
|
||||
}
|
||||
state.used = used;
|
||||
Ok(Reservation {
|
||||
@@ -104,20 +95,11 @@ impl BackgroundMemoryBudget {
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct BackgroundMemoryExhausted {
|
||||
requested: u64,
|
||||
used: u64,
|
||||
limit: u64,
|
||||
expired: bool,
|
||||
}
|
||||
struct BackgroundMemoryExhausted;
|
||||
|
||||
impl std::fmt::Display for BackgroundMemoryExhausted {
|
||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"background buffer memory budget exhausted (requested={}, used={}, limit={}, expired={})",
|
||||
self.requested, self.used, self.limit, self.expired,
|
||||
)
|
||||
formatter.write_str("background buffer memory budget exhausted")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -339,31 +321,6 @@ impl io::Write for BackgroundEncodeBuffer {
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn rejected_reservation_reports_capacity_and_expiration_without_charging() {
|
||||
let owner = BackgroundMemoryBudget::new();
|
||||
owner.set_sampled_limit(64, Duration::from_secs(3));
|
||||
let held = owner.reserve(32).expect("initial reservation fits");
|
||||
let error = owner.reserve(33).err().expect("one byte over remaining capacity");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"background buffer memory budget exhausted (requested=33, used=32, limit=64, expired=false)"
|
||||
);
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
|
||||
tokio::time::advance(Duration::from_secs(3)).await;
|
||||
let error = owner.reserve(1).err().expect("expired lease rejects a fitting request");
|
||||
assert!(is_memory_pressure(&error));
|
||||
assert_eq!(
|
||||
error.to_string(),
|
||||
"background buffer memory budget exhausted (requested=1, used=32, limit=64, expired=true)"
|
||||
);
|
||||
assert_eq!(owner.snapshot().used, 32);
|
||||
drop(held);
|
||||
assert_eq!(owner.snapshot().used, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn encoded_buffer_growth_retains_both_allocations_and_the_last_slice() {
|
||||
use std::io::Write;
|
||||
|
||||
@@ -140,28 +140,6 @@ pub(crate) fn admit(binding: &PhysicalDiskBinding) -> io::Result<()> {
|
||||
.unwrap_or(Ok(()))
|
||||
}
|
||||
|
||||
/// Wait only at the namespace RPC entry, before storage locks or native I/O.
|
||||
/// The caller owns the deadline; dropping this future releases its queue turn.
|
||||
pub(crate) async fn admit_scanner_preflight(binding: &PhysicalDiskBinding) -> io::Result<()> {
|
||||
let node = REMOTE_RESOURCES
|
||||
.try_with(|context| context.admitted.borrow().is_none().then(|| context.node.clone()))
|
||||
.ok()
|
||||
.flatten();
|
||||
let Some(node) = node else {
|
||||
return admit(binding);
|
||||
};
|
||||
let deferred = || io::Error::from(io::ErrorKind::WouldBlock);
|
||||
let plan = binding.plan().ok_or_else(deferred)?;
|
||||
let permit = plan.acquire(&node).await.ok_or_else(deferred)?;
|
||||
REMOTE_RESOURCES.with(|context| {
|
||||
let mut admitted = context.admitted.borrow_mut();
|
||||
if admitted.is_none() {
|
||||
*admitted = Some(permit);
|
||||
}
|
||||
});
|
||||
admit(binding)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -184,76 +162,6 @@ mod tests {
|
||||
node
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn scanner_preflight_cancellation_and_timeout_release_queued_capacity() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("device plan");
|
||||
for deadline in [false, true] {
|
||||
let held = gate.try_acquire().expect("concurrent disk operation");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let mut wait = Box::pin(admit_scanner_preflight(&binding));
|
||||
assert!(futures_util::poll!(&mut wait).is_pending());
|
||||
assert_eq!(node.snapshot().active, 0, "queued disk work cannot reserve node capacity");
|
||||
if deadline {
|
||||
assert!(tokio::time::timeout(std::time::Duration::from_secs(1), wait).await.is_err());
|
||||
} else {
|
||||
drop(wait);
|
||||
}
|
||||
drop(held);
|
||||
admit(&binding).expect("cancelled queue turn cannot block the immediate path");
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_preflight_rejects_a_plan_replaced_while_waiting() {
|
||||
let node = node();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
let binding = PhysicalDiskBinding::default();
|
||||
binding.set_test_capacity(vec![gate.clone()]).expect("old plan");
|
||||
let held = gate.try_acquire().expect("busy disk");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let wait = admit_scanner_preflight(&binding);
|
||||
tokio::pin!(wait);
|
||||
assert!(futures_util::poll!(&mut wait).is_pending());
|
||||
binding
|
||||
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 1)])
|
||||
.expect("new plan");
|
||||
drop(held);
|
||||
assert_eq!(wait.await.expect_err("old plan is invalid").kind(), io::ErrorKind::WouldBlock);
|
||||
assert!(resources().is_none());
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!(node.completed(), Some(0));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scanner_preflight_never_waits_for_additional_devices_under_existing_admission() {
|
||||
let node = node();
|
||||
let first = PhysicalDiskBinding::default();
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
first.set_test_capacity(vec![gate.clone()]).expect("first plan");
|
||||
let other = PhysicalDiskBinding::default();
|
||||
other
|
||||
.set_test_capacity(vec![AdaptiveAdmission::new(NonZeroU16::MIN, 0)])
|
||||
.expect("closed plan");
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
admit(&first).expect("existing request admission");
|
||||
let other = admit_scanner_preflight(&other);
|
||||
tokio::pin!(other);
|
||||
assert!(matches!(futures_util::poll!(other), std::task::Poll::Ready(Err(error)) if error.kind() == io::ErrorKind::WouldBlock));
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn receiver_reuses_one_receipt_across_phases_and_releases_only_native_subsets() {
|
||||
let node = node();
|
||||
|
||||
@@ -3692,17 +3692,10 @@ pub(crate) fn has_delete_version_test_failpoints() -> bool {
|
||||
#[cfg(not(test))]
|
||||
pub(crate) fn has_delete_version_test_failpoints() -> bool {
|
||||
#[cfg(all(feature = "test-util", not(windows)))]
|
||||
let prepared_hooks = os::prepared_publication_test_hooks::has_any_hooks();
|
||||
#[cfg(not(all(feature = "test-util", not(windows))))]
|
||||
let prepared_hooks = false;
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
{
|
||||
prepared_hooks || std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some()
|
||||
}
|
||||
#[cfg(not(feature = "e2e-test-hooks"))]
|
||||
{
|
||||
prepared_hooks
|
||||
if os::prepared_publication_test_hooks::has_any_hooks() {
|
||||
return true;
|
||||
}
|
||||
cfg!(feature = "e2e-test-hooks") && std::env::var_os("RUSTFS_DELETE_RECOVERY_BARRIER_DIR").is_some()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -12488,13 +12481,9 @@ impl LocalDisk {
|
||||
// Files written here (format.json, ...) stay where they land — no
|
||||
// rename follows — so the new directory entry must be fsynced too.
|
||||
// System-critical volumes are pinned to strict by effective_durability;
|
||||
// A consumed shutdown receipt must never survive a later writer crash,
|
||||
// including when data-shard syncing is disabled.
|
||||
let sync = if volume == RUSTFS_META_BUCKET && path == super::UNCLEAN_SHUTDOWN_MARKER_PATH {
|
||||
SyncMode::FileAndDir
|
||||
} else {
|
||||
effective_durability(volume).metadata_sync_mode()
|
||||
};
|
||||
// only the legacy full-off switch (historical semantics) skips this.
|
||||
let durability = effective_durability(volume);
|
||||
let sync = durability.metadata_sync_mode();
|
||||
self.write_all_private(volume, path, data, sync, &volume_dir).await?;
|
||||
|
||||
Ok(())
|
||||
@@ -22563,33 +22552,6 @@ mod test {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn shutdown_receipt_is_durable_with_legacy_sync_disabled() {
|
||||
let _mode = durability_mode_override::set(DurabilityMode::LegacyOff);
|
||||
let dir = tempfile::tempdir().expect("shutdown marker disk");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("disk path")).expect("local endpoint");
|
||||
let disk = LocalDisk::new(&endpoint, false).await.expect("local disk");
|
||||
ensure_test_volume(&disk, RUSTFS_META_BUCKET).await;
|
||||
disk.write_all(
|
||||
RUSTFS_META_BUCKET,
|
||||
crate::disk::UNCLEAN_SHUTDOWN_MARKER_PATH,
|
||||
Bytes::from_static(b"running"),
|
||||
)
|
||||
.await
|
||||
.expect("consume shutdown receipt");
|
||||
let parent = disk
|
||||
.get_object_path(RUSTFS_META_BUCKET, crate::disk::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.expect("marker path")
|
||||
.parent()
|
||||
.expect("marker parent")
|
||||
.to_path_buf();
|
||||
assert!(
|
||||
os::fsync_dir_recorder::was_fsynced(&parent),
|
||||
"a stale clean receipt must not reappear after a crash"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[allow(clippy::await_holding_lock)]
|
||||
async fn test_write_all_public_still_fsyncs_parent_dir() {
|
||||
|
||||
@@ -122,27 +122,17 @@ fn validate_certificate_shape(
|
||||
}
|
||||
|
||||
impl LocalDisk {
|
||||
// Separate inspection, abort and recovery layouts, and construct the
|
||||
// selected boxed future outside caller poll frames.
|
||||
// Give readonly inspection its own future layout while retaining one
|
||||
// selected Box per request and constructing it outside caller poll frames.
|
||||
#[inline(never)]
|
||||
pub(super) fn recover_multipart_source_native(
|
||||
&self,
|
||||
request: MultipartSourceRequest,
|
||||
) -> impl std::future::Future<Output = Result<MultipartSourceResponse>> + Send + '_ {
|
||||
match &request.operation {
|
||||
MultipartSourceOperation::Inspect { .. } => {
|
||||
futures::future::Either::Left(Box::pin(self.inspect_multipart_source_native(request)))
|
||||
}
|
||||
MultipartSourceOperation::UpgradeLegacy { .. }
|
||||
| MultipartSourceOperation::PrepareAbort { .. }
|
||||
| MultipartSourceOperation::CommitAbort { .. }
|
||||
| MultipartSourceOperation::DeleteAborted { .. }
|
||||
| MultipartSourceOperation::ConfirmCleanupTarget { .. } => futures::future::Either::Right(
|
||||
futures::future::Either::Left(Box::pin(self.abort_multipart_source_native(request))),
|
||||
),
|
||||
_ => futures::future::Either::Right(futures::future::Either::Right(Box::pin(
|
||||
self.recover_multipart_source_native_inner(request),
|
||||
))),
|
||||
if matches!(&request.operation, MultipartSourceOperation::Inspect { .. }) {
|
||||
futures::future::Either::Left(Box::pin(self.inspect_multipart_source_native(request)))
|
||||
} else {
|
||||
futures::future::Either::Right(Box::pin(self.recover_multipart_source_native_inner(request)))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -190,24 +180,20 @@ impl LocalDisk {
|
||||
}))
|
||||
}
|
||||
|
||||
async fn abort_multipart_source_native(&self, request: MultipartSourceRequest) -> Result<MultipartSourceResponse> {
|
||||
request.validate()?;
|
||||
let store = self.open_multipart_source_store(&request.path, None).await?;
|
||||
self.recover_multipart_abort_owned(store, request.operation).await
|
||||
}
|
||||
|
||||
async fn recover_multipart_source_native_inner(&self, request: MultipartSourceRequest) -> Result<MultipartSourceResponse> {
|
||||
request.validate()?;
|
||||
// Source RW ownership precedes source namespace, target metadata, and
|
||||
// target publication ownership. Every filesystem tail retains it.
|
||||
let store = self.open_multipart_source_store(&request.path, None).await?;
|
||||
match request.operation {
|
||||
MultipartSourceOperation::Inspect { .. }
|
||||
| MultipartSourceOperation::UpgradeLegacy { .. }
|
||||
operation @ (MultipartSourceOperation::UpgradeLegacy { .. }
|
||||
| MultipartSourceOperation::PrepareAbort { .. }
|
||||
| MultipartSourceOperation::CommitAbort { .. }
|
||||
| MultipartSourceOperation::DeleteAborted { .. }
|
||||
| MultipartSourceOperation::ConfirmCleanupTarget { .. } => Err(DiskError::UnsupportedDisk),
|
||||
| MultipartSourceOperation::ConfirmCleanupTarget { .. }) => {
|
||||
self.recover_multipart_abort_owned(store, operation).await
|
||||
}
|
||||
MultipartSourceOperation::Inspect { .. } => Err(DiskError::UnsupportedDisk),
|
||||
MultipartSourceOperation::Claim { expected, round } => {
|
||||
let loaded = store.load().await?;
|
||||
require_expected(&loaded, &expected)?;
|
||||
|
||||
@@ -316,18 +316,11 @@ pub(super) fn put_active_multipart_source_head(source: impl AsRef<Path>, raw: By
|
||||
}
|
||||
|
||||
fn stamp_matches_active_disk(source: &Path, expected_head_stamp: FileStamp) -> bool {
|
||||
let stamp_matches = FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp);
|
||||
#[cfg(test)]
|
||||
{
|
||||
stamp_matches
|
||||
&& !source.join(STATE).exists()
|
||||
&& !source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
&& !source.join(".mpu-completions").exists()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
stamp_matches
|
||||
}
|
||||
FileStamp::from_path(&source.join(STORAGE_FORMAT_FILE)) == Some(expected_head_stamp)
|
||||
&& !(cfg!(test)
|
||||
&& (source.join(STATE).exists()
|
||||
|| source.join(crate::disk::MULTIPART_RECOVERY_DIR).exists()
|
||||
|| source.join(".mpu-completions").exists()))
|
||||
}
|
||||
|
||||
pub(super) fn get_active_multipart_source_head(source: &Path) -> Option<(Bytes, FileInfo)> {
|
||||
@@ -607,13 +600,11 @@ impl SourceStore {
|
||||
let disk_id = self.disk_id;
|
||||
let physical_slot = self.physical_slot;
|
||||
let disk_ids = self.disk_ids.clone();
|
||||
// Keep the loaded state out of each blocking-queue and oneshot poll frame.
|
||||
os::run_blocking_namespace_operation(self.lease.clone(), move || {
|
||||
Ok(load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids).map(Box::new))
|
||||
Ok(load_source(&source, &base, &root, &path, disk_id, physical_slot, &disk_ids))
|
||||
})
|
||||
.await
|
||||
.map_err(DiskError::from)?
|
||||
.map(|loaded| *loaded)
|
||||
}
|
||||
pub async fn persist(&self, state: &SourceState) -> Result<()> {
|
||||
self.persist_with_source_sync(state, false).await.map(|_| ())
|
||||
|
||||
@@ -45,7 +45,7 @@ pub use local::pool_meta_native::{PoolMetaNativeCasCondition, PoolMetaNativeFile
|
||||
pub(crate) use io_admission::{DiskIoClass, test_resource_units};
|
||||
pub(crate) use io_admission::{
|
||||
PhysicalHealBinding, acquire_context_disk_io_permit, current_disk_io_context, current_resource_unit, has_resource_io_scope,
|
||||
with_detached_disk_io_context, with_optional_disk_io_context, with_optional_resource_unit,
|
||||
with_optional_disk_io_context, with_optional_resource_unit,
|
||||
};
|
||||
|
||||
pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys";
|
||||
@@ -77,8 +77,6 @@ pub fn validate_format_control_path(path: &str) -> Result<()> {
|
||||
/// Per-disk marker present while an erasure-set heal is rebuilding this disk.
|
||||
/// `LocalDisk::disk_info` reports `healing = true` while the file exists.
|
||||
pub const HEALING_MARKER_PATH: &str = "healing.bin";
|
||||
/// Boot-time crash fence and clean scanner continuity receipt.
|
||||
pub const UNCLEAN_SHUTDOWN_MARKER_PATH: &str = "unclean-shutdown";
|
||||
pub const STORAGE_FORMAT_FILE: &str = "xl.meta";
|
||||
pub const STORAGE_FORMAT_FILE_BACKUP: &str = "xl.meta.bkp";
|
||||
/// Original upload authority after it stops admitting active part mutations.
|
||||
|
||||
@@ -417,15 +417,11 @@ pub(crate) mod prepared_publication_test_hooks {
|
||||
}
|
||||
|
||||
pub(crate) fn has_any_hooks() -> bool {
|
||||
let before_publication = !BEFORE_PUBLICATION.lock().is_empty();
|
||||
#[cfg(test)]
|
||||
{
|
||||
before_publication || !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty()
|
||||
}
|
||||
#[cfg(not(test))]
|
||||
{
|
||||
before_publication
|
||||
if !RENAME_DESTINATIONS.lock().is_empty() || !HARDLINK_DESTINATIONS.lock().is_empty() {
|
||||
return true;
|
||||
}
|
||||
!BEFORE_PUBLICATION.lock().is_empty()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -1045,13 +1045,6 @@ impl InstanceContext {
|
||||
self.local_disk_map.clone()
|
||||
}
|
||||
|
||||
/// Snapshot the boot-time disk registry before a store is published.
|
||||
/// Missing local drives remain explicit so startup receipts cannot treat
|
||||
/// an available subset as the complete instance inventory.
|
||||
pub async fn local_disk_inventory(&self) -> Vec<Option<DiskStore>> {
|
||||
self.local_disk_map.read().await.values().cloned().collect()
|
||||
}
|
||||
|
||||
/// Handle to this instance's local disk-id→endpoint map.
|
||||
pub(crate) fn local_disk_id_map(&self) -> Arc<RwLock<HashMap<Uuid, String>>> {
|
||||
self.local_disk_id_map.clone()
|
||||
|
||||
@@ -5958,7 +5958,6 @@ mod tests {
|
||||
#[test]
|
||||
fn scanner_dirty_usage_acknowledgement_preserves_newer_pending_work() {
|
||||
let activity = |dirty_usage_pending| ScannerPeerActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 1,
|
||||
maintenance_generation: 1,
|
||||
|
||||
@@ -4638,11 +4638,10 @@ impl SetDisks {
|
||||
write_quorum,
|
||||
};
|
||||
let resource_unit = resource_unit.clone();
|
||||
let producer = crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
tasks.spawn(crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
let outcome = Self::run_unary_rename_slot(context).await;
|
||||
let _ = outcome_tx.send(outcome).await;
|
||||
});
|
||||
tasks.spawn(crate::disk::with_detached_disk_io_context(producer));
|
||||
}));
|
||||
}
|
||||
for group in fanout_plan.batch_groups {
|
||||
let outcome_tx = outcome_tx.clone();
|
||||
@@ -4653,7 +4652,7 @@ impl SetDisks {
|
||||
let group_disks = disks.clone();
|
||||
let group_tokens = fence_tokens.clone();
|
||||
let resource_unit = resource_unit.clone();
|
||||
let producer = crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
tasks.spawn(crate::disk::with_optional_resource_unit(resource_unit, async move {
|
||||
let prepared = join_all(group.slots.into_iter().map(|slot| {
|
||||
let context = RenameSlotContext {
|
||||
slot,
|
||||
@@ -4816,8 +4815,7 @@ impl SetDisks {
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
tasks.spawn(crate::disk::with_detached_disk_io_context(producer));
|
||||
}));
|
||||
}
|
||||
drop(outcome_tx);
|
||||
RenameOutcomeProducers {
|
||||
@@ -4872,7 +4870,7 @@ impl SetDisks {
|
||||
|
||||
let coordinator_failure_receipt = rollback_receipt.clone();
|
||||
let early_ack_spawn_to_quorum_wait_started = rustfs_io_metrics::put_stage_timer();
|
||||
let tail_drain = tokio::spawn(crate::disk::with_detached_disk_io_context({
|
||||
let tail_drain = tokio::spawn({
|
||||
let fanout_src_bucket = src_bucket.clone();
|
||||
let fanout_src_object = src_object.clone();
|
||||
let fanout_dst_bucket = dst_bucket.clone();
|
||||
@@ -5126,7 +5124,7 @@ impl SetDisks {
|
||||
all_online_local_succeeded,
|
||||
})
|
||||
}
|
||||
}));
|
||||
});
|
||||
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
rustfs_io_metrics::PUT_STAGE_SET_DISK_RENAME_EARLY_ACK_SPAWN_TO_QUORUM_WAIT,
|
||||
@@ -5307,7 +5305,6 @@ impl SetDisks {
|
||||
rollback_fault_injection::after_fanout(&fanout_dst_object);
|
||||
(results, fanout_file_infos)
|
||||
};
|
||||
let fanout_future = crate::disk::with_detached_disk_io_context(fanout_future);
|
||||
let fanout = if protected_multipart {
|
||||
tokio::spawn(crate::disk::with_optional_resource_unit(
|
||||
crate::disk::current_resource_unit(),
|
||||
|
||||
@@ -6601,7 +6601,7 @@ impl SetDisks {
|
||||
let diagnostic_context = crate::cluster::rpc::runtime_sources::ReadVersionDiagnosticContext::capture_entry(
|
||||
rustfs_io_metrics::internode_metrics::ReadVersionEntrySource::PutCommitOwner,
|
||||
);
|
||||
let result = tokio::spawn(crate::disk::with_detached_disk_io_context(async move {
|
||||
let result = tokio::spawn(async move {
|
||||
let commit = Box::pin(commit(commit_submit_to_closure_enter_started, Some(child_token)));
|
||||
#[cfg(feature = "diagnostic-read-version-caller")]
|
||||
return crate::cluster::rpc::runtime_sources::with_read_version_diagnostic_context(
|
||||
@@ -6612,7 +6612,7 @@ impl SetDisks {
|
||||
.await;
|
||||
#[cfg(not(feature = "diagnostic-read-version-caller"))]
|
||||
commit.await
|
||||
}))
|
||||
})
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("put_object commit task failed: {err}")))?;
|
||||
rustfs_io_metrics::record_put_object_stage_duration_from(
|
||||
@@ -25096,64 +25096,6 @@ mod put_object_tmp_cleanup_tests {
|
||||
wait_for_tmp_workspace_to_drain(&dirs, "cancelled full-tail commit should release staging ownership").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn background_put_retains_disk_admission_across_owned_commit() {
|
||||
use crate::disk::with_background_disk_io;
|
||||
|
||||
let (_dirs, disks, set) = hermetic_set_disks(4).await;
|
||||
let object = "background-cache-commit";
|
||||
let barriers: Vec<_> = (0..disks.len())
|
||||
.map(|index| rename_fanout_barrier::arm(object, index, rename_fanout_barrier::PHASE_RENAME))
|
||||
.collect();
|
||||
let writer = Arc::clone(&set);
|
||||
let put = tokio::spawn(async move {
|
||||
with_background_disk_io(CancellationToken::new(), async {
|
||||
let mut reader = PutObjReader::from_vec(b"scanner checkpoint".to_vec());
|
||||
writer
|
||||
.put_object(
|
||||
RUSTFS_META_BUCKET,
|
||||
object,
|
||||
&mut reader,
|
||||
&ObjectOptions {
|
||||
write_completion: WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
});
|
||||
for barrier in &barriers {
|
||||
tokio::time::timeout(Duration::from_secs(30), barrier.wait_until_paused())
|
||||
.await
|
||||
.expect("every disk reaches the commit boundary");
|
||||
}
|
||||
let local_disks: Vec<_> = disks
|
||||
.iter()
|
||||
.map(|disk| match disk.as_ref() {
|
||||
disk::Disk::Local(local) => local,
|
||||
disk::Disk::Remote(_) => panic!("hermetic fixture must use local disks"),
|
||||
})
|
||||
.collect();
|
||||
let before: Vec<_> = local_disks
|
||||
.iter()
|
||||
.map(|disk| disk.background_io_admissions_for_tests())
|
||||
.collect();
|
||||
for barrier in barriers {
|
||||
barrier.release();
|
||||
}
|
||||
put.await.expect("owned put task").expect("background checkpoint commits");
|
||||
for ((disk, local), before) in disks.iter().zip(local_disks).zip(before) {
|
||||
assert!(
|
||||
local.background_io_admissions_for_tests() > before,
|
||||
"the detached rename must retain background admission on every disk"
|
||||
);
|
||||
disk.read_version("", RUSTFS_META_BUCKET, object, "", &ReadOptions::default())
|
||||
.await
|
||||
.expect("committed checkpoint remains readable");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(capacity_dirty_scope)]
|
||||
async fn borrowed_write_context_rejects_wrong_namespace_and_cache_flag() {
|
||||
|
||||
@@ -33,10 +33,9 @@ pub(crate) mod internode {
|
||||
PUT_FILE_AUTH_TRAILER_MAC_LEN, PUT_FILE_AUTH_TRAILER_MAGIC, PUT_FILE_AUTH_V1, PUT_FILE_CAPABILITY_CHALLENGE_QUERY,
|
||||
PUT_FILE_CAPABILITY_QUERY, PUT_FILE_CAPABILITY_VERSION, PUT_FILE_NONCE_QUERY, PUT_FILE_SERVER_EPOCH_QUERY,
|
||||
PutFileCapabilityResponse, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION, SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY,
|
||||
WALK_DIR_STREAM_COMPLETION_V1,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
WALK_DIR_BODY_SHA256_QUERY, WALK_DIR_STREAM_COMPLETION_QUERY, WALK_DIR_STREAM_COMPLETION_V1,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -469,8 +469,8 @@ pub use peer::external_rpc_disk;
|
||||
use peer::init_local_peer;
|
||||
pub use peer::{
|
||||
BootstrapLocalTarget, all_local_disk, all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc,
|
||||
find_local_disk_by_ref_for_scanner_rpc, get_disk_infos, init_local_disks, init_local_disks_with_instance_ctx,
|
||||
init_lock_clients, prewarm_local_disk_id_map, prewarm_local_disk_id_map_with_instance_ctx,
|
||||
get_disk_infos, init_local_disks, init_local_disks_with_instance_ctx, init_lock_clients, prewarm_local_disk_id_map,
|
||||
prewarm_local_disk_id_map_with_instance_ctx,
|
||||
};
|
||||
|
||||
pub struct ECStore {
|
||||
|
||||
@@ -261,7 +261,6 @@ impl ECStore {
|
||||
}
|
||||
|
||||
/// Execute on this instance's active local disk through the physical owner.
|
||||
/// A verified scanner publication uses the receiver's background disk admission.
|
||||
pub async fn rename_local_data(
|
||||
&self,
|
||||
disk_ref: &str,
|
||||
@@ -279,7 +278,7 @@ impl ECStore {
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let rename = rename_local_data_with_ctx(
|
||||
rename_local_data_with_ctx(
|
||||
&self.ctx,
|
||||
disk_ref,
|
||||
source,
|
||||
@@ -290,12 +289,8 @@ impl ECStore {
|
||||
external_guard,
|
||||
namespace_owner: None,
|
||||
},
|
||||
);
|
||||
if scanner_token.is_some() {
|
||||
crate::disk::with_background_disk_io(tokio_util::sync::CancellationToken::new(), rename).await
|
||||
} else {
|
||||
rename.await
|
||||
}
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Recovery stays bound to this admitted instance even after RPC cancellation.
|
||||
@@ -343,24 +338,10 @@ async fn write_local_format_control_with_ctx(
|
||||
}
|
||||
|
||||
async fn local_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
local_disk_candidate_with_admission(ctx, disk_ref, RpcDiskAdmission::Immediate).await
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum RpcDiskAdmission {
|
||||
Immediate,
|
||||
ScannerPreflight,
|
||||
}
|
||||
|
||||
async fn local_disk_candidate_with_admission(
|
||||
ctx: &Arc<InstanceContext>,
|
||||
disk_ref: &str,
|
||||
admission: RpcDiskAdmission,
|
||||
) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let map = ctx.local_disk_map();
|
||||
let path_disk = map.read().await.get(disk_ref).and_then(Option::as_ref).cloned();
|
||||
if let Some(disk) = path_disk {
|
||||
admit_rpc_disk(&disk, admission).await?;
|
||||
admit_rpc_disk(&disk)?;
|
||||
return Ok((disk, None));
|
||||
}
|
||||
let disk_id = Uuid::parse_str(disk_ref).map_err(|_| DiskError::DiskNotFound)?;
|
||||
@@ -370,7 +351,7 @@ async fn local_disk_candidate_with_admission(
|
||||
if let Some(disk) = cached_disk
|
||||
&& matches!(disk.as_ref(), Disk::Local(_))
|
||||
{
|
||||
admit_rpc_disk(&disk, admission).await?;
|
||||
admit_rpc_disk(&disk)?;
|
||||
if disk.get_disk_id().await? == Some(disk_id) {
|
||||
return Ok((disk, None));
|
||||
}
|
||||
@@ -389,24 +370,13 @@ async fn local_disk_candidate_with_admission(
|
||||
Err(DiskError::DiskNotFound)
|
||||
}
|
||||
|
||||
async fn local_scanner_disk_candidate(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<(DiskStore, Option<Uuid>)> {
|
||||
let (disk, id) = local_disk_candidate_with_admission(ctx, disk_ref, RpcDiskAdmission::ScannerPreflight).await?;
|
||||
// A queued preflight must not resume on a replaced or quarantined attachment.
|
||||
admit_local_disk(ctx, &disk, id, false, None).await?;
|
||||
Ok((disk, id))
|
||||
}
|
||||
|
||||
async fn admit_rpc_disk(disk: &DiskStore, admission: RpcDiskAdmission) -> DiskResult<()> {
|
||||
fn admit_rpc_disk(disk: &DiskStore) -> DiskResult<()> {
|
||||
use crate::disk::remote_io_admission;
|
||||
if remote_io_admission::active() {
|
||||
let binding = disk
|
||||
.physical_admission()
|
||||
.ok_or_else(|| DiskError::Io(std::io::Error::from(std::io::ErrorKind::WouldBlock)))?;
|
||||
match admission {
|
||||
RpcDiskAdmission::Immediate => remote_io_admission::admit(&binding),
|
||||
RpcDiskAdmission::ScannerPreflight => remote_io_admission::admit_scanner_preflight(&binding).await,
|
||||
}
|
||||
.map_err(DiskError::Io)?;
|
||||
remote_io_admission::admit(&binding).map_err(DiskError::Io)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -597,17 +567,6 @@ pub async fn find_local_disk_by_ref_for_rpc(disk_ref: &str) -> DiskResult<Option
|
||||
Ok(Some(disk))
|
||||
}
|
||||
|
||||
/// Resolve an authenticated namespace scan before its storage locks, waiting for
|
||||
/// initial background capacity. The caller must bound the wait with a deadline.
|
||||
/// Cancellation returns queued capacity; unknown or replaced targets are rejected.
|
||||
pub async fn find_local_disk_by_ref_for_scanner_rpc(disk_ref: &str) -> DiskResult<Option<DiskStore>> {
|
||||
if !crate::disk::remote_io_admission::active() {
|
||||
return Ok(find_local_disk_by_ref(disk_ref).await);
|
||||
}
|
||||
let (disk, _) = local_scanner_disk_candidate(&crate::runtime::global::current_ctx(), disk_ref).await?;
|
||||
Ok(Some(disk))
|
||||
}
|
||||
|
||||
/// Resolve the target from the verified listener's instance and acquire any background admission.
|
||||
#[cfg(feature = "internode-transport")]
|
||||
pub async fn external_rpc_disk(ctx: &Arc<InstanceContext>, disk_ref: &str) -> DiskResult<DiskStore> {
|
||||
@@ -1082,123 +1041,6 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_target_lookup_waits_for_busy_preflight_capacity() {
|
||||
use crate::disk::{RemoteBackgroundIo, with_remote_background_io};
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("scanner target directory");
|
||||
let disk = target_disk(&ctx, root.path(), Uuid::new_v4()).await;
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
disk.physical_admission()
|
||||
.expect("local binding")
|
||||
.set_test_capacity(vec![gate.clone()])
|
||||
.expect("physical device");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let held = gate.try_acquire().expect("concurrent scanner operation");
|
||||
let path = disk.endpoint().to_string();
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let lookup = local_scanner_disk_candidate(&ctx, &path);
|
||||
tokio::pin!(lookup);
|
||||
assert!(
|
||||
futures_util::poll!(lookup.as_mut()).is_pending(),
|
||||
"a namespace preflight must wait for busy capacity before storage locks, not fail the bucket"
|
||||
);
|
||||
assert_eq!(node.snapshot().active, 0, "waiting for the disk must not reserve node capacity");
|
||||
drop(held);
|
||||
let (selected, _) = tokio::time::timeout(std::time::Duration::from_secs(5), lookup)
|
||||
.await
|
||||
.expect("capacity release wakes namespace preflight")
|
||||
.expect("namespace target admission");
|
||||
assert!(Arc::ptr_eq(&selected, &disk));
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [1, 1]);
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn namespace_target_wait_rejects_quarantined_or_replaced_disk() {
|
||||
use crate::disk::{RemoteBackgroundIo, with_remote_background_io};
|
||||
use rustfs_concurrency::adaptive::{
|
||||
AdaptiveAdmission, FeedbackController, FeedbackObservation, NodeAdmission, ResourcePressure,
|
||||
};
|
||||
use std::num::NonZeroU16;
|
||||
|
||||
for replace in [false, true] {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
let root = tempfile::tempdir().expect("target directory");
|
||||
let id = Uuid::new_v4();
|
||||
let disk = target_disk(&ctx, root.path(), id).await;
|
||||
assert!(record_local_disk_id_if_active(&ctx, &disk, id).await);
|
||||
let replacement = if replace {
|
||||
let replacement = target_disk(&ctx, root.path(), id).await;
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(disk.clone()));
|
||||
Some(replacement)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let gate = AdaptiveAdmission::new(NonZeroU16::MIN, 1);
|
||||
disk.physical_admission()
|
||||
.expect("local binding")
|
||||
.set_test_capacity(vec![gate.clone()])
|
||||
.expect("physical device");
|
||||
let node = NodeAdmission::new(NonZeroU16::MIN);
|
||||
node.update(
|
||||
&mut FeedbackController::new(),
|
||||
Some(FeedbackObservation {
|
||||
generation: 1,
|
||||
max_concurrency: 1,
|
||||
pressure: ResourcePressure::Unknown,
|
||||
completed_units: None,
|
||||
}),
|
||||
);
|
||||
let held = gate.try_acquire().expect("busy disk");
|
||||
// Both lookup forms must revalidate the registry after queueing.
|
||||
let disk_ref = if replace {
|
||||
id.to_string()
|
||||
} else {
|
||||
disk.endpoint().to_string()
|
||||
};
|
||||
with_remote_background_io(Some(RemoteBackgroundIo::new(node.clone())), async {
|
||||
let lookup = local_scanner_disk_candidate(&ctx, &disk_ref);
|
||||
tokio::pin!(lookup);
|
||||
assert!(futures_util::poll!(&mut lookup).is_pending());
|
||||
if let Some(replacement) = replacement {
|
||||
assert!(!Arc::ptr_eq(&replacement, &disk));
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(disk.endpoint().to_string(), Some(replacement));
|
||||
} else {
|
||||
ctx.local_disk_map().write().await.insert(disk.endpoint().to_string(), None);
|
||||
}
|
||||
drop(held);
|
||||
assert!(matches!(lookup.await, Err(DiskError::DiskNotFound)));
|
||||
})
|
||||
.await;
|
||||
assert_eq!([node.snapshot().active, gate.snapshot().active], [0, 0]);
|
||||
assert_eq!([node.completed(), gate.completed()], [Some(0), Some(0)]);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn target_uuid_cache_cannot_admit_a_different_format_at_the_same_path() {
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
@@ -1542,10 +1384,6 @@ mod tests {
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
let at_entry = (ctx.namespace_commits_pending(), ctx.namespace_commit_generation());
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
assert!(!wrapper.background_io_active_for_tests(), "an ordinary rename stays foreground");
|
||||
assert!(!rename.is_finished(), "caller must still await the paused physical rename");
|
||||
rename.abort();
|
||||
let cancelled = tokio::time::timeout(Duration::from_secs(5), &mut rename)
|
||||
@@ -1664,18 +1502,7 @@ mod tests {
|
||||
})
|
||||
.await
|
||||
.expect("bounded physical entry");
|
||||
let Disk::Local(wrapper) = disk.as_ref() else {
|
||||
panic!("fixture must use a local disk");
|
||||
};
|
||||
assert!(
|
||||
wrapper.background_io_active_for_tests(),
|
||||
"a verified scanner publication must retain background admission on the receiving disk"
|
||||
);
|
||||
drop(rename);
|
||||
assert!(
|
||||
wrapper.background_io_active_for_tests(),
|
||||
"native publication keeps admission after RPC cancellation"
|
||||
);
|
||||
assert!(
|
||||
!store.scanner_data_usage_publication_blocked().await,
|
||||
"a pending namespace commit is not a local publication barrier"
|
||||
|
||||
@@ -165,26 +165,22 @@ impl HealManager {
|
||||
let mut recovery_disks = Vec::new();
|
||||
for disk in &local_disks {
|
||||
let endpoint = disk.endpoint();
|
||||
if let Some(previous) = super::super::shutdown_marker::previous_shutdown_was_unclean(&endpoint.to_string()) {
|
||||
unclean |= previous;
|
||||
} else {
|
||||
match disk
|
||||
.read_all(super::super::RUSTFS_META_BUCKET, super::super::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.await
|
||||
{
|
||||
Ok(_) => unclean = true,
|
||||
Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker check failed"
|
||||
);
|
||||
}
|
||||
match disk
|
||||
.read_all(super::super::RUSTFS_META_BUCKET, super::super::UNCLEAN_SHUTDOWN_MARKER_PATH)
|
||||
.await
|
||||
{
|
||||
Ok(_) => unclean = true,
|
||||
Err(DiskError::FileNotFound) | Err(DiskError::VolumeNotFound) => {}
|
||||
Err(err) => {
|
||||
debug!(
|
||||
target: "rustfs::heal::manager",
|
||||
event = EVENT_HEAL_UNCLEAN_SHUTDOWN,
|
||||
component = LOG_COMPONENT_HEAL,
|
||||
subsystem = LOG_SUBSYSTEM_MANAGER,
|
||||
endpoint = %endpoint,
|
||||
error = ?err,
|
||||
"Unclean-shutdown marker check failed"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -23,7 +23,6 @@ pub mod progress;
|
||||
mod replacement_execution;
|
||||
pub(crate) mod replacement_readiness;
|
||||
pub mod resume;
|
||||
mod shutdown_marker;
|
||||
pub mod storage;
|
||||
pub(crate) mod storage_api;
|
||||
pub mod task;
|
||||
@@ -39,7 +38,6 @@ use storage_api::owner::{
|
||||
pub use erasure_healer::ErasureSetHealer;
|
||||
pub use manager::{HealAdmissionTelemetry, HealManager, HealOperationsSnapshot, HealPriorityCounts, HealSourceCounts};
|
||||
pub use resume::{CheckpointManager, ResumeCheckpoint, ResumeManager, ResumeState, ResumeUtils};
|
||||
pub use shutdown_marker::{consume_read_only_scanner_handoff, save_read_only_scanner_handoff};
|
||||
pub use task::{HealOptions, HealPriority, HealRequest, HealTask, HealType};
|
||||
|
||||
pub(crate) const DATA_USAGE_CACHE_NAME: &str = ECSTORE_DATA_USAGE_CACHE_NAME;
|
||||
@@ -51,7 +49,7 @@ pub(crate) const POOL_META_NAME: &str = ECSTORE_POOL_META_NAME;
|
||||
/// [`clear_unclean_shutdown_markers`] on graceful shutdown. Finding it at
|
||||
/// startup means the previous run crashed or lost power, so the heal manager
|
||||
/// proactively re-verifies all local erasure sets.
|
||||
pub(crate) const UNCLEAN_SHUTDOWN_MARKER_PATH: &str = storage_api::ECSTORE_UNCLEAN_SHUTDOWN_MARKER_PATH;
|
||||
pub(crate) const UNCLEAN_SHUTDOWN_MARKER_PATH: &str = "unclean-shutdown";
|
||||
|
||||
/// Remove the unclean-shutdown markers from all local disks. Call at the end of
|
||||
/// a graceful shutdown, after the data plane has stopped accepting writes.
|
||||
|
||||
@@ -1,214 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
//! A clean, read-only handoff reuses the existing shutdown marker. Receipt-aware
|
||||
//! readers must consume it before legacy rollback; legacy startup alone cannot
|
||||
//! guarantee that checkpoint continuity expires before a write.
|
||||
|
||||
use super::{DiskError, DiskStore, EcstoreDiskAPI, RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH, local_disk_map_read};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{LazyLock, Mutex};
|
||||
use uuid::Uuid;
|
||||
|
||||
static PREVIOUS_SHUTDOWN: LazyLock<Mutex<HashMap<String, bool>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct ReadOnlyHandoff {
|
||||
schema: u32,
|
||||
owner: String,
|
||||
disk_id: Uuid,
|
||||
inventory: [u8; 32],
|
||||
checkpoint_instance_id: String,
|
||||
}
|
||||
|
||||
fn valid_instance(instance: &str) -> bool {
|
||||
instance.len() == 32
|
||||
&& instance
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
&& instance.bytes().any(|byte| byte != b'0')
|
||||
}
|
||||
|
||||
fn decode(bytes: &[u8], owner: &str, disk_id: Option<Uuid>, inventory: Option<[u8; 32]>) -> Option<ReadOnlyHandoff> {
|
||||
if bytes.len() > 2048 || owner.is_empty() || disk_id.is_none_or(|id| id.is_nil()) {
|
||||
return None;
|
||||
}
|
||||
let record: ReadOnlyHandoff = serde_json::from_slice(bytes).ok()?;
|
||||
(record.schema == 1
|
||||
&& record.owner == owner
|
||||
&& Some(record.disk_id) == disk_id
|
||||
&& Some(record.inventory) == inventory
|
||||
&& valid_instance(&record.checkpoint_instance_id))
|
||||
.then_some(record)
|
||||
}
|
||||
|
||||
async fn local_inventory(disks: &[Option<DiskStore>]) -> Vec<(DiskStore, Option<Uuid>)> {
|
||||
let mut inventory = Vec::with_capacity(disks.len());
|
||||
for disk in disks.iter().flatten() {
|
||||
let id = disk.get_disk_id().await.ok().flatten();
|
||||
inventory.push((disk.clone(), id));
|
||||
}
|
||||
inventory
|
||||
}
|
||||
|
||||
fn inventory_digest(disks: &[(DiskStore, Option<Uuid>)]) -> Option<[u8; 32]> {
|
||||
if disks.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut members = disks
|
||||
.iter()
|
||||
.map(|(disk, id)| (disk.endpoint().to_string(), *id))
|
||||
.collect::<Vec<_>>();
|
||||
members.sort_unstable_by(|left, right| left.0.cmp(&right.0));
|
||||
if members.windows(2).any(|pair| pair[0].0 == pair[1].0) {
|
||||
return None;
|
||||
}
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"scanner-read-only-local-disks-v1");
|
||||
for (endpoint, id) in members {
|
||||
let id = id.filter(|id| !id.is_nil())?;
|
||||
hash.update(u64::try_from(endpoint.len()).ok()?.to_be_bytes());
|
||||
hash.update(endpoint.as_bytes());
|
||||
hash.update(id.as_bytes());
|
||||
}
|
||||
Some(hash.finalize().into())
|
||||
}
|
||||
|
||||
pub(super) fn previous_shutdown_was_unclean(endpoint: &str) -> Option<bool> {
|
||||
PREVIOUS_SHUTDOWN
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(endpoint)
|
||||
.copied()
|
||||
}
|
||||
|
||||
/// Consume every local receipt before starting mutation producers or Scanner.
|
||||
/// Continuity requires matching receipts from the complete local disk inventory.
|
||||
/// The fresh unclean marker must be durable before returning a reusable identity.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a storage error if a valid handoff cannot be consumed. Starting a
|
||||
/// writer in that case could leave an older receipt authorizing stale coverage.
|
||||
pub async fn consume_read_only_scanner_handoff(owner: &str, registry: &[Option<DiskStore>]) -> crate::Result<Option<String>> {
|
||||
let disks = local_inventory(registry).await;
|
||||
let inventory = if registry.iter().any(Option::is_none) {
|
||||
None
|
||||
} else {
|
||||
inventory_digest(&disks)
|
||||
};
|
||||
let mut candidate = None;
|
||||
let mut complete = inventory.is_some() && !owner.is_empty();
|
||||
let mut previous = HashMap::new();
|
||||
for (disk, disk_id) in disks {
|
||||
let (record, unclean, absent) = match disk.read_all(RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH).await {
|
||||
Ok(bytes) => {
|
||||
let record = decode(&bytes, owner, disk_id, inventory);
|
||||
let unclean = record.is_none();
|
||||
(record, unclean, false)
|
||||
}
|
||||
Err(DiskError::FileNotFound | DiskError::VolumeNotFound) => (None, false, true),
|
||||
Err(_) => (None, true, false),
|
||||
};
|
||||
previous.insert(disk.endpoint().to_string(), unclean);
|
||||
if let Some(record) = &record {
|
||||
if candidate
|
||||
.as_ref()
|
||||
.is_some_and(|instance| instance != &record.checkpoint_instance_id)
|
||||
{
|
||||
complete = false;
|
||||
}
|
||||
candidate = Some(record.checkpoint_instance_id.clone());
|
||||
} else {
|
||||
complete = false;
|
||||
}
|
||||
// This system-volume write uses the disk's metadata durability path.
|
||||
// A crash after consumption leaves no clean receipt to reuse.
|
||||
if let Err(error) = disk
|
||||
.write_all(RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH, bytes::Bytes::from_static(b"running"))
|
||||
.await
|
||||
{
|
||||
if !absent {
|
||||
return Err(error.into());
|
||||
}
|
||||
complete = false;
|
||||
}
|
||||
}
|
||||
*PREVIOUS_SHUTDOWN.lock().unwrap_or_else(std::sync::PoisonError::into_inner) = previous;
|
||||
Ok(complete.then_some(candidate).flatten())
|
||||
}
|
||||
|
||||
/// Publish a handoff only after frontends, background producers and detached
|
||||
/// mutations have drained, and only if the caller observed no namespace or
|
||||
/// maintenance mutation during this process. No dirty hint is cleared here.
|
||||
///
|
||||
/// # Errors
|
||||
/// Fails closed on a missing disk identity, invalid owner, or persistence error.
|
||||
/// A partially written inventory cannot authorize continuity on the next start.
|
||||
pub async fn save_read_only_scanner_handoff(owner: &str, instance: &str) -> crate::Result<()> {
|
||||
if owner.is_empty() || !valid_instance(instance) {
|
||||
return Err(crate::Error::InvalidCheckpoint("invalid scanner handoff identity".to_owned()));
|
||||
}
|
||||
let registry = local_disk_map_read().await.values().cloned().collect::<Vec<_>>();
|
||||
if registry.iter().any(Option::is_none) {
|
||||
return Err(crate::Error::InvalidCheckpoint(
|
||||
"scanner handoff has an unavailable local disk".to_owned(),
|
||||
));
|
||||
}
|
||||
let disks = local_inventory(®istry).await;
|
||||
let inventory = inventory_digest(&disks)
|
||||
.ok_or_else(|| crate::Error::InvalidCheckpoint("scanner handoff has no complete disk inventory".to_owned()))?;
|
||||
for (disk, disk_id) in disks {
|
||||
let disk_id = disk_id
|
||||
.filter(|id| !id.is_nil())
|
||||
.ok_or_else(|| crate::Error::InvalidCheckpoint("scanner handoff has no disk identity".to_owned()))?;
|
||||
let record = ReadOnlyHandoff {
|
||||
schema: 1,
|
||||
owner: owner.to_owned(),
|
||||
disk_id,
|
||||
inventory,
|
||||
checkpoint_instance_id: instance.to_owned(),
|
||||
};
|
||||
let bytes = serde_json::to_vec(&record).map_err(|error| crate::Error::Serialization(error.to_string()))?;
|
||||
disk.write_all(RUSTFS_META_BUCKET, UNCLEAN_SHUTDOWN_MARKER_PATH, bytes.into())
|
||||
.await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn read_only_handoff_requires_exact_owner_disk_and_supported_state() {
|
||||
let id = Uuid::new_v4();
|
||||
let record = ReadOnlyHandoff {
|
||||
schema: 1,
|
||||
owner: "node-a".to_owned(),
|
||||
disk_id: id,
|
||||
inventory: [7; 32],
|
||||
checkpoint_instance_id: "1".repeat(32),
|
||||
};
|
||||
let bytes = serde_json::to_vec(&record).expect("valid handoff");
|
||||
assert!(decode(&bytes, "node-a", Some(id), Some([7; 32])).is_some());
|
||||
assert!(decode(&bytes, "node-a", Some(id), Some([8; 32])).is_none());
|
||||
for (owner, disk) in [("node-b", Some(id)), ("node-a", Some(Uuid::new_v4())), ("node-a", None)] {
|
||||
assert!(decode(&bytes, owner, disk, Some([7; 32])).is_none());
|
||||
}
|
||||
for bytes in [b"running".as_slice(), b"1730000000", b"{}", b"{\"schema\":1}"] {
|
||||
assert!(decode(bytes, "node-a", Some(id), Some([7; 32])).is_none());
|
||||
}
|
||||
for field in ["schema", "checkpoint_instance_id", "unknown"] {
|
||||
let mut value = serde_json::to_value(&record).expect("handoff JSON");
|
||||
value[field] = if field == "schema" {
|
||||
serde_json::json!(2)
|
||||
} else {
|
||||
serde_json::json!("0".repeat(32))
|
||||
};
|
||||
assert!(decode(&serde_json::to_vec(&value).expect("invalid handoff"), "node-a", Some(id), Some([7; 32])).is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -25,7 +25,6 @@ pub(crate) use rustfs_ecstore::api::disk::{
|
||||
DiskStore as EcstoreDiskStore, HEAL_REPLACEMENT_RECOVERY_DIR as ECSTORE_HEAL_REPLACEMENT_RECOVERY_DIR,
|
||||
HEALING_MARKER_PATH as ECSTORE_HEALING_MARKER_PATH, RUSTFS_META_BUCKET as ECSTORE_RUSTFS_META_BUCKET,
|
||||
ReplacementExecutionLease as EcstoreReplacementExecutionLease,
|
||||
UNCLEAN_SHUTDOWN_MARKER_PATH as ECSTORE_UNCLEAN_SHUTDOWN_MARKER_PATH,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::disk::{
|
||||
|
||||
@@ -1786,7 +1786,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None)
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
|
||||
.await
|
||||
.map(|(updated_at, status, _guard)| (updated_at, status))
|
||||
}
|
||||
@@ -1796,6 +1796,7 @@ where
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(
|
||||
OffsetDateTime,
|
||||
AccountStatus,
|
||||
@@ -1824,6 +1825,51 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order is parent, then the presenting STS credential. Deletion and
|
||||
// session refresh use the same STS lock. Release it before the revocation sweep.
|
||||
let _session_guard = match session {
|
||||
Some(session) => {
|
||||
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
|
||||
let current = self
|
||||
.api
|
||||
.load_user_identity(&session.access_key, UserType::Sts)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if is_err_no_such_user(&error) {
|
||||
Error::IAMActionNotAllowed
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
let parent = if current.credentials.parent_user.is_empty() {
|
||||
current.credentials.claims_or_empty().get("parent").and_then(Value::as_str)
|
||||
} else {
|
||||
Some(current.credentials.parent_user.as_str())
|
||||
};
|
||||
use subtle::ConstantTimeEq as _;
|
||||
if !session.is_valid()
|
||||
|| !current.credentials.is_valid()
|
||||
|| !current.credentials.is_temp()
|
||||
|| current.credentials.is_service_account()
|
||||
|| current.credentials.access_key != session.access_key
|
||||
|| parent != Some(access_key)
|
||||
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|
||||
|| !bool::from(
|
||||
current
|
||||
.credentials
|
||||
.session_token
|
||||
.as_bytes()
|
||||
.ct_eq(session.session_token.as_bytes()),
|
||||
)
|
||||
|| current.sts_generation != u.sts_generation
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Some(guard)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
|
||||
+183
-6
@@ -1421,11 +1421,11 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
|
||||
}
|
||||
|
||||
/// Change an enabled regular caller's secret only if its signing secret is
|
||||
/// still current under the parent mutation lock. Status and policies are kept.
|
||||
/// Change an enabled regular user's secret only if the supplied current
|
||||
/// secret still matches under the parent mutation lock. Status and policies are kept.
|
||||
/// Persistence and revocation continue after caller cancellation.
|
||||
pub async fn change_own_secret_key(
|
||||
&self,
|
||||
@@ -1433,7 +1433,27 @@ impl<T: Store> IamSys<T> {
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Change a local IAM password using an authenticated STS session and its parent password.
|
||||
///
|
||||
/// The parent and session mutation locks protect revalidation through the secret write.
|
||||
/// Persistence and session revocation continue after request cancellation.
|
||||
/// A revocation failure is returned after the new secret has been persisted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
|
||||
/// incorrect current password. Storage and revocation errors are propagated.
|
||||
pub async fn change_own_secret_key_from_session(
|
||||
&self,
|
||||
access_key: &str,
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
session: &Credentials,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1442,16 +1462,18 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
let access_key = access_key.to_string();
|
||||
let secret_key = secret_key.to_string();
|
||||
let expected_secret = expected_secret.map(str::to_string);
|
||||
let session = session.cloned();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
|
||||
.await
|
||||
};
|
||||
#[cfg(test)]
|
||||
@@ -1471,6 +1493,7 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1482,7 +1505,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, status, parent_guard) = self
|
||||
.store
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
|
||||
.await?;
|
||||
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
drop(parent_guard);
|
||||
@@ -4815,6 +4838,160 @@ mod tests {
|
||||
assert!(matches!(issuer.check_key(&legacy.access_key).await, Ok((None, false))));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
|
||||
ensure_test_global_credentials();
|
||||
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token", "generation"] {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
|
||||
let parent = "session-change-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
|
||||
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
|
||||
let pending = {
|
||||
let iam = Arc::clone(&iam);
|
||||
let session = session.clone();
|
||||
tokio::spawn(async move {
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
|
||||
.await
|
||||
})
|
||||
};
|
||||
if mutation == "revoke" {
|
||||
iam.delete_temp_account(&session.access_key, false)
|
||||
.await
|
||||
.expect("revoke presenting session");
|
||||
} else {
|
||||
let mut saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
let current = saved.get_mut(&session.access_key).expect("durable session");
|
||||
match mutation {
|
||||
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
|
||||
"disable" => current.credentials.status = "off".to_string(),
|
||||
"parent" => current.credentials.parent_user = "another-parent".to_string(),
|
||||
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
|
||||
"token" => current.credentials.session_token = "replacement-token".to_string(),
|
||||
"generation" => {
|
||||
assert!(current.sts_generation.take().is_some());
|
||||
}
|
||||
_ => unreachable!("known fixture mutation"),
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
|
||||
.await
|
||||
.expect("queued rotation finishes")
|
||||
.expect("rotation task")
|
||||
.expect_err("stale session must fail");
|
||||
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
|
||||
let saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "stale-password-parent";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "cachedParentSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "fresh-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
iam.store
|
||||
.api
|
||||
.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved identities")
|
||||
.get_mut(parent)
|
||||
.expect("durable parent")
|
||||
.credentials
|
||||
.secret_key = "durableParentSecret123".to_string();
|
||||
assert_eq!(
|
||||
iam.store
|
||||
.get_user(parent)
|
||||
.await
|
||||
.expect("stale cached parent")
|
||||
.credentials
|
||||
.secret_key,
|
||||
"cachedParentSecret123"
|
||||
);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
|
||||
)
|
||||
.await
|
||||
.expect("durable current password authorizes the change");
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "current-session-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "current-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let (_, _, revoked) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("rotation must release its session lock before revocation")
|
||||
.expect("rotate current session");
|
||||
assert_eq!(revoked, 1);
|
||||
assert!(iam.get_user(&session.access_key).await.is_none());
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -3351,9 +3351,6 @@ pub struct AllocatorMemoryObservation {
|
||||
pub reserved_bytes: Option<u64>,
|
||||
pub committed_bytes: Option<u64>,
|
||||
pub page_committed_bytes: Option<u64>,
|
||||
/// Allocator-reported block bytes, including size-class rounding and delayed statistics updates.
|
||||
/// This is not live requested bytes or resident memory.
|
||||
pub allocated_bytes: Option<u64>,
|
||||
pub malloc_requested_bytes: Option<u64>,
|
||||
pub malloc_requested_peak_bytes: Option<u64>,
|
||||
pub malloc_requested_total_bytes: Option<u64>,
|
||||
@@ -3388,11 +3385,6 @@ pub fn record_allocator_memory_observation(backend: &'static str, observation: A
|
||||
.set(if observation.reserved_bytes.is_some() { 1.0 } else { 0.0 });
|
||||
gauge!("rustfs_memory_allocator_committed_bytes_available", "backend" => backend)
|
||||
.set(if observation.committed_bytes.is_some() { 1.0 } else { 0.0 });
|
||||
gauge!("rustfs_memory_allocator_allocated_bytes_available", "backend" => backend)
|
||||
.set(if observation.allocated_bytes.is_some() { 1.0 } else { 0.0 });
|
||||
if let Some(allocated_bytes) = observation.allocated_bytes {
|
||||
gauge!("rustfs_memory_allocator_allocated_bytes", "backend" => backend).set(allocated_bytes as f64);
|
||||
}
|
||||
if let Some(reserved_bytes) = observation.reserved_bytes {
|
||||
gauge!("rustfs_memory_allocator_reserved_bytes", "backend" => backend).set(reserved_bytes as f64);
|
||||
}
|
||||
@@ -3867,41 +3859,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocator_block_bytes_preserve_decreases_missing_and_zero() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
let sequence = [
|
||||
(None, None, 0.0),
|
||||
(Some(128), Some(128.0), 1.0),
|
||||
(Some(64), Some(64.0), 1.0),
|
||||
(None, Some(64.0), 0.0),
|
||||
(Some(0), Some(0.0), 1.0),
|
||||
];
|
||||
for (index, (_, expected, available)) in sequence.into_iter().enumerate() {
|
||||
// Debugging snapshots consume values; replay each prefix before its only snapshot.
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
set_metrics_enabled(true);
|
||||
for (current, ..) in sequence.iter().take(index + 1) {
|
||||
record_allocator_memory_observation(
|
||||
"mimalloc",
|
||||
AllocatorMemoryObservation {
|
||||
allocated_bytes: *current,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
set_metrics_enabled(false);
|
||||
});
|
||||
let rows = snapshotter.snapshot().into_vec();
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_allocated_bytes_available"), Some(available));
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_allocated_bytes"), expected);
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_malloc_requested_bytes_available"), Some(0.0));
|
||||
assert_eq!(gauge_value(&rows, "rustfs_memory_allocator_malloc_requested_bytes"), None);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn allocator_current_availability_tracks_missing_zero_and_recovery() {
|
||||
let _guard = METRICS_FLAG_LOCK.lock().unwrap_or_else(|e| e.into_inner());
|
||||
|
||||
@@ -77,18 +77,7 @@ pub async fn initialize(config: Config) -> Result<(), NotificationError> {
|
||||
|
||||
let task_system = system.clone();
|
||||
tokio::spawn(async move {
|
||||
// The one-shot API keeps its strict contract: report an activation that
|
||||
// left quarantined targets so the caller can retry with a fixed config.
|
||||
// The reconciliation path quarantines instead and stays admin-usable.
|
||||
let result = task_system.set_targets_enabled(true, Some(config)).await.and_then(|()| {
|
||||
if task_system.runtime_lifecycle_all_targets_activated() {
|
||||
Ok(())
|
||||
} else {
|
||||
Err(NotificationError::Initialization(
|
||||
"one or more notification targets failed to activate".to_string(),
|
||||
))
|
||||
}
|
||||
});
|
||||
let result = task_system.set_targets_enabled(true, Some(config)).await;
|
||||
let mut legacy = LEGACY_INITIALIZATION.lock().unwrap_or_else(|err| err.into_inner());
|
||||
if matches!(
|
||||
legacy.as_ref(),
|
||||
|
||||
@@ -393,20 +393,11 @@ impl NotificationSystem {
|
||||
self.services.config_manager.lifecycle().state()
|
||||
}
|
||||
|
||||
/// Returns whether the latest accepted lifecycle generation has been handed
|
||||
/// to the runtime. A published generation serves admin operations even when
|
||||
/// some targets are quarantined; per-target health is reported by the target
|
||||
/// list, and the reconciler must not rebuild a published generation.
|
||||
/// Returns whether the latest accepted lifecycle generation is active.
|
||||
pub fn runtime_lifecycle_is_converged(&self) -> bool {
|
||||
self.services.config_manager.lifecycle().is_converged()
|
||||
}
|
||||
|
||||
/// Returns whether the last published generation activated every configured
|
||||
/// target. Used by the one-shot `initialize` API's strict contract.
|
||||
pub fn runtime_lifecycle_all_targets_activated(&self) -> bool {
|
||||
self.services.config_manager.lifecycle().all_targets_activated()
|
||||
}
|
||||
|
||||
/// Validates a bucket notification configuration against the runtime targets
|
||||
/// without publishing it, so callers can reject a request before persisting.
|
||||
pub async fn validate_bucket_notification_config(
|
||||
|
||||
@@ -21,12 +21,12 @@ use rustfs_config::server_config::Config;
|
||||
use rustfs_targets::Target;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicBool, AtomicU64, Ordering};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, Mutex as StdMutex};
|
||||
use tokio::sync::{Mutex, RwLock};
|
||||
use tokio::task::JoinHandle;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, info};
|
||||
|
||||
type CurrentGeneration = Arc<dyn Fn() -> bool + Send + Sync>;
|
||||
type CommitGeneration = Box<dyn FnOnce(bool) + Send>;
|
||||
@@ -238,18 +238,11 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
|
||||
close_old?;
|
||||
close_rejected?;
|
||||
close_activation_rejected?;
|
||||
// A target that cannot activate is quarantined and reported by the
|
||||
// admin target list. Failing the whole generation here would leave
|
||||
// the lifecycle permanently unconverged and block every admin call.
|
||||
if !activation_failures.is_empty() {
|
||||
warn!(
|
||||
event = EVENT_NOTIFY_RUNTIME_LIFECYCLE,
|
||||
component = LOG_COMPONENT_NOTIFY,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "degraded",
|
||||
failures = %activation_failures.join("; "),
|
||||
"notify runtime lifecycle"
|
||||
);
|
||||
return Err(NotificationError::Initialization(format!(
|
||||
"one or more notification targets failed to activate: {}",
|
||||
activation_failures.join("; ")
|
||||
)));
|
||||
}
|
||||
Ok(())
|
||||
})))
|
||||
@@ -288,7 +281,6 @@ impl LifecycleRuntime for DefaultLifecycleRuntime {
|
||||
struct LifecycleCoordinatorInner {
|
||||
applied_generation: AtomicU64,
|
||||
apply_lock: Mutex<()>,
|
||||
all_targets_activated: AtomicBool,
|
||||
desired: StdMutex<DesiredState>,
|
||||
publication_gate: Arc<StdMutex<()>>,
|
||||
runtime: Arc<dyn LifecycleRuntime>,
|
||||
@@ -317,7 +309,6 @@ impl NotifyLifecycleCoordinator {
|
||||
inner: Arc::new(LifecycleCoordinatorInner {
|
||||
applied_generation: AtomicU64::new(0),
|
||||
apply_lock: Mutex::new(()),
|
||||
all_targets_activated: AtomicBool::new(true),
|
||||
desired: StdMutex::new(DesiredState {
|
||||
cancellation: CancellationToken::new(),
|
||||
config: initial_config,
|
||||
@@ -366,22 +357,11 @@ impl NotifyLifecycleCoordinator {
|
||||
*self.inner.state.lock().unwrap_or_else(|err| err.into_inner())
|
||||
}
|
||||
|
||||
/// Whether the latest accepted generation has been handed to the runtime.
|
||||
/// This is the rebuild trigger and the admin availability gate: a published
|
||||
/// generation stays published until the configuration changes, so one
|
||||
/// unreachable target cannot keep the runtime churning through rebuilds.
|
||||
pub(crate) fn is_converged(&self) -> bool {
|
||||
let desired_generation = self.inner.desired.lock().unwrap_or_else(|err| err.into_inner()).generation;
|
||||
self.inner.applied_generation.load(Ordering::Acquire) >= desired_generation
|
||||
}
|
||||
|
||||
/// Whether the last published generation activated every configured target.
|
||||
/// Per-target health is reported by the admin target list; this only drives
|
||||
/// the one-shot `initialize` API's strict contract.
|
||||
pub(crate) fn all_targets_activated(&self) -> bool {
|
||||
self.inner.all_targets_activated.load(Ordering::Acquire)
|
||||
}
|
||||
|
||||
fn accept<F>(&self, allow_terminated: bool, update: F) -> NotificationLifecycleTransition
|
||||
where
|
||||
F: FnOnce(&mut DesiredState) -> Result<bool, NotificationError>,
|
||||
@@ -474,8 +454,9 @@ impl LifecycleCoordinatorInner {
|
||||
|
||||
fn mark_published(&self, generation: u64, state: NotificationRuntimeState, complete: bool) {
|
||||
*self.state.lock().unwrap_or_else(|err| err.into_inner()) = state;
|
||||
self.all_targets_activated.store(complete, Ordering::Release);
|
||||
self.applied_generation.store(generation, Ordering::Release);
|
||||
if complete {
|
||||
self.applied_generation.store(generation, Ordering::Release);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -998,7 +979,7 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_activation_quarantines_failed_target_without_rebuild_churn() {
|
||||
async fn partial_activation_reports_error_and_same_config_can_retry() {
|
||||
// The template's single-failure init budget is shared by every clone the factory hands
|
||||
// out, so the "bad" instance fails once in the first generation and recovers on retry.
|
||||
let bad_template = MockTarget::new("bad", RETRY_INIT_TARGET_TYPE).with_init_failures(1);
|
||||
@@ -1025,25 +1006,14 @@ mod tests {
|
||||
.insert("bad".to_string(), bad);
|
||||
let (coordinator, runtime_view, _) = real_runtime(config.clone(), Arc::new(TargetRegistry::with_plugins(plugins)), None);
|
||||
|
||||
// A quarantined target is a per-target health outcome, so the mutation
|
||||
// itself succeeds and admin callers are not locked out.
|
||||
coordinator
|
||||
.set_mode(true, Some(config.clone()))
|
||||
.wait()
|
||||
.await
|
||||
.expect("a quarantined target must not fail the lifecycle transition");
|
||||
.expect_err("the originating transition must report the failed target");
|
||||
let partial = runtime_view.runtime_status_snapshot().await;
|
||||
assert_eq!(partial.target_count, 2, "a quarantined target stays in the runtime beside healthy ones");
|
||||
// A published generation must not look stale: otherwise the reconciler
|
||||
// tears the whole runtime down and rebuilds it on every tick.
|
||||
assert!(
|
||||
coordinator.is_converged(),
|
||||
"a quarantined target must not leave the generation unpublished"
|
||||
);
|
||||
assert!(
|
||||
!coordinator.all_targets_activated(),
|
||||
"the quarantined target is still reported as not fully activated"
|
||||
);
|
||||
assert_eq!(partial.target_count, 1, "the healthy target should remain fault-isolated and active");
|
||||
assert!(!coordinator.is_converged(), "a partial generation must remain retryable");
|
||||
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 1 });
|
||||
|
||||
coordinator
|
||||
@@ -1054,7 +1024,6 @@ mod tests {
|
||||
let recovered = runtime_view.runtime_status_snapshot().await;
|
||||
assert_eq!(recovered.target_count, 2);
|
||||
assert!(coordinator.is_converged());
|
||||
assert!(coordinator.all_targets_activated(), "the transient failure recovers on the next apply");
|
||||
assert_eq!(coordinator.state(), NotificationRuntimeState::TargetsEnabled { generation: 2 });
|
||||
|
||||
coordinator
|
||||
|
||||
@@ -1,300 +0,0 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::{INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT, dial_channel, proto_gen::node_service};
|
||||
use prost::Message;
|
||||
use std::{
|
||||
collections::HashSet,
|
||||
io,
|
||||
sync::{
|
||||
Arc,
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::{
|
||||
io::{AsyncReadExt, AsyncWriteExt},
|
||||
net::{TcpListener, TcpStream},
|
||||
sync::{mpsc, oneshot},
|
||||
task::JoinSet,
|
||||
};
|
||||
|
||||
const BATCHES: usize = 80;
|
||||
const HEADERS: u8 = 1;
|
||||
const RST_STREAM: u8 = 3;
|
||||
const SETTINGS: u8 = 4;
|
||||
const PING: u8 = 6;
|
||||
const GOAWAY: u8 = 7;
|
||||
const WINDOW_UPDATE: u8 = 8;
|
||||
const END_HEADERS: u8 = 4;
|
||||
const END_STREAM: u8 = 1;
|
||||
|
||||
struct Frame {
|
||||
kind: u8,
|
||||
flags: u8,
|
||||
stream: u32,
|
||||
payload: Vec<u8>,
|
||||
}
|
||||
|
||||
async fn read_frame(socket: &mut TcpStream) -> io::Result<Frame> {
|
||||
let mut header = [0; 9];
|
||||
socket.read_exact(&mut header).await?;
|
||||
let length = usize::from(header[0]) * 65536 + usize::from(header[1]) * 256 + usize::from(header[2]);
|
||||
if length > 16384 {
|
||||
return Err(io::Error::other("test peer received an oversized frame"));
|
||||
}
|
||||
let mut payload = vec![0; length];
|
||||
socket.read_exact(&mut payload).await?;
|
||||
Ok(Frame {
|
||||
kind: header[3],
|
||||
flags: header[4],
|
||||
stream: u32::from_be_bytes(header[5..9].try_into().expect("four stream ID bytes")) & 0x7fff_ffff,
|
||||
payload,
|
||||
})
|
||||
}
|
||||
|
||||
async fn write_frame(socket: &mut TcpStream, kind: u8, flags: u8, stream: u32, payload: &[u8]) -> io::Result<()> {
|
||||
let length = u32::try_from(payload.len()).expect("bounded test frame length").to_be_bytes();
|
||||
assert_eq!(length[0], 0, "HTTP/2 frame length fits 24 bits");
|
||||
let mut header = [0; 9];
|
||||
header[..3].copy_from_slice(&length[1..]);
|
||||
header[3] = kind;
|
||||
header[4] = flags;
|
||||
header[5..].copy_from_slice(&stream.to_be_bytes());
|
||||
socket.write_all(&header).await?;
|
||||
socket.write_all(payload).await
|
||||
}
|
||||
|
||||
async fn write_probe_response(socket: &mut TcpStream, stream: u32) -> io::Result<()> {
|
||||
// HPACK's static :status=200 and literal content-type=application/grpc.
|
||||
write_frame(socket, HEADERS, END_HEADERS, stream, b"\x88\x0f\x10\x10application/grpc").await?;
|
||||
let response = node_service::ReadVersionResponse {
|
||||
success: true,
|
||||
..Default::default()
|
||||
}
|
||||
.encode_to_vec();
|
||||
let mut body = vec![0];
|
||||
body.extend_from_slice(&u32::try_from(response.len()).expect("small probe response").to_be_bytes());
|
||||
body.extend(response);
|
||||
write_frame(socket, 0, 0, stream, &body).await?;
|
||||
// A literal grpc-status=0 trailer ends the live probe RPC.
|
||||
write_frame(socket, HEADERS, END_HEADERS | END_STREAM, stream, b"\x00\x0bgrpc-status\x010").await
|
||||
}
|
||||
|
||||
async fn accept_peer(listener: TcpListener) -> io::Result<TcpStream> {
|
||||
let (mut socket, _) = listener.accept().await?;
|
||||
socket.set_nodelay(true)?;
|
||||
let mut preface = [0; 24];
|
||||
socket.read_exact(&mut preface).await?;
|
||||
assert_eq!(&preface, b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n");
|
||||
write_frame(&mut socket, SETTINGS, 0, 0, &[]).await?;
|
||||
Ok(socket)
|
||||
}
|
||||
|
||||
async fn delayed_response_peer(listener: TcpListener, ready: mpsc::Sender<()>, error_resets: Arc<AtomicUsize>) -> io::Result<()> {
|
||||
let mut socket = accept_peer(listener).await?;
|
||||
let mut cancelled = Vec::with_capacity(INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT);
|
||||
let mut resets = HashSet::new();
|
||||
let mut probe = None;
|
||||
loop {
|
||||
let frame = read_frame(&mut socket).await?;
|
||||
match frame.kind {
|
||||
SETTINGS | PING if frame.flags & 1 == 0 => {
|
||||
let payload = if frame.kind == PING { frame.payload.as_slice() } else { &[] };
|
||||
write_frame(&mut socket, frame.kind, 1, 0, payload).await?;
|
||||
}
|
||||
0 if !frame.payload.is_empty() => {
|
||||
let credit = u32::try_from(frame.payload.len()).expect("bounded DATA frame").to_be_bytes();
|
||||
write_frame(&mut socket, WINDOW_UPDATE, 0, 0, &credit).await?;
|
||||
}
|
||||
HEADERS if cancelled.len() < INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT => {
|
||||
cancelled.push(frame.stream);
|
||||
if cancelled.len() == INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
ready.send(()).await.map_err(|_| io::Error::other("test driver stopped"))?;
|
||||
}
|
||||
}
|
||||
HEADERS => {
|
||||
assert!(probe.replace(frame.stream).is_none(), "one live probe per batch");
|
||||
}
|
||||
RST_STREAM => {
|
||||
let reason = u32::from_be_bytes(frame.payload.as_slice().try_into().expect("RST_STREAM error code"));
|
||||
if reason == 5 {
|
||||
error_resets.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
if cancelled.contains(&frame.stream) {
|
||||
resets.insert(frame.stream);
|
||||
}
|
||||
}
|
||||
GOAWAY => return Err(io::Error::other(format!("client GOAWAY: {:?}", frame.payload))),
|
||||
_ => {}
|
||||
}
|
||||
if let Some(stream) = probe
|
||||
&& resets.len() == INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT
|
||||
{
|
||||
// A raw peer controls bytes already in flight at cancellation. A normal
|
||||
// server would consume RST_STREAM and suppress these delayed responses.
|
||||
for cancelled_stream in &cancelled {
|
||||
write_frame(&mut socket, HEADERS, END_HEADERS | END_STREAM, *cancelled_stream, b"\x88").await?;
|
||||
}
|
||||
write_probe_response(&mut socket, stream).await?;
|
||||
cancelled.clear();
|
||||
resets.clear();
|
||||
probe = None;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn assert_cancelled_responses_are_ignored(bulk: bool) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind isolated peer");
|
||||
let address = format!("http://{}", listener.local_addr().expect("peer address"));
|
||||
let (ready_tx, mut ready_rx) = mpsc::channel(1);
|
||||
let error_resets = Arc::new(AtomicUsize::new(0));
|
||||
let peer = tokio::spawn(delayed_response_peer(listener, ready_tx, error_resets.clone()));
|
||||
let cache_key = if bulk {
|
||||
super::bulk_cache_key(&address, 0)
|
||||
} else {
|
||||
address.clone()
|
||||
};
|
||||
let result = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
let (channel, _) = dial_channel(&address, &cache_key, Duration::from_secs(10))
|
||||
.await
|
||||
.expect("dial real internode channel");
|
||||
let mut client = node_service::node_service_client::NodeServiceClient::new(channel);
|
||||
for batch in 0..BATCHES {
|
||||
let mut requests = JoinSet::new();
|
||||
for _ in 0..INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT {
|
||||
let mut pending_client = client.clone();
|
||||
requests.spawn(async move { pending_client.read_version(node_service::ReadVersionRequest::default()).await });
|
||||
}
|
||||
ready_rx.recv().await.expect("peer observed every pending request");
|
||||
requests.abort_all();
|
||||
while let Some(result) = requests.join_next().await {
|
||||
assert!(result.expect_err("pending RPC task was cancelled").is_cancelled());
|
||||
}
|
||||
let response = client
|
||||
.read_version(node_service::ReadVersionRequest::default())
|
||||
.await
|
||||
.map_err(|error| format!("live probe after cancellation batch {batch}: {error}"))?;
|
||||
assert!(response.into_inner().success);
|
||||
}
|
||||
Ok::<_, String>(())
|
||||
})
|
||||
.await;
|
||||
peer.abort();
|
||||
let peer_result = peer.await;
|
||||
assert!(matches!(result, Ok(Ok(()))), "shared channel failed: {result:?}; peer: {peer_result:?}");
|
||||
assert_eq!(
|
||||
error_resets.load(Ordering::Relaxed),
|
||||
0,
|
||||
"late cancelled responses must not spend the error budget"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn cancelled_metadata_responses_keep_the_shared_channel_usable() {
|
||||
assert_cancelled_responses_are_ignored(false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn cancelled_metadata_responses_keep_the_bulk_channel_usable() {
|
||||
temp_env::async_with_vars(
|
||||
[(rustfs_config::ENV_INTERNODE_BULK_CONCURRENCY_LIMIT, Some("64"))],
|
||||
assert_cancelled_responses_are_ignored(true),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
|
||||
async fn unsolicited_response_peer(listener: TcpListener, attack: oneshot::Receiver<()>) -> io::Result<(usize, Vec<u8>)> {
|
||||
let mut socket = accept_peer(listener).await?;
|
||||
let mut completed = Vec::with_capacity(1025);
|
||||
while completed.len() < 1025 {
|
||||
let frame = read_frame(&mut socket).await?;
|
||||
match frame.kind {
|
||||
HEADERS => {
|
||||
write_probe_response(&mut socket, frame.stream).await?;
|
||||
completed.push(frame.stream);
|
||||
}
|
||||
SETTINGS if frame.flags & 1 == 0 => write_frame(&mut socket, SETTINGS, 1, 0, &[]).await?,
|
||||
PING if frame.flags & 1 == 0 => write_frame(&mut socket, PING, 1, 0, &frame.payload).await?,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
attack
|
||||
.await
|
||||
.map_err(|_| io::Error::other("test driver stopped before attack"))?;
|
||||
// These streams completed normally, so duplicate responses are protocol errors,
|
||||
// not late responses to a cancellation that the grace period should absorb.
|
||||
let mut resets = 0;
|
||||
for stream in completed {
|
||||
write_frame(&mut socket, HEADERS, END_HEADERS | END_STREAM, stream, b"\x88").await?;
|
||||
// Drain each reset before provoking the next one: GOAWAY may discard
|
||||
// queued frames, so a burst's wire count is not the internal error count.
|
||||
loop {
|
||||
let frame = read_frame(&mut socket).await?;
|
||||
match frame.kind {
|
||||
RST_STREAM => {
|
||||
assert_eq!(frame.stream, stream);
|
||||
assert_eq!(frame.payload, 5_u32.to_be_bytes());
|
||||
resets += 1;
|
||||
break;
|
||||
}
|
||||
GOAWAY => return Ok((resets, frame.payload)),
|
||||
PING if frame.flags & 1 == 0 => write_frame(&mut socket, PING, 1, 0, &frame.payload).await?,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(io::Error::other("peer accepted more than 1024 protocol errors"))
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn cancelled_stream_retention_preserves_the_protocol_error_limit() {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind isolated peer");
|
||||
let address = format!("http://{}", listener.local_addr().expect("peer address"));
|
||||
let (attack_tx, attack_rx) = oneshot::channel();
|
||||
let mut peer = tokio::spawn(unsolicited_response_peer(listener, attack_rx));
|
||||
let result = tokio::time::timeout(Duration::from_secs(30), async {
|
||||
let (channel, _) = dial_channel(&address, &address, Duration::from_secs(10))
|
||||
.await
|
||||
.expect("dial real internode channel");
|
||||
let mut client = node_service::node_service_client::NodeServiceClient::new(channel);
|
||||
for _ in 0..1025 {
|
||||
assert!(
|
||||
client
|
||||
.read_version(node_service::ReadVersionRequest::default())
|
||||
.await
|
||||
.expect("initial live probe")
|
||||
.into_inner()
|
||||
.success
|
||||
);
|
||||
}
|
||||
attack_tx
|
||||
.send(())
|
||||
.expect("release duplicate responses after the RPC completes");
|
||||
(&mut peer).await
|
||||
})
|
||||
.await;
|
||||
if result.is_err() {
|
||||
peer.abort();
|
||||
let _ = peer.await;
|
||||
}
|
||||
let (resets, payload) = result
|
||||
.expect("error budget is bounded")
|
||||
.expect("peer task")
|
||||
.expect("peer exchange");
|
||||
assert_eq!(resets, 1024, "the existing local protocol-error reset limit remains enabled");
|
||||
assert!(payload.len() >= 8);
|
||||
assert_eq!(u32::from_be_bytes(payload[4..8].try_into().expect("GOAWAY reason")), 11);
|
||||
assert_eq!(&payload[8..], b"too_many_internal_resets");
|
||||
}
|
||||
@@ -1588,10 +1588,6 @@ pub struct ScannerActivityResponse {
|
||||
pub movement_generation: ::core::option::Option<u64>,
|
||||
#[prost(bool, optional, tag = "11")]
|
||||
pub publication_blocked: ::core::option::Option<bool>,
|
||||
/// v8 binds checkpoint reuse to a verified clean, read-only handoff.
|
||||
/// The process instance above remains fresh for RPC and ACK fencing.
|
||||
#[prost(string, optional, tag = "12")]
|
||||
pub checkpoint_instance_id: ::core::option::Option<::prost::alloc::string::String>,
|
||||
}
|
||||
#[allow(clippy::all)]
|
||||
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
|
||||
|
||||
@@ -12,8 +12,6 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
#[cfg(test)]
|
||||
mod channel_reset_tests;
|
||||
pub mod compat_manifest;
|
||||
// SAFETY: `generated` is prost/tonic-generated protocol code. The allowance is
|
||||
// scoped to that module so generated internals do not relax lints elsewhere.
|
||||
@@ -709,21 +707,6 @@ pub fn canonical_scanner_activity_v7_response_body(
|
||||
pub mod scoped_dirty_usage;
|
||||
pub mod usage_watch;
|
||||
|
||||
/// Binds checkpoint continuity independently of the fresh RPC process ID.
|
||||
pub fn canonical_scanner_activity_v8_response_body(
|
||||
challenge: &[u8],
|
||||
response: &proto_gen::node_service::ScannerActivityResponse,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
let mut body = canonical_scanner_activity_v7_response_body(challenge, response)?;
|
||||
body.extend_from_slice(b"rustfs-scanner-checkpoint-continuity-v1\0");
|
||||
body.push(u8::from(response.checkpoint_instance_id.is_some()));
|
||||
if let Some(instance) = &response.checkpoint_instance_id {
|
||||
body.extend_from_slice(&u64::try_from(instance.len())?.to_be_bytes());
|
||||
body.extend_from_slice(instance.as_bytes());
|
||||
}
|
||||
Ok(body)
|
||||
}
|
||||
|
||||
pub fn canonical_scanner_dirty_usage_snapshot_request_body(
|
||||
request: &proto_gen::node_service::ScannerDirtyUsageSnapshotRequest,
|
||||
) -> Result<Vec<u8>, std::num::TryFromIntError> {
|
||||
@@ -2515,9 +2498,9 @@ mod scanner_activity_tests {
|
||||
use super::{
|
||||
canonical_scanner_activity_request_body, canonical_scanner_activity_response_body,
|
||||
canonical_scanner_activity_v4_response_body, canonical_scanner_activity_v7_response_body,
|
||||
canonical_scanner_activity_v8_response_body, canonical_scanner_dirty_usage_snapshot_request_body,
|
||||
canonical_scanner_dirty_usage_snapshot_response_body, canonical_scanner_publication_lease_release_request_body,
|
||||
canonical_scanner_publication_lease_request_body, canonical_scanner_publication_lease_response_body,
|
||||
canonical_scanner_dirty_usage_snapshot_request_body, canonical_scanner_dirty_usage_snapshot_response_body,
|
||||
canonical_scanner_publication_lease_release_request_body, canonical_scanner_publication_lease_request_body,
|
||||
canonical_scanner_publication_lease_response_body,
|
||||
proto_gen::node_service::{
|
||||
ScannerActivityRequest, ScannerActivityResponse, ScannerDirtyUsageBucket, ScannerDirtyUsageSnapshotRequest,
|
||||
ScannerDirtyUsageSnapshotResponse, ScannerPublicationLeaseRequest, ScannerPublicationLeaseResponse,
|
||||
@@ -2669,7 +2652,6 @@ mod scanner_activity_tests {
|
||||
#[test]
|
||||
fn canonical_scanner_activity_response_binds_challenge_and_every_status_field() {
|
||||
let response = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -2752,36 +2734,9 @@ mod scanner_activity_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_v8_authenticates_checkpoint_continuity() {
|
||||
let response = ScannerActivityResponse {
|
||||
protocol_version: 8,
|
||||
instance_id: "1".repeat(32),
|
||||
checkpoint_instance_id: Some("2".repeat(32)),
|
||||
..Default::default()
|
||||
};
|
||||
let baseline = canonical_scanner_activity_v8_response_body(&[1; 16], &response).expect("v8 body");
|
||||
for checkpoint_instance_id in [None, Some("3".repeat(32))] {
|
||||
let changed = ScannerActivityResponse {
|
||||
checkpoint_instance_id,
|
||||
..response.clone()
|
||||
};
|
||||
assert_ne!(
|
||||
baseline,
|
||||
canonical_scanner_activity_v8_response_body(&[1; 16], &changed).expect("changed body")
|
||||
);
|
||||
assert_eq!(
|
||||
canonical_scanner_activity_v7_response_body(&[1; 16], &response).expect("v7 body"),
|
||||
canonical_scanner_activity_v7_response_body(&[1; 16], &changed).expect("v7 compatibility"),
|
||||
"v7 does not authenticate continuity and its decoder must reject this field"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_activity_v4_response_canonicalization_ignores_v5_fields() {
|
||||
let response = ScannerActivityResponse {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: "0123456789abcdef0123456789abcdef".to_string(),
|
||||
namespace_generation: 7,
|
||||
maintenance_generation: 3,
|
||||
@@ -3778,9 +3733,9 @@ fn complete_pending_rebuild_on_cache_hit(dial_addr: &str, cache_key: &str) -> bo
|
||||
pub const INTERNODE_CONTROL_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
|
||||
// Quorum reads cancel surplus metadata RPCs in bursts. Keep bounded reset state
|
||||
// for their in-flight replies through the HTTP/2 grace period without spending
|
||||
// the connection's protocol-error budget (h2 remembers only 50 resets by default).
|
||||
const INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS: usize = 8192;
|
||||
// for their in-flight replies without spending the connection's protocol-error
|
||||
// budget (h2 remembers only 50 resets by default).
|
||||
const INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS: usize = 4096;
|
||||
|
||||
/// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout).
|
||||
fn channel_isolation_enabled() -> bool {
|
||||
|
||||
@@ -1011,9 +1011,6 @@ message ScannerActivityResponse {
|
||||
// response shape while newer readers fail closed when they are absent.
|
||||
optional uint64 movement_generation = 10;
|
||||
optional bool publication_blocked = 11;
|
||||
// v8 binds checkpoint reuse to a verified clean, read-only handoff.
|
||||
// The process instance above remains fresh for RPC and ACK fencing.
|
||||
optional string checkpoint_instance_id = 12;
|
||||
}
|
||||
|
||||
message ScannerDirtyUsageBucket {
|
||||
|
||||
@@ -58,7 +58,7 @@ impl From<Region> for legacy_s3s::region::Region {
|
||||
fn from(region: Region) -> Self {
|
||||
match Self::new(region.into_boxed_str()) {
|
||||
Ok(region) => region,
|
||||
Err(invalid) => unreachable!("rustfs_s3_types::Region and the legacy region type share one grammar: {invalid}"),
|
||||
Err(invalid) => unreachable!("rustfs_s3_types::Region and s3s::region::Region share one grammar: {invalid}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -67,7 +67,7 @@ impl From<legacy_s3s::region::Region> for Region {
|
||||
fn from(region: legacy_s3s::region::Region) -> Self {
|
||||
match Self::new(region.into_boxed_str()) {
|
||||
Ok(region) => region,
|
||||
Err(invalid) => unreachable!("the legacy region type and rustfs_s3_types::Region share one grammar: {invalid}"),
|
||||
Err(invalid) => unreachable!("s3s::region::Region and rustfs_s3_types::Region share one grammar: {invalid}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -133,16 +133,6 @@ impl From<S3Error> for legacy_s3s::S3Error {
|
||||
}
|
||||
}
|
||||
|
||||
impl S3Error {
|
||||
/// The pinned legacy serializer's XML response. Call sites stay on this type.
|
||||
/// DELETE BY T4.2 with the rest of this file.
|
||||
pub fn to_http_response(self) -> Result<http::Response<crate::Body>, Self> {
|
||||
let response = legacy_s3s::S3Error::from(self).to_http_response().map_err(Self::from)?;
|
||||
let (parts, body) = response.into_parts();
|
||||
Ok(http::Response::from_parts(parts, crate::Body::from(body)))
|
||||
}
|
||||
}
|
||||
|
||||
impl From<legacy_s3s::S3Error> for S3Error {
|
||||
fn from(error: legacy_s3s::S3Error) -> Self {
|
||||
let code: S3ErrorCode = error.code().clone().into();
|
||||
|
||||
@@ -933,8 +933,8 @@ pub struct DataUsageCacheInfo {
|
||||
#[serde(default)]
|
||||
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
/// Full activity and inventory scope of a set scan; only a complete
|
||||
/// snapshot proves coverage. A completed bucket stores its checkpoint
|
||||
/// coverage plan here until the authoritative aggregate includes it.
|
||||
/// snapshot proves coverage. Bucket caches bind this scope into their
|
||||
/// opaque scan plan digest instead.
|
||||
#[serde(default)]
|
||||
pub scan_coverage_digest: Option<DataUsageScanPlanDigest>,
|
||||
#[serde(default)]
|
||||
|
||||
@@ -15,7 +15,6 @@
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CachePublicationAdmissionUnavailable;
|
||||
@@ -626,22 +625,14 @@ impl DataUsageCache {
|
||||
self.save_inner(store, name, Some(revisions), None).await
|
||||
}
|
||||
|
||||
/// All epoch-fenced cache writes use background disk admission. Incremental
|
||||
/// checkpoints pass their scan token; finalization uses a fresh token so a
|
||||
/// spent scan budget can still persist progress within the save timeout.
|
||||
pub(crate) async fn save_with_revisions_for_epoch<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
&self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revisions: &DataUsageCacheRevisions,
|
||||
expected_epoch: u64,
|
||||
cancellation: CancellationToken,
|
||||
) -> StorageResult<()> {
|
||||
crate::storage_api::ecstore_with_background_disk_io(
|
||||
cancellation,
|
||||
self.save_inner(store, name, Some(revisions), Some(expected_epoch)),
|
||||
)
|
||||
.await
|
||||
self.save_inner(store, name, Some(revisions), Some(expected_epoch)).await
|
||||
}
|
||||
|
||||
async fn save_inner<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
|
||||
@@ -5,7 +5,6 @@ use super::*;
|
||||
use std::hint::black_box;
|
||||
use std::sync::atomic::AtomicU64;
|
||||
use std::time::Instant as WallInstant;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
const MAX_WIRE_BYTES: u64 = 32 * 1024 * 1024;
|
||||
const CACHE_NAME: &str = "bucket/cache-cost.bin";
|
||||
@@ -231,7 +230,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
baseline
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0)
|
||||
.await
|
||||
.expect("baseline save");
|
||||
|
||||
@@ -288,7 +287,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
store.reset_counts();
|
||||
let started = WallInstant::now();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("measured save");
|
||||
save_ns.push(elapsed_ns(started));
|
||||
@@ -310,7 +309,7 @@ async fn profile_case(objects: usize, scenario: &str, samples: usize) {
|
||||
conflicting.info.next_cycle += 1;
|
||||
assert!(matches!(
|
||||
conflicting
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &initial, 0)
|
||||
.await,
|
||||
Err(Error::PreconditionFailed)
|
||||
));
|
||||
|
||||
@@ -114,10 +114,8 @@ pub use scanner_io::{
|
||||
probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
|
||||
record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
|
||||
restore_scanner_checkpoint_instance_id, scanner_activity_epoch, scanner_checkpoint_instance_id,
|
||||
scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, scanner_process_was_read_only, set_scanner_dirty_usage_clear_observer,
|
||||
set_scanner_dirty_usage_mutation_observer,
|
||||
scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
pub use segment_invalidation::SegmentInvalidationProducerIdentity;
|
||||
pub use sleeper::{
|
||||
@@ -127,9 +125,9 @@ pub use sleeper::{
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
pub use storage_api::ScannerReplicationConfig as ReplicationConfig;
|
||||
pub use storage_api::scan::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
pub use workload_admission::{set_scanner_node_admission, set_scanner_workload_admission_snapshot_provider};
|
||||
|
||||
|
||||
@@ -595,13 +595,7 @@ pub(crate) async fn write_budget_drain_fixture_frames<W: AsyncWrite + Unpin>(
|
||||
cache.info.scan_plan_digest = Some(request.scan_plan_digest);
|
||||
cache.info.last_update = Some(std::time::SystemTime::now());
|
||||
cache
|
||||
.save_with_revisions_for_epoch(
|
||||
store.clone(),
|
||||
&cache_name,
|
||||
&revisions,
|
||||
identity.publication_epoch,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.save_with_revisions_for_epoch(store.clone(), &cache_name, &revisions, identity.publication_epoch)
|
||||
.await
|
||||
.expect("durable current-request root CAS");
|
||||
let mut persisted = DataUsageCache::default();
|
||||
@@ -854,7 +848,6 @@ async fn classify_remote_usage_watch_outcome(
|
||||
// The folder completes under its coverage plan; finished cache
|
||||
// reuse and the response remain bound to the requested reuse plan.
|
||||
cache.info.scan_plan_digest = Some(scan_plan_digest);
|
||||
cache.info.scan_coverage_digest = checkpoint_plan_digest;
|
||||
(cache, None)
|
||||
}
|
||||
}
|
||||
@@ -989,27 +982,6 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
let tier_registry_generation = crate::runtime_tier_registry_for_cycle(next_cycle, leader_epoch)
|
||||
.await
|
||||
.generation;
|
||||
// Only a leader handoff needs this authoritative floor; ordinary cycles
|
||||
// avoid an extra config read. An unavailable/legacy floor forces a walk.
|
||||
let completed_checkpoint_cycle_floor = if cache.info.snapshot_complete
|
||||
&& cache.info.leader_epoch < leader_epoch
|
||||
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
&& checkpoint_plan_digest.is_some()
|
||||
&& cache.info.scan_coverage_digest == checkpoint_plan_digest
|
||||
&& cache.info.next_cycle < next_cycle
|
||||
{
|
||||
let baseline = read_config(store.clone(), crate::DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.ok();
|
||||
let sources = store
|
||||
.all_set_disks()
|
||||
.iter()
|
||||
.map(|set| DataUsageCacheSource::new(set.pool_index, set.set_index))
|
||||
.collect();
|
||||
crate::scanner_io::completed_checkpoint_cycle_floor(baseline.as_deref(), &sources, next_cycle, leader_epoch)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let current_identity = crate::scanner_io::scanner_bucket_checkpoint_identity(
|
||||
&set,
|
||||
&bucket,
|
||||
@@ -1040,7 +1012,6 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
tier_registry_generation: Some(tier_registry_generation),
|
||||
checkpoint_identity: current_identity,
|
||||
checkpoint_plan_digest,
|
||||
completed_checkpoint_cycle_floor,
|
||||
},
|
||||
);
|
||||
match scan_state {
|
||||
@@ -1293,7 +1264,7 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
// the scan. A movement transition that starts and ends during the scan
|
||||
// therefore cannot admit the stale cache under the new epoch.
|
||||
let save_result = cache
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, expected_publication_epoch)
|
||||
.await;
|
||||
done_save();
|
||||
if let Err(err) = save_result {
|
||||
|
||||
@@ -591,7 +591,6 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
|
||||
tier_registry_generation: Some(tier_generation),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: Some(plan),
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
crate::scanner_io::bind_usage_watch_to_checkpoint(&mut cache, Some(&proof), plan);
|
||||
|
||||
@@ -4451,10 +4451,9 @@ pub use activity::scanner_topology_digest;
|
||||
pub(crate) use activity::{
|
||||
LOCAL_SCANNER_ACTIVITY_NODE, ScannerActivityGuardrailVeto, ScannerActivitySnapshot, ScannerDirtyUsageAcknowledgement,
|
||||
ScannerDirtyUsageAcknowledgementKind, probe_scanner_activity, scanner_activity_allows_usage_publication,
|
||||
scanner_activity_bucket_reuse_digest, scanner_activity_coverage_digest, scanner_activity_dirty_usage_state_for_host,
|
||||
scanner_activity_guardrail_veto, scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest,
|
||||
scanner_activity_structural_digest, scanner_activity_usage_watch_structure_digest, scanner_dirty_usage_acknowledgements,
|
||||
scanner_peer_dirty_usage_moved,
|
||||
scanner_activity_coverage_digest, scanner_activity_dirty_usage_state_for_host, scanner_activity_guardrail_veto,
|
||||
scanner_activity_publication_lease_targets, scanner_activity_snapshot_digest, scanner_activity_structural_digest,
|
||||
scanner_activity_usage_watch_structure_digest, scanner_dirty_usage_acknowledgements, scanner_peer_dirty_usage_moved,
|
||||
};
|
||||
pub(crate) use activity::{ScannerCycleOutcome, scanner_cycle_outcome_with_pending_maintenance};
|
||||
pub use backlog::{
|
||||
|
||||
@@ -535,7 +535,6 @@ pub(crate) const LOCAL_SCANNER_ACTIVITY_NODE: &str = "<local>";
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub(crate) struct ScannerNodeActivity {
|
||||
pub(super) instance_id: String,
|
||||
pub(super) checkpoint_instance_id: Option<String>,
|
||||
pub(super) namespace_generation: u64,
|
||||
pub(super) maintenance_generation: u64,
|
||||
pub(super) protocol_version: u32,
|
||||
@@ -552,7 +551,6 @@ impl ScannerNodeActivity {
|
||||
pub(crate) fn e2e_observation(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"instance_id": self.instance_id,
|
||||
"checkpoint_instance_id": self.checkpoint_instance_id,
|
||||
"namespace_generation": self.namespace_generation,
|
||||
"maintenance_generation": self.maintenance_generation,
|
||||
"protocol_version": self.protocol_version,
|
||||
@@ -577,7 +575,6 @@ pub(crate) fn scanner_node_activity_for_tests(
|
||||
) -> ScannerNodeActivity {
|
||||
ScannerNodeActivity {
|
||||
instance_id: instance_id.to_string(),
|
||||
checkpoint_instance_id: None,
|
||||
namespace_generation,
|
||||
maintenance_generation: 0,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -1098,32 +1095,11 @@ where
|
||||
}
|
||||
|
||||
pub(crate) fn scanner_activity_snapshot_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
hash_scanner_activity_members(snapshot.iter().map(|(host, activity)| (host.as_str(), activity)), false)
|
||||
}
|
||||
|
||||
/// Completed bucket reuse retains all mutation and maintenance fences while
|
||||
/// allowing a verified read-only handoff to replace only the RPC process epoch.
|
||||
pub(crate) fn scanner_activity_bucket_reuse_digest(snapshot: &ScannerActivitySnapshot, local_node: &str) -> [u8; 32] {
|
||||
hash_scanner_activity_members(scanner_activity_members(snapshot, local_node).into_iter(), true)
|
||||
}
|
||||
|
||||
fn hash_scanner_activity_members<'a>(
|
||||
members: impl ExactSizeIterator<Item = (&'a str, &'a ScannerNodeActivity)>,
|
||||
checkpoint_continuity: bool,
|
||||
) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
if checkpoint_continuity {
|
||||
hasher.update(b"scanner-bucket-reuse-v1");
|
||||
}
|
||||
hasher.update(u64::try_from(members.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in members {
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in snapshot {
|
||||
let host = host.as_bytes();
|
||||
let instance_id = if checkpoint_continuity {
|
||||
activity.checkpoint_instance_id.as_deref().unwrap_or(&activity.instance_id)
|
||||
} else {
|
||||
&activity.instance_id
|
||||
}
|
||||
.as_bytes();
|
||||
let instance_id = activity.instance_id.as_bytes();
|
||||
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host);
|
||||
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
@@ -1142,11 +1118,11 @@ fn hash_scanner_activity_members<'a>(
|
||||
|
||||
/// The reuse identity: hash the activity inputs that make an existing
|
||||
/// scanner result unsafe to reuse without walking again. Besides the
|
||||
/// coverage identity below it binds each peer's checkpoint continuity and
|
||||
/// `maintenance_generation`. A read-only, clean shutdown may preserve that
|
||||
/// continuity; a writer or crash starts fresh because its last dirty-journal
|
||||
/// window may be missing. A changed lifecycle or replication rule also needs
|
||||
/// a maintenance pass. With
|
||||
/// coverage identity below it binds each peer's `instance_id` and
|
||||
/// `maintenance_generation`: a restarted node may have lost the dirty hints
|
||||
/// of its last journal window and a changed lifecycle or replication rule
|
||||
/// needs a maintenance pass, so both invalidate the authoritative baseline
|
||||
/// and the set caches and force the next cycle to walk every bucket. With
|
||||
/// the bucket list it is the plan under which set caches and the
|
||||
/// scoped-scan baseline are reused (`scanner_bucket_plan_digest`).
|
||||
///
|
||||
@@ -1155,17 +1131,12 @@ fn hash_scanner_activity_members<'a>(
|
||||
/// `publication_blocked` is omitted from every identity: a movement pause
|
||||
/// vetoes the cycle it moves in (`scanner_activity_guardrail_veto`) and
|
||||
/// describes nothing about the heap.
|
||||
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot, local_node: &str) -> [u8; 32] {
|
||||
pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"scanner-reuse-members-v2");
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in scanner_activity_members(snapshot, local_node) {
|
||||
for (host, activity) in snapshot {
|
||||
let host = host.as_bytes();
|
||||
let instance_id = activity
|
||||
.checkpoint_instance_id
|
||||
.as_deref()
|
||||
.unwrap_or(&activity.instance_id)
|
||||
.as_bytes();
|
||||
let instance_id = activity.instance_id.as_bytes();
|
||||
hasher.update(u64::try_from(host.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host);
|
||||
hasher.update(u64::try_from(instance_id.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
@@ -1179,64 +1150,33 @@ pub(crate) fn scanner_activity_structural_digest(snapshot: &ScannerActivitySnaps
|
||||
hasher.finalize().into()
|
||||
}
|
||||
|
||||
fn scanner_activity_members<'a>(
|
||||
snapshot: &'a ScannerActivitySnapshot,
|
||||
local_node: &'a str,
|
||||
) -> Vec<(&'a str, &'a ScannerNodeActivity)> {
|
||||
// The local marker moves to another member when leadership changes.
|
||||
// Hash the actual member identities in canonical order instead.
|
||||
let mut nodes = snapshot
|
||||
.iter()
|
||||
.map(|(host, activity)| {
|
||||
(
|
||||
if host == LOCAL_SCANNER_ACTIVITY_NODE {
|
||||
local_node
|
||||
} else {
|
||||
host.as_str()
|
||||
},
|
||||
activity,
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
nodes.sort_unstable_by_key(|(host, _)| *host);
|
||||
nodes
|
||||
}
|
||||
|
||||
/// The coverage identity: hash the activity inputs under which a bucket
|
||||
/// sweep's traversal stays one traversal, so a checkpoint resumed in a later
|
||||
/// cycle is not mixed coverage. It binds the peer set, the protocol, the
|
||||
/// topology and the data-movement state. A coordinator restart, a
|
||||
/// maintenance publication or a publication pause changes none of the
|
||||
/// namespace a sweep walks, so the RPC `instance_id`, `maintenance_generation`
|
||||
/// and `publication_blocked` are left out. Protocol v8 instead binds the
|
||||
/// checkpoint identity inherited only from a durable read-only handoff;
|
||||
/// a lost write hint must invalidate partial as well as completed coverage. The
|
||||
/// standalone node's namespace and dirty generations are left out too: the
|
||||
/// namespace a sweep walks, so `instance_id`, `maintenance_generation` and
|
||||
/// `publication_blocked` are left out; hashing them made every resumed sweep
|
||||
/// restart as mixed coverage after such an event (rustfs/rustfs#8494). The
|
||||
/// local node's namespace and dirty generations are left out too: the
|
||||
/// bucket's own dirty generation joins the checkpoint plan in
|
||||
/// `scanner_bucket_cache_digest`, which is what certification compares.
|
||||
/// Distributed plans retain every member's generations, including the
|
||||
/// coordinator's, so a leader rotation cannot remove a mutation witness.
|
||||
pub(crate) fn scanner_activity_coverage_digest(snapshot: &ScannerActivitySnapshot, local_node: &str) -> [u8; 32] {
|
||||
/// `scanner_bucket_cache_digest`, which is what certification compares. A
|
||||
/// remote peer's generations stay in, because its dirty buckets are still
|
||||
/// acknowledged per cycle rather than per bucket, and a write there between
|
||||
/// the two halves of a sweep would otherwise be acknowledged without having
|
||||
/// been walked.
|
||||
pub(crate) fn scanner_activity_coverage_digest(snapshot: &ScannerActivitySnapshot) -> [u8; 32] {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(b"scanner-checkpoint-coverage-v2");
|
||||
hasher.update(u64::try_from(snapshot.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
for (host, activity) in scanner_activity_members(snapshot, local_node) {
|
||||
for (host, activity) in snapshot {
|
||||
let host_bytes = host.as_bytes();
|
||||
hasher.update(u64::try_from(host_bytes.len()).unwrap_or(u64::MAX).to_be_bytes());
|
||||
hasher.update(host_bytes);
|
||||
if let Some(instance) = &activity.checkpoint_instance_id {
|
||||
hasher.update(instance.as_bytes());
|
||||
}
|
||||
hasher.update(activity.protocol_version.to_be_bytes());
|
||||
hasher.update(activity.topology_digest);
|
||||
hasher.update([u8::from(activity.data_movement_active)]);
|
||||
hasher.update(activity.movement_generation.to_be_bytes());
|
||||
if host == LOCAL_SCANNER_ACTIVITY_NODE {
|
||||
// Standalone writes are fenced by the bucket's own generation.
|
||||
hasher.update(0_u64.to_be_bytes());
|
||||
hasher.update(0_u64.to_be_bytes());
|
||||
hasher.update([0]);
|
||||
} else {
|
||||
if host.as_str() != LOCAL_SCANNER_ACTIVITY_NODE {
|
||||
hasher.update(activity.namespace_generation.to_be_bytes());
|
||||
hasher.update(activity.dirty_usage_generation.to_be_bytes());
|
||||
hasher.update([u8::from(activity.dirty_usage_pending)]);
|
||||
@@ -1374,10 +1314,8 @@ pub(crate) fn scanner_peer_dirty_usage_moved(
|
||||
pub(crate) fn scanner_activity_allows_usage_publication(snapshot: &ScannerActivitySnapshot) -> bool {
|
||||
!snapshot.is_empty()
|
||||
&& snapshot.values().all(|activity| {
|
||||
matches!(
|
||||
activity.protocol_version,
|
||||
crate::SCANNER_ACTIVITY_V7_PROTOCOL_VERSION | SCANNER_ACTIVITY_PROTOCOL_VERSION
|
||||
) && activity.movement_generation != u64::MAX
|
||||
activity.protocol_version == SCANNER_ACTIVITY_PROTOCOL_VERSION
|
||||
&& activity.movement_generation != u64::MAX
|
||||
&& !activity.data_movement_active
|
||||
&& !activity.publication_blocked
|
||||
})
|
||||
@@ -1484,7 +1422,6 @@ where
|
||||
LOCAL_SCANNER_ACTIVITY_NODE.to_string(),
|
||||
ScannerNodeActivity {
|
||||
instance_id: local_instance_id,
|
||||
checkpoint_instance_id: Some(crate::scanner_checkpoint_instance_id().to_owned()),
|
||||
namespace_generation,
|
||||
maintenance_generation,
|
||||
protocol_version: SCANNER_ACTIVITY_PROTOCOL_VERSION,
|
||||
@@ -1534,7 +1471,7 @@ where
|
||||
SCANNER_ACTIVITY_V6_PROTOCOL_VERSION
|
||||
));
|
||||
}
|
||||
crate::SCANNER_ACTIVITY_V7_PROTOCOL_VERSION | SCANNER_ACTIVITY_PROTOCOL_VERSION => (
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION => (
|
||||
activity
|
||||
.topology_digest
|
||||
.ok_or_else(|| format!("scanner activity peer {host} omitted its storage topology"))?,
|
||||
@@ -1573,7 +1510,6 @@ where
|
||||
host.clone(),
|
||||
ScannerNodeActivity {
|
||||
instance_id: activity.instance_id,
|
||||
checkpoint_instance_id: activity.checkpoint_instance_id,
|
||||
namespace_generation: activity.namespace_generation,
|
||||
maintenance_generation: activity.maintenance_generation,
|
||||
protocol_version: activity.protocol_version,
|
||||
|
||||
@@ -899,7 +899,7 @@ async fn checkpoint_fixture_runtime_deadline_save_reload_resume() {
|
||||
.await
|
||||
.expect("initial cache revisions");
|
||||
checkpoint
|
||||
.save_with_revisions_for_epoch(store.clone(), cache_name, &cache_revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), cache_name, &cache_revisions, 0)
|
||||
.await
|
||||
.expect("publish the current checkpoint into the fenced store");
|
||||
let cache_path = rustfs_utils::path::path_join_buf(&[crate::BUCKET_META_PREFIX, cache_name]);
|
||||
@@ -10357,7 +10357,6 @@ async fn recovery_wake_interrupts_normal_cycle_wait() {
|
||||
|
||||
fn scanner_node_activity(epoch: &str, namespace_generation: u64, maintenance_generation: u64) -> ScannerNodeActivity {
|
||||
ScannerNodeActivity {
|
||||
checkpoint_instance_id: None,
|
||||
instance_id: epoch.to_string(),
|
||||
namespace_generation,
|
||||
maintenance_generation,
|
||||
@@ -10378,10 +10377,7 @@ fn scoped_scan_remote_dirty_coverage_invalidates_local_bucket_current() {
|
||||
let remote = after.get_mut("remote").expect("remote activity should exist");
|
||||
remote.dirty_usage_generation += 1;
|
||||
remote.dirty_usage_pending = true;
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
assert_eq!(scanner_activity_structural_digest(&before), scanner_activity_structural_digest(&after));
|
||||
let old_plan = crate::scanner_io::checkpoint_fixture_bucket_digest(
|
||||
DataUsageScanPlanDigest(scanner_activity_snapshot_digest(&before)),
|
||||
None,
|
||||
@@ -10423,7 +10419,6 @@ fn scoped_scan_remote_dirty_coverage_invalidates_local_bucket_current() {
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
),
|
||||
crate::scanner_io::DataUsageCacheScanState::Prepared {
|
||||
@@ -10613,14 +10608,6 @@ fn scanner_activity_snapshot_fences_data_movement() {
|
||||
assert!(scanner_activity_allows_usage_publication(&idle));
|
||||
assert!(!scanner_activity_allows_usage_publication(&moving));
|
||||
assert_ne!(scanner_activity_snapshot_digest(&idle), scanner_activity_snapshot_digest(&moving));
|
||||
let mut v7 = idle;
|
||||
v7.get_mut("node-2").expect("node").protocol_version = crate::SCANNER_ACTIVITY_V7_PROTOCOL_VERSION;
|
||||
assert!(
|
||||
scanner_activity_allows_usage_publication(&v7),
|
||||
"v7 still authenticates terminal movement state"
|
||||
);
|
||||
v7.get_mut("node-2").expect("node").protocol_version = crate::SCANNER_ACTIVITY_V6_PROTOCOL_VERSION;
|
||||
assert!(!scanner_activity_allows_usage_publication(&v7));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -10649,8 +10636,8 @@ fn scanner_activity_structural_digest_ignores_regular_bucket_writes() {
|
||||
|
||||
assert_ne!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&written));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&written, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&written),
|
||||
"bucket writes are refreshed through the dirty-bucket scope rather than invalidating every cache"
|
||||
);
|
||||
}
|
||||
@@ -10672,24 +10659,18 @@ fn scanner_activity_structural_digest_fences_restart_and_maintenance_but_not_pub
|
||||
// maintenance rule needs a maintenance pass, so both invalidate every
|
||||
// reusable result (the baseline and the set caches).
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&restarted, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&restarted)
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&maintained, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&maintained)
|
||||
);
|
||||
// A publication pause describes nothing about the heap: it vetoes the
|
||||
// cycle it moves in and is part of no identity.
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&paused, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
assert_eq!(scanner_activity_structural_digest(&baseline), scanner_activity_structural_digest(&paused));
|
||||
assert_eq!(scanner_activity_snapshot_digest(&baseline), scanner_activity_snapshot_digest(&paused));
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&paused, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
assert_eq!(scanner_activity_coverage_digest(&baseline), scanner_activity_coverage_digest(&paused));
|
||||
assert_eq!(
|
||||
scanner_activity_guardrail_veto(&baseline, &paused).map(|veto| veto.field),
|
||||
Some("publication_blocked")
|
||||
@@ -10718,8 +10699,8 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut changed = baseline.clone();
|
||||
mutate(changed.get_mut(host).expect("host should be present"));
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&changed),
|
||||
"{field} on {host} must not change the coverage identity"
|
||||
);
|
||||
assert_eq!(
|
||||
@@ -10738,8 +10719,8 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut changed = baseline.clone();
|
||||
mutate(changed.get_mut(LOCAL_SCANNER_ACTIVITY_NODE).expect("local activity"));
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&changed),
|
||||
"local {field} is certified per bucket through the bucket's own dirty generation"
|
||||
);
|
||||
// A peer's generations stay in: its dirty buckets are acknowledged
|
||||
@@ -10748,8 +10729,8 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut peer_changed = baseline.clone();
|
||||
mutate(peer_changed.get_mut("node-2").expect("peer activity"));
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&peer_changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&peer_changed),
|
||||
"remote {field} must change the coverage identity"
|
||||
);
|
||||
}
|
||||
@@ -10764,63 +10745,20 @@ fn scanner_activity_coverage_digest_survives_restart_and_maintenance_but_fences_
|
||||
let mut changed = baseline.clone();
|
||||
mutate(changed.get_mut(host).expect("host should be present"));
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&baseline),
|
||||
scanner_activity_coverage_digest(&changed),
|
||||
"{field} on {host} must change the coverage identity"
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&changed, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&baseline),
|
||||
scanner_activity_structural_digest(&changed),
|
||||
"{field} on {host} must change the reuse identity"
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut peer_lost = baseline.clone();
|
||||
peer_lost.remove("node-2");
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&baseline, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_coverage_digest(&peer_lost, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn scanner_checkpoint_coverage_survives_clean_leader_rotation() {
|
||||
let first = BTreeMap::from([
|
||||
(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("leader-a", 0, 0)),
|
||||
("node-b".to_string(), scanner_node_activity("peer-b", 0, 0)),
|
||||
("node-c".to_string(), scanner_node_activity("peer-c", 0, 0)),
|
||||
]);
|
||||
let second = BTreeMap::from([
|
||||
(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), scanner_node_activity("leader-b", 0, 0)),
|
||||
("node-a".to_string(), scanner_node_activity("peer-a", 0, 0)),
|
||||
("node-c".to_string(), scanner_node_activity("peer-c", 0, 0)),
|
||||
]);
|
||||
assert_eq!(
|
||||
scanner_activity_coverage_digest(&first, "node-a"),
|
||||
scanner_activity_coverage_digest(&second, "node-b"),
|
||||
"a quiet namespace must retain checkpoint coverage when a different node becomes leader"
|
||||
);
|
||||
assert_ne!(
|
||||
scanner_activity_structural_digest(&first, "node-a"),
|
||||
scanner_activity_structural_digest(&second, "node-b"),
|
||||
"restarted processes must invalidate completed results even with reset mutation counters"
|
||||
);
|
||||
let mut rotated = first.clone();
|
||||
let previous_leader = rotated.remove(LOCAL_SCANNER_ACTIVITY_NODE).expect("previous leader");
|
||||
let next_leader = rotated.remove("node-b").expect("next leader");
|
||||
rotated.insert("node-a".to_string(), previous_leader);
|
||||
rotated.insert(LOCAL_SCANNER_ACTIVITY_NODE.to_string(), next_leader);
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&first, "node-a"),
|
||||
scanner_activity_structural_digest(&rotated, "node-b"),
|
||||
"a leadership change alone must retain completed coverage"
|
||||
);
|
||||
rotated.get_mut("node-a").expect("previous leader").namespace_generation += 1;
|
||||
assert_ne!(
|
||||
scanner_activity_coverage_digest(&first, "node-a"),
|
||||
scanner_activity_coverage_digest(&rotated, "node-b"),
|
||||
"a write on the previous coordinator must invalidate its completed coverage"
|
||||
);
|
||||
assert_ne!(scanner_activity_coverage_digest(&baseline), scanner_activity_coverage_digest(&peer_lost));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -10852,18 +10790,10 @@ fn bucket_checkpoint_plan_survives_a_coordinator_restart() {
|
||||
// restarted coordinator requests, or every resumed sweep restarts as
|
||||
// mixed coverage and the bucket can never certify (rustfs/rustfs#8494).
|
||||
// The reuse plan does change: finished results are walked again.
|
||||
let coverage_plan = |snapshot| {
|
||||
crate::scanner_io::scanner_bucket_plan_digest(
|
||||
&buckets,
|
||||
scanner_activity_coverage_digest(snapshot, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
)
|
||||
};
|
||||
let reuse_plan = |snapshot| {
|
||||
crate::scanner_io::scanner_bucket_plan_digest(
|
||||
&buckets,
|
||||
scanner_activity_structural_digest(snapshot, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
)
|
||||
};
|
||||
let coverage_plan =
|
||||
|snapshot| crate::scanner_io::scanner_bucket_plan_digest(&buckets, scanner_activity_coverage_digest(snapshot));
|
||||
let reuse_plan =
|
||||
|snapshot| crate::scanner_io::scanner_bucket_plan_digest(&buckets, scanner_activity_structural_digest(snapshot));
|
||||
assert_eq!(coverage_plan(&before), coverage_plan(&restarted));
|
||||
assert_ne!(reuse_plan(&before), reuse_plan(&restarted));
|
||||
|
||||
@@ -10871,10 +10801,7 @@ fn bucket_checkpoint_plan_survives_a_coordinator_restart() {
|
||||
extra_bucket.push(bucket_info("gamma"));
|
||||
assert_ne!(
|
||||
coverage_plan(&before),
|
||||
crate::scanner_io::scanner_bucket_plan_digest(
|
||||
&extra_bucket,
|
||||
scanner_activity_coverage_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
),
|
||||
crate::scanner_io::scanner_bucket_plan_digest(&extra_bucket, scanner_activity_coverage_digest(&before)),
|
||||
"the bucket list stays part of the plan"
|
||||
);
|
||||
}
|
||||
@@ -11070,8 +10997,8 @@ fn generation_movement_shades_buckets_without_a_guardrail_veto() {
|
||||
assert_eq!(scanner_activity_guardrail_veto(&before, &after), None, "{field} on {host} must not veto");
|
||||
assert!(SCANNER_ACTIVITY_SHADE_FIELDS.contains(field));
|
||||
assert_eq!(
|
||||
scanner_activity_structural_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
scanner_activity_structural_digest(&before),
|
||||
scanner_activity_structural_digest(&after),
|
||||
"{field} must stay outside the structural digest"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -395,7 +395,7 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
progress.started_plan.0[0] ^= 1;
|
||||
let epoch = checkpoint.info.scan_identity.expect("checkpoint identity").publication_epoch;
|
||||
checkpoint
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, epoch, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(set.clone(), &cache_name, &revisions, epoch)
|
||||
.await
|
||||
.expect("persist a prefix from a different process plan");
|
||||
|
||||
|
||||
@@ -932,7 +932,7 @@ async fn usage_durable_root_ack_requires_fresh_contributions_from_every_source()
|
||||
progress.started_plan.0[0] ^= 1;
|
||||
let epoch = cache.info.scan_identity.expect("checkpoint identity").publication_epoch;
|
||||
cache
|
||||
.save_with_revisions_for_epoch(set, &cache_name, &revisions, epoch, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(set, &cache_name, &revisions, epoch)
|
||||
.await
|
||||
.expect("persist a resumed interval mismatch on one source");
|
||||
mixed_source = Some(index);
|
||||
|
||||
@@ -623,7 +623,7 @@ async fn checkpoint_fixture_existing_uncovered_cursor_cannot_skip_to_complete()
|
||||
cache.info.scan_coverage_receipt.as_mut().expect("receipt").through = "bucket/z-stale".into();
|
||||
}
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist corrupted existing cursor");
|
||||
let mut loaded = store.strict_load().await;
|
||||
@@ -663,7 +663,7 @@ async fn checkpoint_fixture_existing_uncovered_cursor_cannot_skip_to_complete()
|
||||
assert_eq!(cache.checked_flatten("bucket/b-pending").expect("revisited prefix").size, 7);
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist verified partial");
|
||||
assert!(!store.strict_load().await.info.snapshot_complete);
|
||||
@@ -827,7 +827,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
.await
|
||||
.expect("fixture revision");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist sampling partial");
|
||||
if resume_mode == HealScanMode::Deep {
|
||||
@@ -876,7 +876,7 @@ async fn check_complete_sampling_resumption(resume_mode: HealScanMode) {
|
||||
assert_eq!(cache.find("bucket/prefix/0000").expect("Deep revisits earlier prefix").size, 7);
|
||||
}
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save bounded recovery");
|
||||
if complete {
|
||||
@@ -964,7 +964,6 @@ async fn run_hot_bucket_certification(advance_epoch: bool, logs: &CapturedLogs)
|
||||
tier_registry_generation: Some(identity.tier_registry_generation),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
cache.info.skip_healing = true;
|
||||
@@ -996,7 +995,7 @@ async fn run_hot_bucket_certification(advance_epoch: bool, logs: &CapturedLogs)
|
||||
};
|
||||
assert_eq!(budget.budget_elapsed(), max_objects.is_some(), "round {round} budget");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save hot bucket sweep");
|
||||
let saved = store.strict_load().await;
|
||||
@@ -1210,7 +1209,7 @@ pub(super) async fn scan_budgeted_metadata_repair_tail() -> DataUsageCache {
|
||||
assert!(!cache.info.skip_healing);
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist scanner repair obligation with CAS");
|
||||
let saved = store.strict_load().await;
|
||||
@@ -1283,7 +1282,6 @@ where
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
let prepared = retained(&cache);
|
||||
@@ -1320,7 +1318,7 @@ where
|
||||
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Objects));
|
||||
let scanned = retained(&cache);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist partial checkpoint");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
@@ -1359,14 +1357,14 @@ where
|
||||
.await
|
||||
.expect("read durable checkpoint bytes");
|
||||
let epoch_error = loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 1)
|
||||
.await
|
||||
.expect_err("stale publication epoch must reject persistence");
|
||||
assert!(epoch_error.to_string().contains(crate::SCANNER_PUBLICATION_EPOCH_CHANGED));
|
||||
store.reject_save.store(true, Ordering::SeqCst);
|
||||
loaded.info.next_cycle += 1;
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect_err("injected save failure must not report durable progress");
|
||||
assert_eq!(
|
||||
@@ -1424,7 +1422,6 @@ where
|
||||
tier_registry_generation: Some(identity.tier_registry_generation),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
cache.info.skip_healing = true;
|
||||
@@ -1455,7 +1452,7 @@ where
|
||||
ScannerDiskScanOutcome::NamespaceNotFound(_) => panic!("fixture namespace exists"),
|
||||
};
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save each bounded sweep");
|
||||
let saved = store.strict_load().await;
|
||||
|
||||
@@ -134,7 +134,7 @@ async fn assert_completed_compaction_resumes(mode: HealScanMode) {
|
||||
.await
|
||||
.expect("initial fixture revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist compacted partial cache through the production codec and CAS path");
|
||||
let mut reloaded = store.strict_load().await;
|
||||
|
||||
@@ -42,7 +42,7 @@ async fn save_reload(store: &Arc<FixtureStore>, cache: &DataUsageCache) -> DataU
|
||||
.await
|
||||
.expect("fixture save revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save compacted checkpoint through real codec and revision checks");
|
||||
let loaded = store.strict_load().await;
|
||||
|
||||
@@ -150,7 +150,7 @@ async fn nested_periodic_snapshot(
|
||||
.await
|
||||
.expect("initial periodic fixture revisions");
|
||||
snapshot
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist real mid-walk periodic snapshot");
|
||||
let mut loaded = DataUsageCache::default();
|
||||
@@ -223,7 +223,7 @@ async fn periodic_mixed_tail_requires_terminal_persistence_before_fresh_certific
|
||||
.await
|
||||
.expect("initial periodic fixture revisions");
|
||||
periodic
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("seed durable periodic snapshot");
|
||||
|
||||
@@ -281,7 +281,7 @@ async fn periodic_mixed_tail_requires_terminal_persistence_before_fresh_certific
|
||||
|
||||
store.reject_save.store(round == 0, Ordering::SeqCst);
|
||||
let saved = terminal
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await;
|
||||
if round == 0 {
|
||||
assert!(saved.is_err(), "injected terminal persistence failure must be observable");
|
||||
@@ -369,7 +369,7 @@ async fn periodic_nested_snapshot_keeps_durable_frontier_across_early_repeated_h
|
||||
.await
|
||||
.expect("initial repeated-handoff revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save the inherited valid subtree frontier");
|
||||
let (mut scanner, unused_root) = build_test_scanner().await;
|
||||
@@ -442,7 +442,7 @@ async fn periodic_nested_snapshot_keeps_durable_frontier_across_early_repeated_h
|
||||
"inherited completed object and both newly read objects must survive"
|
||||
);
|
||||
candidate
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist later coverage without losing the inherited prefix");
|
||||
let mut latest = DataUsageCache::default();
|
||||
@@ -958,7 +958,7 @@ async fn periodic_failed_subtree_resumes_healthy_descendants_and_compacted_debt(
|
||||
.await
|
||||
.expect("fresh revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist failure-bearing coverage");
|
||||
let mut adopted = DataUsageCache::default();
|
||||
@@ -1220,7 +1220,7 @@ async fn checkpoint_debt_restart_round(request: &CheckpointDebtRequest) -> serde
|
||||
let expected = checkpoint_debt_observation(&cache, state.identity, frontier, failed_objects);
|
||||
assert_eq!(cache.info.leader_epoch, 8 + u64::from(request.round));
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist combined healthy coverage and failure debt with CAS");
|
||||
let saved = store.strict_load().await;
|
||||
|
||||
@@ -303,7 +303,7 @@ async fn walk_and_save(observe: bool) -> (Vec<String>, serde_json::Value) {
|
||||
.expect("read empty fixture revisions");
|
||||
scanner
|
||||
.new_cache
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0, CancellationToken::new())
|
||||
.save_with_revisions_for_epoch(store.clone(), CACHE_NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save actual walker output through the cache codec and revision gate");
|
||||
let loaded = store.strict_load().await;
|
||||
|
||||
@@ -931,7 +931,6 @@ pub struct ScannerBucketScanPlan {
|
||||
/// coordinator restart or a maintenance publication does not turn a
|
||||
/// resumed sweep into mixed coverage.
|
||||
checkpoint_coverage_digest: DataUsageScanPlanDigest,
|
||||
completed_checkpoint_cycle_floor: Option<u64>,
|
||||
requires_full_scan: bool,
|
||||
service_cohort: Option<Arc<StdMutex<ScannerServiceCohort>>>,
|
||||
// Cache work must invalidate on namespace completion even when its scoped baseline remains reusable.
|
||||
@@ -2146,8 +2145,8 @@ use guards::*;
|
||||
|
||||
pub(crate) use cache::{
|
||||
DataUsageCacheReuseOptions, DataUsageCacheScanState, ScannerCheckpointPersistContext, ScannerCheckpointPersistResult,
|
||||
acquire_scanner_cache_locks, completed_checkpoint_cycle_floor, current_cache_root_or_prepare_with_generation,
|
||||
log_saved_partial_bucket_cache, persist_scanner_checkpoint,
|
||||
acquire_scanner_cache_locks, current_cache_root_or_prepare_with_generation, log_saved_partial_bucket_cache,
|
||||
persist_scanner_checkpoint,
|
||||
};
|
||||
pub use dirty_usage::{
|
||||
ScannerDirtyUsageAckError, ScannerDirtyUsageBucket, ScannerDirtyUsageClearObserver, ScannerDirtyUsageMutationObserver,
|
||||
@@ -2157,10 +2156,8 @@ pub use dirty_usage::{
|
||||
probe_scoped_dirty_usage, record_dirty_usage_bucket, record_dirty_usage_bucket_from_producer,
|
||||
record_dirty_usage_bucket_from_producers, record_dirty_usage_object, record_dirty_usage_object_from_producer,
|
||||
record_scanner_maintenance_change, record_status_class_mutation, replay_durable_dirty_usage_producer_record,
|
||||
restore_scanner_checkpoint_instance_id, scanner_activity_epoch, scanner_checkpoint_instance_id,
|
||||
scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, scanner_process_was_read_only, set_scanner_dirty_usage_clear_observer,
|
||||
set_scanner_dirty_usage_mutation_observer,
|
||||
scanner_activity_epoch, scanner_dirty_usage_bucket_generation, scanner_dirty_usage_snapshot, scanner_dirty_usage_state,
|
||||
scanner_maintenance_generation, set_scanner_dirty_usage_clear_observer, set_scanner_dirty_usage_mutation_observer,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use dirty_usage::{clear_dirty_usage_buckets_for_tests, dirty_usage_buckets_for_tests};
|
||||
|
||||
@@ -97,15 +97,11 @@ where
|
||||
return ScannerCheckpointPersistResult::FenceChanged;
|
||||
}
|
||||
|
||||
let checkpoint_save = checkpoint
|
||||
.save_with_revisions_for_epoch(
|
||||
store.clone(),
|
||||
cache_name,
|
||||
revisions,
|
||||
context.expected_publication_epoch,
|
||||
context.ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
let checkpoint_save = crate::storage_api::ecstore_with_background_disk_io(
|
||||
context.ctx.clone(),
|
||||
checkpoint.save_with_revisions_for_epoch(store.clone(), cache_name, revisions, context.expected_publication_epoch),
|
||||
)
|
||||
.await;
|
||||
if let Err(error) = checkpoint_save {
|
||||
record_checkpoint_save_failure(cache_name, checkpoint, &error);
|
||||
return if matches!(error, StorageError::PreconditionFailed) {
|
||||
@@ -363,39 +359,6 @@ pub(crate) struct DataUsageCacheReuseOptions {
|
||||
/// (`scanner_activity_coverage_digest`); `None` resumes under the reuse
|
||||
/// plan itself for requests from older remote coordinators.
|
||||
pub(crate) checkpoint_plan_digest: Option<DataUsageScanPlanDigest>,
|
||||
/// The last authoritative aggregate's cycle. Later completed bucket
|
||||
/// sweeps are still checkpoints of an unpublished aggregate.
|
||||
pub(crate) completed_checkpoint_cycle_floor: Option<u64>,
|
||||
}
|
||||
|
||||
pub(crate) fn completed_checkpoint_cycle_floor(
|
||||
data: Option<&[u8]>,
|
||||
expected_sources: &HashSet<DataUsageCacheSource>,
|
||||
next_cycle: u64,
|
||||
leader_epoch: u64,
|
||||
) -> Option<u64> {
|
||||
let usage: DataUsageInfo = serde_json::from_slice(data?).ok()?;
|
||||
if usage.scanner_epoch.is_some_and(|epoch| epoch > leader_epoch) {
|
||||
return None;
|
||||
}
|
||||
if crate::scanner::data_usage_info_is_bootstrap_pending(&usage) {
|
||||
return Some(0);
|
||||
}
|
||||
let cycle = usage.scanner_cycle.filter(|cycle| *cycle < next_cycle)?;
|
||||
let plan = DataUsageScanPlanDigest(usage.usage_snapshot_set_states.first()?.scan_plan_digest?);
|
||||
complete_scanner_cache_snapshot_plan_digest(
|
||||
&usage,
|
||||
ScannerCacheBaselineProof {
|
||||
authoritative_data: None,
|
||||
observed_candidate_data: None,
|
||||
expected_sources,
|
||||
leader_epoch: usage.scanner_epoch?,
|
||||
want_cycle: cycle,
|
||||
scan_plan_digest: plan,
|
||||
},
|
||||
None,
|
||||
)?;
|
||||
Some(cycle)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -420,7 +383,6 @@ pub(crate) fn current_cache_root_or_prepare(
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
@@ -435,38 +397,6 @@ pub(crate) fn current_cache_root_or_prepare_with_generation(
|
||||
options: DataUsageCacheReuseOptions,
|
||||
) -> DataUsageCacheScanState {
|
||||
let checkpoint_plan_digest = options.checkpoint_plan_digest.unwrap_or(scan_plan_digest);
|
||||
// A new leader must not discard finished progress while other buckets
|
||||
// still need budget. Adopt only completed, unpublished coverage; after
|
||||
// the aggregate is published the cycle floor forces the usual refresh.
|
||||
// Keep the saved cycle unchanged so reuse cannot extend this authority.
|
||||
// The reuse plan still binds every process incarnation: a restart may
|
||||
// lose unflushed dirty hints and must invalidate completed coverage.
|
||||
let unpublished_completion = options.completed_checkpoint_cycle_floor.is_some_and(|floor| {
|
||||
cache.info.next_cycle > floor && cache.info.next_cycle < next_cycle && cache.info.leader_epoch < leader_epoch
|
||||
}) && options.checkpoint_identity.is_some_and(|identity| {
|
||||
identity.is_valid()
|
||||
&& cache.info.scan_identity == Some(identity)
|
||||
&& cache.info.tier_registry_generation == Some(identity.tier_registry_generation)
|
||||
&& options.tier_registry_generation == Some(identity.tier_registry_generation)
|
||||
}) && name != DATA_USAGE_ROOT
|
||||
&& cache.info.name == name
|
||||
&& cache.info.source == Some(source)
|
||||
&& cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT
|
||||
&& cache.info.snapshot_complete
|
||||
&& cache.info.last_update.is_some()
|
||||
&& cache.info.scan_progress.is_none()
|
||||
&& !cache.has_scan_position()
|
||||
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
|
||||
&& options.checkpoint_plan_digest.is_some()
|
||||
&& cache.info.scan_coverage_digest == Some(checkpoint_plan_digest)
|
||||
&& !cache
|
||||
.info
|
||||
.replication
|
||||
.as_ref()
|
||||
.is_some_and(|config| config.has_active_rules("", true));
|
||||
if unpublished_completion && let Ok(root) = cache_root_entry_info(cache) {
|
||||
return DataUsageCacheScanState::Current(Box::new(root));
|
||||
}
|
||||
if cache.info.next_cycle <= next_cycle
|
||||
&& cache.info.leader_epoch <= leader_epoch
|
||||
&& cache.info.scan_identity != options.checkpoint_identity
|
||||
@@ -1549,13 +1479,7 @@ pub(super) async fn persist_scanner_service_ledger(
|
||||
ledger.leader_epoch = leader_epoch;
|
||||
persisted.info.scan_bucket_service = Some(ledger);
|
||||
match persisted
|
||||
.save_with_revisions_for_epoch(
|
||||
store,
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
&revisions,
|
||||
expected_publication_epoch,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.save_with_revisions_for_epoch(store, DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
|
||||
.await
|
||||
{
|
||||
Ok(()) => true,
|
||||
@@ -1738,13 +1662,7 @@ pub(super) async fn persist_and_publish_cache_snapshot(
|
||||
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
if let Err(e) = cache_snapshot
|
||||
.save_with_revisions_for_epoch(
|
||||
store.clone(),
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
&revisions,
|
||||
expected_publication_epoch,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.save_with_revisions_for_epoch(store.clone(), DATA_USAGE_CACHE_NAME, &revisions, expected_publication_epoch)
|
||||
.await
|
||||
{
|
||||
error!(
|
||||
|
||||
@@ -40,8 +40,6 @@ static DIRTY_USAGE_MUTATION_OBSERVER: LazyLock<StdRwLock<Option<ScannerDirtyUsag
|
||||
pub(super) static DIRTY_USAGE_PRODUCER_COVERAGE: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static DIRTY_USAGE_BUCKET_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
pub(super) static SCANNER_ACTIVITY_EPOCH: LazyLock<String> = LazyLock::new(|| format!("{:032x}", rand::random::<u128>()));
|
||||
static SCANNER_CHECKPOINT_INSTANCE: std::sync::OnceLock<String> = std::sync::OnceLock::new();
|
||||
static SCANNER_PROCESS_MUTATED: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);
|
||||
pub(super) static SCANNER_MAINTENANCE_GENERATION: AtomicU64 = AtomicU64::new(0);
|
||||
pub(super) static SCANNER_MAINTENANCE_NOTIFY: LazyLock<Notify> = LazyLock::new(Notify::new);
|
||||
const SCANNER_DURABLE_DIRTY_USAGE_REPLAY_SCHEMA: u16 = 1;
|
||||
@@ -154,7 +152,6 @@ fn notify_dirty_usage_mutation(
|
||||
if bucket.is_empty() {
|
||||
return;
|
||||
}
|
||||
SCANNER_PROCESS_MUTATED.store(true, Ordering::Release);
|
||||
let observer = DIRTY_USAGE_MUTATION_OBSERVER
|
||||
.read()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
@@ -1375,30 +1372,6 @@ pub fn scanner_activity_epoch() -> &'static str {
|
||||
SCANNER_ACTIVITY_EPOCH.as_str()
|
||||
}
|
||||
|
||||
/// Identity used only for checkpoint continuity, never for RPC or dirty ACKs.
|
||||
/// Without a consumed clean-shutdown receipt it is the fresh process identity.
|
||||
pub fn scanner_checkpoint_instance_id() -> &'static str {
|
||||
SCANNER_CHECKPOINT_INSTANCE.get_or_init(|| scanner_activity_epoch().to_owned())
|
||||
}
|
||||
|
||||
/// Dirty replay is not a new mutation. Live invalidations prevent handing
|
||||
/// checkpoint authority to another process even after their dirty ACKs drain.
|
||||
pub fn scanner_process_was_read_only() -> bool {
|
||||
!SCANNER_PROCESS_MUTATED.load(Ordering::Acquire) && scanner_maintenance_generation() == 0
|
||||
}
|
||||
|
||||
/// Restore continuity before starting any scanner or mutation producer.
|
||||
/// The caller must first consume matching, durable, read-only shutdown receipts
|
||||
/// on every local disk. Returns false for invalid or already initialized state.
|
||||
pub fn restore_scanner_checkpoint_instance_id(instance: &str) -> bool {
|
||||
instance.len() == 32
|
||||
&& instance
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
|
||||
&& instance.bytes().any(|byte| byte != b'0')
|
||||
&& SCANNER_CHECKPOINT_INSTANCE.set(instance.to_owned()).is_ok()
|
||||
}
|
||||
|
||||
pub fn scanner_dirty_usage_state() -> ScannerDirtyUsageState {
|
||||
let dirty_buckets = dirty_usage_buckets();
|
||||
ScannerDirtyUsageState {
|
||||
|
||||
@@ -389,7 +389,6 @@ impl ScannerIOCache for SetDisks {
|
||||
digest: scan_plan_digest,
|
||||
bucket_coverage_digest,
|
||||
checkpoint_coverage_digest,
|
||||
completed_checkpoint_cycle_floor,
|
||||
requires_full_scan,
|
||||
service_cohort,
|
||||
execution_digest,
|
||||
@@ -1516,7 +1515,6 @@ impl ScannerIOCache for SetDisks {
|
||||
tier_registry_generation: Some(tier_registry_generation),
|
||||
checkpoint_identity,
|
||||
checkpoint_plan_digest: Some(bucket_checkpoint_plan_digest),
|
||||
completed_checkpoint_cycle_floor,
|
||||
},
|
||||
);
|
||||
let outcome = match scan_state {
|
||||
@@ -1876,11 +1874,10 @@ impl ScannerIOCache for SetDisks {
|
||||
} else {
|
||||
// The sweep finished under its coverage plan; the
|
||||
// complete cache is current under the reuse plan,
|
||||
// while its coverage plan remains available until
|
||||
// the coordinated aggregate is published.
|
||||
// which a restart or a maintenance publication
|
||||
// changes so the bucket is walked again next time.
|
||||
if completed_cache.info.snapshot_complete {
|
||||
completed_cache.info.scan_plan_digest = Some(bucket_scan_plan_digest);
|
||||
completed_cache.info.scan_coverage_digest = Some(bucket_checkpoint_plan_digest);
|
||||
}
|
||||
(completed_cache, false)
|
||||
}
|
||||
@@ -1932,7 +1929,6 @@ impl ScannerIOCache for SetDisks {
|
||||
cache_name.as_str(),
|
||||
&revisions,
|
||||
expected_publication_epoch_clone,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
@@ -2046,7 +2042,6 @@ impl ScannerIOCache for SetDisks {
|
||||
&cache_name,
|
||||
&revisions,
|
||||
expected_publication_epoch_clone,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
{
|
||||
|
||||
@@ -619,14 +619,6 @@ where
|
||||
};
|
||||
|
||||
let distributed = store.setup_is_dist_erasure().await;
|
||||
let local_node = if distributed {
|
||||
store
|
||||
.scanner_local_node_name()
|
||||
.filter(|name| !name.is_empty())
|
||||
.ok_or_else(|| Error::other("scanner local member identity is unavailable"))?
|
||||
} else {
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE.to_string()
|
||||
};
|
||||
#[cfg(test)]
|
||||
let distributed = distributed || peer_observations.is_some();
|
||||
let activity_before = match scanner_activity_preflight(
|
||||
@@ -731,23 +723,15 @@ where
|
||||
let activity_before = remote_start
|
||||
.as_ref()
|
||||
.map_or(activity_before, ScannerRemoteDirtyUsageWindow::activity_for_walk);
|
||||
let structural_scan_plan_digest = scanner_bucket_plan_digest(
|
||||
&all_buckets,
|
||||
crate::scanner::scanner_activity_structural_digest(&activity_before, &local_node),
|
||||
);
|
||||
let structural_scan_plan_digest =
|
||||
scanner_bucket_plan_digest(&all_buckets, crate::scanner::scanner_activity_structural_digest(&activity_before));
|
||||
let scan_plan_digest = scanner_bucket_work_digest(structural_scan_plan_digest, scan_mode, requires_full_scan);
|
||||
let activity_digest = crate::scanner::scanner_activity_snapshot_digest(&activity_before);
|
||||
let bucket_coverage_digest = scanner_bucket_plan_digest(
|
||||
&all_buckets,
|
||||
crate::scanner::scanner_activity_bucket_reuse_digest(&activity_before, &local_node),
|
||||
);
|
||||
// A bucket checkpoint must survive unrelated inventory changes.
|
||||
let checkpoint_coverage_digest =
|
||||
DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&activity_before, &local_node));
|
||||
let completed_checkpoint_cycle_floor =
|
||||
completed_checkpoint_cycle_floor(persisted_usage_baseline.as_deref(), &expected_sources, want_cycle, leader_epoch);
|
||||
let execution_digest =
|
||||
scanner_bucket_work_digest(scanner_bucket_plan_digest(&all_buckets, activity_digest), scan_mode, requires_full_scan);
|
||||
let bucket_coverage_digest = scanner_bucket_plan_digest(&all_buckets, activity_digest);
|
||||
// A bucket checkpoint must not be invalidated by another bucket joining
|
||||
// the inventory. The set plan above still binds the complete inventory.
|
||||
let checkpoint_coverage_digest = DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&activity_before));
|
||||
let execution_digest = scanner_bucket_work_digest(bucket_coverage_digest, scan_mode, requires_full_scan);
|
||||
let dirty_usage_snapshot = Arc::new(snapshot_dirty_usage_buckets_after_listing(&all_buckets, dirty_usage_cutoff));
|
||||
let baseline_proof = ScannerCacheBaselineProof {
|
||||
authoritative_data: persisted_usage_baseline.as_ref(),
|
||||
@@ -1030,7 +1014,6 @@ where
|
||||
digest: structural_scan_plan_digest,
|
||||
bucket_coverage_digest,
|
||||
checkpoint_coverage_digest,
|
||||
completed_checkpoint_cycle_floor,
|
||||
requires_full_scan,
|
||||
service_cohort: service_cohort.clone(),
|
||||
execution_digest,
|
||||
|
||||
@@ -635,7 +635,6 @@ mod tests {
|
||||
tier_registry_generation: None,
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
};
|
||||
assert!(matches!(
|
||||
current_cache_root_or_prepare_with_generation(&mut cache.clone(), "cold-bucket", source, 7, 11, old_plan, options()),
|
||||
|
||||
@@ -827,7 +827,6 @@ fn current_cache_root_with_new_tier_generation_resets_old_cache() {
|
||||
tier_registry_generation: Some(2),
|
||||
checkpoint_identity: None,
|
||||
checkpoint_plan_digest: None,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -547,102 +547,11 @@ async fn setup_two_pool_scanner_store() -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
pub(crate) async fn setup_local_scanner_store(pool_count: usize, drives_per_set: usize) -> (tempfile::TempDir, Arc<ECStore>) {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("multi-pool scanner test directory should be created");
|
||||
let store = open_local_scanner_store(temp_dir.path(), pool_count, drives_per_set, false).await;
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn epoch_fenced_cache_save_uses_background_admission_on_every_disk() {
|
||||
#[derive(Clone)]
|
||||
struct AdmissionRecorder {
|
||||
root: String,
|
||||
counts: Arc<std::sync::Mutex<HashMap<String, Arc<AtomicU64>>>>,
|
||||
}
|
||||
impl metrics::Recorder for AdmissionRecorder {
|
||||
fn describe_counter(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn describe_gauge(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn describe_histogram(&self, _: metrics::KeyName, _: Option<metrics::Unit>, _: metrics::SharedString) {}
|
||||
fn register_counter(&self, key: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Counter {
|
||||
if key.name() != "rustfs_disk_io_admission_total"
|
||||
|| !key
|
||||
.labels()
|
||||
.any(|label| label.key() == "class" && label.value() == "background")
|
||||
{
|
||||
return metrics::Counter::noop();
|
||||
}
|
||||
let Some(disk) = key
|
||||
.labels()
|
||||
.find(|label| label.key() == "disk" && label.value().starts_with(&self.root))
|
||||
else {
|
||||
return metrics::Counter::noop();
|
||||
};
|
||||
metrics::Counter::from_arc(
|
||||
self.counts
|
||||
.lock()
|
||||
.expect("admission counters")
|
||||
.entry(disk.value().to_string())
|
||||
.or_default()
|
||||
.clone(),
|
||||
)
|
||||
}
|
||||
fn register_gauge(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Gauge {
|
||||
metrics::Gauge::noop()
|
||||
}
|
||||
fn register_histogram(&self, _: &metrics::Key, _: &metrics::Metadata<'_>) -> metrics::Histogram {
|
||||
metrics::Histogram::noop()
|
||||
}
|
||||
}
|
||||
let (root, store) = setup_local_scanner_store(1, 4).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let epoch = scanner_publication_epoch(set.clone()).await.expect("publication epoch");
|
||||
let cache_name = "background-admission/.usage-cache.bin";
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "background-admission".to_string();
|
||||
let recorder = AdmissionRecorder {
|
||||
root: root.path().to_str().expect("fixture path").to_string(),
|
||||
counts: Arc::default(),
|
||||
};
|
||||
let counts = recorder.counts.clone();
|
||||
// nextest runs this test in its own process; the recorder also observes the
|
||||
// detached commit tasks and filters out every disk outside this fixture.
|
||||
metrics::set_global_recorder(recorder).expect("native save recorder");
|
||||
cache.save(set.clone(), cache_name).await.expect("unclassified control save");
|
||||
assert!(counts.lock().expect("control counters").is_empty());
|
||||
let revisions = DataUsageCache::read_revisions(set.clone(), cache_name)
|
||||
.await
|
||||
.expect("cache revisions");
|
||||
cache.info.next_cycle = 7;
|
||||
cache
|
||||
.save_with_revisions_for_epoch(set.clone(), cache_name, &revisions, epoch, CancellationToken::new())
|
||||
.await
|
||||
.expect("background primary and backup save");
|
||||
{
|
||||
let counts = counts.lock().expect("background counters");
|
||||
assert_eq!(counts.len(), 4, "every local disk must receive background cache I/O");
|
||||
assert!(counts.values().all(|counter| counter.load(Ordering::Relaxed) > 0));
|
||||
}
|
||||
for name in [cache_name.to_string(), format!("{cache_name}.bkp")] {
|
||||
let body = crate::read_config(set.clone(), &path_join_buf(&[crate::BUCKET_META_PREFIX, &name]))
|
||||
.await
|
||||
.expect("persisted cache copy");
|
||||
let saved: DataUsageCache = rmp_serde::from_slice(&body).expect("decode persisted cache");
|
||||
assert_eq!(saved.info.name, cache.info.name);
|
||||
assert_eq!(saved.info.next_cycle, 7);
|
||||
}
|
||||
}
|
||||
|
||||
async fn open_local_scanner_store(
|
||||
root: &std::path::Path,
|
||||
pool_count: usize,
|
||||
drives_per_set: usize,
|
||||
process_boot: bool,
|
||||
) -> Arc<ECStore> {
|
||||
init_ecstore_config_for_scanner_tests();
|
||||
let mut pools = Vec::new();
|
||||
for pool_index in 0..pool_count {
|
||||
let mut endpoints = Vec::new();
|
||||
for disk_index in 0..drives_per_set {
|
||||
let disk_path = root.join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
let disk_path = temp_dir.path().join(format!("pool{pool_index}-disk{disk_index}"));
|
||||
tokio::fs::create_dir_all(&disk_path)
|
||||
.await
|
||||
.expect("multi-pool scanner test disk should be created");
|
||||
@@ -664,16 +573,6 @@ async fn open_local_scanner_store(
|
||||
}
|
||||
|
||||
let endpoint_pools = EndpointServerPools::from(pools);
|
||||
if process_boot {
|
||||
crate::storage_api::ecstore_init_local_disks(endpoint_pools.clone())
|
||||
.await
|
||||
.expect("process disk registry");
|
||||
let store = ECStore::new("127.0.0.1:0".parse().expect("test address"), endpoint_pools, CancellationToken::new())
|
||||
.await
|
||||
.expect("process scanner store");
|
||||
init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
|
||||
return store;
|
||||
}
|
||||
let instance_ctx = Arc::new(InstanceContext::new());
|
||||
instance_ctx.set_endpoints(endpoint_pools.clone());
|
||||
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
|
||||
@@ -702,7 +601,7 @@ async fn open_local_scanner_store(
|
||||
.expect("scanner fixture must expose its live metadata owner");
|
||||
assert!(Arc::ptr_eq(&owner, &store), "scanner watch metadata owner must be this fixture");
|
||||
|
||||
store
|
||||
(temp_dir, store)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2595,7 +2494,6 @@ async fn set_snapshot_reuse_requires_execution_identity_and_fences_stale_writers
|
||||
digest: DataUsageScanPlanDigest([6; 32]),
|
||||
bucket_coverage_digest: DataUsageScanPlanDigest([6; 32]),
|
||||
checkpoint_coverage_digest: DataUsageScanPlanDigest([6; 32]),
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: false,
|
||||
execution_digest: empty_execution,
|
||||
leader_epoch: 11,
|
||||
|
||||
@@ -14,143 +14,6 @@ use super::*;
|
||||
const HOT_OBJECTS: [&str; 6] = ["p0/a", "p1/b", "p2/c", "p3/d", "p4/e", "p5/f"];
|
||||
const LEADER_EPOCH: u64 = 11;
|
||||
|
||||
#[test]
|
||||
fn completed_checkpoint_reuse_requires_unpublished_unchanged_coverage() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
let coverage = DataUsageScanPlanDigest([2; 32]);
|
||||
let identity = crate::DataUsageScanIdentity {
|
||||
version: 1,
|
||||
bucket_incarnation: Uuid::new_v4(),
|
||||
set_layout: DataUsageScanPlanDigest([3; 32]),
|
||||
publication_epoch: 1,
|
||||
tier_registry_generation: 2,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
};
|
||||
let mut saved = DataUsageCache {
|
||||
info: DataUsageCacheInfo {
|
||||
name: "completed".to_string(),
|
||||
next_cycle: 8,
|
||||
leader_epoch: 11,
|
||||
last_update: Some(SystemTime::now()),
|
||||
source: Some(source),
|
||||
snapshot_complete: true,
|
||||
scan_plan_digest: Some(plan),
|
||||
scan_coverage_digest: Some(coverage),
|
||||
scan_identity: Some(identity),
|
||||
tier_registry_generation: Some(2),
|
||||
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
saved.replace(
|
||||
"completed",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 3,
|
||||
size: 48,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
for case in [
|
||||
"unpublished",
|
||||
"published",
|
||||
"missing_floor",
|
||||
"restart",
|
||||
"mutation",
|
||||
"legacy",
|
||||
"newer_cycle",
|
||||
"newer_leader",
|
||||
"same_leader",
|
||||
"tier",
|
||||
"incarnation",
|
||||
"source",
|
||||
"deep",
|
||||
"corrupt",
|
||||
"partial",
|
||||
] {
|
||||
let mut cache = saved.clone();
|
||||
let mut options = DataUsageCacheReuseOptions {
|
||||
require_source: true,
|
||||
tier_registry_generation: Some(2),
|
||||
checkpoint_identity: Some(identity),
|
||||
checkpoint_plan_digest: Some(coverage),
|
||||
completed_checkpoint_cycle_floor: Some(7),
|
||||
};
|
||||
match case {
|
||||
"published" => options.completed_checkpoint_cycle_floor = Some(8),
|
||||
"missing_floor" => options.completed_checkpoint_cycle_floor = None,
|
||||
"restart" => cache.info.scan_plan_digest = Some(DataUsageScanPlanDigest([4; 32])),
|
||||
"mutation" => cache.info.scan_coverage_digest = Some(DataUsageScanPlanDigest([4; 32])),
|
||||
"legacy" => cache.info.scan_coverage_digest = None,
|
||||
"newer_cycle" => cache.info.next_cycle = 10,
|
||||
"newer_leader" => cache.info.leader_epoch = 13,
|
||||
"same_leader" => cache.info.leader_epoch = 12,
|
||||
"tier" => options.tier_registry_generation = Some(3),
|
||||
"incarnation" => options.checkpoint_identity.as_mut().expect("identity").bucket_incarnation = Uuid::new_v4(),
|
||||
"source" => cache.info.source = Some(DataUsageCacheSource::new(1, 0)),
|
||||
"deep" => options.checkpoint_identity.as_mut().expect("identity").scan_mode = HealScanMode::Deep,
|
||||
"corrupt" => cache.cache.clear(),
|
||||
"partial" => cache.info.scan_resume_after = Some("completed/a".to_string()),
|
||||
"unpublished" => {}
|
||||
_ => unreachable!("all fixture cases are handled"),
|
||||
}
|
||||
let state = current_cache_root_or_prepare_with_generation(&mut cache, "completed", source, 9, 12, plan, options);
|
||||
assert_eq!(matches!(state, DataUsageCacheScanState::Current(_)), case == "unpublished", "{case}");
|
||||
if case == "unpublished" {
|
||||
assert_eq!(cache.info.next_cycle, 8, "adoption must not extend the saved receipt's lifetime");
|
||||
let DataUsageCacheScanState::Current(root) = state else { unreachable!() };
|
||||
assert_eq!(root.entry.objects, 3);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn completed_checkpoint_floor_requires_an_authoritative_baseline() {
|
||||
let source = DataUsageCacheSource::new(0, 0);
|
||||
let sources = HashSet::from([source]);
|
||||
let baseline: serde_json::Value =
|
||||
serde_json::from_slice(&complete_usage_baseline(source, DataUsageScanPlanDigest([1; 32]), 7, 11))
|
||||
.expect("complete baseline");
|
||||
for case in [
|
||||
"complete",
|
||||
"bootstrap",
|
||||
"observational",
|
||||
"partial",
|
||||
"incomplete",
|
||||
"legacy",
|
||||
"no_cycle",
|
||||
] {
|
||||
let mut value = baseline.clone();
|
||||
match case {
|
||||
"bootstrap" => {
|
||||
value = serde_json::to_value(crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(11)))
|
||||
.expect("bootstrap marker");
|
||||
}
|
||||
"observational" => value["usage_snapshot_observational"] = true.into(),
|
||||
"partial" => value["usage_snapshot_partial"] = true.into(),
|
||||
"incomplete" => value["usage_snapshot_complete"] = false.into(),
|
||||
"legacy" => value["scanner_epoch"] = serde_json::Value::Null,
|
||||
"no_cycle" => value["scanner_cycle"] = serde_json::Value::Null,
|
||||
"complete" => {}
|
||||
_ => unreachable!("all fixture cases are handled"),
|
||||
}
|
||||
let bytes = serde_json::to_vec(&value).expect("baseline fixture");
|
||||
assert_eq!(
|
||||
completed_checkpoint_cycle_floor(Some(&bytes), &sources, 9, 12),
|
||||
match case {
|
||||
"complete" => Some(7),
|
||||
"bootstrap" => Some(0),
|
||||
_ => None,
|
||||
},
|
||||
"{case}"
|
||||
);
|
||||
}
|
||||
assert_eq!(completed_checkpoint_cycle_floor(None, &sources, 9, 12), None);
|
||||
assert_eq!(completed_checkpoint_cycle_floor(Some(b"invalid"), &sources, 9, 12), None);
|
||||
}
|
||||
|
||||
async fn put_fixture_object(store: &Arc<ECStore>, bucket: &str, object: &str) {
|
||||
let mut reader = ScannerPutObjReader::from_vec(vec![7; 16]);
|
||||
store
|
||||
@@ -164,18 +27,6 @@ async fn run_full_cycle(
|
||||
cycle: u64,
|
||||
max_objects: Option<u64>,
|
||||
baseline: Option<&DataUsageInfo>,
|
||||
) -> (ScannerCycleResult, Option<DataUsageInfo>, Arc<ScannerCycleBudget>) {
|
||||
let baseline = baseline
|
||||
.map(|snapshot| Bytes::from(serde_json::to_vec(snapshot).expect("the actual candidate must encode as a baseline")));
|
||||
run_full_cycle_at_epoch(store, cycle, LEADER_EPOCH, max_objects, baseline).await
|
||||
}
|
||||
|
||||
async fn run_full_cycle_at_epoch(
|
||||
store: &Arc<ECStore>,
|
||||
cycle: u64,
|
||||
leader_epoch: u64,
|
||||
max_objects: Option<u64>,
|
||||
persisted_usage_baseline: Option<Bytes>,
|
||||
) -> (ScannerCycleResult, Option<DataUsageInfo>, Arc<ScannerCycleBudget>) {
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
@@ -198,10 +49,12 @@ async fn run_full_cycle_at_epoch(
|
||||
budget: Arc::clone(&budget),
|
||||
updates,
|
||||
want_cycle: cycle,
|
||||
leader_epoch,
|
||||
leader_epoch: LEADER_EPOCH,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
scan_scope: ScannerBucketScanScope::default(),
|
||||
persisted_usage_baseline,
|
||||
persisted_usage_baseline: baseline.map(|snapshot| {
|
||||
Bytes::from(serde_json::to_vec(snapshot).expect("the actual candidate must encode as a baseline"))
|
||||
}),
|
||||
observed_usage_candidate: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
@@ -220,265 +73,6 @@ async fn run_full_cycle_at_epoch(
|
||||
(result, snapshot, budget)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_small_budget_accumulates_completed_buckets_across_leaders() {
|
||||
let (_temp_dir, store) = setup_local_scanner_store(1, 1).await;
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let names = ["bounded-a", "bounded-b", "bounded-c", "bounded-d"];
|
||||
for bucket in names {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("bounded fixture bucket");
|
||||
for object in ["a", "b", "c"] {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
record_dirty_usage_bucket(bucket);
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let baseline = Bytes::from(
|
||||
serde_json::to_vec(&crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(LEADER_EPOCH)))
|
||||
.expect("bootstrap baseline"),
|
||||
);
|
||||
let mut observations = Vec::new();
|
||||
for cycle in 1..=20 {
|
||||
let (result, usage, budget) =
|
||||
run_full_cycle_at_epoch(&store, cycle, LEADER_EPOCH + cycle, Some(2), Some(baseline.clone())).await;
|
||||
observations.push((cycle, result.status, budget.progress()));
|
||||
if result.status == ScannerCycleStatus::Complete {
|
||||
let usage = usage.expect("completed bounded sweep publishes usage");
|
||||
assert_eq!(usage.objects_total_count, 12);
|
||||
assert_eq!(usage.usage_snapshot_converged, Some(true));
|
||||
for bucket in names {
|
||||
assert_eq!(usage.buckets_usage[bucket].objects_count, 3);
|
||||
}
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("fixed budget must accumulate the completed buckets: {observations:?}");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_small_budget_accumulates_across_process_restarts() {
|
||||
const ROOT_ENV: &str = "RUSTFS_CHECKPOINT_RESTART_ROOT";
|
||||
const CYCLE_ENV: &str = "RUSTFS_CHECKPOINT_RESTART_CYCLE";
|
||||
const TEST_NAME: &str =
|
||||
"scanner_io::tests::checkpoint_coverage_identity::checkpoint_small_budget_accumulates_across_process_restarts";
|
||||
let names = ["restart-a", "restart-b", "restart-c", "restart-d"];
|
||||
if let Some(root) = std::env::var_os(ROOT_ENV) {
|
||||
let root = std::path::PathBuf::from(root);
|
||||
let cycle: u64 = std::env::var(CYCLE_ENV)
|
||||
.expect("worker cycle")
|
||||
.parse()
|
||||
.expect("numeric cycle");
|
||||
let store = open_local_scanner_store(&root, 1, 1, true).await;
|
||||
let registry = scanner_set_disk_inventory(&store.pools[0].disk_set[0])
|
||||
.await
|
||||
.into_iter()
|
||||
.map(Some)
|
||||
.collect::<Vec<_>>();
|
||||
let previous = rustfs_heal::heal::consume_read_only_scanner_handoff("test-node", ®istry)
|
||||
.await
|
||||
.expect("consume previous receipt");
|
||||
if let Some(previous) = previous {
|
||||
assert!(crate::restore_scanner_checkpoint_instance_id(&previous));
|
||||
}
|
||||
if cycle == 0 {
|
||||
for bucket in names {
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket");
|
||||
for object in ["a", "b", "c"] {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
rustfs_heal::heal::clear_unclean_shutdown_markers().await;
|
||||
return;
|
||||
}
|
||||
let mutation_path = root.join("mutate.json");
|
||||
let mutated = root.join("mutated");
|
||||
if mutation_path.exists() && !mutated.exists() {
|
||||
let mutation: serde_json::Value =
|
||||
serde_json::from_slice(&tokio::fs::read(&mutation_path).await.expect("mutation request")).expect("mutation JSON");
|
||||
let bucket = mutation["bucket"].as_str().expect("mutation bucket");
|
||||
assert!(names.contains(&bucket));
|
||||
put_fixture_object(&store, bucket, "after-checkpoint").await;
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
assert!(store.scanner_namespace_mutation_generation() > 0 || !crate::scanner_process_was_read_only());
|
||||
if mutation["clean"] == true {
|
||||
rustfs_heal::heal::clear_unclean_shutdown_markers().await;
|
||||
}
|
||||
tokio::fs::write(&mutated, b"committed").await.expect("mutation receipt");
|
||||
let report = serde_json::json!({"instance": scanner_activity_epoch(), "mutated": true});
|
||||
tokio::fs::write(
|
||||
root.join(format!("round-{cycle}.json")),
|
||||
serde_json::to_vec(&report).expect("writer report"),
|
||||
)
|
||||
.await
|
||||
.expect("writer report file");
|
||||
return;
|
||||
}
|
||||
let baseline = if root.join("published.json").exists() {
|
||||
Bytes::from(tokio::fs::read(root.join("published.json")).await.expect("published floor"))
|
||||
} else {
|
||||
Bytes::from(
|
||||
serde_json::to_vec(&crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(LEADER_EPOCH)))
|
||||
.expect("bootstrap baseline"),
|
||||
)
|
||||
};
|
||||
let (result, usage, budget) = run_full_cycle_at_epoch(&store, cycle, LEADER_EPOCH + cycle, Some(2), Some(baseline)).await;
|
||||
let complete = result.status == ScannerCycleStatus::Complete;
|
||||
if complete {
|
||||
let usage = usage.expect("complete sweep must publish");
|
||||
let mutation: Option<serde_json::Value> = if mutated.exists() {
|
||||
Some(
|
||||
serde_json::from_slice(&tokio::fs::read(&mutation_path).await.expect("mutation request"))
|
||||
.expect("mutation JSON"),
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
assert_eq!(usage.objects_total_count, if mutation.is_some() { 13 } else { 12 });
|
||||
assert_eq!(usage.usage_snapshot_converged, Some(true));
|
||||
for bucket in names {
|
||||
let expected = if mutation.as_ref().is_some_and(|mutation| mutation["bucket"] == bucket) {
|
||||
4
|
||||
} else {
|
||||
3
|
||||
};
|
||||
assert_eq!(usage.buckets_usage[bucket].objects_count, expected, "{bucket}");
|
||||
}
|
||||
tokio::fs::write(root.join("published.json"), serde_json::to_vec(&usage).expect("complete usage"))
|
||||
.await
|
||||
.expect("published baseline");
|
||||
}
|
||||
let mut completed_buckets = Vec::new();
|
||||
for bucket in names {
|
||||
let mut cache = DataUsageCache::default();
|
||||
if cache
|
||||
.load(store.pools[0].disk_set[0].clone(), &path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]))
|
||||
.await
|
||||
.is_ok()
|
||||
&& cache.info.snapshot_complete
|
||||
&& !cache.has_scan_position()
|
||||
{
|
||||
completed_buckets.push(bucket);
|
||||
}
|
||||
}
|
||||
assert_eq!(
|
||||
store.scanner_namespace_mutation_generation(),
|
||||
0,
|
||||
"scanner must not mint namespace mutations"
|
||||
);
|
||||
assert!(
|
||||
crate::scanner_process_was_read_only(),
|
||||
"a scanner-only process must be eligible for a read-only handoff"
|
||||
);
|
||||
rustfs_heal::heal::save_read_only_scanner_handoff("test-node", crate::scanner_checkpoint_instance_id())
|
||||
.await
|
||||
.expect("persist read-only handoff");
|
||||
let report = serde_json::json!({
|
||||
"instance": scanner_activity_epoch(), "checkpoint_instance": crate::scanner_checkpoint_instance_id(),
|
||||
"complete": complete, "objects_processed": budget.progress().0, "completed_buckets": completed_buckets,
|
||||
});
|
||||
tokio::fs::write(
|
||||
root.join(format!("round-{cycle}.json")),
|
||||
serde_json::to_vec(&report).expect("worker report"),
|
||||
)
|
||||
.await
|
||||
.expect("worker report file");
|
||||
return;
|
||||
}
|
||||
for scenario in ["quiet", "clean-writer", "crashed-writer"] {
|
||||
let root = tempfile::tempdir().expect("restart fixture root");
|
||||
let mut instances = HashSet::new();
|
||||
let mut observations = Vec::new();
|
||||
let mut injected = false;
|
||||
let mut complete = false;
|
||||
let mut before_mutation = None;
|
||||
let mut expect_fresh = false;
|
||||
for cycle in 0..=40 {
|
||||
let log_path = root.path().join(format!("round-{cycle}.log"));
|
||||
let output = std::fs::File::create(&log_path).expect("worker log");
|
||||
let mut child = std::process::Command::new(std::env::current_exe().expect("test executable"))
|
||||
.args(["--exact", TEST_NAME, "--nocapture"])
|
||||
.env(ROOT_ENV, root.path())
|
||||
.env(CYCLE_ENV, cycle.to_string())
|
||||
.env("RUST_MIN_STACK", "4194304")
|
||||
.stdout(output.try_clone().expect("worker stdout"))
|
||||
.stderr(output)
|
||||
.spawn()
|
||||
.expect("restart worker");
|
||||
let status = tokio::time::timeout(Duration::from_secs(75), async {
|
||||
loop {
|
||||
if let Some(status) = child.try_wait().expect("worker status") {
|
||||
break status;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(25)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap_or_else(|_| {
|
||||
child.kill().expect("stop expired worker");
|
||||
child.wait().expect("reap expired worker");
|
||||
panic!("{scenario} round {cycle} exceeded its deadline");
|
||||
});
|
||||
assert!(
|
||||
status.success(),
|
||||
"{scenario} round {cycle}: {}",
|
||||
std::fs::read_to_string(log_path).expect("worker diagnostics")
|
||||
);
|
||||
if cycle == 0 {
|
||||
continue;
|
||||
}
|
||||
let report: serde_json::Value = serde_json::from_slice(
|
||||
&tokio::fs::read(root.path().join(format!("round-{cycle}.json")))
|
||||
.await
|
||||
.expect("worker report file"),
|
||||
)
|
||||
.expect("worker report JSON");
|
||||
assert!(instances.insert(report["instance"].as_str().expect("process identity").to_owned()));
|
||||
if report["mutated"] == true {
|
||||
expect_fresh = true;
|
||||
continue;
|
||||
}
|
||||
if expect_fresh {
|
||||
assert_ne!(
|
||||
before_mutation.as_ref(),
|
||||
Some(&report["checkpoint_instance"]),
|
||||
"writer must invalidate continuity"
|
||||
);
|
||||
assert_eq!(report["complete"], false, "old completed roots must not hide the mutation");
|
||||
expect_fresh = false;
|
||||
}
|
||||
if scenario != "quiet" && !injected && !report["completed_buckets"].as_array().expect("completed buckets").is_empty()
|
||||
{
|
||||
assert_eq!(report["complete"], false, "mutation must precede global publication");
|
||||
let mutation = serde_json::json!({"bucket": report["completed_buckets"][0], "clean": scenario == "clean-writer"});
|
||||
tokio::fs::write(root.path().join("mutate.json"), serde_json::to_vec(&mutation).expect("mutation request"))
|
||||
.await
|
||||
.expect("mutation request file");
|
||||
before_mutation = Some(report["checkpoint_instance"].clone());
|
||||
injected = true;
|
||||
}
|
||||
if report["complete"] == true {
|
||||
assert!(cycle > 1);
|
||||
assert!(scenario == "quiet" || injected);
|
||||
complete = true;
|
||||
eprintln!("{scenario}: exact convergence in {cycle} bounded fresh processes");
|
||||
break;
|
||||
}
|
||||
observations.push(report);
|
||||
}
|
||||
assert!(complete, "{scenario}: fixed budget did not converge in 40 restarts: {observations:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn checkpoint_resumes_across_a_maintenance_publication() {
|
||||
@@ -585,13 +179,13 @@ async fn checkpoint_resumes_across_a_maintenance_publication() {
|
||||
.await
|
||||
.expect("the fixture activity should be observable");
|
||||
assert_ne!(
|
||||
crate::scanner::scanner_activity_structural_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_structural_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_structural_digest(&before),
|
||||
crate::scanner::scanner_activity_structural_digest(&after),
|
||||
"a maintenance publication invalidates reusable results"
|
||||
);
|
||||
assert_eq!(
|
||||
crate::scanner::scanner_activity_coverage_digest(&before, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_coverage_digest(&after, crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_coverage_digest(&before),
|
||||
crate::scanner::scanner_activity_coverage_digest(&after),
|
||||
"a maintenance publication keeps the coverage plan"
|
||||
);
|
||||
|
||||
@@ -629,31 +223,6 @@ async fn checkpoint_resumes_across_a_maintenance_publication() {
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_changes() {
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
#[derive(serde::Deserialize)]
|
||||
struct Envelope {
|
||||
payload: Vec<u8>,
|
||||
}
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct Progress {
|
||||
objects_scanned: u64,
|
||||
}
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum FrameResult {
|
||||
Progress,
|
||||
Complete(serde::de::IgnoredAny),
|
||||
Partial,
|
||||
NamespaceNotFound,
|
||||
CycleAhead(serde::de::IgnoredAny),
|
||||
Error(serde::de::IgnoredAny),
|
||||
}
|
||||
#[derive(Debug, serde::Deserialize)]
|
||||
struct Frame {
|
||||
progress: Progress,
|
||||
result: FrameResult,
|
||||
}
|
||||
#[derive(serde::Serialize)]
|
||||
struct WorkerRequest<'a> {
|
||||
version: u16,
|
||||
@@ -676,24 +245,20 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
move || resolved.upgrade()
|
||||
)));
|
||||
rustfs_credentials::set_global_rpc_secret("remote-checkpoint-test-secret".to_string()).expect("test RPC secret");
|
||||
let bucket = "remote-checkpoint";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("remote bucket");
|
||||
for object in HOT_OBJECTS {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let disk = scanner_set_disk_inventory(&set).await.pop().expect("local worker disk");
|
||||
let cache_name = path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]);
|
||||
|
||||
for (index, restart) in [true, false].into_iter().enumerate() {
|
||||
let bucket = if restart {
|
||||
"remote-checkpoint-restart"
|
||||
} else {
|
||||
"remote-checkpoint-maintenance"
|
||||
};
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("remote bucket");
|
||||
for object in HOT_OBJECTS {
|
||||
put_fixture_object(&store, bucket, object).await;
|
||||
}
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
let cache_name = path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]);
|
||||
let before = if restart {
|
||||
std::collections::BTreeMap::from([(
|
||||
"<local>".to_string(),
|
||||
@@ -704,64 +269,36 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
.await
|
||||
.expect("activity before maintenance")
|
||||
};
|
||||
let coverage = DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(
|
||||
&before,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE,
|
||||
));
|
||||
let reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(
|
||||
&before,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE,
|
||||
));
|
||||
let cycle = 100 + u64::try_from(index).expect("fixture index") * 4;
|
||||
for step in 0..4_u64 {
|
||||
let resumed = step > 0;
|
||||
let coverage = DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&before));
|
||||
let reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(&before));
|
||||
let cycle = 100 + u64::try_from(index).expect("fixture index") * 2;
|
||||
for resumed in [false, true] {
|
||||
let activity = if resumed && restart {
|
||||
std::collections::BTreeMap::from([(
|
||||
"<local>".to_string(),
|
||||
crate::scanner::scanner_node_activity_for_tests("instance-after", 0, 0, false),
|
||||
)])
|
||||
} else if resumed {
|
||||
if step == 1 {
|
||||
record_scanner_maintenance_change(bucket);
|
||||
}
|
||||
record_scanner_maintenance_change(bucket);
|
||||
crate::scanner::probe_scanner_activity(store.as_ref(), false)
|
||||
.await
|
||||
.expect("activity after maintenance")
|
||||
} else {
|
||||
before.clone()
|
||||
};
|
||||
let requested_reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(
|
||||
&activity,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE,
|
||||
));
|
||||
let requested_reuse = DataUsageScanPlanDigest(crate::scanner::scanner_activity_structural_digest(&activity));
|
||||
assert_eq!(
|
||||
coverage,
|
||||
DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(
|
||||
&activity,
|
||||
crate::scanner::LOCAL_SCANNER_ACTIVITY_NODE
|
||||
))
|
||||
DataUsageScanPlanDigest(crate::scanner::scanner_activity_coverage_digest(&activity))
|
||||
);
|
||||
if resumed {
|
||||
assert_ne!(reuse, requested_reuse, "the event must invalidate completed cache reuse");
|
||||
}
|
||||
let next_cycle = cycle + step;
|
||||
let leader_epoch = LEADER_EPOCH + next_cycle;
|
||||
if step >= 2 {
|
||||
let baseline = if step == 2 {
|
||||
serde_json::to_vec(&crate::scanner::scanner_usage_bootstrap_marker(SystemTime::now(), Some(leader_epoch)))
|
||||
.expect("bootstrap baseline")
|
||||
} else {
|
||||
complete_usage_baseline(DataUsageCacheSource::new(0, 0), requested_reuse, cycle + 1, leader_epoch - 2)
|
||||
.to_vec()
|
||||
};
|
||||
save_config(store.clone(), crate::DATA_USAGE_OBJ_NAME_PATH.as_str(), baseline)
|
||||
.await
|
||||
.expect("authoritative aggregate floor");
|
||||
}
|
||||
let next_cycle = cycle + u64::from(resumed);
|
||||
save_config(
|
||||
store.clone(),
|
||||
DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
crate::scanner::encode_scanner_cycle_fence_for_test(next_cycle, leader_epoch),
|
||||
crate::scanner::encode_scanner_cycle_fence_for_test(next_cycle, LEADER_EPOCH),
|
||||
)
|
||||
.await
|
||||
.expect("remote request cycle fence");
|
||||
@@ -773,39 +310,28 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
session_sequence: 0,
|
||||
bucket,
|
||||
next_cycle,
|
||||
leader_epoch,
|
||||
leader_epoch: LEADER_EPOCH,
|
||||
scan_plan_digest: requested_reuse,
|
||||
checkpoint_plan_digest: coverage,
|
||||
skip_healing: true,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
budget: serde_json::json!({"max_objects": if step == 1 { None } else { Some(2_u64) }}),
|
||||
budget: serde_json::json!({"max_objects": if resumed { None } else { Some(2_u64) }}),
|
||||
};
|
||||
let body = rmp_serde::to_vec_named(&body).expect("remote worker request body");
|
||||
let request = crate::decode_remote_scanner_request(&body).expect("remote worker decodes coverage identity");
|
||||
let mut response = Vec::new();
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(60),
|
||||
crate::serve_remote_scanner_request(disk.clone(), request, &mut response, CancellationToken::new()),
|
||||
crate::serve_remote_scanner_request(disk.clone(), request, tokio::io::sink(), CancellationToken::new()),
|
||||
)
|
||||
.await
|
||||
.expect("bounded remote worker")
|
||||
.expect("remote worker stream");
|
||||
let mut frames = response.as_slice();
|
||||
let mut terminal: Option<Frame> = None;
|
||||
while !frames.is_empty() {
|
||||
let length = usize::try_from(frames.read_u32().await.expect("frame length")).expect("frame size");
|
||||
let envelope: Envelope = rmp_serde::from_slice(&frames[..length]).expect("response envelope");
|
||||
terminal = Some(rmp_serde::from_slice(&envelope.payload).expect("response frame"));
|
||||
frames = &frames[length..];
|
||||
}
|
||||
let terminal = terminal.expect("terminal worker response");
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache
|
||||
.load(set.clone(), &cache_name)
|
||||
.await
|
||||
.expect("durable remote worker cache");
|
||||
if step == 1 || step == 2 {
|
||||
assert!(matches!(terminal.result, FrameResult::Complete(_)), "step {step}: {terminal:?}");
|
||||
if resumed {
|
||||
assert!(
|
||||
cache.info.snapshot_complete,
|
||||
"a resumed remote sweep must complete without mixed coverage"
|
||||
@@ -813,12 +339,7 @@ async fn remote_checkpoint_resumes_across_restart_and_maintenance_identity_chang
|
||||
assert!(cache.info.scan_progress.is_none());
|
||||
assert_eq!(cache.info.scan_plan_digest, Some(requested_reuse));
|
||||
assert_eq!(cache.flatten(cache.find(bucket).expect("bucket root")).objects, 6);
|
||||
if step == 2 {
|
||||
assert_eq!(terminal.progress.objects_scanned, 0);
|
||||
assert_eq!(cache.info.next_cycle, cycle + 1, "reuse must not renew the completed receipt");
|
||||
}
|
||||
} else {
|
||||
assert!(matches!(terminal.result, FrameResult::Partial), "step {step}: {terminal:?}");
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
assert!(cache.has_scan_position(), "budget must leave real checkpoint progress");
|
||||
let progress = cache.info.scan_progress.expect("interrupted worker records coverage");
|
||||
|
||||
@@ -27,7 +27,6 @@ async fn checkpoint_plan(set: &Arc<SetDisks>, owner: &CancellationToken) -> Scan
|
||||
digest,
|
||||
bucket_coverage_digest: digest,
|
||||
checkpoint_coverage_digest: digest,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: None,
|
||||
execution_digest: digest,
|
||||
|
||||
@@ -130,7 +130,6 @@ async fn scan_plan(set: &Arc<SetDisks>, names: &[&str], cycle: u64) -> ScannerBu
|
||||
digest,
|
||||
bucket_coverage_digest: digest,
|
||||
checkpoint_coverage_digest: digest,
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: true,
|
||||
service_cohort: Some(Arc::new(StdMutex::new(cohort))),
|
||||
execution_digest: digest,
|
||||
|
||||
@@ -8,22 +8,6 @@ const EPOCH: u64 = 7;
|
||||
const RETIRED: &str = "retirement-generation";
|
||||
const LIVE: &str = "retirement-survivor";
|
||||
|
||||
async fn close_fixture(directory: tempfile::TempDir, store: Arc<ECStore>) {
|
||||
store.background_cancel_token().expect("fixture shutdown token").cancel();
|
||||
tokio::time::timeout(Duration::from_secs(30), async {
|
||||
for pool in &store.pools {
|
||||
pool.stop_endpoint_monitor_for_test().await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("endpoint monitors must stop before removing fixture disks");
|
||||
wait_for_namespace_commit_tails(&store).await;
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
async fn run_turn(
|
||||
set: &Arc<SetDisks>,
|
||||
buckets: &[BucketInfo],
|
||||
@@ -53,7 +37,6 @@ async fn run_turn(
|
||||
digest: DataUsageScanPlanDigest([31; 32]),
|
||||
bucket_coverage_digest: DataUsageScanPlanDigest([32; 32]),
|
||||
checkpoint_coverage_digest: DataUsageScanPlanDigest([33; 32]),
|
||||
completed_checkpoint_cycle_floor: None,
|
||||
requires_full_scan: true,
|
||||
execution_digest: DataUsageScanPlanDigest([34; 32]),
|
||||
leader_epoch: EPOCH,
|
||||
@@ -160,7 +143,10 @@ async fn assert_retired_source_turn_discards_only_deleted_generation_and_retains
|
||||
failures.partial.lock().await.contains(RETIRED),
|
||||
"responsibility remains until namespace replanning"
|
||||
);
|
||||
close_fixture(directory, store).await;
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
async fn assert_recreated_source_turn_rescans_successor_without_old_value_or_identity() {
|
||||
@@ -202,7 +188,10 @@ async fn assert_recreated_source_turn_rescans_successor_without_old_value_or_ide
|
||||
assert_eq!((successor.objects, successor.size), (2, 28));
|
||||
assert_eq!(after.info.scan_bucket_incarnations[RETIRED], new_identity);
|
||||
assert_eq!(after.find(LIVE).expect("unrelated bucket").size, 13);
|
||||
close_fixture(directory, store).await;
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
async fn assert_missing_retirement_proof_and_foreign_source_fences_cannot_clear_completion() {
|
||||
@@ -229,7 +218,10 @@ async fn assert_missing_retirement_proof_and_foreign_source_fences_cannot_clear_
|
||||
.expect_err("unconfirmed identity changes cannot retire completed work");
|
||||
assert!(error.to_string().contains("identity"), "{field}: {error}");
|
||||
}
|
||||
close_fixture(directory, store).await;
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
async fn assert_missing_directories_and_unreadable_retirement_proof_remain_unconfirmed() {
|
||||
@@ -259,7 +251,10 @@ async fn assert_missing_directories_and_unreadable_retirement_proof_remain_uncon
|
||||
.await
|
||||
.expect_err("an unreadable retirement must preserve failed completion validation");
|
||||
assert!(error.to_string().contains("identity"));
|
||||
close_fixture(directory, store).await;
|
||||
drop(store);
|
||||
let path = directory.path().to_path_buf();
|
||||
directory.close().expect("remove source retirement fixture");
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -168,8 +168,8 @@ fn watch_structure_ignores_unrelated_dirty_generations_but_requires_every_owner(
|
||||
unrelated.insert("peer".into(), crate::scanner::scanner_node_activity_for_tests("remote", 103, 104, false));
|
||||
assert_eq!(original, watch_structure_digest(&unrelated, &routes).unwrap());
|
||||
assert_ne!(
|
||||
crate::scanner::scanner_activity_coverage_digest(&before, LOCAL_SCANNER_ACTIVITY_NODE),
|
||||
crate::scanner::scanner_activity_coverage_digest(&unrelated, LOCAL_SCANNER_ACTIVITY_NODE)
|
||||
crate::scanner::scanner_activity_coverage_digest(&before),
|
||||
crate::scanner::scanner_activity_coverage_digest(&unrelated)
|
||||
);
|
||||
unrelated.remove("peer");
|
||||
assert!(watch_structure_digest(&unrelated, &routes).is_none());
|
||||
|
||||
@@ -170,8 +170,6 @@ pub(crate) use rustfs_ecstore::api::set_disk::SetDisks as EcstoreSetDisks;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::SCANNER_PUBLICATION_LEASE_TTL_MS as ECSTORE_SCANNER_PUBLICATION_LEASE_TTL_MS;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks as ecstore_init_local_disks;
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::storage::init_local_disks_with_instance_ctx as ecstore_init_local_disks_with_instance_ctx;
|
||||
pub(crate) use rustfs_ecstore::api::storage::{
|
||||
ECStore as EcstoreStore, ScannerDataMovementPauseStatus as EcstoreScannerDataMovementPauseStatus,
|
||||
@@ -412,9 +410,9 @@ pub(crate) mod scan {
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::storage_contracts::{DeleteBucketOptions, MakeBucketOptions, ObjectIO, StorageAdminApi};
|
||||
pub use super::storage_contracts::{
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_ACTIVITY_V7_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES, SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE, SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
SCANNER_ACTIVITY_PROTOCOL_VERSION, SCANNER_ACTIVITY_V6_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES,
|
||||
SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION, SCANNER_DIRTY_USAGE_SNAPSHOT_RPC_MAX_MESSAGE_SIZE,
|
||||
SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use super::{
|
||||
@@ -462,9 +460,6 @@ pub(crate) trait ScannerStorage:
|
||||
+ storage_contracts::NamespaceLocking<Error = EcstoreErrorType, NamespaceLock = NamespaceLockWrapper>
|
||||
{
|
||||
fn scanner_topology_digest(&self) -> [u8; 32];
|
||||
fn scanner_local_node_name(&self) -> Option<String> {
|
||||
None
|
||||
}
|
||||
fn scanner_namespace_mutation_generation(&self) -> u64;
|
||||
async fn scanner_data_movement_activity(&self) -> (bool, bool, u64);
|
||||
async fn scanner_data_usage_publication_blocked(&self) -> bool;
|
||||
@@ -517,15 +512,6 @@ impl ScannerStorage for EcstoreStore {
|
||||
crate::scanner::scanner_topology_digest(self)
|
||||
}
|
||||
|
||||
fn scanner_local_node_name(&self) -> Option<String> {
|
||||
self.endpoints()
|
||||
.0
|
||||
.iter()
|
||||
.flat_map(|pool| pool.endpoints.as_ref().iter())
|
||||
.find(|endpoint| endpoint.is_local)
|
||||
.map(|endpoint| endpoint.grid_host())
|
||||
}
|
||||
|
||||
fn scanner_namespace_mutation_generation(&self) -> u64 {
|
||||
EcstoreStore::scanner_namespace_mutation_generation(self)
|
||||
}
|
||||
|
||||
@@ -58,10 +58,7 @@ pub const SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION: u32 = 5;
|
||||
/// a scanner must not use it as a publication proof because terminal movement
|
||||
/// state is not authenticated by that version.
|
||||
pub const SCANNER_ACTIVITY_V6_PROTOCOL_VERSION: u32 = 6;
|
||||
/// Authenticates terminal movement state, but carries no checkpoint continuity.
|
||||
pub const SCANNER_ACTIVITY_V7_PROTOCOL_VERSION: u32 = 7;
|
||||
/// Authenticates read-only checkpoint continuity separately from the RPC epoch.
|
||||
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 8;
|
||||
pub const SCANNER_ACTIVITY_PROTOCOL_VERSION: u32 = 7;
|
||||
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_PROTOCOL_VERSION: u32 = 2;
|
||||
pub const SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES: usize = 32;
|
||||
pub const SCANNER_DIRTY_USAGE_SNAPSHOT_MAX_ENTRIES: usize = SCANNER_SCOPED_DIRTY_USAGE_ACK_MAX_ENTRIES;
|
||||
|
||||
@@ -162,6 +162,7 @@ where
|
||||
) -> PreparedActivation<E> {
|
||||
let mut prepared = Vec::with_capacity(targets.len());
|
||||
let mut failures = Vec::new();
|
||||
let mut rejected_targets = Vec::new();
|
||||
let mut targets = targets.into_iter();
|
||||
while let Some(target) = targets.next() {
|
||||
match prepare_target(target, cancellation).await {
|
||||
@@ -178,6 +179,19 @@ where
|
||||
});
|
||||
prepared.push(target);
|
||||
}
|
||||
PrepareTargetResult::Failed { error, target } => {
|
||||
drop(error);
|
||||
let target_id = target.id().to_string();
|
||||
tracing::warn!(
|
||||
target_id,
|
||||
reason = "initialization_failed",
|
||||
"Target initialization failed during lifecycle preparation"
|
||||
);
|
||||
failures.push(TargetActivationFailure {
|
||||
detail: format!("{target_id}: initialization failed"),
|
||||
});
|
||||
rejected_targets.push(Arc::from(target));
|
||||
}
|
||||
PrepareTargetResult::Cancelled(target) => {
|
||||
prepared.push(Arc::from(target));
|
||||
prepared.extend(targets.map(Arc::from));
|
||||
@@ -187,7 +201,7 @@ where
|
||||
}
|
||||
PreparedActivation {
|
||||
failures,
|
||||
rejected_targets: Vec::new(),
|
||||
rejected_targets,
|
||||
targets: prepared,
|
||||
}
|
||||
}
|
||||
@@ -601,15 +615,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn builtin_adapter_keeps_non_store_target_when_init_fails() {
|
||||
async fn builtin_adapter_skips_non_store_target_when_init_fails() {
|
||||
let adapter = builtin_adapter();
|
||||
let target = MockTarget::new("primary", "webhook").with_init_failures(usize::MAX);
|
||||
|
||||
// A target without a queue store has nothing to replay, but it stays in
|
||||
// the runtime so it self-heals and remains visible to the target list.
|
||||
let activation = adapter.activate_with_replay(vec![Box::new(target)]).await;
|
||||
|
||||
assert_eq!(activation.targets.len(), 1);
|
||||
assert!(activation.targets.is_empty());
|
||||
assert!(activation.replay_workers.is_empty());
|
||||
}
|
||||
|
||||
|
||||
@@ -87,7 +87,14 @@ where
|
||||
E: PluginEvent,
|
||||
{
|
||||
Ready(SharedTarget<E>),
|
||||
Degraded { error: TargetError, target: SharedTarget<E> },
|
||||
Degraded {
|
||||
error: TargetError,
|
||||
target: SharedTarget<E>,
|
||||
},
|
||||
Failed {
|
||||
error: TargetError,
|
||||
target: Box<dyn Target<E> + Send + Sync>,
|
||||
},
|
||||
Cancelled(Box<dyn Target<E> + Send + Sync>),
|
||||
}
|
||||
|
||||
@@ -530,7 +537,12 @@ where
|
||||
) -> (mpsc::Sender<()>, JoinHandle<()>),
|
||||
{
|
||||
let shared = match prepare_target(target, None).await {
|
||||
PrepareTargetResult::Ready(target) | PrepareTargetResult::Degraded { target, .. } => target,
|
||||
PrepareTargetResult::Ready(target) => target,
|
||||
PrepareTargetResult::Degraded { target, .. } => target,
|
||||
PrepareTargetResult::Failed { target, .. } => {
|
||||
let _ = target.close().await;
|
||||
return None;
|
||||
}
|
||||
PrepareTargetResult::Cancelled(_) => unreachable!("preparation without a cancellation token cannot be cancelled"),
|
||||
};
|
||||
let target_id = shared.id().to_string();
|
||||
@@ -555,6 +567,7 @@ where
|
||||
E: PluginEvent,
|
||||
{
|
||||
let target_id = target.id().to_string();
|
||||
let has_store = target.store().is_some();
|
||||
|
||||
let init_result = match cancellation {
|
||||
Some(cancellation) => {
|
||||
@@ -569,9 +582,13 @@ where
|
||||
|
||||
if let Err(err) = init_result {
|
||||
tracing::error!(target_id = %target_id, reason = "initialization_failed", "Failed to initialize target");
|
||||
// Keep the target in the runtime whether or not it has a queue store: a
|
||||
// quarantined target self-heals through delivery retries and is reported
|
||||
// by the target list, so recovery never needs a runtime rebuild.
|
||||
if !has_store {
|
||||
return PrepareTargetResult::Failed { error: err, target };
|
||||
}
|
||||
tracing::warn!(
|
||||
target_id = %target_id,
|
||||
"Proceeding with store-backed target despite init failure"
|
||||
);
|
||||
return PrepareTargetResult::Degraded {
|
||||
error: err,
|
||||
target: Arc::from(target),
|
||||
|
||||
@@ -255,7 +255,18 @@ walk. Field roles:
|
||||
| `maintenance_generation` | veto: no cycle today records whether it published maintenance results, and the field is already part of the structural digest; it becomes a shade signal only once that distinction exists |
|
||||
| `namespace_generation`, `dirty_usage_generation`, `dirty_usage_pending` | shade signal: consumed per bucket; never a veto on their own |
|
||||
|
||||
The activity snapshot feeds separate traversal, bucket-reuse and publication identities. Traversal binds the canonical peer set, bucket list and creation times, topology, protocol, movement state and checkpoint continuity. Bucket reuse also binds namespace, maintenance and dirty generations. Protocol v8 authenticates a checkpoint identity separately from the fresh RPC process identity: only a clean, read-only shutdown with `RUSTFS_SCANNER_READ_ONLY_HANDOFF=true` can hand that identity to the next process. The flag defaults to false and requires reader-first deployment plus receipt-clearing preparation before legacy rollback. Missing, corrupt, mismatched or unconsumed receipts, a writer, or a crash expire it and invalidate old coverage. Protocol v7 remains readable and retains its fresh process fence. `publication_blocked` belongs to neither cache identity; it remains a cycle veto. Publication, dirty ACKs and leases always bind the actual process identity, even across a verified handoff. See [the shutdown receipt contract](scanner-usage-publication.md#read-only-restart-continuity).
|
||||
The activity snapshot feeds two plan identities with different jobs. The
|
||||
coverage identity (bucket list and creation times, `topology_digest`,
|
||||
`protocol_version`, `data_movement_active`, `movement_generation`) decides
|
||||
whether a resumed bucket checkpoint still describes the same traversal; only
|
||||
it may mark a resumed sweep as mixed coverage. The reuse identity adds
|
||||
`instance_id` and `maintenance_generation` and decides whether a completed
|
||||
cache or a scoped baseline may be reused: a peer restart can lose dirty hints
|
||||
inside the journal's coalescing window and a maintenance change needs a fresh
|
||||
discovery walk, so both force a full walk, but neither changes what a
|
||||
full-walk checkpoint has already covered. `publication_blocked` belongs to
|
||||
neither identity. The guardrail compares the reuse identity plus the movement
|
||||
fields.
|
||||
|
||||
The guardrail is bounded in three ways:
|
||||
|
||||
|
||||
@@ -110,17 +110,21 @@ snapshot for a missing or corrupt authoritative object.
|
||||
|
||||
## Cache Execution Identity
|
||||
|
||||
The structural scan-plan digest remains stable across ordinary bucket writes so a scoped scan can retain unaffected baseline buckets. Bucket reuse additionally binds every member's namespace, maintenance and dirty generations; its process component may survive only the read-only handoff described below. The current cycle's execution digest always binds the full activity snapshot with fresh RPC process identities. A set cache carrying buckets whose walk stopped early also binds those bucket names (`scanner_carried_execution_digest`), so it cannot replace a full walk of the same cycle. Completed bucket progress from an older leader is reusable only above the authoritative aggregate's published cycle floor, with identical incarnation, source, layout, tier, scan mode, checkpoint coverage and bucket reuse identity. Adoption never advances the saved cycle. A published aggregate expires this unpublished-progress exception.
|
||||
|
||||
## Read-Only Restart Continuity
|
||||
|
||||
`RUSTFS_SCANNER_READ_ONLY_HANDOFF` defaults to `false`. Unset, false or invalid values disable both checkpoint-identity restoration and receipt creation; startup still durably consumes existing receipts. Deploy these readers to every possible writer before enabling receipt writes with `true`. Cross-process reuse of unpublished completed buckets requires this opt-in.
|
||||
|
||||
The existing `.rustfs.sys/unclean-shutdown` marker can contain a versioned read-only handoff receipt. It binds the node owner, disk UUID, complete local endpoint/disk inventory and checkpoint identity. Startup consumes every receipt by durably replacing it with `running` before constructing the store, because store construction can start recovery writers. Missing or incompatible receipts provide no continuity; an existing or unreadable receipt that cannot be overwritten aborts startup. Marker writes always sync the file and parent directory, including under the legacy data-sync-off setting. A missing local drive, replacement disk, different inventory or inconsistent receipt identity expires continuity.
|
||||
|
||||
When enabled, shutdown saves matching receipts only after frontend, heal, capacity, table-maintenance, scanner and detached-mutation shutdown fences succeed, and only if the process observed no namespace or maintenance mutation. Live dirty notifications permanently disqualify that process even if the dirty generation is later acknowledged; replaying an existing durable dirty hint is not a new mutation. A writer follows ordinary marker cleanup and receives a fresh checkpoint identity on restart. Cancellation, failed shutdown, crash and partial receipt writes cannot grant continuity. Activity protocol v8 signs the checkpoint identity while keeping RPC instances, ACKs, publication proofs and leases fresh. v7 responses omit this field and use the old fresh-process fence.
|
||||
|
||||
Before any older binary opens the storage, disable the flag on every node, start the current reader with all local drives present, wait for readiness on every node, then shut down cleanly. This preparation consumes receipts without restoring their identities or writing replacements. If a node cannot complete it, rollback must wait. Direct rollback from an enabled writer is unsupported: an older process can start recovery before replacing its marker, and a 1.0.4 writer with Scanner and Heal disabled can acknowledge a foreground write and crash while leaving the receipt unchanged. Metadata synchronization alone does not close that gap. A later opt-in restart after the supported preparation starts a fresh checkpoint lineage.
|
||||
The structural scan-plan digest can remain stable across ordinary bucket writes
|
||||
so a scoped scan can retain unaffected baseline buckets. It is not sufficient
|
||||
proof for reusing a completed result within the same cycle. Bucket work uses an
|
||||
execution digest combining the structural plan and the full activity snapshot,
|
||||
with the bucket's dirty generation included in its cache identity. A bucket
|
||||
sweep's checkpoint is keyed separately by the coverage identity
|
||||
(`scanner_activity_coverage_digest`, see
|
||||
[scanner-per-bucket-convergence.md](scanner-per-bucket-convergence.md)), so
|
||||
the reuse identity can change without restarting a sweep in progress. Completed
|
||||
set caches carry the same execution digest separately from their structural
|
||||
plan; a set cache that stands in for buckets whose walk stopped early binds
|
||||
the carried bucket names into it (`scanner_carried_execution_digest`), so it
|
||||
is never adopted in place of a full walk of the same cycle, nor the reverse.
|
||||
The persisted set-root fast path requires equal execution identities as well as
|
||||
the existing source, cycle, leader, tier, and cache-structure checks.
|
||||
|
||||
Remote workers receive the reuse plan as `scan_plan_digest` and the independent
|
||||
coverage plan as optional `checkpoint_plan_digest`. The v3 capability proof
|
||||
|
||||
@@ -73,8 +73,6 @@ All names are defined in `crates/io-metrics/src/internode_metrics.rs`.
|
||||
|
||||
Explicit channel dial metrics start after endpoint settings validate and cover TLS configuration, waiting for its blocking task, and connection establishment. Each completed or cancelled attempt records one result. TLS configuration errors and acquisition cancellation contribute to `rustfs_system_network_internode_dial_errors_total`. Only connection failures follow the existing peer-unreachable classification. Cache hits reuse the existing channel, and coalesced callers share the dial owner's result. Tonic's internal TCP reconnects and HTTP/2 PING outcomes require their own transport observations.
|
||||
|
||||
Internode clients retain up to 8,192 locally reset streams per connection so late responses to cancelled quorum reads can be ignored. The locked HTTP/2 library's one-second retention period and 1,024 local protocol-error reset limit remain unchanged. This increases bounded transport bookkeeping on both shared and bulk channels; measure memory alongside cancellation and GOAWAY counts. Responses arriving after expiration or beyond the retention capacity can still consume the error budget. This client-side setting requires no wire-format migration; rollback restores the previous capacity.
|
||||
|
||||
| Metric | Stage signal |
|
||||
| --- | --- |
|
||||
| `rustfs_system_network_internode_operation_duration_ms{operation,backend}` | control-plane RTT (P0), lock/bulk latency (P1), first-op latency (P3) |
|
||||
|
||||
@@ -29,8 +29,6 @@ Use rustfs_memory_allocator_malloc_requested_bytes only when rustfs_memory_alloc
|
||||
|
||||
A previously published gauge can remain visible after a statistic becomes unavailable. Filter it by the availability metric. The malloc_requested_total_bytes metric is cumulative allocation activity and can grow indefinitely in a healthy process. Do not use its slope as evidence of retained memory.
|
||||
|
||||
The `rustfs_memory_allocator_allocated_bytes` gauge reports the allocator's `allocated.current` block accounting when `rustfs_memory_allocator_allocated_bytes_available` is one. It includes size-class rounding and can lag page or thread statistics merges. Availability means the field was present and valid, not that accounting was enabled. This observation is distinct from live requested bytes, committed bytes and RSS; zero or a change alone does not prove a drained heap or a leak.
|
||||
|
||||
When live requested bytes are unavailable, capture a supported heap profile or allocation/free trace. Preserve attribution to allocation stacks and lifetime; committed bytes and RSS alone cannot establish a leak. Other raw allocator statistics, including page accounting, depend on the allocator build and are not interchangeable with live application bytes.
|
||||
|
||||
### Diagnostic allocator and runtime metrics
|
||||
|
||||
@@ -314,8 +314,6 @@ last successful timestamp and values unchanged. Missing/invalid limits, invisibl
|
||||
membership, or oversized files must not be interpreted as host capacity or zero
|
||||
pressure. Other platforms and cgroup v1 have no sample from this observer.
|
||||
|
||||
A missing `cpu.max` or `memory.max` is accepted only when `cgroup.controllers` proves that the controller belongs to the verified v2 hierarchy but is not enabled for that cgroup. Visible ancestor limits and sibling memory use still apply. Missing limits for an available controller, unreadable controller lists, and controllers absent from the v2 root remain unavailable; missing pressure counters remain unknown.
|
||||
|
||||
The `cpu_usage_microseconds`, `cpu_throttled_microseconds`, `cpu_stall_microseconds`,
|
||||
`memory_stall_microseconds`, and `io_stall_microseconds` gauges, under the same
|
||||
prefix, expose cumulative leaf-cgroup counters. Each requires its own
|
||||
|
||||
@@ -566,7 +566,7 @@ Scanner startup records an empty scope to establish a genuine zero observation.
|
||||
|
||||
`RUSTFS_HEAL_ADAPTIVE_ADMISSION=true` enables Linux cgroup-v2 feedback shared by the heal manager’s four object-heal entry points and local Scanner namespace reads and object accounting. Scanner directory and local metadata reads acquire the physical leaves backing their local disk; object Heal admits the selected erasure sets across its requested pools. It is off by default while full SSD performance acceptance is pending. Unsupported platforms reject an enabled setting. Physical admission does not yet cover cache publication, lifecycle actions, the abandoned-child streaming list coordinator or independent bucket/format repair. Existing worker, disk and byte budgets still apply.
|
||||
|
||||
The node samples its effective CPU quota/cpuset, ancestor memory headroom, CPU usage/throttling and CPU/memory/I/O pressure once per second. Sampling uses one blocking worker at a time. After a failed resource read, it may make one complete fresh read while time remains in the original 20 ms bracket; it never substitutes a cached observation. Both attempts share their original start time, and two failed reads leave admission closed. Resource reads spanning over 20 ms or older than one second after all inputs return cannot renew admission. `rustfs_background_resource_rereads_total{result="success"|"error"}` counts second-read outcomes; a successful read can still be rejected by the timing checks. Counter rates use both read brackets conservatively. The concurrency cap is at most the rounded-up effective CPU count and 256 scheduling units; existing operator limits still apply independently. At most 1% memory headroom blocks new work. This scheduling safeguard alone does not reserve memory for repair buffers.
|
||||
The node samples its effective CPU quota/cpuset, ancestor memory headroom, CPU usage/throttling and CPU/memory/I/O pressure once per second. Sampling uses one blocking worker at a time. Resource reads retain their real start/end times: reads spanning over 20 ms or older than one second after all inputs return cannot renew admission. Counter rates use both read brackets conservatively. The concurrency cap is at most the rounded-up effective CPU count and 256 scheduling units; existing operator limits still apply independently. At most 1% memory headroom blocks new work. This scheduling safeguard alone does not reserve memory for repair buffers.
|
||||
|
||||
The same sampler also controls the reusable staging storage of admitted buffered file readers and writers, whole-file results, and positioned-read results. Its process-wide cap is the smaller of 10% of the effective memory limit and the remaining headroom above 20% of that limit. Missing or expired samples block new allocations. A buffer reserves its payload size before allocation; growth counts both the old and replacement allocations until the old storage is freed. Shrinking the cap does not revoke an existing allocation, and reuse needs no new reservation. A denied allocation returns `WouldBlock`; reusable file streams accept no write bytes and preserve their read position. Cancellation and runtime shutdown keep storage charged until the actual native job releases it. Returned `Bytes` retain the allocation's charge through the last clone or slice. Standard-backend direct reads charge both the aligned bounce allocation and logical output while they coexist. Native io_uring reads retain their full allocation, including alignment padding, through the returned result; chunked reads also charge the assembled output while a native chunk is pending. Memory-budget denial does not switch either read path to a different backend.
|
||||
|
||||
@@ -584,8 +584,6 @@ Namespace frame payload and envelope encoding use the memory owner captured by t
|
||||
|
||||
This cap does not yet cover other encode/decode paths, other RPC serializers and transport copies, allocator overhead or total process RSS. Already resident format-cache bytes are shared without a new payload allocation. The `rustfs_background_buffer_memory_limit_bytes` and `rustfs_background_buffer_memory_retained_bytes` gauges expose the sampled cap and retained payload. Like the node gauges, they can remain stale when the sampler stalls; the in-process admission deadline still expires after three seconds.
|
||||
|
||||
`rustfs_background_resource_observations_total` distinguishes `valid`, `unavailable`, `stale`, `wide` and `invalid_timing` observations used by admission. `rustfs_background_resource_sample_duration_seconds` records the available sampling brackets. These metrics use the admission sample, unlike the independent process-memory sampler, and stop progressing if that worker stalls. Buffer-denial errors include the requested bytes, retained bytes, configured limit and whether the lease expired, captured under the same budget lock. These diagnostics do not change the cap, freshness limits or retry behavior.
|
||||
|
||||
Each complete sample leases admission for three seconds. Missing resource caps block new work; missing or low-confidence feedback retains at most one maintenance unit. CPU use above 90%, throttling or pressure above 1% of elapsed time, memory headroom at most 10%, foreground errors/cancellations, a foreground queue or a comparable foreground p99 regression reduce concurrency immediately. Recovery requires CPU use at most 70%, throttling/stalls at most 0.1% of elapsed time, at least 20% memory headroom, fresh foreground evidence and actual successful work. The existing controller probes one additional unit only after eight consecutive healthy seconds and cooldown. Limits affect new admission and do not revoke active work.
|
||||
|
||||
The Scanner captures the same node owner once per bucket walk. Directory open/refill, entry type and size lookup, and object accounting try admission before their first storage poll; each scheduling unit ends when its operation future finishes. Physical leaves take FIFO turns in a stable order before node admission, and partial grants are returned before any resource wait. On contention, local and remote coordinators suspend the same traversal, release the bucket cache lease, and keep their physical/node FIFO position while topology remains current. The existing cycle budget and cancellation bound this wait. Topology validity is rechecked every 250 ms; expired or replaced plans yield a typed deferral. The coordinators reacquire the cache lease and verify both cache revisions before resuming with a permit or a typed deferral that preserves partial progress. A changed cache rejects the suspended result. Cancellation also drops the queued request.
|
||||
@@ -596,13 +594,11 @@ Native directory work and the local metadata read retain only that disk's physic
|
||||
|
||||
Resource-controlled peer requests carry `x-rustfs-rpc-background-admission: v1`, bound to their HTTP or gRPC signature. Stream retries preserve that classification without retaining the sender's disk quota. The receiver uses its existing shared node owner and admits the selected local physical device before storage locks or I/O. Repeated phases reuse the request's reservation, including when capacity is one. Missing capacity, an unknown UUID mapping, or an unavailable physical plan defers the request instead of probing other disks or reporting object absence. Cancellation returns node capacity; native work retains only its selected physical resources until it finishes.
|
||||
|
||||
An authenticated namespace-scan POST can queue its initial target admission for up to 15 seconds, before storage locks or body processing. The caller's existing response-header deadline and cancellation still apply. The receiver revalidates the active disk attachment after waiting; unknown UUID mappings and unavailable physical plans defer immediately. Other RPC entry points and additional device attachments within an admitted request remain nonblocking.
|
||||
|
||||
An HTTP file response releases node admission after opening its reader, but records progress only after producing the declared byte count or reaching an unbounded EOF. Body errors, early EOF and cancellation do not credit successful preparation. The deferred progress record retains counter identities without keeping node slots or physical units.
|
||||
|
||||
Enable this experimental setting only after all peers support the admission marker. Older peers reject marked signatures, and a new peer with admission disabled rejects marked work; neither path silently runs it outside resource control. Requests without the marker retain their existing signature and foreground behavior. The receiver reports deferred admission as HTTP 503 or gRPC `RESOURCE_EXHAUSTED`.
|
||||
|
||||
Storage operations without a coordinating cache lease retain immediate resource deferral and useful partial-cache handling. Existing remote single-session ownership and local-disk admission shares remain in force. Disabled admission creates no request channel. Enabled walks use one bounded channel, with a response allocation and cache revision reads only on contention; the uncontended per-entry path adds no global lookup or task. The streaming abandoned-child coordinator requires a separate ownership boundary: retaining a node unit while it waits for a bounded-channel consumer that needs the same unit would deadlock at capacity one.
|
||||
Calls without a coordinating cache lease retain immediate resource deferral and useful partial-cache handling. Existing remote single-session ownership and local-disk admission shares remain in force. Disabled admission creates no request channel. Enabled walks use one bounded channel, with a response allocation and cache revision reads only on contention; the uncontended per-entry path adds no global lookup or task. The streaming abandoned-child coordinator requires a separate ownership boundary: retaining a node unit while it waits for a bounded-channel consumer that needs the same unit would deadlock at capacity one.
|
||||
|
||||
Foreground comparison uses the existing complete response-body windows and shared admission snapshots. Each active class needs at least 100 completions and a finite p99 interval; this permits the percentile rank but is not a statistical confidence guarantee. Comparisons require the same active classes and rates within 80–125% of a retained reference. A changed workload resets recovery. The reference is an observed workload window, not an independent background-off baseline. A p99 interval can justify stability only when its upper bound is within 5% of the reference lower bound; a lower bound above 105% of the reference upper bound is pressure. Overlapping/uncertain comparisons cannot justify growth. Missing queues, stale or repeated cuts, uncertain cut boundaries and low counts are not idle evidence.
|
||||
|
||||
|
||||
@@ -85,8 +85,3 @@ RUST_MIN_STACK=4194304 cargo test -p rustfs-ecstore --lib segment_observation_eq
|
||||
```
|
||||
|
||||
[W19](https://github.com/rustfs/backlog/issues/2272) remains open for trustworthy producer coverage, source/incarnation binding, and production shadow observations. No production stream, durable journal, runtime feature switch, scan skipping, or performance claim is introduced here. No restart/gap detection or restart-safe production coverage is established, and the revision-aware file backend does not prove EC publication durability.
|
||||
|
||||
|
||||
## Whole-cycle read-only restart regression
|
||||
|
||||
`scanner_io::tests::checkpoint_coverage_identity::checkpoint_small_budget_accumulates_across_process_restarts` runs real scanner/store workers in fresh OS processes against four buckets. Every scan keeps the same two-object budget and must publish exact per-bucket totals within 40 rounds. Separate cases inject a write into an already completed bucket before global publication, then either cleanly stop the writer or leave its running marker behind. Both must expire checkpoint continuity and eventually include the extra object; accepting the old total as converged fails the test. This is a local storage regression, separate from whole-daemon multi-node and physical-device acceptance.
|
||||
|
||||
@@ -195,27 +195,14 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let request: ChangePasswordRequest = serde_json::from_slice(&body)
|
||||
.map_err(|e| s3::error(S3ErrorCode::InvalidRequest, format!("invalid change-password request: {e}")))?;
|
||||
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let Some(stored) = iam_store.get_user(&caller.access_key).await else {
|
||||
// Reached only if the identity was deleted between authentication
|
||||
// and this lookup.
|
||||
emit_audit(
|
||||
&audit,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&caller.access_key,
|
||||
caller.identity_type,
|
||||
AccountAuditFailure::Internal,
|
||||
),
|
||||
);
|
||||
return Err(s3::error(S3ErrorCode::InvalidRequest, "the calling identity no longer exists"));
|
||||
// An STS session signs with its own secret. A direct IAM request must
|
||||
// have been signed with the same password the caller is replacing.
|
||||
let expected_signing_secret = if caller.identity_type == IdentityType::Sts {
|
||||
&caller.credentials.secret_key
|
||||
} else {
|
||||
&request.current_secret_key
|
||||
};
|
||||
|
||||
if !constant_time_eq(&request.current_secret_key, &stored.credentials.secret_key)
|
||||
|| !constant_time_eq(&request.current_secret_key, &signing_secret)
|
||||
{
|
||||
if !constant_time_eq(expected_signing_secret, &signing_secret) {
|
||||
warn!(
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ACCOUNT,
|
||||
@@ -256,8 +243,10 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
}
|
||||
|
||||
let access_key = caller.access_key.clone();
|
||||
let new_secret_key = request.new_secret_key.clone();
|
||||
let current_secret_key = request.current_secret_key;
|
||||
let new_secret_key = request.new_secret_key;
|
||||
let identity_type = caller.identity_type;
|
||||
let session_credentials = (identity_type == IdentityType::Sts).then(|| caller.credentials.clone());
|
||||
let session_access_key = caller.session_access_key.clone();
|
||||
let audit_for_task = audit.clone();
|
||||
|
||||
@@ -268,16 +257,35 @@ impl Operation for ChangeOwnPasswordHandler {
|
||||
let iam_store =
|
||||
current_ready_iam_handle().map_err(|_| s3::error(S3ErrorCode::InternalError, "iam is not initialized"))?;
|
||||
|
||||
let (updated_at, status, revoked) = iam_store
|
||||
.change_own_secret_key(&access_key, &signing_secret, &new_secret_key)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
let rotation = match session_credentials.as_ref() {
|
||||
Some(session) => {
|
||||
iam_store
|
||||
.change_own_secret_key_from_session(&access_key, ¤t_secret_key, &new_secret_key, session)
|
||||
.await
|
||||
}
|
||||
None => {
|
||||
iam_store
|
||||
.change_own_secret_key(&access_key, ¤t_secret_key, &new_secret_key)
|
||||
.await
|
||||
}
|
||||
};
|
||||
let (updated_at, status, revoked) = rotation.map_err(|error| {
|
||||
if matches!(error, rustfs_iam::error::Error::IAMActionNotAllowed) {
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
AccountAuditRecord::failure(
|
||||
AccountAuditOperation::ChangeOwnPassword,
|
||||
&access_key,
|
||||
identity_type,
|
||||
AccountAuditFailure::InvalidCurrentSecret,
|
||||
)
|
||||
.with_session_access_key(session_access_key.as_deref()),
|
||||
);
|
||||
s3::error(S3ErrorCode::InvalidRequest, "the current secret key is incorrect")
|
||||
} else {
|
||||
iam_error_to_s3_error(error)
|
||||
}
|
||||
})?;
|
||||
|
||||
emit_audit(
|
||||
&audit_for_task,
|
||||
|
||||
@@ -221,13 +221,6 @@ pub fn register_notification_target_route(r: &mut S3Router<AdminOperation>) -> s
|
||||
AdminOperation(&NotificationTarget {}),
|
||||
)?;
|
||||
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}").as_str(),
|
||||
AdminOperation(&RemoveNotificationTarget {}),
|
||||
)?;
|
||||
|
||||
// Legacy spelling kept for consoles that still call the reset suffix.
|
||||
r.insert(
|
||||
Method::DELETE,
|
||||
format!("{}{}", ADMIN_PREFIX, "/v3/target/{target_type}/{target_name}/reset").as_str(),
|
||||
|
||||
@@ -259,7 +259,7 @@ mod tests {
|
||||
}
|
||||
|
||||
// Note: Testing the actual async handler implementations requires:
|
||||
// 1. Owned admin request setup with proper headers, URI, and credentials
|
||||
// 1. S3Request setup with proper headers, URI, and credentials
|
||||
// 2. Global object store initialization
|
||||
// 3. IAM system initialization
|
||||
// 4. Mock or real backend services
|
||||
|
||||
@@ -13921,18 +13921,18 @@ mod tests {
|
||||
env!("CARGO_MANIFEST_DIR"),
|
||||
"/../crates/ecstore/tests/fixtures/bucket-config-goldens/lifecycle/minio.in.xml"
|
||||
));
|
||||
// Answer of s3s 5761ddfe (`Timestamp::parse(DateTime)` on this document's
|
||||
// ExpiryUpdatedAt). The owned axis must keep that instant.
|
||||
let legacy_axis =
|
||||
parse_lifecycle_date("2026-07-07T15:58:57.337315Z").expect("the legacy expiry axis is the minio.in.xml value");
|
||||
let mut old_reader = s3s::xml::Deserializer::new(raw);
|
||||
let old: s3s::dto::BucketLifecycleConfiguration =
|
||||
s3s::xml::Deserialize::deserialize(&mut old_reader).expect("read the actual old lifecycle expiry axis");
|
||||
old_reader.expect_eof().expect("the old expiry reader consumes the document");
|
||||
let old_axis = OffsetDateTime::from(old.expiry_updated_at.expect("the old config contains its expiry axis"));
|
||||
let new = deserialize(raw).expect("read the persisted lifecycle expiry axis");
|
||||
let new_axis = local_lifecycle_staleness_axis(Some(&new), OffsetDateTime::UNIX_EPOCH)
|
||||
.expect("the parsed lifecycle axis is readable");
|
||||
let incoming = parse_lifecycle_date("2026-07-07T15:58:57.337100Z").expect("the incoming expiry axis is valid");
|
||||
assert_eq!(new_axis, legacy_axis, "the parsed axis must keep the legacy submillisecond instant");
|
||||
assert_eq!(
|
||||
is_stale_update(new_axis, Some(incoming)),
|
||||
is_stale_update(legacy_axis, Some(incoming)),
|
||||
is_stale_update(old_axis, Some(incoming)),
|
||||
"a peer update older than the full-precision local axis must remain stale"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -77,6 +77,37 @@ fn table_catalog_handler_request(context: Arc<AppContext>, access_key: &str, sec
|
||||
load_table_request_builder(context, access_key, secret_key).build()
|
||||
}
|
||||
|
||||
/// Converts a legacy request the way the admin router edge does, without its body.
|
||||
fn legacy_request_view(req: &s3s::S3Request<s3s::Body>) -> AdminRequest {
|
||||
AdminRequest::new(RequestEnvelope::from_s3s(req), Body::empty())
|
||||
}
|
||||
|
||||
/// Builds a legacy request to exercise conversion into the owned request.
|
||||
fn legacy_table_catalog_handler_request(
|
||||
context: Arc<AppContext>,
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> s3s::S3Request<s3s::Body> {
|
||||
let slot = ServerContextSlot::new();
|
||||
assert!(slot.install(context));
|
||||
let mut extensions = http::Extensions::new();
|
||||
extensions.insert(slot);
|
||||
s3s::S3Request {
|
||||
input: s3s::Body::empty(),
|
||||
method: Method::GET,
|
||||
uri: LOAD_TABLE_URI.parse().expect("load table URI"),
|
||||
headers: HeaderMap::new(),
|
||||
extensions,
|
||||
credentials: Some(s3s::auth::Credentials {
|
||||
access_key: access_key.to_string(),
|
||||
secret_key: s3s::auth::SecretKey::from(secret_key.to_string()),
|
||||
}),
|
||||
region: None,
|
||||
service: None,
|
||||
trailing_headers: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_access_delegation(request: TestRequest, values: &[&str]) -> AdminRequest {
|
||||
values
|
||||
.iter()
|
||||
@@ -150,13 +181,15 @@ async fn call_catalog_backup_prepare_handler(request: AdminRequest) -> S3Result<
|
||||
PrepareTableCatalogBackupObjectHandler {}.call(request, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_disable_table_bucket_handler(request: AdminRequest) -> S3Result<AdminResponse> {
|
||||
/// Calls the disable handler with a request converted the same way the admin router edge converts it.
|
||||
async fn call_disable_table_bucket_handler(request: s3s::S3Request<s3s::Body>) -> S3Result<AdminResponse> {
|
||||
let mut router = matchit::Router::new();
|
||||
router.insert("/iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
router.insert("/_iceberg/v1/buckets/{warehouse}", ()).unwrap();
|
||||
let path = request.uri().path().to_string();
|
||||
let path = request.uri.path().to_string();
|
||||
let matched = router.at(&path).unwrap();
|
||||
DisableTableBucketHandler {}.call(request, matched.params).await
|
||||
let owned = AdminRequest::new(RequestEnvelope::from_s3s(&request), request.input.into());
|
||||
DisableTableBucketHandler {}.call(owned, matched.params).await
|
||||
}
|
||||
|
||||
async fn call_maintenance_worker_handler(request: AdminRequest) -> S3Result<AdminResponse> {
|
||||
@@ -1334,11 +1367,14 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
..Default::default()
|
||||
}));
|
||||
// Exercise the actual owned internal authorization path, including its temporary target.
|
||||
let mut owned = table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
let legacy = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
let mut owned = legacy_request_view(&legacy);
|
||||
let principal = table_catalog_request_principal(&owned).await.expect("resolve principal");
|
||||
// The owned request keeps the region it was given. An already installed region wins.
|
||||
// Copying a legacy s3s region into the envelope is covered by from_s3s_copies_every_named_extension.
|
||||
// Both request boundaries retain the effective region and the authenticated principal.
|
||||
{
|
||||
let mut legacy_region = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
legacy_region.region = Some("eu-west-1".parse().expect("valid legacy region"));
|
||||
let legacy_request = legacy_request_view(&legacy_region);
|
||||
let owned_request = load_table_request_builder(context.clone(), access_key, secret_key)
|
||||
.region(Some("eu-west-1".to_owned()))
|
||||
.build();
|
||||
@@ -1351,6 +1387,7 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
})
|
||||
.build();
|
||||
for (mut request, expected_region, expected_bucket) in [
|
||||
(legacy_request, "eu-west-1", None),
|
||||
(owned_request, "eu-west-1", None),
|
||||
(preserved_request, "ap-south-1", Some("original-bucket")),
|
||||
] {
|
||||
@@ -1393,9 +1430,11 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
// The exact-key check selects internal-object mode: a request header the object
|
||||
// write does not apply (a malformed x-amz-tagging) must not reach the decision,
|
||||
// while the ordinary owned entry point still evaluates it and rejects it.
|
||||
let mut tagged = load_table_request_builder(context.clone(), access_key, secret_key)
|
||||
.header("x-amz-tagging", "classification=%ZZ")
|
||||
.build();
|
||||
let mut tagged = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
tagged
|
||||
.headers
|
||||
.insert("x-amz-tagging", http::HeaderValue::from_static("classification=%ZZ"));
|
||||
let mut tagged = legacy_request_view(&tagged);
|
||||
install_table_catalog_s3_request_info(&mut tagged, &principal).expect("install authorization context");
|
||||
authorize_table_catalog_s3_actions(&mut tagged, "warehouse", "owned/writable", &[S3Action::PutObjectAction])
|
||||
.await
|
||||
@@ -1416,7 +1455,8 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
.await;
|
||||
objects.seed_object("warehouse", "shared/denied", b"unchanged".to_vec()).await;
|
||||
let backend = || {
|
||||
let mut request = table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
let legacy = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
let mut request = legacy_request_view(&legacy);
|
||||
install_table_catalog_s3_request_info(&mut request, &principal).expect("install authorization context");
|
||||
TableCommitObjectBackend::for_request(objects.clone(), request)
|
||||
};
|
||||
@@ -1548,7 +1588,7 @@ async fn warehouse_location_claim_checks_request_identity_and_relocation() {
|
||||
] {
|
||||
let backend = TableCommitObjectBackend::for_request(
|
||||
TestTableCatalogObjectBackend::default(),
|
||||
table_catalog_handler_request(context.clone(), access_key, secret_key),
|
||||
legacy_request_view(&legacy_table_catalog_handler_request(context.clone(), access_key, secret_key)),
|
||||
);
|
||||
let mut target = metadata.clone();
|
||||
target["location"] = serde_json::json!(location);
|
||||
@@ -18310,13 +18350,10 @@ async fn cleanup_empty_table_bucket_handler_enforces_request_object_permissions(
|
||||
let metadata_objects =
|
||||
create_dropped_empty_tables_for_cleanup(&catalog, &backend, object_store.as_ref(), &bucket, 2).await;
|
||||
let request = |prefix: &str| {
|
||||
table_catalog_request_builder(
|
||||
context.clone(),
|
||||
Some((access_key, secret_key)),
|
||||
Method::DELETE,
|
||||
&format!("{prefix}/v1/buckets/{bucket}?cleanup=true"),
|
||||
)
|
||||
.build()
|
||||
let mut request = legacy_table_catalog_handler_request(context.clone(), access_key, secret_key);
|
||||
request.method = Method::DELETE;
|
||||
request.uri = format!("{prefix}/v1/buckets/{bucket}?cleanup=true").parse().unwrap();
|
||||
request
|
||||
};
|
||||
let object_resource = format!("arn:aws:s3:::{bucket}/*");
|
||||
let first_metadata_resource = format!("arn:aws:s3:::{bucket}/{}", metadata_objects[0]);
|
||||
|
||||
@@ -305,12 +305,6 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
SET_BUCKET_TARGET,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/rustfs/admin/v3/target/{target_type}/{target_name}",
|
||||
SET_BUCKET_TARGET,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Delete,
|
||||
"/rustfs/admin/v3/target/{target_type}/{target_name}/reset",
|
||||
|
||||
@@ -212,7 +212,6 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
admin_route(Method::GET, "/v3/idp/builtin/policy-entities"),
|
||||
admin_route(Method::GET, "/v3/target/list"),
|
||||
admin_route_sample(Method::PUT, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
|
||||
admin_route_sample(Method::DELETE, "/v3/target/{target_type}/{target_name}", "/v3/target/webhook/test-target"),
|
||||
admin_route_sample(
|
||||
Method::DELETE,
|
||||
"/v3/target/{target_type}/{target_name}/reset",
|
||||
|
||||
@@ -87,6 +87,8 @@ pub(crate) enum CredentialMutationDenial {
|
||||
ServiceAccount,
|
||||
/// A derived credential whose parent could not be determined.
|
||||
UnresolvedParent,
|
||||
/// A storage delegation has no account-management authority.
|
||||
TableStorageSession,
|
||||
}
|
||||
|
||||
impl CredentialMutationDenial {
|
||||
@@ -97,6 +99,7 @@ impl CredentialMutationDenial {
|
||||
}
|
||||
Self::FederatedIdentity => "federated identities are managed by their identity provider",
|
||||
Self::ServiceAccount => "service account credentials cannot change the credentials of their parent identity",
|
||||
Self::TableStorageSession => "table-scoped storage credentials cannot manage account credentials",
|
||||
Self::UnresolvedParent => "the parent identity of this session could not be resolved",
|
||||
}
|
||||
}
|
||||
@@ -187,7 +190,7 @@ impl CallerIdentity {
|
||||
// root", and an unresolved parent must never fall through to a
|
||||
// permissive answer.
|
||||
//
|
||||
// These three denials apply to both capabilities: a machine credential
|
||||
// These denials apply to both capabilities: a machine credential
|
||||
// must not take over its parent, a federated identity is owned by its
|
||||
// IdP, and an unresolvable parent fails closed.
|
||||
let shared_denial = if unresolved_parent {
|
||||
@@ -196,6 +199,11 @@ impl CallerIdentity {
|
||||
Some(CredentialMutationDenial::ServiceAccount)
|
||||
} else if federated {
|
||||
Some(CredentialMutationDenial::FederatedIdentity)
|
||||
} else if ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"]
|
||||
.iter()
|
||||
.any(|claim| credentials.claims_or_empty().contains_key(*claim))
|
||||
{
|
||||
Some(CredentialMutationDenial::TableStorageSession)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
@@ -501,6 +509,20 @@ mod tests {
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::FederatedIdentity));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn table_storage_sessions_cannot_manage_parent_credentials() {
|
||||
for claim in ["rustfs:table-bucket", "rustfs:table-id", "rustfs:credential-scope-prefix"] {
|
||||
let mut credentials = sts_session("TABLEKEY", "table-owner");
|
||||
credentials.claims = Some(HashMap::from([(claim.to_string(), Value::String("scope".to_string()))]));
|
||||
let caller = CallerIdentity::from_credentials(credentials, false);
|
||||
|
||||
assert_eq!(caller.mutation_denial, Some(CredentialMutationDenial::TableStorageSession));
|
||||
assert!(!caller.mutability().password);
|
||||
assert!(caller.ensure_credential_mutation_allowed().is_err());
|
||||
assert!(caller.ensure_mfa_management_allowed().is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn derived_credential_without_a_parent_is_denied_rather_than_allowed() {
|
||||
// Fail closed: an unresolvable parent must not be treated as a
|
||||
|
||||
@@ -1083,10 +1083,20 @@ pub(crate) mod runtime {
|
||||
}
|
||||
|
||||
pub(crate) mod s3 {
|
||||
pub(crate) use http::header;
|
||||
// Keep auth types behind the existing s3 facade so test-only callers do
|
||||
// not widen the repository's direct s3s import footprint.
|
||||
pub(crate) use rustfs_s3_types::{S3Error, S3ErrorCode, S3Result};
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) use s3s::auth;
|
||||
pub(crate) use s3s::header;
|
||||
|
||||
/// Build an owned `S3Error` from a code and a message.
|
||||
/// Build an `S3Error` without reaching for the `s3s` error macro.
|
||||
///
|
||||
/// The macro expands to the very constructor this calls, but it puts an
|
||||
/// `s3s` dependency in every file that reports an error. Routing the
|
||||
/// construction through here keeps that dependency in this facade, which is
|
||||
/// the boundary the s3gate migration replaces
|
||||
/// (`scripts/check_s3s_footprint.sh`, rustfs/backlog#1677 F1).
|
||||
pub(crate) fn error(code: S3ErrorCode, message: impl Into<std::borrow::Cow<'static, str>>) -> S3Error {
|
||||
S3Error::with_message(code, message)
|
||||
}
|
||||
|
||||
@@ -49,7 +49,8 @@ pub(crate) async fn read_compatible_admin_body(
|
||||
path: &str,
|
||||
secret_key: &str,
|
||||
) -> S3Result<Vec<u8>> {
|
||||
// Callers pass an owned body. One length limit applies to every read.
|
||||
// Legacy handlers still hand over the s3s body; owned handlers pass the
|
||||
// RustFS body. Both read through the owned type so one limit applies.
|
||||
let mut input: rustfs_s3_types::Body = input.into();
|
||||
let body = input.store_all_limited(max_len).await.map_err(|e| {
|
||||
if e.is::<rustfs_s3_types::BodyTooLarge>() {
|
||||
@@ -241,6 +242,13 @@ mod tests {
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
assert_eq!(err.message(), Some("failed to read request body: length limit exceeded"));
|
||||
|
||||
let legacy_body = s3s::Body::http_body(http_body_util::Full::new(bytes::Bytes::from_static(b"nine bytes!")));
|
||||
let legacy_err = read_compatible_admin_body(legacy_body, 8, "/rustfs/admin/v3/list-users", "secret")
|
||||
.await
|
||||
.expect_err("legacy streaming bodies retain the same limit error");
|
||||
assert_eq!(legacy_err.code(), err.code());
|
||||
assert_eq!(legacy_err.message(), err.message());
|
||||
|
||||
let exact =
|
||||
read_compatible_admin_body(rustfs_s3_types::Body::from("12345678"), 8, "/rustfs/admin/v3/list-users", "secret")
|
||||
.await
|
||||
|
||||
@@ -906,7 +906,9 @@ mod tests {
|
||||
.expect_err("full queue rejects immediately");
|
||||
assert_eq!(error.code(), &S3ErrorCode::SlowDown);
|
||||
assert_eq!(error.headers().expect("retry headers")[http::header::RETRY_AFTER], "1");
|
||||
let response = error.to_http_response().expect("serialize admission error");
|
||||
let response = s3s::S3Error::from(error)
|
||||
.to_http_response()
|
||||
.expect("serialize admission error");
|
||||
assert_eq!(response.status(), http::StatusCode::SERVICE_UNAVAILABLE);
|
||||
assert_eq!(response.headers()[http::header::RETRY_AFTER], "1");
|
||||
use http_body_util::BodyExt as _;
|
||||
|
||||
@@ -208,7 +208,7 @@ impl DefaultMultipartUsecase {
|
||||
let (body, total) =
|
||||
crate::app::object::cuobject::rdma_put_body(bucket, key, &token, true).map_err(S3Error::from)?;
|
||||
let total = i64::try_from(total).map_err(|_| S3Error::new(S3ErrorCode::EntityTooLarge))?;
|
||||
(Some(body), Some(total), Some(total))
|
||||
(Some(crate::app::object::legacy_put_body(body)), Some(total), Some(total))
|
||||
}
|
||||
_ => (body, content_length, None),
|
||||
};
|
||||
|
||||
@@ -27,6 +27,7 @@ use crate::app::storage_api::object_usecase::runtime::get_global_endpoints_opt;
|
||||
#[cfg(not(feature = "nixl"))]
|
||||
use crate::app::storage_api::object_usecase::runtime::global_rustfs_port;
|
||||
use bytes::Bytes;
|
||||
use futures::Stream;
|
||||
use http::HeaderMap;
|
||||
use rustfs_config::{
|
||||
DEFAULT_ERASURE_SHARD_BYTES, DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT, DEFAULT_RUSTFS_CUOBJECT_GET_FANOUT_BUDGET,
|
||||
@@ -55,8 +56,9 @@ pub(crate) mod transport {
|
||||
};
|
||||
}
|
||||
use crate::app::storage_api::object_usecase::rdma::{RegisteredSlabArena, ShardBuf, ShardEmitter, install_shard_arena};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Result};
|
||||
use rustfs_gateway_stream::Body;
|
||||
use crate::app::storage_api::s3::dto::StreamingBlob;
|
||||
use crate::app::storage_api::s3::stream::{ByteStream, RemainingLength};
|
||||
use crate::app::storage_api::s3::{S3Error, S3ErrorCode, S3Result, StdError};
|
||||
use rustfs_utils::{get_env_bool, get_env_opt_str, get_env_u64, get_env_usize};
|
||||
use std::collections::VecDeque;
|
||||
use std::io;
|
||||
@@ -778,6 +780,36 @@ pub(crate) async fn serve_get_over_rdma(
|
||||
Ok(moved)
|
||||
}
|
||||
|
||||
/// Request body of a cuObject PUT: the object bytes are RDMA-read from the
|
||||
/// client's buffer into staging, one chunk ahead of the consumer, and handed
|
||||
/// to the normal PUT pipeline as an ordinary byte stream.
|
||||
struct RdmaPutBody {
|
||||
inner: Pin<Box<dyn Stream<Item = Result<Bytes, StdError>> + Send + Sync + 'static>>,
|
||||
remaining: usize,
|
||||
}
|
||||
|
||||
impl Stream for RdmaPutBody {
|
||||
type Item = Result<Bytes, StdError>;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let next = self.inner.as_mut().poll_next(cx);
|
||||
if let Poll::Ready(Some(Ok(bytes))) = &next {
|
||||
self.remaining = self.remaining.saturating_sub(bytes.len());
|
||||
}
|
||||
next
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> (usize, Option<usize>) {
|
||||
(self.remaining, Some(self.remaining))
|
||||
}
|
||||
}
|
||||
|
||||
impl ByteStream for RdmaPutBody {
|
||||
fn remaining_length(&self) -> RemainingLength {
|
||||
RemainingLength::new_exact(self.remaining)
|
||||
}
|
||||
}
|
||||
|
||||
type StagedRead = JoinHandle<Result<StagingChunk, CuObjectError>>;
|
||||
|
||||
/// The staged bytes of `chunk`: with `zero_copy`, a `Bytes` that owns the
|
||||
@@ -814,14 +846,13 @@ fn spawn_staged_read(server: &Arc<CuObjectServer>, object: &str, token: &RdmaTok
|
||||
/// consumer that keeps a chunk keeps it out of the pool: only callers whose
|
||||
/// body is consumed and dropped with the request (UploadPart) may set it.
|
||||
/// PutObject may hand its body to the object data cache, so it copies.
|
||||
pub(crate) fn rdma_put_body(bucket: &str, key: &str, token: &RdmaToken, zero_copy: bool) -> S3Result<(Body, u64)> {
|
||||
pub(crate) fn rdma_put_body(bucket: &str, key: &str, token: &RdmaToken, zero_copy: bool) -> S3Result<(StreamingBlob, u64)> {
|
||||
let service = SERVICE
|
||||
.get()
|
||||
.ok_or_else(|| S3Error::with_message(S3ErrorCode::NotImplemented, "rdma put is not enabled"))?;
|
||||
let server = Arc::clone(service.pick());
|
||||
let read_ahead = service.put_read_ahead;
|
||||
let total = token.buffer_len;
|
||||
let length = usize::try_from(total).map_err(|_| S3Error::with_message(S3ErrorCode::EntityTooLarge, "rdma put too large"))?;
|
||||
let chunk_bytes = server.chunk_bytes();
|
||||
let object = format!("{bucket}/{key}");
|
||||
let token = token.clone();
|
||||
@@ -862,7 +893,13 @@ pub(crate) fn rdma_put_body(bucket: &str, key: &str, token: &RdmaToken, zero_cop
|
||||
Ok::<_, io::Error>(Some((bytes, (pending, next_offset))))
|
||||
}
|
||||
});
|
||||
Ok((super::native_body::from_io_stream_with_length(stream, Some(length)), total))
|
||||
let stream = futures::TryStreamExt::map_err(stream, |error: io::Error| -> StdError { Box::new(error) });
|
||||
let body = RdmaPutBody {
|
||||
inner: Box::pin(stream),
|
||||
remaining: usize::try_from(total)
|
||||
.map_err(|_| S3Error::with_message(S3ErrorCode::EntityTooLarge, "rdma put too large"))?,
|
||||
};
|
||||
Ok((StreamingBlob::new(body), total))
|
||||
}
|
||||
|
||||
/// State shared by the segment writers of one RDMA GET: the server, the
|
||||
|
||||
@@ -929,7 +929,6 @@ pin_project! {
|
||||
struct GetObjectReaderStream<R> {
|
||||
#[pin]
|
||||
reader: Option<R>,
|
||||
primed_byte: Option<u8>,
|
||||
capacity: usize,
|
||||
strategy: &'static str,
|
||||
buffer_source: &'static str,
|
||||
@@ -994,7 +993,6 @@ where
|
||||
}
|
||||
Self {
|
||||
reader: Some(reader),
|
||||
primed_byte: None,
|
||||
capacity,
|
||||
strategy,
|
||||
buffer_source,
|
||||
@@ -1439,14 +1437,9 @@ where
|
||||
Some(reader) => reader,
|
||||
None => return Poll::Ready(None),
|
||||
};
|
||||
let (poll_read, buf) = if let Some(byte) = this.primed_byte.take() {
|
||||
(Poll::Ready(Ok(1)), BytesMut::from(&[byte][..]))
|
||||
} else {
|
||||
let read_capacity = (*this.capacity).min(*this.remaining);
|
||||
let mut buf = BytesMut::with_capacity(read_capacity);
|
||||
let poll_read = poll_read_buf(reader, cx, &mut buf);
|
||||
(poll_read, buf)
|
||||
};
|
||||
let read_capacity = (*this.capacity).min(*this.remaining);
|
||||
let mut buf = BytesMut::with_capacity(read_capacity);
|
||||
let poll_read = poll_read_buf(reader, cx, &mut buf);
|
||||
|
||||
let result: Poll<Option<Self::Item>> = match poll_read {
|
||||
Poll::Ready(Ok(bytes_read)) if bytes_read > 0 => {
|
||||
@@ -2521,21 +2514,19 @@ impl DefaultObjectUsecase {
|
||||
lifecycle,
|
||||
resume,
|
||||
);
|
||||
let primed_byte = if expected == 0 {
|
||||
None
|
||||
let mut prefix = [0_u8; 1];
|
||||
let prefix_len = if expected == 0 {
|
||||
0
|
||||
} else {
|
||||
let mut prefix = [0_u8; 1];
|
||||
reader
|
||||
.read_exact(&mut prefix)
|
||||
.await
|
||||
.map_err(|error| map_get_object_reader_error(StorageError::from(error)))?;
|
||||
Some(prefix[0])
|
||||
1
|
||||
};
|
||||
let mut stream =
|
||||
GetObjectReaderStream::new(reader, stream_buffer_size, expected, stream_strategy.as_str(), buffer_source)
|
||||
.with_diagnostics(bucket, key, request_id);
|
||||
// Keep the preflight byte in the stream until the transport asks for it.
|
||||
stream.primed_byte = primed_byte;
|
||||
let reader = std::io::Cursor::new(prefix).take(prefix_len).chain(reader);
|
||||
let stream = GetObjectReaderStream::new(reader, stream_buffer_size, expected, stream_strategy.as_str(), buffer_source)
|
||||
.with_diagnostics(bucket, key, request_id);
|
||||
let blob = GetObjectBody::Stream(Box::pin(stream));
|
||||
if let Some(handoff_start) = handoff_start {
|
||||
rustfs_io_metrics::record_get_object_response_handoff(
|
||||
@@ -9019,7 +9010,7 @@ mod tests {
|
||||
let error = result.expect_err("a read quorum failure before the first byte must reject response construction");
|
||||
assert_eq!(error.code(), &S3ErrorCode::Custom("SlowDownRead".into()));
|
||||
assert_eq!(error.status_code(), Some(StatusCode::SERVICE_UNAVAILABLE));
|
||||
assert_eq!(error.message(), Some("The object could not be read from storage."));
|
||||
assert_eq!(error.message(), Some("Resource requested is unreadable, please reduce your request rate"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -9046,20 +9037,7 @@ mod tests {
|
||||
.expect("the first byte should be available before response handoff");
|
||||
|
||||
assert_eq!(reads.load(AtomicOrdering::Relaxed), 1, "response construction must prime one byte");
|
||||
let first = body
|
||||
.next()
|
||||
.await
|
||||
.expect("the primed byte should be present")
|
||||
.expect("the first chunk should be readable");
|
||||
assert_eq!(&first[..], b"h");
|
||||
let first = first.try_into_mut().expect("the first chunk should own its buffer");
|
||||
assert!(first.capacity() < 5, "the preflight byte must not reserve the remaining body capacity");
|
||||
assert_eq!(
|
||||
reads.load(AtomicOrdering::Relaxed),
|
||||
1,
|
||||
"emitting the primed byte must not poll storage again"
|
||||
);
|
||||
let mut received = first.to_vec();
|
||||
let mut received = Vec::new();
|
||||
while let Some(chunk) = body.next().await {
|
||||
received.extend_from_slice(&chunk.expect("the primed body should remain readable"));
|
||||
}
|
||||
|
||||
@@ -52,31 +52,38 @@ fn website_and_object_lambda_ranges_match_the_legacy_wire_reader() {
|
||||
|
||||
#[test]
|
||||
fn website_and_object_lambda_conditions_match_the_legacy_wire_reader() {
|
||||
// Answers measured on s3s 5761ddfe (`ETagCondition::parse_http_header` and
|
||||
// `Timestamp::parse(HttpDate)`). The owned checks must keep those answers.
|
||||
for (value, accepted) in [
|
||||
(b"*".as_slice(), true),
|
||||
(b"abc-2", true),
|
||||
(b"\"\"", true),
|
||||
(b"W/\"x\"", true),
|
||||
(b"\"a \tb\"", true),
|
||||
(b"\"a\"b\"", true),
|
||||
(b"", false),
|
||||
(b"bad etag", false),
|
||||
(b"w/\"x\"", false),
|
||||
(b"\"\xff\"", false),
|
||||
(b"\"\x7f\"", false),
|
||||
(b"\"x", false),
|
||||
use s3s::dto::{ETagCondition, Timestamp, TimestampFormat};
|
||||
for value in [
|
||||
b"*".as_slice(),
|
||||
b"abc-2",
|
||||
b"\"\"",
|
||||
b"W/\"x\"",
|
||||
b"\"a \tb\"",
|
||||
b"\"a\"b\"",
|
||||
b"",
|
||||
b"bad etag",
|
||||
b"w/\"x\"",
|
||||
b"\"\xff\"",
|
||||
b"\"\x7f\"",
|
||||
b"\"x",
|
||||
] {
|
||||
assert_eq!(GetObjectFields::valid_etag_condition(value), accepted, "{value:?}");
|
||||
assert_eq!(
|
||||
GetObjectFields::valid_etag_condition(value),
|
||||
ETagCondition::parse_http_header(value).is_ok(),
|
||||
"{value:?}"
|
||||
);
|
||||
}
|
||||
for (value, accepted) in [
|
||||
("Sun, 06 Nov 1994 08:49:37 GMT", true),
|
||||
("Sunday, 06-Nov-94 08:49:37 GMT", false),
|
||||
("2025-01-01", false),
|
||||
("Sun, 32 Nov 1994 08:49:37 GMT", false),
|
||||
for value in [
|
||||
"Sun, 06 Nov 1994 08:49:37 GMT",
|
||||
"Sunday, 06-Nov-94 08:49:37 GMT",
|
||||
"2025-01-01",
|
||||
"Sun, 32 Nov 1994 08:49:37 GMT",
|
||||
] {
|
||||
assert_eq!(GetObjectFields::valid_date_condition(value), accepted, "{value}");
|
||||
assert_eq!(
|
||||
GetObjectFields::valid_date_condition(value),
|
||||
Timestamp::parse(TimestampFormat::HttpDate, value).is_ok(),
|
||||
"{value}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -182,26 +182,17 @@ impl Stream for BodyChunks {
|
||||
}
|
||||
|
||||
pub(super) fn from_io_stream<S>(stream: S) -> Body
|
||||
where
|
||||
S: Stream<Item = std::io::Result<Bytes>> + Send + 'static,
|
||||
{
|
||||
from_io_stream_with_length(stream, None)
|
||||
}
|
||||
|
||||
pub(super) fn from_io_stream_with_length<S>(stream: S, length: Option<usize>) -> Body
|
||||
where
|
||||
S: Stream<Item = std::io::Result<Bytes>> + Send + 'static,
|
||||
{
|
||||
Body::from_payload(Payload::Stream(Box::pin(IoPayload {
|
||||
stream: Box::pin(stream),
|
||||
remaining: length,
|
||||
ended: false,
|
||||
})))
|
||||
}
|
||||
|
||||
struct IoPayload<S> {
|
||||
stream: Pin<Box<S>>,
|
||||
remaining: Option<usize>,
|
||||
ended: bool,
|
||||
}
|
||||
|
||||
@@ -214,12 +205,7 @@ impl<S: Stream<Item = std::io::Result<Bytes>>> PayloadStream for IoPayload<S> {
|
||||
match self.stream.as_mut().poll_next(cx) {
|
||||
Poll::Pending => return Poll::Pending,
|
||||
Poll::Ready(Some(Ok(bytes))) if bytes.is_empty() => continue,
|
||||
Poll::Ready(Some(Ok(bytes))) => {
|
||||
if let Some(remaining) = &mut self.remaining {
|
||||
*remaining = remaining.saturating_sub(bytes.len());
|
||||
}
|
||||
return Poll::Ready(Ok(PayloadRead::Chunk(bytes)));
|
||||
}
|
||||
Poll::Ready(Some(Ok(bytes))) => return Poll::Ready(Ok(PayloadRead::Chunk(bytes))),
|
||||
Poll::Ready(Some(Err(error))) => {
|
||||
self.ended = true;
|
||||
return Poll::Ready(Err(StreamError::upstream(Box::new(ProducerSource(Box::new(error))))));
|
||||
@@ -236,12 +222,10 @@ impl<S: Stream<Item = std::io::Result<Bytes>>> PayloadStream for IoPayload<S> {
|
||||
Poll::Pending
|
||||
}
|
||||
fn caps(&self) -> PayloadCaps {
|
||||
let mut caps = PayloadCaps::PUSH;
|
||||
caps.set(PayloadCaps::KNOWN_LENGTH, self.remaining.is_some());
|
||||
caps
|
||||
PayloadCaps::PUSH
|
||||
}
|
||||
fn len_hint(&self) -> Option<u64> {
|
||||
self.remaining.map(|length| length as u64)
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
@@ -419,35 +403,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn io_stream_with_length_tracks_data_and_preserves_producer_error() {
|
||||
let mut body = from_io_stream_with_length(
|
||||
futures::stream::iter([
|
||||
Ok(Bytes::new()),
|
||||
Ok(Bytes::from_static(b"abc")),
|
||||
Err(std::io::Error::new(std::io::ErrorKind::TimedOut, InternalProducerFailure)),
|
||||
]),
|
||||
Some(5),
|
||||
);
|
||||
assert_eq!(body.size_hint().exact(), Some(5));
|
||||
let frame = std::future::poll_fn(|cx| Pin::new(&mut body).poll_frame(cx))
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert_eq!(frame.into_data().unwrap(), Bytes::from_static(b"abc"));
|
||||
assert_eq!(body.size_hint().exact(), Some(2));
|
||||
|
||||
let (mut chunks, trailers) = BodyChunks::new(body);
|
||||
let error = chunks.next().await.unwrap().unwrap_err();
|
||||
let error = error
|
||||
.downcast_ref::<std::io::Error>()
|
||||
.expect("the original io::Error reaches RIO");
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::TimedOut);
|
||||
assert!(error.get_ref().unwrap().is::<InternalProducerFailure>());
|
||||
assert_eq!(trailers.lookup("x-test"), TrailerValue::Pending);
|
||||
assert!(chunks.next().await.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reader_boundary_is_send_and_sync() {
|
||||
fn assert_send_sync<T: Send + Sync>() {}
|
||||
|
||||
@@ -1421,7 +1421,7 @@ impl DefaultObjectUsecase {
|
||||
!i64::try_from(token.buffer_len).is_ok_and(|size| put_body_may_fill_cache(&self.object_data_cache(), size));
|
||||
let (body, total) = super::cuobject::rdma_put_body(&bucket, &key, &token, zero_copy)?;
|
||||
let total = i64::try_from(total).map_err(|_| S3Error::new(S3ErrorCode::EntityTooLarge))?;
|
||||
(Some(body), Some(total), Some(total))
|
||||
(Some(super::native_body::from_legacy(body)), Some(total), Some(total))
|
||||
}
|
||||
_ => (body, content_length, None),
|
||||
};
|
||||
|
||||
@@ -578,8 +578,9 @@ impl DefaultObjectUsecase {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::app::RequestHead;
|
||||
use http::{Extensions, HeaderMap, Method, Uri};
|
||||
use http::Method;
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::{RestoreObjectInput, RestoreRequest};
|
||||
|
||||
fn ongoing_metadata(operation_id: Uuid) -> HashMap<String, String> {
|
||||
let mut metadata = HashMap::new();
|
||||
@@ -711,31 +712,28 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
/// The same head `build_request` plus `RequestEnvelope::from_s3s` produces:
|
||||
/// POST `/`, empty headers, and no attached payloads.
|
||||
fn restore_envelope() -> RequestEnvelope {
|
||||
RequestEnvelope::from_parts(
|
||||
RequestHead {
|
||||
method: Method::POST,
|
||||
uri: Uri::from_static("/"),
|
||||
headers: HeaderMap::new(),
|
||||
credentials: None,
|
||||
region: None,
|
||||
service: None,
|
||||
},
|
||||
Extensions::new(),
|
||||
)
|
||||
fn restore_input(bucket: &str, key: &str, rreq: RestoreRequest) -> RestoreObjectInput {
|
||||
RestoreObjectInput::builder()
|
||||
.bucket(bucket.to_string())
|
||||
.key(key.to_string())
|
||||
.restore_request(Some(rreq))
|
||||
.build()
|
||||
.expect("restore input should build")
|
||||
}
|
||||
|
||||
/// backlog#2205: a missing restore body is a client error, not a 500.
|
||||
#[tokio::test]
|
||||
async fn execute_restore_object_rejects_missing_restore_request() {
|
||||
let input = RestoreObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::POST);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_restore_object_owned(restore_envelope(), "test-bucket".to_owned(), "test-key".to_owned(), None, None)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let err = usecase.execute_restore_object(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::MalformedXML);
|
||||
}
|
||||
|
||||
@@ -747,14 +745,12 @@ mod tests {
|
||||
#[tokio::test]
|
||||
async fn execute_restore_object_rejects_select_type() {
|
||||
let mut rreq = restore_request(None);
|
||||
rreq.type_ = Some("SELECT".to_owned());
|
||||
rreq.type_ = Some(s3s::dto::RestoreRequestType::from_static(s3s::dto::RestoreRequestType::SELECT));
|
||||
|
||||
let req = build_request(restore_input("test-bucket", "test-key", rreq), Method::POST);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_restore_object_owned(restore_envelope(), "test-bucket".to_owned(), "test-key".to_owned(), None, Some(rreq))
|
||||
.await
|
||||
.unwrap_err();
|
||||
let err = usecase.execute_restore_object(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::NotImplemented);
|
||||
}
|
||||
|
||||
@@ -786,40 +782,30 @@ mod tests {
|
||||
// An illegal `Days` is a client error, rejected before any lock or
|
||||
// object read.
|
||||
let err = usecase
|
||||
.execute_restore_object_owned(
|
||||
restore_envelope(),
|
||||
bucket.clone(),
|
||||
"local-object".to_owned(),
|
||||
None,
|
||||
Some(restore_request(Some(0))),
|
||||
)
|
||||
.execute_restore_object(build_request(
|
||||
restore_input(&bucket, "local-object", restore_request(Some(0))),
|
||||
Method::POST,
|
||||
))
|
||||
.await
|
||||
.expect_err("days=0 must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidRequest);
|
||||
|
||||
// A malformed version-id keeps InvalidArgument instead of being
|
||||
// flattened inside `post_restore_opts`.
|
||||
let mut input = restore_input(&bucket, "local-object", restore_request(Some(1)));
|
||||
input.version_id = Some("not-a-uuid".to_string());
|
||||
let err = usecase
|
||||
.execute_restore_object_owned(
|
||||
restore_envelope(),
|
||||
bucket.clone(),
|
||||
"local-object".to_owned(),
|
||||
Some("not-a-uuid".to_owned()),
|
||||
Some(restore_request(Some(1))),
|
||||
)
|
||||
.execute_restore_object(build_request(input, Method::POST))
|
||||
.await
|
||||
.expect_err("malformed version-id must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidArgument);
|
||||
|
||||
// A missing key stays NoSuchKey.
|
||||
let err = usecase
|
||||
.execute_restore_object_owned(
|
||||
restore_envelope(),
|
||||
bucket.clone(),
|
||||
"missing-object".to_owned(),
|
||||
None,
|
||||
Some(restore_request(Some(1))),
|
||||
)
|
||||
.execute_restore_object(build_request(
|
||||
restore_input(&bucket, "missing-object", restore_request(Some(1))),
|
||||
Method::POST,
|
||||
))
|
||||
.await
|
||||
.expect_err("missing key must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::NoSuchKey);
|
||||
@@ -827,13 +813,10 @@ mod tests {
|
||||
// Restoring an object that was never transitioned is the S3
|
||||
// InvalidObjectState case, not an internal error.
|
||||
let err = usecase
|
||||
.execute_restore_object_owned(
|
||||
restore_envelope(),
|
||||
bucket,
|
||||
"local-object".to_owned(),
|
||||
None,
|
||||
Some(restore_request(Some(1))),
|
||||
)
|
||||
.execute_restore_object(build_request(
|
||||
restore_input(&bucket, "local-object", restore_request(Some(1))),
|
||||
Method::POST,
|
||||
))
|
||||
.await
|
||||
.expect_err("untransitioned object must be rejected");
|
||||
assert_eq!(err.code(), &S3ErrorCode::InvalidObjectState);
|
||||
@@ -841,18 +824,26 @@ mod tests {
|
||||
|
||||
#[tokio::test]
|
||||
async fn execute_restore_object_returns_internal_error_when_store_uninitialized() {
|
||||
let restore_request = RestoreRequest {
|
||||
days: Some(1),
|
||||
description: None,
|
||||
glacier_job_parameters: None,
|
||||
output_location: None,
|
||||
select_parameters: None,
|
||||
tier: None,
|
||||
type_: None,
|
||||
};
|
||||
let input = RestoreObjectInput::builder()
|
||||
.bucket("test-bucket".to_string())
|
||||
.key("test-key".to_string())
|
||||
.restore_request(Some(restore_request))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
let req = build_request(input, Method::POST);
|
||||
let usecase = DefaultObjectUsecase::without_context();
|
||||
|
||||
let err = usecase
|
||||
.execute_restore_object_owned(
|
||||
restore_envelope(),
|
||||
"test-bucket".to_owned(),
|
||||
"test-key".to_owned(),
|
||||
None,
|
||||
Some(restore_request(Some(1))),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
let err = usecase.execute_restore_object(req).await.unwrap_err();
|
||||
assert_eq!(err.code(), &S3ErrorCode::InternalError);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -265,27 +265,22 @@ mod tests {
|
||||
] {
|
||||
let retention = stored_retention(&stored(value)).expect("representable retention instant");
|
||||
assert_eq!(retention.mode.as_ref().expect("mode").as_str(), "COMPLIANCE");
|
||||
// The response codec writes RetainUntilDate as ISO 8601.
|
||||
let serialized = retention
|
||||
.retain_until_date
|
||||
.expect("stored retention date")
|
||||
.render(rustfs_gateway_types::TimestampFormat::Iso8601)
|
||||
.expect("serialize the response timestamp");
|
||||
let returned = OffsetDateTime::parse(&serialized, &Rfc3339).expect("parse wire timestamp");
|
||||
let stored_at = OffsetDateTime::parse(value, &Rfc3339).expect("parse stored timestamp");
|
||||
assert_eq!(returned.unix_timestamp(), stored_at.unix_timestamp(), "stored date {value}");
|
||||
let output = get_object_retention::output_to_s3s(GetObjectRetentionOutput {
|
||||
retention: Some(retention),
|
||||
})
|
||||
.expect("convert the response");
|
||||
let seam = OffsetDateTime::from(
|
||||
output
|
||||
.retention
|
||||
.expect("retention")
|
||||
.retain_until_date
|
||||
.expect("stored retention date"),
|
||||
);
|
||||
assert_eq!(seam.unix_timestamp(), stored_at.unix_timestamp(), "stored date {value}");
|
||||
let mut serialized = Vec::new();
|
||||
output
|
||||
.retention
|
||||
.expect("retention")
|
||||
.retain_until_date
|
||||
.expect("stored retention date")
|
||||
.format(s3s::dto::TimestampFormat::DateTime, &mut serialized)
|
||||
.expect("serialize the response timestamp");
|
||||
let serialized = String::from_utf8(serialized).expect("UTF-8 timestamp");
|
||||
let returned = OffsetDateTime::parse(&serialized, &Rfc3339).expect("parse wire timestamp");
|
||||
let stored = OffsetDateTime::parse(value, &Rfc3339).expect("parse stored timestamp");
|
||||
assert_eq!(returned.unix_timestamp(), stored.unix_timestamp(), "stored date {value}");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1727,7 +1727,9 @@ mod tests {
|
||||
}
|
||||
|
||||
async fn http_xml_error(error: S3Error) -> (StatusCode, String) {
|
||||
let response = error.to_http_response().expect("S3 error should serialize to HTTP");
|
||||
let response = s3s::S3Error::from(error)
|
||||
.to_http_response()
|
||||
.expect("S3 error should serialize to HTTP");
|
||||
let status = response.status();
|
||||
let body = http_body_util::BodyExt::collect(response.into_body())
|
||||
.await
|
||||
|
||||
@@ -12,13 +12,65 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! SelectObjectContent requests the query engine reads.
|
||||
//! SelectObjectContent requests from either S3 edge.
|
||||
//!
|
||||
//! The gateway edge passes the request in the shape the query engine reads.
|
||||
//! Both S3 edges pass the request in the shape the query engine reads.
|
||||
//! The select use case validates it before building the engine.
|
||||
//! The s3s conversion is deleted with the legacy edge (rustfs/backlog#2783).
|
||||
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::compat::ConversionError;
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::shapes::input_serialization::input_serialization_from_s3s;
|
||||
#[cfg(test)]
|
||||
use rustfs_gateway_types::compat::s3s_5761ddfe::generated::shapes::output_serialization::output_serialization_from_s3s;
|
||||
use rustfs_s3select_api::request as engine;
|
||||
|
||||
/// Converts the decoded request, keeping the SSE-C members as the client sent them.
|
||||
#[cfg(test)]
|
||||
pub(crate) fn select_input_from_s3s(
|
||||
input: s3s::dto::SelectObjectContentInput,
|
||||
) -> Result<engine::SelectObjectContentInput, ConversionError> {
|
||||
let s3s::dto::SelectObjectContentInput {
|
||||
bucket,
|
||||
expected_bucket_owner,
|
||||
key,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
request,
|
||||
} = input;
|
||||
let s3s::dto::SelectObjectContentRequest {
|
||||
expression,
|
||||
expression_type,
|
||||
input_serialization,
|
||||
output_serialization,
|
||||
request_progress,
|
||||
scan_range,
|
||||
} = request;
|
||||
Ok(engine::SelectObjectContentInput {
|
||||
bucket,
|
||||
expected_bucket_owner,
|
||||
key,
|
||||
sse_customer_algorithm,
|
||||
sse_customer_key,
|
||||
sse_customer_key_md5,
|
||||
request: engine::SelectObjectContentRequest {
|
||||
expression,
|
||||
expression_type: engine::ExpressionType::custom(expression_type.as_str().to_owned()),
|
||||
input_serialization: input_serialization_from_s3s(input_serialization)?,
|
||||
output_serialization: output_serialization_from_s3s(output_serialization)?,
|
||||
request_progress: request_progress.map(|progress| engine::RequestProgress {
|
||||
enabled: progress.enabled,
|
||||
}),
|
||||
scan_range: scan_range.map(|range| engine::ScanRange {
|
||||
start: range.start,
|
||||
end: range.end,
|
||||
}),
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
/// Moves the gateway's flat request into the shape the query engine reads.
|
||||
pub(crate) fn select_input_from_gateway(
|
||||
input: rustfs_gateway_types::dto::SelectObjectContentInput,
|
||||
@@ -57,57 +109,60 @@ pub(crate) fn select_input_from_gateway(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use rustfs_gateway_types::dto::{
|
||||
CompressionType, CsvInput, CsvOutput, ExpressionType, FileHeaderInfo, InputSerialization, JsonInput, OutputSerialization,
|
||||
RequestProgress, ScanRange, SelectObjectContentInput, Type as JsonType,
|
||||
use super::select_input_from_s3s;
|
||||
use s3s::dto::{
|
||||
CSVInput, CSVOutput, CompressionType, ExpressionType, FileHeaderInfo, InputSerialization, JSONInput, JSONType,
|
||||
OutputSerialization, RequestProgress, ScanRange, SelectObjectContentInput, SelectObjectContentRequest,
|
||||
};
|
||||
use rustfs_gateway_types::{BucketName, ObjectKey, SseCustomerKey};
|
||||
|
||||
fn input(input_serialization: InputSerialization) -> SelectObjectContentInput {
|
||||
SelectObjectContentInput {
|
||||
bucket: BucketName::new("bucket").expect("bucket"),
|
||||
bucket: "bucket".to_owned(),
|
||||
expected_bucket_owner: Some("owner".to_owned()),
|
||||
key: ObjectKey::new("key").expect("key"),
|
||||
key: "key".to_owned(),
|
||||
sse_customer_algorithm: Some("AES256".to_owned()),
|
||||
sse_customer_key: Some(SseCustomerKey::new("key-text".to_owned())),
|
||||
sse_customer_key: Some("key-text".to_owned()),
|
||||
sse_customer_key_md5: Some("client-md5".to_owned()),
|
||||
expression: "SELECT * FROM S3Object".to_owned(),
|
||||
expression_type: ExpressionType::SQL,
|
||||
input_serialization,
|
||||
output_serialization: OutputSerialization {
|
||||
csv: Some(CsvOutput {
|
||||
field_delimiter: Some(";".to_owned()),
|
||||
quote_fields: None,
|
||||
quote_character: Some("'".to_owned()),
|
||||
quote_escape_character: None,
|
||||
record_delimiter: Some("\r\n".to_owned()),
|
||||
request: SelectObjectContentRequest {
|
||||
expression: "SELECT * FROM S3Object".to_owned(),
|
||||
expression_type: ExpressionType::from_static(ExpressionType::SQL),
|
||||
input_serialization,
|
||||
output_serialization: OutputSerialization {
|
||||
csv: Some(CSVOutput {
|
||||
field_delimiter: Some(";".to_owned()),
|
||||
quote_fields: None,
|
||||
quote_character: Some("'".to_owned()),
|
||||
quote_escape_character: None,
|
||||
record_delimiter: Some("\r\n".to_owned()),
|
||||
}),
|
||||
json: None,
|
||||
},
|
||||
request_progress: Some(RequestProgress { enabled: Some(true) }),
|
||||
scan_range: Some(ScanRange {
|
||||
start: Some(3),
|
||||
end: Some(9),
|
||||
}),
|
||||
json: None,
|
||||
},
|
||||
request_progress: Some(RequestProgress { enabled: Some(true) }),
|
||||
scan_range: Some(ScanRange {
|
||||
start: Some(3),
|
||||
end: Some(9),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_every_member_the_engine_reads() {
|
||||
let converted = super::select_input_from_gateway(input(InputSerialization {
|
||||
csv: Some(CsvInput {
|
||||
let converted = select_input_from_s3s(input(InputSerialization {
|
||||
csv: Some(CSVInput {
|
||||
allow_quoted_record_delimiter: Some(true),
|
||||
comments: Some("#".to_owned()),
|
||||
field_delimiter: Some("|".to_owned()),
|
||||
file_header_info: Some(FileHeaderInfo::USE),
|
||||
file_header_info: Some(FileHeaderInfo::from_static(FileHeaderInfo::USE)),
|
||||
quote_character: Some("\"".to_owned()),
|
||||
quote_escape_character: Some("\\".to_owned()),
|
||||
record_delimiter: Some("\n".to_owned()),
|
||||
}),
|
||||
compression_type: Some(CompressionType::GZIP),
|
||||
compression_type: Some(CompressionType::from_static(CompressionType::GZIP)),
|
||||
json: None,
|
||||
parquet: None,
|
||||
}));
|
||||
}))
|
||||
.expect("convert");
|
||||
|
||||
assert_eq!((converted.bucket.as_str(), converted.key.as_str()), ("bucket", "key"));
|
||||
assert_eq!(converted.expected_bucket_owner.as_deref(), Some("owner"));
|
||||
@@ -144,10 +199,12 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn gateway_projection_preserves_the_query_and_read_parameters() {
|
||||
let owned = input(InputSerialization {
|
||||
csv: Some(CsvInput::default()),
|
||||
let source = input(InputSerialization {
|
||||
csv: Some(CSVInput::default()),
|
||||
..Default::default()
|
||||
});
|
||||
let owned = rustfs_gateway_types::compat::s3s_5761ddfe::select::input_from_s3s(source, &Default::default())
|
||||
.expect("materialize the test request");
|
||||
let converted = super::select_input_from_gateway(owned);
|
||||
assert_eq!((converted.bucket.as_str(), converted.key.as_str()), ("bucket", "key"));
|
||||
assert_eq!(converted.expected_bucket_owner.as_deref(), Some("owner"));
|
||||
@@ -168,13 +225,13 @@ mod tests {
|
||||
#[test]
|
||||
fn keeps_the_json_type_and_an_unknown_expression_type() {
|
||||
let mut source = input(InputSerialization {
|
||||
json: Some(JsonInput {
|
||||
r#type: Some(JsonType::LINES),
|
||||
json: Some(JSONInput {
|
||||
type_: Some(JSONType::from_static(JSONType::LINES)),
|
||||
}),
|
||||
..Default::default()
|
||||
});
|
||||
source.expression_type = ExpressionType::custom("SQL2");
|
||||
let converted = super::select_input_from_gateway(source);
|
||||
source.request.expression_type = ExpressionType::from("SQL2".to_owned());
|
||||
let converted = select_input_from_s3s(source).expect("convert");
|
||||
let json = converted.request.input_serialization.json.as_ref().expect("json input");
|
||||
assert_eq!(json.r#type.as_ref().map(|value| value.as_str()), Some("LINES"));
|
||||
assert_eq!(converted.request.expression_type.as_str(), "SQL2");
|
||||
|
||||
@@ -55,7 +55,7 @@ pub(crate) mod s3 {
|
||||
PersistedReplicationFilter, PersistedReplicationRule, PersistedReplicationTag,
|
||||
};
|
||||
pub(crate) use rustfs_s3_types::StdError;
|
||||
#[cfg(test)]
|
||||
#[cfg(any(test, all(any(feature = "cuobject", feature = "nixl"), target_os = "linux")))]
|
||||
pub(crate) use s3s::Body;
|
||||
#[cfg(test)]
|
||||
pub(crate) use s3s::auth::{Credentials, SimpleAuth};
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user