mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-20 15:07:34 +00:00
Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| c29fd6a394 | |||
| be4ce9af80 | |||
| 19be0df928 | |||
| c6c4877855 | |||
| a29919c275 | |||
| 177d45b4d1 | |||
| c369d5e7c4 | |||
| 811dfdf0c5 | |||
| 56d87c8d0c | |||
| 3d7c0cd26a | |||
| dada3276a3 |
@@ -1,2 +1,2 @@
|
||||
sha256-linux=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
sha256-darwin=348fa5eed9ae58962c06b33283ff4f15122cff97bce9b4eb1e3ffcb227264b0b
|
||||
sha256-linux=983efba6508e684db29bbdb923485fcc1c4fb312fc47fe101f7f27620906978b
|
||||
sha256-darwin=983efba6508e684db29bbdb923485fcc1c4fb312fc47fe101f7f27620906978b
|
||||
|
||||
@@ -1,2 +1,2 @@
|
||||
sha256-darwin=9901a4950099b94d3d2618524d540f1ed4bf6023044903886f98b6c4bf5f30fb
|
||||
sha256-linux=09fd5f48f5556c9fc576357222dccd7ebb0b90b790798f1834db2ec5d67b4952
|
||||
sha256-darwin=34ba36bdda087d0fe2af13ceaf94fc463043a674ea32ed98fd65833a6daa5fa6
|
||||
sha256-linux=4c4fc9019f3d9a76b189642591d1dced1d7590695e9a167ae4e7d5ef4d668bbd
|
||||
|
||||
Generated
+5
-1
@@ -10586,7 +10586,6 @@ name = "rustfs-protos"
|
||||
version = "1.0.6"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
"prost 0.14.4",
|
||||
"rmp-serde",
|
||||
"rustfs-common",
|
||||
@@ -10602,7 +10601,9 @@ dependencies = [
|
||||
"tonic",
|
||||
"tonic-prost",
|
||||
"tonic-prost-build",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -10852,7 +10853,9 @@ dependencies = [
|
||||
"rustfs-storage-api",
|
||||
"rustfs-test-utils",
|
||||
"rustfs-utils",
|
||||
"rustix",
|
||||
"s3s",
|
||||
"same-file",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serial_test",
|
||||
@@ -10866,6 +10869,7 @@ dependencies = [
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -32,6 +32,9 @@ pub const DEFAULT_HEAL_BITROT_CYCLE_SECS: u64 = 30 * 24 * 60 * 60;
|
||||
/// - Note: Enabling auto-heal can improve system resilience by automatically addressing issues, but may increase resource usage; evaluate based on your operational requirements.
|
||||
pub const ENV_HEAL_AUTO_HEAL_ENABLE: &str = "RUSTFS_HEAL_AUTO_HEAL_ENABLE";
|
||||
|
||||
/// Colon-separated Linux mount roots for automatic replacement of data directories below a mount point.
|
||||
pub const ENV_HEAL_REPLACEMENT_MOUNT_ROOTS: &str = "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS";
|
||||
|
||||
/// Environment variable name that specifies the heal queue size.
|
||||
///
|
||||
/// - Purpose: Set the maximum number of heal requests that can be queued.
|
||||
|
||||
@@ -66,6 +66,12 @@ pub const LEGACY_DATA_USAGE_OBJECT_NAME: &str = ".usage.json";
|
||||
/// stale tier names from growing persisted per-tier maps without bound.
|
||||
pub const UNKNOWN_TIER: &str = "UNKNOWN_TIER";
|
||||
|
||||
/// Whether a remote tier name can be represented in scanner accounting.
|
||||
/// Local storage classes are checked separately by the configuration owner.
|
||||
pub fn is_scannable_tier_name(name: &str) -> bool {
|
||||
!name.is_empty() && name.len() <= 256 && !name.bytes().any(|byte| byte.is_ascii_control()) && name != UNKNOWN_TIER
|
||||
}
|
||||
|
||||
/// Returns true when `existing_last_update` is ahead of `now` by more than
|
||||
/// [`USAGE_LAST_UPDATE_FUTURE_TOLERANCE`], i.e. the persisted timestamp cannot be
|
||||
/// trusted for staleness comparisons and a fresh snapshot save must be allowed.
|
||||
@@ -1229,23 +1235,66 @@ impl Serialize for DataUsageEntry {
|
||||
where
|
||||
S: serde::Serializer,
|
||||
{
|
||||
self.serialize_fields(serializer, &self.children, &self.replication_stats, &self.all_tier_stats)
|
||||
}
|
||||
}
|
||||
|
||||
struct OrderedUsageEntry<'a>(&'a DataUsageEntry);
|
||||
|
||||
impl Serialize for OrderedUsageEntry<'_> {
|
||||
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
|
||||
let mut children: Vec<_> = self.0.children.iter().collect();
|
||||
children.sort_unstable();
|
||||
// MessagePack decoders accept struct fields as positional sequences.
|
||||
// Include the nested target and tier maps in stable order as well.
|
||||
let replication = self.0.replication_stats.as_ref().map(|stats| {
|
||||
let ReplicationAllStats {
|
||||
targets,
|
||||
replica_size,
|
||||
replica_count,
|
||||
} = stats;
|
||||
let targets: std::collections::BTreeMap<_, _> = targets.iter().collect();
|
||||
(targets, replica_size, replica_count)
|
||||
});
|
||||
let tiers = self.0.all_tier_stats.as_ref().map(|stats| {
|
||||
let AllTierStats { tiers } = stats;
|
||||
let tiers: std::collections::BTreeMap<_, _> = tiers.iter().collect();
|
||||
(tiers,)
|
||||
});
|
||||
self.0.serialize_fields(serializer, &children, &replication, &tiers)
|
||||
}
|
||||
}
|
||||
|
||||
impl DataUsageEntry {
|
||||
/// Stable MessagePack encoding for independently reusable scanner checkpoint pages.
|
||||
pub fn ordered_encoding(&self) -> impl Serialize + '_ {
|
||||
OrderedUsageEntry(self)
|
||||
}
|
||||
|
||||
fn serialize_fields<S: serde::Serializer, C: Serialize, R: Serialize, T: Serialize>(
|
||||
&self,
|
||||
serializer: S,
|
||||
children: &C,
|
||||
replication: &R,
|
||||
tiers: &T,
|
||||
) -> Result<S::Ok, S::Error> {
|
||||
// Keep entries map-encoded so older readers can ignore fields appended
|
||||
// by newer scanner versions during rolling upgrades. The derived
|
||||
// (array) encoding made any appended field a decode error for them.
|
||||
let mut state = serializer.serialize_map(Some(
|
||||
11 + usize::from(self.unknown_tier_stats.is_some()) + usize::from(self.tier_accounting_proof.is_some()),
|
||||
))?;
|
||||
state.serialize_entry("children", &self.children)?;
|
||||
state.serialize_entry("children", children)?;
|
||||
state.serialize_entry("size", &self.size)?;
|
||||
state.serialize_entry("objects", &self.objects)?;
|
||||
state.serialize_entry("versions", &self.versions)?;
|
||||
state.serialize_entry("delete_markers", &self.delete_markers)?;
|
||||
state.serialize_entry("obj_sizes", &self.obj_sizes)?;
|
||||
state.serialize_entry("obj_versions", &self.obj_versions)?;
|
||||
state.serialize_entry("replication_stats", &self.replication_stats)?;
|
||||
state.serialize_entry("replication_stats", replication)?;
|
||||
state.serialize_entry("compacted", &self.compacted)?;
|
||||
state.serialize_entry("failed_objects", &self.failed_objects)?;
|
||||
state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
|
||||
state.serialize_entry("all_tier_stats", tiers)?;
|
||||
// Keep the legacy no-unknown shape byte-for-byte stable. Once unknown
|
||||
// accounting exists, append its map field before the optional proof.
|
||||
if let Some(unknown_tier_stats) = self.unknown_tier_stats.as_ref() {
|
||||
@@ -2528,6 +2577,32 @@ mod tests {
|
||||
entry
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn ordered_checkpoint_entry_round_trips_and_stays_identical_after_reload() {
|
||||
let mut entry = DataUsageEntry::default();
|
||||
let mut replication = ReplicationAllStats {
|
||||
replica_size: 998,
|
||||
replica_count: 3,
|
||||
..Default::default()
|
||||
};
|
||||
let mut tiers = AllTierStats::default();
|
||||
for index in 1..=32 {
|
||||
let key = format!("source-{index}");
|
||||
entry.children.insert(key.clone());
|
||||
replication.targets.insert(key.clone(), ReplicationTargetUsage::default());
|
||||
tiers.tiers.insert(key, TierStats::default());
|
||||
}
|
||||
entry.replication_stats = Some(replication);
|
||||
entry.all_tier_stats = Some(tiers);
|
||||
let bytes = rmp_serde::to_vec(&entry.ordered_encoding()).expect("encode ordered checkpoint");
|
||||
let reloaded: DataUsageEntry = rmp_serde::from_slice(&bytes).expect("read with the existing codec");
|
||||
assert_eq!(reloaded.children, entry.children);
|
||||
let replicas = reloaded.replication_stats.as_ref().expect("replication stats");
|
||||
assert_eq!((replicas.targets.len(), replicas.replica_size, replicas.replica_count), (32, 998, 3));
|
||||
assert_eq!(reloaded.all_tier_stats.as_ref().expect("tier stats").tiers.len(), 32);
|
||||
assert_eq!(rmp_serde::to_vec(&reloaded.ordered_encoding()).expect("encode reloaded entry"), bytes);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tier_stats_survive_entry_merge() {
|
||||
let mut left = tier_entry(
|
||||
|
||||
@@ -34,7 +34,10 @@
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::common::{RustFSTestEnvironment, init_logging, local_http_client};
|
||||
use crate::common::{
|
||||
AdminTransport, RustFSTestEnvironment, admin_attach_user_policy_via, admin_create_user, admin_request_with_session_token,
|
||||
init_logging, local_http_client, parse_assume_role_credentials,
|
||||
};
|
||||
use hmac::{Hmac, KeyInit as _, Mac};
|
||||
use http::header::HOST;
|
||||
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
|
||||
@@ -396,6 +399,54 @@ mod tests {
|
||||
let (status, _) = signed_request(&env.url, http::Method::GET, ACCOUNT_INFO_PATH, None, user_ak, old_sk).await?;
|
||||
assert!(status.is_client_error(), "the old secret must stop working, status: {status}");
|
||||
|
||||
env.stop_server();
|
||||
drop(env);
|
||||
console_session_rotates_the_iam_password_and_revokes_the_old_session().await
|
||||
}
|
||||
|
||||
async fn console_session_rotates_the_iam_password_and_revokes_the_old_session() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
env.start_rustfs_server(vec![]).await?;
|
||||
|
||||
let user = "console-password-user";
|
||||
let old_password = uuid::Uuid::new_v4().to_string();
|
||||
let new_password = uuid::Uuid::new_v4().to_string();
|
||||
admin_create_user(&env, user, &old_password).await?;
|
||||
admin_attach_user_policy_via(AdminTransport::Signed, &env.url, &env.access_key, &env.secret_key, "readonly", user)
|
||||
.await?;
|
||||
|
||||
let (status, response) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the current password must sign in");
|
||||
let (session_key, session_secret, token) = parse_assume_role_credentials(&response)?;
|
||||
let session_request = |method, path, body| {
|
||||
admin_request_with_session_token(&env.url, method, path, body, &session_key, &session_secret, Some(&token))
|
||||
};
|
||||
let password_request = |current: &str| {
|
||||
Some(serde_json::json!({ "current_secret_key": current, "new_secret_key": new_password }).to_string())
|
||||
};
|
||||
|
||||
let (status, body) =
|
||||
session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&session_secret)).await?;
|
||||
assert_eq!(
|
||||
status,
|
||||
reqwest::StatusCode::BAD_REQUEST,
|
||||
"the session secret must not be accepted as the IAM password"
|
||||
);
|
||||
assert!(body.contains("InvalidRequest") && body.contains("the current secret key is incorrect"));
|
||||
|
||||
let (status, body) = session_request(http::Method::POST, ACCOUNT_PASSWORD_PATH, password_request(&old_password)).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the same session must accept the current IAM password");
|
||||
assert_eq!(serde_json::from_str::<serde_json::Value>(&body)?["sessions_revoked"], 1);
|
||||
|
||||
let (status, _) = session_request(http::Method::GET, ACCOUNT_INFO_PATH, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the pre-rotation session must be revoked");
|
||||
let (status, _) = assume_role(&env.url, user, &old_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::FORBIDDEN, "the old password must no longer sign in");
|
||||
let (status, response) = assume_role(&env.url, user, &new_password, None).await?;
|
||||
assert_eq!(status, reqwest::StatusCode::OK, "the new password must sign in");
|
||||
parse_assume_role_credentials(&response)?;
|
||||
|
||||
env.stop_server();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -36,6 +36,7 @@ mod replication_delete_marker_test;
|
||||
mod replication_quota_test;
|
||||
mod s3_basic_test;
|
||||
mod s3_during_data_movement_test;
|
||||
mod scanner_cluster_observation_test;
|
||||
mod scanner_heal_topology_fixture;
|
||||
mod scanner_heal_topology_test;
|
||||
mod scanner_heal_topology_tier_test;
|
||||
|
||||
@@ -159,6 +159,7 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
|
||||
|
||||
let info_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/info", None).await?;
|
||||
let info: Value = serde_json::from_str(&info_body)?;
|
||||
assert!(info.get("inventory").is_none(), "inventory requires explicit opt-in");
|
||||
let servers = info["info"]["servers"]
|
||||
.as_array()
|
||||
.ok_or_else(|| format!("admin info omitted servers: {info}"))?;
|
||||
@@ -200,6 +201,24 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
|
||||
assert!(sample.finally, "node {node_idx} metrics sample was not terminal");
|
||||
assert!(sample.errors.is_empty(), "node {node_idx} metrics reported errors: {:?}", sample.errors);
|
||||
assert!(!sample.hosts.is_empty(), "node {node_idx} metrics omitted hosts");
|
||||
assert!(sample.scanner_node.is_none(), "default metrics must not expose the internal peer sample");
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let frame: Value = serde_json::from_str(
|
||||
&node_admin_body(&dist, node_idx, "/rustfs/admin/v3/realtime?scope=cluster&types=1&n=1").await?,
|
||||
)?;
|
||||
assert_eq!(frame["final"], true);
|
||||
let summary = &frame["cluster_scanner"];
|
||||
assert_eq!(summary["nodes_total"], 4, "known cluster membership");
|
||||
assert!(summary["sampled_at"].as_str().is_some());
|
||||
assert!(matches!(summary["state"].as_str(), Some("scanning" | "idle" | "unknown")));
|
||||
assert_eq!(summary["current"].is_object(), summary["state"] == "scanning");
|
||||
Ok(summary["nodes_responded"] == 4)
|
||||
},
|
||||
"every request node reaches the full scanner peer set",
|
||||
)
|
||||
.await?;
|
||||
}
|
||||
|
||||
let targets_body = cluster_admin_ok(&dist.cluster, Method::GET, "/rustfs/admin/v3/audit/target/list", None).await?;
|
||||
@@ -234,6 +253,38 @@ async fn four_node_health_inventory_metrics_and_audit_delivery_are_consistent()
|
||||
"audit entry leaked the root secret key"
|
||||
);
|
||||
|
||||
let pending = unique_bucket("inventory");
|
||||
dist.create_bucket(&pending).await?;
|
||||
let listed = dist.client(1)?.list_buckets().send().await?;
|
||||
assert_eq!(listed.buckets().len(), 2, "expected the two test buckets");
|
||||
wait_until(
|
||||
Duration::from_secs(30),
|
||||
|| async {
|
||||
let body = node_admin_body(&dist, 1, "/rustfs/admin/v3/info?include=inventory").await?;
|
||||
let response: Value = serde_json::from_str(&body)?;
|
||||
let inventory: rustfs_madmin::ClusterInventory = serde_json::from_value(response["inventory"].clone())?;
|
||||
if inventory.current_buckets != Some(2) {
|
||||
return Ok(false);
|
||||
}
|
||||
match inventory.measured_buckets {
|
||||
None | Some(0) => {
|
||||
assert_eq!(inventory.measured_objects, None);
|
||||
assert_eq!(inventory.measured_bytes, None);
|
||||
}
|
||||
Some(1 | 2) => {
|
||||
assert!(inventory.measured_objects.is_some_and(|objects| objects <= 1));
|
||||
}
|
||||
Some(count) => panic!("measured {count} buckets across a two-bucket namespace"),
|
||||
}
|
||||
if inventory.measured_buckets != Some(2) {
|
||||
assert_ne!(inventory.converged, Some(true));
|
||||
}
|
||||
Ok(true)
|
||||
},
|
||||
"inventory reflects the current namespace and reports scanner coverage independently",
|
||||
)
|
||||
.await?;
|
||||
|
||||
let result = verify_write_observations_during_peer_failure(&dist, &bucket).await;
|
||||
collector.abort();
|
||||
result
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
use super::harness::{NODE_COUNT, TestResult, wait_until};
|
||||
use super::scanner_peer_bucket_fixture::PeerBucketRun;
|
||||
use crate::common::init_logging;
|
||||
use futures::future::try_join_all;
|
||||
use serde_json::Value;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
|
||||
const CLUSTER_SCANNER: &str = "/rustfs/admin/v3/realtime?n=1&types=1&scope=cluster";
|
||||
const OBSERVATION_TIMEOUT: Duration = Duration::from_secs(35);
|
||||
|
||||
async fn scanner_frames(run: &PeerBucketRun) -> TestResult<Vec<Value>> {
|
||||
try_join_all((0..NODE_COUNT).map(|node| run.admin_json(node, CLUSTER_SCANNER))).await
|
||||
}
|
||||
|
||||
fn full_sample(frame: &Value) -> bool {
|
||||
frame["final"] == true
|
||||
&& frame["cluster_scanner"]["nodes_total"] == NODE_COUNT
|
||||
&& frame["cluster_scanner"]["nodes_responded"] == NODE_COUNT
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn followers_report_the_coordinators_active_progress_and_committed_completion() -> TestResult {
|
||||
init_logging();
|
||||
let mut run = PeerBucketRun::start("cluster-scanner-observation").await?;
|
||||
let baseline = run.quiet_baseline().await?;
|
||||
let leader = run.leader().await?;
|
||||
let pid = run.pid(leader)?;
|
||||
let hot = run.buckets[2].clone();
|
||||
run.barrier.arm("before_start_snapshot", pid, None)?;
|
||||
run.barrier.start_session(&run.buckets)?;
|
||||
// All four child processes and the harness use the same host clock.
|
||||
let written_after = OffsetDateTime::now_utc();
|
||||
run.put(leader, &hot, "wake-cluster-observation", 149).await?;
|
||||
|
||||
let start = run.barrier.wait_stage("before_start_snapshot", pid, None, true).await?;
|
||||
let epoch = start["detail"]["epoch"].as_u64().ok_or("start gate omitted epoch")?;
|
||||
let cycle = start["detail"]["cycle"].as_u64().ok_or("start gate omitted cycle")?;
|
||||
if baseline.scanner_epoch != Some(epoch) || baseline.scanner_cycle.is_none_or(|previous| previous >= cycle) {
|
||||
return Err("controlled cycle did not follow the quiet baseline under the same coordinator".into());
|
||||
}
|
||||
run.barrier.arm("root_durable", pid, Some((epoch, cycle)))?;
|
||||
run.barrier.release("before_start_snapshot", pid)?;
|
||||
run.barrier
|
||||
.wait_stage("root_durable", pid, Some((epoch, cycle)), true)
|
||||
.await?;
|
||||
|
||||
// The leader is held after the real walk and before committing its cycle completion.
|
||||
// Each ingress node maintains its own short-lived cluster sample cache.
|
||||
let expected_leader = &run.dist.cluster.nodes[leader].url;
|
||||
let active = wait_until(
|
||||
OBSERVATION_TIMEOUT,
|
||||
|| async {
|
||||
let frames = scanner_frames(&run).await?;
|
||||
let started = frames[leader]["cluster_scanner"]["current"]["started_at"].as_str();
|
||||
let consistent = started.is_some()
|
||||
&& frames.iter().all(|frame| {
|
||||
let summary = &frame["cluster_scanner"];
|
||||
let current = &summary["current"];
|
||||
full_sample(frame)
|
||||
&& summary["state"] == "scanning"
|
||||
&& current["cycle"] == cycle
|
||||
&& current["leader"].as_str() == Some(expected_leader.as_str())
|
||||
&& current["started_at"].as_str() == started
|
||||
&& current["objects_scanned"].as_u64().is_some_and(|objects| objects > 0)
|
||||
});
|
||||
if !consistent {
|
||||
tracing::info!(
|
||||
?frames,
|
||||
cycle,
|
||||
expected_leader,
|
||||
"waiting for active scanner progress on every ingress node"
|
||||
);
|
||||
}
|
||||
Ok(consistent)
|
||||
},
|
||||
"leader and all three followers report the identified active cycle with nonzero work",
|
||||
)
|
||||
.await;
|
||||
run.barrier.disarm()?;
|
||||
run.barrier.release("root_durable", pid)?;
|
||||
active?;
|
||||
|
||||
wait_until(
|
||||
OBSERVATION_TIMEOUT,
|
||||
|| async {
|
||||
let frames = scanner_frames(&run).await?;
|
||||
let settled = frames.iter().all(|frame| {
|
||||
let summary = &frame["cluster_scanner"];
|
||||
full_sample(frame)
|
||||
&& summary["state"] == "idle"
|
||||
&& summary.get("current").is_none()
|
||||
&& summary["last_attempt"]["cycle"] == cycle
|
||||
&& summary["last_attempt"]["result"] == "success"
|
||||
&& summary["last_success_at"].as_str().is_some_and(|value| {
|
||||
OffsetDateTime::parse(value, &Rfc3339).is_ok_and(|completed| completed >= written_after)
|
||||
})
|
||||
});
|
||||
if !settled {
|
||||
tracing::info!(?frames, cycle, "waiting for the controlled scanner cycle to finish");
|
||||
}
|
||||
Ok(settled)
|
||||
},
|
||||
"every ingress node reports the completed cycle and an idle cluster scanner",
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -96,6 +96,9 @@ mod replication_target_matrix_test;
|
||||
#[cfg(test)]
|
||||
mod data_usage_test;
|
||||
|
||||
#[cfg(test)]
|
||||
mod scanner_wide_directory_test;
|
||||
|
||||
// KMS-specific test modules
|
||||
#[cfg(test)]
|
||||
mod kms;
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
#[cfg(all(test, target_os = "linux"))]
|
||||
mod tests {
|
||||
use crate::chaos::{VersionShardCensus, census_object_version_on_disk};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, admin_request, init_logging};
|
||||
use crate::common::{ClusterTopology, RustFSTestClusterEnvironment, admin_request, init_logging, local_http_client};
|
||||
use aws_sdk_s3::Client;
|
||||
use aws_sdk_s3::primitives::ByteStream;
|
||||
use aws_sdk_s3::types::{BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, VersioningConfiguration};
|
||||
@@ -412,7 +412,7 @@ mod tests {
|
||||
.arg(test_binary)
|
||||
.arg("--exact")
|
||||
.arg(test_name)
|
||||
.arg("--ignored")
|
||||
.arg("--include-ignored")
|
||||
.arg("--nocapture")
|
||||
.env(NAMESPACE_ENV, "1")
|
||||
.status()?;
|
||||
@@ -1524,6 +1524,110 @@ mod tests {
|
||||
assert!(assert_no_replacement_status_records_in_status(&definitive, target).is_ok());
|
||||
}
|
||||
|
||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn test_privileged_4x1_subdirectory_replacement_waits_for_write_quorum() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
init_logging();
|
||||
if !privileged_run_enabled()? {
|
||||
return Ok(());
|
||||
}
|
||||
if std::env::var_os(NAMESPACE_ENV).is_none() {
|
||||
return run_current_test_in_mount_namespace(
|
||||
"replacement_privileged_e2e_test::tests::test_privileged_4x1_subdirectory_replacement_waits_for_write_quorum",
|
||||
);
|
||||
}
|
||||
|
||||
let mut mounts = MountNamespaceGuard::new()?;
|
||||
let mut cluster = RustFSTestClusterEnvironment::new(4).await?;
|
||||
let filesystem: serde_json::Value = serde_json::from_str(&run_command_stdout(
|
||||
"findmnt",
|
||||
&["--json", "--target", &cluster.temp_dir, "--output", "TARGET,FSTYPE"],
|
||||
)?)?;
|
||||
let xfs_root = filesystem["filesystems"][0]["target"]
|
||||
.as_str()
|
||||
.filter(|root| *root != "/" && filesystem["filesystems"][0]["fstype"] == "xfs");
|
||||
for (index, node) in cluster.nodes.iter_mut().enumerate() {
|
||||
let mount = PathBuf::from(&node.data_dir);
|
||||
if xfs_root.is_none() {
|
||||
mounts.mount_tmpfs(&mount, &format!("rustfs-replacement-4x1-{index}"))?;
|
||||
}
|
||||
let data = mount.join("data");
|
||||
fs::create_dir(&data)?;
|
||||
node.data_dir = path_to_string(&data, "replacement subdirectory")?;
|
||||
node.data_dirs[0] = node.data_dir.clone();
|
||||
}
|
||||
let target = PathBuf::from(&cluster.nodes[TARGET_NODE].data_dir);
|
||||
let mount = xfs_root.unwrap_or(target.parent().and_then(Path::to_str).ok_or("missing target mount root")?);
|
||||
eprintln!(
|
||||
"4x1 replacement fixture filesystem = {}",
|
||||
if xfs_root.is_some() { "xfs" } else { "tmpfs" }
|
||||
);
|
||||
cluster.set_node_env(TARGET_NODE, "RUSTFS_HEAL_REPLACEMENT_MOUNT_ROOTS", mount)?;
|
||||
cluster.set_env("RUSTFS_STORAGE_CLASS_STANDARD", "EC:2");
|
||||
cluster.set_env("RUSTFS_HEAL_ENABLED", "true");
|
||||
cluster.set_env("RUSTFS_HEAL_INTERVAL_SECS", "1");
|
||||
cluster.set_env("RUSTFS_SCANNER_START_DELAY_SECS", "0");
|
||||
cluster.set_env("RUSTFS_PUT_RENAME_EARLY_ACK_ENABLE", "false");
|
||||
cluster.start().await?;
|
||||
let clients = cluster.create_all_clients()?;
|
||||
let versions = seed_baseline(&clients[0], &target, 0).await?;
|
||||
|
||||
cluster.stop_node_gracefully(TARGET_NODE).await?;
|
||||
cluster.stop_node_gracefully(0).await?;
|
||||
fs::remove_dir_all(&target)?;
|
||||
fs::create_dir(&target)?;
|
||||
|
||||
let metadata = ["format.json", "pool.bin/xl.meta", "pool.bin.identity/xl.meta"];
|
||||
let system_dir = Path::new(".rustfs.sys");
|
||||
let survivor_files = cluster.nodes[2..]
|
||||
.iter()
|
||||
.flat_map(|node| metadata.map(|name| Path::new(&node.data_dir).join(system_dir).join(name)))
|
||||
.map(|file| fs::read(&file).map(|bytes| (file, bytes)))
|
||||
.collect::<std::io::Result<Vec<_>>>()?;
|
||||
|
||||
cluster.start_node_process(TARGET_NODE).await?;
|
||||
let write_health = format!("{}/minio/health/cluster", cluster.nodes[2].url);
|
||||
let http = local_http_client();
|
||||
timeout(Duration::from_secs(30), async {
|
||||
loop {
|
||||
let status = http.get(&write_health).send().await?.status();
|
||||
if status.as_u16() == 503 {
|
||||
break;
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
Ok::<_, reqwest::Error>(())
|
||||
})
|
||||
.await??;
|
||||
verify_bodies(&clients[2], &versions).await?;
|
||||
let version = &versions[0];
|
||||
let failed_delete = clients[2]
|
||||
.delete_object()
|
||||
.bucket(&version.bucket)
|
||||
.key(&version.key)
|
||||
.send()
|
||||
.await
|
||||
.expect_err("two of four drives must not delete an existing object");
|
||||
assert_eq!(failed_delete.raw_response().map(|response| response.status().as_u16()), Some(503));
|
||||
for name in metadata {
|
||||
assert!(!target.join(system_dir).join(name).try_exists()?, "unadmitted target published {name}");
|
||||
}
|
||||
for (file, before) in survivor_files {
|
||||
assert_eq!(fs::read(&file)?, before, "surviving pool evidence changed below write quorum");
|
||||
}
|
||||
|
||||
cluster.start_node(0).await?;
|
||||
wait_for_completed_replacement_with_census(&cluster, &target, &versions, 90).await?;
|
||||
for name in metadata {
|
||||
assert!(fs::metadata(target.join(system_dir).join(name))?.is_file());
|
||||
}
|
||||
cluster.stop_node_gracefully(TARGET_NODE).await?;
|
||||
cluster.start_node(TARGET_NODE).await?;
|
||||
assert!(incomplete_versions(&target, &versions)?.is_empty());
|
||||
verify_bodies(&clients[TARGET_NODE], &versions).await
|
||||
}
|
||||
|
||||
/// Linux mount namespaces are per-thread; keep mount setup and process
|
||||
/// spawning on one OS thread so child RustFS nodes inherit the test mounts.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
|
||||
@@ -0,0 +1,208 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
use aws_sdk_s3::{Client, primitives::ByteStream};
|
||||
use futures::{StreamExt, TryStreamExt, stream};
|
||||
use rustfs_data_usage::{BucketUsageValueState, DataUsageInfo};
|
||||
use serde::{Deserialize, de::DeserializeOwned};
|
||||
use tokio::time::{Instant, sleep, timeout};
|
||||
|
||||
use crate::common::{RustFSTestEnvironment, signed_request};
|
||||
use crate::storage_api::scanner_peer_bucket::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
|
||||
|
||||
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
|
||||
|
||||
const BUCKET: &str = "wide-directory-usage";
|
||||
const WIDTH: usize = 512;
|
||||
// The per-disk token bucket allows an initial burst of one second. Even the
|
||||
// first pass can read at most (1 + 3) * 96 = 384 of these 512 root entries.
|
||||
const SCAN_ENV: &[(&str, &str)] = &[
|
||||
("RUST_LOG", "warn,rustfs::scanner::io=debug,rustfs::scanner::data_usage=debug"),
|
||||
("RUSTFS_SCANNER_ENABLED", "true"),
|
||||
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
|
||||
("RUSTFS_SCANNER_CYCLE", "1"),
|
||||
("RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS", "3"),
|
||||
("RUSTFS_SCANNER_SPEED", "fastest"),
|
||||
("RUSTFS_SCANNER_DISK_IO_BUDGET_IOPS", "96"),
|
||||
("RUSTFS_SCANNER_DISK_IO_BUDGET_BYTES_PER_SEC", "off"),
|
||||
];
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Status {
|
||||
metrics: Metrics,
|
||||
disk_io_budget: Vec<DiskBudget>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct Metrics {
|
||||
current_cycle_active: bool,
|
||||
active_paths: Vec<String>,
|
||||
cycle_max_duration_seconds: f64,
|
||||
partial_cycles_runtime: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct DiskBudget {
|
||||
iops: Option<u64>,
|
||||
}
|
||||
|
||||
async fn admin<T: DeserializeOwned>(env: &RustFSTestEnvironment, resource: &str) -> TestResult<T> {
|
||||
let url = format!("{}/rustfs/admin/v3/{resource}", env.url);
|
||||
let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
|
||||
Ok(response.error_for_status()?.json().await?)
|
||||
}
|
||||
|
||||
async fn inventory(client: &Client) -> TestResult<BTreeMap<String, u64>> {
|
||||
let mut pages = client.list_objects_v2().bucket(BUCKET).max_keys(37).into_paginator().send();
|
||||
let mut sizes = BTreeMap::new();
|
||||
while let Some(page) = pages.next().await {
|
||||
for object in page?.contents() {
|
||||
let key = object.key().ok_or("S3 inventory omitted object key")?;
|
||||
let size = u64::try_from(object.size().ok_or("S3 inventory omitted object size")?)?;
|
||||
assert!(sizes.insert(key.to_owned(), size).is_none(), "duplicate inventory key {key}");
|
||||
}
|
||||
}
|
||||
Ok(sizes)
|
||||
}
|
||||
|
||||
async fn scan_status(env: &RustFSTestEnvironment) -> TestResult<Status> {
|
||||
let status: Status = admin(env, "scanner/status").await?;
|
||||
assert_eq!(status.metrics.cycle_max_duration_seconds, 3.0, "effective cycle budget");
|
||||
assert!(status.disk_io_budget.iter().all(|disk| disk.iops == Some(96)), "effective disk budget");
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn certifies_wide_directory_after_mid_scan_process_restart() -> TestResult {
|
||||
let mut env = RustFSTestEnvironment::new().await?;
|
||||
let temporary_log = tempfile::NamedTempFile::new()?;
|
||||
let server_log = env
|
||||
.capture_log_path
|
||||
.get_or_insert_with(|| temporary_log.path().display().to_string())
|
||||
.clone();
|
||||
let bucket_metadata = Path::new(&env.temp_dir)
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(BUCKET_META_PREFIX)
|
||||
.join(BUCKET);
|
||||
// Seed a cold bucket without allowing any earlier, larger-budget sweep to count it.
|
||||
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_SCANNER_ENABLED", "false")])
|
||||
.await?;
|
||||
let client = env.create_s3_client();
|
||||
client.create_bucket().bucket(BUCKET).send().await?;
|
||||
stream::iter(0..WIDTH)
|
||||
.map(Ok)
|
||||
.try_for_each_concurrent(12, |i| {
|
||||
let client = &client;
|
||||
async move {
|
||||
client
|
||||
.put_object()
|
||||
.bucket(BUCKET)
|
||||
.key(format!("object-{i:064x}"))
|
||||
.body(ByteStream::from(vec![42; 1 + i % 67]))
|
||||
.send()
|
||||
.await
|
||||
.map(|_| ())
|
||||
}
|
||||
})
|
||||
.await?;
|
||||
let expected = inventory(&client).await?;
|
||||
assert_eq!(expected.len(), WIDTH);
|
||||
|
||||
env.restart_server_preserving_data(vec![], SCAN_ENV).await?;
|
||||
let first_pid = env.process.as_ref().ok_or("missing scanner process")?.id();
|
||||
timeout(Duration::from_secs(60), async {
|
||||
loop {
|
||||
let status = scan_status(&env).await?;
|
||||
if status.metrics.partial_cycles_runtime >= 1
|
||||
&& bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
|
||||
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir()
|
||||
&& status.metrics.current_cycle_active
|
||||
&& status
|
||||
.metrics
|
||||
.active_paths
|
||||
.iter()
|
||||
.any(|path| path.split('/').any(|part| part == BUCKET))
|
||||
{
|
||||
let log = tokio::fs::read_to_string(&server_log).await?;
|
||||
if !log
|
||||
.lines()
|
||||
.any(|line| line.contains("partial_saved_not_published") && line.contains(BUCKET))
|
||||
{
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
continue;
|
||||
}
|
||||
assert!(!status.disk_io_budget.is_empty(), "scanner never admitted disk I/O");
|
||||
break Ok::<_, Box<dyn std::error::Error + Send + Sync>>(());
|
||||
}
|
||||
sleep(Duration::from_millis(200)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.map_err(|_| "scanner did not re-enter the bucket after exhausting the fixed runtime budget")??;
|
||||
let partial: DataUsageInfo = admin(&env, "datausageinfo").await?;
|
||||
assert!(
|
||||
partial
|
||||
.buckets_usage
|
||||
.get(BUCKET)
|
||||
.is_none_or(|value| value.usage_certification.is_none()),
|
||||
"the first process certified the bucket before the injected crash"
|
||||
);
|
||||
|
||||
env.stop_server();
|
||||
// The terminal save has returned and the old process can no longer write.
|
||||
assert!(
|
||||
bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
|
||||
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir(),
|
||||
"the interrupted process must have committed a paged checkpoint"
|
||||
);
|
||||
let restart_log_offset = usize::try_from(tokio::fs::metadata(&server_log).await?.len())?;
|
||||
let restarted_at = SystemTime::now();
|
||||
env.start_rustfs_server_without_cleanup_with_env(SCAN_ENV).await?;
|
||||
let second_pid = env.process.as_ref().ok_or("missing restarted scanner process")?.id();
|
||||
assert_ne!(first_pid, second_pid);
|
||||
let deadline = Instant::now() + Duration::from_secs(180);
|
||||
loop {
|
||||
let usage: DataUsageInfo = admin(&env, "datausageinfo").await?;
|
||||
let status = scan_status(&env).await?;
|
||||
if let Some(bucket) = usage.buckets_usage.get(BUCKET)
|
||||
&& bucket.usage_value_state == Some(BucketUsageValueState::Certified)
|
||||
{
|
||||
let log = tokio::fs::read_to_string(&server_log).await?;
|
||||
let first_admission = log
|
||||
.get(restart_log_offset..)
|
||||
.ok_or("server log truncated across restart")?
|
||||
.lines()
|
||||
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
|
||||
.find(|record| record["state"] == "bucket_checkpoint_prepared" && record["bucket"] == BUCKET)
|
||||
.ok_or("no checkpoint admission in the restarted process")?;
|
||||
assert_eq!(first_admission["resume_position"], true, "restarted scanner discarded the checkpoint");
|
||||
assert!(usage.is_complete_bucket_usage_snapshot());
|
||||
let certificate = bucket
|
||||
.usage_certification
|
||||
.ok_or("missing certificate for the measured bucket")?;
|
||||
assert!(certificate.certified_at >= restarted_at, "stale pre-restart certificate");
|
||||
assert!(status.metrics.partial_cycles_runtime > 0, "new process must cross a cycle deadline");
|
||||
assert!(!status.disk_io_budget.is_empty(), "restarted scanner never admitted disk I/O");
|
||||
assert_eq!(inventory(&client).await?, expected, "durable S3 inventory changed across restart");
|
||||
let count = u64::try_from(expected.len())?;
|
||||
let bytes = expected.values().sum::<u64>();
|
||||
assert_eq!((bucket.objects_count, bucket.size), (count, bytes));
|
||||
assert_eq!((usage.objects_total_count, usage.objects_total_size), (count, bytes));
|
||||
eprintln!(
|
||||
"PIDs {first_pid}/{second_pid}, runtime-limited cycles since restart {}, certificate epoch {} cycle {}, S3 and admin {} objects/{} bytes",
|
||||
status.metrics.partial_cycles_runtime, certificate.epoch, certificate.cycle, count, bytes
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
assert!(
|
||||
Instant::now() < deadline,
|
||||
"no certified bucket after restart: {status:?}, value={:?}",
|
||||
usage.buckets_usage.get(BUCKET)
|
||||
);
|
||||
sleep(Duration::from_millis(500)).await;
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,7 @@ pub mod diagnostic {
|
||||
}
|
||||
|
||||
pub mod admin {
|
||||
pub use crate::diagnostics::admin_server_info::{get_local_server_property, get_server_info};
|
||||
pub use crate::diagnostics::admin_server_info::{get_local_server_property, get_server_info, get_server_info_with_inventory};
|
||||
}
|
||||
|
||||
pub mod bitrot {
|
||||
@@ -455,6 +455,7 @@ pub mod disk {
|
||||
pub use crate::cluster::rpc::background_pipe::{BackgroundPipeReader, BackgroundPipeWriter, background_pipe};
|
||||
pub use crate::cluster::rpc::background_reader_stream::BackgroundReaderStream;
|
||||
pub use crate::disk::BOUNDED_DELETE_PREPARE_PROTOCOL_VERSION;
|
||||
pub use crate::disk::ReplacementMount;
|
||||
pub use crate::disk::background_memory::BackgroundEncodeBuffer;
|
||||
pub use crate::disk::disk_store::get_object_disk_read_timeout;
|
||||
#[cfg(feature = "test-util")]
|
||||
@@ -728,6 +729,9 @@ pub mod store_list {
|
||||
pub mod storage {
|
||||
pub use crate::core::pools::{HealLifecycleExpiryContext, POOL_META_NAME};
|
||||
pub use crate::store::HealWalkVersion;
|
||||
pub use crate::store::delete_recovery_inventory::{
|
||||
DeleteResidueInventoryPage, DeleteResidueSource, DeleteResidueSourceStatus,
|
||||
};
|
||||
pub use crate::store::{
|
||||
BootstrapLocalTarget, ECStore, SCANNER_PUBLICATION_LEASE_TTL_MS, ScannerDataMovementPauseStatus, all_local_disk,
|
||||
all_local_disk_path, find_local_disk_by_ref, find_local_disk_by_ref_for_rpc, init_local_disks,
|
||||
|
||||
@@ -26,6 +26,8 @@ use crate::bucket::target::BucketTargetType;
|
||||
use crate::bucket::target::{self, BucketTarget, BucketTargets, Credentials};
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use aws_sdk_s3::config::interceptors::BeforeDeserializationInterceptorContextRef;
|
||||
use aws_sdk_s3::config::{ConfigBag, Intercept, RuntimeComponents};
|
||||
use aws_sdk_s3::error::ProvideErrorMetadata;
|
||||
use aws_sdk_s3::error::SdkError;
|
||||
use aws_sdk_s3::operation::complete_multipart_upload::CompleteMultipartUploadOutput;
|
||||
@@ -53,8 +55,9 @@ use http::{HeaderMap, HeaderName, HeaderValue, StatusCode};
|
||||
use reqwest::Client as HttpClient;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_LOCK_BYPASS_GOVERNANCE, AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header,
|
||||
is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header,
|
||||
AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_OBJECT_TAGGING, AMZ_OBJECT_TAGGING_LOWER, AMZ_STORAGE_CLASS,
|
||||
AMZ_TAGGING_DIRECTIVE_LOWER, AMZ_WEBSITE_REDIRECT_LOCATION, is_amz_header, is_minio_header, is_rustfs_header,
|
||||
is_standard_header, is_storageclass_header,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
SUFFIX_FORCE_DELETE, SUFFIX_SOURCE_DELETEMARKER, SUFFIX_SOURCE_ETAG, SUFFIX_SOURCE_MTIME, SUFFIX_SOURCE_PROXY_REQUEST,
|
||||
@@ -132,6 +135,36 @@ impl From<&BucketTarget> for RemoteS3EndpointSpec {
|
||||
|
||||
pub type HeadObjectSdkError = Box<SdkError<HeadObjectError>>;
|
||||
|
||||
/// Captures the `x-amz-tagging` response header of a replication HEAD, which
|
||||
/// the SDK drops while deserializing `HeadObjectOutput`.
|
||||
#[derive(Clone, Debug, Default)]
|
||||
struct HeadObjectTaggingCapture(Arc<std::sync::Mutex<Option<String>>>);
|
||||
|
||||
impl HeadObjectTaggingCapture {
|
||||
fn take(&self) -> Option<String> {
|
||||
self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner).take()
|
||||
}
|
||||
}
|
||||
|
||||
impl Intercept for HeadObjectTaggingCapture {
|
||||
fn name(&self) -> &'static str {
|
||||
"HeadObjectTaggingCapture"
|
||||
}
|
||||
|
||||
fn read_before_deserialization(
|
||||
&self,
|
||||
context: &BeforeDeserializationInterceptorContextRef<'_>,
|
||||
_runtime_components: &RuntimeComponents,
|
||||
_cfg: &mut ConfigBag,
|
||||
) -> Result<(), aws_sdk_s3::error::BoxError> {
|
||||
// Overwrite on every attempt so a retried HEAD reports the tags of the
|
||||
// response that was actually deserialized.
|
||||
*self.0.lock().unwrap_or_else(std::sync::PoisonError::into_inner) =
|
||||
context.response().headers().get(AMZ_OBJECT_TAGGING_LOWER).map(str::to_string);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Whether an edited bucket target still addresses the same remote service
|
||||
/// (endpoint, bucket, path style, TLS and identity), so a verdict learned
|
||||
/// about that service stays valid across the edit.
|
||||
@@ -2089,12 +2122,19 @@ impl TargetClient {
|
||||
// object with an identical ETag, and the worker concludes the object
|
||||
// already converged — so it never actually replicates it.
|
||||
insert_header(&mut headers, SUFFIX_SOURCE_PROXY_REQUEST, "false");
|
||||
self.client
|
||||
// Ask for the tag set itself (MinIO `x-amz-tagging-directive: ACCESS`):
|
||||
// a plain HEAD reports only the tag count, which cannot tell the
|
||||
// replica's tags from the source's.
|
||||
headers.insert(HeaderName::from_static(AMZ_TAGGING_DIRECTIVE_LOWER), HeaderValue::from_static("ACCESS"));
|
||||
let target_tagging = HeadObjectTaggingCapture::default();
|
||||
let mut output = self
|
||||
.client
|
||||
.head_object()
|
||||
.bucket(bucket)
|
||||
.key(object)
|
||||
.set_version_id(resolve_read_api_version_id(version_id))
|
||||
.customize()
|
||||
.interceptor(target_tagging.clone())
|
||||
.map_request(move |mut req| {
|
||||
for (k, v) in headers.clone().into_iter() {
|
||||
if let Some(key_str) = k.map(|k| k.as_str().to_string()) {
|
||||
@@ -2106,7 +2146,17 @@ impl TargetClient {
|
||||
})
|
||||
.send()
|
||||
.await
|
||||
.map_err(Box::new)
|
||||
.map_err(Box::new)?;
|
||||
// The SDK's typed output has no field for the tag set; carry it where
|
||||
// the replication comparison reads it. A target that ignores the
|
||||
// directive leaves the key absent, which compares as "tags unknown".
|
||||
if let Some(tagging) = target_tagging.take() {
|
||||
output
|
||||
.metadata
|
||||
.get_or_insert_default()
|
||||
.insert(AMZ_OBJECT_TAGGING.to_string(), tagging);
|
||||
}
|
||||
Ok(output)
|
||||
}
|
||||
|
||||
/// Candidate replicas by content identity on a target that mints its own
|
||||
@@ -4542,6 +4592,66 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
struct TaggingHeadConnector {
|
||||
target_tagging: Option<&'static str>,
|
||||
tagging_directives: Arc<std::sync::Mutex<Vec<Option<String>>>>,
|
||||
}
|
||||
|
||||
impl SmithyHttpConnector for TaggingHeadConnector {
|
||||
fn call(&self, request: HttpRequest) -> HttpConnectorFuture {
|
||||
self.tagging_directives
|
||||
.lock()
|
||||
.expect("recorded directive lock should not be poisoned")
|
||||
.push(request.headers().get(AMZ_TAGGING_DIRECTIVE_LOWER).map(str::to_string));
|
||||
let mut response = HttpResponse::new(
|
||||
aws_smithy_runtime_api::http::StatusCode::try_from(200_u16).expect("200 should be a valid response status"),
|
||||
SdkBody::empty(),
|
||||
);
|
||||
response.headers_mut().insert("x-amz-meta-owner", "team-a");
|
||||
// Both targets expose the standard count, including the older target
|
||||
// that does not implement the optional tag-set response header.
|
||||
response.headers_mut().insert(rustfs_utils::http::AMZ_TAG_COUNT, "2");
|
||||
if let Some(tagging) = self.target_tagging {
|
||||
response.headers_mut().insert(AMZ_OBJECT_TAGGING_LOWER, tagging);
|
||||
}
|
||||
HttpConnectorFuture::ready(Ok(response))
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn head_object_requests_and_reports_the_target_tag_set() {
|
||||
for target_tagging in [Some("env=prod&team=a"), None] {
|
||||
let tagging_directives = Arc::new(std::sync::Mutex::new(Vec::new()));
|
||||
let connector = SharedHttpConnector::new(TaggingHeadConnector {
|
||||
target_tagging,
|
||||
tagging_directives: Arc::clone(&tagging_directives),
|
||||
});
|
||||
let http_client = http_client_fn(move |_settings, _components| connector.clone());
|
||||
let (mut client, _) = recording_target_client();
|
||||
client.client = Arc::new(s3_client_for_test(443, Some(http_client)));
|
||||
|
||||
let head = client
|
||||
.head_object("target-bucket", "object", None)
|
||||
.await
|
||||
.expect("canned HEAD should succeed");
|
||||
|
||||
assert_eq!(
|
||||
*tagging_directives
|
||||
.lock()
|
||||
.expect("recorded directive lock should not be poisoned"),
|
||||
vec![Some("ACCESS".to_string())],
|
||||
"the replication HEAD must ask the target for its tag set"
|
||||
);
|
||||
assert_eq!(head.tag_count(), Some(2));
|
||||
let metadata = head.metadata.expect("user metadata should be parsed");
|
||||
assert_eq!(metadata.get("owner").map(String::as_str), Some("team-a"));
|
||||
// Absent means "tags unknown": the comparison then keeps treating a
|
||||
// tagged replica as different instead of trusting the count alone.
|
||||
assert_eq!(metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str), target_tagging);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn put_object_sends_source_version_id_query_to_target() {
|
||||
// MinIO reads the replicated version only from the `versionId` query
|
||||
|
||||
@@ -115,9 +115,9 @@ use tracing::{debug, error, warn};
|
||||
use uuid::Uuid;
|
||||
use xxhash_rust::xxh64;
|
||||
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
|
||||
const EVENT_LIFECYCLE_WORKER_STATE: &str = "lifecycle_worker_state";
|
||||
pub(super) const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
pub(super) const LOG_SUBSYSTEM_LIFECYCLE: &str = "lifecycle";
|
||||
pub(super) const EVENT_LIFECYCLE_WORKER_STATE: &str = "lifecycle_worker_state";
|
||||
const EVENT_LIFECYCLE_TRANSITION_COMPENSATION: &str = "lifecycle_transition_compensation";
|
||||
const EVENT_LIFECYCLE_STALE_MULTIPART_CLEANUP: &str = "lifecycle_stale_multipart_cleanup";
|
||||
const METRIC_STALE_MULTIPART_CLEANUP_BLOCKED_TOTAL: &str = "rustfs_stale_multipart_cleanup_blocked_total";
|
||||
@@ -3082,6 +3082,7 @@ async fn recover_configured_tier_free_versions(
|
||||
next_object_marker: None,
|
||||
scanned_entries: 0,
|
||||
buckets_scanned: 0,
|
||||
unreadable_objects: 0,
|
||||
truncated: false,
|
||||
});
|
||||
}
|
||||
@@ -3221,6 +3222,7 @@ async fn run_tier_free_version_recovery_loop<F, Fut>(
|
||||
scanned = stats.scanned,
|
||||
scanned_entries = stats.scanned_entries,
|
||||
buckets_scanned = stats.buckets_scanned,
|
||||
unreadable_objects = stats.unreadable_objects,
|
||||
enqueued = stats.enqueued,
|
||||
failed = stats.failed,
|
||||
truncated = stats.truncated,
|
||||
@@ -6565,6 +6567,7 @@ mod tests {
|
||||
next_object_marker: truncated.then(|| "object".to_string()),
|
||||
scanned_entries: 1,
|
||||
buckets_scanned: 1,
|
||||
unreadable_objects: 0,
|
||||
truncated,
|
||||
}
|
||||
}
|
||||
@@ -14818,6 +14821,37 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
async fn seed_corrupt_free_version(disk_paths: &[PathBuf], bucket: &str, object: &str) -> Vec<u8> {
|
||||
use rustfs_utils::http::metadata_compat::{SUFFIX_TRANSITIONED_VERSION_STATE, insert_bytes};
|
||||
|
||||
seed_recoverable_free_version(disk_paths, bucket, object, None, None).await;
|
||||
let path = PathBuf::from(bucket).join(object).join(STORAGE_FORMAT_FILE);
|
||||
let original = fs::read(disk_paths[0].join(&path))
|
||||
.await
|
||||
.expect("read the seeded free version");
|
||||
let mut metadata = FileMeta::load(&original).expect("load the seeded metadata");
|
||||
let mut version = metadata.versions[0].parse_version_meta().expect("decode the seeded version");
|
||||
let marker = version.delete_marker.as_mut().expect("the free version is a delete marker");
|
||||
insert_bytes(&mut marker.meta_sys, SUFFIX_TRANSITIONED_VERSION_STATE, b"invalid".to_vec());
|
||||
metadata.versions[0] = version
|
||||
.try_into()
|
||||
.expect("encode the malformed marker with a consistent header");
|
||||
let encoded = metadata.marshal_msg().expect("encode the metadata with a valid checksum");
|
||||
let reloaded = FileMeta::load(&encoded).expect("the metadata envelope must remain readable");
|
||||
assert_eq!(
|
||||
reloaded
|
||||
.get_file_info_versions(bucket, object, false)
|
||||
.expect_err("decoding the marker must fail"),
|
||||
rustfs_filemeta::Error::FileCorrupt
|
||||
);
|
||||
for disk in disk_paths {
|
||||
fs::write(disk.join(&path), &encoded)
|
||||
.await
|
||||
.expect("persist the same malformed marker on every disk");
|
||||
}
|
||||
encoded
|
||||
}
|
||||
|
||||
async fn remove_seeded_free_version(disk_paths: &[PathBuf], bucket: &str, object: &str) {
|
||||
for disk_path in disk_paths {
|
||||
fs::remove_dir_all(disk_path.join(bucket).join(object))
|
||||
@@ -14890,6 +14924,7 @@ mod tests {
|
||||
next_object_marker: None,
|
||||
scanned_entries: 0,
|
||||
buckets_scanned: 0,
|
||||
unreadable_objects: 0,
|
||||
truncated: false,
|
||||
}
|
||||
);
|
||||
@@ -15086,12 +15121,14 @@ mod tests {
|
||||
);
|
||||
assert_eq!(check.reasons.len(), 1);
|
||||
assert_eq!(check.reasons[0].kind, crate::services::tier::tier::TierDeletionReasonKind::CleanupPending);
|
||||
seed_corrupt_free_version(&disks, &bucket, "corrupt").await;
|
||||
let report = retry_tier_free_version_cleanup(api.clone(), "WARM")
|
||||
.await
|
||||
.expect("retry should enqueue durable cleanup");
|
||||
assert_eq!(report.status, TierCleanupRetryStatus::Accepted);
|
||||
assert_eq!(report.enqueued, 1);
|
||||
assert_eq!(report.failed, 0);
|
||||
assert_eq!(report.unreadable_objects, 1);
|
||||
let rx = Arc::clone(&state.read().await.tasks_rx[0]);
|
||||
let task = rx.lock().await.try_recv().expect("cleanup was queued").expect("queue item");
|
||||
let task = task
|
||||
@@ -15111,21 +15148,27 @@ mod tests {
|
||||
let other = retry_tier_free_version_cleanup(api.clone(), "OTHER")
|
||||
.await
|
||||
.expect("other tier retry should be bounded");
|
||||
assert_eq!(other.status, TierCleanupRetryStatus::Accepted);
|
||||
assert_eq!(other.enqueued, 0, "retry must not enqueue a different tier's marker");
|
||||
assert_eq!(other.failed, 0);
|
||||
let json = serde_json::to_value(&other).expect("serialize the public retry report");
|
||||
assert_eq!(json["unreadableObjects"], 1, "unreadable keys cannot be attributed to a tier");
|
||||
assert!(rx.lock().await.try_recv().is_err());
|
||||
api.tier_config_mgr()
|
||||
.write()
|
||||
.await
|
||||
.remove("OTHER", true)
|
||||
.await
|
||||
.expect("remove other mock tier");
|
||||
reset_runtime_expiry_state(&api).await;
|
||||
let failed = retry_tier_free_version_cleanup(api.clone(), "WARM")
|
||||
.await
|
||||
.expect("queue failure remains retryable");
|
||||
assert_eq!(failed.failed, 1);
|
||||
assert_eq!(failed.enqueued, 0);
|
||||
assert_eq!(failed.unreadable_objects, 1);
|
||||
assert_eq!(crate::services::tier::test_util::free_version_count(&disks[0], &bucket, "owned").await, 1);
|
||||
remove_seeded_free_version(&disks, &bucket, "corrupt").await;
|
||||
api.tier_config_mgr()
|
||||
.write()
|
||||
.await
|
||||
.remove("OTHER", true)
|
||||
.await
|
||||
.expect("remove other mock tier");
|
||||
remove_seeded_free_version(&disks, &bucket, "owned").await;
|
||||
api.tier_config_mgr()
|
||||
.write()
|
||||
@@ -15341,6 +15384,121 @@ mod tests {
|
||||
reset_runtime_expiry_state(&ecstore).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_enqueues_past_a_corrupt_free_version() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let prefix = format!("zzzz-recovery-corrupt-{}", Uuid::new_v4().simple());
|
||||
let first_bucket = format!("{prefix}-a");
|
||||
let second_bucket = format!("{prefix}-b");
|
||||
for bucket in [&first_bucket, &second_bucket] {
|
||||
create_test_bucket(&ecstore, bucket).await;
|
||||
}
|
||||
let healthy = [
|
||||
(first_bucket.as_str(), "a"),
|
||||
(first_bucket.as_str(), "c"),
|
||||
(second_bucket.as_str(), "d"),
|
||||
];
|
||||
for (bucket, object) in healthy {
|
||||
seed_recoverable_free_version(&disk_paths, bucket, object, None, None).await;
|
||||
}
|
||||
let corrupt = seed_corrupt_free_version(&disk_paths, &first_bucket, "b").await;
|
||||
let state = install_unconsumed_runtime_expiry_worker(&ecstore, healthy.len()).await;
|
||||
|
||||
let stats = recover_tier_free_versions_with_cancel(
|
||||
Arc::clone(&ecstore),
|
||||
healthy.len() + 1,
|
||||
Some(first_bucket.clone()),
|
||||
None,
|
||||
CancellationToken::new(),
|
||||
)
|
||||
.await
|
||||
.expect("a corrupt free version must not discard other recovery candidates");
|
||||
assert!(!stats.truncated);
|
||||
assert_eq!(stats.enqueued, healthy.len());
|
||||
let receiver = Arc::clone(&state.read().await.tasks_rx[0]);
|
||||
let mut receiver = receiver.lock().await;
|
||||
for (bucket, object) in healthy {
|
||||
let task = receiver
|
||||
.try_recv()
|
||||
.expect("receive a healthy cleanup task")
|
||||
.expect("the queue carries a task");
|
||||
let task = task
|
||||
.as_any()
|
||||
.downcast_ref::<FreeVersionTask>()
|
||||
.expect("a free-version cleanup task");
|
||||
assert_eq!((task.0.bucket.as_str(), task.0.name.as_str()), (bucket, object));
|
||||
}
|
||||
drop(receiver);
|
||||
reset_runtime_expiry_state(&ecstore).await;
|
||||
for disk in &disk_paths {
|
||||
let stored = fs::read(disk.join(&first_bucket).join("b").join(STORAGE_FORMAT_FILE))
|
||||
.await
|
||||
.expect("discovery must preserve the unreadable marker");
|
||||
assert_eq!(stored, corrupt);
|
||||
}
|
||||
for (bucket, object) in healthy.into_iter().chain([(first_bucket.as_str(), "b")]) {
|
||||
remove_seeded_free_version(&disk_paths, bucket, object).await;
|
||||
}
|
||||
for bucket in [&first_bucket, &second_bucket] {
|
||||
ecstore
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete the empty test bucket");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_counts_a_corrupt_boundary_once_per_sweep() {
|
||||
let (disk_paths, ecstore) = setup_test_env().await;
|
||||
let bucket = format!("zzzz-recovery-boundary-{}", Uuid::new_v4().simple());
|
||||
create_test_bucket(&ecstore, &bucket).await;
|
||||
seed_corrupt_free_version(&disk_paths, &bucket, "b").await;
|
||||
seed_recoverable_free_version(&disk_paths, &bucket, "c", None, None).await;
|
||||
let page_from = |marker| {
|
||||
list_tier_free_versions_with_budget(
|
||||
Arc::clone(&ecstore),
|
||||
2,
|
||||
Some(bucket.clone()),
|
||||
marker,
|
||||
CancellationToken::new(),
|
||||
RecoveryWorkBudget {
|
||||
max_objects: 1,
|
||||
max_buckets: 1,
|
||||
},
|
||||
)
|
||||
};
|
||||
|
||||
let first = page_from(None)
|
||||
.await
|
||||
.expect("the unreadable key should consume the first page");
|
||||
assert!(first.items.is_empty());
|
||||
assert_eq!(first.unreadable_objects, 1);
|
||||
assert!(first.truncated);
|
||||
assert_eq!(first.next_bucket_marker.as_deref(), Some(bucket.as_str()));
|
||||
assert_eq!(first.next_object_marker.as_deref(), Some("b"));
|
||||
|
||||
let second = page_from(first.next_object_marker)
|
||||
.await
|
||||
.expect("resume past the unreadable boundary");
|
||||
assert_eq!(second.unreadable_objects, 0);
|
||||
assert_eq!(second.items.len(), 1);
|
||||
assert_eq!(second.items[0].name, "c");
|
||||
|
||||
let next_sweep = page_from(None).await.expect("a later sweep must revisit the unreadable key");
|
||||
assert_eq!(next_sweep.unreadable_objects, 1);
|
||||
assert_eq!(next_sweep.next_object_marker.as_deref(), Some("b"));
|
||||
|
||||
for object in ["b", "c"] {
|
||||
remove_seeded_free_version(&disk_paths, &bucket, object).await;
|
||||
}
|
||||
ecstore
|
||||
.delete_bucket(&bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete the empty test bucket");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn tier_free_version_recovery_real_enqueue_failure_retries_same_object() {
|
||||
|
||||
@@ -20,7 +20,9 @@ use std::time::Duration;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::enqueue_recovered_free_version_with_cancel;
|
||||
use crate::bucket::lifecycle::bucket_lifecycle_ops::{
|
||||
EVENT_LIFECYCLE_WORKER_STATE, LOG_COMPONENT_ECSTORE, LOG_SUBSYSTEM_LIFECYCLE, enqueue_recovered_free_version_with_cancel,
|
||||
};
|
||||
use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::Result;
|
||||
use crate::object_api::ObjectInfo;
|
||||
@@ -28,9 +30,10 @@ use crate::services::tier::tier::{ERR_TIER_INVALID_CONFIG, TierConfigMgr};
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::storage_api_contracts::{
|
||||
bucket::{BucketOperations, BucketOptions},
|
||||
list::{ListOperations as _, StorageObjectInfoOrErr, StorageWalkOptions},
|
||||
list::StorageWalkOptions,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::list_objects::TierRecoveryEntry;
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_lock::error::LockError;
|
||||
use rustfs_s3_client::admin_handler_utils::AdminError;
|
||||
@@ -45,7 +48,6 @@ const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
#[cfg(test)]
|
||||
const BACKGROUND_WALK_SHUTDOWN_TIMEOUT: Duration = Duration::from_millis(100);
|
||||
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, crate::error::Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&FileInfo) -> bool>;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
@@ -143,6 +145,8 @@ pub struct TierCleanupRetryReport {
|
||||
pub scanned: usize,
|
||||
pub enqueued: usize,
|
||||
pub failed: usize,
|
||||
/// Unreadable logical objects encountered in this pass, whose tier is unknown.
|
||||
pub unreadable_objects: usize,
|
||||
pub scan_truncated: bool,
|
||||
}
|
||||
|
||||
@@ -161,6 +165,7 @@ impl TierCleanupRetryReport {
|
||||
scanned: 0,
|
||||
enqueued: 0,
|
||||
failed: 0,
|
||||
unreadable_objects: 0,
|
||||
scan_truncated: false,
|
||||
}
|
||||
}
|
||||
@@ -267,6 +272,7 @@ async fn retry_tier_free_version_cleanup_page(
|
||||
.map_err(|_| tier_cleanup_retry_error())?;
|
||||
let mut report = TierCleanupRetryReport::new(tier_name, TierCleanupRetryStatus::Accepted);
|
||||
report.scanned = page.scanned_entries;
|
||||
report.unreadable_objects = page.unreadable_objects;
|
||||
report.scan_truncated = page.truncated;
|
||||
for object in page.items {
|
||||
if cancel.is_cancelled() {
|
||||
@@ -290,6 +296,7 @@ pub struct FreeVersionRecoveryStats {
|
||||
pub next_object_marker: Option<String>,
|
||||
pub scanned_entries: usize,
|
||||
pub buckets_scanned: usize,
|
||||
pub unreadable_objects: usize,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
@@ -300,6 +307,7 @@ pub struct FreeVersionRecoveryPage {
|
||||
pub next_object_marker: Option<String>,
|
||||
pub scanned_entries: usize,
|
||||
pub buckets_scanned: usize,
|
||||
pub unreadable_objects: usize,
|
||||
pub truncated: bool,
|
||||
}
|
||||
|
||||
@@ -343,6 +351,7 @@ pub(super) async fn recover_tier_free_versions_with_cancel(
|
||||
next_object_marker: page.next_object_marker,
|
||||
scanned_entries: page.scanned_entries,
|
||||
buckets_scanned: page.buckets_scanned,
|
||||
unreadable_objects: page.unreadable_objects,
|
||||
truncated: page.truncated,
|
||||
};
|
||||
|
||||
@@ -472,6 +481,7 @@ async fn list_tier_free_versions_for_tier(
|
||||
next_object_marker: None,
|
||||
scanned_entries: 0,
|
||||
buckets_scanned: 0,
|
||||
unreadable_objects: 0,
|
||||
truncated: false,
|
||||
};
|
||||
|
||||
@@ -505,6 +515,7 @@ async fn list_tier_free_versions_for_tier(
|
||||
let mut bucket_seen = bucket_marker.is_none();
|
||||
let mut truncated_after: Option<RecoveryCursor> = None;
|
||||
let mut remaining_scan_objects = work_budget.max_objects;
|
||||
let mut first_unreadable = None;
|
||||
|
||||
for bucket in buckets {
|
||||
if cancel_token.is_cancelled() {
|
||||
@@ -535,7 +546,7 @@ async fn list_tier_free_versions_for_tier(
|
||||
};
|
||||
let bucket_walk_limit = remaining_scan_objects;
|
||||
|
||||
let (tx, mut rx) = mpsc::channel::<ObjectInfoOrErr>(100);
|
||||
let (tx, mut rx) = mpsc::channel::<Result<TierRecoveryEntry>>(100);
|
||||
let cancel = cancel_token.child_token();
|
||||
let mut draining_after_truncation = false;
|
||||
let mut drain_deadline = None;
|
||||
@@ -559,14 +570,7 @@ async fn list_tier_free_versions_for_tier(
|
||||
match action {
|
||||
RecoveryWalkTestAction::SendItems(items) => {
|
||||
for item in items {
|
||||
if tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: Some(item),
|
||||
err: None,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
if tx.send(Ok(TierRecoveryEntry::Version(Box::new(item)))).await.is_err() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -574,35 +578,16 @@ async fn list_tier_free_versions_for_tier(
|
||||
}
|
||||
RecoveryWalkTestAction::SendItemsThenError(items, err) => {
|
||||
for item in items {
|
||||
if tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: Some(item),
|
||||
err: None,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
if tx.send(Ok(TierRecoveryEntry::Version(Box::new(item)))).await.is_err() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
let _ = tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: None,
|
||||
err: Some(err),
|
||||
})
|
||||
.await;
|
||||
let _ = tx.send(Err(err)).await;
|
||||
return Ok(());
|
||||
}
|
||||
RecoveryWalkTestAction::SendItemsThenHang(items, started) => {
|
||||
for item in items {
|
||||
if tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: Some(item),
|
||||
err: None,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
if tx.send(Ok(TierRecoveryEntry::Version(Box::new(item)))).await.is_err() {
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
@@ -614,14 +599,11 @@ async fn list_tier_free_versions_for_tier(
|
||||
let mut index = 0usize;
|
||||
loop {
|
||||
if tx
|
||||
.send(ObjectInfoOrErr {
|
||||
item: Some(ObjectInfo {
|
||||
bucket: bucket_name.clone(),
|
||||
name: format!("nonrecoverable-{index:08}"),
|
||||
..Default::default()
|
||||
}),
|
||||
err: None,
|
||||
})
|
||||
.send(Ok(TierRecoveryEntry::Version(Box::new(ObjectInfo {
|
||||
bucket: bucket_name.clone(),
|
||||
name: format!("nonrecoverable-{index:08}"),
|
||||
..Default::default()
|
||||
}))))
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
@@ -639,7 +621,7 @@ async fn list_tier_free_versions_for_tier(
|
||||
}
|
||||
}
|
||||
|
||||
api.walk(cancel, &bucket_name, "", tx, recovery_walk_options(bucket_walk_limit, object_marker))
|
||||
api.walk_for_tier_recovery(cancel, &bucket_name, tx, recovery_walk_options(bucket_walk_limit, object_marker))
|
||||
.await
|
||||
}
|
||||
});
|
||||
@@ -669,21 +651,25 @@ async fn list_tier_free_versions_for_tier(
|
||||
},
|
||||
};
|
||||
page.scanned_entries += 1;
|
||||
if let Some(err) = item.err {
|
||||
cancel.cancel();
|
||||
receive_error = Some(err);
|
||||
break;
|
||||
}
|
||||
let entry = match item {
|
||||
Ok(entry) => entry,
|
||||
Err(err) => {
|
||||
cancel.cancel();
|
||||
receive_error = Some(err);
|
||||
break;
|
||||
}
|
||||
};
|
||||
if draining_after_truncation {
|
||||
continue;
|
||||
}
|
||||
let Some(oi) = item.item else {
|
||||
continue;
|
||||
let name = match &entry {
|
||||
TierRecoveryEntry::Version(oi) => oi.name.as_str(),
|
||||
TierRecoveryEntry::Unreadable { object } => object.as_str(),
|
||||
};
|
||||
if last_seen_object.as_deref() != Some(oi.name.as_str()) && scanned_objects >= bucket_walk_limit {
|
||||
if last_seen_object.as_deref() != Some(name) && scanned_objects >= bucket_walk_limit {
|
||||
// The disk listing limit follows S3-visible counting rules and
|
||||
// does not charge metadata containing only hidden/free versions.
|
||||
// Enforce the repair budget again at the decoded-object boundary.
|
||||
// Charge the recovery budget by key, including unreadable metadata.
|
||||
page.truncated = true;
|
||||
page.next_bucket_marker = Some(bucket.name.clone());
|
||||
page.next_object_marker = last_seen_object.clone();
|
||||
@@ -692,9 +678,9 @@ async fn list_tier_free_versions_for_tier(
|
||||
drain_deadline = Some(tokio::time::Instant::now() + BACKGROUND_WALK_SHUTDOWN_TIMEOUT);
|
||||
continue;
|
||||
}
|
||||
record_scanned_object(&mut last_seen_object, &mut scanned_objects, &oi.name);
|
||||
record_scanned_object(&mut last_seen_object, &mut scanned_objects, name);
|
||||
if let Some(cursor) = &truncated_after
|
||||
&& (cursor.bucket.as_str() != bucket.name.as_str() || cursor.object.as_str() != oi.name.as_str())
|
||||
&& (cursor.bucket.as_str() != bucket.name.as_str() || cursor.object.as_str() != name)
|
||||
{
|
||||
page.truncated = true;
|
||||
cancel.cancel();
|
||||
@@ -702,6 +688,14 @@ async fn list_tier_free_versions_for_tier(
|
||||
drain_deadline = Some(tokio::time::Instant::now() + BACKGROUND_WALK_SHUTDOWN_TIMEOUT);
|
||||
continue;
|
||||
}
|
||||
let oi = match entry {
|
||||
TierRecoveryEntry::Version(oi) => *oi,
|
||||
TierRecoveryEntry::Unreadable { object } => {
|
||||
page.unreadable_objects += 1;
|
||||
first_unreadable.get_or_insert_with(|| (bucket.name.clone(), object));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
if is_recoverable_tier_free_version_for_tier(&oi, tier_name) {
|
||||
let current_cursor = RecoveryCursor {
|
||||
bucket: bucket.name.clone(),
|
||||
@@ -756,6 +750,18 @@ async fn list_tier_free_versions_for_tier(
|
||||
page.next_object_marker = None;
|
||||
}
|
||||
|
||||
if let Some((bucket, object)) = first_unreadable {
|
||||
tracing::warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
state = "free_version_recovery_unreadable_metadata",
|
||||
unreadable_objects = page.unreadable_objects,
|
||||
sample_bucket = ?bucket,
|
||||
sample_object = ?object,
|
||||
"Retained unreadable objects during tier free-version recovery"
|
||||
);
|
||||
}
|
||||
Ok(page)
|
||||
}
|
||||
|
||||
@@ -823,6 +829,7 @@ mod tests {
|
||||
next_object_marker: None,
|
||||
scanned_entries: 1,
|
||||
buckets_scanned: 1,
|
||||
unreadable_objects: 0,
|
||||
truncated: false,
|
||||
};
|
||||
|
||||
@@ -843,6 +850,7 @@ mod tests {
|
||||
next_object_marker: Some("object".to_string()),
|
||||
scanned_entries: 1,
|
||||
buckets_scanned: 1,
|
||||
unreadable_objects: 0,
|
||||
truncated: false,
|
||||
};
|
||||
|
||||
@@ -863,6 +871,7 @@ mod tests {
|
||||
next_object_marker: Some("object-a".to_string()),
|
||||
scanned_entries: 1,
|
||||
buckets_scanned: 1,
|
||||
unreadable_objects: 0,
|
||||
truncated: false,
|
||||
};
|
||||
let mut cursor = RetryCursor::new(stats.next_bucket_marker.clone(), stats.next_object_marker.clone());
|
||||
@@ -973,6 +982,7 @@ mod tests {
|
||||
next_object_marker: None,
|
||||
scanned_entries: DEFAULT_FREE_VERSION_RECOVERY_SCAN_LIMIT,
|
||||
buckets_scanned: 1,
|
||||
unreadable_objects: 0,
|
||||
truncated: false,
|
||||
};
|
||||
|
||||
|
||||
@@ -27,6 +27,7 @@ use crate::disk::RUSTFS_META_BUCKET;
|
||||
use crate::error::{Error, Result, is_err_bucket_not_found, is_err_strict_volume_not_found};
|
||||
use crate::object_api::ObjectOptions;
|
||||
use crate::runtime::sources as runtime_sources;
|
||||
use crate::runtime::usage_watch::UsageWatchMutationGuard;
|
||||
use crate::storage_api_contracts::heal::HealOperations as _;
|
||||
use crate::storage_api_contracts::namespace::NamespaceLocking as _;
|
||||
use crate::store::{ECStore, await_bucket_namespace_operation};
|
||||
@@ -270,7 +271,12 @@ pub async fn init_bucket_metadata_sys(api: Arc<ECStore>, buckets: Vec<String>) {
|
||||
|
||||
let sys = Arc::new(RwLock::new(sys));
|
||||
|
||||
instance_ctx.init_bucket_metadata_sys(sys.clone());
|
||||
{
|
||||
// Installing a whole cache can also withdraw buckets absent from its
|
||||
// initial listing. Fence replacement of the instance's previous cache.
|
||||
let _watch = instance_ctx.usage_watches.begin(None);
|
||||
instance_ctx.init_bucket_metadata_sys(sys.clone());
|
||||
}
|
||||
|
||||
if is_dist_erasure {
|
||||
start_refresh_buckets_metadata_loop(sys);
|
||||
@@ -1955,6 +1961,46 @@ pub(crate) async fn has_authoritative_never_versioned_state(bucket: &str) -> Res
|
||||
bucket_meta_sys.has_authoritative_never_versioned_state(bucket).await
|
||||
}
|
||||
|
||||
/// Hold the shared bucket config transaction through an operator's inspection and
|
||||
/// discard. Read the authoritative disk snapshot under that lock without migrating
|
||||
/// historical metadata or accepting a stale notification cache.
|
||||
pub(crate) async fn legacy_discard_config_guard_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
incarnation: Uuid,
|
||||
) -> Result<(rustfs_lock::NamespaceLockGuard, bool)> {
|
||||
let sys = bucket_metadata_sys_of(ctx)?.read().await.clone();
|
||||
let guard =
|
||||
acquire_metadata_transaction_guard(sys.object_store(), bucket, "delete_residue_config", rustfs_lock::LockType::Shared)
|
||||
.await?;
|
||||
// The caller already holds the bucket lifecycle fence. The config transaction
|
||||
// guard excludes writers while this scoped read and the eventual discard run.
|
||||
let authority = sys.read_authoritative_metadata_from_disk_under_guard(bucket, &guard).await?;
|
||||
let eligible = match authority {
|
||||
BucketMetadataAuthority::Authoritative(metadata)
|
||||
if metadata.bucket_incarnation_sidecar && metadata.bucket_incarnation_id == incarnation =>
|
||||
{
|
||||
authoritative_metadata_never_versioned(bucket, &metadata)?
|
||||
&& matches!(
|
||||
object_lock_config_state_from_authoritative_metadata(&metadata)?,
|
||||
ObjectLockConfigState::ConfirmedAbsent
|
||||
)
|
||||
}
|
||||
_ => false,
|
||||
};
|
||||
if guard.is_lock_lost() {
|
||||
return Err(Error::PreconditionFailed);
|
||||
}
|
||||
Ok((guard, eligible))
|
||||
}
|
||||
|
||||
fn authoritative_metadata_never_versioned(bucket: &str, metadata: &BucketMetadata) -> Result<bool> {
|
||||
if let Some(raw_len) = metadata.xml_config_unreadable_len(super::metadata::BUCKET_VERSIONING_CONFIG) {
|
||||
return Err(unreadable_config_error(bucket, super::metadata::BUCKET_VERSIONING_CONFIG, raw_len));
|
||||
}
|
||||
Ok(metadata.versioning_config.is_none() && metadata.versioning_config_xml.is_empty())
|
||||
}
|
||||
|
||||
pub(crate) async fn has_authoritative_never_versioned_state_in(
|
||||
ctx: &crate::runtime::instance::InstanceContext,
|
||||
bucket: &str,
|
||||
@@ -2063,6 +2109,37 @@ pub(crate) fn current_local_meta_mutation_gen() -> u64 {
|
||||
LOCAL_META_MUTATION_GEN.load(std::sync::atomic::Ordering::Acquire)
|
||||
}
|
||||
|
||||
// These are the cached inputs used by the scanner's lifecycle, retention and
|
||||
// replication decisions. Raw bytes distinguish absent/unreadable records;
|
||||
// typed values also cover publications of already-parsed metadata. A periodic
|
||||
// reload of the same content or an unrelated config edit preserves the watch.
|
||||
fn same_scanner_configuration(previous: &BucketMetadata, next: &BucketMetadata) -> bool {
|
||||
fn target_resets(metadata: &BucketMetadata) -> impl Iterator<Item = (&str, &str, Option<OffsetDateTime>)> {
|
||||
metadata.bucket_target_config.iter().flat_map(|targets| {
|
||||
targets
|
||||
.targets
|
||||
.iter()
|
||||
.map(|target| (target.arn.as_str(), target.reset_id.as_str(), target.reset_before_date))
|
||||
})
|
||||
}
|
||||
|
||||
previous.bucket_incarnation_id == next.bucket_incarnation_id
|
||||
&& previous.bucket_incarnation_sidecar == next.bucket_incarnation_sidecar
|
||||
&& previous.bucket_creation_committed == next.bucket_creation_committed
|
||||
&& previous.lock_enabled == next.lock_enabled
|
||||
&& previous.lifecycle_config_xml == next.lifecycle_config_xml
|
||||
&& previous.lifecycle_config == next.lifecycle_config
|
||||
&& previous.object_lock_config_xml == next.object_lock_config_xml
|
||||
&& previous.object_lock_config == next.object_lock_config
|
||||
&& previous.versioning_config_xml == next.versioning_config_xml
|
||||
&& previous.versioning_config == next.versioning_config
|
||||
&& previous.replication_config_xml == next.replication_config_xml
|
||||
&& previous.replication_config == next.replication_config
|
||||
&& previous.bucket_targets_config_json == next.bucket_targets_config_json
|
||||
&& previous.bucket_targets_unreadable() == next.bucket_targets_unreadable()
|
||||
&& target_resets(previous).eq(target_resets(next))
|
||||
}
|
||||
|
||||
type LocalAuthorityCache = Arc<StdMutex<HashMap<String, (u64, Arc<BucketMetadata>)>>>;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@@ -2147,6 +2224,33 @@ impl BucketMetadataSys {
|
||||
self.api.upgrade()
|
||||
}
|
||||
|
||||
/// The bucket publish lock precedes the metadata map write lock, then the
|
||||
/// instance's watch registry. A caller keeps the returned guard until
|
||||
/// dependent caches (including replication targets) have also been updated.
|
||||
fn replace_cached_metadata(
|
||||
&self,
|
||||
map: &mut HashMap<String, Arc<BucketMetadata>>,
|
||||
bucket: &str,
|
||||
next: Option<Arc<BucketMetadata>>,
|
||||
) -> (bool, Option<UsageWatchMutationGuard>) {
|
||||
let changed = match (map.get(bucket), next.as_deref()) {
|
||||
(Some(previous), Some(next)) => !same_scanner_configuration(previous, next),
|
||||
(None, None) => false,
|
||||
_ => true,
|
||||
};
|
||||
let watch = if changed {
|
||||
self.object_store_if_live()
|
||||
.map(|store| store.ctx.usage_watches.begin(Some(bucket)))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let previous = match next {
|
||||
Some(next) => map.insert(bucket.to_owned(), next),
|
||||
None => map.remove(bucket),
|
||||
};
|
||||
(previous.is_some(), watch)
|
||||
}
|
||||
|
||||
/// Whether the backing `ECStore` has been dropped.
|
||||
///
|
||||
/// Used by `InstanceContext::clear_bucket_metadata_sys_if_stale` during
|
||||
@@ -2354,7 +2458,7 @@ impl BucketMetadataSys {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "stale bucket metadata removal")
|
||||
.await?;
|
||||
let removed = self.metadata_map.write().await.remove(bucket).is_some();
|
||||
let (removed, _watch) = self.replace_cached_metadata(&mut *self.metadata_map.write().await, bucket, None);
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.insert(bucket.to_string(), ()).await;
|
||||
if removed {
|
||||
@@ -2393,7 +2497,8 @@ impl BucketMetadataSys {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, namespace_guard, "initial bucket metadata publish")
|
||||
.await?;
|
||||
self.metadata_map.write().await.insert(bucket.to_string(), Arc::clone(&bm));
|
||||
let (_, _watch) =
|
||||
self.replace_cached_metadata(&mut *self.metadata_map.write().await, bucket, Some(Arc::clone(&bm)));
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &bm).await;
|
||||
@@ -2438,7 +2543,7 @@ impl BucketMetadataSys {
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
let removed = map.remove(bucket).is_some();
|
||||
let (removed, _watch) = self.replace_cached_metadata(&mut map, bucket, None);
|
||||
drop(map);
|
||||
self.fabricated_metadata.write().await.insert(bucket.to_string());
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
@@ -2462,7 +2567,7 @@ impl BucketMetadataSys {
|
||||
if !unchanged {
|
||||
return Ok(());
|
||||
}
|
||||
map.insert(bucket.to_string(), Arc::clone(&metadata));
|
||||
let (_, _watch) = self.replace_cached_metadata(&mut map, bucket, Some(Arc::clone(&metadata)));
|
||||
drop(map);
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
@@ -2491,7 +2596,7 @@ impl BucketMetadataSys {
|
||||
let publish_lock = self.metadata_publish_lock(&bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
let mut map = self.metadata_map.write().await;
|
||||
map.insert(bucket.clone(), bm.clone());
|
||||
let (_, _watch) = self.replace_cached_metadata(&mut map, &bucket, Some(Arc::clone(&bm)));
|
||||
drop(map);
|
||||
self.fabricated_metadata.write().await.remove(&bucket);
|
||||
self.missing_buckets.invalidate(&bucket).await;
|
||||
@@ -2512,7 +2617,7 @@ impl BucketMetadataSys {
|
||||
let publish_lock = self.metadata_publish_lock(bucket);
|
||||
let _publish_guard = publish_lock.lock().await;
|
||||
let mut map = self.metadata_map.write().await;
|
||||
let removed = map.remove(bucket).is_some();
|
||||
let (removed, _watch) = self.replace_cached_metadata(&mut map, bucket, None);
|
||||
drop(map);
|
||||
let removed_fabricated = self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.insert(bucket.to_string(), ()).await;
|
||||
@@ -2528,6 +2633,7 @@ impl BucketMetadataSys {
|
||||
async fn _reset(&mut self) {
|
||||
bump_local_meta_mutation_gen();
|
||||
let mut map = self.metadata_map.write().await;
|
||||
let _watch = self.object_store_if_live().map(|store| store.ctx.usage_watches.begin(None));
|
||||
map.clear();
|
||||
drop(map);
|
||||
self.fabricated_metadata.write().await.clear();
|
||||
@@ -2849,7 +2955,7 @@ impl BucketMetadataSys {
|
||||
if let Some(current) = map.get(bucket) {
|
||||
return Ok((Arc::clone(current), true));
|
||||
}
|
||||
map.insert(bucket.to_string(), bm.clone());
|
||||
let (_, _watch) = self.replace_cached_metadata(&mut map, bucket, Some(Arc::clone(&bm)));
|
||||
drop(map);
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
@@ -2905,11 +3011,7 @@ impl BucketMetadataSys {
|
||||
return Ok(false);
|
||||
};
|
||||
|
||||
if let Some(raw_len) = metadata.xml_config_unreadable_len(super::metadata::BUCKET_VERSIONING_CONFIG) {
|
||||
return Err(unreadable_config_error(bucket, super::metadata::BUCKET_VERSIONING_CONFIG, raw_len));
|
||||
}
|
||||
|
||||
Ok(metadata.versioning_config.is_none() && metadata.versioning_config_xml.is_empty())
|
||||
authoritative_metadata_never_versioned(bucket, &metadata)
|
||||
}
|
||||
|
||||
pub async fn get_bucket_policy(&self, bucket: &str) -> Result<(BucketPolicy, OffsetDateTime)> {
|
||||
@@ -3167,10 +3269,8 @@ impl BucketMetadataSys {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, &namespace_guard, "bucket creation commitment publish")
|
||||
.await?;
|
||||
self.metadata_map
|
||||
.write()
|
||||
.await
|
||||
.insert(bucket.to_string(), Arc::clone(&metadata));
|
||||
let (_, _watch) =
|
||||
self.replace_cached_metadata(&mut *self.metadata_map.write().await, bucket, Some(Arc::clone(&metadata)));
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
Ok(BucketMetadataAuthority::Authoritative(metadata))
|
||||
@@ -3225,7 +3325,7 @@ impl BucketMetadataSys {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, &namespace_guard, "missing legacy bucket metadata publish")
|
||||
.await?;
|
||||
self.metadata_map.write().await.remove(bucket);
|
||||
let (_, _watch) = self.replace_cached_metadata(&mut *self.metadata_map.write().await, bucket, None);
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.insert(bucket.to_string(), ()).await;
|
||||
return Ok(BucketMetadataAuthority::MissingBucket);
|
||||
@@ -3296,10 +3396,8 @@ impl BucketMetadataSys {
|
||||
let _publish_guard = self
|
||||
.lock_metadata_publish(bucket, &namespace_guard, "legacy bucket metadata publish")
|
||||
.await?;
|
||||
self.metadata_map
|
||||
.write()
|
||||
.await
|
||||
.insert(bucket.to_string(), Arc::clone(&metadata));
|
||||
let (_, _watch) =
|
||||
self.replace_cached_metadata(&mut *self.metadata_map.write().await, bucket, Some(Arc::clone(&metadata)));
|
||||
self.fabricated_metadata.write().await.remove(bucket);
|
||||
self.missing_buckets.invalidate(bucket).await;
|
||||
sync_bucket_target_sys(bucket, &metadata).await;
|
||||
@@ -5545,6 +5643,185 @@ mod tests {
|
||||
(dirs, node_a, node_b)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_watch_metadata_comparison_covers_scanner_inputs_and_skips_unrelated_config() {
|
||||
let original = BucketMetadata::new("scanner-config");
|
||||
for config in [
|
||||
BUCKET_LIFECYCLE_CONFIG,
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
OBJECT_LOCK_CONFIG,
|
||||
BUCKET_REPLICATION_CONFIG,
|
||||
] {
|
||||
let (_, bytes) = NEW_WRITER_CONFIGS
|
||||
.iter()
|
||||
.find(|(file, _)| *file == config)
|
||||
.expect("scanner config fixture");
|
||||
let mut parsed = original.clone();
|
||||
parsed.update_config(config, bytes.to_vec()).expect("valid scanner config");
|
||||
parsed.parse_all_configs().expect("parse scanner config");
|
||||
assert!(!same_scanner_configuration(&original, &parsed), "{config} must fence the watch");
|
||||
let mut unreadable = parsed.clone();
|
||||
match config {
|
||||
BUCKET_LIFECYCLE_CONFIG => unreadable.lifecycle_config = None,
|
||||
BUCKET_VERSIONING_CONFIG => unreadable.versioning_config = None,
|
||||
OBJECT_LOCK_CONFIG => unreadable.object_lock_config = None,
|
||||
BUCKET_REPLICATION_CONFIG => unreadable.replication_config = None,
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
!same_scanner_configuration(&parsed, &unreadable),
|
||||
"{config} parser state must fence the watch"
|
||||
);
|
||||
}
|
||||
|
||||
let mut changed = original.clone();
|
||||
changed.lock_enabled = !original.lock_enabled;
|
||||
assert!(!same_scanner_configuration(&original, &changed));
|
||||
let mut changed = original.clone();
|
||||
changed.bucket_target_config = Some(BucketTargets {
|
||||
targets: vec![BucketTarget {
|
||||
arn: "arn:replica".into(),
|
||||
reset_id: "resync".into(),
|
||||
..Default::default()
|
||||
}],
|
||||
});
|
||||
assert!(
|
||||
!same_scanner_configuration(&original, &changed),
|
||||
"in-memory replication resync targets changed"
|
||||
);
|
||||
let mut unrelated = original.clone();
|
||||
unrelated.tagging_config_xml = b"<Tagging/>".to_vec();
|
||||
unrelated.lifecycle_config_updated_at = OffsetDateTime::UNIX_EPOCH;
|
||||
assert!(same_scanner_configuration(&original, &unrelated));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn periodic_metadata_refresh_fences_usage_watch_after_missed_peer_notification() {
|
||||
use crate::bucket::versioning::VersioningApi as _;
|
||||
use crate::layout::endpoints::EndpointServerPools;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::runtime::usage_watch::{ScannerUsageWatchKnown, ScannerUsageWatchRequest};
|
||||
use crate::store::init_local_disks_with_instance_ctx;
|
||||
|
||||
async fn observe(store: &ECStore, request: &ScannerUsageWatchRequest) -> ScannerUsageWatchKnown {
|
||||
store.scanner_usage_watch(std::slice::from_ref(request)).await[0]
|
||||
.known
|
||||
.clone()
|
||||
.expect("a healthy peer must return a bucket watch")
|
||||
}
|
||||
|
||||
let (_dirs, writer) = isolated_store_over_temp_disks().await;
|
||||
let endpoints = EndpointServerPools::from(vec![writer.pools[0].endpoints.clone()]);
|
||||
let peer_ctx = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&peer_ctx, endpoints.clone())
|
||||
.await
|
||||
.expect("initialize the second instance's disks");
|
||||
let peer = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("test address"),
|
||||
endpoints,
|
||||
CancellationToken::new(),
|
||||
peer_ctx,
|
||||
)
|
||||
.await
|
||||
.expect("open the same deployment with an independent instance");
|
||||
assert_eq!(writer.id, peer.id);
|
||||
assert_ne!(writer.scanner_usage_watch_session_id(), peer.scanner_usage_watch_session_id());
|
||||
init_bucket_metadata_sys(writer.clone(), Vec::new()).await;
|
||||
let bucket = "missed-config-notification";
|
||||
writer
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let other = "unchanged-watch-bucket";
|
||||
writer
|
||||
.make_bucket(other, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create control bucket");
|
||||
init_bucket_metadata_sys(peer.clone(), vec![bucket.to_owned(), other.to_owned()]).await;
|
||||
let peer_sys = peer.ctx.bucket_metadata_sys().expect("peer metadata cache");
|
||||
let writer_sys = writer.ctx.bucket_metadata_sys().expect("writer metadata cache");
|
||||
assert!(!Arc::ptr_eq(&peer_sys, &writer_sys));
|
||||
let (initial, _) = peer_sys
|
||||
.read()
|
||||
.await
|
||||
.get_config(bucket)
|
||||
.await
|
||||
.expect("cache original metadata on peer");
|
||||
assert!(initial.versioning_config_xml.is_empty());
|
||||
let request = ScannerUsageWatchRequest {
|
||||
bucket: bucket.into(),
|
||||
bucket_incarnation: peer
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("bucket incarnation"),
|
||||
watch_id: None,
|
||||
session_id: None,
|
||||
release: false,
|
||||
};
|
||||
let control = ScannerUsageWatchRequest {
|
||||
bucket: other.into(),
|
||||
bucket_incarnation: peer
|
||||
.bucket_incarnation_id_from_disk(other)
|
||||
.await
|
||||
.expect("control incarnation"),
|
||||
..request.clone()
|
||||
};
|
||||
let first = observe(&peer, &request).await;
|
||||
let unaffected = observe(&peer, &control).await;
|
||||
let enabled = br#"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>"#;
|
||||
writer_sys
|
||||
.read()
|
||||
.await
|
||||
.update(bucket, BUCKET_VERSIONING_CONFIG, enabled.to_vec())
|
||||
.await
|
||||
.expect("persist versioning");
|
||||
assert!(
|
||||
writer_sys
|
||||
.read()
|
||||
.await
|
||||
.get_versioning_config(bucket)
|
||||
.await
|
||||
.expect("writer versioning")
|
||||
.0
|
||||
.enabled()
|
||||
);
|
||||
assert!(Arc::ptr_eq(
|
||||
&initial,
|
||||
&peer_sys.read().await.get(bucket).await.expect("stale cached snapshot")
|
||||
));
|
||||
assert!(observe(&peer, &request).await.unchanged_since(&first), "the peer missed the notification");
|
||||
|
||||
refresh_buckets_metadata_once(peer_sys.clone()).await;
|
||||
let installed = peer_sys.read().await.get(bucket).await.expect("refreshed cached snapshot");
|
||||
assert!(installed.versioning_config.as_ref().is_some_and(|config| config.enabled()));
|
||||
let refreshed = observe(&peer, &request).await;
|
||||
assert!(
|
||||
!refreshed.unchanged_since(&first),
|
||||
"installing changed scanner configuration must fence that peer's old watch"
|
||||
);
|
||||
assert!(observe(&peer, &control).await.unchanged_since(&unaffected));
|
||||
refresh_buckets_metadata_once(peer_sys.clone()).await;
|
||||
assert!(
|
||||
observe(&peer, &request).await.unchanged_since(&refreshed),
|
||||
"an unchanged periodic reload must preserve progress"
|
||||
);
|
||||
|
||||
assert!(peer_sys.read().await.remove(bucket).await);
|
||||
assert!(!observe(&peer, &request).await.unchanged_since(&refreshed));
|
||||
assert!(observe(&peer, &control).await.unchanged_since(&unaffected));
|
||||
peer_sys.write().await._reset().await;
|
||||
let old_control = ScannerUsageWatchRequest {
|
||||
watch_id: Some(unaffected.watch_id),
|
||||
session_id: Some(unaffected.session_id),
|
||||
..control
|
||||
};
|
||||
assert!(
|
||||
peer.scanner_usage_watch(&[old_control]).await[0].known.is_none(),
|
||||
"reset retires all cached policy proofs"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn disk_incarnation_read_detects_stale_cache_until_peer_reload() {
|
||||
|
||||
@@ -19,7 +19,7 @@ use super::{
|
||||
use crate::bucket::metadata_sys::{
|
||||
BucketMetadataSys, update, update_if_incarnation, update_quota_with_sys_if_incarnation_and_revision,
|
||||
};
|
||||
use crate::data_usage::{get_bucket_usage_memory, reload_unknown_bucket_usage};
|
||||
use crate::data_usage::{get_bucket_usage_for_statistics, get_bucket_usage_memory, reload_unknown_bucket_usage};
|
||||
use crate::error::StorageError;
|
||||
use rustfs_config::QUOTA_CONFIG_FILE;
|
||||
use rustfs_scanner_metrics::metrics::Metric;
|
||||
@@ -96,7 +96,7 @@ impl QuotaChecker {
|
||||
// reported as absent instead of failing the write (rustfs/backlog#2904).
|
||||
// The snapshot protocol still admits from this value and fails closed.
|
||||
let current_usage = if uses_durable_reservations {
|
||||
self.report_only_usage(bucket).await
|
||||
Self::report_usage(bucket, get_bucket_usage_memory(bucket).await)
|
||||
} else {
|
||||
Some(self.get_real_time_usage(bucket).await?)
|
||||
};
|
||||
@@ -235,8 +235,12 @@ impl QuotaChecker {
|
||||
Ok(updated_at)
|
||||
}
|
||||
|
||||
/// Return quota configuration with optional scanner usage, independently
|
||||
/// of the usage requirements enforced by write admission.
|
||||
/// Return the configured limit and best-effort usage for read-only statistics.
|
||||
/// Single-process reports reflect known deletes before quota reconciliation;
|
||||
/// distributed reports read the persisted scanner baseline.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error for a missing bucket or unreadable quota configuration.
|
||||
pub async fn get_quota_stats(&self, bucket: &str) -> Result<(BucketQuota, Option<u64>), QuotaError> {
|
||||
// If bucket doesn't exist, return ConfigNotFound error
|
||||
if !self.bucket_exists(bucket).await {
|
||||
@@ -246,16 +250,12 @@ impl QuotaChecker {
|
||||
}
|
||||
|
||||
let quota = self.get_quota_config(bucket).await?;
|
||||
let current_usage = self.report_only_usage(bucket).await;
|
||||
let current_usage = Self::report_usage(bucket, get_bucket_usage_for_statistics(bucket).await);
|
||||
|
||||
Ok((quota, current_usage))
|
||||
}
|
||||
|
||||
/// Best-effort usage for statistics and durable admission reports.
|
||||
/// Missing scanner coverage is unknown, not an error or a zero-sized bucket.
|
||||
async fn report_only_usage(&self, bucket: &str) -> Option<u64> {
|
||||
// The plain lookup: a report must not wait for an early snapshot reload.
|
||||
let usage = get_bucket_usage_memory(bucket).await;
|
||||
fn report_usage(bucket: &str, usage: Option<u64>) -> Option<u64> {
|
||||
if usage.is_none() {
|
||||
metrics::counter!(METRIC_QUOTA_USAGE_REPORT_UNAVAILABLE_TOTAL).increment(1);
|
||||
debug!(
|
||||
|
||||
@@ -353,6 +353,41 @@ async fn quota_statistics_remain_available_without_a_complete_scanner_snapshot()
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_statistics_exclude_local_deletes_while_snapshot_admission_retains_them() {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
let bucket = format!("quota-delete-report-{}", Uuid::new_v4().simple());
|
||||
create_bucket_with_quota_json(&store, &bucket, legacy_quota_json(100)).await;
|
||||
// Keep the snapshot-protocol control from initiating a background migration.
|
||||
let metadata_sys = BucketMetadataSys::new(store.clone());
|
||||
let (metadata, _) = metadata_sys.get_config(&bucket).await.expect("load quota metadata");
|
||||
metadata_sys.set(bucket.clone(), metadata).await;
|
||||
let checker = QuotaChecker::new(Arc::new(RwLock::new(metadata_sys)));
|
||||
install_reporting_snapshot(store.clone(), &bucket, "complete", 60).await;
|
||||
crate::data_usage::record_bucket_object_write_memory(&bucket, None, 40).await;
|
||||
|
||||
for (deleted, expected) in [(60, 40), (40, 0)] {
|
||||
crate::data_usage::record_bucket_object_delete_memory(&bucket, deleted, true).await;
|
||||
let (quota, usage) = checker.get_quota_stats(&bucket).await.expect("read quota statistics");
|
||||
assert_eq!(quota.quota, Some(100));
|
||||
assert_eq!(usage, Some(expected), "read-only quota GETs must report the surviving bytes");
|
||||
|
||||
let check = checker
|
||||
.check_quota(&bucket, QuotaOperation::PutObject, 1)
|
||||
.await
|
||||
.expect("check snapshot quota");
|
||||
assert_eq!(check.current_usage, Some(100));
|
||||
assert!(!check.allowed, "reading statistics must not release held bytes for writes");
|
||||
}
|
||||
|
||||
crate::data_usage::prepare_bucket_usage_for_namespace_change(&bucket, None)
|
||||
.await
|
||||
.expect("fixture cache cleanup");
|
||||
crate::data_usage::invalidate_data_usage_snapshot_cache().await;
|
||||
stop_reporting_fixture(&store).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn quota_statistics_preserve_configuration_errors() {
|
||||
let (_dirs, store, _) = isolated_durable_quota_store().await;
|
||||
|
||||
@@ -1602,6 +1602,7 @@ fn shard_object(ledger_object: &str, shard_index: u16, bucket_incarnation: Uuid,
|
||||
}
|
||||
|
||||
fn shard_index(object: &str) -> u16 {
|
||||
let object = rustfs_utils::path::encode_dir_object_ref(object);
|
||||
let digest = Sha256::digest(object.as_bytes());
|
||||
u16::from_be_bytes([digest[0], digest[1]]) % SHARDED_LEDGER_COUNT
|
||||
}
|
||||
@@ -3306,6 +3307,35 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn quota_shard_reconciliation_matches_the_directory_object_write() {
|
||||
use crate::object_api::PutObjReader;
|
||||
use crate::storage_api_contracts::object::ObjectIO;
|
||||
|
||||
let (_directories, store) = metadata_sys::test_support::isolated_store_over_temp_disks().await;
|
||||
metadata_sys::init_bucket_metadata_sys(Arc::clone(&store), Vec::new()).await;
|
||||
let bucket = "quota-directory-marker";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create test bucket");
|
||||
let listed_name = "photos/";
|
||||
let stored_name = rustfs_utils::path::encode_dir_object(listed_name);
|
||||
let mut body = PutObjReader::from_vec(vec![1, 2, 3]);
|
||||
store
|
||||
.put_object(bucket, listed_name, &mut body, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("write a directory object with billable bytes");
|
||||
|
||||
let usages = exact_shard_usages(&store, bucket)
|
||||
.await
|
||||
.expect("reconcile the version listing");
|
||||
assert_eq!(usages.iter().sum::<u64>(), 3);
|
||||
assert_eq!(usages[usize::from(shard_index(&stored_name))], 3);
|
||||
assert_eq!(shard_index(listed_name), shard_index(&stored_name));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn sharded_reservation_persists_a_populated_lazy_shard_before_refill() {
|
||||
|
||||
@@ -52,9 +52,9 @@ use super::replication_resync_boundary::{
|
||||
#[cfg(test)]
|
||||
use super::replication_resync_boundary::{RESYNC_META_FORMAT, RESYNC_META_VERSION, WIRE_ZERO_TIME_UNIX};
|
||||
use super::replication_storage_boundary::{
|
||||
AdvancedGetOptions, EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions,
|
||||
ObjectToDelete, ReplicationObjectIO, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, ReplicationStorage,
|
||||
StatObjectOptions, StorageObjectInfoOrErr, WalkOptions,
|
||||
EcstoreObjectOperations, GetObjectReader, HTTPPreconditions, HTTPRangeSpec, ObjectInfo, ObjectOptions, ObjectToDelete,
|
||||
ReplicationObjectIO, ReplicationStatusWritebackCondition, ReplicationStatusWritebackMode, ReplicationStorage,
|
||||
StorageObjectInfoOrErr, WalkOptions,
|
||||
};
|
||||
#[cfg(test)]
|
||||
use super::replication_storage_boundary::{NamespaceLockFence, NamespaceLockSignalTestFence, ReplicationDeletedObject};
|
||||
@@ -67,8 +67,8 @@ use super::replication_target_boundary::{
|
||||
replication_complete_multipart_options, replication_delete_marker_purge_remove_options, replication_delete_remove_options,
|
||||
replication_force_delete_remove_options, replication_object_is_ssec_encrypted, replication_put_object_header_size,
|
||||
replication_put_object_options, replication_source_etag_is_comparable, replication_target_head_is_newer_null_version,
|
||||
resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate, version_identity_capability_from_put,
|
||||
version_identity_drifted,
|
||||
replication_target_tags_match, resolve_read_api_version_id, ssec_passthrough_evidence_present, ssec_passthrough_gate,
|
||||
version_identity_capability_from_put, version_identity_drifted,
|
||||
};
|
||||
use super::replication_versioning_boundary::ReplicationVersioningStore;
|
||||
use super::runtime_boundary as runtime_sources;
|
||||
@@ -87,9 +87,9 @@ use metrics::counter;
|
||||
use rmp_serde;
|
||||
use rustfs_s3_types::EventName;
|
||||
use rustfs_utils::http::{
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, AMZ_TAGGING_DIRECTIVE,
|
||||
SUFFIX_REPLICATION_RESET, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX, has_internal_suffix,
|
||||
insert_str, replication_target_versions,
|
||||
AMZ_OBJECT_LOCK_LEGAL_HOLD, AMZ_OBJECT_LOCK_MODE, AMZ_OBJECT_LOCK_RETAIN_UNTIL_DATE, SUFFIX_REPLICATION_RESET,
|
||||
SUFFIX_REPLICATION_RESET_ARN_PREFIX, SUFFIX_REPLICATION_STATUS, SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX,
|
||||
has_internal_suffix, insert_str, replication_target_versions,
|
||||
};
|
||||
use rustfs_utils::{DEFAULT_SIP_HASH_KEY, get_env_usize, sip_hash};
|
||||
#[cfg(test)]
|
||||
@@ -850,7 +850,8 @@ fn user_defined_value<'a>(user_defined: &'a HashMap<String, String>, name: &str)
|
||||
/// difference (retention is only ever extended here — shortening or clearing
|
||||
/// it needs a governance bypass the replication client does not hold, so the
|
||||
/// replica keeps its longer window and the difference is logged). Heal and
|
||||
/// resync attempts only touch what the HEAD proves diverged.
|
||||
/// resync attempts apply known differences. An existing-object reset also
|
||||
/// reconciles a nonempty tag set whose values the target HEAD omits.
|
||||
async fn sync_drifted_replica_metadata(
|
||||
tgt_client: &TargetClient,
|
||||
object: &str,
|
||||
@@ -863,10 +864,15 @@ async fn sync_drifted_replica_metadata(
|
||||
let mut applied = false;
|
||||
|
||||
let source_tags = ReplicationTagFilter::decode_tags_to_map(&source.user_tags);
|
||||
let source_tag_count = i32::try_from(source_tags.len()).unwrap_or(i32::MAX);
|
||||
if op_type == ReplicationType::Metadata || replica.tag_count.unwrap_or_default() != source_tag_count {
|
||||
let replica_tag_count = replica.tag_count.unwrap_or_default();
|
||||
let update_tags = match op_type {
|
||||
ReplicationType::Metadata => true,
|
||||
ReplicationType::ExistingObject => !replication_target_tags_match(&source_tags, replica_tag_count, replica.metadata()),
|
||||
_ => usize::try_from(replica_tag_count) != Ok(source_tags.len()),
|
||||
};
|
||||
if update_tags {
|
||||
if source_tags.is_empty() {
|
||||
if replica.tag_count.unwrap_or_default() > 0 {
|
||||
if replica_tag_count > 0 || op_type == ReplicationType::ExistingObject {
|
||||
tgt_client
|
||||
.delete_object_tagging(&tgt_client.bucket, object, version_id.clone())
|
||||
.await
|
||||
@@ -4116,6 +4122,17 @@ fn replication_status_writeback_mode(state_update_needed: bool) -> ReplicationSt
|
||||
}
|
||||
}
|
||||
|
||||
/// Source-object metadata a replication worker publishes with its terminal
|
||||
/// status.
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct ReplicationStatusWriteback<'a> {
|
||||
replication_internal: Option<&'a String>,
|
||||
/// `(arn, version id the target assigned)` for targets minting their own ids.
|
||||
target_version_ledger: &'a [(String, String)],
|
||||
/// `(arn, resync stamp)` for targets this attempt resynced.
|
||||
resync_stamps: &'a [(String, String)],
|
||||
}
|
||||
|
||||
/// Publish a worker's status with a storage-enforced compare-and-set token.
|
||||
/// `put_object_metadata` checks the token while it owns the object write lock,
|
||||
/// avoiding both a read/write race and any need to hold a hot object lock
|
||||
@@ -4123,12 +4140,16 @@ fn replication_status_writeback_mode(state_update_needed: bool) -> ReplicationSt
|
||||
fn replication_status_writeback_options(
|
||||
roi: &ReplicateObjectInfo,
|
||||
replication_lock_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
new_replication_internal: Option<&String>,
|
||||
target_version_ledger: &[(String, String)],
|
||||
writeback: ReplicationStatusWriteback<'_>,
|
||||
mode: ReplicationStatusWritebackMode,
|
||||
) -> ObjectOptions {
|
||||
let ReplicationStatusWriteback {
|
||||
replication_internal,
|
||||
target_version_ledger,
|
||||
resync_stamps,
|
||||
} = writeback;
|
||||
let mut eval_metadata = HashMap::new();
|
||||
if let Some(status) = new_replication_internal {
|
||||
if let Some(status) = replication_internal {
|
||||
insert_str(&mut eval_metadata, SUFFIX_REPLICATION_STATUS, status.clone());
|
||||
}
|
||||
// One durable key per drifting target: the version id it assigned to this
|
||||
@@ -4141,6 +4162,12 @@ fn replication_status_writeback_options(
|
||||
assigned_version_id.clone(),
|
||||
);
|
||||
}
|
||||
// The per-target existing-object resync stamp. Without it the object keeps
|
||||
// answering "not yet resynced" for the target's reset id, and the scanner
|
||||
// re-queues it on every cycle.
|
||||
for (arn, stamp) in resync_stamps {
|
||||
insert_str(&mut eval_metadata, &format!("{SUFFIX_REPLICATION_RESET_ARN_PREFIX}{arn}"), stamp.clone());
|
||||
}
|
||||
let mut write_opts = ObjectOptions {
|
||||
version_id: roi.version_id.map(|version_id| version_id.to_string()),
|
||||
eval_metadata: Some(eval_metadata),
|
||||
@@ -4162,13 +4189,11 @@ async fn persist_replication_state_if_current<S: ReplicationStorage>(
|
||||
roi: &ReplicateObjectInfo,
|
||||
storage: &Arc<S>,
|
||||
replication_lock_guard: &rustfs_lock::NamespaceLockGuard,
|
||||
new_replication_internal: Option<&String>,
|
||||
target_version_ledger: &[(String, String)],
|
||||
writeback: ReplicationStatusWriteback<'_>,
|
||||
mode: ReplicationStatusWritebackMode,
|
||||
object_info: &mut ObjectInfo,
|
||||
) -> Result<ReplicationStatePersistOutcome> {
|
||||
let write_opts =
|
||||
replication_status_writeback_options(roi, replication_lock_guard, new_replication_internal, target_version_ledger, mode);
|
||||
let write_opts = replication_status_writeback_options(roi, replication_lock_guard, writeback, mode);
|
||||
match storage.put_object_metadata(&roi.bucket, &roi.name, &write_opts).await {
|
||||
Ok(updated) => {
|
||||
*object_info = updated;
|
||||
@@ -4334,6 +4359,12 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
.filter(|target| !target.is_empty())
|
||||
.filter_map(|target| Some((target.arn.clone(), target.target_version_id.clone()?)))
|
||||
.collect();
|
||||
let resync_stamps: Vec<(String, String)> = rinfos
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|target| !target.is_empty() && !target.resync_timestamp.is_empty())
|
||||
.map(|target| (target.arn.clone(), target.resync_timestamp.clone()))
|
||||
.collect();
|
||||
let state_update_needed = roi.replication_status_internal != new_replication_internal
|
||||
|| rinfos.replication_resynced()
|
||||
|| !target_version_ledger.is_empty();
|
||||
@@ -4343,8 +4374,11 @@ pub(crate) async fn replicate_object_with_outcome<S: ReplicationStorage>(
|
||||
&roi,
|
||||
&storage,
|
||||
&obj_lock_guard,
|
||||
new_replication_internal.as_ref(),
|
||||
&target_version_ledger,
|
||||
ReplicationStatusWriteback {
|
||||
replication_internal: new_replication_internal.as_ref(),
|
||||
target_version_ledger: &target_version_ledger,
|
||||
resync_stamps: &resync_stamps,
|
||||
},
|
||||
writeback_mode,
|
||||
&mut object_info,
|
||||
)
|
||||
@@ -5005,8 +5039,6 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
let _sopts = replicate_all_stat_options(&object_info, &bucket, &tgt_client);
|
||||
|
||||
let Some((replication_action, object_info)) = resolve_replicate_all_action(
|
||||
ReplicateAllActionContext {
|
||||
siblings: &storage,
|
||||
@@ -5090,6 +5122,13 @@ impl ReplicateObjectInfoExt for ReplicateObjectInfo {
|
||||
return rinfo;
|
||||
}
|
||||
|
||||
// The transfer is this target's resync of the object. Stamp it like the
|
||||
// already-converged outcome above, or the next scanner cycle admits the
|
||||
// object for the same reset id again.
|
||||
if self.op_type == ReplicationType::ExistingObject && !tgt_client.reset_id.is_empty() {
|
||||
apply_replication_resync_timestamp(&mut rinfo, &tgt_client.reset_id);
|
||||
}
|
||||
|
||||
rinfo
|
||||
}
|
||||
|
||||
@@ -5238,34 +5277,6 @@ fn note_replicate_all_target_bucket_empty(bucket: &str, tgt_client: &TargetClien
|
||||
});
|
||||
}
|
||||
|
||||
/// Build the stat options for the target metadata comparison, logging (without
|
||||
/// failing) when the tagging directive header cannot be set.
|
||||
fn replicate_all_stat_options(object_info: &ObjectInfo, bucket: &str, tgt_client: &TargetClient) -> StatObjectOptions {
|
||||
let mut sopts = StatObjectOptions {
|
||||
version_id: object_info.version_id.map(|v| v.to_string()).unwrap_or_default(),
|
||||
internal: AdvancedGetOptions {
|
||||
replication_proxy_request: "false".to_string(),
|
||||
..Default::default()
|
||||
},
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if let Err(err) = sopts.set(AMZ_TAGGING_DIRECTIVE, "ACCESS") {
|
||||
debug!(
|
||||
event = EVENT_RESYNC_RUNTIME_SKIPPED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
|
||||
bucket = %bucket,
|
||||
arn = %tgt_client.arn,
|
||||
error = %err,
|
||||
reason = "tagging_directive_header_invalid",
|
||||
"Skipped replication tagging directive header detail"
|
||||
);
|
||||
}
|
||||
|
||||
sopts
|
||||
}
|
||||
|
||||
/// Record a failed payload transfer: mark the outcome FAILED, log the target
|
||||
/// operation failure, and take the target offline when the error is a network
|
||||
/// failure.
|
||||
@@ -5465,7 +5476,15 @@ async fn resolve_replicate_all_action(
|
||||
// records the sync without running the PUT.
|
||||
replication_action = ReplicationAction::None;
|
||||
} else {
|
||||
replication_action = if replica_matches {
|
||||
// A current-version fallback may omit its version ID. Avoid
|
||||
// mutating an unpinned version's tags or certifying stale tags.
|
||||
replication_action = if replica_matches
|
||||
&& (roi.op_type != ReplicationType::ExistingObject
|
||||
|| replication_target_tags_match(
|
||||
&ReplicationTagFilter::decode_tags_to_map(&object_info.user_tags),
|
||||
oi.tag_count.unwrap_or_default(),
|
||||
oi.metadata(),
|
||||
)) {
|
||||
ReplicationAction::None
|
||||
} else {
|
||||
ReplicationAction::All
|
||||
@@ -6395,8 +6414,7 @@ mod tests {
|
||||
let opts = replication_status_writeback_options(
|
||||
&ReplicateObjectInfo::default(),
|
||||
&guard,
|
||||
None,
|
||||
&[],
|
||||
ReplicationStatusWriteback::default(),
|
||||
ReplicationStatusWritebackMode::ValidateOnly,
|
||||
);
|
||||
forced_lost.store(true, std::sync::atomic::Ordering::Release);
|
||||
@@ -6415,6 +6433,76 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn terminal_writeback_persists_resync_stamp_that_ends_existing_object_resync() {
|
||||
let lock = rustfs_lock::NamespaceLock::new(
|
||||
"replication-status-writeback-resync-stamp".to_string(),
|
||||
Arc::new(rustfs_lock::LocalClient::new()),
|
||||
);
|
||||
let guard = lock
|
||||
.get_write_lock(
|
||||
rustfs_lock::ObjectKey::new("bucket", "/[replicate]/object"),
|
||||
"worker-a",
|
||||
std::time::Duration::from_secs(2),
|
||||
)
|
||||
.await
|
||||
.expect("replication lock should be acquired");
|
||||
let arn = "arn:rustfs:replication::target:bucket";
|
||||
let reset_before = OffsetDateTime::now_utc();
|
||||
let mut rinfo = ReplicatedTargetInfo {
|
||||
arn: arn.to_string(),
|
||||
replication_status: ReplicationStatusType::Completed,
|
||||
..Default::default()
|
||||
};
|
||||
apply_replication_resync_timestamp(&mut rinfo, "reset-1");
|
||||
let resync_decision = |user_defined: &HashMap<String, String>, reset_id: &str| {
|
||||
super::super::replication_object_decision_boundary::resync_target_for_object(
|
||||
&super::super::replication_object_decision_boundary::ReplicationResyncTargetObject {
|
||||
mod_time: Some(reset_before - time::Duration::hours(1)),
|
||||
user_defined,
|
||||
},
|
||||
arn,
|
||||
reset_id,
|
||||
Some(reset_before),
|
||||
ReplicationStatusType::Completed,
|
||||
)
|
||||
.replicate
|
||||
};
|
||||
let persisted = |resync_stamps: &[(String, String)]| {
|
||||
replication_status_writeback_options(
|
||||
&ReplicateObjectInfo::default(),
|
||||
&guard,
|
||||
ReplicationStatusWriteback {
|
||||
resync_stamps,
|
||||
..Default::default()
|
||||
},
|
||||
ReplicationStatusWritebackMode::Update,
|
||||
)
|
||||
.eval_metadata
|
||||
.expect("writeback always carries evaluated metadata")
|
||||
};
|
||||
|
||||
let unstamped = persisted(&[]);
|
||||
assert!(
|
||||
resync_decision(&unstamped, "reset-1"),
|
||||
"an object older than the reset date stays admitted until it carries the stamp"
|
||||
);
|
||||
|
||||
let stamped = persisted(&[(rinfo.arn.clone(), rinfo.resync_timestamp.clone())]);
|
||||
let suffix = format!("{SUFFIX_REPLICATION_RESET_ARN_PREFIX}{arn}");
|
||||
for prefix in [
|
||||
rustfs_utils::http::RUSTFS_INTERNAL_PREFIX,
|
||||
rustfs_utils::http::MINIO_INTERNAL_PREFIX,
|
||||
] {
|
||||
assert_eq!(stamped.get(&format!("{prefix}{suffix}")), Some(&rinfo.resync_timestamp));
|
||||
}
|
||||
assert!(
|
||||
!resync_decision(&stamped, "reset-1"),
|
||||
"the persisted stamp must end the resync for the same reset id"
|
||||
);
|
||||
assert!(resync_decision(&stamped, "reset-2"), "a new reset id must admit the object again");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unavailable_object_target_is_persisted_as_failed() {
|
||||
let arn = "arn:object-target";
|
||||
@@ -8058,43 +8146,28 @@ mod tests {
|
||||
assert!(purge_backoff_active(&dobj));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_update_on_a_drifted_target_uses_the_tagging_api_instead_of_a_put() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(3, |line| {
|
||||
if line.starts_with("HEAD ") && line.contains(&format!("versionId={DRIFTED_ASSIGNED_VERSION_ID}")) {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nETag: \"{DRIFTED_ETAG}\"\r\nContent-Length: 4\r\nx-amz-version-id: {DRIFTED_ASSIGNED_VERSION_ID}\r\nx-amz-tagging-count: 1\r\nLast-Modified: Sun, 06 Sep 2026 10:00:00 GMT\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
} else if line.starts_with("HEAD ") {
|
||||
empty_response("404 Not Found")
|
||||
} else if line.starts_with("PUT ")
|
||||
&& line.contains("tagging")
|
||||
&& line.contains(&format!("versionId={DRIFTED_ASSIGNED_VERSION_ID}"))
|
||||
{
|
||||
empty_response("200 OK")
|
||||
} else {
|
||||
empty_response("500 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
async fn resolve_tagged_test_replica(
|
||||
target: &Arc<TargetClient>,
|
||||
op_type: ReplicationType,
|
||||
has_ledger: bool,
|
||||
) -> (Option<ReplicationAction>, ReplicatedTargetInfo) {
|
||||
let (mut roi, mut object_info) = drifted_roi_and_object();
|
||||
roi.op_type = ReplicationType::Metadata;
|
||||
roi.op_type = op_type;
|
||||
roi.user_tags = "phase=after".to_string();
|
||||
object_info.user_tags = Arc::new("phase=after".to_string());
|
||||
let mut user_defined = HashMap::new();
|
||||
insert_str(
|
||||
&mut user_defined,
|
||||
&format!("{SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX}{}", target.arn),
|
||||
DRIFTED_ASSIGNED_VERSION_ID.to_string(),
|
||||
);
|
||||
object_info.user_defined = Arc::new(user_defined);
|
||||
let mut rinfo = replicate_all_target_info(&roi, &target);
|
||||
|
||||
object_info.user_tags = Arc::new(roi.user_tags.clone());
|
||||
if has_ledger {
|
||||
insert_str(
|
||||
Arc::make_mut(&mut object_info.user_defined),
|
||||
&format!("{SUFFIX_REPLICATION_TARGET_VERSION_ARN_PREFIX}{}", target.arn),
|
||||
DRIFTED_ASSIGNED_VERSION_ID.to_string(),
|
||||
);
|
||||
}
|
||||
let mut rinfo = replicate_all_target_info(&roi, target);
|
||||
let action = resolve_replicate_all_action(
|
||||
ReplicateAllActionContext {
|
||||
siblings: &NoSiblingLedger,
|
||||
roi: &roi,
|
||||
tgt_client: &target,
|
||||
tgt_client: target,
|
||||
bucket: &roi.bucket,
|
||||
object: &roi.name,
|
||||
start_time: OffsetDateTime::now_utc(),
|
||||
@@ -8104,24 +8177,93 @@ mod tests {
|
||||
&mut rinfo,
|
||||
)
|
||||
.await;
|
||||
(action.map(|(action, _)| action), rinfo)
|
||||
}
|
||||
|
||||
assert!(
|
||||
matches!(action, Some((ReplicationAction::None, _))),
|
||||
"metadata is applied in place; the PUT transport must not run: {action:?}"
|
||||
);
|
||||
assert!(rinfo.error.is_none(), "{:?}", rinfo.error);
|
||||
assert!(
|
||||
rinfo.target_version_id.is_none(),
|
||||
"an id the ledger already records must not trigger another metadata writeback"
|
||||
);
|
||||
async fn assert_versioned_tagging_sync(
|
||||
op_type: ReplicationType,
|
||||
tag_header: Option<&'static str>,
|
||||
expect_write: bool,
|
||||
put_status: &'static str,
|
||||
) {
|
||||
let wrote = Arc::new(AtomicBool::new(false));
|
||||
let server_wrote = Arc::clone(&wrote);
|
||||
let (endpoint, server) = spawn_scripted_target_server(2 + usize::from(expect_write), move |line| {
|
||||
let assigned = line.contains(&format!("versionId={DRIFTED_ASSIGNED_VERSION_ID}"));
|
||||
if line.starts_with("HEAD ") && assigned {
|
||||
let tags = tag_header
|
||||
.map(|value| format!("x-amz-tagging: {value}\r\n"))
|
||||
.unwrap_or_default();
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nETag: \"{DRIFTED_ETAG}\"\r\nContent-Length: 4\r\nx-amz-version-id: {DRIFTED_ASSIGNED_VERSION_ID}\r\nx-amz-tagging-count: 1\r\n{tags}Last-Modified: Sun, 06 Sep 2026 10:00:00 GMT\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
} else if line.starts_with("HEAD ") {
|
||||
empty_response("404 Not Found")
|
||||
} else if line.starts_with("PUT ") && line.contains("tagging") && assigned {
|
||||
server_wrote.store(true, Ordering::Release);
|
||||
empty_response(put_status)
|
||||
} else {
|
||||
empty_response("400 Unexpected")
|
||||
}
|
||||
});
|
||||
|
||||
let mut target = test_target_client(endpoint);
|
||||
Arc::get_mut(&mut target).expect("unshared test target").reset_id = "reset-1".to_string();
|
||||
let (action, rinfo) = resolve_tagged_test_replica(&target, op_type, true).await;
|
||||
assert_eq!(wrote.load(Ordering::Acquire), expect_write, "{tag_header:?}");
|
||||
if put_status == "200 OK" {
|
||||
assert_eq!(action, Some(ReplicationAction::None), "the payload PUT must not run");
|
||||
assert!(rinfo.error.is_none(), "{:?}", rinfo.error);
|
||||
assert!(rinfo.target_version_id.is_none(), "the ledger already records the replica version");
|
||||
} else {
|
||||
assert_eq!(action, None, "a failed tag write must stop before the caller can mark the reset complete");
|
||||
assert_eq!(rinfo.replication_status, ReplicationStatusType::Failed);
|
||||
assert!(rinfo.error.is_some() && rinfo.resync_timestamp.is_empty());
|
||||
}
|
||||
let seen = server.join().expect("test HTTP server should finish");
|
||||
assert_eq!(seen.len(), 3, "HEAD by source id, HEAD by ledger id, PutObjectTagging: {seen:?}");
|
||||
assert!(
|
||||
seen[1].starts_with("HEAD ") && seen[1].contains(DRIFTED_ASSIGNED_VERSION_ID),
|
||||
"{}",
|
||||
seen[1]
|
||||
);
|
||||
assert!(seen[2].starts_with("PUT ") && seen[2].contains("tagging"), "{}", seen[2]);
|
||||
assert_eq!(seen.len(), 2 + usize::from(expect_write));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn metadata_update_on_a_drifted_target_uses_the_tagging_api_instead_of_a_put() {
|
||||
assert_versioned_tagging_sync(ReplicationType::Metadata, None, true, "200 OK").await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_resync_verifies_drifted_replica_tags_before_stamping() {
|
||||
for (tag_header, write, status) in [
|
||||
(Some("phase=after"), false, "200 OK"),
|
||||
(Some("phase=before"), true, "200 OK"),
|
||||
(None, true, "200 OK"),
|
||||
(Some("phase=before"), true, "403 Forbidden"),
|
||||
] {
|
||||
assert_versioned_tagging_sync(ReplicationType::ExistingObject, tag_header, write, status).await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn existing_object_resync_does_not_stamp_unaddressed_replica_with_stale_tags() {
|
||||
let (endpoint, server) = spawn_scripted_target_server(2, |line| {
|
||||
if line.starts_with("HEAD ") && line.contains("versionId") {
|
||||
empty_response("400 Bad Request")
|
||||
} else if line.starts_with("HEAD ") {
|
||||
format!(
|
||||
"HTTP/1.1 200 OK\r\nETag: \"{DRIFTED_ETAG}\"\r\nContent-Length: 4\r\nx-amz-tagging-count: 1\r\nx-amz-tagging: phase=before\r\nConnection: close\r\n\r\n"
|
||||
)
|
||||
} else {
|
||||
empty_response("400 Unexpected")
|
||||
}
|
||||
});
|
||||
let target = test_target_client(endpoint);
|
||||
let (action, rinfo) = resolve_tagged_test_replica(&target, ReplicationType::ExistingObject, false).await;
|
||||
assert_eq!(action, Some(ReplicationAction::All));
|
||||
assert!(rinfo.resync_timestamp.is_empty());
|
||||
assert_eq!(server.join().expect("test HTTP server should finish").len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -33,7 +33,6 @@ pub(crate) use crate::storage_api_contracts::object::{
|
||||
};
|
||||
pub(crate) use crate::storage_api_contracts::range::HTTPRangeSpec;
|
||||
pub(crate) use rustfs_replication::{DeletedObject as ReplicationDeletedObject, ObjectToDelete as ReplicationObjectToDelete};
|
||||
pub(crate) use rustfs_s3_client::api_get_options::{AdvancedGetOptions, StatObjectOptions};
|
||||
|
||||
type ListObjectsV2Info = StorageListObjectsV2Info<ObjectInfo>;
|
||||
type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
|
||||
@@ -45,6 +45,7 @@ pub(crate) use crate::bucket::bucket_target_sys::{
|
||||
pub(crate) use crate::bucket::target::BucketTarget;
|
||||
pub(crate) use crate::bucket::target::BucketTargets;
|
||||
pub use rustfs_replication::SsecPassthroughCapability;
|
||||
pub(crate) use rustfs_replication::object::replication_target_tags_match;
|
||||
pub use rustfs_replication::{ObjectLockIntegrity, object_lock_put_integrity};
|
||||
pub(crate) use rustfs_replication::{
|
||||
SsecPassthroughGate, is_replication_target_offline_error, ssec_passthrough_gate, version_identity_drifted,
|
||||
|
||||
@@ -1102,7 +1102,38 @@ impl RemoteDisk {
|
||||
}
|
||||
}
|
||||
|
||||
async fn recover_delete_transaction(
|
||||
pub(crate) async fn probe_delete_residue_disk(&self, volume: &str) -> Result<Uuid> {
|
||||
match self
|
||||
.recover_delete_transaction(volume, DeleteRecoveryRpcOperation::ProbeLegacyDiscard)
|
||||
.await
|
||||
{
|
||||
Ok(DeleteRecoveryRpcReply::LegacyDiscardDisk { disk_id }) if !disk_id.is_nil() => Ok(disk_id),
|
||||
Ok(_) => Err(Error::FileCorrupt),
|
||||
Err(Error::Io(source))
|
||||
if crate::cluster::rpc::client::embedded_tonic_status(&source)
|
||||
.is_some_and(|status| matches!(status.code(), tonic::Code::Unimplemented | tonic::Code::InvalidArgument)) =>
|
||||
{
|
||||
Err(Error::UnsupportedDisk)
|
||||
}
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn list_delete_residue_on_disk(
|
||||
&self,
|
||||
volume: &str,
|
||||
disk_id: Uuid,
|
||||
cursor: Option<DeleteRecoveryCursor>,
|
||||
limit: u32,
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
let operation = DeleteRecoveryRpcOperation::ListResidueOnDisk { disk_id, cursor, limit };
|
||||
match self.recover_delete_transaction(volume, operation).await? {
|
||||
DeleteRecoveryRpcReply::ListedResidueOnDisk { disk_id: observed, page } if observed == disk_id => Ok(page),
|
||||
_ => Err(Error::FileCorrupt),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn recover_delete_transaction(
|
||||
&self,
|
||||
volume: &str,
|
||||
operation: DeleteRecoveryRpcOperation,
|
||||
@@ -1115,6 +1146,7 @@ impl RemoteDisk {
|
||||
}
|
||||
_ => DELETE_RECOVERY_PROTOCOL_VERSION,
|
||||
};
|
||||
let bounded_inventory_page = matches!(&operation, DeleteRecoveryRpcOperation::ListResidueOnDisk { .. });
|
||||
let payload = rmp_serde::to_vec_named(&operation)?;
|
||||
if payload.len() > operation.max_wire_payload_bytes() {
|
||||
return Err(Error::other("delete recovery operation exceeds payload limit"));
|
||||
@@ -1142,6 +1174,9 @@ impl RemoteDisk {
|
||||
} else {
|
||||
self.get_client().await?
|
||||
};
|
||||
if bounded_inventory_page {
|
||||
client = client.max_decoding_message_size(rustfs_protos::DELETE_RECOVERY_MAX_CONTROL_PAYLOAD_BYTES);
|
||||
}
|
||||
let mut request = Request::new(RecoverDeleteTransactionRequest {
|
||||
disk: self.endpoint.to_string(),
|
||||
volume: volume.to_owned(),
|
||||
@@ -5622,7 +5657,9 @@ mod tests {
|
||||
);
|
||||
assert_eq!(
|
||||
remote
|
||||
.pool_meta_native_recovery(PoolMetaNativeRecoveryRequest::Probe {})
|
||||
.pool_meta_native_recovery(PoolMetaNativeRecoveryRequest::Inspect {
|
||||
object: crate::disk::PoolMetaNativeObject::PoolBin,
|
||||
})
|
||||
.await,
|
||||
Err(DiskError::UnsupportedDisk),
|
||||
);
|
||||
@@ -5633,7 +5670,7 @@ mod tests {
|
||||
"/node_service.NodeService/UndoPoolMetaWrite",
|
||||
"/node_service.NodeService/PoolMetaNativeRecovery"
|
||||
],
|
||||
"protected mutations and capability probes must never fall back to legacy RPCs",
|
||||
"protected mutations and native inspection must never fall back to legacy RPCs",
|
||||
);
|
||||
peer.stop().await;
|
||||
}
|
||||
|
||||
@@ -207,10 +207,10 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let mut state = clean_state(&fixture);
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
assert!(
|
||||
repair(&fixture.store, &mut state, &fence).await.is_err(),
|
||||
"reconstruction alone must never authorize a COMMIT"
|
||||
);
|
||||
let error = repair(&fixture.store, &mut state, &fence)
|
||||
.await
|
||||
.expect_err("reconstruction alone must never authorize a COMMIT");
|
||||
assert!(error.to_string().contains("(no_ordinary_v3_authority)"), "{error}");
|
||||
assert_eq!(
|
||||
snapshots(&fixture.store).await,
|
||||
before,
|
||||
@@ -222,6 +222,24 @@ async fn pool_meta_native_physical_split_without_ordinary_commit_performs_no_mut
|
||||
.expect("missing ordinary authority alone must not install a permanent corruption latch");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(pool_meta_version_env)]
|
||||
async fn pool_meta_native_inspection_rejects_an_absent_slot_without_changing_evidence() {
|
||||
let fixture = fixture(1).await;
|
||||
let before = snapshots(&fixture.store).await;
|
||||
let fence = PoolMetaPersistenceFence::Owned(owner(&fixture.store).await);
|
||||
let last_pool = fixture.store.pools.last().expect("fixture has a last pool");
|
||||
let set = last_pool.get_disks_by_key(POOL_META_NAME);
|
||||
let missing = set.disks.write().await[0].take().expect("fixture has a disk in slot zero");
|
||||
|
||||
let result = inspect(&fixture.store.pools, &fence).await;
|
||||
|
||||
set.disks.write().await[0] = Some(missing);
|
||||
let error = result.err().expect("an absent slot cannot provide native evidence");
|
||||
assert!(error.to_string().contains("(missing_pool_metadata_slot)"), "{error}");
|
||||
assert_eq!(snapshots(&fixture.store).await, before);
|
||||
}
|
||||
|
||||
async fn interrupted_first_commit_fixture() -> (PhysicalFixture, PoolMetaCommittedCandidate) {
|
||||
let fixture = fixture(3).await;
|
||||
let PoolMetaReplica::Valid {
|
||||
|
||||
@@ -19,6 +19,7 @@ use crate::disk::{
|
||||
PoolMetaNativeRecoveryResponse as Response,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::TryStreamExt as _;
|
||||
|
||||
#[cfg(test)]
|
||||
#[derive(Default)]
|
||||
@@ -38,8 +39,16 @@ struct InspectedPool {
|
||||
snapshots: Vec<PoolMetaNativeInspection>,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("native pool metadata recovery requires one proven V3 transaction and complete matching disk evidence ({0})")]
|
||||
struct MissingEvidence(&'static str);
|
||||
|
||||
fn missing_evidence(reason: &'static str) -> Error {
|
||||
Error::other(MissingEvidence(reason))
|
||||
}
|
||||
|
||||
fn invalid() -> Error {
|
||||
Error::other("native pool metadata recovery requires one proven V3 transaction and complete matching disk evidence")
|
||||
missing_evidence("inconsistent_evidence")
|
||||
}
|
||||
|
||||
fn known_validation(source: Error, phase: &'static str) -> Error {
|
||||
@@ -71,13 +80,19 @@ fn latch_validation(state: &mut PoolMetaWriteState, error: Error) -> Error {
|
||||
fn current(pool: &InspectedPool) -> Result<Vec<Bytes>> {
|
||||
pool.snapshots
|
||||
.iter()
|
||||
.map(|snapshot| snapshot.current.clone().ok_or_else(invalid))
|
||||
.map(|snapshot| {
|
||||
snapshot
|
||||
.current
|
||||
.clone()
|
||||
.ok_or_else(|| missing_evidence("missing_pool_metadata_head"))
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> Result<Vec<InspectedPool>> {
|
||||
let mut inspected = Vec::with_capacity(pools.len());
|
||||
// Pool -> disk order is stable; all capability probes precede every mutation.
|
||||
// Check every slot before starting I/O. The authenticated, read-only
|
||||
// Inspect RPC proves both protocol support and the disk's evidence.
|
||||
for pool in pools {
|
||||
fence.ensure_held()?;
|
||||
let set = pool.get_disks_by_key(POOL_META_NAME);
|
||||
@@ -87,20 +102,8 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.ok_or_else(invalid))
|
||||
.map(|disk| disk.ok_or_else(|| missing_evidence("missing_pool_metadata_slot")))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let probes = futures::stream::iter(disks.iter().cloned().map(|disk| async move {
|
||||
match disk.pool_meta_native_recovery(Request::Probe {}).await? {
|
||||
Response::Probed => Ok(()),
|
||||
_ => Err(invalid()),
|
||||
}
|
||||
}))
|
||||
.buffered(4)
|
||||
.collect::<Vec<Result<()>>>()
|
||||
.await;
|
||||
for result in probes {
|
||||
result?;
|
||||
}
|
||||
inspected.push(InspectedPool {
|
||||
disks,
|
||||
snapshots: Vec::new(),
|
||||
@@ -108,7 +111,7 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
|
||||
}
|
||||
for pool in &mut inspected {
|
||||
fence.ensure_held()?;
|
||||
let results = futures::stream::iter(pool.disks.iter().cloned().map(|disk| async move {
|
||||
pool.snapshots = futures::stream::iter(pool.disks.iter().cloned().map(|disk| async move {
|
||||
match disk
|
||||
.pool_meta_native_recovery(Request::Inspect {
|
||||
object: PoolMetaNativeObject::PoolBin,
|
||||
@@ -116,15 +119,12 @@ async fn inspect(pools: &[Arc<Sets>], fence: &PoolMetaPersistenceFence<'_>) -> R
|
||||
.await?
|
||||
{
|
||||
Response::Inspected(snapshot) if !snapshot.disk_id.is_nil() => Ok(snapshot),
|
||||
_ => Err(invalid()),
|
||||
_ => Err(missing_evidence("invalid_disk_identity_evidence")),
|
||||
}
|
||||
}))
|
||||
.buffered(4)
|
||||
.collect::<Vec<Result<PoolMetaNativeInspection>>>()
|
||||
.await;
|
||||
for result in results {
|
||||
pool.snapshots.push(result?);
|
||||
}
|
||||
.try_collect()
|
||||
.await?;
|
||||
}
|
||||
fence.ensure_held()?;
|
||||
Ok(inspected)
|
||||
@@ -142,7 +142,7 @@ async fn validate_identities(pools: &[Arc<Sets>], canonical: &[u8], fence: &Pool
|
||||
.await
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|disk| disk.ok_or_else(invalid))
|
||||
.map(|disk| disk.ok_or_else(|| missing_evidence("missing_pool_identity_slot")))
|
||||
.collect::<Result<Vec<_>>>()?;
|
||||
let results = futures::stream::iter(disks.into_iter().enumerate().map(|(index, disk)| async move {
|
||||
match disk.pool_meta_native_recovery(Request::InspectIdentity {}).await? {
|
||||
@@ -151,7 +151,7 @@ async fn validate_identities(pools: &[Arc<Sets>], canonical: &[u8], fence: &Pool
|
||||
.await
|
||||
.map_err(|error| codec_validation(error, "native_identity_validation"))
|
||||
}
|
||||
_ => Err(invalid()),
|
||||
_ => Err(missing_evidence("invalid_disk_identity_evidence")),
|
||||
}
|
||||
}))
|
||||
.buffered(4)
|
||||
@@ -479,7 +479,7 @@ pub(super) async fn repair(
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
if ordinary_commits.is_empty() && ordinary_prepares.is_empty() {
|
||||
return Err(invalid());
|
||||
return Err(missing_evidence("no_ordinary_v3_authority"));
|
||||
}
|
||||
let selection = select_pool_meta_replica_reads(reads)
|
||||
.map_err(|error| latch_validation(state, known_validation(error, "native_metadata_validation")))?;
|
||||
@@ -535,7 +535,7 @@ pub(super) async fn repair(
|
||||
.iter()
|
||||
.any(|read| !matches!(read.replica, PoolMetaIdentityReplica::Valid(_)))
|
||||
{
|
||||
return Err(invalid());
|
||||
return Err(missing_evidence("pool_identity_unavailable"));
|
||||
}
|
||||
let identity = select_pool_meta_identity(identities, cluster_id)
|
||||
.map_err(|error| latch_validation(state, known_validation(error, "native_identity_validation")))?;
|
||||
@@ -847,7 +847,10 @@ mod tests {
|
||||
async fn pool_meta_native_plan_rejects_missing_or_corrupt_evidence() {
|
||||
let mut f = fixture().await;
|
||||
let before = f.pools[1].snapshots[2].current.take();
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
let error = plan(&f.pools, &f.commit, f.revision, &f.identity, None)
|
||||
.await
|
||||
.expect_err("missing current shard must block the repair plan");
|
||||
assert!(error.to_string().contains("(missing_pool_metadata_head)"), "{error}");
|
||||
f.pools[1].snapshots[2].current = before;
|
||||
f.pools[1].snapshots[2].backups[0].data = Bytes::from_static(b"invalid metadata");
|
||||
assert!(plan(&f.pools, &f.commit, f.revision, &f.identity, None).await.is_err());
|
||||
|
||||
@@ -222,33 +222,32 @@ impl Sets {
|
||||
let mut set_endpoints = Vec::with_capacity(set_drive_count);
|
||||
for j in 0..set_drive_count {
|
||||
let idx = i * set_drive_count + j;
|
||||
let mut disk = disks[idx].clone();
|
||||
|
||||
let endpoint = endpoints.endpoints.as_ref()[idx].clone();
|
||||
|
||||
set_endpoints.push(endpoint);
|
||||
|
||||
if disk.is_none() {
|
||||
let Some(mut disk) = disks[idx].clone() else {
|
||||
warn!("sets new set_drive {}-{} is none", i, j);
|
||||
set_drive.push(None);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if disk.as_ref().unwrap().is_local() && instance_ctx.is_dist_erasure().await {
|
||||
let local_disk = runtime_sources::local_disk_set_drive(&instance_ctx, pool_idx, i, j).await;
|
||||
|
||||
if local_disk.is_none() {
|
||||
if disk.is_local() && instance_ctx.is_dist_erasure().await {
|
||||
let Some(local_disk) = runtime_sources::local_disk_set_drive(&instance_ctx, pool_idx, i, j).await else {
|
||||
warn!("sets new set_drive {}-{} local_disk is none", i, j);
|
||||
set_drive.push(None);
|
||||
continue;
|
||||
};
|
||||
|
||||
// Format initialization can reuse the registered slot's handle.
|
||||
if !Arc::ptr_eq(&disk, &local_disk) {
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
|
||||
let _ = disk.as_ref().unwrap().close().await;
|
||||
|
||||
disk = local_disk;
|
||||
}
|
||||
|
||||
let has_disk_id = disk.as_ref().unwrap().get_disk_id().await.unwrap_or_else(|err| {
|
||||
let has_disk_id = disk.get_disk_id().await.unwrap_or_else(|err| {
|
||||
if err == DiskError::UnformattedDisk {
|
||||
error!("get_disk_id err {:?}", err);
|
||||
} else {
|
||||
@@ -259,7 +258,7 @@ impl Sets {
|
||||
});
|
||||
|
||||
if let Some(_disk_id) = has_disk_id {
|
||||
set_drive.push(disk);
|
||||
set_drive.push(Some(disk));
|
||||
} else {
|
||||
error!("sets new set_drive {}-{} get_disk_id is none", i, j);
|
||||
set_drive.push(None);
|
||||
@@ -2317,6 +2316,57 @@ mod tests {
|
||||
(format, pool_endpoints)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn distributed_sets_only_retire_superseded_local_disk_handles() {
|
||||
let (_dirs, format, fixture) = setup_heal_format_sets(2, false).await;
|
||||
let mut pool = fixture.endpoints.clone();
|
||||
let mut endpoints = pool.endpoints.as_ref().clone();
|
||||
let mut peer = Endpoint::try_from("http://127.0.0.1:9001/data").expect("peer endpoint");
|
||||
peer.set_pool_index(0);
|
||||
peer.set_set_index(0);
|
||||
peer.set_disk_index(2);
|
||||
endpoints[2] = peer;
|
||||
pool.endpoints = endpoints.clone().into();
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
let options = DiskOption::default();
|
||||
runtime_sources::initialize_local_disk_maps(&ctx, vec![pool.clone()].into(), &options)
|
||||
.await
|
||||
.expect("initialize distributed drive slots");
|
||||
|
||||
// Slot 0 reuses the registered handle. Removing only the endpoint
|
||||
// route for slot 1 makes init_disks open a separate bootstrap handle.
|
||||
ctx.local_disk_map().write().await.remove(&endpoints[1].to_string());
|
||||
let local_endpoints = Endpoints::from(endpoints[..2].to_vec());
|
||||
let (mut disks, errors) = crate::store::init_format::init_disks(&ctx, &local_endpoints, &options).await;
|
||||
assert!(errors.iter().all(Option::is_none), "initialize local disks: {errors:?}");
|
||||
let registered = runtime_sources::local_disk_set_drive(&ctx, 0, 0, 0)
|
||||
.await
|
||||
.expect("registered disk");
|
||||
let replacement = runtime_sources::local_disk_set_drive(&ctx, 0, 0, 1)
|
||||
.await
|
||||
.expect("replacement disk");
|
||||
let bootstrap = disks[1].as_ref().expect("bootstrap disk");
|
||||
assert!(Arc::ptr_eq(®istered, disks[0].as_ref().expect("reused disk")));
|
||||
assert!(!Arc::ptr_eq(&replacement, bootstrap));
|
||||
let bootstrap_retired = bootstrap.local_retirement_token().expect("local bootstrap token");
|
||||
disks.push(None); // The remote peer has not started.
|
||||
|
||||
let sets = Sets::new_with_instance_ctx(disks, &pool, &format, 0, 1, ctx)
|
||||
.await
|
||||
.expect("construct distributed sets");
|
||||
let inventory = sets.disk_set[0].disk_inventory().await;
|
||||
for (slot, expected) in [registered, replacement].into_iter().enumerate() {
|
||||
let active = inventory[slot].as_ref().expect("active local disk");
|
||||
assert!(Arc::ptr_eq(active, &expected));
|
||||
assert!(
|
||||
!active.local_retirement_token().expect("local disk token").is_cancelled(),
|
||||
"set construction retired the active disk in slot {slot}"
|
||||
);
|
||||
}
|
||||
assert!(bootstrap_retired.is_cancelled(), "superseded bootstrap handle must be retired");
|
||||
}
|
||||
|
||||
// Phase 5 follow-up (backlog#1052): a pool built through the ctx-explicit
|
||||
// constructor carries the caller's context through Sets AND every SetDisks,
|
||||
// so nothing in the object graph silently binds to the process bootstrap.
|
||||
|
||||
@@ -57,7 +57,7 @@ const DATA_COMPRESSION_TOTAL_NAME: &str = ".compression.json";
|
||||
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
|
||||
const DATA_USAGE_CACHE_TTL_SECS: u64 = 30;
|
||||
const ADMIN_USAGE_INCARNATION_CONCURRENCY: usize = 4;
|
||||
const ADMIN_USAGE_INCARNATION_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
pub(crate) const ADMIN_USAGE_INCARNATION_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
// A scanner save invalidates only the saving process's snapshot cache; every
|
||||
// other node sees it only after reloading. A fail-closed usage lookup for a
|
||||
// bucket the cached snapshot does not mention at all may reload a copy this
|
||||
@@ -1429,6 +1429,11 @@ fn select_admin_data_usage_snapshot(
|
||||
}
|
||||
|
||||
fn merge_partial_observation_for_admin(mut authoritative: DataUsageInfo, observed: DataUsageInfo) -> DataUsageInfo {
|
||||
// The response takes the observation's cycle below. Buckets absent from
|
||||
// that observation retain their old value and original certificate.
|
||||
for usage in authoritative.buckets_usage.values_mut() {
|
||||
usage.mark_carried();
|
||||
}
|
||||
for (bucket, usage) in observed.buckets_usage {
|
||||
authoritative.buckets_usage.insert(bucket, usage);
|
||||
}
|
||||
@@ -1473,7 +1478,7 @@ async fn project_admin_data_usage_for_current_buckets(
|
||||
) -> DataUsageInfo {
|
||||
use futures::StreamExt as _;
|
||||
let snapshot_measured = response.is_complete_bucket_usage_snapshot() || response.is_valid_partial_snapshot();
|
||||
let overlay_authoritative = with_memory_overlay && !runtime_sources::setup_is_dist_erasure().await;
|
||||
let overlay_authoritative = with_memory_overlay && snapshot_measured && !runtime_sources::setup_is_dist_erasure().await;
|
||||
let deadline = tokio::time::Instant::now() + ADMIN_USAGE_INCARNATION_TIMEOUT;
|
||||
let mut incarnations = HashMap::new();
|
||||
{
|
||||
@@ -1514,10 +1519,7 @@ async fn project_admin_data_usage_for_current_buckets(
|
||||
}
|
||||
}
|
||||
response.project_bucket_observations_for_admin(&incarnations);
|
||||
if with_memory_overlay
|
||||
&& apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await
|
||||
&& snapshot_measured
|
||||
{
|
||||
if apply_bucket_usage_memory_overlay_if_authoritative(&mut response, overlay_authoritative).await {
|
||||
response.usage_snapshot_complete = false;
|
||||
response.usage_snapshot_partial = true;
|
||||
response.usage_snapshot_converged = Some(false);
|
||||
@@ -1784,8 +1786,8 @@ pub async fn load_admin_data_usage_from_backend_cached(store: Arc<ECStore>) -> R
|
||||
|
||||
/// Load the admin view with the existing authoritative single-process overlay.
|
||||
///
|
||||
/// The overlay is applied before current-identity projection so it cannot
|
||||
/// reintroduce statistics from a retired or unverified bucket generation.
|
||||
/// Only a complete or valid partial scanner snapshot can receive memory values.
|
||||
/// The final identity check excludes statistics from retired or unverified buckets.
|
||||
/// Distributed nodes continue to use scanner statistics without local totals.
|
||||
/// Cancellation, unknown values, and per-bucket read semantics are the same as
|
||||
/// [`load_admin_data_usage_from_backend_cached`]. No overlay is persisted.
|
||||
@@ -2667,6 +2669,16 @@ pub async fn reload_unknown_bucket_usage(bucket: &str) -> Option<u64> {
|
||||
}
|
||||
|
||||
pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
|
||||
cached_bucket_usage_size(bucket, |cached| cached.usage.size.saturating_add(cached.pending_negative_delta)).await
|
||||
}
|
||||
|
||||
/// Read-only statistics show the post-delete value. The local hold on deleted
|
||||
/// bytes is an admission precaution, not part of the bucket's measured size.
|
||||
pub(crate) async fn get_bucket_usage_for_statistics(bucket: &str) -> Option<u64> {
|
||||
cached_bucket_usage_size(bucket, |cached| cached.usage.size).await
|
||||
}
|
||||
|
||||
async fn cached_bucket_usage_size(bucket: &str, local_size: impl FnOnce(&CachedBucketUsage) -> u64) -> Option<u64> {
|
||||
// A process-local mutation cache is authoritative only when every request
|
||||
// is handled by this process. In a distributed deployment it contains an
|
||||
// absolute count derived from one node's requests, so applying it as the
|
||||
@@ -2679,10 +2691,7 @@ pub async fn get_bucket_usage_memory(bucket: &str) -> Option<u64> {
|
||||
update_usage_cache_if_needed().await;
|
||||
|
||||
let cache = memory_cache().read().await;
|
||||
cache
|
||||
.get(bucket)
|
||||
.filter(|cached| cached.authoritative)
|
||||
.map(|cached| cached.usage.size.saturating_add(cached.pending_negative_delta))
|
||||
cache.get(bucket).filter(|cached| cached.authoritative).map(local_size)
|
||||
}
|
||||
|
||||
async fn update_usage_cache_if_needed() {
|
||||
@@ -4158,6 +4167,16 @@ mod tests {
|
||||
let mut authoritative = data_usage_info_for_test("cold", 152_318, 80 * 1024 * 1024 * 1024, baseline_time);
|
||||
authoritative.scanner_epoch = Some(4);
|
||||
authoritative.scanner_cycle = Some(10);
|
||||
let cold_certificate = rustfs_data_usage::BucketUsageCertification {
|
||||
epoch: 4,
|
||||
cycle: 10,
|
||||
certified_at: baseline_time,
|
||||
};
|
||||
authoritative
|
||||
.buckets_usage
|
||||
.get_mut("cold")
|
||||
.expect("cold baseline")
|
||||
.certify(cold_certificate);
|
||||
authoritative.buckets_usage.insert(
|
||||
"hot".to_string(),
|
||||
BucketUsageInfo {
|
||||
@@ -4219,6 +4238,7 @@ mod tests {
|
||||
objects_count: 34,
|
||||
versions_count: 34,
|
||||
size: 8 * 1024 * 1024,
|
||||
usage_value_state: Some(rustfs_data_usage::BucketUsageValueState::Observed),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
@@ -4234,6 +4254,13 @@ mod tests {
|
||||
assert!(selected.is_valid_partial_snapshot());
|
||||
assert_eq!(selected.usage_snapshot_converged, Some(false));
|
||||
assert_eq!(selected.buckets_count, 2);
|
||||
assert_eq!(selected.scanner_cycle, Some(11));
|
||||
let cold = &selected.buckets_usage["cold"];
|
||||
assert_eq!(cold.usage_value_state, Some(rustfs_data_usage::BucketUsageValueState::Carried));
|
||||
assert_eq!(cold.usage_certification, Some(cold_certificate));
|
||||
let hot = &selected.buckets_usage["hot"];
|
||||
assert_eq!(hot.usage_value_state, Some(rustfs_data_usage::BucketUsageValueState::Observed));
|
||||
assert_eq!(hot.usage_certification, None);
|
||||
assert_eq!(
|
||||
selected
|
||||
.buckets_usage
|
||||
|
||||
@@ -402,6 +402,54 @@ async fn cached_canonical_admin_usage_rechecks_current_identity_without_peer_inv
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn admin_memory_overlay_requires_a_measured_scanner_baseline() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::storage_api_contracts::bucket::MakeBucketOptions;
|
||||
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
assert!(!runtime_sources::setup_is_dist_erasure().await);
|
||||
let bucket = "usage-unmeasured-overlay";
|
||||
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
let incarnation = store.bucket_incarnation_id_from_disk(bucket).await.unwrap();
|
||||
let mut baseline = data_usage_info_for_test(bucket, 1, 100, SystemTime::UNIX_EPOCH);
|
||||
let usage = baseline.buckets_usage.get_mut(bucket).unwrap();
|
||||
usage.bucket_incarnation = Some(incarnation.to_string());
|
||||
let memory_usage = BucketUsageInfo {
|
||||
size: 150,
|
||||
..usage.clone()
|
||||
};
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
replace_bucket_usage_memory_from_authoritative(bucket, memory_usage, SystemTime::now()).await;
|
||||
|
||||
// A missing snapshot and a mismatched bucket count both leave usage unknown.
|
||||
for (buckets_count, expected_size) in [(None, None), (Some(2), None), (Some(1), Some(150))] {
|
||||
if let Some(count) = buckets_count {
|
||||
baseline.buckets_count = count;
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&baseline).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
}
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let response = load_admin_data_usage_with_memory_overlay(store.clone()).await.unwrap();
|
||||
assert_eq!(
|
||||
response.buckets_usage.get(bucket).map(|usage| usage.size),
|
||||
expected_size,
|
||||
"baseline count {buckets_count:?}"
|
||||
);
|
||||
assert_eq!(response.bucket_sizes.get(bucket).copied(), expected_size);
|
||||
assert_eq!(response.objects_total_size, expected_size.unwrap_or_default());
|
||||
assert_eq!(response.usage_snapshot_admin_projection, expected_size.is_some());
|
||||
assert!(!response.is_complete_bucket_usage_snapshot());
|
||||
assert!(!response.is_valid_partial_snapshot());
|
||||
}
|
||||
clear_usage_memory_cache_for_test().await;
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial]
|
||||
async fn admin_memory_overlay_cannot_reintroduce_a_predecessor_identity() {
|
||||
|
||||
@@ -50,6 +50,8 @@ const EVENT_SERVER_INFO: &str = "server_info";
|
||||
const LOG_COMPONENT_ECSTORE: &str = "ecstore";
|
||||
const LOG_SUBSYSTEM_DIAGNOSTICS: &str = "diagnostics";
|
||||
|
||||
mod inventory;
|
||||
|
||||
use shadow_rs::shadow;
|
||||
|
||||
shadow!(build);
|
||||
@@ -276,6 +278,14 @@ fn collect_runtime_server_stats() -> (MemStats, u64, u64) {
|
||||
}
|
||||
|
||||
pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
get_server_info_with_inventory(get_pools, false).await.0
|
||||
}
|
||||
|
||||
pub async fn get_server_info_with_inventory(
|
||||
get_pools: bool,
|
||||
include_inventory: bool,
|
||||
) -> (InfoMessage, Option<rustfs_madmin::ClusterInventory>) {
|
||||
let mut inventory = include_inventory.then(rustfs_madmin::ClusterInventory::default);
|
||||
let started = tracing::enabled!(tracing::Level::TRACE).then(Instant::now);
|
||||
|
||||
let local = get_local_server_property().await;
|
||||
@@ -301,25 +311,25 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
|
||||
if let Some(store) = runtime_sources::object_store_handle() {
|
||||
mode = ITEM_ONLINE;
|
||||
apply_data_usage_result(
|
||||
load_server_info_usage_snapshot(store.clone()).await,
|
||||
&mut buckets,
|
||||
&mut objects,
|
||||
&mut versions,
|
||||
&mut delete_markers,
|
||||
&mut usage,
|
||||
);
|
||||
let namespace_options = BucketOptions {
|
||||
cached: true,
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
};
|
||||
let deadline = tokio::time::Instant::now() + crate::data_usage::ADMIN_USAGE_INCARNATION_TIMEOUT;
|
||||
let namespace = if include_inventory {
|
||||
Some(inventory::bounded_namespace(deadline, store.list_bucket_for_scanner(&namespace_options)).await)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let snapshot = load_server_info_usage_snapshot(store.clone()).await;
|
||||
if let Some(namespace) = &namespace {
|
||||
inventory = Some(inventory::summarize(namespace.as_ref().ok(), snapshot.as_ref().ok()));
|
||||
}
|
||||
apply_data_usage_result(snapshot, &mut buckets, &mut objects, &mut versions, &mut delete_markers, &mut usage);
|
||||
if buckets.error.is_some() {
|
||||
apply_bucket_namespace_count(
|
||||
store
|
||||
.list_bucket_for_scanner(&BucketOptions {
|
||||
cached: true,
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await,
|
||||
&mut buckets,
|
||||
);
|
||||
let listing = inventory::namespace_or_fallback(namespace, store.list_bucket_for_scanner(&namespace_options)).await;
|
||||
apply_bucket_namespace_count(listing, &mut buckets);
|
||||
}
|
||||
|
||||
let backend_info = StorageAdminApi::backend_info(store.as_ref()).await;
|
||||
@@ -375,7 +385,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
);
|
||||
}
|
||||
|
||||
InfoMessage {
|
||||
let info = InfoMessage {
|
||||
mode: Some(mode.to_string()),
|
||||
domain: None,
|
||||
region: None,
|
||||
@@ -390,7 +400,8 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
services: Some(services),
|
||||
servers: Some(servers),
|
||||
pools: Some(pools),
|
||||
}
|
||||
};
|
||||
(info, inventory)
|
||||
}
|
||||
|
||||
fn is_online_drive_state(state: &str) -> bool {
|
||||
|
||||
@@ -0,0 +1,270 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::ScannerBucketListing;
|
||||
use rustfs_data_usage::DataUsageInfo;
|
||||
use rustfs_madmin::ClusterInventory;
|
||||
use std::collections::HashSet;
|
||||
|
||||
pub(super) async fn bounded_namespace(
|
||||
deadline: tokio::time::Instant,
|
||||
read: impl Future<Output = crate::error::Result<ScannerBucketListing>>,
|
||||
) -> crate::error::Result<ScannerBucketListing> {
|
||||
tokio::time::timeout_at(deadline, read)
|
||||
.await
|
||||
.map_err(|_| crate::error::Error::other("inventory namespace read timed out"))?
|
||||
}
|
||||
|
||||
pub(super) async fn namespace_or_fallback(
|
||||
previous: Option<crate::error::Result<ScannerBucketListing>>,
|
||||
read: impl Future<Output = crate::error::Result<ScannerBucketListing>>,
|
||||
) -> crate::error::Result<ScannerBucketListing> {
|
||||
// A failed or timed-out optional listing is already an observation.
|
||||
// Retrying it without the deadline would stall the independent health response.
|
||||
match previous {
|
||||
Some(result) => result,
|
||||
None => read.await,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn summarize(namespace: Option<&ScannerBucketListing>, snapshot: Option<&DataUsageInfo>) -> ClusterInventory {
|
||||
// An admin projection is a validated display-only view produced by the loader.
|
||||
let snapshot = snapshot.filter(|info| {
|
||||
info.usage_snapshot_admin_projection || info.is_complete_bucket_usage_snapshot() || info.is_valid_partial_snapshot()
|
||||
});
|
||||
let mut inventory = ClusterInventory {
|
||||
snapshot_at: snapshot
|
||||
.and_then(|info| info.last_update)
|
||||
.and_then(|time| time.try_into().ok()),
|
||||
..Default::default()
|
||||
};
|
||||
let Some(namespace) = namespace.filter(|listing| listing.topology_complete) else {
|
||||
return inventory;
|
||||
};
|
||||
let names: HashSet<_> = namespace
|
||||
.buckets
|
||||
.iter()
|
||||
.map(|bucket| bucket.name.as_str())
|
||||
.filter(|name| !name.starts_with('.'))
|
||||
.collect();
|
||||
inventory.current_buckets = u64::try_from(names.len()).ok();
|
||||
let Some(snapshot) = snapshot else {
|
||||
return inventory;
|
||||
};
|
||||
|
||||
let mut measured = 0_u64;
|
||||
let mut objects = Some(0_u64);
|
||||
let mut bytes = Some(0_u64);
|
||||
for &name in &names {
|
||||
let Some(usage) = snapshot.buckets_usage.get(name) else { continue };
|
||||
// The admin loader has already checked each retained value against the
|
||||
// current disk incarnation after this request listed the namespace.
|
||||
measured += 1;
|
||||
objects = objects.and_then(|total| total.checked_add(usage.objects_count));
|
||||
bytes = bytes.and_then(|total| total.checked_add(usage.size));
|
||||
}
|
||||
inventory.measured_buckets = Some(measured);
|
||||
if measured != 0 || names.is_empty() {
|
||||
inventory.measured_objects = objects;
|
||||
inventory.measured_bytes = bytes;
|
||||
}
|
||||
inventory.converged = if objects.is_none() || bytes.is_none() {
|
||||
None
|
||||
} else if inventory.current_buckets != Some(measured)
|
||||
|| snapshot.buckets_usage.len() != names.len()
|
||||
|| snapshot.usage_snapshot_pending_bucket_count > 0
|
||||
|| !snapshot.usage_snapshot_pending_buckets.is_empty()
|
||||
|| !snapshot.is_complete_bucket_usage_snapshot()
|
||||
{
|
||||
Some(false)
|
||||
} else {
|
||||
snapshot.usage_snapshot_converged
|
||||
};
|
||||
inventory
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_data_usage::BucketUsageInfo;
|
||||
use std::time::{Duration, SystemTime};
|
||||
|
||||
fn namespace(names: &[&str]) -> ScannerBucketListing {
|
||||
ScannerBucketListing {
|
||||
buckets: names
|
||||
.iter()
|
||||
.map(|name| crate::storage_api_contracts::bucket::BucketInfo {
|
||||
name: name.to_string(),
|
||||
..Default::default()
|
||||
})
|
||||
.collect(),
|
||||
set_buckets: vec![],
|
||||
topology_complete: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn snapshot(names: &[(&str, u64)]) -> DataUsageInfo {
|
||||
DataUsageInfo {
|
||||
last_update: Some(SystemTime::UNIX_EPOCH + Duration::from_secs(1)),
|
||||
usage_snapshot_complete: true,
|
||||
usage_snapshot_converged: Some(true),
|
||||
buckets_count: u64::try_from(names.len()).unwrap(),
|
||||
buckets_usage: names
|
||||
.iter()
|
||||
.map(|&(name, count)| {
|
||||
(
|
||||
name.to_owned(),
|
||||
BucketUsageInfo {
|
||||
objects_count: count,
|
||||
size: count,
|
||||
bucket_incarnation: Some("11111111-1111-4111-8111-111111111111".into()),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn view(namespace: Option<&ScannerBucketListing>, snapshot: Option<&DataUsageInfo>) -> ClusterInventory {
|
||||
summarize(namespace, snapshot)
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn inventory_namespace_timeout_is_reused_by_the_legacy_fallback() {
|
||||
use tokio::time::{Duration, Instant};
|
||||
let deadline = Instant::now() + Duration::from_secs(2);
|
||||
let result = bounded_namespace(deadline, std::future::pending()).await;
|
||||
assert!(result.is_err());
|
||||
assert_eq!(Instant::now(), deadline);
|
||||
let result = namespace_or_fallback(Some(result), async { panic!("must not retry a failed namespace read") }).await;
|
||||
assert!(result.is_err());
|
||||
let inventory = view(result.as_ref().ok(), None);
|
||||
assert_eq!(inventory.current_buckets, None);
|
||||
assert_eq!(inventory.measured_objects, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn inventory_excludes_stale_bucket_incarnations_from_a_stored_snapshot() {
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::config::com::save_config;
|
||||
use crate::data_usage::{DATA_USAGE_OBJ_NAME_PATH, invalidate_data_usage_snapshot_cache};
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations as _, BucketOptions, MakeBucketOptions};
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
for bucket in ["unchanged", "recreated", "new"] {
|
||||
store.make_bucket(bucket, &MakeBucketOptions::default()).await.unwrap();
|
||||
}
|
||||
let mut scanned = snapshot(&[("unchanged", 4), ("recreated", 99), ("deleted", 6)]);
|
||||
let current = store.bucket_incarnation_id_from_disk("unchanged").await.unwrap();
|
||||
scanned.buckets_usage.get_mut("unchanged").unwrap().bucket_incarnation = Some(current.to_string());
|
||||
scanned.usage_snapshot_converged = None;
|
||||
scanned.calculate_totals();
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&scanned).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let listing = store
|
||||
.list_bucket_for_scanner(&BucketOptions {
|
||||
no_metadata: true,
|
||||
..Default::default()
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let loaded = super::super::load_server_info_usage_snapshot(store.clone()).await.unwrap();
|
||||
let result = summarize(Some(&listing), Some(&loaded));
|
||||
assert_eq!(result.current_buckets, Some(3));
|
||||
assert_eq!(result.measured_buckets, Some(1));
|
||||
assert_eq!(result.measured_objects, Some(4));
|
||||
assert_eq!(result.converged, Some(false));
|
||||
// A complete canonical snapshot from before the convergence field was
|
||||
// added inherits the admin loader's existing convergence interpretation.
|
||||
let mut legacy = snapshot(&[("unchanged", 4), ("recreated", 0), ("new", 0)]);
|
||||
legacy.usage_snapshot_converged = None;
|
||||
for (name, usage) in &mut legacy.buckets_usage {
|
||||
usage.bucket_incarnation = Some(store.bucket_incarnation_id_from_disk(name).await.unwrap().to_string());
|
||||
}
|
||||
legacy.calculate_totals();
|
||||
save_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str(), serde_json::to_vec(&legacy).unwrap())
|
||||
.await
|
||||
.unwrap();
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
let loaded = super::super::load_server_info_usage_snapshot(store.clone()).await.unwrap();
|
||||
let result = summarize(Some(&listing), Some(&loaded));
|
||||
assert_eq!(result.measured_objects, Some(4));
|
||||
assert_eq!(result.converged, Some(true));
|
||||
|
||||
invalidate_data_usage_snapshot_cache().await;
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inventory_reports_only_the_intersection_of_current_and_measured_buckets() {
|
||||
let scope = namespace(&["existing", "new", ".rustfs.sys"]);
|
||||
let before = snapshot(&[("existing", 300_000), ("deleted", 7)]);
|
||||
let result = view(Some(&scope), Some(&before));
|
||||
assert_eq!(result.current_buckets, Some(2));
|
||||
assert_eq!(result.measured_buckets, Some(1));
|
||||
assert_eq!(result.measured_objects, Some(300_000));
|
||||
assert_eq!(result.measured_bytes, Some(300_000));
|
||||
assert_eq!(result.converged, Some(false));
|
||||
assert_eq!(result.snapshot_at.unwrap().as_second(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inventory_distinguishes_unavailable_and_confirmed_empty_measurements() {
|
||||
let empty = namespace(&[]);
|
||||
let missing = view(Some(&empty), Some(&DataUsageInfo::default()));
|
||||
assert_eq!(missing.current_buckets, Some(0));
|
||||
assert_eq!(missing.measured_objects, None);
|
||||
assert_eq!(missing.measured_buckets, None);
|
||||
let available = view(Some(&empty), Some(&snapshot(&[])));
|
||||
assert_eq!(available.measured_objects, Some(0));
|
||||
assert_eq!(available.converged, Some(true));
|
||||
let scope = namespace(&["new"]);
|
||||
let no_overlap = view(Some(&scope), Some(&snapshot(&[("deleted", 7)])));
|
||||
assert_eq!(no_overlap.measured_buckets, Some(0));
|
||||
assert_eq!(no_overlap.measured_objects, None);
|
||||
let known_empty = view(Some(&scope), Some(&snapshot(&[("new", 0)])));
|
||||
assert_eq!(known_empty.measured_objects, Some(0));
|
||||
assert_eq!(known_empty.measured_buckets, Some(1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inventory_keeps_the_recorded_snapshot_convergence_and_age() {
|
||||
let scope = namespace(&["a"]);
|
||||
let mut scanned = snapshot(&[("a", 12)]);
|
||||
scanned.usage_snapshot_converged = None;
|
||||
assert_eq!(view(Some(&scope), Some(&scanned)).converged, None);
|
||||
scanned.usage_snapshot_converged = Some(true);
|
||||
scanned.usage_snapshot_pending_buckets = vec!["a".into()];
|
||||
assert_eq!(view(Some(&scope), Some(&scanned)).converged, Some(false));
|
||||
scanned.usage_snapshot_pending_buckets.clear();
|
||||
scanned.usage_snapshot_converged = Some(false);
|
||||
scanned.usage_snapshot_complete = false;
|
||||
scanned.usage_snapshot_admin_projection = true;
|
||||
let result = view(Some(&scope), Some(&scanned));
|
||||
assert_eq!(result.measured_objects, Some(12));
|
||||
assert_eq!(result.converged, Some(false));
|
||||
assert_eq!(result.snapshot_at.unwrap().as_second(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn inventory_requires_complete_namespace_and_checks_each_sum() {
|
||||
let mut scope = namespace(&["a", "b"]);
|
||||
let mut scanned = snapshot(&[("a", u64::MAX), ("b", 1)]);
|
||||
scanned.buckets_usage.get_mut("a").unwrap().size = 1;
|
||||
scope.topology_complete = false;
|
||||
let missing = view(Some(&scope), Some(&scanned));
|
||||
assert_eq!(missing.current_buckets, None);
|
||||
assert_eq!(missing.measured_buckets, None);
|
||||
assert_eq!(missing.converged, None);
|
||||
scope.topology_complete = true;
|
||||
let overflow = view(Some(&scope), Some(&scanned));
|
||||
assert_eq!(overflow.measured_buckets, Some(2));
|
||||
assert_eq!(overflow.measured_objects, None);
|
||||
assert_eq!(overflow.measured_bytes, Some(2));
|
||||
assert_eq!(overflow.converged, None);
|
||||
}
|
||||
}
|
||||
@@ -1623,6 +1623,8 @@ pub struct LocalDiskWrapper {
|
||||
health_check: bool,
|
||||
/// Cancellation token for monitoring tasks
|
||||
cancel_token: CancellationToken,
|
||||
/// Retirement belongs to this wrapper instance, not its monitoring policy.
|
||||
retired: CancellationToken,
|
||||
/// Disk ID for stale checking
|
||||
disk_id: Arc<RwLock<Option<Uuid>>>,
|
||||
/// Timeout policy for scanner-sensitive operations, loaded once on wrapper initialization.
|
||||
@@ -1776,6 +1778,7 @@ impl LocalDiskWrapper {
|
||||
physical_admission: Arc::new(crate::disk::io_admission::PhysicalDiskBinding::default()),
|
||||
health_check: health_check && env_health_check,
|
||||
cancel_token: CancellationToken::new(),
|
||||
retired: CancellationToken::new(),
|
||||
disk_id: Arc::new(RwLock::new(None)),
|
||||
timeout_health_policy: get_drive_timeout_health_policy(),
|
||||
};
|
||||
@@ -1983,6 +1986,31 @@ impl LocalDiskWrapper {
|
||||
self.cancel_token.cancel();
|
||||
}
|
||||
|
||||
/// Retire a superseded wrapper and stop any monitoring tied to that attachment.
|
||||
pub(crate) fn retire_handle(&self) {
|
||||
self.retired.cancel();
|
||||
self.cancel_token.cancel();
|
||||
}
|
||||
|
||||
/// Check whether the scanner can still use this handle without issuing disk I/O.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `DiskNotFound` after retirement or `FaultyDisk` for an offline disk.
|
||||
pub fn check_scanner_handle(&self) -> Result<()> {
|
||||
if self.retired.is_cancelled() {
|
||||
return Err(DiskError::DiskNotFound);
|
||||
}
|
||||
if self.health.is_faulty() {
|
||||
return Err(DiskError::FaultyDisk);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Observe retirement without granting callers authority to retire this wrapper.
|
||||
pub fn retirement_token(&self) -> CancellationToken {
|
||||
self.retired.child_token()
|
||||
}
|
||||
|
||||
fn spawn_recovery_monitor_if_needed(&self) {
|
||||
if !self.health_check {
|
||||
return;
|
||||
@@ -2980,7 +3008,7 @@ impl DiskAPI for LocalDiskWrapper {
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<()> {
|
||||
self.stop_monitoring().await;
|
||||
self.retire_handle();
|
||||
self.disk.close().await
|
||||
}
|
||||
|
||||
|
||||
@@ -7592,8 +7592,9 @@ pub struct LocalDisk {
|
||||
io_root: PathBuf,
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: Arc<std::fs::File>,
|
||||
// The descriptor pins this identity even if the configured path is remounted.
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id: Option<u64>,
|
||||
mount_lease_identity: Option<ReplacementMountIdentity>,
|
||||
/// Public path for callers that need the configured disk layout. Internal
|
||||
/// disk I/O uses `io_format_path`, which is rooted at `mount_lease`.
|
||||
pub format_path: PathBuf,
|
||||
@@ -7787,72 +7788,50 @@ struct ReplacementMountIdentity {
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity(metadata: &std::fs::Metadata, mount_id: u64) -> ReplacementMountIdentity {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
ReplacementMountIdentity {
|
||||
device: metadata.dev(),
|
||||
inode: metadata.ino(),
|
||||
mount_id,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_for_fd(fd: &std::fs::File) -> Option<u64> {
|
||||
use rustix::fs::{AtFlags, StatxFlags};
|
||||
use std::os::fd::AsRawFd as _;
|
||||
|
||||
rustix::fs::statx(fd, "", AtFlags::EMPTY_PATH, StatxFlags::MNT_ID)
|
||||
.ok()
|
||||
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
|
||||
.map(|statx| statx.stx_mnt_id)
|
||||
.or_else(|| mount_id_from_fdinfo(fd.as_raw_fd()))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_for_path(path: &Path) -> Option<u64> {
|
||||
use rustix::fs::{AtFlags, CWD, StatxFlags};
|
||||
|
||||
rustix::fs::statx(CWD, path, AtFlags::empty(), StatxFlags::MNT_ID)
|
||||
.ok()
|
||||
.filter(|statx| StatxFlags::from_bits_retain(statx.stx_mask).contains(StatxFlags::MNT_ID))
|
||||
.map(|statx| statx.stx_mnt_id)
|
||||
.or_else(|| {
|
||||
std::fs::canonicalize(path)
|
||||
.ok()
|
||||
.and_then(|path| mount_id_from_mountinfo(&path))
|
||||
fn statx_replacement_mount_identity(
|
||||
dir: impl std::os::fd::AsFd,
|
||||
path: &Path,
|
||||
flags: rustix::fs::AtFlags,
|
||||
) -> Option<ReplacementMountIdentity> {
|
||||
use rustix::fs::{FileType, StatxFlags, makedev, statx};
|
||||
let fields = StatxFlags::TYPE | StatxFlags::INO | StatxFlags::MNT_ID;
|
||||
let stat = statx(dir, path, flags, fields).ok()?;
|
||||
(StatxFlags::from_bits_retain(stat.stx_mask).contains(fields) && FileType::from_raw_mode(stat.stx_mode.into()).is_dir())
|
||||
.then_some(ReplacementMountIdentity {
|
||||
device: makedev(stat.stx_dev_major, stat.stx_dev_minor),
|
||||
inode: stat.stx_ino,
|
||||
mount_id: stat.stx_mnt_id,
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_fdinfo(fd: std::os::fd::RawFd) -> Option<u64> {
|
||||
std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
|
||||
fn replacement_mount_identity_for_fd(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
|
||||
statx_replacement_mount_identity(directory, Path::new(""), rustix::fs::AtFlags::EMPTY_PATH)
|
||||
.or_else(|| replacement_mount_identity_from_fdinfo(directory))
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity_for_path(path: &Path) -> Option<ReplacementMountIdentity> {
|
||||
use rustix::fs::{AtFlags, CWD, Mode, OFlags, open};
|
||||
statx_replacement_mount_identity(CWD, path, AtFlags::empty()).or_else(|| {
|
||||
let directory = std::fs::File::from(open(path, OFlags::PATH | OFlags::DIRECTORY | OFlags::CLOEXEC, Mode::empty()).ok()?);
|
||||
replacement_mount_identity_from_fdinfo(&directory)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn replacement_mount_identity_from_fdinfo(directory: &std::fs::File) -> Option<ReplacementMountIdentity> {
|
||||
use std::os::{fd::AsRawFd as _, unix::fs::MetadataExt as _};
|
||||
let fd = directory.as_raw_fd();
|
||||
let mount_id = std::fs::read_to_string(format!("/proc/self/fdinfo/{fd}"))
|
||||
.ok()?
|
||||
.lines()
|
||||
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_mountinfo(path: &Path) -> Option<u64> {
|
||||
let mountinfo = std::fs::read_to_string("/proc/self/mountinfo").ok()?;
|
||||
mount_id_from_mountinfo_contents(&mountinfo, path)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn mount_id_from_mountinfo_contents(mountinfo: &str, path: &Path) -> Option<u64> {
|
||||
let mountpoint = path
|
||||
.to_string_lossy()
|
||||
.replace('\\', "\\134")
|
||||
.replace('\t', "\\011")
|
||||
.replace('\n', "\\012")
|
||||
.replace(' ', "\\040");
|
||||
mountinfo.lines().find_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
let mount_id = fields.next()?.parse().ok()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
(fields.next()? == mountpoint).then_some(mount_id)
|
||||
.find_map(|line| line.strip_prefix("mnt_id:")?.trim().parse().ok())?;
|
||||
let metadata = directory.metadata().ok()?;
|
||||
Some(ReplacementMountIdentity {
|
||||
device: metadata.dev(),
|
||||
inode: metadata.ino(),
|
||||
mount_id,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -7910,26 +7889,55 @@ impl LocalDisk {
|
||||
/// configured endpoint still names the directory held by `mount_lease`.
|
||||
pub fn has_replacement_mount_lease(&self) -> bool {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let Ok(configured) = std::fs::metadata(self.endpoint.get_file_path()) else {
|
||||
return false;
|
||||
};
|
||||
let Ok(pinned) = self.mount_lease.metadata() else {
|
||||
return false;
|
||||
};
|
||||
let Some(configured_mount_id) = mount_id_for_path(Path::new(&self.endpoint.get_file_path())) else {
|
||||
return false;
|
||||
};
|
||||
let Some(pinned_mount_id) = self.mount_lease_mount_id else {
|
||||
return false;
|
||||
};
|
||||
replacement_mount_identity(&configured, configured_mount_id) == replacement_mount_identity(&pinned, pinned_mount_id)
|
||||
}
|
||||
return self
|
||||
.current_replacement_mount_identity(Path::new(&self.endpoint.get_file_path()))
|
||||
.is_some();
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
false
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn current_replacement_mount_identity(&self, configured_path: &Path) -> Option<ReplacementMountIdentity> {
|
||||
let pinned = self.mount_lease_identity?;
|
||||
(replacement_mount_identity_for_path(configured_path)? == pinned).then_some(pinned)
|
||||
}
|
||||
|
||||
pub(super) fn replacement_mount(&self, _mount_roots: &[PathBuf]) -> Option<super::ReplacementMount> {
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
let configured_path = self.endpoint.get_file_path();
|
||||
let configured_path = Path::new(&configured_path);
|
||||
let canonical_path = std::fs::canonicalize(&self.io_root).ok()?;
|
||||
let expected = _mount_roots
|
||||
.iter()
|
||||
.filter(|root| root.is_absolute() && (configured_path.starts_with(root) || canonical_path.starts_with(root)))
|
||||
.max_by_key(|root| root.components().count())
|
||||
.unwrap_or(&canonical_path);
|
||||
// The expected root must be its own mount boundary. If it is
|
||||
// unmounted, the exposed directory and its parent share a mount ID.
|
||||
if !canonical_path.starts_with(expected) {
|
||||
return None;
|
||||
}
|
||||
// When the configured endpoint is the expected mount root, this
|
||||
// live observation also supplies the mount ID for the boundary.
|
||||
let identity = self.current_replacement_mount_identity(configured_path)?;
|
||||
if (expected != configured_path && replacement_mount_identity_for_path(expected)?.mount_id != identity.mount_id)
|
||||
|| replacement_mount_identity_for_path(expected.parent()?)?.mount_id == identity.mount_id
|
||||
{
|
||||
return None;
|
||||
}
|
||||
Some(super::ReplacementMount {
|
||||
canonical_path,
|
||||
physical_device_ids: rustfs_utils::os::get_physical_device_ids(self.io_root.to_str()?).ok()?,
|
||||
filesystem_identity: format!("{}:{}:{}", identity.mount_id, identity.device, identity.inode),
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
None
|
||||
}
|
||||
|
||||
/// Return the descriptor-rooted path for the mount instance admitted for
|
||||
/// automatic replacement. Callers must not derive destructive identity
|
||||
/// from the mutable endpoint pathname.
|
||||
@@ -7980,10 +7988,10 @@ impl LocalDisk {
|
||||
let root = publication_root.path().to_path_buf();
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
let (mount_lease, io_root, mount_lease_mount_id) = (
|
||||
let (mount_lease, io_root, mount_lease_identity) = (
|
||||
publication_root.directory().clone(),
|
||||
publication_root.io_path().to_path_buf(),
|
||||
mount_id_for_fd(publication_root.directory()),
|
||||
replacement_mount_identity_for_fd(publication_root.directory()),
|
||||
);
|
||||
#[cfg(not(target_os = "linux"))]
|
||||
let io_root = Self::open_mount_lease(&root)?;
|
||||
@@ -8139,7 +8147,7 @@ impl LocalDisk {
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease: mount_lease.clone(),
|
||||
#[cfg(target_os = "linux")]
|
||||
mount_lease_mount_id,
|
||||
mount_lease_identity,
|
||||
endpoint: ep.clone(),
|
||||
format_path,
|
||||
io_format_path,
|
||||
@@ -37809,10 +37817,18 @@ mod test {
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[test]
|
||||
fn mountinfo_fallback_uses_the_exact_escaped_mountpoint() {
|
||||
let mountinfo = "101 1 0:30 / / rw - rootfs rootfs rw\n202 101 0:42 / /mnt/replacement\\040disk rw - tmpfs tmpfs rw\n";
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement disk")), Some(202));
|
||||
assert_eq!(mount_id_from_mountinfo_contents(mountinfo, Path::new("/mnt/replacement")), None);
|
||||
fn mount_id_fdinfo_can_identify_a_directory_below_a_mount() {
|
||||
let temp = tempfile::tempdir().expect("mount fixture directory");
|
||||
let child_path = temp.path().join("data");
|
||||
std::fs::create_dir(&child_path).expect("create data subdirectory");
|
||||
let parent = std::fs::File::open(temp.path()).expect("open parent directory");
|
||||
let child = std::fs::File::open(&child_path).expect("open data subdirectory");
|
||||
let parent_identity = replacement_mount_identity_from_fdinfo(&parent).expect("read parent identity");
|
||||
let child_identity = replacement_mount_identity_from_fdinfo(&child).expect("read child identity");
|
||||
assert_eq!(child_identity.mount_id, parent_identity.mount_id);
|
||||
assert_ne!(child_identity.inode, parent_identity.inode);
|
||||
assert_eq!(replacement_mount_identity_for_fd(&child), Some(child_identity));
|
||||
assert_eq!(replacement_mount_identity_for_path(&child_path), Some(child_identity));
|
||||
}
|
||||
|
||||
#[cfg(not(windows))]
|
||||
|
||||
@@ -1777,6 +1777,28 @@ impl LocalDisk {
|
||||
})
|
||||
}
|
||||
|
||||
/// Read the format through the owning disk's pinned mount, bypassing cached IDs.
|
||||
pub(crate) async fn delete_residue_disk_id(&self) -> Result<Uuid> {
|
||||
let bytes = self.read_format_json().await?;
|
||||
let format = FormatV3::try_from(bytes.as_slice()).map_err(|_| DiskError::CorruptedBackend)?;
|
||||
let id = format.erasure.this;
|
||||
let (set, slot) = format.find_disk_index_by_disk_id(id)?;
|
||||
if id.is_nil()
|
||||
|| usize::try_from(self.endpoint.set_idx).ok() != Some(set)
|
||||
|| usize::try_from(self.endpoint.disk_idx).ok() != Some(slot)
|
||||
{
|
||||
return Err(DiskError::InconsistentDisk);
|
||||
}
|
||||
Ok(id)
|
||||
}
|
||||
|
||||
pub(crate) async fn require_delete_residue_disk_id(&self, expected: Uuid) -> Result<()> {
|
||||
if expected.is_nil() || self.delete_residue_disk_id().await? != expected {
|
||||
return Err(DiskError::OutdatedXLMeta);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub(crate) async fn delete_recovery_settle(
|
||||
&self,
|
||||
volume: &str,
|
||||
@@ -1967,7 +1989,13 @@ impl LocalDisk {
|
||||
return Ok(observed);
|
||||
}
|
||||
}
|
||||
if action == DeleteRecoveryAction::DiscardLegacy {
|
||||
if matches!(
|
||||
action,
|
||||
DeleteRecoveryAction::DiscardLegacy | DeleteRecoveryAction::DiscardLegacyOnDisk { .. }
|
||||
) {
|
||||
if let DeleteRecoveryAction::DiscardLegacyOnDisk { disk_id } = action {
|
||||
self.require_delete_residue_disk_id(disk_id).await?;
|
||||
}
|
||||
if observed.state != DeleteRecoveryState::LegacyAmbiguous || !observed.explicit_delete_safe {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
@@ -2416,7 +2444,9 @@ impl LocalDisk {
|
||||
}
|
||||
}
|
||||
DeleteRecoveryAction::AdoptCommit { .. } => return Err(DiskError::MethodNotAllowed),
|
||||
DeleteRecoveryAction::DiscardLegacy => return Err(DiskError::MethodNotAllowed),
|
||||
DeleteRecoveryAction::DiscardLegacy | DeleteRecoveryAction::DiscardLegacyOnDisk { .. } => {
|
||||
return Err(DiskError::MethodNotAllowed);
|
||||
}
|
||||
}
|
||||
self.delete_recovery_inspect_owned(volume, object, transaction, owner).await
|
||||
}
|
||||
|
||||
@@ -104,6 +104,7 @@ use sha2::Digest as _;
|
||||
use std::{fmt::Debug, path::PathBuf, sync::Arc, time::Duration};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::io::{AsyncRead, AsyncWrite};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use uuid::Uuid;
|
||||
|
||||
/// Independent admission and physical ownership for one disk rename.
|
||||
@@ -224,6 +225,14 @@ pub enum PoolMetaNativeRecoveryResponse {
|
||||
|
||||
pub type DiskStore = Arc<Disk>;
|
||||
|
||||
/// Filesystem evidence for a replacement data directory on its expected mount.
|
||||
#[derive(Debug)]
|
||||
pub struct ReplacementMount {
|
||||
pub canonical_path: PathBuf,
|
||||
pub physical_device_ids: Vec<String>,
|
||||
pub filesystem_identity: String,
|
||||
}
|
||||
|
||||
pub type FileReader = Box<dyn AsyncRead + Send + Sync + Unpin>;
|
||||
pub type FileWriter = Box<dyn AsyncWrite + Send + Sync + Unpin>;
|
||||
|
||||
@@ -325,6 +334,76 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Identify the physical participant that supports identity-bound legacy discard.
|
||||
pub async fn probe_delete_residue_disk(&self, volume: &str) -> Result<Uuid> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().delete_residue_disk_id().await,
|
||||
Self::Remote(disk) => disk.probe_delete_residue_disk(volume).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enumerate one page on the same physical participant verified before and after traversal.
|
||||
pub async fn list_delete_residue_on_disk(
|
||||
&self,
|
||||
volume: &str,
|
||||
expected: Uuid,
|
||||
cursor: Option<DeleteRecoveryCursor>,
|
||||
limit: u32,
|
||||
) -> Result<DeleteRecoveryPage> {
|
||||
if expected.is_nil() || !(1..=MAX_DELETE_RESIDUE_PAGE_SIZE).contains(&limit) {
|
||||
return Err(Error::InvalidPath);
|
||||
}
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
let local = disk.get_disk();
|
||||
local.require_delete_residue_disk_id(expected).await?;
|
||||
let page = disk.discover_delete_transactions(volume, None, cursor, limit).await?;
|
||||
local.require_delete_residue_disk_id(expected).await?;
|
||||
Ok(page)
|
||||
}
|
||||
Self::Remote(disk) => disk.list_delete_residue_on_disk(volume, expected, cursor, limit).await,
|
||||
}
|
||||
}
|
||||
|
||||
/// Bound the physical disk identity to the same owner that performs a single
|
||||
/// bounded discovery or inspection. An old peer cannot substitute an unbound reply.
|
||||
pub async fn read_delete_residue_on_disk(
|
||||
&self,
|
||||
volume: &str,
|
||||
object: &str,
|
||||
disk_id: Uuid,
|
||||
transaction: Option<Uuid>,
|
||||
max_entries: u32,
|
||||
) -> Result<DeleteRecoveryRpcReply> {
|
||||
match self {
|
||||
Self::Local(disk) => {
|
||||
disk.get_disk().require_delete_residue_disk_id(disk_id).await?;
|
||||
match transaction {
|
||||
Some(id) => disk
|
||||
.inspect_delete_transaction_bounded(volume, object, id, max_entries)
|
||||
.await
|
||||
.map(DeleteRecoveryRpcReply::InspectedBounded),
|
||||
None => disk
|
||||
.discover_delete_transactions_bounded(volume, object, max_entries)
|
||||
.await
|
||||
.map(DeleteRecoveryRpcReply::DiscoveredBounded),
|
||||
}
|
||||
}
|
||||
Self::Remote(disk) => {
|
||||
disk.recover_delete_transaction(
|
||||
volume,
|
||||
DeleteRecoveryRpcOperation::ReadResidueOnDisk {
|
||||
object: object.to_owned(),
|
||||
disk_id,
|
||||
transaction,
|
||||
max_entries,
|
||||
},
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A read-only probe; the physical delete repeats the same leaf checks.
|
||||
pub async fn is_plain_orphan_data_dir(&self, volume: &str, path: &str) -> Result<bool> {
|
||||
match self {
|
||||
@@ -1418,6 +1497,15 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe a local wrapper's retirement. Remote handles have their own RPC lifecycle.
|
||||
/// Cancelling the returned child token does not retire the underlying wrapper.
|
||||
pub fn local_retirement_token(&self) -> Option<CancellationToken> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => Some(local_disk.retirement_token()),
|
||||
Disk::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn offline_duration_secs(&self) -> Option<u64> {
|
||||
match self {
|
||||
Disk::Local(local_disk) => local_disk.offline_duration_secs(),
|
||||
@@ -1541,6 +1629,15 @@ impl Disk {
|
||||
}
|
||||
}
|
||||
|
||||
/// Observe the expected mount through the same local disk handle that will
|
||||
/// execute recovery. Call from a blocking context.
|
||||
pub fn replacement_mount(&self, mount_roots: &[PathBuf]) -> Option<ReplacementMount> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().replacement_mount(mount_roots),
|
||||
Self::Remote(_) => None,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn acquire_replacement_execution_lease(&self) -> Result<std::sync::Arc<local::ReplacementExecutionLease>> {
|
||||
match self {
|
||||
Self::Local(disk) => disk.get_disk().acquire_replacement_execution_lease().await,
|
||||
@@ -2570,6 +2667,7 @@ pub enum DeleteRecoveryAction {
|
||||
Rollback,
|
||||
DiscardLegacy,
|
||||
AdoptCommit { authority: Box<DeleteRecoveryJournal> },
|
||||
DiscardLegacyOnDisk { disk_id: Uuid },
|
||||
}
|
||||
|
||||
/// A complete inventory or a work-limit refusal. No public discovery cursor is allocated.
|
||||
@@ -2582,6 +2680,7 @@ pub struct BoundedDeleteRecoveryDiscovery {
|
||||
}
|
||||
|
||||
pub const DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES: u32 = 4096;
|
||||
pub const MAX_DELETE_RESIDUE_PAGE_SIZE: u32 = 64;
|
||||
pub const DELETE_RECOVERY_MAX_INSPECTION_ENTRIES: u32 = 65536;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
@@ -2659,6 +2758,18 @@ pub enum DeleteRecoveryRpcOperation {
|
||||
Exists {
|
||||
object: String,
|
||||
},
|
||||
ReadResidueOnDisk {
|
||||
object: String,
|
||||
disk_id: Uuid,
|
||||
transaction: Option<Uuid>,
|
||||
max_entries: u32,
|
||||
},
|
||||
ProbeLegacyDiscard,
|
||||
ListResidueOnDisk {
|
||||
disk_id: Uuid,
|
||||
cursor: Option<DeleteRecoveryCursor>,
|
||||
limit: u32,
|
||||
},
|
||||
}
|
||||
|
||||
impl DeleteRecoveryRpcOperation {
|
||||
@@ -2693,6 +2804,8 @@ pub enum DeleteRecoveryRpcReply {
|
||||
AppliedVersion,
|
||||
AppliedVersions(Vec<Option<u32>>),
|
||||
Evidence { present: bool },
|
||||
LegacyDiscardDisk { disk_id: Uuid },
|
||||
ListedResidueOnDisk { disk_id: Uuid, page: DeleteRecoveryPage },
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
|
||||
@@ -61,6 +61,12 @@ struct FillRequest {
|
||||
response: oneshot::Sender<FillResult>,
|
||||
}
|
||||
|
||||
#[cfg_attr(test, derive(Debug))]
|
||||
struct FilledStripes {
|
||||
primary: Vec<u8>,
|
||||
queued: VecDeque<Vec<u8>>,
|
||||
}
|
||||
|
||||
struct FillResult {
|
||||
result: io::Result<Option<Vec<u8>>>,
|
||||
queued_buffers: VecDeque<Vec<u8>>,
|
||||
@@ -293,7 +299,7 @@ where
|
||||
Self::new_with_fill_policy_inner(source, engine, total_length, metrics_path, fill_policy)
|
||||
}
|
||||
|
||||
fn poll_fill_result(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<Vec<u8>>>> {
|
||||
fn poll_fill_result(&mut self, cx: &mut Context<'_>) -> Poll<io::Result<Option<FilledStripes>>> {
|
||||
if self.fill.is_none() {
|
||||
let fill_worker_tx = match self.fill_worker_tx() {
|
||||
Ok(tx) => tx,
|
||||
@@ -345,15 +351,6 @@ where
|
||||
if let Some(deferred_error) = deferred_error {
|
||||
self.prefetch_error = Some(deferred_error);
|
||||
}
|
||||
// Queued stripes are delivered to the client via `prefetched_bufs.pop_front()`
|
||||
// in `poll_read` without touching `self.remaining`, so their bytes must be
|
||||
// accounted for here. Otherwise, under multi-in-flight fill policies (e.g. the
|
||||
// default `DualInFlight`), `remaining` never reaches 0 and a fully delivered
|
||||
// multi-block object still terminates the GET with `LessData`. Buffers in
|
||||
// `queued_buffers` come only from `Ok(true)` decodes, so they are non-empty and
|
||||
// their total is bounded by `remaining - main_buf.len()`, ruling out underflow.
|
||||
self.remaining -= queued_buffers.iter().map(Vec::len).sum::<usize>();
|
||||
self.prefetched_bufs.extend(queued_buffers);
|
||||
|
||||
match result {
|
||||
Ok(Some(buf)) => {
|
||||
@@ -362,8 +359,17 @@ where
|
||||
}
|
||||
rustfs_io_metrics::record_get_object_reader_stripe(self.metrics_path);
|
||||
rustfs_io_metrics::record_get_object_reader_bytes(self.metrics_path, buf.len());
|
||||
// Queued stripes are delivered to the client via `prefetched_bufs.pop_front()`
|
||||
// in `poll_read` without touching `self.remaining`, so their bytes must be
|
||||
// accounted for here alongside the primary stripe. Buffers in `queued_buffers`
|
||||
// come only from `Ok(true)` decodes, so they are non-empty and their total is
|
||||
// bounded by `remaining - buf.len()`, ruling out underflow.
|
||||
self.remaining -= buf.len();
|
||||
Poll::Ready(Ok(Some(buf)))
|
||||
self.remaining -= queued_buffers.iter().map(Vec::len).sum::<usize>();
|
||||
Poll::Ready(Ok(Some(FilledStripes {
|
||||
primary: buf,
|
||||
queued: queued_buffers,
|
||||
})))
|
||||
}
|
||||
Ok(None) => {
|
||||
if self.remaining == 0 {
|
||||
@@ -401,8 +407,8 @@ where
|
||||
};
|
||||
|
||||
match fill {
|
||||
Ok(Some(buf)) => {
|
||||
let output_has_remaining = self.output_pos < self.output_buf.len();
|
||||
Ok(Some(FilledStripes { primary: buf, queued })) => {
|
||||
let output_has_remaining = self.output_pos < self.output_buf.len() || !self.prefetched_bufs.is_empty();
|
||||
let mut queued_count = 0usize;
|
||||
let mut queued_bytes = 0usize;
|
||||
|
||||
@@ -420,6 +426,7 @@ where
|
||||
self.output_buf = buf;
|
||||
self.output_pos = 0;
|
||||
}
|
||||
self.prefetched_bufs.extend(queued);
|
||||
|
||||
if !self.prefetched_bufs.is_empty() {
|
||||
let total_prefetched_bytes = self.prefetched_bufs.iter().map(Vec::len).sum::<usize>();
|
||||
@@ -451,7 +458,7 @@ where
|
||||
Poll::Ready(Ok(()))
|
||||
}
|
||||
Err(err) => {
|
||||
if self.output_pos < self.output_buf.len() {
|
||||
if self.output_pos < self.output_buf.len() || !self.prefetched_bufs.is_empty() {
|
||||
rustfs_io_metrics::record_get_object_reader_prefetch(self.metrics_path, GET_READER_PREFETCH_ERROR_DEFERRED);
|
||||
self.prefetch_error = Some(err);
|
||||
Poll::Ready(Ok(()))
|
||||
@@ -1860,6 +1867,42 @@ mod tests {
|
||||
.expect("dual inflight policy should prefetch two future stripes before current output drains");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_decode_reader_dual_inflight_preserves_stripe_order_when_prefetch_completes_before_output_drains() {
|
||||
let erasure = Erasure::new(4, 2, 16);
|
||||
let data = (0..96u8).collect::<Vec<_>>();
|
||||
let read_count = Arc::new(AtomicUsize::new(0));
|
||||
let mut source = source_from_data(&erasure, &data, &[]);
|
||||
source.read_count = Some(Arc::clone(&read_count));
|
||||
let engine = LegacyEcDecodeEngine::new(erasure);
|
||||
let mut reader = ErasureDecodeReader::new_with_fill_policy(
|
||||
source,
|
||||
engine,
|
||||
data.len(),
|
||||
GET_OBJECT_PATH_CODEC_STREAMING,
|
||||
FillPolicy::DualInFlight,
|
||||
)
|
||||
.expect("reader should be constructed");
|
||||
|
||||
let mut decoded = Vec::with_capacity(data.len());
|
||||
let mut chunk = [0u8; 5];
|
||||
loop {
|
||||
let n = reader.read(&mut chunk).await.expect("chunk read should succeed");
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
decoded.extend_from_slice(&chunk[..n]);
|
||||
// Yield between partial reads so the DualInFlight worker completes
|
||||
// a 2-stripe fill (primary + queued_buffers) while output_buf still
|
||||
// has unconsumed bytes.
|
||||
for _ in 0..4 {
|
||||
yield_now().await;
|
||||
}
|
||||
}
|
||||
|
||||
assert_eq!(decoded, data);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn erasure_decode_reader_dual_inflight_reads_multi_block_object_to_end() {
|
||||
// Regression for the byte-accounting bug: under DualInFlight each fill
|
||||
|
||||
@@ -50,6 +50,7 @@ use crate::layout::endpoints::{EndpointServerPools, SetupType};
|
||||
use crate::object_api::ObjectEncryptionResolver;
|
||||
use crate::services::event_notification::EventNotifier;
|
||||
use crate::services::tier::tier::TierConfigMgr;
|
||||
use crate::store::delete_recovery_inventory::DeleteResidueCursors;
|
||||
use crate::store::delete_recovery_runtime::DeleteRecoveryHints;
|
||||
use rustfs_lock::{GlobalLockManager, get_global_lock_manager};
|
||||
use s3s::region::Region;
|
||||
@@ -369,6 +370,7 @@ pub struct InstanceContext {
|
||||
pub(crate) usage_watches: Arc<super::usage_watch::UsageWatchRegistry>,
|
||||
/// Resolves object-encryption material at the application boundary.
|
||||
object_encryption_resolver: OnceLock<Arc<dyn ObjectEncryptionResolver>>,
|
||||
pub(crate) delete_residue_cursors: DeleteResidueCursors,
|
||||
delete_recovery_hints: DeleteRecoveryHints,
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex<HashSet<Uuid>>,
|
||||
#[cfg(test)]
|
||||
@@ -448,6 +450,7 @@ impl InstanceContext {
|
||||
namespace_commit_generation: AtomicU64::new(0),
|
||||
usage_watches: Arc::new(super::usage_watch::UsageWatchRegistry::default()),
|
||||
object_encryption_resolver: OnceLock::new(),
|
||||
delete_residue_cursors: DeleteResidueCursors::default(),
|
||||
delete_recovery_hints: DeleteRecoveryHints::default(),
|
||||
tier_delete_journal_recovery_stores: std::sync::Mutex::new(HashSet::new()),
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -31,11 +31,10 @@ use crate::{
|
||||
layout::endpoints::{EndpointServerPools, SetupType},
|
||||
runtime::global::{
|
||||
GLOBAL_BOOT_TIME, GLOBAL_LIFECYCLE_SYS, GLOBAL_LOCAL_NODE_NAME_FALLBACK, GLOBAL_ROOT_DISK_THRESHOLD,
|
||||
TypeLocalDiskSetDrives, get_background_services_cancel_token, get_global_bucket_monitor, get_global_deployment_id,
|
||||
get_global_endpoints, get_global_endpoints_opt, get_global_lock_client, get_global_lock_clients, get_global_region,
|
||||
get_global_tier_config_mgr, global_rustfs_port, init_global_bucket_monitor, is_dist_erasure, is_erasure, is_erasure_sd,
|
||||
is_first_cluster_node_local, resolve_object_store_handle, set_global_lock_client, set_global_lock_clients,
|
||||
set_object_layer, update_erasure_type,
|
||||
get_background_services_cancel_token, get_global_bucket_monitor, get_global_deployment_id, get_global_endpoints,
|
||||
get_global_endpoints_opt, get_global_lock_client, get_global_lock_clients, get_global_region, get_global_tier_config_mgr,
|
||||
global_rustfs_port, init_global_bucket_monitor, is_dist_erasure, is_erasure, is_erasure_sd, is_first_cluster_node_local,
|
||||
resolve_object_store_handle, set_global_lock_client, set_global_lock_clients, set_object_layer, update_erasure_type,
|
||||
},
|
||||
services::batch_processor::{GlobalBatchProcessors, get_global_processors},
|
||||
services::notification_sys::{NotificationSys, get_global_notification_sys},
|
||||
@@ -410,10 +409,6 @@ pub(crate) fn local_disk_id_map_handle() -> Arc<RwLock<HashMap<Uuid, String>>> {
|
||||
crate::runtime::global::current_ctx().local_disk_id_map()
|
||||
}
|
||||
|
||||
pub(crate) fn local_disk_set_drives_handle() -> Arc<RwLock<TypeLocalDiskSetDrives>> {
|
||||
crate::runtime::global::current_ctx().local_disk_set_drives()
|
||||
}
|
||||
|
||||
pub(crate) fn tier_config_mgr_handle() -> Arc<RwLock<TierConfigMgr>> {
|
||||
get_global_tier_config_mgr()
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{Arc, Mutex},
|
||||
sync::{Arc, Mutex, OnceLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use uuid::Uuid;
|
||||
@@ -88,6 +88,86 @@ pub struct ScannerUsageWatchObservation {
|
||||
pub known: Option<ScannerUsageWatchKnown>,
|
||||
}
|
||||
|
||||
pub(crate) struct UsageWatchOwners {
|
||||
pub(crate) deployment: Uuid,
|
||||
pub(crate) local: Uuid,
|
||||
pub(crate) peers: HashMap<String, Uuid>,
|
||||
}
|
||||
|
||||
impl UsageWatchOwners {
|
||||
pub(crate) fn from_topology(deployment: Uuid, topology: &crate::layout::endpoints::EndpointServerPools) -> Option<Self> {
|
||||
use crate::layout::endpoint::EndpointType;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::collections::hash_map::Entry;
|
||||
|
||||
if deployment.is_nil() || topology.as_ref().is_empty() {
|
||||
return None;
|
||||
}
|
||||
let mut has_paths = false;
|
||||
let mut nodes = HashMap::new();
|
||||
for pool in topology.as_ref() {
|
||||
if pool.endpoints.as_ref().is_empty()
|
||||
|| pool.set_count.checked_mul(pool.drives_per_set) != Some(pool.endpoints.as_ref().len())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
for endpoint in pool.endpoints.as_ref() {
|
||||
if endpoint.get_type() == EndpointType::Path {
|
||||
if !endpoint.is_local {
|
||||
return None;
|
||||
}
|
||||
has_paths = true;
|
||||
continue;
|
||||
}
|
||||
let host = endpoint.host_port();
|
||||
let grid = endpoint.grid_host();
|
||||
if host.is_empty() || grid.is_empty() {
|
||||
return None;
|
||||
}
|
||||
let route = (grid, endpoint.is_local);
|
||||
match nodes.entry(host) {
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(route);
|
||||
}
|
||||
Entry::Occupied(entry) if entry.get() == &route => {}
|
||||
Entry::Occupied(_) => return None,
|
||||
}
|
||||
}
|
||||
}
|
||||
if has_paths {
|
||||
return nodes.is_empty().then_some(Self {
|
||||
deployment,
|
||||
local: deployment,
|
||||
peers: HashMap::new(),
|
||||
});
|
||||
}
|
||||
let mut local = None;
|
||||
let mut peers = HashMap::new();
|
||||
for (host, (grid, is_local)) in nodes {
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"rustfs-scanner-usage-watch-owner-v1");
|
||||
hash.update(deployment.as_bytes());
|
||||
hash.update(host.as_bytes());
|
||||
let digest = hash.finalize();
|
||||
let mut bytes = [0; 16];
|
||||
bytes.copy_from_slice(&digest[..16]);
|
||||
let owner = uuid::Builder::from_custom_bytes(bytes).into_uuid();
|
||||
if is_local {
|
||||
if local.replace(owner).is_some() {
|
||||
return None;
|
||||
}
|
||||
} else if peers.insert(grid, owner).is_some() {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
Some(Self {
|
||||
deployment,
|
||||
local: local?,
|
||||
peers,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct Watch {
|
||||
owner: Uuid,
|
||||
token: Uuid,
|
||||
@@ -108,6 +188,7 @@ struct State {
|
||||
}
|
||||
|
||||
pub(crate) struct UsageWatchRegistry {
|
||||
pub(crate) owners: OnceLock<UsageWatchOwners>,
|
||||
session: Uuid,
|
||||
state: Mutex<State>,
|
||||
}
|
||||
@@ -126,6 +207,7 @@ pub(crate) struct UsageWatchMutationGuard {
|
||||
impl Default for UsageWatchRegistry {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
owners: OnceLock::new(),
|
||||
session: Uuid::new_v4(),
|
||||
state: Mutex::new(State {
|
||||
coverage_epoch: 1,
|
||||
|
||||
@@ -64,6 +64,10 @@ impl MetricType {
|
||||
pub const RPC: MetricType = MetricType(1 << 8);
|
||||
pub const HTTP: MetricType = MetricType(1 << 9);
|
||||
|
||||
/// Internal GetMetrics selector for a compact, generation-bound node sample.
|
||||
/// Excluded from ALL so existing HTTP and rolling-upgrade peers keep their wire shape.
|
||||
pub const SCANNER_SUMMARY: MetricType = MetricType(1 << 10);
|
||||
|
||||
// MetricsAll must be last.
|
||||
pub const ALL: MetricType = MetricType((1 << 10) - 1);
|
||||
|
||||
@@ -160,6 +164,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_scanner_metrics::metrics::ScannerMe
|
||||
last_cycle_lifecycle_transition_actions: metrics.last_cycle_lifecycle_transition_actions,
|
||||
last_cycle_partial_source: metrics.last_cycle_partial_source,
|
||||
last_cycle_partial_source_code: metrics.last_cycle_partial_source_code,
|
||||
last_cycle_partial_stage: metrics.last_cycle_partial_stage,
|
||||
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
|
||||
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
|
||||
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
|
||||
@@ -407,6 +412,27 @@ impl MetricType {
|
||||
pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) -> RealtimeMetrics {
|
||||
debug!("collect_local_metrics");
|
||||
let mut real_time_metrics = RealtimeMetrics::default();
|
||||
if types.0 == MetricType::SCANNER_SUMMARY.0 {
|
||||
let sample = global_metrics().scanner_node_observation().await;
|
||||
real_time_metrics.scanner_node = Some(rustfs_madmin::metrics::ScannerNodeSample {
|
||||
active: sample.active,
|
||||
leader: sample.leader,
|
||||
cycle: sample.cycle,
|
||||
epoch: sample.epoch,
|
||||
started_at: sample.started_at,
|
||||
objects_scanned: sample.objects_scanned,
|
||||
last_attempt: sample
|
||||
.last_attempt
|
||||
.map(|attempt| rustfs_madmin::metrics::ScannerAttemptSample {
|
||||
epoch: attempt.epoch,
|
||||
cycle: attempt.cycle,
|
||||
result: attempt.result.to_string(),
|
||||
partial_reason: attempt.partial_reason.map(str::to_string),
|
||||
duration_seconds: attempt.duration.as_secs_f64(),
|
||||
}),
|
||||
});
|
||||
return real_time_metrics;
|
||||
}
|
||||
if types.0 == MetricType::NONE.0 {
|
||||
info!("types is None, return");
|
||||
return real_time_metrics;
|
||||
@@ -691,6 +717,7 @@ mod test {
|
||||
current_started: chrono_to_jiff_timestamp(current_started),
|
||||
last_cycle_partial_source: "usage".to_string(),
|
||||
last_cycle_partial_source_code: 1,
|
||||
last_cycle_partial_stage: Some("object_budget".into()),
|
||||
cycle_timeout_total: 3,
|
||||
cycle_recovery_required_total: 2,
|
||||
cycle_last_progress_age: 17,
|
||||
@@ -706,6 +733,7 @@ mod test {
|
||||
assert_eq!(scanner.current_started, chrono_to_jiff_timestamp(current_started));
|
||||
assert_eq!(scanner.last_cycle_partial_source, "usage");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 1);
|
||||
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert_eq!(scanner.cycle_timeout_total, 3);
|
||||
assert_eq!(scanner.cycle_recovery_required_total, 2);
|
||||
assert_eq!(scanner.cycle_last_progress_age, 17);
|
||||
|
||||
@@ -3661,6 +3661,9 @@ impl TierConfigMgr {
|
||||
}
|
||||
}
|
||||
}
|
||||
if !rustfs_data_usage::is_scannable_tier_name(&tier_config.name) {
|
||||
return Err(tier_invalid_config("Remote tier name cannot be represented in lifecycle accounting"));
|
||||
}
|
||||
self.ensure_generation_is_idle(&tier_config.name)?;
|
||||
let tier_name = tier_config.name.clone();
|
||||
if tier_name != tier_name.to_uppercase() {
|
||||
@@ -8435,6 +8438,20 @@ mod tests {
|
||||
assert_eq!(legacy_gcs.gcs.as_ref().expect("GCS payload should remain").name, "COLD-LEGACY");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn add_rejects_names_that_scanner_cannot_account_for() {
|
||||
let mut mgr = empty_mgr();
|
||||
let long_name = "X".repeat(257);
|
||||
for name in ["COLD\n", rustfs_data_usage::UNKNOWN_TIER, long_name.as_str()] {
|
||||
let err = mgr
|
||||
.add(build_s3_tier(name), true)
|
||||
.await
|
||||
.expect_err("tier name must fail admission");
|
||||
assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code);
|
||||
assert!(mgr.tiers.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_add_rejects_non_uppercase_name() {
|
||||
let mut mgr = empty_mgr();
|
||||
|
||||
@@ -27,6 +27,8 @@ use std::collections::{BTreeMap, BTreeSet, HashSet};
|
||||
use std::sync::Arc;
|
||||
use uuid::Uuid;
|
||||
|
||||
mod operator;
|
||||
|
||||
const DISCOVERY_PAGE_SIZE: u32 = 128;
|
||||
const MAX_DISCOVERED_TRANSACTIONS: usize = 1_000_000;
|
||||
/// Inline cleanup attempts after a certified decision. Later attempts start
|
||||
@@ -227,6 +229,18 @@ fn validate(inspection: &DeleteRecoveryInspection, object: &str, transaction: Uu
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn explicit_legacy_copy_safe(copy: &DeleteRecoveryInspection) -> bool {
|
||||
copy.current_digest.is_none()
|
||||
&& match copy.state {
|
||||
DeleteRecoveryState::Absent => true,
|
||||
DeleteRecoveryState::LegacyAmbiguous => copy.explicit_delete_safe,
|
||||
DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => {
|
||||
copy.journal.as_ref().is_some_and(|journal| journal.target_digest.is_none())
|
||||
}
|
||||
DeleteRecoveryState::NewPrepared => false,
|
||||
}
|
||||
}
|
||||
|
||||
async fn inspect(
|
||||
disks: &[Option<DiskStore>],
|
||||
bucket: &str,
|
||||
@@ -263,6 +277,15 @@ async fn discover_bounded(
|
||||
) -> Result<Vec<DeleteTransactionId>> {
|
||||
let admitted = budget.admit(crate::disk::DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
|
||||
let page = disk.discover_delete_transactions_bounded(bucket, object, admitted).await?;
|
||||
bounded_discovery_ids(page, object, admitted, budget)
|
||||
}
|
||||
|
||||
fn bounded_discovery_ids(
|
||||
page: crate::disk::BoundedDeleteRecoveryDiscovery,
|
||||
object: &str,
|
||||
admitted: u32,
|
||||
budget: &crate::object_api::DeleteRecoveryWorkBudget,
|
||||
) -> Result<Vec<DeleteTransactionId>> {
|
||||
if page.visited > admitted
|
||||
|| page.transactions.len() > 256
|
||||
|| page
|
||||
@@ -983,18 +1006,7 @@ impl SetDisks {
|
||||
// A fresh authorized removal may discard safe legacy residue beside
|
||||
// certified absence receipts. Every participant must have no live metadata;
|
||||
// the receipts remain behind the key-wide barrier until all groups settle.
|
||||
let explicit_legacy_removal = explicit_delete
|
||||
&& all_states.values().flatten().all(|copy| {
|
||||
copy.current_digest.is_none()
|
||||
&& match copy.state {
|
||||
DeleteRecoveryState::Absent => true,
|
||||
DeleteRecoveryState::LegacyAmbiguous => copy.explicit_delete_safe,
|
||||
DeleteRecoveryState::QuorumCommitted | DeleteRecoveryState::CommitFinished => {
|
||||
copy.journal.as_ref().is_some_and(|journal| journal.target_digest.is_none())
|
||||
}
|
||||
DeleteRecoveryState::NewPrepared => false,
|
||||
}
|
||||
});
|
||||
let explicit_legacy_removal = explicit_delete && all_states.values().flatten().all(explicit_legacy_copy_safe);
|
||||
// This is a fresh authorized delete of an absent unversioned key,
|
||||
// rather than evidence about the outcome of a historical transaction.
|
||||
let fresh_absence_decision = explicit_delete
|
||||
@@ -3102,13 +3114,9 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_set_legacy_requires_explicit_removal() {
|
||||
let bucket = "recover-legacy-remove";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![42; 4096], false).await;
|
||||
async fn interrupt_legacy_delete(disks: &[DiskStore], bucket: &str) -> Uuid {
|
||||
let transaction = Uuid::new_v4();
|
||||
for disk in &disks {
|
||||
for disk in disks {
|
||||
disk.delete_version(
|
||||
bucket,
|
||||
"object",
|
||||
@@ -3125,6 +3133,229 @@ mod tests {
|
||||
.await
|
||||
.expect("create undecided historical delete shape");
|
||||
}
|
||||
transaction
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_operator_discard_requires_complete_unchanged_evidence() {
|
||||
let bucket = "recover-operator-proof";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![42; 4096], false).await;
|
||||
let transaction = interrupt_legacy_delete(&disks, bucket).await;
|
||||
let root = dirs[0].path().join(bucket).join("object");
|
||||
let backup = root.join(transaction.to_string()).join("xl.meta.bkp");
|
||||
let original = std::fs::read(&backup).expect("historical bytes");
|
||||
let preview = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("inspect without writes");
|
||||
assert!(preview.complete, "{preview:?}");
|
||||
assert_eq!(preview.transactions.len(), 1);
|
||||
assert_eq!(preview.participants.len(), 4);
|
||||
assert_eq!(preview.transactions[0].copies.len(), 4);
|
||||
let proof = preview.fingerprint.expect("eligible legacy layout");
|
||||
assert_eq!(proof.len(), 64);
|
||||
|
||||
let sibling = root.join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir(&sibling).expect("introduce a later sibling");
|
||||
std::fs::write(sibling.join("xl.meta.bkp"), &original).expect("seed sibling evidence");
|
||||
assert!(matches!(
|
||||
store.discard_delete_residue(bucket, "object", 0, 0, &proof).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
assert_eq!(std::fs::read(&backup).expect("first sibling retained"), original);
|
||||
assert_eq!(std::fs::read(sibling.join("xl.meta.bkp")).expect("new sibling retained"), original);
|
||||
std::fs::remove_dir_all(sibling).expect("restore fixture to previewed evidence");
|
||||
|
||||
let copy = disks[0]
|
||||
.inspect_delete_transaction(bucket, "object", transaction)
|
||||
.await
|
||||
.expect("disk evidence");
|
||||
let wrong_disk = disks[1].get_disk_id().await.expect("other disk").expect("formatted");
|
||||
for transaction in [None, Some(transaction)] {
|
||||
assert!(matches!(
|
||||
disks[0]
|
||||
.read_delete_residue_on_disk(bucket, "object", wrong_disk, transaction, 4096)
|
||||
.await,
|
||||
Err(DiskError::OutdatedXLMeta)
|
||||
));
|
||||
}
|
||||
assert!(matches!(
|
||||
disks[0]
|
||||
.settle_delete_transaction(
|
||||
bucket,
|
||||
"object",
|
||||
transaction,
|
||||
copy.fingerprint,
|
||||
DeleteRecoveryAction::DiscardLegacyOnDisk { disk_id: wrong_disk }
|
||||
)
|
||||
.await,
|
||||
Err(DiskError::OutdatedXLMeta)
|
||||
));
|
||||
assert_eq!(std::fs::read(&backup).expect("wrong disk retained all evidence"), original);
|
||||
|
||||
// An actual missing slot invalidates the same fingerprint; the preview reports
|
||||
// the incomplete per-drive picture even though the key is visible on a quorum.
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let offline = set.disks.write().await[3].take();
|
||||
let incomplete = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("partial evidence");
|
||||
assert!(!incomplete.complete && incomplete.fingerprint.is_none());
|
||||
assert!(matches!(
|
||||
store.discard_delete_residue(bucket, "object", 0, 0, &proof).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
set.disks.write().await[3] = offline;
|
||||
|
||||
// A live copy appearing after the preview must not become a fresh ordinary DELETE.
|
||||
let current = root.join("xl.meta");
|
||||
std::fs::write(¤t, &original).expect("publish a live generation");
|
||||
let live = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("inspect current object");
|
||||
assert!(live.fingerprint.is_none());
|
||||
assert!(matches!(
|
||||
store.discard_delete_residue(bucket, "object", 0, 0, &proof).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
assert_eq!(std::fs::read(¤t).expect("live object untouched"), original);
|
||||
std::fs::remove_file(current).expect("restore absent fixture");
|
||||
|
||||
let proof = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("fresh preview")
|
||||
.fingerprint
|
||||
.expect("eligible");
|
||||
let discarded = store
|
||||
.discard_delete_residue(bucket, "object", 0, 0, &proof)
|
||||
.await
|
||||
.expect("authorized exact discard");
|
||||
assert_eq!(discarded.reclaimed, 1);
|
||||
assert!(discarded.fingerprint.is_none());
|
||||
assert!(dirs.iter().all(|dir| {
|
||||
!dir.path()
|
||||
.join(bucket)
|
||||
.join("object")
|
||||
.join(transaction.to_string())
|
||||
.join("xl.meta.bkp")
|
||||
.exists()
|
||||
}));
|
||||
assert!(matches!(
|
||||
store.discard_delete_residue(bucket, "object", 0, 0, &proof).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_operator_rejects_versioning_and_object_lock_after_preview() {
|
||||
use crate::bucket::metadata::{BUCKET_VERSIONING_CONFIG, OBJECT_LOCK_CONFIG};
|
||||
for (bucket, config, xml) in [
|
||||
(
|
||||
"recover-operator-enabled",
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
"<VersioningConfiguration><Status>Enabled</Status></VersioningConfiguration>",
|
||||
),
|
||||
(
|
||||
"recover-operator-suspended",
|
||||
BUCKET_VERSIONING_CONFIG,
|
||||
"<VersioningConfiguration><Status>Suspended</Status></VersioningConfiguration>",
|
||||
),
|
||||
(
|
||||
"recover-operator-lock",
|
||||
OBJECT_LOCK_CONFIG,
|
||||
"<ObjectLockConfiguration><ObjectLockEnabled>Enabled</ObjectLockEnabled></ObjectLockConfiguration>",
|
||||
),
|
||||
] {
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![21; 4096], false).await;
|
||||
let transaction = interrupt_legacy_delete(&disks, bucket).await;
|
||||
let backup = dirs[0]
|
||||
.path()
|
||||
.join(bucket)
|
||||
.join("object")
|
||||
.join(transaction.to_string())
|
||||
.join("xl.meta.bkp");
|
||||
let original = std::fs::read(&backup).expect("historical bytes");
|
||||
let pre = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("first inspection");
|
||||
let proof = pre.fingerprint.expect("eligible before bucket config change");
|
||||
crate::bucket::metadata_sys::update_in(&store.ctx, bucket, config, xml.as_bytes().to_vec())
|
||||
.await
|
||||
.expect("publish bucket protection");
|
||||
let blocked = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("inspect protected residue");
|
||||
assert_eq!(blocked.reason, "bucket_protected");
|
||||
assert!(blocked.fingerprint.is_none());
|
||||
assert!(matches!(
|
||||
store.discard_delete_residue(bucket, "object", 0, 0, &proof).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
assert_eq!(std::fs::read(backup).expect("protected evidence retained"), original);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_operator_preview_does_not_survive_bucket_recreation() {
|
||||
let bucket = "recover-operator-recreate";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, b"stable bytes", false).await;
|
||||
interrupt_legacy_delete(&disks, bucket).await;
|
||||
let old = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("old generation");
|
||||
let old_proof = old.fingerprint.expect("historical shape eligible");
|
||||
// Preserve every original byte and drive slot while changing only bucket identity.
|
||||
for dir in &dirs {
|
||||
std::fs::rename(dir.path().join(bucket).join("object"), dir.path().join("saved-residue"))
|
||||
.expect("hold original evidence");
|
||||
}
|
||||
store
|
||||
.delete_bucket(bucket, &DeleteBucketOptions::default())
|
||||
.await
|
||||
.expect("delete empty original bucket");
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("recreate bucket");
|
||||
for dir in &dirs {
|
||||
std::fs::rename(dir.path().join("saved-residue"), dir.path().join(bucket).join("object"))
|
||||
.expect("restore original evidence");
|
||||
}
|
||||
let new = store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("new generation");
|
||||
assert_ne!(old.incarnation, new.incarnation);
|
||||
assert_eq!(old.transactions, new.transactions);
|
||||
assert_ne!(new.fingerprint.as_deref(), Some(old_proof.as_str()));
|
||||
assert!(matches!(
|
||||
store.discard_delete_residue(bucket, "object", 0, 0, &old_proof).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
assert_eq!(
|
||||
store
|
||||
.inspect_delete_residue(bucket, "object", 0, 0)
|
||||
.await
|
||||
.expect("unchanged evidence"),
|
||||
new
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_set_legacy_requires_explicit_removal() {
|
||||
let bucket = "recover-legacy-remove";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
put(&store, bucket, &vec![42; 4096], false).await;
|
||||
let transaction = interrupt_legacy_delete(&disks, bucket).await;
|
||||
let report = store
|
||||
.recover_delete_transactions(bucket, &HealOpts::default())
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Read-only exact-key evidence and conditional legacy-only disposal for administrators.
|
||||
|
||||
use super::*;
|
||||
use crate::disk::{DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES, DELETE_RECOVERY_MAX_INSPECTION_ENTRIES, DeleteRecoveryRpcReply};
|
||||
use crate::error::Result as StorageResult;
|
||||
use crate::object_api::DeleteRecoveryWorkBudget;
|
||||
use rustfs_madmin::heal_commands::{
|
||||
DeleteResidueCopy, DeleteResidueInspection, DeleteResidueParticipant, DeleteResidueTransaction,
|
||||
};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
fn status(error: &DiskError) -> &'static str {
|
||||
if unsupported_recovery(error) {
|
||||
"unsupported"
|
||||
} else {
|
||||
match error {
|
||||
DiskError::DeleteRecoveryWorkLimit => "work_limit",
|
||||
DiskError::FileCorrupt => "corrupt",
|
||||
_ if unavailable(error) => "unavailable",
|
||||
_ => "failed",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn hex_digest(bytes: &[u8]) -> String {
|
||||
hex_simd::encode_to_string(bytes, hex_simd::AsciiCase::Lower)
|
||||
}
|
||||
|
||||
fn evidence_copy(slot: usize, inspection: &DeleteRecoveryInspection) -> DeleteResidueCopy {
|
||||
let state = match inspection.state {
|
||||
DeleteRecoveryState::Absent => "absent",
|
||||
DeleteRecoveryState::NewPrepared => "prepared",
|
||||
DeleteRecoveryState::QuorumCommitted => "committed",
|
||||
DeleteRecoveryState::CommitFinished => "finished",
|
||||
DeleteRecoveryState::LegacyAmbiguous => "legacy_ambiguous",
|
||||
};
|
||||
DeleteResidueCopy {
|
||||
slot,
|
||||
state: state.to_owned(),
|
||||
current_metadata_present: Some(inspection.current_digest.is_some()),
|
||||
backup_present: Some(inspection.backup_digest.is_some()),
|
||||
journal_present: Some(inspection.journal.is_some()),
|
||||
fingerprint: Some(hex_digest(&inspection.fingerprint)),
|
||||
}
|
||||
}
|
||||
|
||||
impl SetDisks {
|
||||
/// Retains the original exact-key and transaction guards through the discard write tail.
|
||||
pub(crate) async fn operator_delete_residue(
|
||||
&self,
|
||||
object: &str,
|
||||
mut report: DeleteResidueInspection,
|
||||
expected: Option<&str>,
|
||||
bucket_allows_discard: bool,
|
||||
expected_disks: &[Uuid],
|
||||
can_commit: &(dyn Fn() -> bool + Sync),
|
||||
) -> StorageResult<DeleteResidueInspection> {
|
||||
let bucket = &report.bucket;
|
||||
// Lock order matches normal recovery: caller's lifecycle and movement fences,
|
||||
// raw object write lock, then its set-local transaction lock.
|
||||
let object_guard = Arc::new(self.acquire_write_lock_diag("inspect_delete_residue", bucket, object).await?);
|
||||
let transaction_guard = self.acquire_delete_transaction_lock(bucket, object).await?;
|
||||
let fence = || can_commit() && !object_guard.is_lock_lost() && !transaction_guard.is_lock_lost();
|
||||
let disks = self.disk_inventory().await;
|
||||
let budget = DeleteRecoveryWorkBudget::for_disks(self.set_drive_count);
|
||||
let pages = join_all(disks.iter().map(|disk| async {
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let id = disk.probe_delete_residue_disk(bucket).await?;
|
||||
let admitted = budget.admit(DELETE_RECOVERY_MAX_DISCOVERY_ENTRIES)?;
|
||||
let page = match disk.read_delete_residue_on_disk(bucket, object, id, None, admitted).await {
|
||||
Ok(DeleteRecoveryRpcReply::DiscoveredBounded(page)) => bounded_discovery_ids(page, object, admitted, &budget),
|
||||
Ok(_) => Err(DiskError::FileCorrupt),
|
||||
Err(error) => Err(error),
|
||||
};
|
||||
Ok::<_, DiskError>((id, page))
|
||||
}))
|
||||
.await;
|
||||
let mut ids = BTreeSet::new();
|
||||
let mut disk_ids = HashSet::new();
|
||||
report.complete = disks.len() == self.set_drive_count && !disks.is_empty() && expected_disks.len() == disks.len();
|
||||
for (slot, page) in pages.into_iter().enumerate() {
|
||||
let (disk_id, discovery) = match page {
|
||||
Ok((id, Ok(page))) if expected_disks.get(slot) == Some(&id) && disk_ids.insert(id) => {
|
||||
ids.extend(page.into_iter().map(|item| item.transaction));
|
||||
(Some(id.to_string()), "complete")
|
||||
}
|
||||
Ok((id, Ok(_))) => (Some(id.to_string()), "disk_identity_mismatch"),
|
||||
Ok((id, Err(error))) => (Some(id.to_string()), status(&error)),
|
||||
Err(error) => (None, status(&error)),
|
||||
};
|
||||
report.complete &= discovery == "complete";
|
||||
report.participants.push(DeleteResidueParticipant {
|
||||
slot,
|
||||
disk_id,
|
||||
discovery: discovery.to_owned(),
|
||||
});
|
||||
}
|
||||
|
||||
let mut groups = BTreeMap::new();
|
||||
let mut legacy = false;
|
||||
let mut valid_generation = true;
|
||||
// A bounded exact-key discovery admits at most 256 records per disk.
|
||||
// A severely divergent set cannot multiply that bound into an unbounded response.
|
||||
let truncated = ids.len() > 256;
|
||||
report.complete &= !truncated;
|
||||
for id in ids.into_iter().take(256) {
|
||||
let observations = join_all(disks.iter().enumerate().map(|(slot, disk)| {
|
||||
let budget = &budget;
|
||||
async move {
|
||||
let disk = disk.as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
let physical_id = *expected_disks.get(slot).ok_or(DiskError::InconsistentDisk)?;
|
||||
let admitted = budget.admit(DELETE_RECOVERY_MAX_INSPECTION_ENTRIES)?;
|
||||
let DeleteRecoveryRpcReply::InspectedBounded(reply) = disk
|
||||
.read_delete_residue_on_disk(bucket, object, physical_id, Some(id), admitted)
|
||||
.await?
|
||||
else {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
};
|
||||
if reply.visited > admitted {
|
||||
return Err(DiskError::FileCorrupt);
|
||||
}
|
||||
budget.charge(reply.visited);
|
||||
validate(&reply.inspection, object, id)?;
|
||||
Ok(reply.inspection)
|
||||
}
|
||||
}))
|
||||
.await;
|
||||
let mut copies = Vec::with_capacity(observations.len());
|
||||
let mut states = Vec::with_capacity(observations.len());
|
||||
for (slot, observation) in observations.into_iter().enumerate() {
|
||||
match observation {
|
||||
Ok(mut state) => {
|
||||
legacy |= state.state == DeleteRecoveryState::LegacyAmbiguous;
|
||||
valid_generation &= state.journal.as_ref().is_none_or(|journal| {
|
||||
journal
|
||||
.bucket_incarnation
|
||||
.is_some_and(|incarnation| incarnation.to_string() == report.incarnation)
|
||||
});
|
||||
copies.push(evidence_copy(slot, &state));
|
||||
// Only the causal edges, classification and fingerprints survive
|
||||
// this iteration. Never retain full version histories or request
|
||||
// payloads across the bounded list of sibling transactions.
|
||||
state.current_versions = Vec::new();
|
||||
if let Some(journal) = &mut state.journal {
|
||||
journal.requests = Vec::new();
|
||||
journal.original_versions = Vec::new();
|
||||
journal.data_dirs = Vec::new();
|
||||
journal.deleted_versions = Vec::new();
|
||||
}
|
||||
states.push(state);
|
||||
}
|
||||
Err(error) => {
|
||||
report.complete = false;
|
||||
copies.push(DeleteResidueCopy {
|
||||
slot,
|
||||
state: status(&error).to_owned(),
|
||||
current_metadata_present: None,
|
||||
backup_present: None,
|
||||
journal_present: None,
|
||||
fingerprint: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
report.transactions.push(DeleteResidueTransaction {
|
||||
id: id.to_string(),
|
||||
copies,
|
||||
});
|
||||
groups.insert(id, states);
|
||||
}
|
||||
|
||||
let reason = if truncated {
|
||||
"work_limit"
|
||||
} else if !report.complete {
|
||||
"incomplete"
|
||||
} else if !legacy {
|
||||
"no_legacy_residue"
|
||||
} else if !bucket_allows_discard {
|
||||
"bucket_protected"
|
||||
} else if !valid_generation || causal_transaction_order(&groups).is_err() {
|
||||
"conflicting_history"
|
||||
} else if groups.values().flatten().any(|copy| copy.current_digest.is_some()) {
|
||||
"current_object_present"
|
||||
} else if !groups.values().flatten().all(explicit_legacy_copy_safe) {
|
||||
"unsafe_layout"
|
||||
} else {
|
||||
"discard_eligible"
|
||||
};
|
||||
report.reason = reason.to_owned();
|
||||
if !fence() {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if reason == "discard_eligible" {
|
||||
// Fixed-order structs and sorted transaction UUIDs cover the exact action,
|
||||
// logical/physical identities, all siblings and every disk fingerprint.
|
||||
let serialized = serde_json::to_vec(&("rustfs:discard-legacy:v1", &report)).map_err(StorageError::other)?;
|
||||
report.fingerprint = Some(hex_digest(&Sha256::digest(serialized)));
|
||||
}
|
||||
let Some(expected) = expected else { return Ok(report) };
|
||||
if report.fingerprint.as_deref() != Some(expected) {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
|
||||
let owner: Arc<dyn Send + Sync> = Arc::new((
|
||||
self.ctx.begin_namespace_commit_for_bucket(bucket),
|
||||
object_guard.clone(),
|
||||
transaction_guard.clone(),
|
||||
));
|
||||
let work = RecoveryWorkScope {
|
||||
namespace_owner: Some(&owner),
|
||||
budget: Some(&budget),
|
||||
};
|
||||
for copies in groups
|
||||
.values()
|
||||
.filter(|copies| copies.iter().any(|copy| copy.state == DeleteRecoveryState::LegacyAmbiguous))
|
||||
{
|
||||
// Compare both the physical disk ID and its original evidence on the owning node.
|
||||
// A late reply or a partial failure requires a new preview of the remaining evidence.
|
||||
let results = join_all(
|
||||
copies
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, copy)| copy.state == DeleteRecoveryState::LegacyAmbiguous)
|
||||
.map(|(slot, copy)| {
|
||||
let fence = &fence;
|
||||
let disks = &disks;
|
||||
async move {
|
||||
let disk = disks[slot].as_ref().ok_or(DiskError::DiskNotFound)?;
|
||||
settle(
|
||||
disk,
|
||||
bucket,
|
||||
copy,
|
||||
DeleteRecoveryAction::DiscardLegacyOnDisk {
|
||||
disk_id: expected_disks[slot],
|
||||
},
|
||||
fence,
|
||||
work,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
for result in results {
|
||||
match result {
|
||||
Err(DiskError::DiskOngoingReq | DiskError::OutdatedXLMeta) => return Err(StorageError::PreconditionFailed),
|
||||
Ok(next) if next.state == DeleteRecoveryState::Absent && next.current_digest.is_none() => {}
|
||||
Ok(_) => return Err(StorageError::PreconditionFailed),
|
||||
Err(error) => return Err(error.into()),
|
||||
}
|
||||
}
|
||||
if !fence() {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
report.reclaimed += 1;
|
||||
}
|
||||
report.reason = "discarded".to_owned();
|
||||
report.fingerprint = None;
|
||||
Ok(report)
|
||||
}
|
||||
}
|
||||
@@ -7079,71 +7079,16 @@ mod heal_result_report_tests {
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn heal_object_with_queued_disk_renewal() {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
use crate::set_disk::core::io_primitives::disk_call_counters;
|
||||
use std::collections::HashMap;
|
||||
use std::future::Future;
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
// renew_disk still registers local disks on the ambient context. Match
|
||||
// the default serial group used by its other setup/registry fixtures,
|
||||
// and restore only this temporary endpoint, including on a failed join.
|
||||
struct RenewDiskTestState {
|
||||
ctx: Arc<InstanceContext>,
|
||||
was_dist_erasure: bool,
|
||||
map: Arc<RwLock<HashMap<String, Option<DiskStore>>>>,
|
||||
endpoint: String,
|
||||
previous_disk: Option<Option<DiskStore>>,
|
||||
}
|
||||
|
||||
impl Drop for RenewDiskTestState {
|
||||
fn drop(&mut self) {
|
||||
let ctx = self.ctx.clone();
|
||||
let was_dist_erasure = self.was_dist_erasure;
|
||||
let map = self.map.clone();
|
||||
let endpoint = self.endpoint.clone();
|
||||
let previous_disk = self.previous_disk.take();
|
||||
let handle = tokio::runtime::Handle::current();
|
||||
std::thread::spawn(move || {
|
||||
handle.block_on(async move {
|
||||
let mut map = map.write().await;
|
||||
match previous_disk {
|
||||
Some(disk) => {
|
||||
map.insert(endpoint, disk);
|
||||
}
|
||||
None => {
|
||||
map.remove(&endpoint);
|
||||
}
|
||||
}
|
||||
drop(map);
|
||||
if was_dist_erasure {
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
}
|
||||
});
|
||||
})
|
||||
.join()
|
||||
.expect("renew fixture state restoration should finish");
|
||||
}
|
||||
}
|
||||
|
||||
let (_temp_dirs, disks, set) = hermetic_set_disks_isolated(4).await;
|
||||
let endpoint = set.set_endpoints[0].clone();
|
||||
let ctx = crate::runtime::global::current_ctx();
|
||||
let map = ctx.local_disk_map();
|
||||
let restore = RenewDiskTestState {
|
||||
ctx: ctx.clone(),
|
||||
was_dist_erasure: ctx.is_dist_erasure().await,
|
||||
map: map.clone(),
|
||||
endpoint: endpoint.to_string(),
|
||||
previous_disk: map.read().await.get(&endpoint.to_string()).cloned(),
|
||||
};
|
||||
// Only distributed erasure needs an override to avoid the ambient slot array.
|
||||
if restore.was_dist_erasure {
|
||||
ctx.update_erasure_type(SetupType::Erasure).await;
|
||||
}
|
||||
crate::runtime::sources::record_local_disks(&set.ctx, disks.clone()).await;
|
||||
let map = set.ctx.local_disk_map();
|
||||
|
||||
let bucket = "heal-disk-renewal-bucket";
|
||||
let object = "heal-disk-renewal-object";
|
||||
@@ -7209,14 +7154,6 @@ mod heal_result_report_tests {
|
||||
)
|
||||
.await
|
||||
.expect("real renewal must queue its topology writer behind the read gate");
|
||||
let registered = map
|
||||
.read()
|
||||
.await
|
||||
.get(&endpoint.to_string())
|
||||
.cloned()
|
||||
.flatten()
|
||||
.expect("renewal must register the connected disk before its inventory write");
|
||||
assert!(!Arc::ptr_eq(®istered, &disks[0]), "renewal must construct a new disk handle");
|
||||
tokio::time::timeout(Duration::from_secs(5), async {
|
||||
while calls.total(disk_call_counters::KIND_READ_VERSION) < 4 {
|
||||
tokio::task::yield_now().await;
|
||||
@@ -7240,6 +7177,14 @@ mod heal_result_report_tests {
|
||||
let renewed = set.get_disks_internal().await[0]
|
||||
.clone()
|
||||
.expect("the renewed slot must remain online");
|
||||
let registered = map
|
||||
.read()
|
||||
.await
|
||||
.get(&endpoint.to_string())
|
||||
.cloned()
|
||||
.flatten()
|
||||
.expect("renewed RPC route");
|
||||
assert!(!Arc::ptr_eq(®istered, &disks[0]), "renewal must replace the original handle");
|
||||
assert!(Arc::ptr_eq(&renewed, ®istered), "the set must publish the newly connected handle");
|
||||
assert_eq!(renewed.endpoint(), endpoint);
|
||||
let format = load_format_erasure(&renewed, false)
|
||||
|
||||
@@ -31,7 +31,6 @@ use futures::{StreamExt, stream, stream::FuturesUnordered};
|
||||
use rand::prelude::SliceRandom;
|
||||
use rustfs_heal_contracts::heal_channel::{create_auto_replacement_disks_request, send_heal_request};
|
||||
use rustfs_utils::path::object_name_disk_identity;
|
||||
#[cfg(test)]
|
||||
use uuid::Uuid;
|
||||
|
||||
// Bound batching when another reconnect is still pending.
|
||||
@@ -446,7 +445,7 @@ impl SetDisks {
|
||||
// Reconnect drives concurrently without holding the inventory lock.
|
||||
let mut reconnects = stream::iter(disks.into_iter().enumerate())
|
||||
.map(|(i, opdisk)| async move {
|
||||
if let Some(disk) = opdisk {
|
||||
if let Some(disk) = &opdisk {
|
||||
if !active_monitoring
|
||||
&& disk.get_disk_location().set_idx.is_some()
|
||||
&& let disk::Disk::Local(local) = disk.as_ref()
|
||||
@@ -465,7 +464,9 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
let endpoint = self.set_endpoints.get(i)?;
|
||||
self.reconnect_disk(endpoint).await.then(|| endpoint.to_string())
|
||||
self.reconnect_disk(endpoint, opdisk.as_ref())
|
||||
.await
|
||||
.then(|| endpoint.to_string())
|
||||
})
|
||||
.buffer_unordered(MAX_ERASURE_SET_DRIVE_COUNT);
|
||||
let mut submissions = FuturesUnordered::new();
|
||||
@@ -515,26 +516,22 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
pub async fn renew_disk(&self, ep: &Endpoint) {
|
||||
if self.reconnect_disk(ep).await {
|
||||
let Some(disk_idx) = self.set_endpoints.iter().position(|endpoint| endpoint == ep) else {
|
||||
return;
|
||||
};
|
||||
let previous = self.disks.read().await[disk_idx].clone();
|
||||
if self.reconnect_disk(ep, previous.as_ref()).await {
|
||||
self.submit_replacement_heal(vec![ep.to_string()]).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// Return whether this reconnect attached a verified unformatted target.
|
||||
async fn reconnect_disk(&self, ep: &Endpoint) -> bool {
|
||||
async fn reconnect_disk(&self, ep: &Endpoint, previous: Option<&DiskStore>) -> bool {
|
||||
debug!("renew_disk: start {:?}", ep);
|
||||
|
||||
let previous_disk = {
|
||||
let disks = self.disks.read().await;
|
||||
disks
|
||||
.iter()
|
||||
.filter_map(|disk| disk.as_ref())
|
||||
.find(|disk| disk.endpoint() == *ep)
|
||||
.cloned()
|
||||
};
|
||||
let previous_health = previous_disk
|
||||
.as_ref()
|
||||
.and_then(|disk| disk.local_health_tracker_epoch_for_reconnect());
|
||||
if previous.is_some_and(|disk| disk.endpoint() != *ep) {
|
||||
return false;
|
||||
}
|
||||
let previous_health = previous.and_then(|disk| disk.local_health_tracker_epoch_for_reconnect());
|
||||
|
||||
let (new_disk, fm) = match Self::connect_endpoint(ep, previous_health).await {
|
||||
Ok(res) => res,
|
||||
@@ -544,7 +541,7 @@ impl SetDisks {
|
||||
return false;
|
||||
}
|
||||
|
||||
let attached = match self.attach_unformatted_replacement_disk(ep).await {
|
||||
let attached = match self.attach_unformatted_replacement_disk(ep, previous).await {
|
||||
Ok(attached) => attached,
|
||||
Err(err) => {
|
||||
warn!(endpoint = %ep, error = ?err, "renew_disk: unformatted replacement probe failed");
|
||||
@@ -580,21 +577,18 @@ impl SetDisks {
|
||||
return false;
|
||||
}
|
||||
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
|
||||
if !new_disk.is_local() {
|
||||
if let disk::Disk::Local(local) = new_disk.as_ref() {
|
||||
local.set_disk_id_state(Some(fm.erasure.this)).await;
|
||||
self.publish_local_reconnect(disk_idx, previous, new_disk, Some(fm.erasure.this))
|
||||
.await;
|
||||
} else {
|
||||
let _ = new_disk.set_disk_id(Some(fm.erasure.this)).await;
|
||||
let displaced = {
|
||||
let mut disks = self.disks.write().await;
|
||||
let slot = &mut disks[disk_idx];
|
||||
let unchanged = match (slot.as_ref(), previous_disk.as_ref()) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(previous)) => Arc::ptr_eq(current, previous),
|
||||
_ => false,
|
||||
};
|
||||
// A GET probe or another reconnect may have filled this slot while
|
||||
// format I/O was pending. Only the successful publisher owns the
|
||||
// displaced monitor; a failed or superseded probe must retain it.
|
||||
if unchanged {
|
||||
// format I/O was pending. Only the winner owns the displaced monitor.
|
||||
if Self::same_disk_handle(slot.as_ref(), previous) {
|
||||
new_disk.enable_health_check();
|
||||
slot.replace(new_disk)
|
||||
} else {
|
||||
@@ -604,39 +598,79 @@ impl SetDisks {
|
||||
if let Some(disk) = displaced {
|
||||
let _ = disk.close().await;
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
fn same_disk_handle(current: Option<&DiskStore>, expected: Option<&DiskStore>) -> bool {
|
||||
match (current, expected) {
|
||||
(None, None) => true,
|
||||
(Some(current), Some(expected)) => Arc::ptr_eq(current, expected),
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Commit a freshly probed attachment to its set and all local RPC routes together.
|
||||
async fn publish_local_reconnect(
|
||||
&self,
|
||||
disk_idx: usize,
|
||||
previous: Option<&DiskStore>,
|
||||
candidate: DiskStore,
|
||||
format_id: Option<Uuid>,
|
||||
) -> bool {
|
||||
let disk::Disk::Local(local) = candidate.as_ref() else { return false };
|
||||
let path = candidate.endpoint().to_string();
|
||||
let dist = self.ctx.is_dist_erasure().await;
|
||||
// A busy set must not block RPC lookups across the instance. Lock the set
|
||||
// first, then follow the registry order used by startup and RPC registration:
|
||||
// distributed drive table -> path map -> ID map.
|
||||
let mut disks = self.disks.write().await;
|
||||
let slot = &mut disks[disk_idx];
|
||||
if !Self::same_disk_handle(slot.as_ref(), previous) {
|
||||
return false;
|
||||
}
|
||||
|
||||
new_disk.enable_health_check();
|
||||
|
||||
if new_disk.is_local() {
|
||||
if let crate::disk::Disk::Local(wrapper) = new_disk.as_ref() {
|
||||
wrapper.get_disk().usage_watch_binding.bind(&self.ctx.usage_watches);
|
||||
}
|
||||
let local_disk_map = runtime_sources::local_disk_map_handle();
|
||||
let mut global_local_disk_map = local_disk_map.write().await;
|
||||
let path = new_disk.endpoint().to_string();
|
||||
global_local_disk_map.insert(path, Some(new_disk.clone()));
|
||||
|
||||
if runtime_sources::setup_is_dist_erasure().await {
|
||||
let local_disk_set_drives = runtime_sources::local_disk_set_drives_handle();
|
||||
let mut local_set_drives = local_disk_set_drives.write().await;
|
||||
local_set_drives[self.pool_index][set_idx][disk_idx] = Some(new_disk.clone());
|
||||
}
|
||||
let table = self.ctx.local_disk_set_drives();
|
||||
let mut table = if dist { Some(table.write().await) } else { None };
|
||||
let route_slot = if let Some(table) = table.as_mut() {
|
||||
let Some(slot) = table
|
||||
.get_mut(self.pool_index)
|
||||
.and_then(|pool| pool.get_mut(self.set_index))
|
||||
.and_then(|set| set.get_mut(disk_idx))
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
Some(slot)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let routes = self.ctx.local_disk_map();
|
||||
let mut routes = routes.write().await;
|
||||
if let Some(id) = format_id {
|
||||
self.ctx.local_disk_id_map().write().await.insert(id, path.clone());
|
||||
}
|
||||
|
||||
debug!("renew_disk: update {:?}", fm.erasure.this);
|
||||
|
||||
let mut disk_lock = self.disks.write().await;
|
||||
disk_lock[disk_idx] = Some(new_disk);
|
||||
false
|
||||
// No await after publication begins: cancellation cannot expose half an attachment.
|
||||
local.get_disk().usage_watch_binding.bind(&self.ctx.usage_watches);
|
||||
candidate.enable_health_check();
|
||||
if let Some(disk::Disk::Local(displaced)) = slot.as_deref() {
|
||||
displaced.retire_handle();
|
||||
}
|
||||
routes.insert(path, Some(candidate.clone()));
|
||||
if let Some(route_slot) = route_slot {
|
||||
*route_slot = Some(candidate.clone());
|
||||
}
|
||||
*slot = Some(candidate);
|
||||
true
|
||||
}
|
||||
|
||||
/// Attach a replacement target only after proving that the exact local slot
|
||||
/// is present and unformatted. A health-checked reconnect may reject a
|
||||
/// blank target before it reaches the format-heal path; that target still
|
||||
/// has to be visible in this set for the formatter to claim it safely.
|
||||
async fn attach_unformatted_replacement_disk(&self, ep: &Endpoint) -> disk::error::Result<bool> {
|
||||
async fn attach_unformatted_replacement_disk(
|
||||
&self,
|
||||
ep: &Endpoint,
|
||||
previous: Option<&DiskStore>,
|
||||
) -> disk::error::Result<bool> {
|
||||
if !ep.is_local
|
||||
|| usize::try_from(ep.pool_idx).ok() != Some(self.pool_index)
|
||||
|| usize::try_from(ep.set_idx).ok() != Some(self.set_index)
|
||||
@@ -662,28 +696,7 @@ impl SetDisks {
|
||||
Err(err) => return Err(err),
|
||||
}
|
||||
|
||||
{
|
||||
let mut disks = self.disks.write().await;
|
||||
if disks[disk_idx].as_ref().is_some_and(|existing| existing.endpoint() != *ep) {
|
||||
warn!(endpoint = %ep, disk_idx, "renew_disk rejected unformatted replacement for an occupied foreign slot");
|
||||
return Ok(false);
|
||||
}
|
||||
disks[disk_idx] = Some(replacement.clone());
|
||||
}
|
||||
|
||||
let local_disk_map = runtime_sources::local_disk_map_handle();
|
||||
local_disk_map
|
||||
.write()
|
||||
.await
|
||||
.insert(replacement.endpoint().to_string(), Some(replacement.clone()));
|
||||
|
||||
if runtime_sources::setup_is_dist_erasure().await {
|
||||
let local_disk_set_drives = runtime_sources::local_disk_set_drives_handle();
|
||||
let mut local_set_drives = local_disk_set_drives.write().await;
|
||||
local_set_drives[self.pool_index][self.set_index][disk_idx] = Some(replacement);
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
Ok(self.publish_local_reconnect(disk_idx, previous, replacement, None).await)
|
||||
}
|
||||
|
||||
pub(in crate::set_disk) fn find_disk_index(&self, fm: &FormatV3) -> Result<(usize, usize)> {
|
||||
@@ -1251,6 +1264,9 @@ mod tests {
|
||||
.await
|
||||
.expect("renewed disk_info should record a drive API metric");
|
||||
renewed_disk.force_runtime_state_for_test(disk::health_state::RuntimeDriveHealthState::Offline);
|
||||
assert!(
|
||||
matches!(renewed_disk.as_ref(), disk::Disk::Local(local) if matches!(local.check_scanner_handle(), Err(DiskError::FaultyDisk)))
|
||||
);
|
||||
|
||||
set_disks.renew_disk(&endpoints[0]).await;
|
||||
|
||||
@@ -1258,6 +1274,13 @@ mod tests {
|
||||
let renewed_again = disks[0]
|
||||
.as_ref()
|
||||
.expect("second renew_disk should keep the recovered disk attached");
|
||||
let (disk::Disk::Local(retired), disk::Disk::Local(current)) = (renewed_disk.as_ref(), renewed_again.as_ref()) else {
|
||||
panic!("local reconnect fixture must retain a local slot");
|
||||
};
|
||||
assert!(matches!(retired.check_scanner_handle(), Err(DiskError::DiskNotFound)));
|
||||
current
|
||||
.check_scanner_handle()
|
||||
.expect("reconnected scanner handle must be usable");
|
||||
assert_eq!(
|
||||
renewed_again
|
||||
.metrics_snapshot()
|
||||
@@ -1269,6 +1292,70 @@ mod tests {
|
||||
drop(temp_dirs);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn stale_local_reconnect_cannot_retire_the_published_rpc_source() {
|
||||
use crate::layout::endpoints::SetupType;
|
||||
use crate::runtime::instance::InstanceContext;
|
||||
|
||||
let format = FormatV3::new(1, 1);
|
||||
let id = format.erasure.sets[0][0];
|
||||
let (directory, endpoint, original) = make_formatted_local_disk(0, &format).await;
|
||||
let path = endpoint.to_string();
|
||||
let ctx = Arc::new(InstanceContext::new());
|
||||
ctx.update_erasure_type(SetupType::DistErasure).await;
|
||||
*ctx.local_disk_set_drives().write().await = vec![vec![vec![Some(original.clone())]]];
|
||||
ctx.local_disk_map()
|
||||
.write()
|
||||
.await
|
||||
.insert(path.clone(), Some(original.clone()));
|
||||
let set = SetDisks::new_with_instance_ctx(
|
||||
"local-reconnect-race".to_owned(),
|
||||
Arc::new(RwLock::new(vec![Some(original.clone())])),
|
||||
1,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
vec![endpoint.clone()],
|
||||
format,
|
||||
Vec::new(),
|
||||
ctx.clone(),
|
||||
)
|
||||
.await;
|
||||
|
||||
// Two reconnects saw the same set slot. Finish one before allowing the
|
||||
// other probe to publish, first with its format and then as a blank target.
|
||||
let delayed = set.get_disks_internal().await[0].clone();
|
||||
set.renew_disk(&endpoint).await;
|
||||
let winner = set.get_disks_internal().await[0].clone().expect("published handle");
|
||||
assert!(
|
||||
original
|
||||
.local_retirement_token()
|
||||
.expect("original local handle")
|
||||
.is_cancelled()
|
||||
);
|
||||
assert!(!Arc::ptr_eq(&original, &winner));
|
||||
for blank in [false, true] {
|
||||
if blank {
|
||||
tokio::fs::remove_file(directory.path().join(disk::RUSTFS_META_BUCKET).join(disk::FORMAT_CONFIG_FILE))
|
||||
.await
|
||||
.expect("turn delayed source into a verified blank replacement");
|
||||
}
|
||||
assert!(
|
||||
!set.reconnect_disk(&endpoint, delayed.as_ref()).await,
|
||||
"superseded blank cannot schedule a heal"
|
||||
);
|
||||
let active = set.get_disks_internal().await[0].clone().expect("set slot");
|
||||
let route = ctx.local_disk_map().read().await[&path].clone().expect("RPC route");
|
||||
let topology = ctx.local_disk_set_drives().read().await[0][0][0]
|
||||
.clone()
|
||||
.expect("distributed drive slot");
|
||||
assert!(Arc::ptr_eq(&active, &winner) && Arc::ptr_eq(&route, &winner) && Arc::ptr_eq(&topology, &winner));
|
||||
assert_eq!(ctx.local_disk_id_map().read().await.get(&id), Some(&path));
|
||||
assert!(!winner.local_retirement_token().expect("current local handle").is_cancelled());
|
||||
}
|
||||
winner.close().await.expect("stop the winning monitor");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn reconnect_sweep_collects_all_blank_targets_and_excludes_original_disks() {
|
||||
use rustfs_heal_contracts::heal_channel::{HealAdmissionResult, HealChannelCommand, init_heal_channel};
|
||||
@@ -1466,7 +1553,7 @@ mod tests {
|
||||
|
||||
assert!(
|
||||
set_disks
|
||||
.attach_unformatted_replacement_disk(&replacement_endpoint)
|
||||
.attach_unformatted_replacement_disk(&replacement_endpoint, None)
|
||||
.await
|
||||
.expect("a blank local replacement should be admitted")
|
||||
);
|
||||
|
||||
@@ -28724,13 +28724,15 @@ mod delete_recovery_inventory_tests {
|
||||
.await
|
||||
.expect("seed actual version");
|
||||
let endpoint = set.set_endpoints[0].clone();
|
||||
let ctx = crate::runtime::global::current_ctx();
|
||||
let was_dist = ctx.is_dist_erasure().await;
|
||||
if was_dist {
|
||||
ctx.update_erasure_type(crate::layout::endpoints::SetupType::Erasure).await;
|
||||
}
|
||||
let map = ctx.local_disk_map();
|
||||
let previous = map.read().await.get(&endpoint.to_string()).cloned();
|
||||
let map = set.ctx.local_disk_map();
|
||||
let original = map
|
||||
.read()
|
||||
.await
|
||||
.get(&endpoint.to_string())
|
||||
.cloned()
|
||||
.flatten()
|
||||
.expect("original RPC route");
|
||||
assert!(Arc::ptr_eq(&original, &disks[0]));
|
||||
|
||||
let barrier = DeleteCleanupBarrier::install(bucket);
|
||||
let delete_store = store.clone();
|
||||
@@ -28767,14 +28769,14 @@ mod delete_recovery_inventory_tests {
|
||||
)
|
||||
.await
|
||||
.expect("queue real renewal writer");
|
||||
let connected = map
|
||||
.read()
|
||||
.await
|
||||
let routed = map
|
||||
.try_read()
|
||||
.expect("waiting for one set must leave RPC lookups available")
|
||||
.get(&endpoint.to_string())
|
||||
.cloned()
|
||||
.flatten()
|
||||
.expect("renewal registered new handle");
|
||||
assert!(!Arc::ptr_eq(&connected, &disks[0]));
|
||||
.expect("original route is still published");
|
||||
assert!(Arc::ptr_eq(&routed, &original));
|
||||
drop(extra_gate);
|
||||
barrier.release();
|
||||
let outcome = tokio::time::timeout(Duration::from_secs(3), async { tokio::join!(renewal.as_mut(), &mut deletion) }).await;
|
||||
@@ -28785,25 +28787,20 @@ mod delete_recovery_inventory_tests {
|
||||
if outcome.is_err() {
|
||||
let _ = deletion.await;
|
||||
}
|
||||
{
|
||||
let mut map = map.write().await;
|
||||
match previous {
|
||||
Some(old) => {
|
||||
map.insert(endpoint.to_string(), old);
|
||||
}
|
||||
None => {
|
||||
map.remove(&endpoint.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
if was_dist {
|
||||
ctx.update_erasure_type(crate::layout::endpoints::SetupType::DistErasure)
|
||||
.await;
|
||||
}
|
||||
let (_, result) =
|
||||
outcome.expect("batch DELETE and real queued disk renewal must finish without nested inventory-read deadlock");
|
||||
let (_, errors) = result.expect("delete task must finish");
|
||||
assert!(errors.iter().all(Option::is_none), "DELETE result must remain successful: {errors:?}");
|
||||
let connected = set.get_disks_internal().await[0].clone().expect("renewed set slot");
|
||||
let registered = map
|
||||
.read()
|
||||
.await
|
||||
.get(&endpoint.to_string())
|
||||
.cloned()
|
||||
.flatten()
|
||||
.expect("renewed RPC route");
|
||||
assert!(Arc::ptr_eq(®istered, &connected), "DELETE and RPC must see the same attachment");
|
||||
assert!(!Arc::ptr_eq(®istered, &original), "renewal must replace the original handle");
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,507 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Read-only paged observation of physical delete evidence. Sources can report
|
||||
//! the same transaction; only an exact-key inspection can classify the set.
|
||||
|
||||
use super::ECStore;
|
||||
use crate::bucket::utils::{check_bucket_and_object_names, check_valid_bucket_name_strict, is_meta_bucketname};
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::disk::{DeleteRecoveryCursor, DeleteRecoveryPage, DeleteTransactionId, Disk, DiskStore, MAX_DELETE_RESIDUE_PAGE_SIZE};
|
||||
use crate::error::{Result, StorageError};
|
||||
use crate::set_disk::SetDisks;
|
||||
use rustfs_utils::path::GLOBAL_DIR_SUFFIX;
|
||||
use serde::Serialize;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex, Weak};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{OwnedSemaphorePermit, Semaphore};
|
||||
use tokio::time::Instant;
|
||||
use uuid::Uuid;
|
||||
|
||||
const MAX_SESSIONS: usize = 8;
|
||||
const CURSOR_TTL: Duration = Duration::from_secs(300);
|
||||
|
||||
/// Coordinates of the single configured drive examined for this page.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize)]
|
||||
pub struct DeleteResidueSource {
|
||||
pub pool: usize,
|
||||
pub set: usize,
|
||||
pub drive: usize,
|
||||
}
|
||||
|
||||
impl DeleteResidueSource {
|
||||
fn erasure_set(self, store: &ECStore) -> Result<&SetDisks> {
|
||||
store
|
||||
.pools
|
||||
.get(self.pool)
|
||||
.and_then(|pool| pool.disk_set.get(self.set))
|
||||
.filter(|set| self.drive < set.set_drive_count)
|
||||
.map(Arc::as_ref)
|
||||
.ok_or(StorageError::ErasureReadQuorum)
|
||||
}
|
||||
|
||||
fn next(self, store: &ECStore) -> Result<Option<Self>> {
|
||||
let set = self.erasure_set(store)?;
|
||||
if self.drive + 1 < set.set_drive_count {
|
||||
return Ok(Some(Self {
|
||||
drive: self.drive + 1,
|
||||
..self
|
||||
}));
|
||||
}
|
||||
if self.set + 1 < store.pools[self.pool].disk_set.len() {
|
||||
return Ok(Some(Self {
|
||||
pool: self.pool,
|
||||
set: self.set + 1,
|
||||
drive: 0,
|
||||
}));
|
||||
}
|
||||
Ok((self.pool + 1 < store.pools.len()).then_some(Self {
|
||||
pool: self.pool + 1,
|
||||
..Self::default()
|
||||
}))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub enum DeleteResidueSourceStatus {
|
||||
Scanned,
|
||||
Unavailable,
|
||||
Unreadable,
|
||||
}
|
||||
|
||||
/// One physical discovery page. A candidate may appear on other drives or
|
||||
/// pages. Completion describes traversal coverage, never the outcome of a delete.
|
||||
#[derive(Debug, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DeleteResidueInventoryPage {
|
||||
pub bucket: String,
|
||||
pub incarnation: Uuid,
|
||||
pub source: DeleteResidueSource,
|
||||
pub source_status: DeleteResidueSourceStatus,
|
||||
pub candidates: Vec<DeleteTransactionId>,
|
||||
pub next_cursor: Option<Uuid>,
|
||||
/// Cumulative count of sources skipped because their contents could not be read.
|
||||
pub incomplete_sources: usize,
|
||||
pub complete: bool,
|
||||
}
|
||||
|
||||
struct PhysicalContinuation {
|
||||
disk: Weak<Disk>,
|
||||
id: Uuid,
|
||||
cursor: DeleteRecoveryCursor,
|
||||
}
|
||||
|
||||
struct InventorySession {
|
||||
store: Uuid,
|
||||
bucket: String,
|
||||
incarnation: Uuid,
|
||||
source: DeleteResidueSource,
|
||||
physical: Option<PhysicalContinuation>,
|
||||
incomplete_sources: usize,
|
||||
expires: Instant,
|
||||
_permit: OwnedSemaphorePermit,
|
||||
}
|
||||
|
||||
/// Tokens are single-use, node-local capabilities. The permit follows the
|
||||
/// session during I/O so concurrent first-page requests also share the bound.
|
||||
pub(crate) struct DeleteResidueCursors {
|
||||
sessions: Mutex<HashMap<Uuid, InventorySession>>,
|
||||
capacity: Arc<Semaphore>,
|
||||
}
|
||||
|
||||
impl Default for DeleteResidueCursors {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
sessions: Mutex::default(),
|
||||
capacity: Arc::new(Semaphore::new(MAX_SESSIONS)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl DeleteResidueCursors {
|
||||
fn take(&self, token: Option<Uuid>, store: Uuid, bucket: &str) -> Result<Option<InventorySession>> {
|
||||
let now = Instant::now();
|
||||
let mut sessions = self.sessions.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
sessions.retain(|_, session| session.expires > now);
|
||||
let Some(token) = token else { return Ok(None) };
|
||||
if !sessions.get(&token).is_some_and(|s| s.store == store && s.bucket == bucket) {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Ok(sessions.remove(&token))
|
||||
}
|
||||
|
||||
fn save(&self, mut session: InventorySession) -> Uuid {
|
||||
let token = Uuid::new_v4();
|
||||
session.expires = Instant::now() + CURSOR_TTL;
|
||||
self.sessions
|
||||
.lock()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.insert(token, session);
|
||||
token
|
||||
}
|
||||
}
|
||||
|
||||
async fn source_disk(set: &SetDisks, drive: usize) -> Result<Option<DiskStore>> {
|
||||
set.disks
|
||||
.read()
|
||||
.await
|
||||
.get(drive)
|
||||
.cloned()
|
||||
.ok_or(StorageError::ErasureReadQuorum)
|
||||
}
|
||||
|
||||
async fn observe_source(
|
||||
set: &SetDisks,
|
||||
session: &mut InventorySession,
|
||||
expected_id: Option<Uuid>,
|
||||
limit: u32,
|
||||
) -> Result<(DeleteResidueSourceStatus, Vec<DeleteTransactionId>)> {
|
||||
let continuation = session.physical.take();
|
||||
let Some(disk) = source_disk(set, session.source.drive).await? else {
|
||||
if continuation.is_some() {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
return Ok((DeleteResidueSourceStatus::Unavailable, Vec::new()));
|
||||
};
|
||||
let id = expected_id.filter(|id| !id.is_nil());
|
||||
if let Some(prior) = &continuation
|
||||
&& (Some(prior.id) != id || !prior.disk.ptr_eq(&Arc::downgrade(&disk)))
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
let Some(id) = id else {
|
||||
return Ok((DeleteResidueSourceStatus::Unreadable, Vec::new()));
|
||||
};
|
||||
let resuming = continuation.is_some();
|
||||
let page = disk
|
||||
.list_delete_residue_on_disk(&session.bucket, id, continuation.map(|state| state.cursor), limit)
|
||||
.await;
|
||||
let DeleteRecoveryPage { mut transactions, next } = match page {
|
||||
Ok(page) => page,
|
||||
Err(DiskError::FileNotFound | DiskError::InvalidPath | DiskError::OutdatedXLMeta | DiskError::InconsistentDisk)
|
||||
if resuming =>
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
Err(_) => return Ok((DeleteResidueSourceStatus::Unreadable, Vec::new())),
|
||||
};
|
||||
if transactions.len() > usize::try_from(limit).map_err(|_| StorageError::FileCorrupt)?
|
||||
|| next.as_ref().is_some_and(|cursor| cursor.token.is_nil())
|
||||
{
|
||||
return Err(StorageError::FileCorrupt);
|
||||
}
|
||||
for candidate in &mut transactions {
|
||||
// A literal suffix can precede the final on-disk directory marker.
|
||||
// Remove exactly one so re-encoding the listed key selects the same physical path.
|
||||
if let Some(prefix) = candidate.object.strip_suffix(GLOBAL_DIR_SUFFIX) {
|
||||
candidate.object = format!("{prefix}/");
|
||||
}
|
||||
if candidate.transaction.is_nil()
|
||||
|| candidate.object.len() > 1024
|
||||
|| check_bucket_and_object_names(&session.bucket, &candidate.object).is_err()
|
||||
{
|
||||
return Err(StorageError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
if !source_disk(set, session.source.drive)
|
||||
.await?
|
||||
.is_some_and(|current| Arc::ptr_eq(¤t, &disk))
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
session.physical = next.map(|cursor| PhysicalContinuation {
|
||||
disk: Arc::downgrade(&disk),
|
||||
id,
|
||||
cursor,
|
||||
});
|
||||
Ok((DeleteResidueSourceStatus::Scanned, transactions))
|
||||
}
|
||||
|
||||
impl ECStore {
|
||||
/// Lists candidates from one bounded raw disk page of a user bucket. Pass
|
||||
/// the single-use continuation to the same node until it returns no token.
|
||||
/// Later disk mutations can change the observations between pages.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns a precondition error for a stale continuation or changed source,
|
||||
/// a throttling error when the cursor limit is reached, or a storage error
|
||||
/// when the bucket identity or peer response cannot be validated. A skipped
|
||||
/// disk is reported explicitly and prevents a complete terminal result.
|
||||
pub async fn list_delete_residue_candidates(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
cursor: Option<Uuid>,
|
||||
limit: u32,
|
||||
) -> Result<DeleteResidueInventoryPage> {
|
||||
if is_meta_bucketname(bucket) || check_valid_bucket_name_strict(bucket).is_err() {
|
||||
return Err(StorageError::BucketNameInvalid(bucket.to_owned()));
|
||||
}
|
||||
if !(1..=MAX_DELETE_RESIDUE_PAGE_SIZE).contains(&limit) {
|
||||
return Err(StorageError::InvalidArgument(
|
||||
bucket.to_owned(),
|
||||
String::new(),
|
||||
"limit must be 1..=64".to_owned(),
|
||||
));
|
||||
}
|
||||
let cursors = &self.ctx.delete_residue_cursors;
|
||||
let mut session = match cursors.take(cursor, self.id, bucket)? {
|
||||
Some(session) => session,
|
||||
None => {
|
||||
let permit = Arc::clone(&cursors.capacity)
|
||||
.try_acquire_owned()
|
||||
.map_err(|_| StorageError::SlowDown)?;
|
||||
InventorySession {
|
||||
store: self.id,
|
||||
bucket: bucket.to_owned(),
|
||||
incarnation: self.bucket_incarnation_id_from_disk(bucket).await?,
|
||||
source: DeleteResidueSource::default(),
|
||||
physical: None,
|
||||
incomplete_sources: 0,
|
||||
expires: Instant::now(),
|
||||
_permit: permit,
|
||||
}
|
||||
}
|
||||
};
|
||||
let fence = self.acquire_bucket_incarnation_fence(bucket, session.incarnation).await?;
|
||||
let incarnation = session.incarnation;
|
||||
let source = session.source;
|
||||
let set = source.erasure_set(self)?;
|
||||
let expected_id = self.pools[source.pool]
|
||||
.format
|
||||
.erasure
|
||||
.sets
|
||||
.get(source.set)
|
||||
.and_then(|ids| ids.get(source.drive))
|
||||
.copied();
|
||||
let (source_status, candidates) = observe_source(set, &mut session, expected_id, limit).await?;
|
||||
if fence.is_lock_lost() {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if source_status != DeleteResidueSourceStatus::Scanned {
|
||||
session.incomplete_sources += 1;
|
||||
}
|
||||
let incomplete_sources = session.incomplete_sources;
|
||||
let next_source = if session.physical.is_some() {
|
||||
Some(source)
|
||||
} else {
|
||||
source.next(self)?
|
||||
};
|
||||
let next_cursor = next_source.map(|next| {
|
||||
session.source = next;
|
||||
cursors.save(session)
|
||||
});
|
||||
Ok(DeleteResidueInventoryPage {
|
||||
bucket: bucket.to_owned(),
|
||||
incarnation,
|
||||
source,
|
||||
source_status,
|
||||
candidates,
|
||||
next_cursor,
|
||||
incomplete_sources,
|
||||
complete: next_cursor.is_none() && incomplete_sources == 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::bucket::metadata_sys::{init_bucket_metadata_sys, test_support::isolated_store_over_temp_disks};
|
||||
use crate::disk::DiskAPI;
|
||||
use crate::storage_api_contracts::bucket::{BucketOperations, MakeBucketOptions};
|
||||
|
||||
async fn fixture(bucket: &str) -> (Vec<tempfile::TempDir>, Arc<ECStore>) {
|
||||
let (dirs, store) = isolated_store_over_temp_disks().await;
|
||||
store.background_cancel_token().expect("fixture background token").cancel();
|
||||
tokio::task::yield_now().await;
|
||||
init_bucket_metadata_sys(store.clone(), Vec::new()).await;
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create discovery bucket");
|
||||
(dirs, store)
|
||||
}
|
||||
|
||||
fn seed_residue(disk: &DiskStore, bucket: &str, key: &str, txn: Uuid) -> std::path::PathBuf {
|
||||
let data = disk.path().join(bucket).join(key).join(Uuid::new_v4().to_string());
|
||||
std::fs::create_dir_all(&data).expect("create historical data directory");
|
||||
let marker = data.join(format!("reserve-delete-data.{txn}"));
|
||||
std::fs::write(&marker, []).expect("seed undiscoverable-by-S3 reservation");
|
||||
marker
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_residue_inventory_pages_cover_configured_sources_and_preserve_hidden_evidence() {
|
||||
let bucket = "residue-inventory";
|
||||
let (_dirs, store) = fixture(bucket).await;
|
||||
let disks = store.pools[0].disk_set[0].disk_inventory().await;
|
||||
let transaction = Uuid::new_v4();
|
||||
let mut markers = Vec::new();
|
||||
for disk in disks.iter().flatten() {
|
||||
markers.push(seed_residue(disk, bucket, "hidden", transaction));
|
||||
}
|
||||
let mut cursor = None;
|
||||
let mut observed = std::collections::HashSet::new();
|
||||
for _ in 0..(disks.len() * 4) {
|
||||
let page = store
|
||||
.list_delete_residue_candidates(bucket, cursor, 1)
|
||||
.await
|
||||
.expect("read disk discovery page");
|
||||
assert!(page.candidates.len() <= 1);
|
||||
assert_eq!(page.source_status, DeleteResidueSourceStatus::Scanned);
|
||||
for item in page.candidates {
|
||||
assert_eq!((item.object.as_str(), item.transaction), ("hidden", transaction));
|
||||
observed.insert(page.source.drive);
|
||||
}
|
||||
cursor = page.next_cursor;
|
||||
if cursor.is_none() {
|
||||
assert!(page.complete);
|
||||
break;
|
||||
}
|
||||
assert!(!page.complete);
|
||||
}
|
||||
assert!(cursor.is_none(), "traversal must reach a terminal page");
|
||||
assert_eq!(observed.len(), disks.len(), "scan every participant independently");
|
||||
for marker in markers {
|
||||
assert!(std::fs::read(marker).expect("retain legacy marker bytes").is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_residue_inventory_wrong_disk_never_certifies_complete() {
|
||||
let bucket = "residue-wrong-disk";
|
||||
let (_dirs, store) = fixture(bucket).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let disks = set.disk_inventory().await;
|
||||
let other = disks[1].as_ref().expect("second fixture drive");
|
||||
seed_residue(other, bucket, "only-on-second-drive", Uuid::new_v4());
|
||||
set.disks.write().await[0] = Some(other.clone());
|
||||
|
||||
let page = store
|
||||
.list_delete_residue_candidates(bucket, None, 4)
|
||||
.await
|
||||
.expect("report unverified slot");
|
||||
assert_eq!(page.source.drive, 0);
|
||||
assert_eq!(page.source_status, DeleteResidueSourceStatus::Unreadable);
|
||||
assert_eq!(page.incomplete_sources, 1);
|
||||
assert!(!page.complete);
|
||||
assert!(page.candidates.is_empty(), "never attribute the other slot's evidence to this one");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_residue_inventory_preserves_literal_suffix_in_directory_key() {
|
||||
use rustfs_utils::path::encode_dir_object;
|
||||
let bucket = "residue-dir-suffix";
|
||||
let (_dirs, store) = fixture(bucket).await;
|
||||
let disk = store.pools[0].disk_set[0].disk_inventory().await[0]
|
||||
.clone()
|
||||
.expect("first drive");
|
||||
let key = format!("object{GLOBAL_DIR_SUFFIX}/");
|
||||
let physical = encode_dir_object(&key);
|
||||
seed_residue(&disk, bucket, &physical, Uuid::new_v4());
|
||||
let page = store
|
||||
.list_delete_residue_candidates(bucket, None, 4)
|
||||
.await
|
||||
.expect("list directory key");
|
||||
assert_eq!(page.candidates.len(), 1);
|
||||
assert_eq!(page.candidates[0].object, key);
|
||||
assert_eq!(encode_dir_object(&page.candidates[0].object), physical);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_residue_inventory_missing_source_never_certifies_empty() {
|
||||
let bucket = "residue-offline";
|
||||
let (_dirs, store) = fixture(bucket).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let failed = set.set_drive_count - 1;
|
||||
set.disks.write().await[failed] = None;
|
||||
let mut cursor = None;
|
||||
for index in 0..set.set_drive_count {
|
||||
let page = store
|
||||
.list_delete_residue_candidates(bucket, cursor, 1)
|
||||
.await
|
||||
.expect("page with an offline source");
|
||||
assert_eq!(page.source.drive, index);
|
||||
assert!(page.candidates.is_empty());
|
||||
assert!(!page.complete);
|
||||
if index == failed {
|
||||
assert_eq!(page.source_status, DeleteResidueSourceStatus::Unavailable);
|
||||
assert_eq!(page.incomplete_sources, 1);
|
||||
}
|
||||
cursor = page.next_cursor;
|
||||
}
|
||||
assert!(cursor.is_none(), "failed source is explicitly accounted for at end of walk");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_residue_inventory_rejects_replaced_source_during_page() {
|
||||
let bucket = "residue-source-change";
|
||||
let (_dirs, store) = fixture(bucket).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let disks = set.disk_inventory().await;
|
||||
let first = disks[0].as_ref().expect("first fixture participant");
|
||||
seed_residue(first, bucket, "first", Uuid::new_v4());
|
||||
seed_residue(first, bucket, "second", Uuid::new_v4());
|
||||
let page = store
|
||||
.list_delete_residue_candidates(bucket, None, 1)
|
||||
.await
|
||||
.expect("open cursor on first drive");
|
||||
assert_eq!(page.candidates.len(), 1);
|
||||
let cursor = page.next_cursor.expect("additional physical pages");
|
||||
assert!(matches!(
|
||||
store
|
||||
.list_delete_residue_candidates("different-bucket", Some(cursor), 1)
|
||||
.await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
set.disks.write().await[0] = None;
|
||||
assert!(matches!(
|
||||
store.list_delete_residue_candidates(bucket, Some(cursor), 1).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
assert!(matches!(
|
||||
store.list_delete_residue_candidates(bucket, Some(cursor), 1).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn delete_residue_inventory_expired_cursor_releases_capacity() {
|
||||
let cursors = DeleteResidueCursors::default();
|
||||
let store = Uuid::new_v4();
|
||||
let mut token = Uuid::nil();
|
||||
for _ in 0..MAX_SESSIONS {
|
||||
token = cursors.save(InventorySession {
|
||||
store,
|
||||
bucket: "scope".to_owned(),
|
||||
incarnation: Uuid::new_v4(),
|
||||
source: DeleteResidueSource::default(),
|
||||
physical: None,
|
||||
incomplete_sources: 0,
|
||||
expires: Instant::now(),
|
||||
_permit: cursors.capacity.clone().try_acquire_owned().expect("bounded capacity"),
|
||||
});
|
||||
}
|
||||
assert!(cursors.capacity.clone().try_acquire_owned().is_err());
|
||||
assert!(matches!(
|
||||
cursors.take(Some(token), Uuid::new_v4(), "scope"),
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
tokio::time::advance(CURSOR_TTL).await;
|
||||
assert!(matches!(cursors.take(Some(token), store, "scope"), Err(StorageError::PreconditionFailed)));
|
||||
assert_eq!(cursors.capacity.available_permits(), MAX_SESSIONS);
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@
|
||||
use super::ECStore;
|
||||
use crate::bucket::utils::is_meta_bucketname;
|
||||
use crate::disk::{DeleteRecoveryCursor, DiskAPI, DiskStore};
|
||||
use crate::error::StorageError;
|
||||
use futures::{StreamExt, future::BoxFuture, stream::FuturesUnordered};
|
||||
use rustfs_heal_contracts::heal_channel::HealOpts;
|
||||
use std::collections::{HashMap, HashSet, VecDeque};
|
||||
@@ -41,6 +42,8 @@ const EVENT_DELETE_RECOVERY_STATUS: &str = "delete_recovery_status";
|
||||
const KEYS_PER_PASS: usize = 16;
|
||||
const RECOVERY_CONCURRENCY: usize = 4;
|
||||
const INTERVAL: Duration = Duration::from_secs(1);
|
||||
const INITIAL_RETRY_DELAY: Duration = Duration::from_secs(5);
|
||||
const MAX_RETRY_DELAY: Duration = Duration::from_secs(5 * 60);
|
||||
const INVENTORY_INTERVAL: Duration = Duration::from_secs(30);
|
||||
const DISCOVERY_BUDGET: Duration = Duration::from_secs(5);
|
||||
const ANTI_ENTROPY_INTERVAL: Duration = Duration::from_secs(60 * 60);
|
||||
@@ -60,6 +63,7 @@ pub(crate) struct DeleteRecoveryTicket {
|
||||
key: Arc<DeleteRecoveryKey>,
|
||||
generation: Uuid,
|
||||
admission: Option<Arc<AdmissionGuard>>,
|
||||
retry: Option<RetrySchedule>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -82,10 +86,40 @@ impl Drop for AdmissionGuard {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct RetrySchedule {
|
||||
ready_at: Instant,
|
||||
delay: Duration,
|
||||
}
|
||||
|
||||
impl RetrySchedule {
|
||||
fn after_failure(previous: Option<Self>, now: Instant) -> Self {
|
||||
let delay = previous.map_or(INITIAL_RETRY_DELAY, |retry| (retry.delay * 2).clamp(INITIAL_RETRY_DELAY, MAX_RETRY_DELAY));
|
||||
Self {
|
||||
ready_at: now + delay,
|
||||
delay,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PendingHint {
|
||||
generation: Uuid,
|
||||
queue_generation: Uuid,
|
||||
kind: PendingHintKind,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum PendingHintKind {
|
||||
Foreground,
|
||||
Discovered,
|
||||
Retry(RetrySchedule),
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct HintState {
|
||||
pending: HashMap<Arc<DeleteRecoveryKey>, (Uuid, Uuid, bool)>,
|
||||
order: VecDeque<DeleteRecoveryTicket>,
|
||||
pending: HashMap<Arc<DeleteRecoveryKey>, PendingHint>,
|
||||
order: VecDeque<(Arc<DeleteRecoveryKey>, Uuid)>,
|
||||
rescan: Option<Uuid>,
|
||||
started: HashSet<Uuid>,
|
||||
#[cfg(test)]
|
||||
@@ -93,6 +127,18 @@ struct HintState {
|
||||
}
|
||||
|
||||
impl HintState {
|
||||
fn enqueue(&mut self, key: Arc<DeleteRecoveryKey>, generation: Uuid, kind: PendingHintKind) {
|
||||
self.pending.insert(
|
||||
key.clone(),
|
||||
PendingHint {
|
||||
generation,
|
||||
queue_generation: generation,
|
||||
kind,
|
||||
},
|
||||
);
|
||||
self.order.push_back((key, generation));
|
||||
}
|
||||
|
||||
fn compact_order(&mut self, capacity: usize) {
|
||||
// Twice the pending capacity guarantees at least capacity completed
|
||||
// entries between compactions, even with a full retry backlog. This
|
||||
@@ -106,7 +152,7 @@ impl HintState {
|
||||
self.compaction_visits += self.order.len();
|
||||
}
|
||||
let Self { pending, order, .. } = self;
|
||||
order.retain(|ticket| pending.get(&ticket.key).is_some_and(|entry| entry.1 == ticket.generation));
|
||||
order.retain(|(key, generation)| pending.get(key).is_some_and(|entry| entry.queue_generation == *generation));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -142,32 +188,28 @@ impl DeleteRecoveryHints {
|
||||
let key = Arc::new(key);
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let generation = Uuid::new_v4();
|
||||
if let Some((current, _, _)) = state.pending.get_mut(&key) {
|
||||
*current = generation;
|
||||
if let Some(entry) = state.pending.get_mut(&key) {
|
||||
entry.generation = generation;
|
||||
entry.kind = PendingHintKind::Foreground;
|
||||
} else {
|
||||
if state.pending.len() >= self.capacity {
|
||||
// Durable evidence remains the authority. Rotate the oldest
|
||||
// hint so permanently blocked keys cannot exclude later raw
|
||||
// discoveries forever; the next scan revisits evicted keys.
|
||||
while let Some(oldest) = state.order.pop_front() {
|
||||
while let Some((oldest, generation)) = state.order.pop_front() {
|
||||
if state
|
||||
.pending
|
||||
.get(&oldest.key)
|
||||
.is_some_and(|entry| entry.1 == oldest.generation)
|
||||
.get(&oldest)
|
||||
.is_some_and(|entry| entry.queue_generation == generation)
|
||||
{
|
||||
state.pending.remove(&oldest.key);
|
||||
state.pending.remove(&oldest);
|
||||
break;
|
||||
}
|
||||
}
|
||||
state.rescan = Some(Uuid::new_v4());
|
||||
}
|
||||
state.compact_order(self.capacity);
|
||||
state.pending.insert(key.clone(), (generation, generation, false));
|
||||
state.order.push_back(DeleteRecoveryTicket {
|
||||
key: key.clone(),
|
||||
generation,
|
||||
admission: None,
|
||||
});
|
||||
state.enqueue(key.clone(), generation, PendingHintKind::Foreground);
|
||||
}
|
||||
Some(DeleteRecoveryTicket {
|
||||
admission: Some(Arc::new(AdmissionGuard {
|
||||
@@ -177,6 +219,7 @@ impl DeleteRecoveryHints {
|
||||
})),
|
||||
key,
|
||||
generation,
|
||||
retry: None,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -188,28 +231,21 @@ impl DeleteRecoveryHints {
|
||||
if state.pending.len() >= self.capacity {
|
||||
// Only previously attempted work may yield its slot. Unattempted
|
||||
// keys apply backpressure so a fast scan cannot overwrite them.
|
||||
let retried = state.order.iter().position(|ticket| {
|
||||
let retried = state.order.iter().position(|(queued, generation)| {
|
||||
state
|
||||
.pending
|
||||
.get(&ticket.key)
|
||||
.is_some_and(|entry| entry.1 == ticket.generation && entry.2)
|
||||
.get(queued)
|
||||
.is_some_and(|entry| entry.queue_generation == *generation && matches!(entry.kind, PendingHintKind::Retry(_)))
|
||||
});
|
||||
let Some(index) = retried else { return false };
|
||||
if let Some(oldest) = state.order.remove(index) {
|
||||
state.pending.remove(&oldest.key);
|
||||
if let Some((oldest, _)) = state.order.remove(index) {
|
||||
state.pending.remove(&oldest);
|
||||
}
|
||||
// This durable key has already been attempted. The hourly sweep
|
||||
// revisits it; immediate rescans would loop on permanent failures.
|
||||
}
|
||||
state.compact_order(self.capacity);
|
||||
let key = Arc::new(key);
|
||||
let generation = Uuid::new_v4();
|
||||
state.pending.insert(key.clone(), (generation, generation, false));
|
||||
state.order.push_back(DeleteRecoveryTicket {
|
||||
key,
|
||||
generation,
|
||||
admission: None,
|
||||
});
|
||||
state.enqueue(Arc::new(key), Uuid::new_v4(), PendingHintKind::Discovered);
|
||||
true
|
||||
}
|
||||
|
||||
@@ -221,33 +257,36 @@ impl DeleteRecoveryHints {
|
||||
if state
|
||||
.pending
|
||||
.get(&ticket.key)
|
||||
.is_some_and(|entry| entry.0 == ticket.generation)
|
||||
.is_some_and(|entry| entry.generation == ticket.generation)
|
||||
{
|
||||
state.pending.remove(&ticket.key);
|
||||
}
|
||||
}
|
||||
|
||||
fn take_available(&self, active: &HashSet<Arc<DeleteRecoveryKey>>) -> Option<DeleteRecoveryTicket> {
|
||||
fn take_available(&self, active: &HashSet<Arc<DeleteRecoveryKey>>, now: Instant) -> Option<DeleteRecoveryTicket> {
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
let queued = state.order.len();
|
||||
for _ in 0..queued {
|
||||
let ticket = state.order.pop_front()?;
|
||||
if let Some(&(generation, slot, _)) = state.pending.get(&ticket.key)
|
||||
&& slot == ticket.generation
|
||||
{
|
||||
if active.contains(&ticket.key) {
|
||||
// Preserve a newer hint while this key still owns a slot.
|
||||
// Other keys can proceed without waiting on its lock.
|
||||
state.order.push_back(ticket);
|
||||
continue;
|
||||
}
|
||||
state.pending.remove(&ticket.key);
|
||||
return Some(DeleteRecoveryTicket {
|
||||
key: ticket.key,
|
||||
generation,
|
||||
admission: None,
|
||||
});
|
||||
let (key, slot) = state.order.pop_front()?;
|
||||
let Some(entry) = state.pending.get(&key).filter(|entry| entry.queue_generation == slot) else {
|
||||
continue;
|
||||
};
|
||||
let retry = match entry.kind {
|
||||
PendingHintKind::Retry(retry) => Some(retry),
|
||||
PendingHintKind::Discovered | PendingHintKind::Foreground => None,
|
||||
};
|
||||
if retry.is_some_and(|retry| retry.ready_at > now) || active.contains(&key) {
|
||||
state.order.push_back((key, slot));
|
||||
continue;
|
||||
}
|
||||
let ticket = DeleteRecoveryTicket {
|
||||
generation: entry.generation,
|
||||
admission: None,
|
||||
retry,
|
||||
key,
|
||||
};
|
||||
state.pending.remove(&ticket.key);
|
||||
return Some(ticket);
|
||||
}
|
||||
None
|
||||
}
|
||||
@@ -255,24 +294,38 @@ impl DeleteRecoveryHints {
|
||||
#[cfg(test)]
|
||||
fn take(&self, limit: usize) -> Vec<DeleteRecoveryTicket> {
|
||||
let active = HashSet::new();
|
||||
std::iter::from_fn(|| self.take_available(&active)).take(limit).collect()
|
||||
std::iter::from_fn(|| self.take_available(&active, Instant::now()))
|
||||
.take(limit)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn retry(&self, ticket: DeleteRecoveryTicket) {
|
||||
fn retry(&self, ticket: DeleteRecoveryTicket, retry: RetrySchedule) {
|
||||
let mut state = self.state.lock().unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||
if state.pending.contains_key(&ticket.key) {
|
||||
if let Some(entry) = state.pending.get_mut(&ticket.key) {
|
||||
// Discovery can see the same journal while replay is in flight.
|
||||
// Keep its queue slot, but only new foreground work resets backoff.
|
||||
if matches!(entry.kind, PendingHintKind::Discovered) {
|
||||
entry.kind = PendingHintKind::Retry(retry);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if state.pending.len() >= self.capacity {
|
||||
// Attempted work yields to new admissions and the periodic durable sweep.
|
||||
return;
|
||||
}
|
||||
state.compact_order(self.capacity);
|
||||
if state.pending.len() >= self.capacity {
|
||||
// Previously attempted work yields to admitted keys until the
|
||||
// periodic sweep. Foreground admission retains its own rescan guard.
|
||||
return;
|
||||
}
|
||||
state
|
||||
.pending
|
||||
.insert(ticket.key.clone(), (ticket.generation, ticket.generation, true));
|
||||
state.order.push_back(ticket);
|
||||
state.enqueue(ticket.key, ticket.generation, PendingHintKind::Retry(retry));
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
fn retry_now(&self, ticket: DeleteRecoveryTicket) {
|
||||
self.retry(
|
||||
ticket,
|
||||
RetrySchedule {
|
||||
ready_at: Instant::now(),
|
||||
delay: Duration::ZERO,
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -642,7 +695,12 @@ impl<'a> RecoveryWork<'a> {
|
||||
&& Instant::now() < self.deadline
|
||||
&& !self.cancel.is_cancelled()
|
||||
{
|
||||
let Some(ticket) = self.store.ctx.delete_recovery_hints().take_available(&self.active) else {
|
||||
let Some(ticket) = self
|
||||
.store
|
||||
.ctx
|
||||
.delete_recovery_hints()
|
||||
.take_available(&self.active, Instant::now())
|
||||
else {
|
||||
break;
|
||||
};
|
||||
self.active.insert(ticket.key.clone());
|
||||
@@ -688,15 +746,32 @@ async fn recover_hint(
|
||||
) => Some(result),
|
||||
};
|
||||
let mut failures = RecoveryFailures::default();
|
||||
if !matches!(outcome, Some(Ok(_))) {
|
||||
match outcome {
|
||||
Some(Err(crate::error::StorageError::ErasureReadQuorum)) => failures.pending = 1,
|
||||
Some(Err(crate::error::StorageError::SlowDown)) => failures.blocked = 1,
|
||||
Some(Err(crate::error::StorageError::FileCorrupt)) => failures.corrupt = 1,
|
||||
Some(Err(_)) => failures.failed = 1,
|
||||
_ => {}
|
||||
let now = Instant::now();
|
||||
let retry = match outcome {
|
||||
Some(Ok(_)) => None,
|
||||
Some(Err(StorageError::ErasureReadQuorum)) => {
|
||||
failures.pending = 1;
|
||||
// Check returning participants on the normal maintenance cadence.
|
||||
Some(RetrySchedule {
|
||||
ready_at: now + INTERVAL,
|
||||
delay: INTERVAL,
|
||||
})
|
||||
}
|
||||
store.ctx.delete_recovery_hints().retry(ticket);
|
||||
Some(Err(error)) => {
|
||||
match error {
|
||||
StorageError::SlowDown => failures.blocked = 1,
|
||||
StorageError::FileCorrupt => failures.corrupt = 1,
|
||||
_ => failures.failed = 1,
|
||||
}
|
||||
Some(RetrySchedule::after_failure(ticket.retry, now))
|
||||
}
|
||||
None => Some(ticket.retry.unwrap_or(RetrySchedule {
|
||||
ready_at: now,
|
||||
delay: Duration::ZERO,
|
||||
})),
|
||||
};
|
||||
if let Some(retry) = retry {
|
||||
store.ctx.delete_recovery_hints().retry(ticket, retry);
|
||||
}
|
||||
(key, failures)
|
||||
}
|
||||
@@ -826,7 +901,7 @@ mod tests {
|
||||
assert_eq!(taken[0].generation, newer.generation);
|
||||
let latest = hints.register(key("a")).expect("hint during recovery");
|
||||
hints.ack(&taken[0]);
|
||||
hints.retry(taken.into_iter().next().expect("selected ticket"));
|
||||
hints.retry_now(taken.into_iter().next().expect("selected ticket"));
|
||||
let pending = hints.take(2);
|
||||
assert_eq!(pending[0].key.object, "b");
|
||||
assert_eq!(pending[1].generation, latest.generation, "old recovery cannot replace a newer hint");
|
||||
@@ -865,7 +940,7 @@ mod tests {
|
||||
assert!(hints.state.lock().expect("hint state").order.len() <= 2);
|
||||
let mut tickets = hints.take(2);
|
||||
assert_eq!(tickets.len(), 2);
|
||||
hints.retry(tickets.remove(0));
|
||||
hints.retry_now(tickets.remove(0));
|
||||
assert_eq!(hints.take(2).len(), 1, "failed work survives an empty queue");
|
||||
}
|
||||
|
||||
@@ -877,18 +952,48 @@ mod tests {
|
||||
let active = HashSet::from([admitted.key.clone()]);
|
||||
let newer = hints.register(key("active")).expect("new foreground generation");
|
||||
hints.register(key("later")).expect("independent key");
|
||||
let later = hints.take_available(&active).expect("skip the active key");
|
||||
let later = hints.take_available(&active, Instant::now()).expect("skip the active key");
|
||||
assert_eq!(later.key.object, "later");
|
||||
assert!(hints.take_available(&active).is_none(), "a newer generation cannot occupy a second slot");
|
||||
assert!(
|
||||
hints.take_available(&active, Instant::now()).is_none(),
|
||||
"a newer generation cannot occupy a second slot"
|
||||
);
|
||||
hints.ack(&original);
|
||||
hints.retry(admitted);
|
||||
hints.retry_now(admitted);
|
||||
assert!(hints.is_pending(&key("active")), "old success or cancellation cannot clear newer work");
|
||||
let next = hints
|
||||
.take_available(&HashSet::new())
|
||||
.take_available(&HashSet::new(), Instant::now())
|
||||
.expect("active key becomes eligible after completion");
|
||||
assert_eq!(next.generation, newer.generation, "retry cannot overwrite the foreground generation");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_recovery_backoff_survives_rediscovery_and_resets_for_foreground_work() {
|
||||
let hints = DeleteRecoveryHints::with_capacity(2);
|
||||
let active = HashSet::new();
|
||||
let key = key("blocked");
|
||||
let mut now = Instant::now();
|
||||
assert!(hints.register_discovered(key.clone()));
|
||||
let mut ticket = hints.take_available(&active, now).expect("discovery is immediately eligible");
|
||||
for seconds in [5, 10, 20, 40, 80, 160, 300, 300] {
|
||||
assert!(hints.register_discovered(key.clone()), "discovery arrives during replay");
|
||||
let retry = RetrySchedule::after_failure(ticket.retry, now);
|
||||
hints.retry(ticket, retry);
|
||||
assert!(hints.register_discovered(key.clone()), "discovery also sees the queued retry");
|
||||
now += Duration::from_secs(seconds);
|
||||
assert!(hints.take_available(&active, now - Duration::from_nanos(1)).is_none());
|
||||
ticket = hints.take_available(&active, now).expect("admit at the retry deadline");
|
||||
}
|
||||
let retry = RetrySchedule::after_failure(ticket.retry, now);
|
||||
hints.retry(ticket, retry);
|
||||
let foreground = hints.register(key).expect("foreground request during backoff");
|
||||
let ticket = hints
|
||||
.take_available(&active, now)
|
||||
.expect("admit new foreground work promptly");
|
||||
assert_eq!(ticket.generation, foreground.generation);
|
||||
assert!(ticket.retry.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn delete_recovery_hint_compaction_stays_amortized_near_capacity() {
|
||||
let capacity = HINT_CAPACITY;
|
||||
@@ -925,7 +1030,7 @@ mod tests {
|
||||
hints.register(key("blocked-a")).expect("first blocked key");
|
||||
hints.register(key("blocked-b")).expect("second blocked key");
|
||||
for ticket in hints.take(2) {
|
||||
hints.retry(ticket);
|
||||
hints.retry_now(ticket);
|
||||
}
|
||||
hints
|
||||
.register(key("healthy-later"))
|
||||
@@ -1118,7 +1223,8 @@ mod tests {
|
||||
publish_one(&disks[0], bucket, transaction).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let offline = set.disks.write().await[3].take();
|
||||
pass.run(&store, &CancellationToken::new()).await;
|
||||
let cancel = CancellationToken::new();
|
||||
assert_eq!(recover_hints(&store, &cancel).await.pending, 1, "the participant is required");
|
||||
assert_eq!(
|
||||
store
|
||||
.ctx
|
||||
@@ -1131,7 +1237,10 @@ mod tests {
|
||||
1
|
||||
);
|
||||
set.disks.write().await[3] = offline;
|
||||
pass.run(&store, &CancellationToken::new()).await;
|
||||
tokio::time::pause();
|
||||
tokio::time::advance(INTERVAL).await;
|
||||
tokio::time::resume();
|
||||
pass.run(&store, &cancel).await;
|
||||
assert!(
|
||||
store.ctx.delete_recovery_hints().take(1).is_empty(),
|
||||
"returning participant releases retained work"
|
||||
@@ -1184,7 +1293,7 @@ mod tests {
|
||||
assert!(hints.register_discovered(key(&format!("blocked-{index}"))));
|
||||
}
|
||||
for ticket in hints.take(KEYS_PER_PASS) {
|
||||
hints.retry(ticket);
|
||||
hints.retry_now(ticket);
|
||||
}
|
||||
for index in 0..KEYS_PER_PASS {
|
||||
assert!(
|
||||
@@ -1194,7 +1303,7 @@ mod tests {
|
||||
}
|
||||
let in_flight = hints.take(1).pop().expect("one admitted recovery attempt");
|
||||
assert!(hints.register_discovered(key("arrived-during-recovery")));
|
||||
hints.retry(in_flight);
|
||||
hints.retry_now(in_flight);
|
||||
let mut pass = RecoveryPass {
|
||||
next_inventory: Some(Instant::now() + ANTI_ENTROPY_INTERVAL),
|
||||
scan_generation: Some(scanning),
|
||||
@@ -1221,6 +1330,65 @@ mod tests {
|
||||
assert_eq!(hints.rescan(), Some(foreground), "an older scan cannot clear a foreground obligation");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_blocked_legacy_key_backs_off_without_delaying_other_keys() {
|
||||
let bucket = "runtime-legacy-backoff";
|
||||
let (dirs, store, disks) = fixture(bucket).await;
|
||||
let transaction = Uuid::new_v4();
|
||||
for disk in &disks {
|
||||
disk.delete_version(
|
||||
bucket,
|
||||
"object",
|
||||
FileInfo {
|
||||
name: "object".to_owned(),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("leave an undecided historical delete");
|
||||
}
|
||||
let backups: Vec<_> = dirs
|
||||
.iter()
|
||||
.map(|dir| {
|
||||
dir.path()
|
||||
.join(bucket)
|
||||
.join("object")
|
||||
.join(transaction.to_string())
|
||||
.join("xl.meta.bkp")
|
||||
})
|
||||
.collect();
|
||||
let before: Vec<_> = backups
|
||||
.iter()
|
||||
.map(|path| std::fs::read(path).expect("read original evidence"))
|
||||
.collect();
|
||||
register(&store, bucket).await;
|
||||
let cancel = CancellationToken::new();
|
||||
assert_eq!(recover_hints(&store, &cancel).await.blocked, 1);
|
||||
let hints = store.ctx.delete_recovery_hints();
|
||||
let healthy = DeleteRecoveryKey {
|
||||
object: "absent-key".to_owned(),
|
||||
bucket: bucket.to_owned(),
|
||||
incarnation: Some(store.bucket_incarnation_id_from_disk(bucket).await.expect("bucket identity")),
|
||||
..key("absent-key")
|
||||
};
|
||||
assert!(hints.register_discovered(healthy.clone()));
|
||||
assert_eq!(
|
||||
recover_hints(&store, &cancel).await.blocked,
|
||||
0,
|
||||
"do not re-inspect a blocked key on the next pass"
|
||||
);
|
||||
assert!(!hints.is_pending(&healthy), "the independent key still settles promptly");
|
||||
assert_eq!(hints.pending_len(), 1, "the blocked key remains eligible for a later retry");
|
||||
for (path, bytes) in backups.iter().zip(before) {
|
||||
assert_eq!(std::fs::read(path).expect("retained historical evidence"), bytes);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delete_recovery_admission_stays_within_each_tick_budget() {
|
||||
let bucket = "runtime-admission-budget";
|
||||
@@ -1413,7 +1581,10 @@ mod tests {
|
||||
.find(|(key, _)| key.object == format!("waiting-{index}"))
|
||||
.expect("unattempted key remains queued")
|
||||
.1;
|
||||
assert!(!queued.2, "budget exhaustion must not grant discovery permission to evict untouched work");
|
||||
assert!(
|
||||
matches!(queued.kind, PendingHintKind::Foreground),
|
||||
"budget exhaustion must not grant discovery permission to evict untouched work"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1434,7 +1605,7 @@ mod tests {
|
||||
for ticket in hints.take(KEYS_PER_PASS) {
|
||||
let index: usize = ticket.key.object.parse().expect("numeric test key");
|
||||
if index % 1_024 < KEYS_PER_PASS {
|
||||
hints.retry(ticket);
|
||||
hints.retry_now(ticket);
|
||||
} else {
|
||||
recovered.insert(index);
|
||||
}
|
||||
|
||||
@@ -29,6 +29,7 @@ const LOG_SUBSYSTEM_HEAL: &str = "heal";
|
||||
const EVENT_HEAL_ABANDONED_PARTS: &str = "heal_abandoned_parts";
|
||||
const EVENT_HEAL_FORMAT_COMPLETED: &str = "heal_format_completed";
|
||||
const EVENT_HEAL_OBJECT_STARTED: &str = "heal_object_started";
|
||||
const EVENT_HEAL_DELETE_RECOVERY_BLOCKED: &str = "heal_delete_recovery_blocked";
|
||||
|
||||
/// An explicit bucket-heal admission owns one lifecycle generation through its write tail.
|
||||
pub(crate) struct BucketHealScope {
|
||||
@@ -314,6 +315,104 @@ impl ECStore {
|
||||
.map_err(|error| Error::other_with_context("bucket heal owner task failed", error))?
|
||||
}
|
||||
|
||||
/// Inspect every participant and sibling transaction for one user key in its selected set.
|
||||
/// A partial inspection never produces a fingerprint authorizing a subsequent discard.
|
||||
pub async fn inspect_delete_residue(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pool: usize,
|
||||
set: usize,
|
||||
) -> Result<rustfs_madmin::heal_commands::DeleteResidueInspection> {
|
||||
self.admin_delete_residue(bucket, object, pool, set, None).await
|
||||
}
|
||||
|
||||
/// Discard only eligible historical evidence under a fresh exact-key inspection.
|
||||
/// A stale or ineligible preview returns PreconditionFailed. If a drive fails after
|
||||
/// others have discarded their copies, inspect again before retrying.
|
||||
pub async fn discard_delete_residue(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pool: usize,
|
||||
set: usize,
|
||||
expected_fingerprint: &str,
|
||||
) -> Result<rustfs_madmin::heal_commands::DeleteResidueInspection> {
|
||||
if expected_fingerprint.len() != 64
|
||||
|| !expected_fingerprint
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b))
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
self.admin_delete_residue(bucket, object, pool, set, Some(expected_fingerprint.to_owned()))
|
||||
.await
|
||||
}
|
||||
|
||||
async fn admin_delete_residue(
|
||||
self: &Arc<Self>,
|
||||
bucket: &str,
|
||||
object: &str,
|
||||
pool_index: usize,
|
||||
set_index: usize,
|
||||
expected: Option<String>,
|
||||
) -> Result<rustfs_madmin::heal_commands::DeleteResidueInspection> {
|
||||
use rustfs_madmin::heal_commands::DeleteResidueInspection;
|
||||
check_valid_bucket_name_strict(bucket)?;
|
||||
check_object_args(bucket, object)?;
|
||||
let incarnation = self.bucket_incarnation_id_from_disk(bucket).await?;
|
||||
let object = object.to_owned();
|
||||
self.run_bucket_heal_at_incarnation(bucket, incarnation, &HealOpts::default(), move |store, bucket, _| async move {
|
||||
let scope = bucket_heal_scope(&bucket).ok_or(StorageError::PreconditionFailed)?;
|
||||
let gate = store.ctx.data_movement_operation_gate();
|
||||
let _movement_guard = gate.read().await;
|
||||
let pool = store
|
||||
.pools
|
||||
.get(pool_index)
|
||||
.ok_or_else(|| invalid_heal_pool_index(pool_index, store.pools.len()))?;
|
||||
let physical_key = encode_dir_object(&object);
|
||||
let set = pool
|
||||
.disk_set
|
||||
.get(set_index)
|
||||
.filter(|set| Arc::ptr_eq(set, &pool.get_disks_by_key(&physical_key)))
|
||||
.ok_or_else(|| {
|
||||
StorageError::InvalidArgument(
|
||||
"delete residue".to_owned(),
|
||||
"set".to_owned(),
|
||||
"selected set does not own the key".to_owned(),
|
||||
)
|
||||
})?;
|
||||
if store.is_suspended(pool_index).await || store.is_pool_rebalancing(pool_index).await {
|
||||
return Err(StorageError::SlowDown);
|
||||
}
|
||||
let (config_guard, allows_discard) =
|
||||
metadata_sys::legacy_discard_config_guard_in(&store.ctx, &bucket, incarnation).await?;
|
||||
let report = DeleteResidueInspection {
|
||||
bucket,
|
||||
object,
|
||||
pool: pool_index,
|
||||
set: set_index,
|
||||
incarnation: incarnation.to_string(),
|
||||
complete: false,
|
||||
reason: String::new(),
|
||||
participants: Vec::new(),
|
||||
transactions: Vec::new(),
|
||||
fingerprint: None,
|
||||
reclaimed: 0,
|
||||
};
|
||||
set.operator_delete_residue(
|
||||
&physical_key,
|
||||
report,
|
||||
expected.as_deref(),
|
||||
allows_discard,
|
||||
pool.format.erasure.sets.get(set_index).map(Vec::as_slice).unwrap_or_default(),
|
||||
&|| scope.check().is_ok() && !config_guard.is_lock_lost(),
|
||||
)
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
/// Recover hidden delete journals through the lifecycle-owned heal boundary.
|
||||
pub async fn recover_delete_transactions(self: &Arc<Self>, bucket: &str, opts: &HealOpts) -> Result<HealResultItem> {
|
||||
Ok(self
|
||||
@@ -409,6 +508,24 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(object) = &object
|
||||
&& !opts.dry_run
|
||||
&& summary.blocked > 0
|
||||
{
|
||||
tracing::debug!(
|
||||
event = EVENT_HEAL_DELETE_RECOVERY_BLOCKED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_HEAL,
|
||||
state = "insufficient_evidence",
|
||||
bucket = ?bucket,
|
||||
object = ?decode_dir_object(object),
|
||||
pool = opts.pool,
|
||||
set = opts.set,
|
||||
legacy_ambiguous = summary.legacy_ambiguous,
|
||||
other_blocked = summary.blocked - summary.legacy_ambiguous,
|
||||
"Exact-key delete recovery is blocked"
|
||||
);
|
||||
}
|
||||
if object.is_some() && summary.pending > 0 {
|
||||
return Err(StorageError::ErasureReadQuorum);
|
||||
}
|
||||
@@ -1702,6 +1819,74 @@ mod tests {
|
||||
.expect("minimal pool should build")
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn delete_recovery_operator_discard_is_scoped_to_the_selected_pool() {
|
||||
let (root, store, shutdown) = multi_pool_heal_store().await;
|
||||
let bucket = "operator-cross-pool";
|
||||
let key = "object";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let options = ObjectOptions {
|
||||
write_completion: crate::object_api::WriteCompletion::TailDrained,
|
||||
..Default::default()
|
||||
};
|
||||
for (pool, contents) in store.pools.iter().zip([b"old bytes", b"new bytes"]) {
|
||||
pool.put_object(bucket, key, &mut PutObjReader::from_vec(contents.to_vec()), &options)
|
||||
.await
|
||||
.expect("put distinct pool copy");
|
||||
}
|
||||
let transaction = Uuid::new_v4();
|
||||
for disk in store.pools[0].disk_set[0].disk_inventory().await.into_iter().flatten() {
|
||||
disk.delete_version(
|
||||
bucket,
|
||||
key,
|
||||
rustfs_filemeta::FileInfo {
|
||||
name: key.to_owned(),
|
||||
..Default::default()
|
||||
},
|
||||
false,
|
||||
DeleteOptions {
|
||||
old_data_dir: Some(transaction),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create legacy source pool residue");
|
||||
}
|
||||
let other_pool_metadata = (0..4)
|
||||
.map(|slot| root.path().join(format!("pool1-disk{slot}/{bucket}/{key}/xl.meta")))
|
||||
.map(|path| {
|
||||
let bytes = std::fs::read(&path).expect("other pool's live copy");
|
||||
(path, bytes)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
let proof = store
|
||||
.inspect_delete_residue(bucket, key, 0, 0)
|
||||
.await
|
||||
.expect("inspect source pool")
|
||||
.fingerprint
|
||||
.expect("source residue");
|
||||
assert!(matches!(
|
||||
store.discard_delete_residue(bucket, key, 1, 0, &proof).await,
|
||||
Err(StorageError::PreconditionFailed)
|
||||
));
|
||||
assert_eq!(
|
||||
store
|
||||
.discard_delete_residue(bucket, key, 0, 0, &proof)
|
||||
.await
|
||||
.expect("scoped cleanup")
|
||||
.reclaimed,
|
||||
1
|
||||
);
|
||||
for (path, contents) in other_pool_metadata {
|
||||
assert_eq!(std::fs::read(path).expect("other pool remains live"), contents);
|
||||
}
|
||||
shutdown.cancel();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn admin_bucket_recovery_recreates_missing_directories_at_current_incarnation() {
|
||||
|
||||
@@ -40,8 +40,8 @@ use crate::storage_api_contracts::list::{
|
||||
AuthoritativeListOperations, ListObjectsInfo as StorageListObjectsInfo, StorageListObjectVersionsInfo,
|
||||
StorageListObjectsV2Info, StorageObjectInfoOrErr, StorageWalkOptions, VersionMarker, WalkVersionsSortOrder,
|
||||
};
|
||||
use crate::store::ECStore;
|
||||
use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||
use crate::store::{ECStore, VersioningConfiguration};
|
||||
use futures::future::join_all;
|
||||
use rand::seq::SliceRandom;
|
||||
use rustfs_filemeta::{
|
||||
@@ -78,6 +78,113 @@ type ListObjectVersionsInfo = StorageListObjectVersionsInfo<ObjectInfo>;
|
||||
type ObjectInfoOrErr = StorageObjectInfoOrErr<ObjectInfo, Error>;
|
||||
type WalkOptions = StorageWalkOptions<fn(&rustfs_filemeta::FileInfo) -> bool>;
|
||||
|
||||
/// Evidence from a single named object during free-version recovery.
|
||||
#[derive(Debug)]
|
||||
pub(crate) enum TierRecoveryEntry {
|
||||
Version(Box<ObjectInfo>),
|
||||
Unreadable { object: String },
|
||||
}
|
||||
|
||||
enum ListingMergeOutput {
|
||||
Entries(Sender<MetaCacheEntry>),
|
||||
TierRecovery {
|
||||
sender: Sender<Result<TierRecoveryEntry>>,
|
||||
bucket: String,
|
||||
versioning: Option<VersioningConfiguration>,
|
||||
marker: Option<String>,
|
||||
},
|
||||
}
|
||||
|
||||
impl From<Sender<MetaCacheEntry>> for ListingMergeOutput {
|
||||
fn from(sender: Sender<MetaCacheEntry>) -> Self {
|
||||
Self::Entries(sender)
|
||||
}
|
||||
}
|
||||
|
||||
impl ListingMergeOutput {
|
||||
fn accepts_key(&self, key: &str) -> Result<bool> {
|
||||
if let Self::TierRecovery { marker, .. } = self {
|
||||
if key.is_empty() {
|
||||
return Err(Error::FileCorrupt);
|
||||
}
|
||||
// The drive may repeat the last key. A recovery cursor covers every
|
||||
// version (or the unreadable metadata) at that key.
|
||||
return Ok(marker.as_deref().is_none_or(|marker| key > marker));
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
async fn send_group(
|
||||
&self,
|
||||
cancel: &CancellationToken,
|
||||
mut entry: MetaCacheEntry,
|
||||
others: std::vec::Drain<'_, MetaCacheEntry>,
|
||||
last_emitted: &mut String,
|
||||
) -> Result<bool> {
|
||||
if !self.accepts_key(&entry.name)? {
|
||||
return Ok(true);
|
||||
}
|
||||
let merge = if others.as_slice().is_empty() {
|
||||
Ok(())
|
||||
} else {
|
||||
merge_object_entry_versions(
|
||||
&mut entry,
|
||||
others,
|
||||
match self {
|
||||
Self::Entries(_) => log_listing_merge_corruption,
|
||||
// Recovery reports unreadable keys once per page.
|
||||
Self::TierRecovery { .. } => |_, _, _| {},
|
||||
},
|
||||
)
|
||||
};
|
||||
let is_new = entry.name != *last_emitted;
|
||||
if is_new {
|
||||
last_emitted.clone_from(&entry.name);
|
||||
}
|
||||
match (self, merge) {
|
||||
(Self::TierRecovery { sender, .. }, Err(Error::FileCorrupt)) if is_new => {
|
||||
send_or_cancel(cancel, sender, Ok(TierRecoveryEntry::Unreadable { object: entry.name })).await
|
||||
}
|
||||
(Self::TierRecovery { .. }, Err(Error::FileCorrupt)) => Ok(true),
|
||||
(_, Err(err)) => Err(err),
|
||||
(_, Ok(())) if is_new => self.send(cancel, entry).await,
|
||||
(_, Ok(())) => Ok(true),
|
||||
}
|
||||
}
|
||||
|
||||
async fn send(&self, cancel: &CancellationToken, entry: MetaCacheEntry) -> Result<bool> {
|
||||
match self {
|
||||
Self::Entries(sender) => send_or_cancel(cancel, sender, entry).await,
|
||||
Self::TierRecovery {
|
||||
sender,
|
||||
bucket,
|
||||
versioning,
|
||||
..
|
||||
} => {
|
||||
if entry.is_dir() {
|
||||
return Ok(true);
|
||||
}
|
||||
// This operation decodes an already enumerated buffer. Even
|
||||
// parser errors wrapping UnexpectedEof are local to this key.
|
||||
let versions = match entry.file_info_versions_with_free_versions(bucket) {
|
||||
Ok(versions) => versions,
|
||||
Err(_) => {
|
||||
return send_or_cancel(cancel, sender, Ok(TierRecoveryEntry::Unreadable { object: entry.name })).await;
|
||||
}
|
||||
};
|
||||
let versioned = versioning.as_ref().is_some_and(|v| v.versioned(&entry.name));
|
||||
for fi in versions.versions.into_iter().chain(versions.free_versions) {
|
||||
let info = ObjectInfo::from_owned_file_info(fi, bucket, &entry.name, versioned);
|
||||
if !send_or_cancel(cancel, sender, Ok(TierRecoveryEntry::Version(Box::new(info)))).await? {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn unversioned_authoritative_page(bucket: &str, page: ListPage) -> Result<ListObjectVersionsInfo> {
|
||||
let mut objects = Vec::new();
|
||||
for entry in page.entries.entries() {
|
||||
@@ -1358,7 +1465,7 @@ impl ECStore {
|
||||
return Ok(Vec::new());
|
||||
};
|
||||
if entries.len() > 0 {
|
||||
merge_object_entry_versions(&mut entry, entries)?;
|
||||
merge_object_entry_versions(&mut entry, entries, log_listing_merge_corruption)?;
|
||||
}
|
||||
// This projection excludes tier free-version bookkeeping but retains
|
||||
// pending version purges for the evaluator's replication safeguards.
|
||||
@@ -1714,12 +1821,63 @@ impl ECStore {
|
||||
check_list_objs_args(bucket, prefix, &None)?;
|
||||
|
||||
let rx = rx.child_token();
|
||||
let (sender, receiver) = mpsc::channel(100);
|
||||
// Some strict consumers drain the result only after walk returns.
|
||||
let _forwarder = spawn_walk_entry_forwarder(rx.clone(), receiver, result, opts.clone(), bucket.to_owned());
|
||||
self.walk_merged_entries(rx, bucket, prefix, opts, sender.into()).await
|
||||
}
|
||||
|
||||
pub(crate) async fn walk_for_tier_recovery(
|
||||
self: Arc<Self>,
|
||||
cancel: CancellationToken,
|
||||
bucket: &str,
|
||||
sender: Sender<Result<TierRecoveryEntry>>,
|
||||
mut opts: WalkOptions,
|
||||
) -> Result<()> {
|
||||
check_list_objs_args(bucket, "", &None)?;
|
||||
let cancel = cancel.child_token();
|
||||
let _cancel_on_drop = cancel.clone().drop_guard();
|
||||
let versioning = tokio::select! {
|
||||
_ = cancel.cancelled() => return Ok(()),
|
||||
_ = sender.closed() => return Ok(()),
|
||||
result = get_versioning_config(bucket) => result.ok().map(|(config, _)| config),
|
||||
};
|
||||
if opts.marker.is_some() && opts.limit > 0 {
|
||||
// Drives include forward_to in their limit; the consumer counts
|
||||
// only distinct keys strictly beyond the recovery marker.
|
||||
opts.limit = opts.limit.saturating_add(1);
|
||||
}
|
||||
let output = ListingMergeOutput::TierRecovery {
|
||||
sender,
|
||||
bucket: bucket.to_owned(),
|
||||
versioning,
|
||||
marker: opts.marker.clone(),
|
||||
};
|
||||
self.walk_merged_entries(cancel, bucket, "", opts, output).await
|
||||
}
|
||||
|
||||
async fn walk_merged_entries(
|
||||
self: Arc<Self>,
|
||||
rx: CancellationToken,
|
||||
bucket: &str,
|
||||
prefix: &str,
|
||||
opts: WalkOptions,
|
||||
output: ListingMergeOutput,
|
||||
) -> Result<()> {
|
||||
let mut futures = Vec::new();
|
||||
let mut inputs = Vec::new();
|
||||
|
||||
for eset in self.pools.iter() {
|
||||
for set in eset.disk_set.iter() {
|
||||
let (mut disks, infos, _) = set.get_online_disks_with_healing_and_info(true).await;
|
||||
let online_disks = set.get_online_disks_with_healing_and_info(true);
|
||||
let (mut disks, infos, _) = match &output {
|
||||
ListingMergeOutput::Entries(_) => online_disks.await,
|
||||
ListingMergeOutput::TierRecovery { sender, .. } => tokio::select! {
|
||||
_ = rx.cancelled() => return Ok(()),
|
||||
_ = sender.closed() => return Ok(()),
|
||||
disks = online_disks => disks,
|
||||
},
|
||||
};
|
||||
let opts = opts.clone();
|
||||
|
||||
let (sender, list_out_rx) = mpsc::channel::<MetaCacheEntry>(1);
|
||||
@@ -1832,13 +1990,7 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
let (merge_tx, merge_rx) = mpsc::channel::<MetaCacheEntry>(100);
|
||||
let bucket = bucket.to_owned();
|
||||
// Not awaited: callers may drain `result` only after this returns, so
|
||||
// waiting on the forwarder here could stall them on the channel bound.
|
||||
let _forwarder = spawn_walk_entry_forwarder(rx.clone(), merge_rx, result, opts, bucket.clone());
|
||||
|
||||
let merge_task = spawn_listing_merge(rx, inputs, merge_tx);
|
||||
let merge_task = spawn_listing_merge(rx, inputs, output);
|
||||
|
||||
let walk_started = std::time::Instant::now();
|
||||
let walk_results = join_all(futures).await;
|
||||
@@ -1921,6 +2073,23 @@ impl ECStore {
|
||||
}
|
||||
}
|
||||
|
||||
/// Stop the producers if recovery drops its receiver, retaining terminal
|
||||
/// producer errors that may already be in flight.
|
||||
async fn run_tier_walk_with_consumer(
|
||||
cancel: &CancellationToken,
|
||||
consumer: &Sender<Result<TierRecoveryEntry>>,
|
||||
walk: impl Future<Output = Result<()>>,
|
||||
) -> Result<()> {
|
||||
tokio::pin!(walk);
|
||||
tokio::select! {
|
||||
result = &mut walk => result,
|
||||
_ = consumer.closed() => {
|
||||
cancel.cancel();
|
||||
walk.await
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Forwards merged raw entries to a walk consumer as `ObjectInfoOrErr`,
|
||||
/// applying the walk's version projection, sort order and filter. The first
|
||||
/// entry that fails to decode ends the stream with that error; a consumer that
|
||||
@@ -2476,7 +2645,7 @@ async fn send_or_cancel<T: Send>(rx: &CancellationToken, out_channel: &Sender<T>
|
||||
}
|
||||
}
|
||||
|
||||
fn listing_merge_corruption(object: &str, version_id: Option<Uuid>, validation: &'static str) -> Error {
|
||||
fn log_listing_merge_corruption(object: &str, version_id: Option<Uuid>, validation: &'static str) {
|
||||
error!(
|
||||
event = EVENT_LISTING_MERGE_REJECTED,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
@@ -2487,30 +2656,39 @@ fn listing_merge_corruption(object: &str, version_id: Option<Uuid>, validation:
|
||||
validation,
|
||||
"Listing version merge rejected metadata"
|
||||
);
|
||||
Error::FileCorrupt
|
||||
}
|
||||
|
||||
/// Each input has already been resolved inside its own erasure set. This is a
|
||||
/// union of version histories, never a quorum vote between unrelated pools.
|
||||
fn merge_object_entry_versions(first: &mut MetaCacheEntry, others: impl Iterator<Item = MetaCacheEntry>) -> Result<()> {
|
||||
let name = first.name.clone();
|
||||
fn merge_object_entry_versions(
|
||||
first: &mut MetaCacheEntry,
|
||||
others: impl Iterator<Item = MetaCacheEntry>,
|
||||
report_corruption: fn(&str, Option<Uuid>, &'static str),
|
||||
) -> Result<()> {
|
||||
let name = first.name.as_str();
|
||||
let corrupt = |version_id, validation| {
|
||||
report_corruption(name, version_id, validation);
|
||||
Error::FileCorrupt
|
||||
};
|
||||
let mut versions: HashMap<(Option<Uuid>, bool), (FileMetaShallowVersion, ObjectInfo)> = HashMap::new();
|
||||
for mut entry in std::iter::once(std::mem::take(first)).chain(others) {
|
||||
let meta = match entry.cached.take() {
|
||||
let inputs = std::iter::once((first.cached.take(), std::mem::take(&mut first.metadata)))
|
||||
.chain(others.map(|entry| (entry.cached, entry.metadata)));
|
||||
for (cached, metadata) in inputs {
|
||||
let meta = match cached {
|
||||
Some(meta) => meta,
|
||||
None => FileMeta::load(&entry.metadata).map_err(|_| listing_merge_corruption(&name, None, "metadata_decode"))?,
|
||||
None => FileMeta::load(&metadata).map_err(|_| corrupt(None, "metadata_decode"))?,
|
||||
};
|
||||
if meta.versions.is_empty() {
|
||||
return Err(listing_merge_corruption(&name, None, "empty_version_stack"));
|
||||
return Err(corrupt(None, "empty_version_stack"));
|
||||
}
|
||||
for version in meta.versions {
|
||||
let corrupt = |validation| listing_merge_corruption(&name, version.header.version_id, validation);
|
||||
let corrupt = |validation| corrupt(version.header.version_id, validation);
|
||||
let parsed = version.parse_version_meta().map_err(|_| corrupt("version_decode"))?;
|
||||
if !parsed.valid() || parsed.version_type != version.header.version_type {
|
||||
return Err(corrupt("version_type_or_validity"));
|
||||
}
|
||||
let fi = parsed
|
||||
.into_fileinfo("", &name, true)
|
||||
.into_fileinfo("", name, true)
|
||||
.map_err(|_| corrupt("file_info_decode"))?;
|
||||
let version_id = fi.version_id.filter(|id| !id.is_nil());
|
||||
if version_id != version.header.version_id.filter(|id| !id.is_nil())
|
||||
@@ -2519,7 +2697,7 @@ fn merge_object_entry_versions(first: &mut MetaCacheEntry, others: impl Iterator
|
||||
{
|
||||
return Err(corrupt("version_header_mismatch"));
|
||||
}
|
||||
let info = ObjectInfo::from_file_info(&fi, "", &name, true);
|
||||
let info = ObjectInfo::from_file_info(&fi, "", name, true);
|
||||
let identity = (version_id, version.header.free_version());
|
||||
match versions.entry(identity) {
|
||||
std::collections::hash_map::Entry::Vacant(slot) => {
|
||||
@@ -2588,13 +2766,9 @@ fn merge_object_entry_versions(first: &mut MetaCacheEntry, others: impl Iterator
|
||||
std::cmp::Ordering::Equal
|
||||
}
|
||||
});
|
||||
let metadata = merged.marshal_msg()?;
|
||||
*first = MetaCacheEntry {
|
||||
name,
|
||||
metadata,
|
||||
cached: Some(merged),
|
||||
reusable: true,
|
||||
};
|
||||
first.metadata = merged.marshal_msg()?;
|
||||
first.cached = Some(merged);
|
||||
first.reusable = true;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -2603,11 +2777,25 @@ fn merge_object_entry_versions(first: &mut MetaCacheEntry, others: impl Iterator
|
||||
fn spawn_listing_merge(
|
||||
rx: CancellationToken,
|
||||
inputs: Vec<Receiver<MetaCacheEntry>>,
|
||||
sender: Sender<MetaCacheEntry>,
|
||||
sender: impl Into<ListingMergeOutput>,
|
||||
) -> AbortOnDropHandle<Result<()>> {
|
||||
let sender = sender.into();
|
||||
AbortOnDropHandle::new(tokio::spawn(
|
||||
async move {
|
||||
let result = merge_entry_channels(rx.clone(), inputs, sender, 1).await;
|
||||
let consumer = match &sender {
|
||||
ListingMergeOutput::TierRecovery { sender, .. } => Some(sender.clone()),
|
||||
ListingMergeOutput::Entries(_) => None,
|
||||
};
|
||||
let merge = merge_entry_channels(rx.clone(), inputs, sender, 1);
|
||||
let result = match consumer {
|
||||
Some(consumer) => {
|
||||
// End the event stream as soon as merge finishes, even if a
|
||||
// drive is still winding down. Recovery bounds that join.
|
||||
let _stop_producers = rx.clone().drop_guard();
|
||||
run_tier_walk_with_consumer(&rx, &consumer, merge).await
|
||||
}
|
||||
None => merge.await,
|
||||
};
|
||||
if result.is_err() {
|
||||
rx.cancel();
|
||||
}
|
||||
@@ -2620,7 +2808,7 @@ fn spawn_listing_merge(
|
||||
async fn merge_entry_channels(
|
||||
rx: CancellationToken,
|
||||
in_channels: Vec<Receiver<MetaCacheEntry>>,
|
||||
out_channel: Sender<MetaCacheEntry>,
|
||||
out_channel: impl Into<ListingMergeOutput>,
|
||||
read_quorum: usize,
|
||||
) -> Result<()> {
|
||||
let completions = (0..in_channels.len()).map(|_| None).collect();
|
||||
@@ -2633,9 +2821,10 @@ async fn merge_entry_channels_with_cuts(
|
||||
rx: CancellationToken,
|
||||
in_channels: Vec<Receiver<MetaCacheEntry>>,
|
||||
mut completions: Vec<Option<MergeCompletion>>,
|
||||
out_channel: Sender<MetaCacheEntry>,
|
||||
out_channel: impl Into<ListingMergeOutput>,
|
||||
read_quorum: usize,
|
||||
) -> Result<Option<String>> {
|
||||
let out_channel = out_channel.into();
|
||||
if in_channels.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
@@ -2648,8 +2837,7 @@ async fn merge_entry_channels_with_cuts(
|
||||
tokio::select! {
|
||||
has_entry = in_channels[0].recv()=>{
|
||||
if let Some(entry) = has_entry{
|
||||
// warn!("merge_entry_channels entry {}", &entry.name);
|
||||
if !send_or_cancel(&rx, &out_channel, entry).await? {
|
||||
if out_channel.accepts_key(&entry.name)? && !out_channel.send(&rx, entry).await? {
|
||||
return Ok(None);
|
||||
}
|
||||
} else {
|
||||
@@ -2755,21 +2943,12 @@ async fn merge_entry_channels_with_cuts(
|
||||
}
|
||||
}
|
||||
|
||||
if !object_entries.is_empty()
|
||||
&& let Some(winner) = object_winner.as_mut()
|
||||
{
|
||||
merge_object_entry_versions(&mut winner.entry, object_entries.drain(..))?;
|
||||
}
|
||||
|
||||
if let Some(head) = object_winner.or(dir_winner)
|
||||
&& head.entry.name != last_emitted
|
||||
&& !out_channel
|
||||
.send_group(&rx, head.entry, object_entries.drain(..), &mut last_emitted)
|
||||
.await?
|
||||
{
|
||||
last_emitted.clear();
|
||||
last_emitted.push_str(&head.entry.name);
|
||||
let MergeHead { entry, .. } = *head;
|
||||
if !send_or_cancel(&rx, &out_channel, entry).await? {
|
||||
return Ok(None);
|
||||
}
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
for &idx in &refill {
|
||||
@@ -3608,6 +3787,137 @@ mod test {
|
||||
assert!(!parent.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_recovery_walk_isolates_named_decode_and_cross_pool_merge_failures() {
|
||||
use super::{ListingMergeOutput, TierRecoveryEntry};
|
||||
let logs = CapturedLogs::default();
|
||||
let subscriber = tracing_subscriber::fmt().with_writer(logs.clone()).finish();
|
||||
let _guard = tracing::subscriber::set_default(subscriber);
|
||||
let corrupt = |name: &str| MetaCacheEntry {
|
||||
name: name.to_owned(),
|
||||
metadata: vec![0xff],
|
||||
..Default::default()
|
||||
};
|
||||
let time = time::OffsetDateTime::from_unix_timestamp(1_705_312_300).expect("valid version timestamp");
|
||||
let mut inputs = Vec::new();
|
||||
for entries in [
|
||||
vec![
|
||||
corrupt("a-marker"),
|
||||
test_object_meta_entry("b-healthy"),
|
||||
corrupt("c-bad-buffer"),
|
||||
test_object_meta_entry_with_erasure_versions("d-conflict", &[(time, "first-etag", 2, 2)]),
|
||||
],
|
||||
vec![
|
||||
test_object_meta_entry_with_erasure_versions("d-conflict", &[(time, "different-etag", 2, 2)]),
|
||||
test_object_meta_entry("e-later"),
|
||||
],
|
||||
] {
|
||||
let (tx, rx) = mpsc::channel(entries.len());
|
||||
for entry in entries {
|
||||
tx.try_send(entry).expect("buffer each sorted set stream");
|
||||
}
|
||||
inputs.push(rx);
|
||||
}
|
||||
let (sender, mut events) = mpsc::channel(8);
|
||||
let output = ListingMergeOutput::TierRecovery {
|
||||
sender,
|
||||
bucket: "bucket".to_owned(),
|
||||
versioning: None,
|
||||
marker: Some("a-marker".to_owned()),
|
||||
};
|
||||
merge_entry_channels(CancellationToken::new(), inputs, output, 1)
|
||||
.await
|
||||
.expect("isolated metadata failures should allow the walk to finish");
|
||||
assert!(
|
||||
!logs.contents().contains("Listing version merge rejected metadata"),
|
||||
"recovery reports its unreadable keys in one page summary"
|
||||
);
|
||||
let mut actual = Vec::new();
|
||||
while let Some(event) = events.recv().await {
|
||||
actual.push(match event.expect("each event belongs to an enumerated key") {
|
||||
TierRecoveryEntry::Version(info) => (info.name, false),
|
||||
TierRecoveryEntry::Unreadable { object } => (object, true),
|
||||
});
|
||||
}
|
||||
assert_eq!(
|
||||
actual,
|
||||
[
|
||||
("b-healthy".to_owned(), false),
|
||||
("c-bad-buffer".to_owned(), true),
|
||||
("d-conflict".to_owned(), true),
|
||||
("e-later".to_owned(), false),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_recovery_walk_closes_event_stream_before_joining_drives() {
|
||||
let (source, input) = mpsc::channel(1);
|
||||
let (sender, mut events) = mpsc::channel(1);
|
||||
let output = super::ListingMergeOutput::TierRecovery {
|
||||
sender,
|
||||
bucket: "bucket".to_owned(),
|
||||
versioning: None,
|
||||
marker: None,
|
||||
};
|
||||
let cancel = CancellationToken::new();
|
||||
let merge = super::spawn_listing_merge(cancel.clone(), vec![input], output);
|
||||
// A later drive join may still be pending. Recovery must be able to
|
||||
// observe EOF and apply its shutdown timeout as soon as the merge ends.
|
||||
drop(source);
|
||||
timeout(Duration::from_secs(5), async {
|
||||
assert!(events.recv().await.is_none());
|
||||
cancel.cancelled().await;
|
||||
merge.await.expect("merge worker must not panic").expect("clean stream");
|
||||
})
|
||||
.await
|
||||
.expect("end of stream must release producers before their final join");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_recovery_walk_dropped_receiver_cancels_producers_and_preserves_error() {
|
||||
let (sender, receiver) = mpsc::channel(1);
|
||||
let cancel = CancellationToken::new();
|
||||
let (started_tx, started) = oneshot::channel();
|
||||
let producer = async {
|
||||
started_tx.send(()).expect("notify that the producer is running");
|
||||
cancel.cancelled().await;
|
||||
Err(StorageError::FaultyDisk)
|
||||
};
|
||||
let close_receiver = async {
|
||||
started.await.expect("wait for the active producer");
|
||||
drop(receiver);
|
||||
};
|
||||
let (result, ()) = timeout(Duration::from_secs(5), async {
|
||||
tokio::join!(super::run_tier_walk_with_consumer(&cancel, &sender, producer), close_receiver)
|
||||
})
|
||||
.await
|
||||
.expect("receiver loss must cancel the active walk");
|
||||
assert_eq!(result.expect_err("terminal disk errors must still propagate"), StorageError::FaultyDisk);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn tier_recovery_walk_propagates_unattributed_failures() {
|
||||
let (source, input) = mpsc::channel(1);
|
||||
let (complete, completion) = oneshot::channel();
|
||||
drop(source);
|
||||
complete
|
||||
.send(Err(StorageError::FileCorrupt))
|
||||
.expect("set should report its terminal error");
|
||||
let (sender, mut events) = mpsc::channel(1);
|
||||
let output = super::ListingMergeOutput::TierRecovery {
|
||||
sender,
|
||||
bucket: "bucket".to_owned(),
|
||||
versioning: None,
|
||||
marker: None,
|
||||
};
|
||||
let error = merge_entry_channels_with_cuts(CancellationToken::new(), vec![input], vec![Some(completion)], output, 1)
|
||||
.await
|
||||
.expect_err("corruption without a trusted object key must abort recovery");
|
||||
assert_eq!(error, StorageError::FileCorrupt);
|
||||
assert!(events.recv().await.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn walk_forwarder_abort_before_poll_cancels_producers() {
|
||||
let (entry_tx, entry_rx) = mpsc::channel(1);
|
||||
@@ -8749,7 +9059,7 @@ mod test {
|
||||
.insert("x-amz-meta-private".to_string(), "sensitive-value-must-not-be-logged".to_string());
|
||||
});
|
||||
assert_eq!(
|
||||
super::merge_object_entry_versions(&mut original, std::iter::once(conflicting))
|
||||
super::merge_object_entry_versions(&mut original, std::iter::once(conflicting), super::log_listing_merge_corruption)
|
||||
.expect_err("conflicting user metadata must still fail closed"),
|
||||
StorageError::FileCorrupt
|
||||
);
|
||||
|
||||
@@ -427,6 +427,7 @@ mod bucket;
|
||||
mod bucket_fence;
|
||||
pub(crate) use bucket::await_bucket_namespace_operation;
|
||||
pub use bucket_fence::BucketIncarnationFenceGuard;
|
||||
pub(crate) mod delete_recovery_inventory;
|
||||
pub(crate) mod delete_recovery_runtime;
|
||||
mod heal;
|
||||
pub use heal::{HealObjectAbsenceProof, HealObjectStorageResult};
|
||||
@@ -927,6 +928,15 @@ impl ECStore {
|
||||
if requests.len() > 32 {
|
||||
return observations;
|
||||
}
|
||||
let Some(owner) = self.scanner_usage_watch_owner_id() else {
|
||||
return requests
|
||||
.iter()
|
||||
.map(|request| ScannerUsageWatchObservation {
|
||||
bucket: request.bucket.clone(),
|
||||
known: None,
|
||||
})
|
||||
.collect();
|
||||
};
|
||||
let local_map = self.ctx.local_disk_map();
|
||||
let local_disks = local_map.read().await;
|
||||
let bindings_complete = !local_disks.is_empty()
|
||||
@@ -941,7 +951,7 @@ impl ECStore {
|
||||
if request.release {
|
||||
// Retirement needs the exact saved token, not current bucket
|
||||
// metadata. Deleted buckets must not retain scarce watch slots.
|
||||
self.ctx.usage_watches.observe(self.id, request);
|
||||
self.ctx.usage_watches.observe(owner, request);
|
||||
observations.push(ScannerUsageWatchObservation {
|
||||
bucket: request.bucket.clone(),
|
||||
known: None,
|
||||
@@ -951,7 +961,7 @@ impl ECStore {
|
||||
let current = self.bucket_incarnation_id_from_disk(&request.bucket).await;
|
||||
let known = match current {
|
||||
Ok(incarnation) if bindings_complete && incarnation == request.bucket_incarnation && !incarnation.is_nil() => {
|
||||
self.ctx.usage_watches.observe(self.id, request)
|
||||
self.ctx.usage_watches.observe(owner, request)
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
@@ -973,11 +983,49 @@ impl ECStore {
|
||||
self.ctx.usage_watches.session_id()
|
||||
}
|
||||
|
||||
/// Producer identity for this node's bucket mutation watches. Its session changes on restart.
|
||||
pub fn scanner_usage_watch_owner_id(&self) -> Option<Uuid> {
|
||||
self.scanner_usage_watch_owner_for_route(None)
|
||||
}
|
||||
|
||||
/// Resolve an authenticated notification peer against this instance's storage topology.
|
||||
pub fn scanner_usage_watch_peer_owner_id(&self, grid_host: &str) -> Option<Uuid> {
|
||||
self.scanner_usage_watch_owner_for_route(Some(grid_host))
|
||||
}
|
||||
|
||||
fn scanner_usage_watch_owner_for_route(&self, grid_host: Option<&str>) -> Option<Uuid> {
|
||||
let slot = &self.ctx.usage_watches.owners;
|
||||
let owners = if let Some(owners) = slot.get() {
|
||||
owners
|
||||
} else if let Some(endpoints) = self.instance_endpoints() {
|
||||
// InstanceContext installs this topology once. Publish only a successfully validated table.
|
||||
let owners = crate::runtime::usage_watch::UsageWatchOwners::from_topology(self.id, &endpoints)?;
|
||||
slot.get_or_init(|| owners)
|
||||
} else {
|
||||
// Direct file-backed stores can open before an instance endpoint registry exists.
|
||||
let all_local_paths = !self.pools.is_empty()
|
||||
&& self.pools.iter().all(|pool| {
|
||||
let endpoints = pool.endpoints.endpoints.as_ref();
|
||||
!endpoints.is_empty() && endpoints.iter().all(|ep| ep.is_local && ep.get_type() == EndpointType::Path)
|
||||
});
|
||||
return (grid_host.is_none() && !self.id.is_nil() && all_local_paths).then_some(self.id);
|
||||
};
|
||||
if owners.deployment != self.id {
|
||||
return None;
|
||||
}
|
||||
match grid_host {
|
||||
Some(host) => owners.peers.get(host).copied(),
|
||||
None => Some(owners.local),
|
||||
}
|
||||
}
|
||||
|
||||
/// Revoke an exact local token without awaiting storage or peer I/O. A
|
||||
/// failed keepalive must not let a later successful query revive its proof.
|
||||
pub fn scanner_release_usage_watch(&self, request: &crate::runtime::usage_watch::ScannerUsageWatchRequest) {
|
||||
if request.release {
|
||||
self.ctx.usage_watches.observe(self.id, request);
|
||||
if request.release
|
||||
&& let Some(owner) = self.scanner_usage_watch_owner_id()
|
||||
{
|
||||
self.ctx.usage_watches.observe(owner, request);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -24,6 +24,130 @@ async fn known(store: &ECStore, request: &ScannerUsageWatchRequest) -> ScannerUs
|
||||
.expect("all counted physical tails settle")
|
||||
}
|
||||
|
||||
fn watch_node_topology(local_nodes: &[usize]) -> EndpointServerPools {
|
||||
let endpoints = (0..4)
|
||||
.map(|slot| {
|
||||
let node = slot % 2;
|
||||
let url = format!("http://node-{node}:9000/data{slot}");
|
||||
let mut endpoint = Endpoint::try_from(url.as_str()).expect("node endpoint");
|
||||
endpoint.is_local = local_nodes.contains(&node);
|
||||
endpoint.set_pool_index(0);
|
||||
endpoint.set_set_index(0);
|
||||
endpoint.set_disk_index(slot);
|
||||
endpoint
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
vec![crate::layout::endpoints::PoolEndpoints {
|
||||
legacy: false,
|
||||
set_count: 1,
|
||||
drives_per_set: endpoints.len(),
|
||||
endpoints: endpoints.into(),
|
||||
cmd_line: String::new(),
|
||||
platform: String::new(),
|
||||
}]
|
||||
.into()
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn shared_deployment_issues_distinct_usage_watch_owners_through_both_node_views() {
|
||||
// Open the same formatted storage with two independent producer registries.
|
||||
// Each sees the same deployment and bucket through its own node's routing view.
|
||||
let (_dirs, first) = isolated_store_over_temp_disks().await;
|
||||
let disks = EndpointServerPools::from(vec![first.pools[0].endpoints.clone()]);
|
||||
let second_ctx = Arc::new(InstanceContext::new());
|
||||
init_local_disks_with_instance_ctx(&second_ctx, disks.clone())
|
||||
.await
|
||||
.expect("second disk registry");
|
||||
let second = ECStore::new_with_instance_ctx(
|
||||
"127.0.0.1:0".parse().expect("local test address"),
|
||||
disks,
|
||||
CancellationToken::new(),
|
||||
second_ctx,
|
||||
)
|
||||
.await
|
||||
.expect("second storage instance");
|
||||
assert_eq!(first.id, second.id, "both producers read the same deployment from disk");
|
||||
assert_eq!(
|
||||
first.scanner_usage_watch_owner_id(),
|
||||
Some(first.id),
|
||||
"a direct file-backed store keeps its owner"
|
||||
);
|
||||
assert_eq!(first.scanner_usage_watch_peer_owner_id("http://node-1:9000"), None);
|
||||
first.ctx.set_endpoints(watch_node_topology(&[0]));
|
||||
second.ctx.set_endpoints(watch_node_topology(&[1]));
|
||||
init_bucket_metadata_sys(first.clone(), Vec::new()).await;
|
||||
init_bucket_metadata_sys(second.clone(), Vec::new()).await;
|
||||
let bucket = "shared-watch-deployment";
|
||||
first
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create shared bucket");
|
||||
let incarnation = first
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("first incarnation");
|
||||
assert_eq!(
|
||||
second
|
||||
.bucket_incarnation_id_from_disk(bucket)
|
||||
.await
|
||||
.expect("second incarnation"),
|
||||
incarnation
|
||||
);
|
||||
let mut request = ScannerUsageWatchRequest {
|
||||
bucket: bucket.into(),
|
||||
bucket_incarnation: incarnation,
|
||||
watch_id: None,
|
||||
session_id: None,
|
||||
release: false,
|
||||
};
|
||||
let a = known(&first, &request).await;
|
||||
let b = known(&second, &request).await;
|
||||
assert_ne!(a.owner_id, b.owner_id, "watch owners must distinguish nodes in one deployment");
|
||||
assert_eq!(first.scanner_usage_watch_owner_id(), Some(a.owner_id));
|
||||
assert_eq!(second.scanner_usage_watch_owner_id(), Some(b.owner_id));
|
||||
assert_eq!(first.scanner_usage_watch_peer_owner_id("http://node-1:9000"), Some(b.owner_id));
|
||||
assert_eq!(second.scanner_usage_watch_peer_owner_id("http://node-0:9000"), Some(a.owner_id));
|
||||
assert_eq!(first.scanner_usage_watch_peer_owner_id("http://node-0:9000"), None);
|
||||
assert_eq!(first.scanner_usage_watch_peer_owner_id("http://unknown:9000"), None);
|
||||
assert_ne!(a.session_id, b.session_id);
|
||||
request.watch_id = Some(a.watch_id);
|
||||
request.session_id = Some(a.session_id);
|
||||
request.release = true;
|
||||
second.scanner_release_usage_watch(&request);
|
||||
assert!(
|
||||
known(
|
||||
&second,
|
||||
&ScannerUsageWatchRequest {
|
||||
watch_id: Some(b.watch_id),
|
||||
session_id: Some(b.session_id),
|
||||
release: false,
|
||||
..request.clone()
|
||||
}
|
||||
)
|
||||
.await
|
||||
.unchanged_since(&b)
|
||||
);
|
||||
first.scanner_release_usage_watch(&request);
|
||||
request.release = false;
|
||||
assert!(
|
||||
first.scanner_usage_watch(&[request]).await[0].known.is_none(),
|
||||
"released token cannot be revived"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn usage_watch_routes_require_one_unambiguous_local_node() {
|
||||
use crate::runtime::usage_watch::UsageWatchOwners;
|
||||
let deployment = Uuid::new_v4();
|
||||
for local_nodes in [&[][..], &[0, 1][..]] {
|
||||
assert!(UsageWatchOwners::from_topology(deployment, &watch_node_topology(local_nodes)).is_none());
|
||||
}
|
||||
assert!(UsageWatchOwners::from_topology(Uuid::nil(), &watch_node_topology(&[0])).is_none());
|
||||
let mut incomplete = watch_node_topology(&[0]);
|
||||
incomplete.0[0].drives_per_set += 1;
|
||||
assert!(UsageWatchOwners::from_topology(deployment, &incomplete).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
async fn usage_watch_api_tracks_real_put_delete_and_bucket_recreation() {
|
||||
let (_dirs, store) = isolated_store_over_temp_disks().await;
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
|
||||
mod storage_api;
|
||||
use storage_api::local_drive_retirement::{Disk, DiskAPI, DiskOption, Endpoint, new_disk};
|
||||
|
||||
#[tokio::test]
|
||||
async fn local_wrapper_retirement_is_shared_and_independent_of_monitoring() {
|
||||
let dir = tempfile::tempdir().expect("retirement fixture directory");
|
||||
let endpoint = Endpoint::try_from(dir.path().to_str().expect("UTF-8 fixture path")).expect("local endpoint");
|
||||
let disk = new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("local disk");
|
||||
let Disk::Local(wrapper) = disk.as_ref() else { panic!("fixture must use a local disk") };
|
||||
let cloned = wrapper.clone();
|
||||
let observer = disk.local_retirement_token().expect("local retirement observer");
|
||||
let cancelled_observer = disk.local_retirement_token().expect("another observer");
|
||||
cancelled_observer.cancel();
|
||||
assert!(!observer.is_cancelled(), "observers cannot retire the wrapper");
|
||||
wrapper.stop_monitoring().await;
|
||||
assert!(!observer.is_cancelled(), "monitoring policy is not retirement");
|
||||
cloned.close().await.expect("close shared wrapper");
|
||||
assert!(observer.is_cancelled(), "clones share retirement");
|
||||
assert!(
|
||||
disk.local_retirement_token().expect("late observer").is_cancelled(),
|
||||
"late observers see retirement"
|
||||
);
|
||||
disk.close().await.expect("retirement is idempotent");
|
||||
}
|
||||
@@ -94,3 +94,8 @@ pub(crate) mod pinned_erasure_fixtures {
|
||||
storage::init_local_disks_with_instance_ctx,
|
||||
};
|
||||
}
|
||||
|
||||
pub(crate) mod local_drive_retirement {
|
||||
pub(crate) use super::{DiskAPI, DiskOption, Endpoint, new_disk};
|
||||
pub(crate) use rustfs_ecstore::api::disk::Disk;
|
||||
}
|
||||
|
||||
@@ -14,12 +14,9 @@
|
||||
|
||||
use super::{
|
||||
DiskError, DiskStore, HEALING_MARKER_PATH, RUSTFS_META_BUCKET,
|
||||
replacement_readiness::{auto_replacement_target_identities, replacement_target_disk},
|
||||
replacement_readiness::{ReplacementTargets, local_replacement_disks, probe_replacement_mounts},
|
||||
resume::ReplacementTargetIdentity,
|
||||
storage_api::{
|
||||
EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease,
|
||||
ecstore_local_disk_map_read,
|
||||
},
|
||||
storage_api::{EcstoreConditionalFileUpdate, EcstoreDiskAPI, EcstoreDiskBytes, EcstoreReplacementExecutionLease},
|
||||
};
|
||||
use crate::{Error, Result};
|
||||
use std::{future::Future, sync::Arc};
|
||||
@@ -35,33 +32,27 @@ pub struct ReplacementExecution {
|
||||
|
||||
impl ReplacementExecution {
|
||||
pub(crate) async fn acquire(targets: &[String]) -> Result<Arc<Self>> {
|
||||
let identities = auto_replacement_target_identities(targets)
|
||||
let targets = ReplacementTargets::open(targets)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
|
||||
let identities = targets
|
||||
.identities()
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("target mount admission failed".to_string()))?;
|
||||
let local_disks = ecstore_local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| EcstoreDiskAPI::is_local(disk.as_ref()))
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
let mut disks = Vec::with_capacity(identities.len());
|
||||
let mut leases = Vec::with_capacity(identities.len());
|
||||
// Identities are sorted by endpoint, the same order on every opener.
|
||||
for identity in &identities {
|
||||
let disk = replacement_target_disk(&identity.endpoint, &local_disks)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is unavailable".to_string()))?;
|
||||
// Take leases in endpoint order. Revalidate those same handles against
|
||||
// the current inventory so a reopened disk cannot supply their proof.
|
||||
for disk in &targets.disks {
|
||||
leases.push(disk.acquire_replacement_execution_lease().await?);
|
||||
disks.push(disk);
|
||||
}
|
||||
if auto_replacement_target_identities(targets).await.as_ref() != Some(&identities) {
|
||||
let local_disks = local_replacement_disks().await;
|
||||
if probe_replacement_mounts(&targets.disks, &local_disks).await.as_ref() != Some(&identities) {
|
||||
return Err(Error::ReplacementTargetNotReady(
|
||||
"target changed while acquiring execution leases".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(Arc::new(Self {
|
||||
disks,
|
||||
disks: targets.disks,
|
||||
identities,
|
||||
_leases: leases,
|
||||
}))
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{fs, path::Path};
|
||||
use std::collections::HashSet;
|
||||
|
||||
use super::{
|
||||
DiskOption, DiskStore, Endpoint, HealDiskExt as _, local_disk_map_read, new_disk, resume::ReplacementTargetIdentity,
|
||||
@@ -24,12 +24,15 @@ use super::{
|
||||
/// Directory-backed deployments already declare, through
|
||||
/// `RUSTFS_UNSAFE_BYPASS_DISK_CHECK`, that their endpoints are plain
|
||||
/// directories sharing a device with the host root. Those endpoints can never
|
||||
/// satisfy [`auto_replacement_target_identity`], so without this fallback a
|
||||
/// pass the independent-mount check, so without this fallback a
|
||||
/// runtime-wiped or replaced directory disk would stay deferred forever. The
|
||||
/// admission check itself is never bypassed; the fallback only routes the heal
|
||||
/// through the ordinary format path that formats every unformatted disk in the
|
||||
/// set, which is exactly what the pre-admission `heal_disk` path did.
|
||||
pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
if cfg!(target_os = "linux") && std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS).is_some() {
|
||||
return false;
|
||||
}
|
||||
rustfs_utils::get_env_bool_with_aliases(
|
||||
rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK,
|
||||
&[rustfs_config::ENV_MINIO_CI],
|
||||
@@ -38,53 +41,66 @@ pub(crate) fn directory_backed_replacement_fallback_enabled() -> bool {
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_ready(disk: &DiskStore, local_disks: &[DiskStore]) -> bool {
|
||||
auto_replacement_target_identity(disk, local_disks).await.is_some()
|
||||
probe_replacement_mounts(std::slice::from_ref(disk), local_disks)
|
||||
.await
|
||||
.is_some()
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_identity(
|
||||
disk: &DiskStore,
|
||||
pub(super) async fn probe_replacement_mounts(
|
||||
disks: &[DiskStore],
|
||||
local_disks: &[DiskStore],
|
||||
) -> Option<ReplacementTargetIdentity> {
|
||||
let lease_root = disk.replacement_mount_lease_root()?;
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let sibling_lease_roots = local_disks
|
||||
.iter()
|
||||
.filter(|sibling| sibling.endpoint().is_local && sibling.endpoint().to_string() != endpoint)
|
||||
.map(|sibling| sibling.replacement_mount_lease_root())
|
||||
.collect::<Option<Vec<_>>>()?;
|
||||
) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
if disks.is_empty() {
|
||||
return Some(Vec::new());
|
||||
}
|
||||
let disks = disks.to_vec();
|
||||
let local_disks = local_disks.to_vec();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let canonical_path = fs::canonicalize(&lease_root).ok()?;
|
||||
let metadata = fs::metadata(&lease_root).ok()?;
|
||||
let Ok(target_device_ids) = rustfs_utils::os::get_physical_device_ids(lease_root.to_string_lossy().as_ref()) else {
|
||||
return None;
|
||||
};
|
||||
let Ok(root_device_ids) = rustfs_utils::os::get_physical_device_ids("/") else {
|
||||
return None;
|
||||
};
|
||||
if target_device_ids.is_empty()
|
||||
|| root_device_ids.is_empty()
|
||||
|| target_device_ids.iter().any(|target| root_device_ids.contains(target))
|
||||
|| !rustfs_utils::os::is_mount_point(&canonical_path).unwrap_or(false)
|
||||
{
|
||||
return None;
|
||||
let mount_roots = std::env::var_os(rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS)
|
||||
.as_ref()
|
||||
.into_iter()
|
||||
.flat_map(std::env::split_paths)
|
||||
.collect::<Vec<_>>();
|
||||
let mut identities = Vec::with_capacity(disks.len());
|
||||
for disk in &disks {
|
||||
let mount = disk.replacement_mount(&mount_roots)?;
|
||||
if mount.physical_device_ids.is_empty() {
|
||||
return None;
|
||||
}
|
||||
identities.push(ReplacementTargetIdentity {
|
||||
endpoint: disk.endpoint().to_string(),
|
||||
canonical_path: mount.canonical_path.into_os_string().into_string().ok()?,
|
||||
physical_device_ids: mount.physical_device_ids,
|
||||
filesystem_identity: mount.filesystem_identity,
|
||||
});
|
||||
}
|
||||
|
||||
if sibling_lease_roots.iter().any(|sibling_lease_root| {
|
||||
rustfs_utils::os::get_physical_device_ids(sibling_lease_root.to_string_lossy().as_ref())
|
||||
.map(|ids| ids.iter().any(|id| target_device_ids.contains(id)))
|
||||
.unwrap_or(true)
|
||||
}) {
|
||||
let mut target_devices = HashSet::new();
|
||||
for target in &identities {
|
||||
if target
|
||||
.physical_device_ids
|
||||
.iter()
|
||||
.any(|device| !target_devices.insert(device.as_str()))
|
||||
{
|
||||
return None;
|
||||
}
|
||||
}
|
||||
let root_devices = rustfs_utils::os::get_physical_device_ids("/").ok()?;
|
||||
if root_devices.is_empty() || root_devices.iter().any(|device| target_devices.contains(device.as_str())) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let filesystem_identity = filesystem_identity(&metadata, &canonical_path)?;
|
||||
|
||||
Some(ReplacementTargetIdentity {
|
||||
endpoint,
|
||||
canonical_path: canonical_path.to_string_lossy().into_owned(),
|
||||
physical_device_ids: target_device_ids,
|
||||
filesystem_identity,
|
||||
})
|
||||
let targets: HashSet<_> = identities.iter().map(|identity| identity.endpoint.as_str()).collect();
|
||||
for sibling in &local_disks {
|
||||
let endpoint = sibling.endpoint();
|
||||
if endpoint.is_local && !targets.contains(endpoint.to_string().as_str()) {
|
||||
let root = sibling.replacement_mount_lease_root()?;
|
||||
let devices = rustfs_utils::os::get_physical_device_ids(root.to_str()?).ok()?;
|
||||
if devices.iter().any(|device| target_devices.contains(device.as_str())) {
|
||||
return None;
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(identities)
|
||||
})
|
||||
.await
|
||||
.ok()
|
||||
@@ -106,72 +122,76 @@ fn local_replacement_endpoint(target: &str, local_grid_hosts: &[String]) -> Opti
|
||||
Some(endpoint)
|
||||
}
|
||||
|
||||
pub(super) async fn replacement_target_disk(target: &str, local_disks: &[DiskStore]) -> Option<DiskStore> {
|
||||
if let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == target) {
|
||||
return Some(disk.clone());
|
||||
}
|
||||
|
||||
let local_grid_hosts = local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>();
|
||||
let endpoint = local_replacement_endpoint(target, &local_grid_hosts)?;
|
||||
|
||||
new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()
|
||||
}
|
||||
|
||||
pub(crate) async fn auto_replacement_target_identities(targets: &[String]) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
let local_disk_map = local_disk_map_read().await;
|
||||
let local_disks = local_disk_map
|
||||
pub(super) async fn local_replacement_disks() -> Vec<DiskStore> {
|
||||
local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
drop(local_disk_map);
|
||||
.collect()
|
||||
}
|
||||
|
||||
let mut identities = Vec::with_capacity(targets.len());
|
||||
for target in targets {
|
||||
let disk = replacement_target_disk(target, &local_disks).await?;
|
||||
identities.push(auto_replacement_target_identity(&disk, &local_disks).await?);
|
||||
pub(super) struct ReplacementTargets {
|
||||
pub(super) disks: Vec<DiskStore>,
|
||||
local_disks: Vec<DiskStore>,
|
||||
}
|
||||
|
||||
impl ReplacementTargets {
|
||||
pub(super) async fn open(targets: &[String]) -> Option<Self> {
|
||||
let mut ordered: Vec<_> = targets.iter().map(String::as_str).collect();
|
||||
ordered.sort_unstable();
|
||||
if ordered.windows(2).any(|pair| pair[0] == pair[1]) {
|
||||
return None;
|
||||
}
|
||||
|
||||
let registry = local_disk_map_read().await;
|
||||
let requested: Vec<_> = ordered
|
||||
.into_iter()
|
||||
.map(|target| {
|
||||
let disk = registry
|
||||
.get(target)
|
||||
.and_then(Option::as_ref)
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned();
|
||||
(target, disk)
|
||||
})
|
||||
.collect();
|
||||
let local_disks = registry
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
drop(registry);
|
||||
let mut disks = Vec::with_capacity(requested.len());
|
||||
let mut hosts = None;
|
||||
for (target, disk) in requested {
|
||||
let disk = match disk {
|
||||
Some(disk) => disk,
|
||||
None => {
|
||||
let local_hosts = hosts
|
||||
.get_or_insert_with(|| local_disks.iter().map(|disk| disk.endpoint().grid_host()).collect::<Vec<_>>());
|
||||
let endpoint = local_replacement_endpoint(target, local_hosts)?;
|
||||
new_disk(
|
||||
&endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.ok()?
|
||||
}
|
||||
};
|
||||
disks.push(disk);
|
||||
}
|
||||
Some(Self { disks, local_disks })
|
||||
}
|
||||
identities.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
identities.dedup_by(|left, right| left.endpoint == right.endpoint);
|
||||
(identities.len() == targets.len()).then_some(identities)
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
fn filesystem_identity(metadata: &fs::Metadata, canonical_path: &Path) -> Option<String> {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
let escaped_path = canonical_path.to_string_lossy().replace(' ', "\\040");
|
||||
let mountinfo = fs::read_to_string("/proc/self/mountinfo").ok()?;
|
||||
let mount_id = mountinfo.lines().find_map(|line| {
|
||||
let mut fields = line.split_whitespace();
|
||||
let mount_id = fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
fields.next()?;
|
||||
(fields.next()? == escaped_path).then_some(mount_id)
|
||||
})?;
|
||||
Some(format!("{mount_id}:{}:{}", metadata.dev(), metadata.ino()))
|
||||
}
|
||||
|
||||
#[cfg(all(unix, not(target_os = "linux")))]
|
||||
fn filesystem_identity(metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
|
||||
use std::os::unix::fs::MetadataExt as _;
|
||||
|
||||
Some(format!("{}:{}", metadata.dev(), metadata.ino()))
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
fn filesystem_identity(_metadata: &fs::Metadata, _canonical_path: &Path) -> Option<String> {
|
||||
None
|
||||
pub(super) async fn identities(&self) -> Option<Vec<ReplacementTargetIdentity>> {
|
||||
probe_replacement_mounts(&self.disks, &self.local_disks).await
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -269,8 +289,8 @@ mod tests {
|
||||
mod linux_privileged_tests {
|
||||
use super::*;
|
||||
use std::error::Error;
|
||||
use std::path::Path;
|
||||
use std::process::Command;
|
||||
use std::{fs, path::Path};
|
||||
|
||||
const ENABLE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS";
|
||||
const NAMESPACE_ENV: &str = "RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS_IN_NAMESPACE";
|
||||
@@ -396,39 +416,86 @@ mod tests {
|
||||
#[ignore = "requires Linux root/CAP_SYS_ADMIN and RUSTFS_PRIVILEGED_MOUNT_READINESS_TESTS=1"]
|
||||
fn auto_replacement_readiness_accepts_an_independent_mount() -> Result<(), Box<dyn Error + Send + Sync>> {
|
||||
run_privileged_mount_test(|mut mounts| async move {
|
||||
let temp = TempDir::new().expect("temporary replacement roots should be created");
|
||||
let target = temp.path().join("target");
|
||||
let temp = TempDir::new()?;
|
||||
let parent = temp.path().join("parent");
|
||||
let target = parent.join("replacement disk");
|
||||
let data = target.join("data");
|
||||
let sibling = temp.path().join("sibling");
|
||||
std::fs::create_dir(&target).expect("target mountpoint should be created");
|
||||
std::fs::create_dir(&sibling).expect("sibling mountpoint should be created");
|
||||
fs::create_dir(&parent)?;
|
||||
mounts.mount_tmpfs(&parent, "rustfs-readiness-parent")?;
|
||||
// Leave a directory at the same path below the target mount.
|
||||
fs::create_dir_all(&data)?;
|
||||
mounts.mount_tmpfs(&target, "rustfs-readiness-target")?;
|
||||
fs::create_dir(&data)?;
|
||||
fs::create_dir(&sibling)?;
|
||||
mounts.mount_tmpfs(&sibling, "rustfs-readiness-sibling")?;
|
||||
|
||||
let target_endpoint = Endpoint::try_from(target.to_string_lossy().as_ref())?;
|
||||
let sibling_endpoint = Endpoint::try_from(sibling.to_string_lossy().as_ref())?;
|
||||
let target_disk = new_disk(
|
||||
&target_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let sibling_disk = new_disk(
|
||||
&sibling_endpoint,
|
||||
&DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
let options = DiskOption {
|
||||
cleanup: false,
|
||||
health_check: false,
|
||||
};
|
||||
let target_disk = new_disk(&target_endpoint, &options).await?;
|
||||
let sibling_disk = new_disk(&sibling_endpoint, &options).await?;
|
||||
|
||||
let identity = auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await;
|
||||
assert!(
|
||||
identity.is_some(),
|
||||
auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
|
||||
"a separately mounted replacement target with no sibling device overlap must be admitted"
|
||||
);
|
||||
Ok(())
|
||||
let disks = [target_disk, sibling_disk];
|
||||
let mut identities = probe_replacement_mounts(&disks, &disks)
|
||||
.await
|
||||
.expect("independently mounted replacement targets should be admitted together");
|
||||
assert_eq!(identities[0].endpoint, disks[0].endpoint().to_string());
|
||||
assert_eq!(identities[1].endpoint, disks[1].endpoint().to_string());
|
||||
identities.sort_unstable_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
let endpoints = [identities[1].endpoint.clone(), identities[0].endpoint.clone()];
|
||||
let execution = crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints).await?;
|
||||
assert_eq!(execution.identities(), identities);
|
||||
assert!(
|
||||
crate::heal::replacement_execution::ReplacementExecution::acquire(&endpoints)
|
||||
.await
|
||||
.is_err(),
|
||||
"a second executor must not take leases on the same targets"
|
||||
);
|
||||
drop(execution);
|
||||
|
||||
let endpoint = Endpoint::try_from(path_to_string(&data, "data directory")?.as_str())?;
|
||||
let disk = new_disk(&endpoint, &options).await?;
|
||||
let key = rustfs_config::ENV_HEAL_REPLACEMENT_MOUNT_ROOTS;
|
||||
temp_env::async_with_vars([(key, None::<&str>)], async {
|
||||
assert!(!auto_replacement_target_ready(&disk, &[]).await);
|
||||
})
|
||||
.await;
|
||||
|
||||
let roots = std::env::join_paths([&parent, &target])?;
|
||||
temp_env::async_with_vars(
|
||||
[
|
||||
(key, Some(roots.to_str().ok_or("mount roots are not UTF-8")?)),
|
||||
(rustfs_config::ENV_UNSAFE_BYPASS_DISK_CHECK, Some("true")),
|
||||
],
|
||||
async {
|
||||
assert!(!directory_backed_replacement_fallback_enabled());
|
||||
let identities = probe_replacement_mounts(std::slice::from_ref(&disk), &[])
|
||||
.await
|
||||
.expect("data directory on its configured independent mount should be admitted");
|
||||
assert_eq!(Path::new(&identities[0].canonical_path), data.canonicalize()?);
|
||||
let reopened = new_disk(&endpoint, &options).await?;
|
||||
assert_eq!(probe_replacement_mounts(&[reopened], &[]).await, Some(identities));
|
||||
|
||||
run_command("umount", &["-l", &path_to_string(&target, "replacement mount")?])?;
|
||||
assert!(!auto_replacement_target_ready(&disk, &[]).await);
|
||||
|
||||
let exposed_disk = new_disk(&endpoint, &options).await?;
|
||||
assert!(
|
||||
!auto_replacement_target_ready(&exposed_disk, &[]).await,
|
||||
"a new handle on the exposed parent filesystem must also be rejected"
|
||||
);
|
||||
Ok(())
|
||||
},
|
||||
)
|
||||
.await
|
||||
})
|
||||
}
|
||||
|
||||
@@ -467,10 +534,20 @@ mod tests {
|
||||
.await?;
|
||||
|
||||
assert!(
|
||||
auto_replacement_target_identity(&target_disk, &[target_disk.clone(), sibling_disk.clone()])
|
||||
!auto_replacement_target_ready(&target_disk, &[target_disk.clone(), sibling_disk.clone()]).await,
|
||||
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
|
||||
);
|
||||
assert!(
|
||||
probe_replacement_mounts(&[target_disk.clone(), sibling_disk], &[])
|
||||
.await
|
||||
.is_none(),
|
||||
"replacement readiness must reject a target sharing its physical device with a sibling endpoint"
|
||||
"a batch must reject overlapping targets even before they are registered"
|
||||
);
|
||||
assert!(auto_replacement_target_ready(&target_disk, &[]).await);
|
||||
mounts.mount_bind(&source, &target)?;
|
||||
assert!(
|
||||
!auto_replacement_target_ready(&target_disk, &[]).await,
|
||||
"rebinding the same inode must invalidate the held mount identity"
|
||||
);
|
||||
Ok(())
|
||||
})
|
||||
|
||||
@@ -519,8 +519,14 @@ pub(crate) fn replacement_target_identities_match(
|
||||
expected: &[ReplacementTargetIdentity],
|
||||
actual: &[ReplacementTargetIdentity],
|
||||
) -> bool {
|
||||
let mut expected = expected.to_vec();
|
||||
let mut actual = actual.to_vec();
|
||||
if expected == actual {
|
||||
return true;
|
||||
}
|
||||
if expected.len() != actual.len() {
|
||||
return false;
|
||||
}
|
||||
let mut expected: Vec<_> = expected.iter().collect();
|
||||
let mut actual: Vec<_> = actual.iter().collect();
|
||||
expected.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
actual.sort_by(|left, right| left.endpoint.cmp(&right.endpoint));
|
||||
expected == actual
|
||||
|
||||
@@ -38,7 +38,7 @@ use super::storage_api::storage::{
|
||||
ObjectIO as _, ObjectOperations as _, StorageAdminApi,
|
||||
};
|
||||
use super::{
|
||||
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError, local_disk_map_read,
|
||||
DiskError, DiskStore, ECStore, HealDiskExt as _, StorageError,
|
||||
resume::{ReplacementRecoveryCandidate, ReplacementTargetIdentity, merge_replacement_recovery_candidate},
|
||||
};
|
||||
pub use super::{HealObjectInfo, HealObjectOptions, HealPutObjReader};
|
||||
@@ -2457,9 +2457,13 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
}
|
||||
|
||||
async fn replacement_target_identities(&self, targets: &[String]) -> Result<Vec<ReplacementTargetIdentity>> {
|
||||
super::replacement_readiness::auto_replacement_target_identities(targets)
|
||||
let error = || Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string());
|
||||
super::replacement_readiness::ReplacementTargets::open(targets)
|
||||
.await
|
||||
.ok_or_else(|| Error::ReplacementTargetNotReady("replacement target is not a stable mounted disk".to_string()))
|
||||
.ok_or_else(error)?
|
||||
.identities()
|
||||
.await
|
||||
.ok_or_else(error)
|
||||
}
|
||||
|
||||
async fn replacement_targets_ready_for_retry(
|
||||
@@ -2468,25 +2472,14 @@ impl HealStorageAPI for ECStoreHealStorage {
|
||||
targets: &[String],
|
||||
expected_identities: &[ReplacementTargetIdentity],
|
||||
) -> Result<bool> {
|
||||
let identities = match self.replacement_target_identities(targets).await {
|
||||
Ok(identities) => identities,
|
||||
Err(_) => return Ok(false),
|
||||
let Some(targets) = super::replacement_readiness::ReplacementTargets::open(targets).await else {
|
||||
return Ok(false);
|
||||
};
|
||||
if identities != expected_identities {
|
||||
if targets.identities().await.as_deref() != Some(expected_identities) {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let local_disks = local_disk_map_read()
|
||||
.await
|
||||
.values()
|
||||
.flatten()
|
||||
.filter(|disk| disk.endpoint().is_local)
|
||||
.cloned()
|
||||
.collect::<Vec<_>>();
|
||||
for target in targets {
|
||||
let Some(disk) = local_disks.iter().find(|disk| disk.endpoint().to_string() == *target) else {
|
||||
return Ok(false);
|
||||
};
|
||||
for disk in targets.disks {
|
||||
let endpoint = disk.endpoint();
|
||||
if crate::heal::utils::format_set_disk_id_from_i32(endpoint.pool_idx, endpoint.set_idx).as_deref()
|
||||
!= Some(set_disk_id)
|
||||
|
||||
@@ -228,10 +228,10 @@ impl HealTask {
|
||||
// first target format write. A task retry keeps this id; a newly
|
||||
// admitted blank replacement gets a fresh id and cannot reuse cursor
|
||||
// progress from an older disk at the same endpoint.
|
||||
let replacement_resume = if is_auto_replacement {
|
||||
let identities = self
|
||||
.await_with_control(self.storage.replacement_target_identities(&self.heal_endpoints))
|
||||
.await?;
|
||||
let replacement_resume = if let Some(execution) = &replacement_execution {
|
||||
self.check_control_flags().await?;
|
||||
// Bind durable intent to the mounts whose execution leases we hold.
|
||||
let identities = execution.identities().to_vec();
|
||||
let disk = replacement_resume_disk.clone().ok_or_else(|| Error::TaskExecutionFailed {
|
||||
message: format!("Replacement resume disk is missing for automatic heal {set_disk_id}"),
|
||||
})?;
|
||||
|
||||
@@ -1258,6 +1258,40 @@ async fn recovered_replacement_never_uses_a_fresh_resume_disk() {
|
||||
assert!(!*storage.listed.lock().unwrap(), "an unavailable durable anchor must not list buckets");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_does_not_format_a_mount_outside_its_execution_lease() {
|
||||
let temp = TempDir::new().expect("resume disk directory");
|
||||
let disk = make_resume_disk(&temp).await;
|
||||
let leased = replacement_identity("replacement-a", "device-a", "mount-a");
|
||||
let remounted = replacement_identity("replacement-a", "device-a", "mount-b");
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![remounted.clone()], vec![remounted]])),
|
||||
replacement_execution_fixture: Mutex::new(Some(crate::heal::storage::ReplacementExecution::for_test(
|
||||
Vec::new(),
|
||||
vec![leased.clone()],
|
||||
))),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
..Default::default()
|
||||
});
|
||||
let mut request = directory_backed_replacement_request();
|
||||
request.heal_endpoints = vec!["replacement-a".to_string()];
|
||||
let task = HealTask::from_request(request, storage.clone());
|
||||
|
||||
let error = task.execute().await.expect_err("a remount must be rejected before format");
|
||||
assert!(error.to_string().contains("changed during format"), "{error}");
|
||||
assert!(storage.replacement_format_calls.lock().unwrap().is_empty());
|
||||
|
||||
let state = ResumeManager::load_replacement_intent(disk, &task.id)
|
||||
.await
|
||||
.expect("durable intent should identify the leased target")
|
||||
.get_state()
|
||||
.await;
|
||||
assert_eq!(state.replacement_phase, ReplacementPhase::Intent);
|
||||
assert_eq!(state.replacement_target_identities, vec![leased]);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn automatic_replacement_rejects_a_new_identity_after_format() {
|
||||
let temp = TempDir::new().expect("temporary resume disk directory should be created");
|
||||
@@ -1267,11 +1301,7 @@ async fn automatic_replacement_rejects_a_new_identity_after_format() {
|
||||
let storage = Arc::new(MockStorage {
|
||||
bucket_incarnation_id: Mutex::new(Some(Uuid::from_u128(1))),
|
||||
replacement_target_identities_ready: Mutex::new(true),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([
|
||||
vec![first_identity.clone()],
|
||||
vec![first_identity.clone()],
|
||||
vec![second_identity],
|
||||
])),
|
||||
replacement_target_identity_sequences: Mutex::new(VecDeque::from([vec![first_identity.clone()], vec![second_identity]])),
|
||||
resume_disk: Mutex::new(Some(disk.clone())),
|
||||
..Default::default()
|
||||
});
|
||||
|
||||
+62
-11
@@ -72,6 +72,15 @@ const TEMP_USER_PERSISTENCE_VERIFY_RETRY_DELAY: Duration = Duration::from_millis
|
||||
const TEMP_USER_PERSISTENCE_VERIFY_RETRY_DELAY: Duration = Duration::from_millis(1);
|
||||
const TEMP_USER_PERSISTENCE_VERIFY_MAX_RETRIES: usize = 5;
|
||||
|
||||
/// Resolve the durable parent of an authenticated or loaded session credential.
|
||||
/// Older identities can store the parent only in the verified token claims.
|
||||
pub fn session_parent_identity(credentials: &Credentials) -> Option<&str> {
|
||||
if !credentials.parent_user.is_empty() {
|
||||
return Some(credentials.parent_user.as_str());
|
||||
}
|
||||
credentials.claims_or_empty().get("parent").and_then(Value::as_str)
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
struct IAMFormat {
|
||||
version: i32,
|
||||
@@ -817,7 +826,7 @@ where
|
||||
.values()
|
||||
.filter_map(|x| {
|
||||
if !access_key.is_empty()
|
||||
&& x.credentials.parent_user.as_str() == access_key
|
||||
&& session_parent_identity(&x.credentials) == Some(access_key)
|
||||
&& x.credentials.is_temp()
|
||||
&& !x.credentials.is_service_account()
|
||||
{
|
||||
@@ -1692,7 +1701,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn update_user_secret_key(&self, access_key: &str, secret_key: &str) -> Result<(OffsetDateTime, AccountStatus)> {
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None)
|
||||
self.update_user_secret_key_with_guard(access_key, secret_key, None, None)
|
||||
.await
|
||||
.map(|(updated_at, status, _guard)| (updated_at, status))
|
||||
}
|
||||
@@ -1702,6 +1711,7 @@ where
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(
|
||||
OffsetDateTime,
|
||||
AccountStatus,
|
||||
@@ -1713,13 +1723,17 @@ where
|
||||
|
||||
let parent_lock = self.cache.parent_mutation_lock(access_key);
|
||||
let parent_guard = parent_lock.lock_owned().await;
|
||||
let cache = self.cache.snapshot();
|
||||
let users = Arc::clone(&cache.users);
|
||||
let u = match users.get(access_key) {
|
||||
Some(u) => u,
|
||||
None if expected_secret.is_some() => return Err(Error::IAMActionNotAllowed),
|
||||
None => return Err(Error::NoSuchUser(access_key.to_string())),
|
||||
};
|
||||
let u = self
|
||||
.api
|
||||
.load_user_identity(access_key, UserType::Reg)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if expected_secret.is_some() && is_err_no_such_user(&error) {
|
||||
Error::IAMActionNotAllowed
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
|
||||
if let Some(expected) = expected_secret {
|
||||
use subtle::ConstantTimeEq as _;
|
||||
@@ -1732,6 +1746,45 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Lock order is parent, then the presenting STS credential. Deletion and
|
||||
// session refresh use the same STS lock. Release it before the revocation sweep.
|
||||
let _session_guard = match session {
|
||||
Some(session) => {
|
||||
let guard = self.cache.sts_account_mutation_lock(&session.access_key).lock_owned().await;
|
||||
let current = self
|
||||
.api
|
||||
.load_user_identity(&session.access_key, UserType::Sts)
|
||||
.await
|
||||
.map_err(|error| {
|
||||
if is_err_no_such_user(&error) {
|
||||
Error::IAMActionNotAllowed
|
||||
} else {
|
||||
error
|
||||
}
|
||||
})?;
|
||||
use subtle::ConstantTimeEq as _;
|
||||
if !session.is_valid()
|
||||
|| !current.credentials.is_valid()
|
||||
|| !current.credentials.is_temp()
|
||||
|| current.credentials.is_service_account()
|
||||
|| current.credentials.access_key != session.access_key
|
||||
|| session_parent_identity(¤t.credentials) != Some(access_key)
|
||||
|| !bool::from(current.credentials.secret_key.as_bytes().ct_eq(session.secret_key.as_bytes()))
|
||||
|| !bool::from(
|
||||
current
|
||||
.credentials
|
||||
.session_token
|
||||
.as_bytes()
|
||||
.ct_eq(session.session_token.as_bytes()),
|
||||
)
|
||||
{
|
||||
return Err(Error::IAMActionNotAllowed);
|
||||
}
|
||||
Some(guard)
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
|
||||
let mut cred = u.credentials.clone();
|
||||
cred.secret_key = secret_key.to_string();
|
||||
|
||||
@@ -1745,8 +1798,6 @@ where
|
||||
};
|
||||
let u = UserIdentity::from(cred);
|
||||
let updated_at = u.update_at.unwrap_or_else(OffsetDateTime::now_utc);
|
||||
drop(cache);
|
||||
drop(users);
|
||||
|
||||
self.api
|
||||
.save_user_identity(access_key, UserType::Reg, u.clone(), None)
|
||||
|
||||
+229
-6
@@ -1206,11 +1206,11 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None).await
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, None, None).await
|
||||
}
|
||||
|
||||
/// Change an enabled regular caller's secret only if its signing secret is
|
||||
/// still current under the parent mutation lock. Status and policies are kept.
|
||||
/// Change an enabled regular user's secret only if the supplied current
|
||||
/// secret still matches under the parent mutation lock. Status and policies are kept.
|
||||
/// Persistence and revocation continue after caller cancellation.
|
||||
pub async fn change_own_secret_key(
|
||||
&self,
|
||||
@@ -1218,7 +1218,27 @@ impl<T: Store> IamSys<T> {
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret))
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), None)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Change a local IAM password using an authenticated STS session and its parent password.
|
||||
///
|
||||
/// The parent and session mutation locks protect revalidation through the secret write.
|
||||
/// Persistence and session revocation continue after request cancellation.
|
||||
/// A revocation failure is returned after the new secret has been persisted.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns `IAMActionNotAllowed` for inactive sessions, parent mismatches, or an
|
||||
/// incorrect current password. Storage and revocation errors are propagated.
|
||||
pub async fn change_own_secret_key_from_session(
|
||||
&self,
|
||||
access_key: &str,
|
||||
expected_secret: &str,
|
||||
secret_key: &str,
|
||||
session: &Credentials,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
self.rotate_user_secret_key_checked(access_key, secret_key, Some(expected_secret), Some(session))
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1227,16 +1247,18 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
let runtime = tokio::runtime::Handle::try_current().map_err(Error::other)?;
|
||||
let iam = self.clone();
|
||||
let access_key = access_key.to_string();
|
||||
let secret_key = secret_key.to_string();
|
||||
let expected_secret = expected_secret.map(str::to_string);
|
||||
let session = session.cloned();
|
||||
#[cfg(test)]
|
||||
let notification_probe = crate::LOAD_USER_NOTIFICATION_PROBE.try_with(Arc::clone).ok();
|
||||
let operation = async move {
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref())
|
||||
iam.rotate_user_secret_key_inner(&access_key, &secret_key, expected_secret.as_deref(), session.as_ref())
|
||||
.await
|
||||
};
|
||||
#[cfg(test)]
|
||||
@@ -1256,6 +1278,7 @@ impl<T: Store> IamSys<T> {
|
||||
access_key: &str,
|
||||
secret_key: &str,
|
||||
expected_secret: Option<&str>,
|
||||
session: Option<&Credentials>,
|
||||
) -> Result<(OffsetDateTime, rustfs_madmin::AccountStatus, usize)> {
|
||||
if !is_access_key_valid(access_key) {
|
||||
return Err(IamError::InvalidAccessKeyLength);
|
||||
@@ -1267,7 +1290,7 @@ impl<T: Store> IamSys<T> {
|
||||
|
||||
let (updated_at, status, parent_guard) = self
|
||||
.store
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret)
|
||||
.update_user_secret_key_with_guard(access_key, secret_key, expected_secret, session)
|
||||
.await?;
|
||||
let revoked = self.revoke_sts_sessions_for_parent(access_key).await?;
|
||||
drop(parent_guard);
|
||||
@@ -3936,6 +3959,206 @@ mod tests {
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn own_secret_change_revalidates_the_presenting_session_under_the_parent_lock() {
|
||||
ensure_test_global_credentials();
|
||||
for mutation in ["revoke", "expire", "disable", "parent", "secret", "token"] {
|
||||
let store = StsTestMockStore::new(false);
|
||||
let iam = Arc::new(IamSys::new(IamCache::new(store.clone()).await.expect("IAM")));
|
||||
let parent = "session-change-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
|
||||
let guard = iam.store.cache.parent_mutation_lock(parent).lock_owned().await;
|
||||
let pending = {
|
||||
let iam = Arc::clone(&iam);
|
||||
let session = session.clone();
|
||||
tokio::spawn(async move {
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session)
|
||||
.await
|
||||
})
|
||||
};
|
||||
if mutation == "revoke" {
|
||||
iam.delete_temp_account(&session.access_key, false)
|
||||
.await
|
||||
.expect("revoke presenting session");
|
||||
} else {
|
||||
let mut saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
let current = saved.get_mut(&session.access_key).expect("durable session");
|
||||
match mutation {
|
||||
"expire" => current.credentials.expiration = Some(OffsetDateTime::UNIX_EPOCH),
|
||||
"disable" => current.credentials.status = "off".to_string(),
|
||||
"parent" => current.credentials.parent_user = "another-parent".to_string(),
|
||||
"secret" => current.credentials.secret_key = "replacementSessionSecret123".to_string(),
|
||||
"token" => current.credentials.session_token = "replacement-token".to_string(),
|
||||
_ => unreachable!("known fixture mutation"),
|
||||
}
|
||||
}
|
||||
drop(guard);
|
||||
let error = tokio::time::timeout(std::time::Duration::from_secs(5), pending)
|
||||
.await
|
||||
.expect("queued rotation finishes")
|
||||
.expect("rotation task")
|
||||
.expect_err("stale session must fail");
|
||||
assert_eq!(error, Error::IAMActionNotAllowed, "{mutation}");
|
||||
let saved = store.saved_sts_users.lock().expect("saved identities");
|
||||
assert_eq!(saved.get(parent).expect("parent survives").credentials.secret_key, password, "{mutation}");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn session_secret_change_uses_the_durable_parent_password_with_a_stale_cache() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "stale-password-parent";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: "cachedParentSecret123".to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "fresh-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("cached parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
iam.store
|
||||
.api
|
||||
.saved_sts_users
|
||||
.lock()
|
||||
.expect("saved identities")
|
||||
.get_mut(parent)
|
||||
.expect("durable parent")
|
||||
.credentials
|
||||
.secret_key = "durableParentSecret123".to_string();
|
||||
assert_eq!(
|
||||
iam.store
|
||||
.get_user(parent)
|
||||
.await
|
||||
.expect("stale cached parent")
|
||||
.credentials
|
||||
.secret_key,
|
||||
"cachedParentSecret123"
|
||||
);
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE
|
||||
.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, "durableParentSecret123", "replacementParentSecret123", &session),
|
||||
)
|
||||
.await
|
||||
.expect("durable current password authorizes the change");
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("updated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn current_session_can_change_its_parent_secret_and_revoke_itself() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "current-session-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
let session = rotation_test_session(parent, "current-password-session");
|
||||
let owner = iam.get_user(parent).await.expect("parent").credentials;
|
||||
iam.set_temp_user_from_parent(&session, &owner)
|
||||
.await
|
||||
.expect("publish session");
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let (_, _, revoked) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("rotation must release its session lock before revocation")
|
||||
.expect("rotate current session");
|
||||
assert_eq!(revoked, 1);
|
||||
assert!(iam.get_user(&session.access_key).await.is_none());
|
||||
assert_eq!(
|
||||
iam.get_user(parent).await.expect("rotated parent").credentials.secret_key,
|
||||
"replacementParentSecret123"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn password_rotation_revokes_sessions_with_a_claim_only_parent() {
|
||||
let iam = test_iam_sys().await;
|
||||
let parent = "claim-only-session-parent";
|
||||
let password = "currentParentSecret123";
|
||||
iam.create_user(
|
||||
parent,
|
||||
&AddOrUpdateUserReq {
|
||||
secret_key: password.to_string(),
|
||||
policy: None,
|
||||
status: rustfs_madmin::AccountStatus::Enabled,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.expect("create parent");
|
||||
|
||||
// Legacy identities can have a verified parent claim without the denormalized field.
|
||||
let mut session = rotation_test_session(parent, "claim-only-password-session");
|
||||
session.parent_user.clear();
|
||||
session.claims = Some(HashMap::from([("parent".to_owned(), Value::String(parent.to_owned()))]));
|
||||
let identity = UserIdentity::new(session.clone());
|
||||
iam.store
|
||||
.api
|
||||
.save_user_identity(&session.access_key, UserType::Sts, identity.clone(), None)
|
||||
.await
|
||||
.expect("seed persisted legacy session");
|
||||
iam.store
|
||||
.cache
|
||||
.add_or_update_sts_account(&session.access_key, &identity, OffsetDateTime::now_utc());
|
||||
|
||||
let probe = IamSys::<StsTestMockStore>::load_user_notification_probe(0, false, false);
|
||||
let (_, _, revoked) = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
crate::LOAD_USER_NOTIFICATION_PROBE.scope(
|
||||
probe,
|
||||
iam.change_own_secret_key_from_session(parent, password, "replacementParentSecret123", &session),
|
||||
),
|
||||
)
|
||||
.await
|
||||
.expect("rotation and revocation finish")
|
||||
.expect("rotate using legacy session");
|
||||
assert_eq!(revoked, 1);
|
||||
assert!(iam.get_user(&session.access_key).await.is_none());
|
||||
assert!(matches!(
|
||||
iam.store.api.load_user_identity(&session.access_key, UserType::Sts).await,
|
||||
Err(Error::NoSuchUser(_))
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn self_secret_change_cannot_undo_concurrent_disable_rotation_or_deletion() {
|
||||
ensure_test_global_credentials();
|
||||
|
||||
@@ -34,6 +34,52 @@ pub struct Infos {
|
||||
/// enum's crate).
|
||||
const DRIVE_STATE_OK: &str = "ok";
|
||||
|
||||
/// Inspection of all discovered transactions for one exact object in an erasure set.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DeleteResidueInspection {
|
||||
pub bucket: String,
|
||||
pub object: String,
|
||||
pub pool: usize,
|
||||
pub set: usize,
|
||||
pub incarnation: String,
|
||||
pub complete: bool,
|
||||
pub reason: String,
|
||||
pub participants: Vec<DeleteResidueParticipant>,
|
||||
pub transactions: Vec<DeleteResidueTransaction>,
|
||||
/// Present only when all participants permit a fresh discard of legacy residue.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub fingerprint: Option<String>,
|
||||
/// Number of transaction groups discarded by this request.
|
||||
pub reclaimed: usize,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DeleteResidueParticipant {
|
||||
pub slot: usize,
|
||||
pub disk_id: Option<String>,
|
||||
pub discovery: String,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DeleteResidueTransaction {
|
||||
pub id: String,
|
||||
pub copies: Vec<DeleteResidueCopy>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct DeleteResidueCopy {
|
||||
pub slot: usize,
|
||||
pub state: String,
|
||||
pub current_metadata_present: Option<bool>,
|
||||
pub backup_present: Option<bool>,
|
||||
pub journal_present: Option<bool>,
|
||||
pub fingerprint: Option<String>,
|
||||
}
|
||||
|
||||
/// Unresolved delete recovery work, independent of ordinary object healing.
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub struct DeleteRecoveryStatus {
|
||||
|
||||
@@ -322,6 +322,17 @@ pub struct Services {
|
||||
pub notifications: Option<Vec<HashMap<String, Vec<TargetIDStatus>>>>,
|
||||
}
|
||||
|
||||
/// Namespace coverage and scanner measurements for the optional RustFS admin inventory.
|
||||
#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ClusterInventory {
|
||||
pub current_buckets: Option<u64>,
|
||||
pub measured_buckets: Option<u64>,
|
||||
pub measured_objects: Option<u64>,
|
||||
pub measured_bytes: Option<u64>,
|
||||
pub snapshot_at: Option<jiff::Timestamp>,
|
||||
pub converged: Option<bool>,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Default)]
|
||||
pub struct Buckets {
|
||||
pub count: u64,
|
||||
|
||||
@@ -709,6 +709,8 @@ pub struct ScannerMetrics {
|
||||
pub last_cycle_partial_source: String,
|
||||
#[serde(rename = "last_cycle_partial_source_code", default)]
|
||||
pub last_cycle_partial_source_code: u64,
|
||||
#[serde(rename = "last_cycle_partial_stage", default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_cycle_partial_stage: Option<String>,
|
||||
#[serde(rename = "last_cycle_duration_seconds", default)]
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
#[serde(rename = "last_cycle_objects_scanned", default)]
|
||||
@@ -853,6 +855,7 @@ impl ScannerMetrics {
|
||||
self.last_cycle_partial_reason_code = other.last_cycle_partial_reason_code;
|
||||
self.last_cycle_partial_source = other.last_cycle_partial_source.clone();
|
||||
self.last_cycle_partial_source_code = other.last_cycle_partial_source_code;
|
||||
self.last_cycle_partial_stage.clone_from(&other.last_cycle_partial_stage);
|
||||
self.last_cycle_duration_seconds = other.last_cycle_duration_seconds;
|
||||
self.throttle_idle_mode_enabled = other.throttle_idle_mode_enabled;
|
||||
self.throttle_sleep_factor = other.throttle_sleep_factor;
|
||||
@@ -1506,6 +1509,27 @@ pub struct KeyRotationInfo {
|
||||
pub objects_failed: i64,
|
||||
}
|
||||
|
||||
/// A single configured node's scanner observation. The epoch fences concurrent coordinator samples.
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct ScannerNodeSample {
|
||||
pub active: bool,
|
||||
pub leader: bool,
|
||||
pub cycle: u64,
|
||||
pub epoch: Option<u64>,
|
||||
pub started_at: Option<Timestamp>,
|
||||
pub objects_scanned: u64,
|
||||
pub last_attempt: Option<ScannerAttemptSample>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
|
||||
pub struct ScannerAttemptSample {
|
||||
pub epoch: u64,
|
||||
pub cycle: u64,
|
||||
pub result: String,
|
||||
pub partial_reason: Option<String>,
|
||||
pub duration_seconds: f64,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Serialize, Deserialize)]
|
||||
pub struct RealtimeMetrics {
|
||||
#[serde(rename = "errors")]
|
||||
@@ -1520,6 +1544,9 @@ pub struct RealtimeMetrics {
|
||||
pub by_disk: HashMap<String, DiskMetric>,
|
||||
#[serde(rename = "final")]
|
||||
pub finally: bool,
|
||||
/// Internal GetMetrics reply to a scanner summary probe. Absent on older peers.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scanner_node: Option<ScannerNodeSample>,
|
||||
}
|
||||
|
||||
impl RealtimeMetrics {
|
||||
@@ -1583,6 +1610,35 @@ mod tests {
|
||||
Timestamp::constant(1_700_000_000, 123_456_000)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cluster_scanner_node_sample_is_additive_on_the_peer_metrics_wire() {
|
||||
let ordinary = serde_json::to_value(RealtimeMetrics::default()).expect("ordinary response");
|
||||
assert!(ordinary.get("scanner_node").is_none());
|
||||
let old = rmp_serde::to_vec_named(&ordinary).expect("previous map reply");
|
||||
assert!(
|
||||
rmp_serde::from_slice::<RealtimeMetrics>(&old)
|
||||
.expect("old peers remain readable")
|
||||
.scanner_node
|
||||
.is_none()
|
||||
);
|
||||
let reply = RealtimeMetrics {
|
||||
scanner_node: Some(ScannerNodeSample {
|
||||
active: true,
|
||||
leader: true,
|
||||
epoch: Some(8),
|
||||
cycle: 40,
|
||||
..Default::default()
|
||||
}),
|
||||
..Default::default()
|
||||
};
|
||||
let encoded = rmp_serde::to_vec_named(&reply).expect("scoped peer sample");
|
||||
let sample = rmp_serde::from_slice::<RealtimeMetrics>(&encoded)
|
||||
.expect("round trip")
|
||||
.scanner_node
|
||||
.expect("new capability field");
|
||||
assert_eq!((sample.active, sample.leader, sample.epoch, sample.cycle), (true, true, Some(8), 40));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn http_metrics_merge_preserves_outcomes_and_missing_node_support() {
|
||||
#[derive(Serialize, Deserialize, Default)]
|
||||
@@ -2138,6 +2194,7 @@ mod tests {
|
||||
collected_at: collected_at + SignedDuration::from_secs(1),
|
||||
last_cycle_partial_source: "lifecycle".to_string(),
|
||||
last_cycle_partial_source_code: 2,
|
||||
last_cycle_partial_stage: Some("object_budget".into()),
|
||||
pacing_pressure: ScannerPacingPressureSnapshot {
|
||||
primary_pressure: "queued_scans".to_string(),
|
||||
current_queued_scans: 4,
|
||||
@@ -2162,6 +2219,7 @@ mod tests {
|
||||
|
||||
assert_eq!(scanner.last_cycle_partial_source, "lifecycle");
|
||||
assert_eq!(scanner.last_cycle_partial_source_code, 2);
|
||||
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert_eq!(scanner.pacing_pressure.primary_pressure, "queued_scans");
|
||||
assert_eq!(scanner.pacing_pressure.current_queued_scans, 4);
|
||||
assert_eq!(scanner.pacing_pressure.current_active_scans, 3);
|
||||
|
||||
@@ -35,8 +35,6 @@ path = "src/main.rs"
|
||||
[features]
|
||||
default = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
"rustfs-common/hotpath",
|
||||
"rustfs-config/hotpath",
|
||||
"rustfs-io-metrics/hotpath",
|
||||
@@ -45,7 +43,6 @@ hotpath = [
|
||||
]
|
||||
hotpath-alloc = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-alloc",
|
||||
"rustfs-common/hotpath-alloc",
|
||||
"rustfs-config/hotpath-alloc",
|
||||
"rustfs-io-metrics/hotpath-alloc",
|
||||
@@ -54,7 +51,6 @@ hotpath-alloc = [
|
||||
]
|
||||
hotpath-cpu = [
|
||||
"hotpath",
|
||||
"hotpath/hotpath-cpu",
|
||||
"rustfs-common/hotpath-cpu",
|
||||
"rustfs-config/hotpath-cpu",
|
||||
"rustfs-io-metrics/hotpath-cpu",
|
||||
@@ -63,7 +59,6 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-heal-contracts.workspace = true
|
||||
rustfs-io-metrics.workspace = true
|
||||
@@ -87,3 +82,5 @@ doctest = false
|
||||
[dev-dependencies]
|
||||
serde_json = { workspace = true }
|
||||
temp-env = { workspace = true }
|
||||
tower = { workspace = true, features = ["util"] }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
@@ -3399,6 +3399,10 @@ pub enum ChannelClass {
|
||||
// Keep multiplexed unary RPCs below h2's per-connection small-frame budget.
|
||||
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
|
||||
|
||||
// Quorum reads cancel surplus metadata RPCs in bursts. Retain their stream IDs
|
||||
// until late replies arrive, with a finite per-connection memory bound.
|
||||
const INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS: usize = 4096;
|
||||
|
||||
/// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout).
|
||||
fn channel_isolation_enabled() -> bool {
|
||||
rustfs_utils::get_env_bool(
|
||||
@@ -3603,6 +3607,7 @@ async fn dial_channel(dial_addr: &str, cache_key: &str, rpc_timeout: Duration) -
|
||||
// Fast connection timeout for dead peer detection
|
||||
.connect_timeout(connect_timeout)
|
||||
.concurrency_limit(INTERNODE_RPC_CONCURRENCY_LIMIT)
|
||||
.http2_max_concurrent_reset_streams(INTERNODE_RPC_MAX_CONCURRENT_RESET_STREAMS)
|
||||
// TCP-level keepalive - OS will probe connection
|
||||
.tcp_keepalive(Some(tcp_keepalive))
|
||||
// Disable Nagle so latency-sensitive control-plane RPCs (locks/health) are not batched
|
||||
@@ -3732,6 +3737,9 @@ pub async fn evict_failed_connection_with_log_level(addr: &str, log_level: Conne
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod reset_retention_tests;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use super::proto_gen::node_service::{PingRequest, node_service_client::NodeServiceClient};
|
||||
use super::{ChannelClass, INTERNODE_RPC_CONCURRENCY_LIMIT, bulk_cache_key, dial_channel};
|
||||
use std::convert::Infallible;
|
||||
use std::fmt::Debug;
|
||||
use std::future::pending;
|
||||
use std::time::Duration;
|
||||
use tokio::net::TcpListener;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::task::JoinSet;
|
||||
use tonic::body::Body;
|
||||
use tonic::codegen::{http::Response, tokio_stream::wrappers::TcpListenerStream};
|
||||
use tracing::{Event, Subscriber, field::Field};
|
||||
use tracing_subscriber::{Layer, layer::Context, prelude::*};
|
||||
|
||||
// Tonic does not expose the applied h2 retention limit. Observe h2's local
|
||||
// accounting on a normal cancellation batch, before the server replies.
|
||||
struct ResetAccounting(mpsc::UnboundedSender<bool>);
|
||||
|
||||
impl<S: Subscriber> Layer<S> for ResetAccounting {
|
||||
fn on_event(&self, event: &Event<'_>, _: Context<'_, S>) {
|
||||
event.record(&mut |field: &Field, value: &dyn Debug| {
|
||||
if field.name() == "message" {
|
||||
let message = format!("{value:?}");
|
||||
if message.starts_with("enqueue_reset_expiration; ") {
|
||||
self.0
|
||||
.send(message.starts_with("enqueue_reset_expiration; added "))
|
||||
.expect("retention receiver");
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
async fn check_cancelled_batch(class: ChannelClass) {
|
||||
let (events, mut received) = mpsc::unbounded_channel();
|
||||
let observer = ResetAccounting(events).with_filter(tracing_subscriber::filter::filter_fn(|metadata| {
|
||||
metadata.target() == "h2::proto::streams::recv"
|
||||
}));
|
||||
let _guard = tracing::subscriber::set_default(tracing_subscriber::registry().with(observer));
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind test server");
|
||||
let address = format!("http://{}", listener.local_addr().expect("read listener address"));
|
||||
let (started, mut dispatches) = mpsc::unbounded_channel();
|
||||
let server = tokio::spawn(async move {
|
||||
let service = tower::service_fn(move |_| {
|
||||
started.send(()).expect("record dispatch");
|
||||
pending::<Result<Response<Body>, Infallible>>()
|
||||
});
|
||||
tonic::transport::Server::builder()
|
||||
.serve_with_incoming(service, TcpListenerStream::new(listener))
|
||||
.await
|
||||
});
|
||||
let key = match class {
|
||||
ChannelClass::Control => address.clone(),
|
||||
ChannelClass::Bulk => bulk_cache_key(&address, 0),
|
||||
};
|
||||
let (channel, _) = dial_channel(&address, &key, Duration::from_secs(10))
|
||||
.await
|
||||
.expect("dial test server");
|
||||
let mut calls = JoinSet::new();
|
||||
for _ in 0..INTERNODE_RPC_CONCURRENCY_LIMIT {
|
||||
let mut client = NodeServiceClient::new(channel.clone());
|
||||
calls.spawn(async move { client.ping(PingRequest::default()).await });
|
||||
}
|
||||
for _ in 0..INTERNODE_RPC_CONCURRENCY_LIMIT {
|
||||
dispatches
|
||||
.recv()
|
||||
.await
|
||||
.expect("all RPCs must reach the service before cancellation");
|
||||
}
|
||||
calls.shutdown().await;
|
||||
let mut retained = 0;
|
||||
for _ in 0..INTERNODE_RPC_CONCURRENCY_LIMIT {
|
||||
retained += usize::from(received.recv().await.expect("observe accounting for each cancelled stream"));
|
||||
}
|
||||
assert_eq!(
|
||||
retained, INTERNODE_RPC_CONCURRENCY_LIMIT,
|
||||
"one complete RPC batch must fit in local reset retention"
|
||||
);
|
||||
server.abort();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn control_channel_retains_cancelled_batch() {
|
||||
tokio::time::timeout(Duration::from_secs(15), check_cancelled_batch(ChannelClass::Control))
|
||||
.await
|
||||
.expect("control batch completes within the test deadline");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bulk_channel_retains_cancelled_batch() {
|
||||
tokio::time::timeout(Duration::from_secs(15), check_cancelled_batch(ChannelClass::Bulk))
|
||||
.await
|
||||
.expect("bulk batch completes within the test deadline");
|
||||
}
|
||||
@@ -194,7 +194,8 @@ pub fn replication_action_for_target(
|
||||
return ReplicationAction::Metadata;
|
||||
}
|
||||
|
||||
if tag_metadata_differs(source, target) {
|
||||
let source_tags = ReplicationTagFilter::decode_tags_to_map(source.user_tags);
|
||||
if !replication_target_tags_match(&source_tags, target.tag_count, target.metadata) {
|
||||
return ReplicationAction::Metadata;
|
||||
}
|
||||
|
||||
@@ -219,16 +220,18 @@ fn content_encoding_differs(source: &ReplicationSourceObject<'_>, target: &Repli
|
||||
false
|
||||
}
|
||||
|
||||
fn tag_metadata_differs(source: &ReplicationSourceObject<'_>, target: &ReplicationTargetObject<'_>) -> bool {
|
||||
let source_tags = ReplicationTagFilter::decode_tags_to_map(source.user_tags);
|
||||
let target_tagging = target
|
||||
.metadata
|
||||
.and_then(|metadata| metadata.get(AMZ_OBJECT_TAGGING).map(String::as_str))
|
||||
.unwrap_or_default();
|
||||
let target_tags = ReplicationTagFilter::decode_tags_to_map(target_tagging);
|
||||
let source_tag_count = i32::try_from(source_tags.len()).unwrap_or(i32::MAX);
|
||||
|
||||
(target.tag_count > 0 && source_tags != target_tags) || target.tag_count != source_tag_count
|
||||
/// Compare a source tag set with a replica HEAD. A count alone cannot confirm
|
||||
/// that a nonempty source tag set reached the replica.
|
||||
pub fn replication_target_tags_match(
|
||||
source_tags: &HashMap<String, String>,
|
||||
target_tag_count: i32,
|
||||
target_metadata: Option<&HashMap<String, String>>,
|
||||
) -> bool {
|
||||
usize::try_from(target_tag_count) == Ok(source_tags.len())
|
||||
&& match target_metadata.and_then(|metadata| metadata.get(AMZ_OBJECT_TAGGING)) {
|
||||
Some(tagging) => ReplicationTagFilter::decode_tags_to_map(tagging) == *source_tags,
|
||||
None => source_tags.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
fn comparable_metadata(metadata: Option<&HashMap<String, String>>) -> HashMap<String, String> {
|
||||
|
||||
@@ -58,7 +58,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
hotpath.workspace = true
|
||||
hotpath = { workspace = true, optional = true }
|
||||
arc-swap.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "macros", "net", "rt-multi-thread", "sync", "time"] }
|
||||
rand = { workspace = true, features = ["serde"] }
|
||||
|
||||
@@ -286,19 +286,6 @@ impl ScannerWorkSource {
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: u8) -> Option<Self> {
|
||||
match code {
|
||||
1 => Some(Self::Usage),
|
||||
2 => Some(Self::Lifecycle),
|
||||
3 => Some(Self::BucketReplication),
|
||||
4 => Some(Self::SiteReplication),
|
||||
5 => Some(Self::Heal),
|
||||
6 => Some(Self::Bitrot),
|
||||
7 => Some(Self::Alerts),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
fn all() -> &'static [Self] {
|
||||
&Self::ALL
|
||||
}
|
||||
@@ -1000,6 +987,34 @@ impl ScannerUsagePublicationObservation {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Default)]
|
||||
struct LastScanCycle {
|
||||
sequence: u64,
|
||||
result: u8,
|
||||
partial_reason: ScanCyclePartialReason,
|
||||
partial_source: Option<ScannerWorkSource>,
|
||||
partial_stage: Option<&'static str>,
|
||||
duration_millis: u64,
|
||||
}
|
||||
|
||||
/// Updates only this observation if a later state write fails. A supervisor
|
||||
/// cancellation leaves the already recorded partial result and budget visible.
|
||||
pub struct PartialCycleObservation<'a> {
|
||||
latest: &'a Mutex<LastScanCycle>,
|
||||
sequence: u64,
|
||||
}
|
||||
|
||||
impl PartialCycleObservation<'_> {
|
||||
pub fn observe_state_write(self, persisted: bool) {
|
||||
if !persisted {
|
||||
let mut last = self.latest.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
if last.sequence == self.sequence {
|
||||
last.partial_stage = Some("cycle_state_not_durable");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Metrics {
|
||||
startup: crate::startup::ScannerStartupObservation,
|
||||
scan_observation: ScannerScanObservation,
|
||||
@@ -1009,6 +1024,7 @@ pub struct Metrics {
|
||||
actions_latency: Vec<LockedLastMinuteLatency>,
|
||||
current_paths: Arc<RwLock<HashMap<String, Arc<CurrentPathTracker>>>>,
|
||||
cycle_info: Arc<RwLock<Option<CurrentCycle>>>,
|
||||
last_cluster_attempt: Mutex<Option<ScannerAttemptRecord>>,
|
||||
// Lock order: cycle_info, usage_publication, then the observation's status. No awaits under either mutex.
|
||||
usage_publication: Mutex<Arc<ScannerUsagePublicationObservation>>,
|
||||
current_scan_mode: AtomicU8,
|
||||
@@ -1046,11 +1062,8 @@ pub struct Metrics {
|
||||
scanner_leader_lock_held: AtomicBool,
|
||||
scanner_leader_lock_last_error: RwLock<String>,
|
||||
scanner_leader_lock_last_update_unix_secs: AtomicU64,
|
||||
last_scan_cycle_result: AtomicU8,
|
||||
last_scan_cycle_partial_reason: AtomicU8,
|
||||
last_scan_cycle_partial_source: AtomicU8,
|
||||
last_scan_cycle: Mutex<LastScanCycle>,
|
||||
last_scan_cycle_end_unix_secs: AtomicU64,
|
||||
last_scan_cycle_duration_millis: AtomicU64,
|
||||
last_scan_cycle_objects_scanned: AtomicU64,
|
||||
last_scan_cycle_directories_scanned: AtomicU64,
|
||||
/// Lifetime count of object versions walked by the data scanner, counted
|
||||
@@ -1166,7 +1179,6 @@ pub struct Metrics {
|
||||
partial_scan_cycles: AtomicU64,
|
||||
}
|
||||
|
||||
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
|
||||
const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
|
||||
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
|
||||
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
|
||||
@@ -1198,12 +1210,12 @@ impl ScanCyclePartialReason {
|
||||
}
|
||||
}
|
||||
|
||||
fn from_code(code: u8) -> Self {
|
||||
match code {
|
||||
1 => Self::Runtime,
|
||||
2 => Self::Objects,
|
||||
3 => Self::Directories,
|
||||
_ => Self::Unknown,
|
||||
fn budget_stage(self) -> Option<&'static str> {
|
||||
match self {
|
||||
Self::Runtime => Some("runtime_budget"),
|
||||
Self::Objects => Some("object_budget"),
|
||||
Self::Directories => Some("directory_budget"),
|
||||
Self::Unknown => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1240,6 +1252,27 @@ impl ScannerUsageSaveResult {
|
||||
}
|
||||
}
|
||||
|
||||
/// Completed coordinator attempt, retained with the generation of its originating worker.
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
pub struct ScannerAttemptRecord {
|
||||
pub epoch: u64,
|
||||
pub cycle: u64,
|
||||
pub result: &'static str,
|
||||
pub partial_reason: Option<&'static str>,
|
||||
pub duration: Duration,
|
||||
}
|
||||
|
||||
/// Compact process-local state for a cluster status probe.
|
||||
pub struct ScannerNodeObservation {
|
||||
pub active: bool,
|
||||
pub leader: bool,
|
||||
pub epoch: Option<u64>,
|
||||
pub cycle: u64,
|
||||
pub started_at: Option<Timestamp>,
|
||||
pub objects_scanned: u64,
|
||||
pub last_attempt: Option<ScannerAttemptRecord>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Default, Serialize, Deserialize)]
|
||||
pub struct CurrentCycle {
|
||||
pub current: u64,
|
||||
@@ -1603,6 +1636,9 @@ pub struct ScannerMetricsReport {
|
||||
pub last_cycle_partial_source: String,
|
||||
#[serde(default)]
|
||||
pub last_cycle_partial_source_code: u64,
|
||||
/// First observed blocker for this partial cycle, including the exact budget when known.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub last_cycle_partial_stage: Option<String>,
|
||||
pub last_cycle_duration_seconds: f64,
|
||||
#[serde(default)]
|
||||
pub last_cycle_objects_scanned: u64,
|
||||
@@ -1742,6 +1778,11 @@ pub struct ScannerRuntimeDetailsReport {
|
||||
}
|
||||
|
||||
impl CurrentCycle {
|
||||
/// Last successful completion recorded in the persisted cycle history.
|
||||
pub fn last_success_at(&self) -> Option<Timestamp> {
|
||||
self.cycle_completed.last().copied().map(chrono_to_jiff_timestamp)
|
||||
}
|
||||
|
||||
pub fn unmarshal(&mut self, buf: &[u8]) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
|
||||
*self = rmp_serde::from_slice(buf)?;
|
||||
Ok(())
|
||||
@@ -2100,8 +2141,18 @@ pub fn emit_scan_cycle_partial_with_source(
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
) {
|
||||
global_metrics().record_scan_cycle_partial_with_source(duration, reason, source);
|
||||
emit_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_partial_with_stage(
|
||||
duration: Duration,
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
stage: Option<&'static str>,
|
||||
) -> PartialCycleObservation<'static> {
|
||||
let observation = global_metrics().record_scan_cycle_partial_with_stage(duration, reason, source, stage);
|
||||
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
|
||||
observation
|
||||
}
|
||||
|
||||
pub fn emit_scan_cycle_superseded(duration: Duration) {
|
||||
@@ -2198,6 +2249,7 @@ impl Metrics {
|
||||
.collect(),
|
||||
current_paths: Arc::new(RwLock::new(HashMap::new())),
|
||||
cycle_info: Arc::new(RwLock::new(None)),
|
||||
last_cluster_attempt: Mutex::new(None),
|
||||
usage_publication: Mutex::new(Arc::new(ScannerUsagePublicationObservation::default())),
|
||||
current_scan_mode: AtomicU8::new(HealScanMode::Unknown as u8),
|
||||
current_scan_cycle_work_active: AtomicBool::new(false),
|
||||
@@ -2231,11 +2283,8 @@ impl Metrics {
|
||||
scanner_leader_lock_held: AtomicBool::new(false),
|
||||
scanner_leader_lock_last_error: RwLock::new(String::new()),
|
||||
scanner_leader_lock_last_update_unix_secs: AtomicU64::new(0),
|
||||
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
|
||||
last_scan_cycle_partial_reason: AtomicU8::new(ScanCyclePartialReason::Unknown as u8),
|
||||
last_scan_cycle_partial_source: AtomicU8::new(0),
|
||||
last_scan_cycle: Mutex::new(LastScanCycle::default()),
|
||||
last_scan_cycle_end_unix_secs: AtomicU64::new(0),
|
||||
last_scan_cycle_duration_millis: AtomicU64::new(0),
|
||||
last_scan_cycle_objects_scanned: AtomicU64::new(0),
|
||||
last_scan_cycle_directories_scanned: AtomicU64::new(0),
|
||||
lifetime_versions_scanned: AtomicU64::new(0),
|
||||
@@ -3166,6 +3215,45 @@ impl Metrics {
|
||||
*current_cycle = Some(cycle);
|
||||
}
|
||||
|
||||
/// Keep delayed results from an older coordinator from replacing a newer attempt.
|
||||
pub fn record_cluster_attempt(&self, attempt: ScannerAttemptRecord) {
|
||||
let mut latest = self.last_cluster_attempt.lock().unwrap_or_else(|err| err.into_inner());
|
||||
if latest.is_none_or(|old| (old.epoch, old.cycle) <= (attempt.epoch, attempt.cycle)) {
|
||||
*latest = Some(attempt);
|
||||
}
|
||||
}
|
||||
|
||||
/// Sample only the fields required by the admin cluster status API. The cycle read
|
||||
/// lock also protects installation of the publication and per-cycle work baseline.
|
||||
pub async fn scanner_node_observation(&self) -> ScannerNodeObservation {
|
||||
let cycle = self.cycle_info.read().await;
|
||||
let active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
|
||||
let generation = self
|
||||
.usage_publication
|
||||
.lock()
|
||||
.unwrap_or_else(|err| err.into_inner())
|
||||
.snapshot();
|
||||
let cycle_id = cycle.as_ref().map_or(0, |c| c.current);
|
||||
let epoch = generation
|
||||
.scanner_epoch
|
||||
.filter(|_| !active || generation.cycle == Some(cycle_id));
|
||||
let last_attempt = *self.last_cluster_attempt.lock().unwrap_or_else(|err| err.into_inner());
|
||||
ScannerNodeObservation {
|
||||
active,
|
||||
leader: self.scanner_leader_lock_held.load(Ordering::Acquire),
|
||||
cycle: cycle_id,
|
||||
epoch,
|
||||
started_at: cycle.as_ref().filter(|_| active).map(|c| chrono_to_jiff_timestamp(c.started)),
|
||||
objects_scanned: if active {
|
||||
self.scan_cycle_work_since(self.current_scan_cycle_work_start())
|
||||
.objects_scanned
|
||||
} else {
|
||||
0
|
||||
},
|
||||
last_attempt,
|
||||
}
|
||||
}
|
||||
|
||||
/// Read the current cycle record.
|
||||
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
|
||||
self.cycle_info.read().await.clone()
|
||||
@@ -3203,36 +3291,43 @@ impl Metrics {
|
||||
self.failed_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
SCAN_CYCLE_RESULT_ERROR
|
||||
};
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result.store(result, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_superseded(&self, duration: Duration) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_SUPERSEDED,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
|
||||
self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_DEFERRED,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
fn record_last_cycle(&self, duration: Duration, mut cycle: LastScanCycle) -> u64 {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason
|
||||
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
cycle.duration_millis = duration_millis_saturated(duration);
|
||||
let mut latest = self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
cycle.sequence = latest.sequence.wrapping_add(1);
|
||||
*latest = cycle;
|
||||
cycle.sequence
|
||||
}
|
||||
|
||||
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
|
||||
@@ -3245,7 +3340,16 @@ impl Metrics {
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
) {
|
||||
self.record_scanner_cycle_end_time();
|
||||
self.record_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
|
||||
}
|
||||
|
||||
fn record_scan_cycle_partial_with_stage(
|
||||
&self,
|
||||
duration: Duration,
|
||||
reason: ScanCyclePartialReason,
|
||||
source: Option<ScannerWorkSource>,
|
||||
stage: Option<&'static str>,
|
||||
) -> PartialCycleObservation<'_> {
|
||||
self.partial_scan_cycles.fetch_add(1, Ordering::Relaxed);
|
||||
match reason {
|
||||
ScanCyclePartialReason::Unknown => &self.partial_scan_cycles_unknown,
|
||||
@@ -3254,18 +3358,25 @@ impl Metrics {
|
||||
ScanCyclePartialReason::Directories => &self.partial_scan_cycles_directories,
|
||||
}
|
||||
.fetch_add(1, Ordering::Relaxed);
|
||||
self.last_scan_cycle_result
|
||||
.store(SCAN_CYCLE_RESULT_PARTIAL, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_reason.store(reason as u8, Ordering::Relaxed);
|
||||
self.last_scan_cycle_partial_source
|
||||
.store(source.map(ScannerWorkSource::code).unwrap_or_default(), Ordering::Relaxed);
|
||||
if let Some(source) = source
|
||||
&& let Some(cycles) = self.partial_scan_cycles_by_source.get(source.index())
|
||||
{
|
||||
cycles.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
self.last_scan_cycle_duration_millis
|
||||
.store(duration_millis_saturated(duration), Ordering::Relaxed);
|
||||
let sequence = self.record_last_cycle(
|
||||
duration,
|
||||
LastScanCycle {
|
||||
result: SCAN_CYCLE_RESULT_PARTIAL,
|
||||
partial_reason: reason,
|
||||
partial_source: source,
|
||||
partial_stage: stage,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
PartialCycleObservation {
|
||||
latest: &self.last_scan_cycle,
|
||||
sequence,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
|
||||
@@ -3854,19 +3965,19 @@ impl Metrics {
|
||||
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
|
||||
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
|
||||
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
|
||||
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
|
||||
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
|
||||
m.last_cycle_result_code = last_cycle_result as u64;
|
||||
let partial_reason = ScanCyclePartialReason::from_code(self.last_scan_cycle_partial_reason.load(Ordering::Relaxed));
|
||||
m.last_cycle_partial_reason = partial_reason.as_str().to_string();
|
||||
m.last_cycle_partial_reason_code = partial_reason as u8 as u64;
|
||||
let partial_source = ScannerWorkSource::from_code(self.last_scan_cycle_partial_source.load(Ordering::Relaxed));
|
||||
m.last_cycle_partial_source = partial_source
|
||||
let last = *self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
m.last_cycle_result = scan_cycle_result_label(last.result).to_string();
|
||||
m.last_cycle_result_code = last.result.into();
|
||||
m.last_cycle_partial_reason = last.partial_reason.as_str().to_string();
|
||||
m.last_cycle_partial_reason_code = last.partial_reason as u64;
|
||||
m.last_cycle_partial_source = last
|
||||
.partial_source
|
||||
.map(ScannerWorkSource::as_str)
|
||||
.unwrap_or(SCAN_CYCLE_RESULT_UNKNOWN_LABEL)
|
||||
.to_string();
|
||||
m.last_cycle_partial_source_code = partial_source.map(|source| source.code().into()).unwrap_or_default();
|
||||
m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
|
||||
m.last_cycle_partial_source_code = last.partial_source.map(|source| source.code().into()).unwrap_or_default();
|
||||
m.last_cycle_partial_stage = last.partial_stage.map(str::to_owned);
|
||||
m.last_cycle_duration_seconds = last.duration_millis as f64 / 1000.0;
|
||||
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
|
||||
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
|
||||
@@ -5284,6 +5395,56 @@ mod tests {
|
||||
assert_eq!(report.partial_cycles, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn delayed_partial_state_write_can_only_update_its_own_observation() {
|
||||
let metrics = Metrics::new();
|
||||
let record = || {
|
||||
metrics.record_scan_cycle_partial_with_stage(
|
||||
Duration::from_secs(1),
|
||||
ScanCyclePartialReason::Runtime,
|
||||
None,
|
||||
Some("runtime_budget"),
|
||||
)
|
||||
};
|
||||
let old = record();
|
||||
let current = record();
|
||||
assert_eq!(
|
||||
metrics.report().await.partial_cycles,
|
||||
2,
|
||||
"both stops are visible before either state write finishes"
|
||||
);
|
||||
old.observe_state_write(false);
|
||||
assert_eq!(metrics.report().await.last_cycle_partial_stage.as_deref(), Some("runtime_budget"));
|
||||
current.observe_state_write(false);
|
||||
let failed = metrics.report().await;
|
||||
assert_eq!(failed.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
|
||||
assert_eq!(failed.partial_cycles, 2);
|
||||
assert_eq!(failed.last_cycle_partial_reason, "runtime");
|
||||
let superseded = record();
|
||||
metrics.record_scan_cycle_complete(true, Duration::ZERO);
|
||||
superseded.observe_state_write(false);
|
||||
assert!(metrics.report().await.last_cycle_partial_stage.is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partial_stage_is_reported_with_the_last_result_and_cleared_on_completion() {
|
||||
let metrics = Metrics::new();
|
||||
metrics.record_scan_cycle_partial(Duration::from_secs(1), ScanCyclePartialReason::Directories);
|
||||
let partial = serde_json::to_value(metrics.report().await).unwrap();
|
||||
assert_eq!(partial["last_cycle_partial_reason"], "directories");
|
||||
assert_eq!(partial["last_cycle_partial_stage"], "directory_budget");
|
||||
metrics.record_scan_cycle_complete(true, Duration::from_secs(1));
|
||||
let finished = serde_json::to_value(metrics.report().await).unwrap();
|
||||
assert!(
|
||||
!finished
|
||||
.as_object()
|
||||
.expect("serialized report")
|
||||
.contains_key("last_cycle_partial_stage")
|
||||
);
|
||||
let legacy: ScannerMetricsReport = serde_json::from_value(finished).expect("older reports omit the stage");
|
||||
assert_eq!(legacy.last_cycle_partial_stage, None);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn report_tracks_partial_scan_cycle_without_failed_increment() {
|
||||
let metrics = Metrics::new();
|
||||
@@ -5754,6 +5915,50 @@ mod tests {
|
||||
assert_eq!(report.last_cycle_usage_saves, 2);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cluster_scanner_keeps_last_attempt_when_leadership_moves() {
|
||||
let metrics = Metrics::new();
|
||||
let previous = ScannerAttemptRecord {
|
||||
epoch: 7,
|
||||
cycle: 40,
|
||||
result: "partial",
|
||||
partial_reason: Some("runtime"),
|
||||
duration: Duration::from_secs(1801),
|
||||
};
|
||||
metrics.record_cluster_attempt(previous);
|
||||
metrics.record_cluster_attempt(ScannerAttemptRecord {
|
||||
epoch: 6,
|
||||
cycle: 100,
|
||||
..previous
|
||||
});
|
||||
metrics.set_cycle(None).await;
|
||||
let follower = metrics.scanner_node_observation().await;
|
||||
assert!(!follower.active);
|
||||
let history = follower.last_attempt.expect("previous leader's terminal record");
|
||||
assert_eq!(
|
||||
(history.epoch, history.cycle, history.result, history.partial_reason),
|
||||
(7, 40, "partial", Some("runtime"))
|
||||
);
|
||||
let cycle = CurrentCycle {
|
||||
current: 41,
|
||||
..Default::default()
|
||||
};
|
||||
let remote = Arc::new(RemoteScanProgress::default());
|
||||
metrics
|
||||
.start_scan_cycle_work_with_publication(
|
||||
cycle,
|
||||
remote.clone(),
|
||||
Arc::new(ScannerUsagePublicationObservation::new(8, 41)),
|
||||
)
|
||||
.await;
|
||||
metrics.record_scanner_leader_liveness("acquired", true, "").await;
|
||||
remote.record(814, 1);
|
||||
let active = metrics.scanner_node_observation().await;
|
||||
assert!(active.active && active.leader);
|
||||
assert_eq!((active.epoch, active.cycle, active.objects_scanned), (Some(8), 41, 814));
|
||||
assert_eq!(active.last_attempt.expect("prior cycle").cycle, 40);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scan_cycle_activity_and_cycle_state_publish_together() {
|
||||
let metrics = Metrics::new();
|
||||
|
||||
@@ -73,6 +73,7 @@ hotpath-cpu = [
|
||||
]
|
||||
|
||||
[dependencies]
|
||||
tempfile.workspace = true
|
||||
hotpath.workspace = true
|
||||
rustfs-config = { workspace = true, features = ["server-config-model"] }
|
||||
rustfs-common = { workspace = true }
|
||||
@@ -108,6 +109,13 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] }
|
||||
bytes.workspace = true
|
||||
hex-simd.workspace = true
|
||||
|
||||
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
|
||||
rustix = { workspace = true, features = ["fs"] }
|
||||
|
||||
[target.'cfg(windows)'.dependencies]
|
||||
same-file.workspace = true
|
||||
winapi-util.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
opentelemetry-proto.workspace = true
|
||||
opentelemetry_sdk.workspace = true
|
||||
@@ -118,7 +126,6 @@ rustfs-test-utils.workspace = true
|
||||
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
|
||||
serial_test = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
|
||||
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
|
||||
# Test-only: pins the emitted scanner alert wire names against the canonical
|
||||
|
||||
@@ -199,10 +199,25 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[derive(Clone, Debug, Eq)]
|
||||
pub(crate) struct DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision,
|
||||
backup: Option<DataUsageCacheRevision>,
|
||||
damaged_pages: DamagedPages,
|
||||
}
|
||||
|
||||
// Admissions compare CAS tokens. A reader's local repair hint does not
|
||||
// indicate that either committed object changed while its locks were released.
|
||||
impl PartialEq for DataUsageCacheRevisions {
|
||||
fn eq(&self, other: &Self) -> bool {
|
||||
self.main == other.main && self.backup == other.backup
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
|
||||
struct DamagedPages {
|
||||
main: bool,
|
||||
backup: bool,
|
||||
}
|
||||
|
||||
/// In-memory provenance of the decoded cache and the revisions observed by its loader.
|
||||
@@ -601,10 +616,31 @@ pub(crate) struct DataUsageDurableProgress {
|
||||
identity: Option<DataUsageScanIdentity>,
|
||||
started_plan: Option<DataUsageScanPlanDigest>,
|
||||
frontier: Option<String>,
|
||||
dormant: HashMap<String, String>,
|
||||
raw: HashMap<String, crate::raw_page_index::RawEnumerationDurableProgress>,
|
||||
compacted: Vec<crate::scanner_folder::DirectoryProgress>,
|
||||
listings: Vec<crate::scanner_folder::DirectoryReadProgress>,
|
||||
active_source: Option<String>,
|
||||
}
|
||||
|
||||
impl DataUsageDurableProgress {
|
||||
fn source_frontier(&self, source: Option<&str>) -> Option<&str> {
|
||||
if source == self.active_source.as_deref() {
|
||||
self.frontier.as_deref()
|
||||
} else {
|
||||
source.and_then(|source| self.dormant.get(source)).map(String::as_str)
|
||||
}
|
||||
}
|
||||
|
||||
fn covers_on(&self, source: Option<&str>, path: &str) -> bool {
|
||||
self.source_frontier(source)
|
||||
.is_some_and(|frontier| scan_frontier_covers_path(path, frontier))
|
||||
|| self
|
||||
.compacted
|
||||
.iter()
|
||||
.any(|done| Some(done.physical.endpoint.as_str()) == source && done.covers(path))
|
||||
}
|
||||
|
||||
pub(crate) fn advanced_from(&self, previous: &Self) -> bool {
|
||||
if self.bucket != previous.bucket
|
||||
|| self.source.is_none()
|
||||
@@ -616,31 +652,69 @@ impl DataUsageDurableProgress {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut advanced = match (&previous.frontier, &self.frontier) {
|
||||
(Some(before), Some(after)) => {
|
||||
if !scan_frontier_covers_path(before, after) {
|
||||
return false;
|
||||
let active = self.active_source.as_deref();
|
||||
let old = previous.active_source.as_deref();
|
||||
let switched = old.is_some() && active != old;
|
||||
// A frontier can move from active to dormant without earning a turn.
|
||||
// Retention and forward movement are compared only on the same source.
|
||||
if previous.frontier.as_deref().is_some_and(|path| !self.covers_on(old, path))
|
||||
|| previous
|
||||
.dormant
|
||||
.iter()
|
||||
.any(|(source, path)| !self.covers_on(Some(source), path))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut advanced = self.frontier.as_deref().is_some_and(|now| {
|
||||
previous
|
||||
.source_frontier(active)
|
||||
.is_none_or(|before| before != now && scan_frontier_covers_path(before, now))
|
||||
});
|
||||
if switched {
|
||||
// Legacy raw-name pages are local to their writer. The physical
|
||||
// root position must still exist for that writer across rotation.
|
||||
if !previous.raw.is_empty()
|
||||
&& !previous.listings.iter().any(|before| {
|
||||
before.parent == self.bucket
|
||||
&& Some(before.physical.endpoint.as_str()) == old
|
||||
&& self.listings.iter().any(|after| after.extends(before))
|
||||
})
|
||||
{
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
for (parent, before) in &previous.raw {
|
||||
match self.raw.get(parent) {
|
||||
Some(after) if after.extends(before) => {}
|
||||
None if self.covers_on(active, parent) => {}
|
||||
_ => return false,
|
||||
}
|
||||
before != after
|
||||
}
|
||||
(Some(_), None) => return false,
|
||||
(None, Some(_)) => true,
|
||||
(None, None) => false,
|
||||
};
|
||||
for (parent, before) in &previous.raw {
|
||||
match self.raw.get(parent) {
|
||||
Some(after) if after.extends(before) => {}
|
||||
None if self
|
||||
.frontier
|
||||
.as_ref()
|
||||
.is_some_and(|frontier| scan_frontier_covers_path(parent, frontier)) => {}
|
||||
_ => return false,
|
||||
advanced |= self.raw.iter().any(|(parent, after)| {
|
||||
after.committed_entries() > previous.raw.get(parent).map_or(0, |before| before.committed_entries())
|
||||
});
|
||||
}
|
||||
for before in &previous.listings {
|
||||
if !self.listings.iter().any(|after| after.extends(before))
|
||||
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
for (parent, after) in &self.raw {
|
||||
let before = previous.raw.get(parent).map_or(0, |value| value.committed_entries());
|
||||
advanced |= after.committed_entries() > before;
|
||||
for before in &previous.compacted {
|
||||
if !self.compacted.iter().any(|after| after.extends(before))
|
||||
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
advanced |= self.listings.iter().any(|after| {
|
||||
Some(after.physical.endpoint.as_str()) == active && !previous.listings.iter().any(|before| before.extends(after))
|
||||
});
|
||||
advanced |= self.compacted.iter().any(|after| {
|
||||
Some(after.physical.endpoint.as_str()) == active && !previous.compacted.iter().any(|before| before.extends(after))
|
||||
});
|
||||
|
||||
advanced
|
||||
}
|
||||
}
|
||||
@@ -929,6 +1003,14 @@ pub struct DataUsageCacheInfo {
|
||||
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
|
||||
/// Source-bound native listing positions and fully committed compacted work.
|
||||
#[serde(default)]
|
||||
pub scan_directory_work: Vec<crate::scanner_folder::DirectoryCheckpoint>,
|
||||
/// Physical source of the regular linked tree and its frontier.
|
||||
#[serde(default)]
|
||||
pub scan_directory_source: Option<String>,
|
||||
#[serde(default)]
|
||||
pub scan_source_trees: std::collections::BTreeMap<String, Arc<source_tree::SourceTree>>,
|
||||
/// Indexes of the other directories on the active walk path. A restart
|
||||
/// must replay every wide ancestor before it reaches the selected one, and
|
||||
/// recharging those replays would stop a fixed budget from ever advancing.
|
||||
@@ -1018,6 +1100,9 @@ impl Serialize for DataUsageCacheInfo {
|
||||
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
|
||||
+ usize::from(self.scan_raw_enumeration_page_index.is_some())
|
||||
+ usize::from(!self.scan_raw_enumeration_path_page_indexes.is_empty())
|
||||
+ usize::from(!self.scan_directory_work.is_empty())
|
||||
+ usize::from(self.scan_directory_source.is_some())
|
||||
+ usize::from(!self.scan_source_trees.is_empty())
|
||||
+ usize::from(self.scan_identity.is_some())
|
||||
+ usize::from(self.scan_progress.is_some())
|
||||
+ usize::from(self.scan_usage_watch.is_some())
|
||||
@@ -1053,6 +1138,15 @@ impl Serialize for DataUsageCacheInfo {
|
||||
if let Some(index) = &self.scan_raw_enumeration_page_index {
|
||||
state.serialize_entry("scan_raw_enumeration_page_index", index)?;
|
||||
}
|
||||
if !self.scan_source_trees.is_empty() {
|
||||
state.serialize_entry("scan_source_trees", &self.scan_source_trees)?;
|
||||
}
|
||||
if let Some(source) = &self.scan_directory_source {
|
||||
state.serialize_entry("scan_directory_source", source)?;
|
||||
}
|
||||
if !self.scan_directory_work.is_empty() {
|
||||
state.serialize_entry("scan_directory_work", &self.scan_directory_work)?;
|
||||
}
|
||||
if !self.scan_raw_enumeration_path_page_indexes.is_empty() {
|
||||
state.serialize_entry("scan_raw_enumeration_path_page_indexes", &self.scan_raw_enumeration_path_page_indexes)?;
|
||||
}
|
||||
@@ -1260,6 +1354,25 @@ impl DataUsageCache {
|
||||
identity: self.info.scan_identity,
|
||||
started_plan: self.info.scan_progress.map(|progress| progress.started_plan),
|
||||
frontier: self.validated_scan_frontier().map(str::to_owned),
|
||||
dormant: self
|
||||
.info
|
||||
.scan_source_trees
|
||||
.iter()
|
||||
.filter_map(|(source, tree)| tree.frontier(&self.info).map(|last| (source.clone(), last.to_owned())))
|
||||
.collect(),
|
||||
active_source: self.info.scan_directory_source.clone(),
|
||||
listings: self
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.filter_map(|d| d.read_progress(&self.info))
|
||||
.collect(),
|
||||
compacted: self
|
||||
.info
|
||||
.scan_directory_work
|
||||
.iter()
|
||||
.filter_map(|d| d.progress(&self.info))
|
||||
.collect(),
|
||||
raw: self
|
||||
.info
|
||||
.scan_raw_enumeration_page_index
|
||||
@@ -1273,7 +1386,8 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
pub(crate) fn has_scan_position(&self) -> bool {
|
||||
self.info.scan_resume_after.is_some()
|
||||
!self.info.scan_directory_work.is_empty()
|
||||
|| self.info.scan_resume_after.is_some()
|
||||
|| self.info.scan_checkpoint.is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| self.info.scan_raw_enumeration_page_index.is_some()
|
||||
@@ -1343,7 +1457,8 @@ impl DataUsageCache {
|
||||
&& self.info.scan_identity == Some(identity)
|
||||
&& (self.validated_scan_frontier().is_some()
|
||||
|| self.validated_raw_enumeration_cursor().is_some()
|
||||
|| self.validated_raw_enumeration_page_index().is_some());
|
||||
|| self.validated_raw_enumeration_page_index().is_some()
|
||||
|| self.info.scan_directory_work.iter().any(|d| d.is_bound_to(&self.info)));
|
||||
let handoff_frontier = cross_epoch_checkpoint
|
||||
.then(|| self.validated_scan_frontier().map(clone_cache_value))
|
||||
.flatten();
|
||||
@@ -1428,7 +1543,10 @@ impl DataUsageCache {
|
||||
.take(MAX_DATA_USAGE_CACHE_DEPTH)
|
||||
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
|
||||
.collect();
|
||||
let directory_work = std::mem::take(&mut self.info.scan_directory_work);
|
||||
self.info.scan_directory_work = directory_work.into_iter().filter(|d| d.is_bound_to(&self.info)).collect();
|
||||
let cursor_is_valid = !self.has_scan_position()
|
||||
|| !self.info.scan_directory_work.is_empty()
|
||||
|| self.validated_scan_frontier().is_some()
|
||||
|| self.info.scan_raw_enumeration_cursor.is_some()
|
||||
|| self.info.scan_raw_enumeration_page_index.is_some();
|
||||
@@ -1471,6 +1589,9 @@ impl DataUsageCache {
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
self.info.scan_directory_work.clear();
|
||||
self.info.scan_directory_source = None;
|
||||
self.info.scan_source_trees.clear();
|
||||
self.info.scan_coverage_receipt = None;
|
||||
}
|
||||
// Old readers do not understand coverage sweeps. An absent plan makes
|
||||
@@ -2216,6 +2337,7 @@ impl DataUsageCache {
|
||||
}
|
||||
|
||||
mod persistence;
|
||||
mod source_tree;
|
||||
|
||||
#[derive(Default, Clone)]
|
||||
struct Inner {
|
||||
|
||||
@@ -12,6 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod pages;
|
||||
mod scan_load;
|
||||
|
||||
use super::*;
|
||||
use bytes::Bytes;
|
||||
use std::sync::Arc;
|
||||
@@ -71,6 +74,7 @@ struct DataUsageCacheLoadResult {
|
||||
main_revision: DataUsageCacheRevision,
|
||||
backup_revision: Option<DataUsageCacheRevision>,
|
||||
loaded_from_main: bool,
|
||||
damaged_pages: DamagedPages,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
@@ -119,46 +123,58 @@ impl DataUsageCache {
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
) -> StorageResult<DataUsageCacheBaselineRevisions> {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let loaded = Self::load_cache(store.clone(), name).await?;
|
||||
let backup = match loaded.backup_revision {
|
||||
Some(revision) => Some(revision),
|
||||
None => match read_config_revision(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
debug!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_LOAD_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_CACHE,
|
||||
cache_name = %name,
|
||||
backup_path = %backup_path,
|
||||
state = "backup_revision_unavailable",
|
||||
error = %err,
|
||||
"Scanner cache backup revision lookup failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
},
|
||||
None => Self::backup_revision(store, name).await,
|
||||
};
|
||||
let main = loaded.main_revision;
|
||||
*self = loaded.cache;
|
||||
|
||||
Ok(DataUsageCacheBaselineRevisions {
|
||||
revisions: DataUsageCacheRevisions { main, backup },
|
||||
revisions: DataUsageCacheRevisions {
|
||||
main,
|
||||
backup,
|
||||
damaged_pages: loaded.damaged_pages,
|
||||
},
|
||||
loaded_from_main: loaded.loaded_from_main,
|
||||
})
|
||||
}
|
||||
|
||||
async fn backup_revision<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> Option<DataUsageCacheRevision> {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
match read_config_revision(store, &backup_path).await {
|
||||
Ok(revision) => Some(revision),
|
||||
Err(err) => {
|
||||
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
|
||||
debug!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_LOAD_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_CACHE,
|
||||
cache_name = %name,
|
||||
backup_path = %backup_path,
|
||||
state = "backup_revision_unavailable",
|
||||
error = %err,
|
||||
"Scanner cache backup revision lookup failed"
|
||||
);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
|
||||
let main_path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let main = read_config_revision(store.clone(), &main_path).await?;
|
||||
let backup = Some(read_config_revision(store, &backup_path).await?);
|
||||
Ok(DataUsageCacheRevisions { main, backup })
|
||||
Ok(DataUsageCacheRevisions {
|
||||
main,
|
||||
backup,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn load_cache<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheLoadResult> {
|
||||
@@ -179,6 +195,7 @@ impl DataUsageCache {
|
||||
main_revision,
|
||||
backup_revision: None,
|
||||
loaded_from_main,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
});
|
||||
}
|
||||
DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => {
|
||||
@@ -203,6 +220,11 @@ impl DataUsageCache {
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_attempt = Self::try_load_inner(store.clone(), &backup_name, Duration::from_secs(30)).await?;
|
||||
let backup_revision = backup_attempt.revision();
|
||||
if backup_revision.is_some()
|
||||
&& let Some(DataUsageCacheRevision::Etag(etag)) = &main_revision
|
||||
{
|
||||
scan_load::PendingBackup::finish(&store, name, etag);
|
||||
}
|
||||
let backup_state = match backup_attempt {
|
||||
DataUsageCacheLoadAttempt::Loaded {
|
||||
cache,
|
||||
@@ -217,6 +239,10 @@ impl DataUsageCache {
|
||||
main_revision,
|
||||
backup_revision: Some(backup_revision),
|
||||
loaded_from_main: false,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: false,
|
||||
},
|
||||
});
|
||||
}
|
||||
DataUsageCacheLoadState::Loaded
|
||||
@@ -241,6 +267,10 @@ impl DataUsageCache {
|
||||
.ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?,
|
||||
backup_revision,
|
||||
loaded_from_main: false,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: backup_state == DataUsageCacheLoadState::Corrupt,
|
||||
},
|
||||
});
|
||||
}
|
||||
(DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing)
|
||||
@@ -261,6 +291,10 @@ impl DataUsageCache {
|
||||
.ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?,
|
||||
backup_revision,
|
||||
loaded_from_main: false,
|
||||
damaged_pages: DamagedPages {
|
||||
main: main_state == DataUsageCacheLoadState::Corrupt,
|
||||
backup: backup_state == DataUsageCacheLoadState::Corrupt,
|
||||
},
|
||||
});
|
||||
}
|
||||
_ => {}
|
||||
@@ -316,10 +350,35 @@ impl DataUsageCache {
|
||||
.filter(|etag| !etag.is_empty())
|
||||
.cloned()
|
||||
.map(DataUsageCacheRevision::Etag);
|
||||
if let Some(DataUsageCacheRevision::Etag(etag)) = &revision
|
||||
&& scan_load::PendingBackup::needs_backup(&store, load_name, etag)
|
||||
{
|
||||
return Ok((None, revision, true));
|
||||
}
|
||||
match reader.read_all().await {
|
||||
Ok(data) => match DataUsageCache::unmarshal(&data) {
|
||||
Ok(data) => match Box::pin(pages::decode(store.clone(), load_name, &data)).await {
|
||||
Ok(cache) => Ok((Some(cache), revision, false)),
|
||||
Err(_) => Ok((None, revision, true)),
|
||||
Err(err)
|
||||
if matches!(err, StorageError::Io(ref io) if io.kind() == std::io::ErrorKind::InvalidData)
|
||||
|| matches!(err, StorageError::FileNotFound | StorageError::ObjectNotFound(..))
|
||||
|| (pages::is_manifest(&data) && pages::unreadable_page(&err)) =>
|
||||
{
|
||||
// A concurrent manifest commit can retire the generation
|
||||
// we were reading. Retry the current root before treating
|
||||
// missing pages as corruption or falling back to a backup.
|
||||
if pages::is_manifest(&data) {
|
||||
if Some(read_config_revision(store.clone(), &path).await?) != revision {
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
if !load_name.ends_with(".bkp")
|
||||
&& let Some(DataUsageCacheRevision::Etag(etag)) = &revision
|
||||
{
|
||||
scan_load::PendingBackup::begin(&store, load_name, etag);
|
||||
}
|
||||
}
|
||||
Ok((None, revision, true))
|
||||
}
|
||||
Err(err) => Err(err),
|
||||
},
|
||||
Err(e) => {
|
||||
// Read error
|
||||
@@ -665,20 +724,44 @@ impl DataUsageCache {
|
||||
expected_epoch: Option<u64>,
|
||||
) -> StorageResult<()> {
|
||||
let serialize_started = Instant::now();
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
// Primary, backup, and retries read the same immutable encoded body.
|
||||
let buf = Bytes::from(buf);
|
||||
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(u64::try_from(buf.len()).unwrap_or(u64::MAX));
|
||||
let paged = matches!((revisions, expected_epoch), (Some(_), Some(_))) && pages::eligible(self, name);
|
||||
let encoded = if paged { Some(pages::encode(self)?) } else { None };
|
||||
let inline = if paged {
|
||||
Bytes::new()
|
||||
} else {
|
||||
let mut buf = Vec::new();
|
||||
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
|
||||
Bytes::from(buf)
|
||||
};
|
||||
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(
|
||||
encoded
|
||||
.as_ref()
|
||||
.map_or_else(|| u64::try_from(inline.len()).unwrap_or(u64::MAX), pages::EncodedPages::length),
|
||||
);
|
||||
histogram!("rustfs_scanner_cache_serialize_seconds").record(serialize_started.elapsed().as_secs_f64());
|
||||
let timeout_duration = Self::cache_save_timeout();
|
||||
|
||||
let (primary_buf, primary_retired) = match (&encoded, revisions, expected_epoch) {
|
||||
(Some(encoded), Some(revisions), Some(epoch)) => {
|
||||
Box::pin(pages::prepare(
|
||||
self,
|
||||
store.clone(),
|
||||
name,
|
||||
&revisions.main,
|
||||
encoded,
|
||||
epoch,
|
||||
revisions.damaged_pages.main,
|
||||
))
|
||||
.await?
|
||||
}
|
||||
_ => (inline.clone(), None),
|
||||
};
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let primary_save_started = Instant::now();
|
||||
Self::save_path_with_retry(
|
||||
store.clone(),
|
||||
&path,
|
||||
&buf,
|
||||
&primary_buf,
|
||||
timeout_duration,
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
revisions.map(|revisions| revisions.main.clone()),
|
||||
@@ -686,6 +769,9 @@ impl DataUsageCache {
|
||||
)
|
||||
.await?;
|
||||
|
||||
if let (Some(retired), Some(epoch)) = (primary_retired, expected_epoch) {
|
||||
let _ = pages::reclaim(store.clone(), name, retired, epoch).await;
|
||||
}
|
||||
let backup_name = format!("{name}.bkp");
|
||||
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
|
||||
let backup_timeout_duration = Self::backup_cache_save_timeout(timeout_duration, primary_save_started.elapsed());
|
||||
@@ -693,17 +779,38 @@ impl DataUsageCache {
|
||||
if revisions.is_some() && backup_revision.is_none() {
|
||||
return Ok(());
|
||||
}
|
||||
if let Err(e) = Self::save_path_with_retry(
|
||||
store,
|
||||
&backup_path,
|
||||
&buf,
|
||||
backup_timeout_duration,
|
||||
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
|
||||
backup_revision,
|
||||
expected_epoch,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let save_backup = async {
|
||||
let (backup_buf, retired) = match (&encoded, &backup_revision, expected_epoch) {
|
||||
(Some(encoded), Some(revision), Some(epoch)) => {
|
||||
Box::pin(pages::prepare(
|
||||
self,
|
||||
store.clone(),
|
||||
&backup_name,
|
||||
revision,
|
||||
encoded,
|
||||
epoch,
|
||||
revisions.is_some_and(|revisions| revisions.damaged_pages.backup),
|
||||
))
|
||||
.await?
|
||||
}
|
||||
_ => (inline, None),
|
||||
};
|
||||
Self::save_path_with_retry(
|
||||
store.clone(),
|
||||
&backup_path,
|
||||
&backup_buf,
|
||||
backup_timeout_duration,
|
||||
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
|
||||
backup_revision,
|
||||
expected_epoch,
|
||||
)
|
||||
.await?;
|
||||
if let (Some(bundle), Some(epoch)) = (retired, expected_epoch) {
|
||||
let _ = pages::reclaim(store, &backup_name, bundle, epoch).await;
|
||||
}
|
||||
Ok::<_, StorageError>(())
|
||||
};
|
||||
if let Err(e) = save_backup.await {
|
||||
warn!(
|
||||
target: "rustfs::scanner::data_usage",
|
||||
event = EVENT_SCANNER_CACHE_SAVE_STATE,
|
||||
|
||||
@@ -0,0 +1,579 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
mod restore;
|
||||
mod save;
|
||||
mod staging;
|
||||
|
||||
use super::*;
|
||||
use futures::future::try_join_all;
|
||||
use sha2::Digest as _;
|
||||
use std::collections::{BTreeMap, HashSet};
|
||||
use std::io::Write;
|
||||
use tokio::io::AsyncReadExt;
|
||||
|
||||
const PAGE_MIN: usize = 8 * 1024;
|
||||
const PAGE_MAX: usize = 64 * 1024;
|
||||
const FANOUT: usize = 128;
|
||||
const IO_CONCURRENCY: usize = 4;
|
||||
const PREFIX: &[u8] = b"\x83\xa7pagesV1";
|
||||
type Digest = [u8; 32];
|
||||
|
||||
/// Older readers see the committed compact bucket or an incomplete empty tree.
|
||||
#[derive(Deserialize)]
|
||||
struct PageEnvelope {
|
||||
#[serde(rename = "pagesV1")]
|
||||
header: Header,
|
||||
info: DataUsageCacheInfo,
|
||||
#[serde(rename = "cache")]
|
||||
_cache: serde::de::IgnoredAny,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct Header {
|
||||
body: Payload,
|
||||
retired: Option<uuid::Uuid>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
enum Payload {
|
||||
Inline,
|
||||
Pages(Manifest),
|
||||
}
|
||||
|
||||
impl Header {
|
||||
fn manifest(&self) -> Option<&Manifest> {
|
||||
match &self.body {
|
||||
Payload::Pages(manifest) => Some(manifest),
|
||||
Payload::Inline => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct Manifest {
|
||||
bundle: uuid::Uuid,
|
||||
finished: bool,
|
||||
cycle: u64,
|
||||
length: u64,
|
||||
level: u8,
|
||||
root: Digest,
|
||||
#[serde(default)]
|
||||
bundle_bytes: u64,
|
||||
#[serde(default)]
|
||||
bundle_commits: u16,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Archive {
|
||||
blocks: BTreeMap<Digest, Bytes>,
|
||||
leaves: Vec<Digest>,
|
||||
buffer: Vec<u8>,
|
||||
rolling: u64,
|
||||
length: u64,
|
||||
}
|
||||
|
||||
fn invalid(message: &str) -> Error {
|
||||
Error::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, message.to_owned()))
|
||||
}
|
||||
|
||||
pub(super) fn unreadable_page(error: &StorageError) -> bool {
|
||||
matches!(error, StorageError::FileCorrupt)
|
||||
|| matches!(error, StorageError::Io(io) if matches!(io.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof))
|
||||
}
|
||||
|
||||
fn digest(data: &[u8]) -> Digest {
|
||||
Sha256::digest(data).into()
|
||||
}
|
||||
|
||||
fn block_path(name: &str, bundle: uuid::Uuid, block: &Digest) -> String {
|
||||
let hash = hex_simd::encode_to_string(block, hex_simd::AsciiCase::Lower);
|
||||
format!("{BUCKET_META_PREFIX}/{name}.pages/{bundle}/{hash}")
|
||||
}
|
||||
|
||||
fn bucket_cache_name(name: &str) -> Option<&str> {
|
||||
let (bucket, leaf) = name.split_once('/')?;
|
||||
(!crate::is_reserved_or_invalid_bucket(bucket, true) && leaf == DATA_USAGE_CACHE_NAME).then_some(name)
|
||||
}
|
||||
|
||||
// The high bits of a multiplicative hash of each byte give a rolling
|
||||
// content-defined boundary. Insertion before a sealed run need not shift the
|
||||
// offsets of every following page. Page length is bounded for every input.
|
||||
const fn byte_gear(byte: u8) -> u64 {
|
||||
let mut x = (byte as u64).wrapping_add(0x9e37_79b9_7f4a_7c15);
|
||||
x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
|
||||
x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
|
||||
x ^ (x >> 31)
|
||||
}
|
||||
|
||||
const GEAR: [u64; 256] = {
|
||||
let mut table = [0; 256];
|
||||
let mut byte = 0u8;
|
||||
loop {
|
||||
table[byte as usize] = byte_gear(byte);
|
||||
if byte == u8::MAX {
|
||||
break;
|
||||
}
|
||||
byte += 1;
|
||||
}
|
||||
table
|
||||
};
|
||||
|
||||
impl Archive {
|
||||
fn intern(&mut self, bytes: Vec<u8>) -> Digest {
|
||||
let key = digest(&bytes);
|
||||
self.blocks.entry(key).or_insert_with(|| Bytes::from(bytes));
|
||||
key
|
||||
}
|
||||
|
||||
fn seal(&mut self) {
|
||||
if !self.buffer.is_empty() {
|
||||
let bytes = std::mem::take(&mut self.buffer);
|
||||
let key = self.intern(bytes);
|
||||
self.leaves.push(key);
|
||||
self.rolling = 0;
|
||||
}
|
||||
}
|
||||
|
||||
fn finish(&mut self, bundle: uuid::Uuid) -> StorageResult<Manifest> {
|
||||
self.seal();
|
||||
let mut children = std::mem::take(&mut self.leaves);
|
||||
let mut level = 0;
|
||||
while children.len() > 1 {
|
||||
children = children
|
||||
.chunks(FANOUT)
|
||||
.map(|chunk| {
|
||||
let bytes = chunk.iter().flatten().copied().collect();
|
||||
self.intern(bytes)
|
||||
})
|
||||
.collect();
|
||||
level += 1;
|
||||
}
|
||||
Ok(Manifest {
|
||||
bundle,
|
||||
finished: false,
|
||||
cycle: 0,
|
||||
length: self.length,
|
||||
level,
|
||||
root: children.pop().ok_or_else(|| invalid("empty scanner checkpoint"))?,
|
||||
bundle_bytes: 0,
|
||||
bundle_commits: 0,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for Archive {
|
||||
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
|
||||
self.length = self
|
||||
.length
|
||||
.checked_add(u64::try_from(bytes.len()).map_err(std::io::Error::other)?)
|
||||
.ok_or_else(|| std::io::Error::other("scanner checkpoint is too large"))?;
|
||||
let mut start = 0;
|
||||
for (offset, byte) in bytes.iter().enumerate() {
|
||||
self.rolling = self.rolling.wrapping_shl(1).wrapping_add(GEAR[usize::from(*byte)]);
|
||||
let end = offset + 1;
|
||||
let len = self.buffer.len() + (end - start);
|
||||
if len == PAGE_MAX || (len >= PAGE_MIN && self.rolling & 0x3fff == 0) {
|
||||
self.buffer.extend_from_slice(&bytes[start..end]);
|
||||
self.seal();
|
||||
start = end;
|
||||
}
|
||||
}
|
||||
self.buffer.extend_from_slice(&bytes[start..]);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> std::io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) struct EncodedPages {
|
||||
archive: Archive,
|
||||
root: Manifest,
|
||||
inline: Option<Bytes>,
|
||||
}
|
||||
|
||||
impl EncodedPages {
|
||||
pub(super) fn length(&self) -> u64 {
|
||||
self.archive.length
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn encode(cache: &DataUsageCache) -> StorageResult<EncodedPages> {
|
||||
let mut archive = Archive::default();
|
||||
archive.write_all(&[0x92])?;
|
||||
cache.info.serialize(&mut rmp_serde::Serializer::new(&mut archive))?;
|
||||
archive.seal();
|
||||
let count = u32::try_from(cache.cache.len()).map_err(|_| invalid("too many scanner checkpoint entries"))?;
|
||||
// A wide root's growing child set must not shift every subtree page.
|
||||
let mut entries: Vec<_> = cache.cache.iter().collect();
|
||||
entries.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
let mut header = Vec::new();
|
||||
if count < 16 {
|
||||
header.push(0x80 | u8::try_from(count).map_err(|_| invalid("invalid fixed map length"))?);
|
||||
} else if let Ok(n) = u16::try_from(count) {
|
||||
header.push(0xde);
|
||||
header.extend_from_slice(&n.to_be_bytes());
|
||||
} else {
|
||||
header.push(0xdf);
|
||||
header.extend_from_slice(&count.to_be_bytes());
|
||||
}
|
||||
archive.write_all(&header)?;
|
||||
for (name, entry) in entries {
|
||||
let mut serializer = rmp_serde::Serializer::new(&mut archive);
|
||||
name.serialize(&mut serializer)?;
|
||||
entry.ordered_encoding().serialize(&mut serializer)?;
|
||||
if name == &cache.info.name {
|
||||
archive.seal();
|
||||
}
|
||||
}
|
||||
archive.seal();
|
||||
let inline = if archive.length <= PAGE_MIN as u64 {
|
||||
let mut buf = Vec::new();
|
||||
for key in &archive.leaves {
|
||||
buf.extend_from_slice(
|
||||
archive
|
||||
.blocks
|
||||
.get(key)
|
||||
.ok_or_else(|| invalid("missing encoded scanner page"))?,
|
||||
);
|
||||
}
|
||||
Some(Bytes::from(buf))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let root = archive.finish(uuid::Uuid::nil())?;
|
||||
Ok(EncodedPages { archive, root, inline })
|
||||
}
|
||||
|
||||
pub(super) fn is_manifest(bytes: &[u8]) -> bool {
|
||||
bytes.starts_with(PREFIX)
|
||||
}
|
||||
|
||||
fn read_envelope(bytes: &[u8], bucket: &str) -> StorageResult<PageEnvelope> {
|
||||
let head: PageEnvelope = rmp_serde::from_slice(bytes).map_err(|_| invalid("invalid scanner checkpoint manifest"))?;
|
||||
if head.info.name != bucket
|
||||
|| head.header.retired.is_some_and(|id| id.is_nil())
|
||||
|| head.header.manifest().is_some_and(|manifest| {
|
||||
manifest.bundle.is_nil() || manifest.length == 0 || manifest.level > 9 || head.header.retired == Some(manifest.bundle)
|
||||
})
|
||||
{
|
||||
return Err(invalid("invalid scanner checkpoint identity or size"));
|
||||
}
|
||||
Ok(head)
|
||||
}
|
||||
|
||||
async fn read_block<S: ScannerObjectIO>(store: Arc<S>, name: &str, bundle: uuid::Uuid, key: Digest) -> StorageResult<Bytes> {
|
||||
let path = block_path(name, bundle, &key);
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if reader.object_info.size < 1 || usize::try_from(reader.object_info.size).map_or(true, |size| size > PAGE_MAX) {
|
||||
return Err(invalid("invalid scanner checkpoint page length"));
|
||||
}
|
||||
let bytes = Bytes::from(reader.read_all().await?);
|
||||
if bytes.len() > PAGE_MAX || digest(&bytes) != key {
|
||||
return Err(invalid("scanner checkpoint page checksum mismatch"));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
|
||||
async fn committed_keys<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
manifest: &Manifest,
|
||||
candidate: &Archive,
|
||||
) -> StorageResult<HashSet<Digest>> {
|
||||
let mut keys = vec![manifest.root];
|
||||
let mut known = HashSet::from([manifest.root]);
|
||||
for _ in 0..manifest.level {
|
||||
let parents = std::mem::take(&mut keys);
|
||||
for batch in parents.chunks(IO_CONCURRENCY) {
|
||||
let mut reads = Vec::new();
|
||||
let mut nodes = Vec::with_capacity(batch.len());
|
||||
for &key in batch {
|
||||
// The committed parent authenticates the table digest.
|
||||
// Unchanged tables are already available in the candidate.
|
||||
if let Some(node) = candidate.blocks.get(&key) {
|
||||
nodes.push(node.clone());
|
||||
} else {
|
||||
reads.push(read_block(store.clone(), name, manifest.bundle, key));
|
||||
}
|
||||
}
|
||||
nodes.extend(try_join_all(reads).await?);
|
||||
for node in nodes {
|
||||
if node.is_empty() || node.len() > FANOUT * 32 || node.len() % 32 != 0 {
|
||||
return Err(invalid("invalid scanner checkpoint page table"));
|
||||
}
|
||||
for &key in node.as_chunks::<32>().0 {
|
||||
keys.push(key);
|
||||
known.insert(key);
|
||||
}
|
||||
if u64::try_from(keys.len()).unwrap_or(u64::MAX) > manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3) {
|
||||
return Err(invalid("scanner checkpoint contains excess page references"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(known)
|
||||
}
|
||||
|
||||
pub(super) async fn decode<S: ScannerObjectIO>(store: Arc<S>, load_name: &str, bytes: &[u8]) -> StorageResult<DataUsageCache> {
|
||||
if !is_manifest(bytes) {
|
||||
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid scanner cache"));
|
||||
}
|
||||
let name = load_name.strip_suffix(".bkp").unwrap_or(load_name);
|
||||
let bucket = bucket_cache_name(name)
|
||||
.and_then(|name| name.split_once('/'))
|
||||
.ok_or_else(|| invalid("misplaced scanner checkpoint"))?
|
||||
.0;
|
||||
let head = read_envelope(bytes, bucket)?;
|
||||
let Some(manifest) = head.header.manifest() else {
|
||||
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid inline scanner checkpoint"));
|
||||
};
|
||||
let cache = restore::restore(store, load_name, manifest).await?;
|
||||
if cache.info.name != bucket
|
||||
|| cache.info.snapshot_complete != manifest.finished
|
||||
|| cache.info.next_cycle != manifest.cycle
|
||||
|| cache.info.scan_identity != head.info.scan_identity
|
||||
|| cache.info.source != head.info.source
|
||||
|| cache.info.leader_epoch != head.info.leader_epoch
|
||||
|| cache.info.next_cycle != head.info.next_cycle
|
||||
{
|
||||
return Err(invalid("scanner checkpoint scope changed"));
|
||||
}
|
||||
Ok(cache)
|
||||
}
|
||||
|
||||
async fn previous_header<S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
) -> StorageResult<Option<Header>> {
|
||||
if *revision == DataUsageCacheRevision::Missing {
|
||||
return Ok(None);
|
||||
}
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let mut reader = store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
&path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if reader
|
||||
.object_info
|
||||
.etag
|
||||
.as_ref()
|
||||
.map(|etag| DataUsageCacheRevision::Etag(etag.clone()))
|
||||
.as_ref()
|
||||
!= Some(revision)
|
||||
{
|
||||
return Err(StorageError::PreconditionFailed);
|
||||
}
|
||||
// Header is the first value in the envelope. A large compatibility
|
||||
// projection or legacy cache does not require reading its body here.
|
||||
let mut prefix = Vec::new();
|
||||
(&mut reader).take(1024).read_to_end(&mut prefix).await?;
|
||||
if !is_manifest(&prefix) {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut deserializer = rmp_serde::Deserializer::new(&prefix[PREFIX.len()..]);
|
||||
// A damaged old header cannot participate in reuse or reclamation.
|
||||
// Rebuilding uses the CAS revision obtained by the normal loader.
|
||||
Ok(Header::deserialize(&mut deserializer).ok().filter(|header| {
|
||||
header.retired.is_none_or(|id| !id.is_nil())
|
||||
&& header
|
||||
.manifest()
|
||||
.is_none_or(|m| !m.bundle.is_nil() && m.length != 0 && m.level <= 9 && header.retired != Some(m.bundle))
|
||||
}))
|
||||
}
|
||||
|
||||
fn build_bundle(name: &str, revision: &DataUsageCacheRevision, cache: &DataUsageCache) -> uuid::Uuid {
|
||||
// Before the first root CAS, retries and competing builders populate the
|
||||
// same staging namespace. Individual page names are immutable digests.
|
||||
let mut hash = Sha256::new();
|
||||
hash.update(b"rustfs-scanner-checkpoint-v1\0");
|
||||
hash.update(name.as_bytes());
|
||||
if let Some(identity) = cache.info.scan_identity {
|
||||
hash.update(identity.bucket_incarnation.as_bytes());
|
||||
}
|
||||
if let DataUsageCacheRevision::Etag(etag) = revision {
|
||||
hash.update(etag.as_bytes());
|
||||
}
|
||||
let mut id = [0; 16];
|
||||
id.copy_from_slice(&hash.finalize()[..16]);
|
||||
uuid::Uuid::from_bytes(id)
|
||||
}
|
||||
|
||||
pub(super) fn eligible(cache: &DataUsageCache, name: &str) -> bool {
|
||||
bucket_cache_name(name).is_some()
|
||||
&& cache.info.scan_identity.is_some()
|
||||
&& cache.info.name == name.split('/').next().unwrap_or("")
|
||||
}
|
||||
|
||||
pub(super) async fn prepare<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
cache: &DataUsageCache,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
encoded: &EncodedPages,
|
||||
epoch: u64,
|
||||
damaged: bool,
|
||||
) -> StorageResult<(Bytes, Option<uuid::Uuid>)> {
|
||||
let previous_header = previous_header(store.clone(), name, revision).await?;
|
||||
let previous = previous_header.as_ref().and_then(Header::manifest);
|
||||
let mut retired = previous_header.as_ref().and_then(|header| header.retired);
|
||||
if let Some(id) = retired
|
||||
&& reclaim(store.clone(), name, id, epoch).await.is_ok()
|
||||
{
|
||||
retired = None;
|
||||
}
|
||||
// Each root has its own page namespace and can retire independently of
|
||||
// the other copy after a successful conditional commit.
|
||||
let allow_rotation = retired.is_none();
|
||||
if let Some(inline) = &encoded.inline
|
||||
&& (previous.is_none() || allow_rotation)
|
||||
{
|
||||
retired = retired.or_else(|| previous.map(|head| head.bundle));
|
||||
let bytes = if retired.is_some() {
|
||||
rmp_serde::to_vec_named(&CompatibilityEnvelope {
|
||||
header: &Header {
|
||||
body: Payload::Inline,
|
||||
retired,
|
||||
},
|
||||
info: &cache.info,
|
||||
cache: &cache.cache,
|
||||
})
|
||||
.map_err(|_| invalid("failed to encode inline scanner checkpoint"))?
|
||||
} else {
|
||||
return Ok((inline.clone(), None));
|
||||
};
|
||||
return Ok((Bytes::from(bytes), retired));
|
||||
}
|
||||
let live_bytes = save::block_bytes(encoded.archive.blocks.values());
|
||||
let reuse = match previous {
|
||||
Some(head) if !damaged && !(head.finished && head.cycle != cache.info.next_cycle) => {
|
||||
save::reuse(store.clone(), name, head, &encoded.archive, live_bytes).await?
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
let verify_backup = name.ends_with(".bkp");
|
||||
let (bundle, known, bundle_bytes, bundle_commits) = if let Some((head, known, total)) = reuse {
|
||||
(head.bundle, known, total, head.bundle_commits + 1)
|
||||
} else {
|
||||
if let Some(head) = previous {
|
||||
if !allow_rotation {
|
||||
// Keep both readable generations and their cleanup obligation.
|
||||
// Further staging into either bundle would defeat the space bound.
|
||||
return Err(Error::other("cannot rotate scanner checkpoint before reclaiming its retired bundle"));
|
||||
}
|
||||
retired = Some(head.bundle);
|
||||
}
|
||||
(build_bundle(name, revision, cache), HashSet::new(), live_bytes, 1)
|
||||
};
|
||||
let mut manifest = encoded.root.clone();
|
||||
manifest.bundle = bundle;
|
||||
manifest.finished = cache.info.snapshot_complete;
|
||||
manifest.cycle = cache.info.next_cycle;
|
||||
manifest.bundle_bytes = bundle_bytes;
|
||||
manifest.bundle_commits = bundle_commits;
|
||||
|
||||
if let Err(error) = save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &known).await {
|
||||
// A corrupt XL header need not yield an ETag for a conditional repair.
|
||||
// Rebuild this backup under a fresh namespace; its old root stays visible
|
||||
// until the manifest CAS. Other I/O failures remain retryable in place.
|
||||
if !(verify_backup && manifest.bundle_commits > 1 && allow_rotation && unreadable_page(&error)) {
|
||||
return Err(error);
|
||||
}
|
||||
retired = Some(manifest.bundle);
|
||||
manifest.bundle = build_bundle(name, revision, cache);
|
||||
manifest.bundle_bytes = live_bytes;
|
||||
manifest.bundle_commits = 1;
|
||||
save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &HashSet::new()).await?;
|
||||
}
|
||||
let header = Header {
|
||||
body: Payload::Pages(manifest),
|
||||
retired,
|
||||
};
|
||||
let projection = |info: &DataUsageCacheInfo, cache: &BTreeMap<String, DataUsageEntry>| {
|
||||
rmp_serde::to_vec_named(&CompatibilityEnvelope {
|
||||
header: &header,
|
||||
info,
|
||||
cache,
|
||||
})
|
||||
.map_err(|_| invalid("failed to encode scanner checkpoint root"))
|
||||
};
|
||||
if cache.info.snapshot_complete {
|
||||
// Older readers can serve compact totals when the compatibility
|
||||
// projection fits. The manifest remains bounded even with many
|
||||
// pending repairs or a large lifecycle configuration.
|
||||
let mut root = cache
|
||||
.checked_flatten_complete_scope(&cache.info.name)
|
||||
.ok_or_else(|| invalid("disconnected complete bucket checkpoint"))?;
|
||||
root.compacted = true;
|
||||
let bytes = projection(&cache.info, &BTreeMap::from([(cache.info.name.clone(), root)]))?;
|
||||
if bytes.len() <= PAGE_MIN {
|
||||
return Ok((Bytes::from(bytes), retired));
|
||||
}
|
||||
}
|
||||
let info = DataUsageCacheInfo {
|
||||
name: cache.info.name.clone(),
|
||||
source: cache.info.source,
|
||||
scan_identity: cache.info.scan_identity,
|
||||
leader_epoch: cache.info.leader_epoch,
|
||||
next_cycle: cache.info.next_cycle,
|
||||
..Default::default()
|
||||
};
|
||||
let bytes = projection(&info, &BTreeMap::new())?;
|
||||
Ok((Bytes::from(bytes), retired))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct CompatibilityEnvelope<'a, T> {
|
||||
#[serde(rename = "pagesV1")]
|
||||
header: &'a Header,
|
||||
info: &'a DataUsageCacheInfo,
|
||||
cache: &'a T,
|
||||
}
|
||||
|
||||
pub(super) async fn reclaim<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
) -> StorageResult<()> {
|
||||
let _admission = scanner_publication_admission_for_epoch(store.clone(), epoch)
|
||||
.await
|
||||
.ok_or_else(cache_publication_epoch_changed)?;
|
||||
let prefix = format!("{BUCKET_META_PREFIX}/{name}.pages/{bundle}/");
|
||||
let opts = ObjectOptions {
|
||||
delete_prefix: true,
|
||||
..Default::default()
|
||||
};
|
||||
match timeout(
|
||||
DataUsageCache::cache_save_timeout(),
|
||||
store.delete_config_object(RUSTFS_META_BUCKET, &prefix, opts),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(_)) | Ok(Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..))) => Ok(()),
|
||||
Ok(Err(error)) => Err(error),
|
||||
Err(_) => Err(Error::other("retired scanner checkpoint cleanup timed out")),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,207 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::super::scan_load::park_restore;
|
||||
use super::*;
|
||||
use std::any::Any;
|
||||
use std::fs::File;
|
||||
use std::io::{Seek, SeekFrom};
|
||||
use std::sync::{Mutex, Weak};
|
||||
use tokio::time::Instant as WallInstant;
|
||||
|
||||
const MEMORY_LIMIT: u64 = 1024 * 1024;
|
||||
const PARKED_LIMIT: usize = 32;
|
||||
|
||||
enum Buffer {
|
||||
Memory(Vec<u8>),
|
||||
File(Arc<Mutex<File>>),
|
||||
}
|
||||
|
||||
struct Position {
|
||||
pending: Vec<(Digest, u8)>,
|
||||
length: u64,
|
||||
leaves: u64,
|
||||
nodes: u64,
|
||||
buffer: Buffer,
|
||||
writer: Option<tokio::task::JoinHandle<std::io::Result<()>>>,
|
||||
decoder: Option<tokio::task::JoinHandle<StorageResult<DataUsageCache>>>,
|
||||
}
|
||||
|
||||
struct Suspended {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
manifest: Manifest,
|
||||
last_polled: WallInstant,
|
||||
position: Position,
|
||||
}
|
||||
|
||||
struct Recovery(Option<Suspended>);
|
||||
|
||||
static PARKED: LazyLock<Mutex<Vec<Suspended>>> = LazyLock::new(|| Mutex::new(Vec::new()));
|
||||
|
||||
impl Recovery {
|
||||
async fn open<S: ScannerObjectIO>(store: &Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<Self> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&erased);
|
||||
let resumed = {
|
||||
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
parked.retain(|job| {
|
||||
job.owner.strong_count() != 0 && (!job.owner.ptr_eq(&owner) || job.name != name || job.manifest == *manifest)
|
||||
});
|
||||
parked
|
||||
.iter()
|
||||
.position(|job| job.owner.ptr_eq(&owner) && job.name == name && job.manifest == *manifest)
|
||||
.map(|index| parked.swap_remove(index))
|
||||
};
|
||||
if let Some(mut job) = resumed {
|
||||
job.last_polled = WallInstant::now();
|
||||
return Ok(Self(Some(job)));
|
||||
}
|
||||
let buffer = if manifest.length <= MEMORY_LIMIT {
|
||||
Buffer::Memory(Vec::new())
|
||||
} else {
|
||||
let file = tokio::task::spawn_blocking(tempfile::tempfile)
|
||||
.await
|
||||
.map_err(Error::other)??;
|
||||
Buffer::File(Arc::new(Mutex::new(file)))
|
||||
};
|
||||
Ok(Self(Some(Suspended {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
manifest: manifest.clone(),
|
||||
last_polled: WallInstant::now(),
|
||||
position: Position {
|
||||
pending: vec![(manifest.root, manifest.level)],
|
||||
length: 0,
|
||||
leaves: 0,
|
||||
nodes: 0,
|
||||
buffer,
|
||||
writer: None,
|
||||
decoder: None,
|
||||
},
|
||||
})))
|
||||
}
|
||||
}
|
||||
|
||||
// Park only verified bytes and traversal state. Outstanding metadata reads
|
||||
// and their admissions are dropped with the waiter. A local spool write or
|
||||
// decode may finish after cancellation and is joined before advancing again.
|
||||
impl Drop for Recovery {
|
||||
fn drop(&mut self) {
|
||||
let Some(mut job) = self.0.take() else { return };
|
||||
if job.position.nodes == 0 && job.position.leaves == 0 {
|
||||
return;
|
||||
}
|
||||
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
job.last_polled = WallInstant::now();
|
||||
if let Some(previous) = parked
|
||||
.iter_mut()
|
||||
.find(|previous| previous.owner.ptr_eq(&job.owner) && previous.name == job.name && previous.manifest == job.manifest)
|
||||
{
|
||||
// Readers of an immutable root traverse the same ordered prefix.
|
||||
// Keep the furthest admission regardless of cancellation order.
|
||||
if job.position.verified_progress() > previous.position.verified_progress() {
|
||||
std::mem::swap(previous, &mut job);
|
||||
}
|
||||
} else {
|
||||
let displaced = park_restore(&mut parked, PARKED_LIMIT, job, |job| job.last_polled);
|
||||
drop(parked);
|
||||
drop(displaced);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Position {
|
||||
fn verified_progress(&self) -> (u64, bool) {
|
||||
(self.nodes + self.leaves, self.decoder.is_some())
|
||||
}
|
||||
|
||||
async fn restore<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
manifest: &Manifest,
|
||||
) -> StorageResult<DataUsageCache> {
|
||||
let max_leaves = manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3);
|
||||
loop {
|
||||
// Retain the local write across cancellation. The verified page already
|
||||
// owns its offset and must not consume another storage read next round.
|
||||
if let Some(writer) = &mut self.writer {
|
||||
writer.await.map_err(Error::other)??;
|
||||
self.writer = None;
|
||||
}
|
||||
let Some(&(key, level)) = self.pending.last() else { break };
|
||||
if level != 0 {
|
||||
let node = read_block(store.clone(), name, manifest.bundle, key).await?;
|
||||
if node.is_empty()
|
||||
|| node.len() > FANOUT * 32
|
||||
|| node.len() % 32 != 0
|
||||
|| self.nodes >= max_leaves.saturating_mul(u64::from(manifest.level))
|
||||
{
|
||||
return Err(invalid("invalid scanner checkpoint restore table"));
|
||||
}
|
||||
self.pending.pop();
|
||||
for &child in node.as_chunks::<32>().0.iter().rev() {
|
||||
self.pending.push((child, level - 1));
|
||||
}
|
||||
self.nodes += 1;
|
||||
continue;
|
||||
}
|
||||
let block = read_block(store.clone(), name, manifest.bundle, key).await?;
|
||||
let next = self
|
||||
.length
|
||||
.checked_add(u64::try_from(block.len()).map_err(|_| invalid("invalid restore length"))?)
|
||||
.filter(|len| *len <= manifest.length)
|
||||
.ok_or_else(|| invalid("scanner checkpoint exceeds its committed length"))?;
|
||||
if self.leaves >= max_leaves {
|
||||
return Err(invalid("too many scanner checkpoint leaves"));
|
||||
}
|
||||
match &mut self.buffer {
|
||||
Buffer::Memory(bytes) => bytes.extend_from_slice(&block),
|
||||
Buffer::File(file) => {
|
||||
let file = file.clone();
|
||||
let offset = self.length;
|
||||
self.writer = Some(tokio::task::spawn_blocking(move || {
|
||||
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
|
||||
file.seek(SeekFrom::Start(offset))?;
|
||||
file.write_all(&block)
|
||||
}));
|
||||
}
|
||||
}
|
||||
self.pending.pop();
|
||||
self.length = next;
|
||||
self.leaves += 1;
|
||||
}
|
||||
if self.length != manifest.length {
|
||||
return Err(invalid("incomplete scanner checkpoint"));
|
||||
}
|
||||
let job = self.decoder.get_or_insert_with(|| {
|
||||
let buffer = std::mem::replace(&mut self.buffer, Buffer::Memory(Vec::new()));
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let result = match buffer {
|
||||
Buffer::Memory(bytes) => rmp_serde::from_slice(&bytes),
|
||||
Buffer::File(file) => {
|
||||
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
|
||||
file.seek(SeekFrom::Start(0))?;
|
||||
rmp_serde::from_read(std::io::BufReader::new(&mut *file))
|
||||
}
|
||||
};
|
||||
result.map_err(|_| invalid("invalid scanner checkpoint payload"))
|
||||
})
|
||||
});
|
||||
job.await.map_err(Error::other)?
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn restore<S: ScannerObjectIO>(store: Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<DataUsageCache> {
|
||||
let mut job = Recovery::open(&store, name, manifest).await?;
|
||||
let result = job
|
||||
.0
|
||||
.as_mut()
|
||||
.ok_or_else(|| invalid("missing checkpoint restore state"))?
|
||||
.position
|
||||
.restore(store, name, manifest)
|
||||
.await;
|
||||
job.0.take();
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
// A losing writer can stage blocks without publishing its accounting. Even
|
||||
// tiny or identical successful checkpoints periodically retire these orphans.
|
||||
const MAX_BUNDLE_COMMITS: u16 = 256;
|
||||
|
||||
pub(super) fn block_bytes<'a>(blocks: impl Iterator<Item = &'a Bytes>) -> u64 {
|
||||
blocks.fold(0u64, |sum, bytes| sum.saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX)))
|
||||
}
|
||||
|
||||
pub(super) async fn reuse<'a, S: ScannerObjectIO>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
head: &'a Manifest,
|
||||
candidate: &Archive,
|
||||
live: u64,
|
||||
) -> StorageResult<Option<(&'a Manifest, HashSet<Digest>, u64)>> {
|
||||
// Legacy manifests did not account for unreachable blocks in this bundle.
|
||||
if head.bundle_bytes == 0 || head.bundle_commits == 0 || head.bundle_commits >= MAX_BUNDLE_COMMITS {
|
||||
return Ok(None);
|
||||
}
|
||||
let known = match committed_keys(store, name, head, candidate).await {
|
||||
Ok(known) => known,
|
||||
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(None),
|
||||
Err(error) if unreadable_page(&error) => return Ok(None),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
// Count reachable allocation once at bundle creation and newly staged
|
||||
// bytes thereafter. A small delta stays incremental even across cycles.
|
||||
let added = block_bytes(
|
||||
candidate
|
||||
.blocks
|
||||
.iter()
|
||||
.filter_map(|(key, bytes)| (!known.contains(key)).then_some(bytes)),
|
||||
);
|
||||
let total = head.bundle_bytes.checked_add(added);
|
||||
Ok(total
|
||||
.filter(|&total| total <= live.saturating_mul(2))
|
||||
.map(|total| (head, known, total)))
|
||||
}
|
||||
|
||||
/// Returns the conditional write necessary to restore this content-addressed
|
||||
/// backup page, or None when the bytes in storage already match the candidate.
|
||||
async fn backup_revision<S: ScannerObjectIO>(
|
||||
store: &S,
|
||||
path: &str,
|
||||
expected: &Bytes,
|
||||
) -> StorageResult<Option<DataUsageCacheRevision>> {
|
||||
let mut reader = match store
|
||||
.get_object_reader(
|
||||
RUSTFS_META_BUCKET,
|
||||
path,
|
||||
None,
|
||||
HeaderMap::new(),
|
||||
&ObjectOptions {
|
||||
no_lock: true,
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(reader) => reader,
|
||||
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(Some(DataUsageCacheRevision::Missing)),
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
if usize::try_from(reader.object_info.size).ok() == Some(expected.len()) {
|
||||
let mut bytes = Vec::with_capacity(expected.len());
|
||||
(&mut reader).take(PAGE_MAX as u64 + 1).read_to_end(&mut bytes).await?;
|
||||
if bytes == expected.as_ref() {
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
let etag = reader
|
||||
.object_info
|
||||
.etag
|
||||
.ok_or_else(|| invalid("damaged scanner backup page has no revision"))?;
|
||||
Ok(Some(DataUsageCacheRevision::Etag(etag)))
|
||||
}
|
||||
|
||||
pub(super) async fn stage<S: ScannerObjectIO + ScannerConfigObjectDelete>(
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
revision: &DataUsageCacheRevision,
|
||||
epoch: u64,
|
||||
manifest: &Manifest,
|
||||
archive: &Archive,
|
||||
known: &HashSet<Digest>,
|
||||
) -> StorageResult<()> {
|
||||
let verify_backup = name.ends_with(".bkp");
|
||||
let staging = staging::Staging::open(&store, name, manifest.bundle, epoch, revision);
|
||||
let mut writes = Vec::with_capacity(IO_CONCURRENCY);
|
||||
for (key, bytes) in &archive.blocks {
|
||||
let staged = staging.contains(key);
|
||||
if !verify_backup && (known.contains(key) || staged) {
|
||||
continue;
|
||||
}
|
||||
let path = block_path(name, manifest.bundle, key);
|
||||
let store = store.clone();
|
||||
let bytes = bytes.clone();
|
||||
let key = *key;
|
||||
let receipts = &staging;
|
||||
writes.push(async move {
|
||||
let needed = if verify_backup && (manifest.bundle_commits > 1 || staged) {
|
||||
// A healthy primary load does not establish the integrity of its
|
||||
// independent backup. Check reused bytes and repair only damaged pages.
|
||||
match timeout(DataUsageCache::cache_save_timeout(), backup_revision(store.as_ref(), &path, &bytes))
|
||||
.await
|
||||
.map_err(|_| Error::other("scanner backup page validation timed out"))??
|
||||
{
|
||||
Some(revision) => revision,
|
||||
None => return Ok(()),
|
||||
}
|
||||
} else {
|
||||
DataUsageCacheRevision::Missing
|
||||
};
|
||||
DataUsageCache::save_path_with_retry(
|
||||
store,
|
||||
&path,
|
||||
&bytes,
|
||||
DataUsageCache::cache_save_timeout(),
|
||||
DATA_USAGE_CACHE_SAVE_RETRIES,
|
||||
Some(needed),
|
||||
Some(epoch),
|
||||
)
|
||||
.await?;
|
||||
receipts.record(key);
|
||||
Ok::<(), Error>(())
|
||||
});
|
||||
if writes.len() == IO_CONCURRENCY {
|
||||
try_join_all(std::mem::take(&mut writes)).await?;
|
||||
}
|
||||
}
|
||||
try_join_all(writes).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use std::any::Any;
|
||||
use std::sync::{Mutex, Weak};
|
||||
use std::time::Instant as WallInstant;
|
||||
|
||||
/// A manifest commit can be interrupted after any group of immutable PUTs.
|
||||
/// Reuse only receipts from the same store, epoch and predecessor revision.
|
||||
struct Attempt {
|
||||
store: Weak<dyn Any + Send + Sync>,
|
||||
path: String,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
revision: DataUsageCacheRevision,
|
||||
seen: Arc<Mutex<HashSet<Digest>>>,
|
||||
last_used: WallInstant,
|
||||
}
|
||||
|
||||
const MAX_ATTEMPTS: usize = 32;
|
||||
static ATTEMPTS: LazyLock<Mutex<Vec<Attempt>>> = LazyLock::new(|| Mutex::new(Vec::new()));
|
||||
|
||||
pub(super) struct Staging(Arc<Mutex<HashSet<Digest>>>);
|
||||
|
||||
impl Staging {
|
||||
pub(super) fn open<S: ScannerObjectIO>(
|
||||
store: &Arc<S>,
|
||||
path: &str,
|
||||
bundle: uuid::Uuid,
|
||||
epoch: u64,
|
||||
revision: &DataUsageCacheRevision,
|
||||
) -> Self {
|
||||
let store: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&store);
|
||||
let mut attempts = ATTEMPTS.lock().unwrap_or_else(|p| p.into_inner());
|
||||
attempts.retain(|job| job.store.strong_count() != 0 && job.last_used.elapsed() < Duration::from_secs(3600));
|
||||
if let Some(attempt) = attempts.iter_mut().find(|attempt| {
|
||||
attempt.store.ptr_eq(&owner)
|
||||
&& attempt.path == path
|
||||
&& attempt.bundle == bundle
|
||||
&& attempt.epoch == epoch
|
||||
&& &attempt.revision == revision
|
||||
}) {
|
||||
attempt.last_used = WallInstant::now();
|
||||
return Self(attempt.seen.clone());
|
||||
}
|
||||
// A successfully committed predecessor naturally retires the receipts
|
||||
// for its previous ETag. Concurrent older writers retain their own Arc.
|
||||
attempts.retain(|attempt| !attempt.store.ptr_eq(&owner) || attempt.path != path);
|
||||
if attempts.len() >= MAX_ATTEMPTS
|
||||
&& let Some((index, _)) = attempts.iter().enumerate().min_by_key(|(_, job)| job.last_used)
|
||||
{
|
||||
attempts.swap_remove(index);
|
||||
}
|
||||
let seen = Arc::new(Mutex::new(HashSet::new()));
|
||||
attempts.push(Attempt {
|
||||
store: owner,
|
||||
path: path.to_owned(),
|
||||
bundle,
|
||||
epoch,
|
||||
revision: revision.clone(),
|
||||
seen: seen.clone(),
|
||||
last_used: WallInstant::now(),
|
||||
});
|
||||
Self(seen)
|
||||
}
|
||||
|
||||
pub(super) fn contains(&self, key: &Digest) -> bool {
|
||||
self.0.lock().unwrap_or_else(|p| p.into_inner()).contains(key)
|
||||
}
|
||||
|
||||
pub(super) fn record(&self, key: Digest) {
|
||||
self.0.lock().unwrap_or_else(|p| p.into_inner()).insert(key);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
use std::any::Any;
|
||||
use std::sync::{Mutex, Weak};
|
||||
use tokio_util::sync::CancellationToken;
|
||||
|
||||
// Keep only a failed root's identity while its independent backup is being
|
||||
// recovered. A fresh primary reader still obtains the current ETag every time.
|
||||
pub(super) struct PendingBackup {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
failed_etag: String,
|
||||
last_polled: tokio::time::Instant,
|
||||
}
|
||||
|
||||
static PENDING_BACKUPS: Mutex<Vec<PendingBackup>> = Mutex::new(Vec::new());
|
||||
const MAX_PENDING_BACKUPS: usize = 32;
|
||||
|
||||
// Age permits reclamation under contention. A scanner whose configured
|
||||
// interval exceeds this age can still resume while slots remain available.
|
||||
pub(super) fn park_restore<T>(
|
||||
slots: &mut Vec<T>,
|
||||
limit: usize,
|
||||
value: T,
|
||||
last_polled: impl Fn(&T) -> tokio::time::Instant,
|
||||
) -> Option<T> {
|
||||
if slots.len() < limit {
|
||||
slots.push(value);
|
||||
return None;
|
||||
}
|
||||
if let Some(oldest) = slots.iter_mut().min_by_key(|slot| last_polled(slot))
|
||||
&& last_polled(oldest).elapsed() >= Duration::from_secs(3600)
|
||||
{
|
||||
return Some(std::mem::replace(oldest, value));
|
||||
}
|
||||
Some(value)
|
||||
}
|
||||
|
||||
impl PendingBackup {
|
||||
fn owner<S: ScannerObjectIO>(store: &Arc<S>) -> Weak<dyn Any + Send + Sync> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
Arc::downgrade(&erased)
|
||||
}
|
||||
|
||||
pub(super) fn needs_backup<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) -> bool {
|
||||
let owner = Self::owner(store);
|
||||
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
|
||||
let Some(index) = pending
|
||||
.iter()
|
||||
.position(|phase| phase.owner.ptr_eq(&owner) && phase.name == name)
|
||||
else {
|
||||
return false;
|
||||
};
|
||||
if pending[index].failed_etag == etag {
|
||||
pending[index].last_polled = tokio::time::Instant::now();
|
||||
true
|
||||
} else {
|
||||
pending.swap_remove(index);
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn begin<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
|
||||
let owner = Self::owner(store);
|
||||
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
|
||||
pending.retain(|phase| phase.owner.strong_count() > 0 && !(phase.owner.ptr_eq(&owner) && phase.name == name));
|
||||
let phase = Self {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
failed_etag: etag.to_owned(),
|
||||
last_polled: tokio::time::Instant::now(),
|
||||
};
|
||||
let _ = park_restore(&mut pending, MAX_PENDING_BACKUPS, phase, |phase| phase.last_polled);
|
||||
}
|
||||
|
||||
pub(super) fn finish<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
|
||||
let owner = Self::owner(store);
|
||||
PENDING_BACKUPS
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.retain(|phase| !(phase.owner.ptr_eq(&owner) && phase.name == name && phase.failed_etag == etag));
|
||||
}
|
||||
}
|
||||
|
||||
struct DecodedCheckpoint {
|
||||
owner: Weak<dyn Any + Send + Sync>,
|
||||
name: String,
|
||||
parked_at: std::time::Instant,
|
||||
cache: DataUsageCache,
|
||||
revisions: DataUsageCacheRevisions,
|
||||
}
|
||||
|
||||
// The scanner may finish a cold restore with too little time for the walk.
|
||||
// Move that exact value to the next admission. Only one decoded bucket can
|
||||
// occupy the handoff and it never owns storage, a permit, or a publication fence.
|
||||
static DECODED: Mutex<Option<DecodedCheckpoint>> = Mutex::new(None);
|
||||
|
||||
impl DataUsageCache {
|
||||
pub(crate) async fn load_for_scan<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
ctx: &CancellationToken,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> StorageResult<Option<DataUsageCacheRevisions>> {
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = ctx.cancelled() => Ok(None),
|
||||
result = self.load_for_scan_window(store, name, budget) => result,
|
||||
}
|
||||
}
|
||||
|
||||
async fn load_for_scan_window<S: ScannerObjectIO>(
|
||||
&mut self,
|
||||
store: Arc<S>,
|
||||
name: &str,
|
||||
budget: &ScannerCycleBudget,
|
||||
) -> StorageResult<Option<DataUsageCacheRevisions>> {
|
||||
let erased: Arc<dyn Any + Send + Sync> = store.clone();
|
||||
let owner = Arc::downgrade(&erased);
|
||||
let is_ours = |ready: &DecodedCheckpoint| ready.owner.ptr_eq(&owner) && ready.name == name;
|
||||
let cached = {
|
||||
let mut decoded = DECODED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
// Give a retired or deleted bucket's slot back to other active
|
||||
// readers. A matching admission can consume it even after a long
|
||||
// scheduled interval, subject to the revision checks below.
|
||||
decoded.take_if(|ready| {
|
||||
ready.owner.strong_count() == 0 || (!is_ours(ready) && ready.parked_at.elapsed() >= Duration::from_secs(3600))
|
||||
});
|
||||
decoded.as_ref().is_some_and(is_ours)
|
||||
};
|
||||
if cached {
|
||||
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
|
||||
let main = read_config_revision(store.clone(), &path).await;
|
||||
let backup = Self::backup_revision(store.clone(), name).await;
|
||||
let ready = DECODED
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.take_if(|ready| is_ours(ready));
|
||||
if let (Some(ready), Ok(main)) = (ready, main)
|
||||
&& ready.revisions.main == main
|
||||
&& ready.revisions.backup == backup
|
||||
{
|
||||
*self = ready.cache;
|
||||
return Ok(Some(ready.revisions));
|
||||
}
|
||||
}
|
||||
|
||||
let started = tokio::time::Instant::now();
|
||||
// Obtain the optional backup head before the expensive decode. A budget
|
||||
// stop after the decoder finishes must not discard the materialized tree
|
||||
// while waiting for an unrelated backup lookup.
|
||||
let backup = Self::backup_revision(store.clone(), name).await;
|
||||
let loaded = Box::pin(Self::load_cache(store, name)).await?;
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: loaded.main_revision,
|
||||
backup: loaded.backup_revision.or(backup),
|
||||
damaged_pages: loaded.damaged_pages,
|
||||
};
|
||||
*self = loaded.cache;
|
||||
if self.info.scan_identity.is_some()
|
||||
&& !self.cache.is_empty()
|
||||
&& budget
|
||||
.remaining_config()
|
||||
.max_duration
|
||||
.is_some_and(|remaining| started.elapsed() > remaining)
|
||||
{
|
||||
let mut slot = DECODED.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if slot.as_ref().is_none_or(|ready| ready.owner.strong_count() == 0) {
|
||||
*slot = Some(DecodedCheckpoint {
|
||||
owner,
|
||||
name: name.to_owned(),
|
||||
parked_at: std::time::Instant::now(),
|
||||
cache: std::mem::take(self),
|
||||
revisions,
|
||||
});
|
||||
return Ok(None);
|
||||
}
|
||||
}
|
||||
Ok(Some(revisions))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn only_reclaim_idle_restore_slots_under_pressure() {
|
||||
let first = tokio::time::Instant::now();
|
||||
tokio::time::advance(Duration::from_secs(7200)).await;
|
||||
let second = tokio::time::Instant::now();
|
||||
let mut slots = vec![first];
|
||||
assert_eq!(park_restore(&mut slots, 2, second, |&time| time), None);
|
||||
assert_eq!(slots, [first, second]);
|
||||
let new = second + Duration::from_secs(1);
|
||||
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(first));
|
||||
assert_eq!(slots, [new, second]);
|
||||
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(new));
|
||||
assert_eq!(slots, [new, second], "active recoveries retain both slots");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
/// A linked tree paused on a different physical replica. It contains only
|
||||
/// traversal fields, so dormant sources cannot recursively contain each other.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct SourceTree {
|
||||
bucket: String,
|
||||
identity: DataUsageScanIdentity,
|
||||
started_plan: DataUsageScanPlanDigest,
|
||||
#[serde(serialize_with = "ordered_entries")]
|
||||
entries: HashMap<String, DataUsageEntry>,
|
||||
resume_after: Option<String>,
|
||||
checkpoint: Option<DataUsageScanCheckpoint>,
|
||||
coverage: Option<DataUsageScanCoverageReceipt>,
|
||||
cursor: Option<DataUsageRawEnumerationCursor>,
|
||||
index: Option<RawEnumerationPageIndex>,
|
||||
ancestors: Vec<RawEnumerationPageIndex>,
|
||||
}
|
||||
|
||||
impl SourceTree {
|
||||
/// An inactive receipt may preserve an existing source position. Only an
|
||||
/// active, verified traversal can contribute new progress.
|
||||
pub(super) fn frontier(&self, info: &DataUsageCacheInfo) -> Option<&str> {
|
||||
if self.bucket != info.name
|
||||
|| Some(self.identity) != info.scan_identity
|
||||
|| Some(self.started_plan) != info.scan_progress.map(|p| p.started_plan)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.coverage.as_ref().map(|receipt| receipt.through.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
// Dormant trees live in the checkpoint header stream. A cold process has
|
||||
// fresh HashMap seeds; preserve stable bytes for immutable Merkle pages.
|
||||
fn ordered_entries<S: serde::Serializer>(
|
||||
entries: &HashMap<String, DataUsageEntry>,
|
||||
serializer: S,
|
||||
) -> std::result::Result<S::Ok, S::Error> {
|
||||
let mut sorted = entries.iter().collect::<Vec<_>>();
|
||||
sorted.sort_unstable_by(|a, b| a.0.cmp(b.0));
|
||||
serializer.collect_map(sorted.into_iter().map(|(name, entry)| (name, entry.ordered_encoding())))
|
||||
}
|
||||
|
||||
impl DataUsageCache {
|
||||
pub(crate) fn activate_directory_source(&mut self, source: String, endpoints: &HashSet<String>) {
|
||||
let (Some(identity), Some(progress)) = (self.info.scan_identity, self.info.scan_progress) else {
|
||||
return;
|
||||
};
|
||||
let previous = self.info.scan_directory_source.replace(source.clone());
|
||||
// Every production endpoint has an erasure-set slot. A disk replacement
|
||||
// cannot retain a backup under the same slot or under a retired endpoint.
|
||||
let current_location = directory_source_location(&source);
|
||||
self.info.scan_directory_work.retain(|work| {
|
||||
work.endpoint() == source
|
||||
|| endpoints.contains(directory_source_location(work.endpoint()))
|
||||
&& directory_source_location(work.endpoint()) != current_location
|
||||
});
|
||||
self.info.scan_source_trees.retain(|key, tree| {
|
||||
tree.bucket == self.info.name
|
||||
&& tree.identity == identity
|
||||
&& tree.started_plan == progress.started_plan
|
||||
&& (key == &source
|
||||
|| endpoints.contains(directory_source_location(key)) && directory_source_location(key) != current_location)
|
||||
});
|
||||
let Some(previous) = previous else {
|
||||
// Pre-source checkpoints identify an erasure set but never recorded
|
||||
// which disk supplied the partial tree. The same keys can have
|
||||
// different metadata on another replica. Keep the certified usage,
|
||||
// repair obligations and observations outside this traversal.
|
||||
if self.has_scan_position() {
|
||||
self.clear_active_directory_traversal();
|
||||
}
|
||||
return;
|
||||
};
|
||||
if previous == source {
|
||||
return;
|
||||
}
|
||||
let entries = std::mem::take(&mut self.cache);
|
||||
let linked = entries.get(&hash_path(&self.info.name).0).is_some_and(|root| !root.compacted);
|
||||
let previous_tree = SourceTree {
|
||||
bucket: self.info.name.clone(),
|
||||
identity,
|
||||
started_plan: progress.started_plan,
|
||||
entries,
|
||||
resume_after: self.info.scan_resume_after.take(),
|
||||
checkpoint: self.info.scan_checkpoint.take(),
|
||||
coverage: self.info.scan_coverage_receipt.take(),
|
||||
cursor: self.info.scan_raw_enumeration_cursor.take(),
|
||||
index: self.info.scan_raw_enumeration_page_index.take(),
|
||||
ancestors: std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes),
|
||||
};
|
||||
if linked
|
||||
&& endpoints.contains(directory_source_location(&previous))
|
||||
&& directory_source_location(&previous) != current_location
|
||||
{
|
||||
self.info.scan_source_trees.insert(previous, Arc::new(previous_tree));
|
||||
}
|
||||
if let Some(tree) = self.info.scan_source_trees.remove(&source) {
|
||||
let tree = Arc::unwrap_or_clone(tree);
|
||||
self.cache = tree.entries;
|
||||
self.info.scan_resume_after = tree.resume_after;
|
||||
self.info.scan_coverage_receipt = tree.coverage;
|
||||
self.info.scan_checkpoint = tree.checkpoint;
|
||||
self.info.scan_raw_enumeration_cursor = tree.cursor;
|
||||
self.info.scan_raw_enumeration_page_index = tree.index;
|
||||
self.info.scan_raw_enumeration_path_page_indexes = tree.ancestors;
|
||||
// A dormant tree must provide the same coverage/graph guarantees
|
||||
// required of an active checkpoint before its child links are used.
|
||||
if self.checked_flatten_complete_scope(&self.info.name).is_none()
|
||||
|| self.info.scan_coverage_receipt.is_some() && self.validated_scan_frontier().is_none()
|
||||
{
|
||||
self.clear_active_directory_traversal();
|
||||
}
|
||||
}
|
||||
}
|
||||
fn clear_active_directory_traversal(&mut self) {
|
||||
self.cache.clear();
|
||||
self.info.scan_resume_after = None;
|
||||
self.info.scan_checkpoint = None;
|
||||
self.info.scan_coverage_receipt = None;
|
||||
self.info.scan_raw_enumeration_cursor = None;
|
||||
self.info.scan_raw_enumeration_page_index = None;
|
||||
self.info.scan_raw_enumeration_path_page_indexes.clear();
|
||||
}
|
||||
}
|
||||
|
||||
fn directory_source_location(source: &str) -> &str {
|
||||
source.split_once('|').map_or(source, |(_, endpoint)| endpoint)
|
||||
}
|
||||
@@ -653,6 +653,7 @@ async fn cache_save_reconciles_an_ambiguous_committed_write() {
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: None,
|
||||
};
|
||||
|
||||
@@ -779,6 +780,7 @@ fn baseline_revisions_allow_only_a_decoded_unchanged_primary_backup_catchup() {
|
||||
revisions: DataUsageCacheRevisions {
|
||||
main: etag("primary"),
|
||||
backup: Some(etag("old-backup")),
|
||||
damaged_pages: DamagedPages::default(),
|
||||
},
|
||||
loaded_from_main: true,
|
||||
};
|
||||
@@ -835,11 +837,11 @@ async fn baseline_revisions_record_the_actual_decoded_cache_source() {
|
||||
let mut expected = DataUsageCache::default();
|
||||
expected.info.name = "decoded".to_owned();
|
||||
let valid = expected.marshal_msg().expect("encode valid cache");
|
||||
for (main, backup, from_main, name) in [
|
||||
(valid.clone(), Some(valid.clone()), true, "decoded"),
|
||||
(b"corrupt".to_vec(), Some(valid.clone()), false, "decoded"),
|
||||
(b"corrupt".to_vec(), None, false, ""),
|
||||
(Vec::new(), None, false, ""),
|
||||
for (main, backup, from_main, damaged_main, name) in [
|
||||
(valid.clone(), Some(valid.clone()), true, false, "decoded"),
|
||||
(b"corrupt".to_vec(), Some(valid.clone()), false, true, "decoded"),
|
||||
(b"corrupt".to_vec(), None, false, true, ""),
|
||||
(Vec::new(), None, false, true, ""),
|
||||
] {
|
||||
let store = Arc::new(CacheReadStore::new(CacheReadBody::Bytes(main), backup));
|
||||
let mut loaded = DataUsageCache::default();
|
||||
@@ -848,6 +850,7 @@ async fn baseline_revisions_record_the_actual_decoded_cache_source() {
|
||||
.await
|
||||
.expect("load actual cache source");
|
||||
assert_eq!(baseline.loaded_from_main, from_main);
|
||||
assert_eq!(baseline.revisions.damaged_pages.main, damaged_main);
|
||||
assert_eq!(loaded.info.name, name);
|
||||
let mut raw_loaded = DataUsageCache::default();
|
||||
let raw = raw_loaded
|
||||
@@ -866,6 +869,7 @@ async fn baseline_revisions_record_the_actual_decoded_cache_source() {
|
||||
assert_eq!(legacy_cache.info.name, "decoded");
|
||||
assert!(!legacy.loaded_from_main);
|
||||
assert_eq!(legacy.revisions.main, DataUsageCacheRevision::Missing);
|
||||
assert_eq!(legacy.revisions.damaged_pages, DamagedPages::default());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2976,6 +2980,7 @@ async fn test_save_backup_budget_tracks_primary_cost() {
|
||||
let store = Arc::new(store);
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
|
||||
};
|
||||
DataUsageCache::default()
|
||||
@@ -3028,6 +3033,7 @@ async fn test_checkpoint_budget_covers_retries_and_reconciliation() {
|
||||
let store = Arc::new(store);
|
||||
let revisions = DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
|
||||
};
|
||||
let started = Instant::now();
|
||||
|
||||
@@ -88,6 +88,7 @@ async fn check_save(store: BackupFallbackStore, revisions: Option<DataUsageCache
|
||||
fn revisions(backup: bool) -> DataUsageCacheRevisions {
|
||||
DataUsageCacheRevisions {
|
||||
main: DataUsageCacheRevision::Missing,
|
||||
damaged_pages: DamagedPages::default(),
|
||||
backup: backup.then_some(DataUsageCacheRevision::Missing),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -28,6 +28,10 @@ pub enum ScannerError {
|
||||
#[error("I/O error: {0}")]
|
||||
Io(#[from] std::io::Error),
|
||||
|
||||
/// The selected disk failed or was replaced. Discard uncommitted walk state.
|
||||
#[error("Scanner source unavailable: {0}")]
|
||||
SourceUnavailable(#[source] Box<crate::StorageError>),
|
||||
|
||||
/// Serialization error
|
||||
#[error("Serialization error: {0}")]
|
||||
Serialization(#[from] serde_json::Error),
|
||||
@@ -65,3 +69,9 @@ pub enum ScannerError {
|
||||
#[error("Scanner namespace path disappeared during the scan")]
|
||||
NamespaceNotFoundCache(Box<DataUsageCache>),
|
||||
}
|
||||
|
||||
impl ScannerError {
|
||||
pub(crate) fn source_unavailable(error: crate::StorageError) -> Self {
|
||||
Self::SourceUnavailable(Box::new(error))
|
||||
}
|
||||
}
|
||||
|
||||
+11
-14
@@ -471,7 +471,6 @@ pub(crate) fn resolve_scanner_server_config() -> Option<ServerConfig> {
|
||||
/// How long the scanner caches the runtime tier-name list before re-reading
|
||||
/// the tier configuration manager.
|
||||
const TIER_NAME_CACHE_TTL: Duration = Duration::from_secs(30);
|
||||
const MAX_TIER_REGISTRY_NAME_BYTES: usize = 256;
|
||||
|
||||
/// Process-wide TTL cache of runtime tier names.
|
||||
///
|
||||
@@ -560,16 +559,10 @@ fn next_tier_registry_generation() -> u64 {
|
||||
}
|
||||
|
||||
fn validate_tier_registry_names(mut names: Vec<String>) -> Result<Arc<[String]>, ()> {
|
||||
if names.iter().any(|name| {
|
||||
name.is_empty()
|
||||
|| name.len() > MAX_TIER_REGISTRY_NAME_BYTES
|
||||
|| name.bytes().any(|byte| byte.is_ascii_control())
|
||||
|| name == UNKNOWN_TIER
|
||||
|| name == storageclass::STANDARD
|
||||
|| name == storageclass::RRS
|
||||
}) {
|
||||
return Err(());
|
||||
}
|
||||
// Quarantine individual invalid names from older persisted configurations.
|
||||
names.retain(|name| {
|
||||
rustfs_data_usage::is_scannable_tier_name(name) && name != storageclass::STANDARD && name != storageclass::RRS
|
||||
});
|
||||
names.sort_unstable();
|
||||
if names.windows(2).any(|pair| pair[0] == pair[1]) {
|
||||
return Err(());
|
||||
@@ -1240,9 +1233,13 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_tier_registry_names_fail_closed_for_refresh() {
|
||||
assert!(validate_tier_registry_names(vec!["COLD\n".to_string()]).is_err());
|
||||
assert!(validate_tier_registry_names(vec![UNKNOWN_TIER.to_string()]).is_err());
|
||||
fn tier_registry_keeps_usable_names_on_refresh() {
|
||||
assert_eq!(
|
||||
validate_tier_registry_names(vec!["COLD\n".into(), UNKNOWN_TIER.into(), "WARM".into()])
|
||||
.expect("valid subset")
|
||||
.as_ref(),
|
||||
&["WARM"]
|
||||
);
|
||||
assert!(validate_tier_registry_names(vec!["COLD".to_string(), "COLD".to_string()]).is_err());
|
||||
assert_eq!(
|
||||
validate_tier_registry_names(vec!["WARM".to_string(), "COLD".to_string()])
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
use crate::data_usage_define::clone_cache_value;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest as _, Sha256};
|
||||
use std::sync::Arc;
|
||||
use thiserror::Error;
|
||||
|
||||
mod writer;
|
||||
@@ -58,7 +59,7 @@ pub struct RawEnumerationPage {
|
||||
parent: String,
|
||||
page_index: u64,
|
||||
entries_start: u64,
|
||||
entries: Vec<String>,
|
||||
entries: Arc<[String]>,
|
||||
terminal: bool,
|
||||
digest: [u8; 32],
|
||||
}
|
||||
@@ -518,32 +519,26 @@ impl RawEnumerationPage {
|
||||
}
|
||||
|
||||
fn new(parent: &str, building: RawEnumerationPageBuilder) -> Self {
|
||||
let digest = raw_page_digest(parent, &building);
|
||||
let digest = raw_page_digest(parent, building.page_index, building.entries_start, building.terminal, &building.entries);
|
||||
Self {
|
||||
version: RAW_PAGE_INDEX_VERSION,
|
||||
parent: clone_cache_value(parent),
|
||||
page_index: building.page_index,
|
||||
entries_start: building.entries_start,
|
||||
entries: building.entries,
|
||||
entries: building.entries.into(),
|
||||
terminal: building.terminal,
|
||||
digest,
|
||||
}
|
||||
}
|
||||
|
||||
fn validate(&self, parent: &str, page_index: u64, entries_start: u64) -> Result<(), RawEnumerationPageIndexError> {
|
||||
let builder = RawEnumerationPageBuilder {
|
||||
page_index: self.page_index,
|
||||
entries_start: self.entries_start,
|
||||
entries: clone_cache_value(&self.entries),
|
||||
terminal: self.terminal,
|
||||
};
|
||||
if self.version != RAW_PAGE_INDEX_VERSION
|
||||
|| self.parent != parent
|
||||
|| self.page_index != page_index
|
||||
|| self.entries_start != entries_start
|
||||
|| self.entries.is_empty()
|
||||
|| !entries_are_normalized(&self.entries)
|
||||
|| self.digest != raw_page_digest(parent, &builder)
|
||||
|| self.digest != raw_page_digest(parent, self.page_index, self.entries_start, self.terminal, &self.entries)
|
||||
{
|
||||
return Err(RawEnumerationPageIndexError::CorruptIndex);
|
||||
}
|
||||
@@ -587,16 +582,16 @@ fn entry_sets_match(left: &[String], right: &[String]) -> bool {
|
||||
left.len() == right.len() && entries_contain_all(left, right)
|
||||
}
|
||||
|
||||
fn raw_page_digest(parent: &str, building: &RawEnumerationPageBuilder) -> [u8; 32] {
|
||||
fn raw_page_digest(parent: &str, page_index: u64, entries_start: u64, terminal: bool, entries: &[String]) -> [u8; 32] {
|
||||
#[cfg(test)]
|
||||
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(building.entries.len())));
|
||||
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(entries.len())));
|
||||
let mut digest = Sha256::new();
|
||||
update_digest(&mut digest, b"version", &RAW_PAGE_INDEX_VERSION.to_le_bytes());
|
||||
update_digest(&mut digest, b"parent", parent.as_bytes());
|
||||
update_digest(&mut digest, b"page_index", &building.page_index.to_le_bytes());
|
||||
update_digest(&mut digest, b"entries_start", &building.entries_start.to_le_bytes());
|
||||
update_digest(&mut digest, b"terminal", &[u8::from(building.terminal)]);
|
||||
for entry in &building.entries {
|
||||
update_digest(&mut digest, b"page_index", &page_index.to_le_bytes());
|
||||
update_digest(&mut digest, b"entries_start", &entries_start.to_le_bytes());
|
||||
update_digest(&mut digest, b"terminal", &[u8::from(terminal)]);
|
||||
for entry in entries {
|
||||
update_digest(&mut digest, b"entry", entry.as_bytes());
|
||||
}
|
||||
digest.finalize().into()
|
||||
|
||||
@@ -64,6 +64,15 @@ impl RawEnumerationPageWriter {
|
||||
})
|
||||
}
|
||||
|
||||
/// A parked native stream delivers the unread tail first. Incomplete
|
||||
/// membership can immediately append that tail; terminal page verification
|
||||
/// still requires observing the entire current directory.
|
||||
pub(crate) fn continue_native_stream(&mut self) {
|
||||
if !self.inner.complete && !self.inner.building.as_ref().is_some_and(|page| page.terminal) {
|
||||
self.observations = self.revalidate_after_entries;
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn indexed_entry_count(&self) -> usize {
|
||||
self.indexed.len()
|
||||
}
|
||||
|
||||
@@ -334,6 +334,18 @@ struct RemoteScannerServerError {
|
||||
}
|
||||
|
||||
impl RemoteScannerServerError {
|
||||
fn source_unavailable(error: StorageError) -> Self {
|
||||
Self {
|
||||
scope: RemoteScannerErrorScope::Bucket,
|
||||
error: ScannerError::source_unavailable(error),
|
||||
retry_bucket: true,
|
||||
}
|
||||
}
|
||||
|
||||
fn check_source(disk: &Disk) -> Result<(), Self> {
|
||||
crate::scanner_io::check_local_scanner_handle(disk).map_err(Self::source_unavailable)
|
||||
}
|
||||
|
||||
fn worker(message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
scope: RemoteScannerErrorScope::Worker,
|
||||
@@ -866,6 +878,26 @@ async fn remote_disk_belongs_to_set(disk: &Disk, set: &crate::SetDisks) -> bool
|
||||
}
|
||||
}
|
||||
|
||||
// Validate the physical slot before reusing any cached value for the requested set.
|
||||
// Only the configured one-drive topology can admit a healing source for usage alone.
|
||||
fn remote_source_heal_suppression(
|
||||
disk: &Disk,
|
||||
set: &crate::SetDisks,
|
||||
info: &crate::DiskInfo,
|
||||
skip_healing: bool,
|
||||
) -> Result<bool, RemoteScannerServerError> {
|
||||
if !info.error.is_empty()
|
||||
|| info.id.is_none()
|
||||
|| info.id != crate::scanner_io::scanner_slot_disk_id(set, disk)
|
||||
|| (info.healing && set.set_drive_count != 1)
|
||||
{
|
||||
return Err(RemoteScannerServerError::retry_bucket(
|
||||
"scanner source format or healing state does not permit this walk",
|
||||
));
|
||||
}
|
||||
Ok(skip_healing || info.healing)
|
||||
}
|
||||
|
||||
async fn scan_and_persist_local_bucket(
|
||||
disk: Arc<Disk>,
|
||||
ctx: CancellationToken,
|
||||
@@ -932,6 +964,17 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
"remote namespace scanner disk does not belong to its storage instance",
|
||||
));
|
||||
}
|
||||
RemoteScannerServerError::check_source(&disk)?;
|
||||
let options = crate::DiskInfoOptions::default();
|
||||
let info = tokio::select! {
|
||||
_ = ctx.cancelled() => return Err(RemoteScannerServerError::retry_bucket("scanner source admission was cancelled")),
|
||||
_ = budget.wait_for_stop() => return Ok(RemoteScannerFrameResult::Partial),
|
||||
probe = tokio::time::timeout(NS_SCANNER_DISK_HEALTH_TIMEOUT, disk.disk_info(&options)) =>
|
||||
probe.map_err(|_| RemoteScannerServerError::retry_bucket("scanner source probe timed out"))?
|
||||
.map_err(|err| RemoteScannerServerError::source_unavailable(err.into()))?,
|
||||
};
|
||||
let skip_healing = remote_source_heal_suppression(&disk, &set, &info, skip_healing)?;
|
||||
|
||||
if let Some(proof) = usage_watch.as_deref()
|
||||
&& !proof.verify_with_store(&store).await
|
||||
{
|
||||
@@ -956,9 +999,14 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
}
|
||||
})?;
|
||||
let mut cache = DataUsageCache::default();
|
||||
let mut revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?;
|
||||
let Some(mut revisions) = Box::pin(cache.load_for_scan(set.clone(), &cache_name, &ctx, &budget))
|
||||
.await
|
||||
.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
|
||||
})?
|
||||
else {
|
||||
return Ok(RemoteScannerFrameResult::Partial);
|
||||
};
|
||||
let expected_incarnation = cache
|
||||
.info
|
||||
.scan_identity
|
||||
@@ -1036,6 +1084,7 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
"remote namespace scanner usage watch changed before cache reuse",
|
||||
));
|
||||
}
|
||||
RemoteScannerServerError::check_source(&disk)?;
|
||||
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
|
||||
source,
|
||||
scan_plan_digest,
|
||||
@@ -1066,10 +1115,12 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
let scan_position_before = cache.durable_scan_progress();
|
||||
let resumed_sweep = cache.has_scan_position();
|
||||
|
||||
let checkpoint_endpoints = crate::scanner_io::scanner_checkpoint_endpoints(&set, &cache);
|
||||
let set_disks = scanner_set_disk_inventory(set.as_ref()).await;
|
||||
let (checkpoint_tx, mut checkpoint_rx) = mpsc::channel::<DataUsageCache>(1);
|
||||
let (node_tx, mut node_rx) = crate::workload_admission::scanner_node_request_channel();
|
||||
let scan_ctx = ctx.child_token();
|
||||
let local_stop = Arc::new(crate::scanner_io::ScannerLocalStop::default());
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let _ = crate::e2e_barrier::checkpoint(
|
||||
"bucket_walk",
|
||||
@@ -1090,9 +1141,11 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
checkpoint_endpoints,
|
||||
scan_mode,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: Some(checkpoint_tx),
|
||||
local_stop: Some(local_stop.clone()),
|
||||
},
|
||||
),
|
||||
);
|
||||
@@ -1127,7 +1180,11 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
}
|
||||
result = &mut scan => {
|
||||
match result {
|
||||
_ if local_stop.reason().is_some() =>
|
||||
return Err(RemoteScannerServerError::retry_bucket("scanner source was interrupted during the walk")),
|
||||
Ok(outcome) => break outcome,
|
||||
Err(err) if crate::scanner_io::is_scanner_source_unavailable(&err) =>
|
||||
return Err(RemoteScannerServerError::source_unavailable(err)),
|
||||
Err(err) => {
|
||||
let disk_online =
|
||||
tokio::time::timeout(NS_SCANNER_DISK_HEALTH_TIMEOUT, crate::scanner_disk_is_online(disk.as_ref()))
|
||||
@@ -1154,6 +1211,11 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
checkpoint_channel_closed = true;
|
||||
continue;
|
||||
};
|
||||
if let Err(error) = RemoteScannerServerError::check_source(&disk) {
|
||||
scan_ctx.cancel();
|
||||
let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
|
||||
return Err(error);
|
||||
}
|
||||
if guard.is_lock_lost() {
|
||||
scan_ctx.cancel();
|
||||
let _ = tokio::time::timeout(NS_SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan.as_mut()).await;
|
||||
@@ -1259,6 +1321,7 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
.map_err(|err| {
|
||||
RemoteScannerServerError::worker(format!("remote namespace scanner leader fence verification failed: {err}"))
|
||||
})?;
|
||||
RemoteScannerServerError::check_source(&disk)?;
|
||||
let done_save = Metrics::time(Metric::SaveUsage);
|
||||
// Each physical main/backup PUT must still prove the epoch captured before
|
||||
// the scan. A movement transition that starts and ends during the scan
|
||||
@@ -1304,6 +1367,7 @@ async fn scan_and_persist_local_bucket_with_store(
|
||||
));
|
||||
}
|
||||
|
||||
RemoteScannerServerError::check_source(&disk)?;
|
||||
if let Some(partial_result) = partial_result {
|
||||
if matches!(partial_result, RemoteScannerFrameResult::Partial) {
|
||||
log_saved_partial_bucket_cache(&cache, &cache_name);
|
||||
|
||||
@@ -109,6 +109,172 @@ fn retry_bucket_terminal_frame_preserves_its_wire_disposition() {
|
||||
assert!(!frame.message.starts_with(NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn retired_remote_source_at_eof_retries_without_certifying_its_walk() {
|
||||
use crate::scanner_io::tests::{retire_source_at_eof, setup_local_scanner_store, wait_for_namespace_commit_tails};
|
||||
use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _};
|
||||
|
||||
let (_directory, store) = setup_local_scanner_store(1, 4).await;
|
||||
let bucket = "remote-retired-source";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
let mut reader = crate::ScannerPutObjReader::from_vec(b"payload".to_vec());
|
||||
store
|
||||
.put_object(bucket, "object", &mut reader, &crate::ScannerObjectOptions::default())
|
||||
.await
|
||||
.expect("write object");
|
||||
wait_for_namespace_commit_tails(&store).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let disk = scanner_set_disk_inventory(&set).await[0].clone();
|
||||
let retirement = disk.local_retirement_token().expect("local source");
|
||||
let _probe = retire_source_at_eof(bucket, set.clone());
|
||||
let mut request = test_request(Uuid::new_v4());
|
||||
request.bucket = bucket.to_owned();
|
||||
request.scan_mode = HealScanMode::Normal;
|
||||
crate::save_config(
|
||||
store.clone(),
|
||||
crate::DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
crate::scanner::encode_scanner_cycle_fence_for_test(request.next_cycle, request.leader_epoch),
|
||||
)
|
||||
.await
|
||||
.expect("persist the worker's cycle fence");
|
||||
let ctx = CancellationToken::new();
|
||||
let frame = scan_and_persist_local_bucket_with_store(
|
||||
disk,
|
||||
ctx.clone(),
|
||||
ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default()),
|
||||
Arc::new(AtomicU8::new(RemoteScannerPhase::Scanning as u8)),
|
||||
Arc::new(Notify::new()),
|
||||
request,
|
||||
store,
|
||||
)
|
||||
.await
|
||||
.expect_err("retired worker must relinquish this bucket")
|
||||
.into_frame();
|
||||
assert!(retirement.is_cancelled(), "fault must fire after the walk reaches EOF");
|
||||
assert!(matches!(frame.scope, RemoteScannerErrorScope::Bucket));
|
||||
assert!(frame.message.starts_with(NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX), "{}", frame.message);
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache
|
||||
.load(set, &path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]))
|
||||
.await
|
||||
.expect("load checkpoint");
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn remote_checkpoint_survives_replica_detach_and_same_uuid_reconnect() {
|
||||
use crate::scanner_io::tests::{setup_local_scanner_store, wait_for_namespace_commit_tails};
|
||||
use crate::storage_api::owner::EcstoreDiskAPI as _;
|
||||
use crate::storage_api::scan::{BucketOperations as _, MakeBucketOptions, ObjectIO as _};
|
||||
|
||||
let (_directory, store) = setup_local_scanner_store(1, 4).await;
|
||||
let set = store.pools[0].disk_set[0].clone();
|
||||
let bucket = "remote-dormant-replica";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("create bucket");
|
||||
for index in 0..12 {
|
||||
let mut reader = crate::ScannerPutObjReader::from_vec(vec![index]);
|
||||
store
|
||||
.put_object(bucket, &format!("key-{index:02}"), &mut reader, &crate::ScannerObjectOptions::default())
|
||||
.await
|
||||
.expect("write object");
|
||||
}
|
||||
wait_for_namespace_commit_tails(&store).await;
|
||||
let request = test_request(Uuid::new_v4());
|
||||
crate::save_config(
|
||||
store.clone(),
|
||||
crate::DATA_USAGE_BLOOM_NAME_PATH.as_str(),
|
||||
crate::scanner::encode_scanner_cycle_fence_for_test(request.next_cycle, request.leader_epoch),
|
||||
)
|
||||
.await
|
||||
.expect("persist cycle fence");
|
||||
let scan = |disk: Arc<Disk>, max_objects| {
|
||||
let store = store.clone();
|
||||
async move {
|
||||
let request = RemoteScannerRequestWire {
|
||||
bucket: bucket.to_owned(),
|
||||
scan_mode: HealScanMode::Normal,
|
||||
skip_healing: false,
|
||||
checkpoint_plan_digest: Some(TEST_PLAN_DIGEST),
|
||||
supports_durable_progress: true,
|
||||
..test_request(Uuid::new_v4())
|
||||
};
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_objects,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
scan_and_persist_local_bucket_with_store(
|
||||
disk,
|
||||
budget.token(),
|
||||
budget,
|
||||
Arc::new(AtomicU8::new(RemoteScannerPhase::Scanning as u8)),
|
||||
Arc::new(Notify::new()),
|
||||
request,
|
||||
store,
|
||||
)
|
||||
.await
|
||||
.expect("run the admitted V4 worker")
|
||||
}
|
||||
};
|
||||
|
||||
let disks = scanner_set_disk_inventory(&set).await;
|
||||
let disk_b = &disks[1];
|
||||
let endpoint_b = disk_b.endpoint();
|
||||
let uuid_b = disk_b.get_disk_id().await.expect("read disk ID").expect("formatted disk");
|
||||
let first = scan(disk_b.clone(), Some(8)).await;
|
||||
let cache_name = path_join_buf(&[bucket, DATA_USAGE_CACHE_NAME]);
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.load(set.clone(), &cache_name).await.expect("load B's checkpoint");
|
||||
assert!(
|
||||
matches!(&first, RemoteScannerFrameResult::DurableProgress(receipt) if receipt.advanced),
|
||||
"B returned {first:?}, persisted identity={:?} progress={:?} source={:?} frontier={:?} work={} trees={} complete={}",
|
||||
cache.info.scan_identity,
|
||||
cache.info.scan_progress,
|
||||
cache.info.scan_directory_source,
|
||||
cache.validated_scan_frontier(),
|
||||
cache.info.scan_directory_work.len(),
|
||||
cache.info.scan_source_trees.len(),
|
||||
cache.info.snapshot_complete
|
||||
);
|
||||
let source_b = cache.info.scan_directory_source.expect("source for the partial traversal");
|
||||
|
||||
let detached = set.disks.write().await[1].take().expect("detach B");
|
||||
assert!(Arc::ptr_eq(&detached, disk_b));
|
||||
detached.close().await.expect("retire B's original handle");
|
||||
assert!(matches!(
|
||||
scan(disks[0].clone(), Some(2)).await,
|
||||
RemoteScannerFrameResult::DurableProgress(_)
|
||||
));
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.load(set.clone(), &cache_name).await.expect("load A's checkpoint");
|
||||
assert!(
|
||||
cache.info.scan_source_trees.contains_key(&source_b)
|
||||
|| cache.info.scan_directory_work.iter().any(|work| work.endpoint() == source_b),
|
||||
"A must retain the disconnected configured replica's progress"
|
||||
);
|
||||
|
||||
set.renew_disk(&endpoint_b).await;
|
||||
let reopened = set.disks.read().await[1].clone().expect("reconnect B");
|
||||
assert!(!Arc::ptr_eq(&reopened, disk_b));
|
||||
assert_eq!(reopened.get_disk_id().await.expect("reopened disk ID"), Some(uuid_b));
|
||||
// A fresh traversal cannot fit all twelve objects in this last budget.
|
||||
let RemoteScannerFrameResult::Complete(result) = scan(reopened, Some(9)).await else {
|
||||
panic!("the reconnected replica should finish from its saved position")
|
||||
};
|
||||
assert_eq!((result.usage.entry.objects, result.usage.entry.size), (12, 12));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn semantic_deadline_is_capped_by_rpc_lifetime_without_overflow() {
|
||||
let now = Instant::now();
|
||||
@@ -610,9 +776,11 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
checkpoint_endpoints: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -638,9 +806,11 @@ async fn remote_worker_rechecks_real_producers_before_persisting_complete_usage(
|
||||
cache,
|
||||
None,
|
||||
ScannerDiskScanOptions {
|
||||
checkpoint_endpoints: None,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
prefix_scan_scope: None,
|
||||
checkpoint_tx: None,
|
||||
local_stop: None,
|
||||
},
|
||||
)
|
||||
.await
|
||||
@@ -828,7 +998,7 @@ async fn remote_sender_preserves_independent_bucket_plan_identities_in_signed_bo
|
||||
server_epoch: Uuid::new_v4(),
|
||||
session_id: Uuid::new_v4(),
|
||||
session_sequence: 0,
|
||||
plans: ScannerBucketPlanDigests::from_work_plans(reuse, coverage, None),
|
||||
plans: ScannerBucketPlanDigests::from_work_plans(reuse, coverage, None, None),
|
||||
tier_registry_generation: 0,
|
||||
skip_healing: true,
|
||||
scan_mode: HealScanMode::Deep,
|
||||
@@ -2020,10 +2190,9 @@ async fn retry_bucket_error_terminal_frame_requeues_bucket_work() {
|
||||
&mut sequence,
|
||||
&RemoteScannerFrame::terminal(
|
||||
RemoteScannerProgress::default(),
|
||||
RemoteScannerFrameResult::Error(RemoteScannerErrorFrame {
|
||||
scope: RemoteScannerErrorScope::Bucket,
|
||||
message: format!("{NS_SCANNER_RETRY_BUCKET_ERROR_PREFIX} cache lock contention"),
|
||||
}),
|
||||
RemoteScannerFrameResult::Error(
|
||||
RemoteScannerServerError::source_unavailable(StorageError::FaultyDisk).into_frame(),
|
||||
),
|
||||
),
|
||||
)
|
||||
.await
|
||||
@@ -2038,7 +2207,7 @@ async fn retry_bucket_error_terminal_frame_requeues_bucket_work() {
|
||||
|
||||
assert!(error.retry_bucket_work());
|
||||
assert!(!error.retire_worker());
|
||||
assert!(error.to_string().contains("cache lock contention"));
|
||||
assert!(error.to_string().contains("Faulty disk"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2916,6 +3085,65 @@ async fn background_envelope_accepts_the_full_wire_limit_and_matches_none() {
|
||||
#[path = "durable_progress.rs"]
|
||||
mod durable_progress;
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn remote_source_admission_preserves_slot_identity_and_single_drive_heal_suppression() {
|
||||
use crate::storage_api::scan::HEALING_MARKER_PATH;
|
||||
|
||||
for width in [1, 4] {
|
||||
let (_directory, store) = crate::scanner_io::tests::setup_local_scanner_store(1, width).await;
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
let disk = scanner_set_disk_inventory(set).await.remove(0);
|
||||
let options = crate::DiskInfoOptions::default();
|
||||
let mut info = disk.disk_info(&options).await.expect("probe formatted source");
|
||||
assert!(!remote_source_heal_suppression(&disk, set, &info, false).expect("formatted source may serve"));
|
||||
assert!(remote_source_heal_suppression(&disk, set, &info, true).expect("preserve requested suppression"));
|
||||
|
||||
info.error = "failed admission probe".to_owned();
|
||||
assert!(
|
||||
remote_source_heal_suppression(&disk, set, &info, false)
|
||||
.expect_err("probe returned a disk error")
|
||||
.retry_bucket
|
||||
);
|
||||
info.error.clear();
|
||||
for id in [None, Some(Uuid::nil()), Some(Uuid::new_v4())] {
|
||||
info.id = id;
|
||||
assert!(
|
||||
remote_source_heal_suppression(&disk, set, &info, false)
|
||||
.expect_err("format mismatch")
|
||||
.retry_bucket
|
||||
);
|
||||
}
|
||||
|
||||
let marker = disk.path().join(crate::RUSTFS_META_BUCKET).join(HEALING_MARKER_PATH);
|
||||
tokio::fs::write(&marker, b"scanner fixture")
|
||||
.await
|
||||
.expect("mark rebuilding drive");
|
||||
let marked = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let info = disk.disk_info(&options).await.expect("probe marked source");
|
||||
if info.healing {
|
||||
break info;
|
||||
}
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("observe healing marker");
|
||||
let admitted = remote_source_heal_suppression(&disk, set, &marked, false);
|
||||
if width == 1 {
|
||||
assert!(admitted.expect("single formatted drive may serve usage with healing suppressed"));
|
||||
} else {
|
||||
assert!(admitted.expect_err("marked multi-drive source must be retried").retry_bucket);
|
||||
assert!(
|
||||
remote_source_heal_suppression(&disk, set, &marked, true)
|
||||
.expect_err("peer cannot override set width")
|
||||
.retry_bucket
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||
#[serial_test::serial]
|
||||
async fn remote_usage_watch_worker_rejects_another_storage_instance_with_identical_coordinates() {
|
||||
|
||||
+166
-34
@@ -34,8 +34,9 @@ use crate::runtime_config::{
|
||||
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
|
||||
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
|
||||
use crate::scanner_io::{
|
||||
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
|
||||
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
|
||||
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerServiceCohort, dirty_usage_bucket_notified,
|
||||
dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed,
|
||||
scanner_maintenance_generation,
|
||||
};
|
||||
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
|
||||
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
|
||||
@@ -55,7 +56,7 @@ use rustfs_lock::{NamespaceLockGuard, error::LockError};
|
||||
use rustfs_scanner_metrics::metrics::{
|
||||
CurrentCycle, Metric, Metrics, RemoteScanProgress, ScanCyclePartialReason, ScanCycleWorkSnapshot,
|
||||
ScannerUsagePublicationObservation, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
|
||||
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
|
||||
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_stage, emit_scan_cycle_superseded, global_metrics,
|
||||
};
|
||||
use rustfs_scanner_metrics::startup::ScannerStartupToken;
|
||||
use serde::{Deserialize, Serialize};
|
||||
@@ -1212,6 +1213,8 @@ where
|
||||
}
|
||||
|
||||
let mut transient_backoff = ScannerRetryBackoff::default();
|
||||
// Only this supervisor, for this storage instance, owns resumable service.
|
||||
let mut service_continuation = None;
|
||||
let mut recovery_retry_count = 0_u32;
|
||||
loop {
|
||||
if ctx_clone.is_cancelled() {
|
||||
@@ -1226,6 +1229,7 @@ where
|
||||
startup_features,
|
||||
startup_maintenance_generation,
|
||||
startup,
|
||||
&mut service_continuation,
|
||||
)
|
||||
.await;
|
||||
if let Err(e) = &run_result {
|
||||
@@ -1781,11 +1785,26 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Scheduling and mutation witnesses retained across a clean local deadline.
|
||||
/// Only an immediately succeeding leadership claim can resume this cohort.
|
||||
struct ScannerServiceContinuation {
|
||||
cohort: Arc<StdMutex<ScannerServiceCohort>>,
|
||||
fenced_epoch: u64,
|
||||
fenced_revision: DataUsageCacheRevision,
|
||||
}
|
||||
|
||||
impl ScannerServiceContinuation {
|
||||
fn matches_claim(&self, previous_revision: &DataUsageCacheRevision, claimed_epoch: u64) -> bool {
|
||||
&self.fenced_revision == previous_revision && self.fenced_epoch.checked_add(1) == Some(claimed_epoch)
|
||||
}
|
||||
}
|
||||
|
||||
struct ScannerCycleDeadlineState<'a> {
|
||||
cycle_info: &'a mut CurrentCycle,
|
||||
cycle_revision: &'a mut DataUsageCacheRevision,
|
||||
leader_epoch: &'a mut u64,
|
||||
cycle_budget: &'a ScannerCycleBudget,
|
||||
service_cohort: Arc<StdMutex<ScannerServiceCohort>>,
|
||||
allow_bootstrap_pending: bool,
|
||||
}
|
||||
|
||||
@@ -1799,7 +1818,8 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
state: ScannerCycleDeadlineState<'_>,
|
||||
worker_stopped: bool,
|
||||
guard: &mut NamespaceLockGuard,
|
||||
) where
|
||||
) -> Option<ScannerServiceContinuation>
|
||||
where
|
||||
Store: ScannerObjectIO + ScannerConfigObjectDelete,
|
||||
{
|
||||
let fenced = fence_scanner_epoch_after_cycle_timeout(
|
||||
@@ -1829,8 +1849,14 @@ async fn handle_scanner_cycle_deadline<Store>(
|
||||
global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
|
||||
// Stop renewing before releasing the lease. A new leader can then claim the
|
||||
// higher persisted generation instead of inheriting the expired worker.
|
||||
let may_resume = !recovery_required && !guard.is_lock_lost() && !ctx.is_cancelled();
|
||||
guard.release();
|
||||
global_metrics().set_cycle(None).await;
|
||||
may_resume.then(|| ScannerServiceContinuation {
|
||||
cohort: state.service_cohort,
|
||||
fenced_epoch: *state.leader_epoch,
|
||||
fenced_revision: state.cycle_revision.clone(),
|
||||
})
|
||||
}
|
||||
|
||||
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
|
||||
@@ -1891,6 +1917,48 @@ where
|
||||
.await
|
||||
}
|
||||
|
||||
/// Observe the outer wait so that an expired drain records the attempt even when
|
||||
/// the worker is dropped before its first poll. Only one terminal record is emitted.
|
||||
async fn await_observed_scanner_cycle(
|
||||
metrics: &rustfs_scanner_metrics::metrics::Metrics,
|
||||
epoch: u64,
|
||||
cycle: u64,
|
||||
ctx: &CancellationToken,
|
||||
budget: &ScannerCycleBudget,
|
||||
worker: impl Future<Output = ScannerCycleOutcome>,
|
||||
lock_lost: impl Future<Output = ()>,
|
||||
) -> ScannerCycleWaitOutcome<ScannerCycleOutcome> {
|
||||
let started = tokio::time::Instant::now();
|
||||
let mut completed = None;
|
||||
let worker = async {
|
||||
let outcome = worker.await;
|
||||
completed = Some(outcome);
|
||||
outcome
|
||||
};
|
||||
let wait = await_scanner_cycle_with_budget_fence(ctx, budget, worker, lock_lost).await;
|
||||
let partial = || ("partial", budget.reason().map(|reason| scan_cycle_partial_reason(Some(reason)).as_str()));
|
||||
let (result, partial_reason) = match (&wait, completed.as_ref()) {
|
||||
(ScannerCycleWaitOutcome::Completed(outcome), _) | (_, Some(outcome)) => match outcome {
|
||||
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance => ("success", None),
|
||||
ScannerCycleOutcome::Partial => partial(),
|
||||
ScannerCycleOutcome::Superseded => ("superseded", None),
|
||||
ScannerCycleOutcome::Deferred(_) => ("deferred", None),
|
||||
ScannerCycleOutcome::Failed | ScannerCycleOutcome::StatePersistenceFailed => ("error", None),
|
||||
},
|
||||
(ScannerCycleWaitOutcome::LockLost, None) => ("lock_lost", None),
|
||||
(ScannerCycleWaitOutcome::Cancelled, None) => ("cancelled", None),
|
||||
(ScannerCycleWaitOutcome::Deadline { .. }, None) => partial(),
|
||||
};
|
||||
metrics.record_cluster_attempt(rustfs_scanner_metrics::metrics::ScannerAttemptRecord {
|
||||
epoch,
|
||||
cycle,
|
||||
result,
|
||||
partial_reason,
|
||||
duration: started.elapsed(),
|
||||
});
|
||||
wait
|
||||
}
|
||||
|
||||
#[instrument(skip_all)]
|
||||
#[hotpath::measure]
|
||||
async fn run_data_scanner_cycle_with_budget<S>(
|
||||
@@ -2468,13 +2536,20 @@ where
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
required_cycle,
|
||||
state = "cache_cycle_ahead",
|
||||
partial_stage = "cache_cycle_ahead",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
required_cycle,
|
||||
partial_cause = "cache_cycle_ahead",
|
||||
"Scanner cycle is recovering to a newer durable cache generation"
|
||||
);
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
ScanCyclePartialReason::Unknown,
|
||||
None,
|
||||
Some("cache_cycle_ahead"),
|
||||
);
|
||||
let persisted = persist_required_scanner_cycle_floor_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
@@ -2488,6 +2563,7 @@ where
|
||||
},
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2538,18 +2614,7 @@ where
|
||||
}
|
||||
|
||||
usage_publication_result.restrict_outcome(usage_persist_outcome);
|
||||
let partial_cause = if scan_cycle_result.has_partial_bucket_progress() && !scan_cycle_result.has_usage_update() {
|
||||
"saved_bucket_progress"
|
||||
} else if scan_cycle_result.has_usage_update()
|
||||
&& matches!(
|
||||
scan_cycle_result.status,
|
||||
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
)
|
||||
{
|
||||
"activity_unverified_observation"
|
||||
} else {
|
||||
"incomplete_coverage"
|
||||
};
|
||||
let (partial_cause, partial_stage) = partial_cycle_diagnostic(&scan_cycle_result);
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
let durable_observation = scan_cycle_result
|
||||
.publication_expectation()
|
||||
@@ -2628,25 +2693,35 @@ where
|
||||
);
|
||||
}
|
||||
if cycle_budget.budget_elapsed() {
|
||||
let budget_reason = cycle_budget.reason();
|
||||
let budget_stage = match budget_reason {
|
||||
Some(ScannerCycleBudgetReason::Runtime) => Some("runtime_budget"),
|
||||
Some(ScannerCycleBudgetReason::Objects) => Some("object_budget"),
|
||||
Some(ScannerCycleBudgetReason::Directories) => Some("directory_budget"),
|
||||
Some(ScannerCycleBudgetReason::Entries) => Some("entry_budget"),
|
||||
None => None,
|
||||
};
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "budget_reached",
|
||||
partial_stage = budget_stage,
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
duration = ?now.elapsed(),
|
||||
reason = ?cycle_budget.reason(),
|
||||
reason = ?budget_reason,
|
||||
max_duration = ?cycle_budget.max_duration(),
|
||||
max_objects = ?cycle_budget.max_objects(),
|
||||
max_directories = ?cycle_budget.max_directories(),
|
||||
state = "budget_reached",
|
||||
"Scanner cycle budget reached"
|
||||
);
|
||||
let budget_reason = cycle_budget.reason();
|
||||
emit_scan_cycle_partial_with_source(
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
scan_cycle_partial_reason(budget_reason),
|
||||
scan_cycle_partial_source(budget_reason),
|
||||
budget_stage,
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
@@ -2658,6 +2733,7 @@ where
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2691,29 +2767,56 @@ where
|
||||
return ScannerCycleOutcome::Failed;
|
||||
}
|
||||
ScannerCycleOutcome::Partial => {
|
||||
if ctx.is_cancelled() {
|
||||
let owner_cancelled = ctx.is_cancelled();
|
||||
let partial_stage = if owner_cancelled { "owner_cancelled" } else { partial_stage };
|
||||
if owner_cancelled {
|
||||
debug!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "incomplete_cancelled",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_stage,
|
||||
"Scanner cycle stopped before a complete usage snapshot was produced"
|
||||
);
|
||||
} else {
|
||||
} else if partial_cause == "saved_bucket_progress"
|
||||
&& matches!(partial_stage, "bucket_checkpoint_saved" | "bucket_walk_incomplete")
|
||||
{
|
||||
warn!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
cycle = cycle_info.current,
|
||||
state = "incomplete",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_cause,
|
||||
"Scanner cycle ended without a complete usage snapshot"
|
||||
partial_stage,
|
||||
"Scanner saved bucket progress before completing coverage"
|
||||
);
|
||||
} else {
|
||||
error!(
|
||||
target: "rustfs::scanner",
|
||||
event = EVENT_SCANNER_CYCLE_STATE,
|
||||
component = LOG_COMPONENT_SCANNER,
|
||||
subsystem = LOG_SUBSYSTEM_RUNTIME,
|
||||
state = "incomplete",
|
||||
cycle = cycle_info.current,
|
||||
leader_epoch,
|
||||
partial_cause,
|
||||
partial_stage,
|
||||
persist_outcome = ?usage_persist_outcome,
|
||||
"Scanner cycle could not certify complete usage"
|
||||
);
|
||||
}
|
||||
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
|
||||
let partial = emit_scan_cycle_partial_with_stage(
|
||||
cycle_start.elapsed(),
|
||||
ScanCyclePartialReason::Unknown,
|
||||
None,
|
||||
Some(partial_stage),
|
||||
);
|
||||
let persisted = finalize_partial_scan_cycle_for_epoch(
|
||||
ctx,
|
||||
storeapi.clone(),
|
||||
@@ -2724,6 +2827,7 @@ where
|
||||
publication_epoch,
|
||||
)
|
||||
.await;
|
||||
partial.observe_state_write(persisted);
|
||||
return if persisted {
|
||||
cycle_budget.mark_cycle_state_persisted();
|
||||
ScannerCycleOutcome::Partial
|
||||
@@ -2917,6 +3021,7 @@ where
|
||||
maintenance_features,
|
||||
maintenance_generation,
|
||||
runtime_guard.startup_token(),
|
||||
&mut None,
|
||||
)
|
||||
.await
|
||||
}
|
||||
@@ -3003,10 +3108,12 @@ async fn run_data_scanner_with_maintenance_state<S>(
|
||||
mut maintenance_features: ScannerMaintenanceFeatures,
|
||||
mut maintenance_generation_seen: Option<u64>,
|
||||
startup: ScannerStartupToken,
|
||||
continuation: &mut Option<ScannerServiceContinuation>,
|
||||
) -> Result<(), ScannerError>
|
||||
where
|
||||
S: ScannerStorage,
|
||||
{
|
||||
let previous = continuation.take();
|
||||
reset_scanner_cycle_schedule();
|
||||
// Normal scanner entry is an attempt, not evidence of leadership, coverage, or publication.
|
||||
global_metrics().startup().record_attempt(startup);
|
||||
@@ -3116,7 +3223,6 @@ where
|
||||
let mut deferred_backoff = ScannerRetryBackoff::default();
|
||||
let mut scope_fallback_backoff = ScannerRetryBackoff::default();
|
||||
let scope_fallback_slot = Arc::new(ScannerScopeFallbackSlot::default());
|
||||
let service_cohort = Arc::new(StdMutex::new(crate::scanner_io::ScannerServiceCohort::default()));
|
||||
let initial_runtime_config = resolve_scanner_runtime_config();
|
||||
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
|
||||
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
|
||||
@@ -3277,6 +3383,7 @@ where
|
||||
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
|
||||
return Ok(());
|
||||
}
|
||||
let previous_revision = cycle_revision.clone();
|
||||
let claim_ctx = ctx.child_token();
|
||||
let leadership_claimed = await_scanner_cycle_with_lock_fence(
|
||||
&claim_ctx,
|
||||
@@ -3326,6 +3433,10 @@ where
|
||||
finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await;
|
||||
return Ok(());
|
||||
}
|
||||
let service_cohort = previous
|
||||
.filter(|previous| previous.matches_claim(&previous_revision, leader_epoch))
|
||||
.map(|previous| previous.cohort)
|
||||
.unwrap_or_default();
|
||||
if rebuild_from_quarantine {
|
||||
record_scanner_cycle_state_rebuilt_from_quarantine(cycle_info.next, leader_epoch);
|
||||
}
|
||||
@@ -3377,7 +3488,10 @@ where
|
||||
let cycle_ctx = ctx.child_token();
|
||||
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
||||
record_scanner_cycle_attempt(None);
|
||||
let initial_outcome = match await_scanner_cycle_with_budget_fence(
|
||||
let initial_outcome = match await_observed_scanner_cycle(
|
||||
global_metrics(),
|
||||
leader_epoch,
|
||||
cycle_info.next,
|
||||
&cycle_ctx,
|
||||
&cycle_budget,
|
||||
run_data_scanner_cycle_with_budget(
|
||||
@@ -3410,7 +3524,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
*continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -3418,6 +3532,7 @@ where
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
service_cohort,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
@@ -3714,7 +3829,10 @@ where
|
||||
let cycle_ctx = ctx.child_token();
|
||||
let cycle_budget = ScannerCycleBudget::new_with_runtime_progress_tracking(&cycle_ctx, scanner_cycle_budget_config());
|
||||
record_scanner_cycle_attempt(Some(wake_reason));
|
||||
let outcome = match await_scanner_cycle_with_budget_fence(
|
||||
let outcome = match await_observed_scanner_cycle(
|
||||
global_metrics(),
|
||||
leader_epoch,
|
||||
cycle_info.next,
|
||||
&cycle_ctx,
|
||||
&cycle_budget,
|
||||
run_data_scanner_cycle_with_budget(
|
||||
@@ -3754,7 +3872,7 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
|
||||
handle_scanner_cycle_deadline(
|
||||
*continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
storeapi.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -3762,6 +3880,7 @@ where
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &cycle_budget,
|
||||
service_cohort,
|
||||
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
|
||||
},
|
||||
worker_stopped,
|
||||
@@ -4341,6 +4460,18 @@ fn finalize_scanner_cycle_result(
|
||||
)
|
||||
}
|
||||
|
||||
fn partial_cycle_diagnostic(scan: &crate::scanner_io::ScannerCycleResult) -> (&'static str, &'static str) {
|
||||
if scan.has_partial_bucket_progress() && !scan.has_usage_update() {
|
||||
("saved_bucket_progress", scan.partial_stage.unwrap_or("bucket_checkpoint_saved"))
|
||||
} else if scan.has_usage_update()
|
||||
&& scan.status == ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
|
||||
{
|
||||
("activity_unverified_observation", "activity_baseline_unverified")
|
||||
} else {
|
||||
("incomplete_coverage", scan.partial_stage.unwrap_or("root_coverage_incomplete"))
|
||||
}
|
||||
}
|
||||
|
||||
fn scanner_cycle_completion_outcome_for_result(
|
||||
scan_cycle_result: &crate::scanner_io::ScannerCycleResult,
|
||||
usage_persist_outcome: DataUsagePersistOutcome,
|
||||
@@ -4469,6 +4600,7 @@ pub use backlog::{
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use cycle_state::encode_scanner_cycle_fence_for_test;
|
||||
pub use cycle_state::read_scanner_status_fence;
|
||||
pub use cycle_state::{
|
||||
SCANNER_RECOVERY_INTENT_ACTION_USAGE_FULL_REBUILD, ScannerCycleRecoveryMarker, ScannerCycleRecoveryStatus,
|
||||
ScannerRecoveryIntentAcceptResult, ScannerRecoveryIntentConflict, ScannerRecoveryIntentRecord, ScannerRecoveryIntentRequest,
|
||||
|
||||
@@ -3881,16 +3881,24 @@ pub(crate) fn encode_scanner_cycle_fence_for_test(next_cycle: u64, leader_epoch:
|
||||
pub(crate) async fn current_scanner_leader_epoch() -> Result<u64, ScannerError> {
|
||||
let store = crate::resolve_scanner_object_store_handle()
|
||||
.ok_or_else(|| ScannerError::Other("scanner object layer is unavailable".to_string()))?;
|
||||
match read_config(store, &DATA_USAGE_BLOOM_NAME_PATH).await {
|
||||
Ok(buf) => {
|
||||
let (_, leader_epoch) = decode_persisted_scanner_cycle_fence(&buf)?;
|
||||
if leader_epoch == 0 {
|
||||
return Err(ScannerError::Other("persisted scanner cycle state has no leader epoch".to_string()));
|
||||
}
|
||||
Ok(leader_epoch)
|
||||
}
|
||||
Err(err) => Err(ScannerError::Other(format!("failed to read persisted scanner leader epoch: {err}"))),
|
||||
Ok(read_scanner_status_fence(store).await?.0)
|
||||
}
|
||||
|
||||
/// Read the committed leader generation, next cycle and successful completion history
|
||||
/// for a point-in-time scanner observation. This does not initialize or repair state.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an error if the requested store has no readable, valid, fenced cycle state.
|
||||
pub async fn read_scanner_status_fence<S: ScannerObjectIO>(store: Arc<S>) -> Result<(u64, CurrentCycle), ScannerError> {
|
||||
let buf = read_config(store, &DATA_USAGE_BLOOM_NAME_PATH)
|
||||
.await
|
||||
.map_err(|err| ScannerError::Other(format!("failed to read persisted scanner leader epoch: {err}")))?;
|
||||
let (cycle, epoch) = decode_scanner_cycle_state(&buf)
|
||||
.map_err(|err| ScannerError::Other(format!("persisted scanner cycle state is invalid: {err}")))?;
|
||||
if epoch == 0 {
|
||||
return Err(ScannerError::Other("persisted scanner cycle state has no leader epoch".to_string()));
|
||||
}
|
||||
Ok((epoch, cycle))
|
||||
}
|
||||
|
||||
pub(super) fn decode_scanner_cycle_state_for_startup(buf: &[u8]) -> Result<(CurrentCycle, u64), ScannerCycleStateError> {
|
||||
|
||||
@@ -51,9 +51,11 @@ const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
|
||||
mod cycle_budget_publication;
|
||||
pub(super) mod cycle_persist_failure;
|
||||
mod gate_self_heal;
|
||||
mod paged_checkpoint;
|
||||
mod quota_reset_preservation;
|
||||
|
||||
mod recovery_control;
|
||||
mod restore_deadline;
|
||||
mod scoped_ack_publication;
|
||||
mod wakeup_metrics;
|
||||
|
||||
@@ -565,6 +567,72 @@ async fn scanner_cycle_lock_fence_bounds_uncooperative_shutdown() {
|
||||
assert!(cycle_ctx.is_cancelled());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cluster_scanner_records_a_budget_deadline_before_the_first_worker_poll() {
|
||||
let metrics = rustfs_scanner_metrics::metrics::Metrics::new();
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(
|
||||
&ctx,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_secs(1)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
budget.wait_for_stop().await;
|
||||
advance(SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT + Duration::from_secs(1)).await;
|
||||
let worker = std::future::poll_fn(|_| panic!("expired drain must win before polling the worker"));
|
||||
let wait = await_observed_scanner_cycle(&metrics, 9, 40, &ctx, &budget, worker, std::future::pending()).await;
|
||||
assert_eq!(wait, ScannerCycleWaitOutcome::Deadline { worker_stopped: false });
|
||||
let attempt = metrics
|
||||
.scanner_node_observation()
|
||||
.await
|
||||
.last_attempt
|
||||
.expect("dropped attempt");
|
||||
assert_eq!(
|
||||
(attempt.epoch, attempt.cycle, attempt.result, attempt.partial_reason),
|
||||
(9, 40, "partial", Some("runtime"))
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cluster_scanner_records_lock_loss_cancellation_and_cooperative_completion() {
|
||||
let metrics = rustfs_scanner_metrics::metrics::Metrics::new();
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, Default::default());
|
||||
let wait = await_observed_scanner_cycle(&metrics, 9, 40, &ctx, &budget, std::future::pending(), std::future::ready(())).await;
|
||||
assert_eq!(wait, ScannerCycleWaitOutcome::LockLost);
|
||||
let attempt = metrics.scanner_node_observation().await.last_attempt.expect("lost lock");
|
||||
assert_eq!(attempt.result, "lock_lost");
|
||||
assert!(attempt.duration >= SCANNER_LOCK_LOSS_SHUTDOWN_TIMEOUT);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, Default::default());
|
||||
ctx.cancel();
|
||||
let wait = await_observed_scanner_cycle(&metrics, 9, 41, &ctx, &budget, std::future::pending(), std::future::pending()).await;
|
||||
assert_eq!(wait, ScannerCycleWaitOutcome::Cancelled);
|
||||
assert_eq!(
|
||||
metrics
|
||||
.scanner_node_observation()
|
||||
.await
|
||||
.last_attempt
|
||||
.expect("cancelled attempt")
|
||||
.result,
|
||||
"cancelled"
|
||||
);
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let budget = ScannerCycleBudget::new(&ctx, Default::default());
|
||||
let worker = std::future::ready(ScannerCycleOutcome::Completed);
|
||||
let wait = await_observed_scanner_cycle(&metrics, 9, 42, &ctx, &budget, worker, std::future::ready(())).await;
|
||||
assert_eq!(wait, ScannerCycleWaitOutcome::LockLost);
|
||||
let attempt = metrics
|
||||
.scanner_node_observation()
|
||||
.await
|
||||
.last_attempt
|
||||
.expect("cooperative result");
|
||||
assert_eq!((attempt.cycle, attempt.result), (42, "success"));
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cycle_budget_fences_late_writer_after_timeout() {
|
||||
let cycle_ctx = CancellationToken::new();
|
||||
@@ -772,7 +840,8 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
);
|
||||
budget.mark_cycle_state_persisted();
|
||||
|
||||
handle_scanner_cycle_deadline(
|
||||
let service_cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default()));
|
||||
let continuation = handle_scanner_cycle_deadline(
|
||||
&ctx,
|
||||
store.clone(),
|
||||
ScannerCycleDeadlineState {
|
||||
@@ -780,12 +849,24 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
|
||||
cycle_revision: &mut cycle_revision,
|
||||
leader_epoch: &mut leader_epoch,
|
||||
cycle_budget: &budget,
|
||||
service_cohort: service_cohort.clone(),
|
||||
allow_bootstrap_pending: false,
|
||||
},
|
||||
true,
|
||||
&mut guard,
|
||||
)
|
||||
.await;
|
||||
.await
|
||||
.expect("a fully drained and fenced deadline retains the live cohort across supervisor iterations");
|
||||
assert!(Arc::ptr_eq(&continuation.cohort, &service_cohort));
|
||||
assert!(continuation.matches_claim(&cycle_revision, 3));
|
||||
assert!(
|
||||
!continuation.matches_claim(&cycle_revision, 4),
|
||||
"an intervening owner ends the continuation"
|
||||
);
|
||||
assert!(
|
||||
!continuation.matches_claim(&DataUsageCacheRevision::Missing, 3),
|
||||
"a replaced cursor ends the continuation"
|
||||
);
|
||||
|
||||
assert!(guard.is_released());
|
||||
assert_eq!(leader_epoch, 2, "deadline handler should claim the next epoch");
|
||||
@@ -988,6 +1069,7 @@ struct MemoryConfigStore {
|
||||
insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
|
||||
read_errors: Mutex<HashMap<String, EcstoreError>>,
|
||||
delayed_gets: Mutex<HashMap<String, Duration>>,
|
||||
checkpoint_page_delay_ms: AtomicUsize,
|
||||
non_regular_objects: Mutex<HashSet<String>>,
|
||||
fail_put_number: Mutex<HashMap<String, usize>>,
|
||||
/// Fail every put whose number is at least the stored value.
|
||||
@@ -999,6 +1081,7 @@ struct MemoryConfigStore {
|
||||
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
|
||||
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
|
||||
error_after_commit_deletes: Mutex<HashSet<String>>,
|
||||
fail_prefix_deletes: AtomicBool,
|
||||
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
|
||||
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
|
||||
put_counts: Mutex<HashMap<String, usize>>,
|
||||
@@ -1045,6 +1128,17 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
|
||||
.await
|
||||
.push(crate::storage_api::ecstore_diagnostic::current_read_version_entry_source());
|
||||
let key = memory_config_key(bucket, object);
|
||||
let _serial_page_read = if object.contains(".pages/") {
|
||||
restore_deadline::admit_page(object).await
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if object.contains(".pages/") {
|
||||
let delay = self.checkpoint_page_delay_ms.load(Ordering::Relaxed);
|
||||
if delay != 0 {
|
||||
tokio::time::sleep(Duration::from_millis(delay as u64)).await;
|
||||
}
|
||||
}
|
||||
if let Some(error) = self.read_errors.lock().await.get(&key).cloned() {
|
||||
return Err(error);
|
||||
}
|
||||
@@ -1637,6 +1731,94 @@ async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn single_drive_cycle_persists_usage_and_advances_with_healing_marker() {
|
||||
use crate::scanner_io::{clear_dirty_usage_buckets_for_tests, tests::wait_for_namespace_commit_tails};
|
||||
use crate::storage_api::scan::{HEALING_MARKER_PATH, MakeBucketOptions};
|
||||
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
|
||||
let store = setup_scanner_cycle_store_at_path_with_layout(temp_dir.path(), true, 1, 1, 1).await;
|
||||
let bucket = "marked-drive-bucket";
|
||||
store
|
||||
.make_bucket(bucket, &MakeBucketOptions::default())
|
||||
.await
|
||||
.expect("fixture bucket should be created");
|
||||
let mut reader = PutObjReader::from_vec(vec![7; 16]);
|
||||
store
|
||||
.put_object(bucket, "object", &mut reader, &ObjectOptions::default())
|
||||
.await
|
||||
.expect("fixture object should persist");
|
||||
wait_for_namespace_commit_tails(store.as_ref()).await;
|
||||
|
||||
let marker = temp_dir
|
||||
.path()
|
||||
.join("pool0/disk0")
|
||||
.join(RUSTFS_META_BUCKET)
|
||||
.join(HEALING_MARKER_PATH);
|
||||
let marker_contents = b"leftover marker";
|
||||
tokio::fs::write(&marker, marker_contents)
|
||||
.await
|
||||
.expect("fixture healing marker should persist");
|
||||
let set = &store.pools[0].disk_set[0];
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
while !set.get_online_disks_with_healing(true).await.1 {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("disk info should observe the marker after its cache expires");
|
||||
|
||||
let ctx = CancellationToken::new();
|
||||
let mut cycle_info = CurrentCycle {
|
||||
next: 1,
|
||||
..Default::default()
|
||||
};
|
||||
let mut revision = DataUsageCacheRevision::Missing;
|
||||
let leader_epoch = 11;
|
||||
let outcome = tokio::time::timeout(
|
||||
Duration::from_secs(60),
|
||||
run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch),
|
||||
)
|
||||
.await
|
||||
.expect("scanner cycle should finish");
|
||||
assert!(
|
||||
matches!(
|
||||
outcome,
|
||||
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
|
||||
),
|
||||
"unexpected cycle outcome: {outcome:?}",
|
||||
);
|
||||
|
||||
let mut set_snapshot = DataUsageCache::default();
|
||||
set_snapshot
|
||||
.load_with_revisions(Arc::clone(set), DATA_USAGE_CACHE_NAME)
|
||||
.await
|
||||
.expect("set usage snapshot should be readable");
|
||||
assert!(set_snapshot.info.snapshot_complete);
|
||||
assert_eq!(set_snapshot.info.next_cycle, 1);
|
||||
|
||||
let usage = read_config(store.clone(), DATA_USAGE_OBJ_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("global usage snapshot should be durable");
|
||||
let usage: DataUsageInfo = serde_json::from_slice(&usage).expect("global usage snapshot should decode");
|
||||
assert!(usage.usage_snapshot_complete);
|
||||
assert!(usage.last_update.is_some());
|
||||
assert_eq!(usage.buckets_usage[bucket].objects_count, 1);
|
||||
|
||||
let persisted_cycle = read_config(store.clone(), DATA_USAGE_BLOOM_NAME_PATH.as_str())
|
||||
.await
|
||||
.expect("scanner cycle cursor should be durable");
|
||||
let (persisted_cycle, persisted_epoch) =
|
||||
decode_scanner_cycle_state(&persisted_cycle).expect("scanner cycle cursor should decode");
|
||||
assert_eq!(persisted_epoch, leader_epoch);
|
||||
assert_eq!(persisted_cycle.next, 2);
|
||||
assert_eq!(tokio::fs::read(marker).await.expect("healing marker should remain"), marker_contents);
|
||||
clear_dirty_usage_buckets_for_tests();
|
||||
global_metrics().set_cycle(None).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn coordinator_publishes_authority_during_pending_put() {
|
||||
@@ -5005,6 +5187,15 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
|
||||
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
|
||||
let key = memory_config_key(bucket, object);
|
||||
let mut objects = self.objects.lock().await;
|
||||
if opts.delete_prefix && !opts.delete_prefix_object {
|
||||
if self.fail_prefix_deletes.load(Ordering::Relaxed) {
|
||||
return Err(EcstoreError::other("injected prefix deletion failure"));
|
||||
}
|
||||
assert!(opts.http_preconditions.is_none(), "fixture prefix deletion cannot condition one object");
|
||||
objects.retain(|object, _| !object.starts_with(&key));
|
||||
self.revisions.lock().await.retain(|object, _| !object.starts_with(&key));
|
||||
return Ok(ObjectInfo::default());
|
||||
}
|
||||
if !objects.contains_key(&key) {
|
||||
return Err(EcstoreError::FileNotFound);
|
||||
}
|
||||
|
||||
@@ -253,6 +253,9 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
.await;
|
||||
assert!(matches!(outcome, ScannerCycleOutcome::Partial));
|
||||
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
|
||||
let report = global_metrics().report().await;
|
||||
assert_eq!(report.last_cycle_partial_reason, "objects");
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("object_budget"));
|
||||
assert!(budget.progress().0 > 0);
|
||||
assert!(budget.cycle_state_persisted());
|
||||
assert_eq!(cycle.next, 353);
|
||||
@@ -410,6 +413,16 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
super::cycle_persist_failure::fail_saves(1, false, SCANNER_PERSIST_CAS_RETRIES + 1)
|
||||
}
|
||||
});
|
||||
use tracing::instrument::WithSubscriber as _;
|
||||
let error_log = tempfile::NamedTempFile::new().expect("error log");
|
||||
let subscriber = tracing_subscriber::fmt()
|
||||
.json()
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_max_level(tracing::Level::ERROR)
|
||||
.with_writer(error_log.reopen().expect("error log writer"))
|
||||
.finish();
|
||||
let _interest_guard = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
|
||||
let tail_outcome = run_data_scanner_cycle_with_budget(
|
||||
&ctx,
|
||||
&store,
|
||||
@@ -423,7 +436,23 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
scope_fallback: None,
|
||||
},
|
||||
)
|
||||
.with_subscriber(subscriber)
|
||||
.await;
|
||||
let error_events = std::fs::read_to_string(error_log.path()).expect("read error log");
|
||||
let event = error_events
|
||||
.lines()
|
||||
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON error event"))
|
||||
.find(|event| event["fields"]["state"] == "incomplete" && event["fields"]["cycle"] == 353)
|
||||
.expect("unbudgeted incomplete observation should be visible at error level");
|
||||
assert_eq!(event["fields"]["leader_epoch"], 1);
|
||||
assert_eq!(event["fields"]["partial_stage"], "mixed_plan_recertification_pending");
|
||||
let report = global_metrics().report().await;
|
||||
assert_eq!(report.last_cycle_partial_reason, "unknown");
|
||||
if fail_cycle_save == Some(false) {
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
|
||||
} else {
|
||||
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("mixed_plan_recertification_pending"));
|
||||
}
|
||||
assert!(!tail_budget.budget_elapsed(), "this is not the budget-exhaustion fast path");
|
||||
checkpoint.load(set, &cache_name).await.expect("reload the saved mixed tail");
|
||||
assert!(!checkpoint.info.snapshot_complete);
|
||||
@@ -527,6 +556,22 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
|
||||
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_diagnostics_keep_the_activity_fence_distinct_from_checkpoint_progress() {
|
||||
let observation =
|
||||
ScannerCycleResult::new(ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable), None)
|
||||
.with_usage_update_emitted(true);
|
||||
assert_eq!(
|
||||
partial_cycle_diagnostic(&observation),
|
||||
("activity_unverified_observation", "activity_baseline_unverified")
|
||||
);
|
||||
let checkpoint = ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_partial_bucket_progress(true);
|
||||
assert_eq!(
|
||||
partial_cycle_diagnostic(&checkpoint),
|
||||
("saved_bucket_progress", "bucket_checkpoint_saved")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn partial_progress_does_not_mask_missing_updates_or_persistence_failure() {
|
||||
for (persisted, expected) in [
|
||||
|
||||
@@ -0,0 +1,473 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::BUCKET_META_PREFIX;
|
||||
|
||||
mod save_review;
|
||||
|
||||
const NAME: &str = "bucket/.usage-cache.bin";
|
||||
|
||||
fn empty_bucket_checkpoint() -> DataUsageCache {
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".into();
|
||||
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
|
||||
cache.info.scan_identity = Some(crate::data_usage_define::DataUsageScanIdentity {
|
||||
version: crate::data_usage_define::DATA_USAGE_SCAN_CHECKPOINT_VERSION,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(42),
|
||||
set_layout: DataUsageScanPlanDigest([7; 32]),
|
||||
publication_epoch: 0,
|
||||
tier_registry_generation: 1,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
});
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
cache
|
||||
}
|
||||
|
||||
fn seeded() -> DataUsageCache {
|
||||
let mut cache = empty_bucket_checkpoint();
|
||||
for index in 0..4096 {
|
||||
cache.replace(
|
||||
&format!("bucket/prefix-{index:05}"),
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
size: index + 1,
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
cache
|
||||
}
|
||||
|
||||
async fn layout(store: &MemoryConfigStore) -> HashMap<String, (usize, usize)> {
|
||||
let puts = store.put_counts.lock().await.clone();
|
||||
store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.iter()
|
||||
.map(|(k, v)| (k.clone(), (puts[k], v.len())))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn page_bytes_after_save(before: &HashMap<String, (usize, usize)>, after: &HashMap<String, (usize, usize)>) -> (usize, usize) {
|
||||
let mut existing = 0;
|
||||
let mut uploaded = 0;
|
||||
for (path, &(writes, size)) in after.iter().filter(|(path, _)| path.contains(".pages/")) {
|
||||
if let Some(&(previous_writes, _)) = before.get(path) {
|
||||
assert_eq!(writes, previous_writes, "a committed immutable page was resubmitted");
|
||||
existing += size;
|
||||
} else {
|
||||
uploaded += size;
|
||||
}
|
||||
}
|
||||
(existing, uploaded)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn compacted_checkpoint_advances_one_child_without_rewriting_300k_names() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = empty_bucket_checkpoint();
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: plan,
|
||||
requested_plan: plan,
|
||||
});
|
||||
cache.info.scan_directory_source = Some(format!("{}|disk-a", uuid::Uuid::from_u128(1)));
|
||||
let [first, second] = crate::scanner_folder::DirectoryCheckpoint::wide_storage_fixture(&cache.info).await;
|
||||
let mut before: Option<HashMap<String, (usize, usize)>> = None;
|
||||
let mut prior_progress = None;
|
||||
for (objects, receipt) in [(121_034, first), (121_035, second)] {
|
||||
cache.info.scan_directory_work = vec![receipt];
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
compacted: true,
|
||||
objects,
|
||||
size: objects * 1024,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save compacted receipt");
|
||||
let after = layout(&store).await;
|
||||
if let Some(before) = before.as_ref() {
|
||||
let (old_bytes, new_bytes) = page_bytes_after_save(before, &after);
|
||||
eprintln!("300k-name checkpoint: {new_bytes} bytes in new pages, {old_bytes} bytes in existing pages");
|
||||
assert!(old_bytes > 3_000_000, "fixture must contain the full physical listing");
|
||||
assert!(
|
||||
new_bytes > 0 && new_bytes < old_bytes / 10,
|
||||
"one more completed child wrote {new_bytes} bytes alongside {old_bytes} reused bytes"
|
||||
);
|
||||
}
|
||||
before = Some(after);
|
||||
let mut restored = DataUsageCache::default();
|
||||
restored
|
||||
.load(store.clone(), NAME)
|
||||
.await
|
||||
.expect("cold read of compacted receipt");
|
||||
assert!(restored.info.scan_directory_work[0].is_bound_to(&restored.info));
|
||||
let progress = restored.durable_scan_progress();
|
||||
if let Some(prior) = &prior_progress {
|
||||
assert!(progress.advanced_from(prior), "the new child must earn another durable work turn");
|
||||
}
|
||||
prior_progress = Some(progress);
|
||||
let total = restored.checked_flatten("bucket").expect("connected root after restore");
|
||||
assert_eq!((total.objects, total.size), (objects, objects * 1024));
|
||||
cache = restored;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn dormant_linked_tree_pages_survive_source_switch_and_cold_reencoding() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
let plan = DataUsageScanPlanDigest([1; 32]);
|
||||
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
|
||||
started_plan: plan,
|
||||
requested_plan: plan,
|
||||
});
|
||||
let endpoints = std::collections::HashSet::from(["disk-a".to_owned(), "disk-b".to_owned()]);
|
||||
let a = format!("{}|disk-a", uuid::Uuid::from_u128(1));
|
||||
let b = format!("{}|disk-b", uuid::Uuid::from_u128(2));
|
||||
cache.info.scan_directory_source = Some(a.clone());
|
||||
cache.activate_directory_source(b.clone(), &endpoints);
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("before first checkpoint");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("persist dormant linked tree");
|
||||
let previous = layout(&store).await;
|
||||
|
||||
let mut cold = DataUsageCache::default();
|
||||
cold.load(store.clone(), NAME)
|
||||
.await
|
||||
.expect("restore checkpoint in a new process");
|
||||
cold.activate_directory_source(a, &endpoints);
|
||||
assert_eq!(cold.checked_flatten("bucket").expect("first source entries").objects, 4096);
|
||||
cold.activate_directory_source(b, &endpoints);
|
||||
assert_eq!(cold.checked_flatten("bucket").expect("second source root").objects, 0);
|
||||
cold.replace(
|
||||
"bucket/new-key",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 99,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("before second checkpoint");
|
||||
cold.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save one new child");
|
||||
let after = layout(&store).await;
|
||||
let (prior_bytes, new_bytes) = page_bytes_after_save(&previous, &after);
|
||||
assert!(prior_bytes > 128 * 1024);
|
||||
assert!(
|
||||
new_bytes > 0 && new_bytes < prior_bytes / 8,
|
||||
"dormant tree rewritten: {new_bytes} new bytes for {prior_bytes} prior bytes"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_writes_only_changed_pages_and_commits_root_last() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
|
||||
.await
|
||||
.expect("initial checkpoint");
|
||||
let before = layout(&store).await;
|
||||
let encoded = cache.marshal_msg().expect("encode baseline").len();
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
assert!(before[&main].1 < 8192, "only a compact manifest belongs in the root");
|
||||
assert!(before.keys().any(|key| key.contains(".pages/")), "large cache must be paged");
|
||||
|
||||
let mut restarted = DataUsageCache::default();
|
||||
let revisions = restarted
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("restart from durable pages");
|
||||
let prior = restarted.checked_flatten("bucket").expect("all old entries");
|
||||
restarted.replace(
|
||||
"bucket/prefix-tail",
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
versions: 1,
|
||||
size: 4711,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
store.fail_puts_from.lock().await.insert(main.clone(), before[&main].0 + 1);
|
||||
restarted
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect_err("commit interrupted after writing pages");
|
||||
let interrupted = layout(&store).await;
|
||||
let mut observer = DataUsageCache::default();
|
||||
observer.load(store.clone(), NAME).await.expect("last root still readable");
|
||||
assert_eq!(observer.checked_flatten("bucket").expect("previous coverage").size, prior.size);
|
||||
store.fail_puts_from.lock().await.remove(&main);
|
||||
restarted
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("retry interrupted commit");
|
||||
observer.load(store.clone(), NAME).await.expect("load committed result");
|
||||
let actual = observer.checked_flatten("bucket").expect("new root");
|
||||
assert_eq!((actual.objects, actual.size), (4097, prior.size + 4711));
|
||||
let after = layout(&store).await;
|
||||
for (key, (writes, _)) in &interrupted {
|
||||
if key.contains(".bin.pages/") {
|
||||
assert_eq!(*writes, after[key].0, "retry restaged a previously successful immutable page");
|
||||
}
|
||||
}
|
||||
let new_bytes: usize = after
|
||||
.iter()
|
||||
.filter(|(key, _)| !before.contains_key(*key))
|
||||
.map(|(_, (_, size))| *size)
|
||||
.sum();
|
||||
assert!(
|
||||
new_bytes < encoded / 3,
|
||||
"one append staged {new_bytes} new bytes for a {encoded}-byte cache"
|
||||
);
|
||||
for (key, (writes, _)) in &before {
|
||||
if key.contains(".pages/") {
|
||||
assert_eq!(*writes, after[key].0, "a committed page was resubmitted");
|
||||
}
|
||||
}
|
||||
restarted.info.next_cycle += 1;
|
||||
let stale = restarted.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0).await;
|
||||
assert!(
|
||||
matches!(stale, Err(EcstoreError::PreconditionFailed)),
|
||||
"stale writer cannot replace the new root"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_cold_restore_spans_fixed_duration_and_repairs_corrupt_primary() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("read revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("write both copies");
|
||||
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let mut completed_at = None;
|
||||
for round in 1..=128 {
|
||||
if let Ok(result) = tokio::time::timeout(Duration::from_millis(26), loaded.load(store.clone(), NAME)).await {
|
||||
result.expect("resume exact committed manifest");
|
||||
completed_at = Some(round);
|
||||
break;
|
||||
}
|
||||
}
|
||||
assert!(
|
||||
completed_at.is_some_and(|round| round > 1),
|
||||
"cold recovery must advance across fixed deadlines"
|
||||
);
|
||||
store.checkpoint_page_delay_ms.store(0, Ordering::Relaxed);
|
||||
let prior = loaded.checked_flatten("bucket").expect("cold cache totals");
|
||||
assert_eq!((prior.objects, prior.size), (4096, 4096 * 4097 / 2));
|
||||
|
||||
{
|
||||
let mut objects = store.objects.lock().await;
|
||||
let (_, block) = objects
|
||||
.iter_mut()
|
||||
.find(|(key, block)| key.contains(".bin.pages/") && block.len() > 8192)
|
||||
.expect("primary data page");
|
||||
block[0] ^= 0xff;
|
||||
}
|
||||
let revisions = loaded
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("read unaffected backup");
|
||||
assert_eq!(
|
||||
revisions,
|
||||
DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("re-enter admission on same heads")
|
||||
);
|
||||
loaded
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("replace damaged primary generation");
|
||||
store.objects.lock().await.retain(|key, _| !key.contains(".bkp"));
|
||||
let mut primary = DataUsageCache::default();
|
||||
primary.load(store, NAME).await.expect("repaired independent primary");
|
||||
assert_eq!(primary.checked_flatten("bucket").expect("primary totals").size, prior.size);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn paged_checkpoint_bucket_admission_uses_the_scan_budget_during_cold_load() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
|
||||
|
||||
let owner = CancellationToken::new();
|
||||
let mut loaded = DataUsageCache::default();
|
||||
for round in 0..128 {
|
||||
let budget = ScannerCycleBudget::new_with_progress_tracking(
|
||||
&owner,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_millis(45)),
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
// Exercise the same admission used by local and remote workers. The
|
||||
// worker may drain after the deadline; only the scan token interrupts
|
||||
// recovery here, as it does in the daemon.
|
||||
let revisions = loaded
|
||||
.load_for_scan(store.clone(), NAME, &budget.token(), &budget)
|
||||
.await
|
||||
.unwrap();
|
||||
if revisions.is_some() {
|
||||
assert!(round > 0, "the first slice cannot finish this physical restore");
|
||||
assert!(!budget.token().is_cancelled(), "return work only to an active scan");
|
||||
let totals = loaded.checked_flatten("bucket").expect("restored full bucket");
|
||||
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("bucket admission repeatedly consumed the entire budget restoring the same root");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn paged_checkpoint_retirement_preserves_independent_backup_and_legacy_projection() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.info.snapshot_complete = true;
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
|
||||
.await
|
||||
.expect("persist first generation");
|
||||
let original = layout(&store).await;
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
let backup = format!("{main}.bkp");
|
||||
let legacy =
|
||||
DataUsageCache::unmarshal(&store.objects.lock().await[&main]).expect("old reader can read compact complete root");
|
||||
let totals = legacy
|
||||
.checked_flatten_complete_scope("bucket")
|
||||
.expect("legacy compact totals");
|
||||
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
|
||||
let initial = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("first generation revisions");
|
||||
cache.info.next_cycle += 1;
|
||||
store
|
||||
.fail_puts_from
|
||||
.lock()
|
||||
.await
|
||||
.insert(backup.clone(), original[&backup].0 + 1);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &initial, 0)
|
||||
.await
|
||||
.expect("commit primary despite failed backup");
|
||||
let updated = layout(&store).await;
|
||||
for key in original.keys().filter(|key| key.contains(".pages/")) {
|
||||
assert_eq!(
|
||||
updated.contains_key(key),
|
||||
key.contains(".bkp.pages/"),
|
||||
"unexpected reclamation scope: {key}"
|
||||
);
|
||||
}
|
||||
let mut latest = DataUsageCache::default();
|
||||
latest.load(store.clone(), NAME).await.expect("new main remains readable");
|
||||
assert_eq!(latest.info.next_cycle, cache.info.next_cycle);
|
||||
store.objects.lock().await.remove(&main);
|
||||
latest
|
||||
.load(store, NAME)
|
||||
.await
|
||||
.expect("previous independent backup remains readable");
|
||||
assert_eq!(latest.info.next_cycle + 1, cache.info.next_cycle);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn paged_checkpoint_scan_handoff_checks_the_latest_root_before_using_its_totals() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.cache.clear();
|
||||
let replace = |cache: &mut DataUsageCache, size| {
|
||||
cache.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
};
|
||||
replace(&mut cache, 17);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
store.delayed_gets.lock().await.insert(main, Duration::from_secs(6));
|
||||
|
||||
let parent = CancellationToken::new();
|
||||
let budget = || {
|
||||
ScannerCycleBudget::new_with_progress_tracking(
|
||||
&parent,
|
||||
ScannerCycleBudgetConfig {
|
||||
max_duration: Some(Duration::from_secs(10)),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
};
|
||||
let first = budget();
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(
|
||||
loaded
|
||||
.load_for_scan(store.clone(), NAME, &first.token(), &first)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
assert!(!first.token().is_cancelled(), "the slow load moved its result to the next admission");
|
||||
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
|
||||
replace(&mut cache, 99);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.unwrap();
|
||||
let next = budget();
|
||||
assert!(
|
||||
loaded
|
||||
.load_for_scan(store, NAME, &next.token(), &next)
|
||||
.await
|
||||
.unwrap()
|
||||
.is_some()
|
||||
);
|
||||
assert_eq!(loaded.checked_flatten("bucket").expect("new root").size, 99);
|
||||
}
|
||||
@@ -0,0 +1,192 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn indefinite_partial_checkpoints_keep_unreachable_bytes_bounded() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
let mut initial_bytes = 0;
|
||||
for round in 0..10 {
|
||||
for (name, entry) in &mut cache.cache {
|
||||
if name != "bucket" {
|
||||
entry.size += 17;
|
||||
}
|
||||
}
|
||||
// A single long-running cycle can checkpoint many times without changing cycle number.
|
||||
assert!(!cache.info.snapshot_complete);
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current heads");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("commit partial");
|
||||
let current: usize = layout(&store)
|
||||
.await
|
||||
.iter()
|
||||
.filter(|(path, _)| path.contains(".pages/"))
|
||||
.map(|(_, (_, size))| size)
|
||||
.sum();
|
||||
if round == 0 {
|
||||
initial_bytes = current;
|
||||
assert!(initial_bytes > 100_000);
|
||||
}
|
||||
assert!(
|
||||
current <= initial_bytes * 4,
|
||||
"after {} partials, {current} page bytes for {initial_bytes} initially",
|
||||
round + 1
|
||||
);
|
||||
}
|
||||
let loaded = &mut DataUsageCache::default();
|
||||
loaded.load(store, NAME).await.expect("restore bounded checkpoint");
|
||||
assert_eq!(
|
||||
loaded.checked_flatten("bucket").expect("valid restored tree").size,
|
||||
cache.checked_flatten("bucket").expect("fixture tree").size
|
||||
);
|
||||
}
|
||||
|
||||
enum BackupDamage {
|
||||
Missing,
|
||||
Corrupt,
|
||||
UnreadableMetadata,
|
||||
}
|
||||
|
||||
async fn survives_lost_primary_after_backup_page_damage(damage: BackupDamage) {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let cache = seeded();
|
||||
let first = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty heads");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
|
||||
.await
|
||||
.expect("initial checkpoint");
|
||||
let modify_tail = |cache: &mut DataUsageCache| {
|
||||
cache.cache.get_mut("bucket/prefix-04095").expect("last leaf").size += 99;
|
||||
};
|
||||
let control = Arc::new(MemoryConfigStore::default());
|
||||
let mut target = cache.clone();
|
||||
modify_tail(&mut target);
|
||||
let empty = DataUsageCache::read_revisions(control.clone(), NAME)
|
||||
.await
|
||||
.expect("empty control store");
|
||||
target
|
||||
.save_with_revisions_for_epoch(control.clone(), NAME, &empty, 0)
|
||||
.await
|
||||
.expect("encode target in an independent fresh bundle");
|
||||
{
|
||||
let control_objects = control.objects.lock().await;
|
||||
let mut objects = store.objects.lock().await;
|
||||
let victim = objects
|
||||
.iter()
|
||||
.find_map(|(path, data)| {
|
||||
// A fresh target bundle only contains currently reachable objects.
|
||||
(path.contains(".bkp.pages/") && data.len() > 8192 && control_objects.get(path) == Some(data))
|
||||
.then(|| path.clone())
|
||||
})
|
||||
.expect("a large unchanged leaf is reachable in both manifests");
|
||||
match damage {
|
||||
BackupDamage::Corrupt => objects.get_mut(&victim).expect("existing backup leaf")[0] ^= 1,
|
||||
BackupDamage::Missing => {
|
||||
objects.remove(&victim);
|
||||
}
|
||||
BackupDamage::UnreadableMetadata => {
|
||||
drop(objects);
|
||||
store.read_errors.lock().await.insert(victim, EcstoreError::FileCorrupt);
|
||||
}
|
||||
}
|
||||
}
|
||||
let mut from_main = DataUsageCache::default();
|
||||
let revisions = from_main
|
||||
.load_with_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("primary remains healthy");
|
||||
modify_tail(&mut from_main);
|
||||
from_main
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("incremental commit while primary is healthy");
|
||||
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
|
||||
store.objects.lock().await.remove(&main);
|
||||
let mut recovered = DataUsageCache::default();
|
||||
recovered.load(store, NAME).await.expect("standalone cold backup recovery");
|
||||
let actual = recovered
|
||||
.checked_flatten("bucket")
|
||||
.expect("a connected backup after losing its primary");
|
||||
assert_eq!((actual.objects, actual.size), (4096, 4096 * 4097 / 2 + 99));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_repairs_missing_backup_leaf() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::Missing).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_repairs_corrupt_backup_leaf() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::Corrupt).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn incremental_save_rebuilds_backup_with_unreadable_page_metadata() {
|
||||
survives_lost_primary_after_backup_page_damage(BackupDamage::UnreadableMetadata).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn unreclaimed_partial_generation_bounds_staging_and_can_retry() {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = seeded();
|
||||
cache.info.snapshot_complete = true;
|
||||
let empty = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("empty store");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &empty, 0)
|
||||
.await
|
||||
.expect("committed predecessor");
|
||||
store.fail_prefix_deletes.store(true, Ordering::Relaxed);
|
||||
cache.info.snapshot_complete = false;
|
||||
cache.info.next_cycle += 1;
|
||||
for _ in 0..8 {
|
||||
for (name, entry) in &mut cache.cache {
|
||||
if name != "bucket" {
|
||||
entry.size += 17;
|
||||
}
|
||||
}
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("current heads");
|
||||
let before = layout(&store).await;
|
||||
if let Err(error) = cache.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0).await {
|
||||
assert!(error.to_string().contains("reclaim"), "unexpected save failure: {error}");
|
||||
assert_eq!(before, layout(&store).await, "a blocked retirement must not stage further objects");
|
||||
assert!(
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert_eq!(
|
||||
before,
|
||||
layout(&store).await,
|
||||
"retry while cleanup is unavailable must leave both generations readable"
|
||||
);
|
||||
let mut previous = DataUsageCache::default();
|
||||
previous.load(store.clone(), NAME).await.expect("last committed checkpoint");
|
||||
assert!(previous.checked_flatten("bucket").is_some());
|
||||
store.fail_prefix_deletes.store(false, Ordering::Relaxed);
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("save after previous retirement is reclaimed");
|
||||
previous.load(store.clone(), NAME).await.expect("read latest checkpoint");
|
||||
assert_eq!(
|
||||
cache.checked_flatten("bucket").expect("fixture").size,
|
||||
previous.checked_flatten("bucket").expect("recovered root").size
|
||||
);
|
||||
return;
|
||||
}
|
||||
}
|
||||
panic!("persistent retirement failure allowed unbounded staging");
|
||||
}
|
||||
@@ -160,6 +160,7 @@ async fn issue_8297_contended_follower_clears_only_transient_cycle_recovery() {
|
||||
ScannerMaintenanceFeatures::default(),
|
||||
None,
|
||||
runtime.startup_token(),
|
||||
&mut None,
|
||||
),
|
||||
)
|
||||
.await
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
use crate::data_usage_define::{DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCacheRevisions, DataUsageScanIdentity};
|
||||
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
|
||||
|
||||
const NAME: &str = "bucket/.usage-cache.bin";
|
||||
const COUNT: usize = 1536;
|
||||
|
||||
struct ReadWindow {
|
||||
pages: usize,
|
||||
device: Arc<Semaphore>,
|
||||
completed: std::sync::Mutex<Vec<String>>,
|
||||
deadline: Notify,
|
||||
}
|
||||
|
||||
tokio::task_local! {
|
||||
static READ_WINDOW: Arc<ReadWindow>;
|
||||
}
|
||||
|
||||
// Model a serial device with one page of I/O time per admission. Completing
|
||||
// a second read would exceed the deadline, so the caller drops that waiter.
|
||||
pub(super) async fn admit_page(object: &str) -> Option<OwnedSemaphorePermit> {
|
||||
let window = READ_WINDOW.try_with(Arc::clone).ok()?;
|
||||
let permit = window.device.clone().acquire_owned().await.expect("device stays open");
|
||||
{
|
||||
let mut completed = window.completed.lock().expect("device history");
|
||||
if completed.len() < window.pages {
|
||||
completed.push(object.into());
|
||||
return Some(permit);
|
||||
}
|
||||
}
|
||||
drop(permit);
|
||||
window.deadline.notify_one();
|
||||
std::future::pending().await
|
||||
}
|
||||
|
||||
async fn window(store: &Arc<MemoryConfigStore>, loaded: &mut DataUsageCache) -> (Option<DataUsageCacheRevisions>, Vec<String>) {
|
||||
let window = Arc::new(ReadWindow {
|
||||
pages: 1,
|
||||
device: Arc::new(Semaphore::new(1)),
|
||||
completed: Default::default(),
|
||||
deadline: Notify::new(),
|
||||
});
|
||||
let result = READ_WINDOW
|
||||
.scope(window.clone(), async {
|
||||
tokio::select! {
|
||||
biased;
|
||||
value = loaded.load_with_revisions(store.clone(), NAME) => Some(value.expect("checkpoint load")),
|
||||
() = window.deadline.notified() => None,
|
||||
}
|
||||
})
|
||||
.await;
|
||||
let pages = std::mem::take(&mut *window.completed.lock().expect("device history"));
|
||||
(result, pages)
|
||||
}
|
||||
|
||||
async fn fixture() -> (Arc<MemoryConfigStore>, DataUsageCache) {
|
||||
let store = Arc::new(MemoryConfigStore::default());
|
||||
let mut cache = DataUsageCache::default();
|
||||
cache.info.name = "bucket".into();
|
||||
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
|
||||
cache.info.scan_identity = Some(DataUsageScanIdentity {
|
||||
version: DATA_USAGE_SCAN_CHECKPOINT_VERSION,
|
||||
bucket_incarnation: uuid::Uuid::from_u128(9),
|
||||
set_layout: DataUsageScanPlanDigest([7; 32]),
|
||||
publication_epoch: 0,
|
||||
tier_registry_generation: 1,
|
||||
scan_mode: HealScanMode::Normal,
|
||||
});
|
||||
cache.replace("bucket", "", DataUsageEntry::default());
|
||||
for key in 0..COUNT {
|
||||
cache.replace(
|
||||
&format!("bucket/key-{key:08}"),
|
||||
"bucket",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: key + 1,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
}
|
||||
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
|
||||
.await
|
||||
.expect("initial revisions");
|
||||
cache
|
||||
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
|
||||
.await
|
||||
.expect("seed both copies");
|
||||
(store, cache)
|
||||
}
|
||||
|
||||
async fn corrupt_primary(store: &MemoryConfigStore) {
|
||||
let mut objects = store.objects.lock().await;
|
||||
let mut pages = 0;
|
||||
for (_, data) in objects.iter_mut().filter(|(path, _)| path.contains(".bin.pages/")) {
|
||||
data[0] ^= 1;
|
||||
pages += 1;
|
||||
}
|
||||
assert!(pages > 4, "fixture must span more than one physical read batch");
|
||||
}
|
||||
|
||||
async fn finish(store: &Arc<MemoryConfigStore>, backup: bool) {
|
||||
finish_across_windows(store, backup, None).await;
|
||||
}
|
||||
|
||||
async fn finish_across_windows(store: &Arc<MemoryConfigStore>, backup: bool, competing: Option<&Arc<MemoryConfigStore>>) {
|
||||
let mut loaded = DataUsageCache::default();
|
||||
let mut main_reads = 0;
|
||||
let mut backup_reads = 0;
|
||||
for _ in 0..128 {
|
||||
let (result, pages) = window(store, &mut loaded).await;
|
||||
for page in pages {
|
||||
if page.contains(".bkp.pages/") {
|
||||
backup_reads += 1;
|
||||
} else {
|
||||
main_reads += 1;
|
||||
}
|
||||
}
|
||||
if result.is_some() {
|
||||
let total = loaded.checked_flatten("bucket").expect("restored bucket");
|
||||
assert_eq!(total.objects, COUNT);
|
||||
assert_eq!(total.size, COUNT * (COUNT + 1) / 2);
|
||||
if backup {
|
||||
assert_eq!(main_reads, 1, "read the corrupt primary once for this revision");
|
||||
assert!(backup_reads > 4, "the independent backup must be restored across admissions");
|
||||
} else {
|
||||
assert!(main_reads > 4);
|
||||
assert_eq!(backup_reads, 0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if let Some(other) = competing {
|
||||
tokio::time::advance(Duration::from_secs(2 * 60 * 60)).await;
|
||||
let _ = window(other, &mut DataUsageCache::default()).await;
|
||||
}
|
||||
}
|
||||
panic!("checkpoint did not converge: {main_reads} primary and {backup_reads} backup page reads");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn serial_backend_restores_one_verified_page_per_deadline() {
|
||||
let (store, _) = fixture().await;
|
||||
finish(&store, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn concurrent_cancellations_keep_the_longer_verified_prefix() {
|
||||
let (store, _) = fixture().await;
|
||||
let longer = Arc::new(ReadWindow {
|
||||
pages: 3,
|
||||
device: Arc::new(Semaphore::new(1)),
|
||||
completed: Default::default(),
|
||||
deadline: Notify::new(),
|
||||
});
|
||||
let mut first = DataUsageCache::default();
|
||||
let mut read = Box::pin(READ_WINDOW.scope(longer.clone(), first.load(store.clone(), NAME)));
|
||||
tokio::select! {
|
||||
result = &mut read => panic!("checkpoint unexpectedly finished within three reads: {result:?}"),
|
||||
() = longer.deadline.notified() => {}
|
||||
}
|
||||
let mut concurrent = DataUsageCache::default();
|
||||
assert!(window(&store, &mut concurrent).await.0.is_none());
|
||||
drop(read);
|
||||
|
||||
let (_, next) = window(&store, &mut concurrent).await;
|
||||
let previous = longer.completed.lock().expect("first reader history");
|
||||
assert_eq!(previous.len(), 3);
|
||||
assert!(
|
||||
!previous.contains(&next[0]),
|
||||
"a shorter concurrent cancellation displaced a verified prefix"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn corrupt_primary_yields_future_admissions_to_the_healthy_backup() {
|
||||
let (store, _) = fixture().await;
|
||||
corrupt_primary(&store).await;
|
||||
finish(&store, true).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_primary_revision_supersedes_an_incomplete_backup_restore() {
|
||||
let (store, _) = fixture().await;
|
||||
corrupt_primary(&store).await;
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(window(&store, &mut loaded).await.0.is_none());
|
||||
let (result, pages) = window(&store, &mut loaded).await;
|
||||
assert!(result.is_none());
|
||||
assert!(pages[0].contains(".bkp.pages/"), "second admission should advance backup");
|
||||
|
||||
let mut new_primary = DataUsageCache::default();
|
||||
new_primary.info.name = "bucket".into();
|
||||
new_primary.replace(
|
||||
"bucket",
|
||||
"",
|
||||
DataUsageEntry {
|
||||
objects: 1,
|
||||
size: 19,
|
||||
..Default::default()
|
||||
},
|
||||
);
|
||||
let root = memory_config_key(RUSTFS_META_BUCKET, &format!("{}/{NAME}", crate::BUCKET_META_PREFIX));
|
||||
let bytes = rmp_serde::to_vec_named(&new_primary).expect("replacement main root");
|
||||
store.objects.lock().await.insert(root.clone(), bytes);
|
||||
*store.revisions.lock().await.get_mut(&root).expect("primary revision") += 1;
|
||||
let (result, pages) = window(&store, &mut loaded).await;
|
||||
assert!(result.is_some(), "must prefer the replacement main head immediately");
|
||||
assert!(pages.is_empty(), "must discard work on the older backup");
|
||||
let total = loaded.checked_flatten("bucket").expect("new primary root");
|
||||
assert_eq!((total.objects, total.size), (1, 19));
|
||||
}
|
||||
|
||||
async fn inject_primary_page_error(store: &MemoryConfigStore, error: EcstoreError) {
|
||||
let errors = store
|
||||
.objects
|
||||
.lock()
|
||||
.await
|
||||
.keys()
|
||||
.filter(|key| key.contains(".bin.pages/"))
|
||||
.map(|key| (key.clone(), error.clone()))
|
||||
.collect();
|
||||
*store.read_errors.lock().await = errors;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn a_transient_primary_page_error_does_not_schedule_the_backup() {
|
||||
let (store, _) = fixture().await;
|
||||
inject_primary_page_error(&store, EcstoreError::SlowDown).await;
|
||||
let mut loaded = DataUsageCache::default();
|
||||
assert!(window(&store, &mut loaded).await.0.is_none());
|
||||
store.read_errors.lock().await.clear();
|
||||
finish(&store, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn backup_phase_is_bound_to_its_storage_instance() {
|
||||
let (damaged, _) = fixture().await;
|
||||
corrupt_primary(&damaged).await;
|
||||
assert!(window(&damaged, &mut DataUsageCache::default()).await.0.is_none());
|
||||
|
||||
let (healthy, _) = fixture().await;
|
||||
let first = DataUsageCache::read_revisions(damaged.clone(), NAME)
|
||||
.await
|
||||
.expect("failed primary revision");
|
||||
let second = DataUsageCache::read_revisions(healthy.clone(), NAME)
|
||||
.await
|
||||
.expect("other storage revision");
|
||||
assert_eq!(first, second, "both stores must expose the same ETags");
|
||||
finish(&healthy, false).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_restore_keeps_verified_pages_until_another_bucket_needs_its_slot() {
|
||||
let (primary, _) = fixture().await;
|
||||
let (other, _) = fixture().await;
|
||||
finish_across_windows(&primary, false, Some(&other)).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn idle_restore_keeps_the_failed_primary_revision_across_scheduled_rounds() {
|
||||
let (primary, _) = fixture().await;
|
||||
let (other, _) = fixture().await;
|
||||
corrupt_primary(&primary).await;
|
||||
corrupt_primary(&other).await;
|
||||
finish_across_windows(&primary, true, Some(&other)).await;
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn physical_primary_page_corruption_falls_back_to_the_healthy_copy() {
|
||||
let (store, _) = fixture().await;
|
||||
inject_primary_page_error(&store, EcstoreError::FileCorrupt).await;
|
||||
finish(&store, true).await;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,720 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
//! Cycle-local execution can borrow an interrupted physical directory stream.
|
||||
//! Only the current scanner owns admission and classification; parked streams
|
||||
//! have no task, budget or authority to update a bucket cache.
|
||||
|
||||
use std::collections::HashMap;
|
||||
|
||||
mod checkpoint;
|
||||
mod pages;
|
||||
pub use checkpoint::DirectoryCheckpoint;
|
||||
pub(super) use checkpoint::Saved;
|
||||
pub(crate) use checkpoint::{DirectoryProgress, DirectoryReadProgress};
|
||||
use checkpoint::{Physical, Recorder, Scope};
|
||||
use pages::NamePages;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, LazyLock, Mutex, Weak};
|
||||
use std::time::{Duration, Instant, SystemTime};
|
||||
|
||||
use super::{DirectoryStream, Disk, FolderScanMode, ScannerBucketPrefixScanScope, native_io};
|
||||
use crate::data_usage_define::{DataUsageCacheInfo, DataUsageCacheSource, DataUsageScanIdentity, DataUsageScanProgress};
|
||||
use crate::scanner_budget::ScannerCycleBudget;
|
||||
|
||||
pub(super) const PARKED_DIRECTORIES_PER_DISK: usize = 32;
|
||||
|
||||
/// Only a positively identified disk can re-adopt a linked tree. A missing
|
||||
/// format on an otherwise visible local drive uses its directory generation;
|
||||
/// genuine format I/O errors propagate before any cached subtotals are used.
|
||||
pub(super) async fn physical_source(disk: &Arc<Disk>) -> Result<String, super::ScannerError> {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
use sha2::{Digest, Sha256};
|
||||
let endpoint = disk.endpoint().to_string();
|
||||
let generation = match disk.get_disk_id().await {
|
||||
Ok(Some(id)) if !id.is_nil() => id.to_string(),
|
||||
Ok(None) | Err(super::DiskError::UnformattedDisk) => {
|
||||
let path = disk.path();
|
||||
let stamp = DirectoryStamp::at(path.to_str().ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput))?).await?;
|
||||
// Bucket creation and scanner startup can change the volume root's
|
||||
// mtime/ctime. Birth time distinguishes inode reuse without making
|
||||
// ordinary directory content changes appear to replace the disk.
|
||||
if let Some(generation) = stamp.volume_generation() {
|
||||
let bytes = rmp_serde::to_vec_named(&(std::env::consts::OS, generation)).map_err(io::Error::other)?;
|
||||
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
|
||||
} else {
|
||||
// The platform cannot attest the generation across processes.
|
||||
uuid::Uuid::new_v4().to_string()
|
||||
}
|
||||
}
|
||||
Ok(Some(_)) => return Err(io::Error::new(io::ErrorKind::InvalidData, "scanner disk format contains a nil ID").into()),
|
||||
Err(error) => {
|
||||
let error = crate::StorageError::from(error);
|
||||
if crate::scanner_io::is_scanner_source_unavailable(&error) {
|
||||
return Err(super::ScannerError::source_unavailable(error));
|
||||
}
|
||||
return Err(io::Error::other(error).into());
|
||||
}
|
||||
};
|
||||
Ok(format!("{generation}|{endpoint}"))
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub(super) struct DirectoryScope {
|
||||
disk: Weak<Disk>,
|
||||
bucket: Arc<str>,
|
||||
source: DataUsageCacheSource,
|
||||
identity: DataUsageScanIdentity,
|
||||
progress: DataUsageScanProgress,
|
||||
mode: FolderScanMode,
|
||||
prefix_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
owner: (u64, u64),
|
||||
endpoint: String,
|
||||
watch: Option<[u8; 32]>,
|
||||
continuing: bool,
|
||||
}
|
||||
|
||||
impl DirectoryScope {
|
||||
pub(super) fn new(
|
||||
disk: &Arc<Disk>,
|
||||
cache: &super::DataUsageCache,
|
||||
mode: FolderScanMode,
|
||||
prefix_scope: Option<ScannerBucketPrefixScanScope>,
|
||||
) -> Option<Self> {
|
||||
use crate::storage_api::owner::EcstoreDiskAPI;
|
||||
let info = &cache.info;
|
||||
Some(Self {
|
||||
disk: Arc::downgrade(disk),
|
||||
bucket: Arc::from(info.name.as_str()),
|
||||
source: info.source?,
|
||||
identity: info.scan_identity.filter(DataUsageScanIdentity::is_valid)?,
|
||||
progress: info.scan_progress?,
|
||||
mode,
|
||||
prefix_scope,
|
||||
owner: (info.leader_epoch, info.next_cycle),
|
||||
endpoint: format!("|{}", disk.endpoint()),
|
||||
watch: info
|
||||
.scan_usage_watch
|
||||
.as_ref()
|
||||
.filter(|proof| proof.is_well_formed())
|
||||
.map(|proof| proof.digest().0),
|
||||
continuing: cache.has_scan_position(),
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn bind_source(&mut self, source: Option<&str>) {
|
||||
if let Some(source) = source {
|
||||
self.endpoint = source.to_owned();
|
||||
}
|
||||
}
|
||||
pub(super) fn endpoint(&self) -> &str {
|
||||
&self.endpoint
|
||||
}
|
||||
fn matches(&self, other: &Self) -> bool {
|
||||
self.endpoint == other.endpoint
|
||||
&& self.disk.ptr_eq(&other.disk)
|
||||
&& self.bucket == other.bucket
|
||||
&& self.source == other.source
|
||||
&& self.identity == other.identity
|
||||
&& self.progress == other.progress
|
||||
&& self.mode == other.mode
|
||||
&& self.watch == other.watch
|
||||
&& self.prefix_scope == other.prefix_scope
|
||||
&& (other.continuing || self.owner == other.owner)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Eq)]
|
||||
struct DirectoryStamp {
|
||||
modified: SystemTime,
|
||||
birth: Option<SystemTime>,
|
||||
#[cfg(unix)]
|
||||
device: u64,
|
||||
#[cfg(unix)]
|
||||
inode: u64,
|
||||
#[cfg(unix)]
|
||||
changed: (i64, i64),
|
||||
#[cfg(windows)]
|
||||
identity: same_file::Handle,
|
||||
#[cfg(windows)]
|
||||
physical: Option<(u64, u64, i64)>,
|
||||
}
|
||||
|
||||
impl DirectoryStamp {
|
||||
fn volume_generation(&self) -> Option<(u64, u64, SystemTime)> {
|
||||
#[cfg(unix)]
|
||||
{
|
||||
Some((self.device, self.inode, self.birth?))
|
||||
}
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let (volume, file, _) = self.physical?;
|
||||
Some((volume, file, self.birth?))
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
{
|
||||
None
|
||||
}
|
||||
}
|
||||
async fn at(path: &str) -> io::Result<Self> {
|
||||
let path = PathBuf::from(path);
|
||||
crate::storage_api::ecstore_spawn_blocking_with_resource_ownership(move || {
|
||||
let meta = std::fs::symlink_metadata(&path)?;
|
||||
if !meta.is_dir() {
|
||||
return Err(io::ErrorKind::NotADirectory.into());
|
||||
}
|
||||
#[cfg(unix)]
|
||||
use std::os::unix::fs::MetadataExt;
|
||||
#[cfg(windows)]
|
||||
let identity = same_file::Handle::from_path(&path)?;
|
||||
#[cfg(windows)]
|
||||
let physical = winapi_util::file::information(identity.as_file()).ok().and_then(|info| {
|
||||
let created = i64::try_from(info.creation_time()?).ok()?;
|
||||
(info.file_index() != 0).then_some((info.volume_serial_number(), info.file_index(), created))
|
||||
});
|
||||
Ok(Self {
|
||||
modified: meta.modified()?,
|
||||
birth: meta.created().ok(),
|
||||
#[cfg(unix)]
|
||||
device: meta.dev(),
|
||||
#[cfg(unix)]
|
||||
inode: meta.ino(),
|
||||
#[cfg(unix)]
|
||||
changed: (meta.ctime(), meta.ctime_nsec()),
|
||||
#[cfg(windows)]
|
||||
identity,
|
||||
#[cfg(windows)]
|
||||
physical,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(io::Error::other)?
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub(super) struct DirectoryWork {
|
||||
names: NamePages,
|
||||
// For a prefix tree the canonical child values live in the bucket cache.
|
||||
// The final compaction threshold caps how many paths this list can hold.
|
||||
pub(super) linked: Vec<String>,
|
||||
pub(super) compacted_total: super::DataUsageEntry,
|
||||
pub(super) gap: bool,
|
||||
compacted: bool,
|
||||
recorder: Option<Recorder>,
|
||||
}
|
||||
|
||||
impl DirectoryWork {
|
||||
pub(super) fn restore_mode(&mut self, compacted: bool) {
|
||||
if self.compacted && !compacted {
|
||||
self.rewind();
|
||||
}
|
||||
}
|
||||
fn maybe_checkpoint(&mut self) {
|
||||
if let Some(recorder) = &mut self.recorder
|
||||
&& recorder.last_update.elapsed() >= Duration::from_secs(1)
|
||||
{
|
||||
recorder.last_update = Instant::now();
|
||||
self.checkpoint();
|
||||
}
|
||||
}
|
||||
pub(super) fn checkpoint(&self) {
|
||||
if let Some(recorder) = &self.recorder {
|
||||
recorder.save(self);
|
||||
}
|
||||
}
|
||||
pub(super) async fn next_name(&mut self) -> io::Result<Option<String>> {
|
||||
self.names.next().await
|
||||
}
|
||||
pub(super) fn acknowledge(&mut self, name: &str, linked: bool, compacted: bool) {
|
||||
if self.names.acknowledge() {
|
||||
self.compacted = compacted;
|
||||
if linked && !compacted {
|
||||
self.linked.push(name.to_owned());
|
||||
}
|
||||
self.maybe_checkpoint();
|
||||
}
|
||||
}
|
||||
pub(super) fn rewind(&mut self) {
|
||||
self.compacted = false;
|
||||
self.names.rewind();
|
||||
self.linked.clear();
|
||||
self.compacted_total = super::DataUsageEntry::default();
|
||||
self.gap = false;
|
||||
}
|
||||
}
|
||||
|
||||
struct Listing {
|
||||
reader: Option<DirectoryStream>,
|
||||
pending: Option<native_io::DirEntry>,
|
||||
delivery: Option<(native_io::DirEntry, bool)>,
|
||||
classified: Option<Option<std::fs::FileType>>,
|
||||
markers: [Option<native_io::DirEntry>; 2],
|
||||
abandoned: Option<super::DataUsageHashMap>,
|
||||
work: DirectoryWork,
|
||||
failed: bool,
|
||||
}
|
||||
|
||||
struct Parked {
|
||||
scope: DirectoryScope,
|
||||
parked_at: Instant,
|
||||
serial: u64,
|
||||
stamp: DirectoryStamp,
|
||||
listing: Box<Listing>,
|
||||
}
|
||||
|
||||
static PARKED: LazyLock<Mutex<HashMap<PathBuf, Parked>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
|
||||
|
||||
fn expired_slots(slots: &mut HashMap<PathBuf, Parked>) -> Vec<(PathBuf, Parked)> {
|
||||
slots
|
||||
.extract_if(|_, parked| {
|
||||
parked.scope.disk.strong_count() == 0 || parked.parked_at.elapsed() > Duration::from_secs(24 * 3600)
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// A routing hint only. The admitted walker revalidates the physical source,
|
||||
/// bucket instance, coverage plan and directory before using a parked stream.
|
||||
pub(crate) fn parked_bucket_disks(bucket: &str) -> Vec<Weak<Disk>> {
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let expired = expired_slots(&mut slots);
|
||||
let matches = slots
|
||||
.values()
|
||||
.filter(|entry| entry.scope.bucket.as_ref() == bucket)
|
||||
.map(|entry| entry.scope.disk.clone())
|
||||
.collect();
|
||||
drop(slots);
|
||||
drop(expired);
|
||||
matches
|
||||
}
|
||||
|
||||
pub(super) fn is_parked(path: &str) -> bool {
|
||||
PARKED.lock().unwrap_or_else(|p| p.into_inner()).contains_key(Path::new(path))
|
||||
}
|
||||
|
||||
pub(super) fn forget_bucket(disk: &Arc<Disk>, info: &DataUsageCacheInfo) {
|
||||
let owner = Arc::downgrade(disk);
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let removed = slots
|
||||
.extract_if(|_, parked| {
|
||||
parked.scope.disk.ptr_eq(&owner)
|
||||
&& parked.scope.bucket.as_ref() == info.name
|
||||
&& parked.scope.owner <= (info.leader_epoch, info.next_cycle)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
drop(slots);
|
||||
drop(removed);
|
||||
}
|
||||
|
||||
// Prefer existing active work over cyclic LRU eviction. Under pressure, a
|
||||
// deleted or replaced directory must not monopolize one of the physical slots.
|
||||
async fn reclaim_retired(scope: &DirectoryScope, path: &Path) {
|
||||
static NEXT_PROBE: AtomicUsize = AtomicUsize::new(0);
|
||||
let candidate = {
|
||||
let slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
if slots.contains_key(path) {
|
||||
return;
|
||||
}
|
||||
let disk_slots: Vec<_> = slots
|
||||
.iter()
|
||||
.filter(|(_, parked)| parked.scope.disk.ptr_eq(&scope.disk))
|
||||
.collect();
|
||||
if disk_slots.len() < PARKED_DIRECTORIES_PER_DISK {
|
||||
return;
|
||||
}
|
||||
let (path, parked) = disk_slots[NEXT_PROBE.fetch_add(1, Ordering::Relaxed) % disk_slots.len()];
|
||||
(path.clone(), parked.serial)
|
||||
};
|
||||
let Some(candidate_path) = candidate.0.to_str() else { return };
|
||||
let current = DirectoryStamp::at(candidate_path).await;
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let stale = slots.get(&candidate.0).is_some_and(|parked| {
|
||||
parked.serial == candidate.1
|
||||
&& match ¤t {
|
||||
Ok(stamp) => stamp != &parked.stamp,
|
||||
Err(error) => matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory),
|
||||
}
|
||||
});
|
||||
let retired = stale.then(|| slots.remove(&candidate.0));
|
||||
drop(slots);
|
||||
drop(retired);
|
||||
}
|
||||
|
||||
fn valid_component(name: &str) -> bool {
|
||||
!name.is_empty()
|
||||
&& name.len() <= 4096
|
||||
&& name != "."
|
||||
&& name != ".."
|
||||
&& !name.as_bytes().contains(&0)
|
||||
&& std::path::Path::new(name)
|
||||
.file_name()
|
||||
.is_some_and(|part| part == std::ffi::OsStr::new(name))
|
||||
}
|
||||
|
||||
pub(super) struct ScannerDirectory {
|
||||
path: Arc<Path>,
|
||||
scope: Option<(DirectoryScope, DirectoryStamp)>,
|
||||
directory_name: Arc<str>,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
listing: Option<Box<Listing>>,
|
||||
marker: usize,
|
||||
pub(super) enumeration_only: bool,
|
||||
resumed: bool,
|
||||
pub(super) directory_replaced: bool,
|
||||
pub(super) batch_entries: usize,
|
||||
}
|
||||
|
||||
impl ScannerDirectory {
|
||||
pub(super) async fn open(
|
||||
path: &str,
|
||||
directory_name: &str,
|
||||
scope: Option<DirectoryScope>,
|
||||
budget: Arc<ScannerCycleBudget>,
|
||||
saved: Option<Saved>,
|
||||
) -> io::Result<Self> {
|
||||
// A different physical directory must never inherit old names. Directory
|
||||
// timestamps supplement (and cannot replace) the scanner's plan fences.
|
||||
let stamp = if scope.is_some() {
|
||||
Some(DirectoryStamp::at(path).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
if let Some(scope) = &scope {
|
||||
reclaim_retired(scope, Path::new(path)).await;
|
||||
}
|
||||
let parked = scope.as_ref().and_then(|current| {
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let key = Path::new(path);
|
||||
if slots
|
||||
.get(key)
|
||||
.is_some_and(|p| p.scope.disk.ptr_eq(¤t.disk) && p.scope.owner > current.owner)
|
||||
{
|
||||
return None;
|
||||
}
|
||||
slots.remove(key)
|
||||
});
|
||||
let directory_replaced = parked.as_ref().is_some_and(|entry| Some(&entry.stamp) != stamp.as_ref());
|
||||
let parked =
|
||||
parked.filter(|entry| !directory_replaced && scope.as_ref().is_some_and(|current| entry.scope.matches(current)));
|
||||
let mut resumed = parked.is_some();
|
||||
let mut previous = saved
|
||||
.as_ref()
|
||||
.and_then(|slot| slot.lock().unwrap_or_else(|p| p.into_inner()).clone());
|
||||
let durable_scope = scope.as_ref().map(Scope::new);
|
||||
let endpoint = scope.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default();
|
||||
let physical = scope
|
||||
.as_ref()
|
||||
.zip(stamp.as_ref())
|
||||
.and_then(|(scope, stamp)| Physical::of(&scope.endpoint, stamp));
|
||||
let mut replaced = directory_replaced
|
||||
|| previous
|
||||
.as_ref()
|
||||
.is_some_and(|old| old.endpoint() == endpoint && old.physical.is_some() && old.physical != physical);
|
||||
previous = previous.filter(|old| {
|
||||
durable_scope
|
||||
.as_ref()
|
||||
.is_some_and(|current| old.restore_allowed(directory_name, current, &physical))
|
||||
});
|
||||
let mut listing = if let Some(parked) = parked {
|
||||
parked.listing
|
||||
} else {
|
||||
let mut reader = None;
|
||||
if let Some(checkpoint) = &previous {
|
||||
if checkpoint.sealed {
|
||||
resumed = true;
|
||||
} else if let Some(anchor) = &checkpoint.anchor {
|
||||
match DirectoryStream::open_at(path, anchor).await {
|
||||
Ok(restored) => {
|
||||
reader = Some(restored);
|
||||
resumed = true;
|
||||
}
|
||||
Err(error)
|
||||
if matches!(
|
||||
error.kind(),
|
||||
io::ErrorKind::InvalidData
|
||||
| io::ErrorKind::Unsupported
|
||||
| io::ErrorKind::NotFound
|
||||
| io::ErrorKind::InvalidInput
|
||||
) =>
|
||||
{
|
||||
previous = None;
|
||||
replaced = true;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
if !resumed {
|
||||
reader = Some(DirectoryStream::open(path, scope.is_some()).await?);
|
||||
}
|
||||
let mut work = DirectoryWork::default();
|
||||
let mut markers = [None, None];
|
||||
if let Some(saved) = &previous {
|
||||
if resumed && let Some(owner) = &scope {
|
||||
tracing::debug!(
|
||||
event = super::EVENT_SCANNER_FOLDER_STATE,
|
||||
component = super::LOG_COMPONENT_SCANNER,
|
||||
subsystem = super::LOG_SUBSYSTEM_FOLDER,
|
||||
state = "directory_checkpoint_restored",
|
||||
bucket = %owner.bucket,
|
||||
directory = directory_name,
|
||||
disk = %endpoint,
|
||||
leader_epoch = owner.owner.0,
|
||||
cycle = owner.owner.1,
|
||||
restore_mode = if saved.sealed { "sealed_membership" } else { "native_position" },
|
||||
entries_read = saved.entries_read,
|
||||
completed_through = saved.aggregate.as_ref().map(|aggregate| aggregate.through.as_str()),
|
||||
"Restored directory checkpoint"
|
||||
);
|
||||
}
|
||||
work.names = NamePages::restore(&saved.pages, saved.sealed, saved.aggregate.as_ref().map(|a| a.through.as_str()));
|
||||
if let Some(aggregate) = &saved.aggregate {
|
||||
work.compacted = true;
|
||||
work.compacted_total = aggregate.value.clone();
|
||||
}
|
||||
let physical_path: Arc<Path> = Arc::from(Path::new(path));
|
||||
if let Some(part) = &saved.part {
|
||||
markers[0] = Some(native_io::DirEntry::recorded(physical_path.clone(), part.into(), None));
|
||||
}
|
||||
if saved.metadata {
|
||||
markers[1] = Some(native_io::DirEntry::recorded(physical_path, super::STORAGE_FORMAT_FILE.into(), None));
|
||||
}
|
||||
}
|
||||
Box::new(Listing {
|
||||
reader,
|
||||
pending: None,
|
||||
delivery: None,
|
||||
classified: None,
|
||||
markers,
|
||||
abandoned: None,
|
||||
work,
|
||||
failed: false,
|
||||
})
|
||||
};
|
||||
if let (Some(slot), Some(durable_scope)) = (saved, durable_scope) {
|
||||
if let Some(recorder) = &mut listing.work.recorder {
|
||||
recorder.slot = slot;
|
||||
} else {
|
||||
let recorder = Recorder::new(slot, directory_name, durable_scope, endpoint, physical, previous.as_ref());
|
||||
|
||||
listing.work.recorder = Some(recorder);
|
||||
}
|
||||
}
|
||||
Ok(Self {
|
||||
path: Arc::from(Path::new(path)),
|
||||
scope: scope.zip(stamp),
|
||||
directory_name: Arc::from(directory_name),
|
||||
budget,
|
||||
listing: Some(listing),
|
||||
marker: 0,
|
||||
enumeration_only: false,
|
||||
resumed,
|
||||
directory_replaced: replaced,
|
||||
batch_entries: 0,
|
||||
})
|
||||
}
|
||||
|
||||
pub(super) fn continues_stream(&self) -> bool {
|
||||
self.resumed
|
||||
}
|
||||
pub(super) fn indexed(&self) -> bool {
|
||||
self.scope.is_some()
|
||||
}
|
||||
pub(super) async fn verify_current(&mut self) -> io::Result<()> {
|
||||
let Some((_, original)) = &self.scope else { return Ok(()) };
|
||||
let path = self.path.to_str().ok_or(io::ErrorKind::InvalidInput)?;
|
||||
let current = DirectoryStamp::at(path).await?;
|
||||
if *original == current {
|
||||
return Ok(());
|
||||
}
|
||||
if let Some(listing) = &mut self.listing {
|
||||
listing.failed = true;
|
||||
}
|
||||
Err(io::Error::new(io::ErrorKind::Interrupted, "scanner directory changed during traversal"))
|
||||
}
|
||||
|
||||
pub(super) fn folder_count(&self) -> Option<usize> {
|
||||
self.indexed()
|
||||
.then(|| self.listing.as_ref().map_or(0, |l| l.work.names.len()))
|
||||
}
|
||||
pub(super) fn finished_listing(&self) -> bool {
|
||||
self.listing.as_ref().is_some_and(|l| l.work.names.ready())
|
||||
}
|
||||
pub(super) fn work(&mut self) -> Option<&mut DirectoryWork> {
|
||||
if !self.indexed() {
|
||||
return None;
|
||||
}
|
||||
self.listing.as_mut().map(|listing| &mut listing.work)
|
||||
}
|
||||
pub(super) fn resolve_abandoned(&mut self, path: &str) {
|
||||
if let Some(abandoned) = self.listing.as_mut().and_then(|l| l.abandoned.as_mut()) {
|
||||
abandoned.remove(path);
|
||||
}
|
||||
}
|
||||
pub(super) fn hold_abandoned(&mut self, baseline: &mut super::DataUsageHashMap) {
|
||||
if self.indexed()
|
||||
&& let Some(listing) = &mut self.listing
|
||||
&& listing.abandoned.is_none()
|
||||
{
|
||||
if !baseline.is_empty() && self.resumed {
|
||||
// Cold restore carries classified members, not the previous
|
||||
// process's transient unresolved heal candidates.
|
||||
for page in listing.work.names.snapshot() {
|
||||
for name in page.names() {
|
||||
baseline.remove(&super::path_join_buf(&[&self.directory_name, name]));
|
||||
}
|
||||
}
|
||||
}
|
||||
listing.abandoned = Some(std::mem::take(baseline));
|
||||
}
|
||||
}
|
||||
pub(super) fn restore_abandoned(&self, baseline: &mut super::DataUsageHashMap) {
|
||||
if let Some(abandoned) = self.listing.as_ref().and_then(|listing| listing.abandoned.as_ref()) {
|
||||
baseline.clone_from(abandoned);
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) async fn peek_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
|
||||
if let Some((entry, enumeration_only)) = self.listing.as_ref().and_then(|listing| listing.delivery.as_ref()) {
|
||||
self.enumeration_only = *enumeration_only;
|
||||
return Ok(Some(entry.clone()));
|
||||
}
|
||||
let entry = self.next_entry().await?;
|
||||
if let (Some(listing), Some(entry)) = (&mut self.listing, &entry) {
|
||||
listing.delivery = Some((entry.clone(), self.enumeration_only));
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
|
||||
pub(super) fn accept_entry(&mut self) {
|
||||
let Some(listing) = &mut self.listing else { return };
|
||||
let Some((entry, _)) = listing.delivery.take() else { return };
|
||||
// Discovering a name can precede the I/O admission wait. Commit its
|
||||
// classification and native position together only after admission;
|
||||
// a cold retry then re-reads any name still pending on cancellation.
|
||||
let Some(kind) = listing.classified.take() else { return };
|
||||
let name = entry.file_name();
|
||||
let text = name.to_string_lossy();
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.anchor = entry.anchor().cloned();
|
||||
state.entries_read = state.entries_read.saturating_add(1);
|
||||
state.valid_names &= name.to_str().is_some_and(valid_component);
|
||||
}
|
||||
if let Some(kind) = kind {
|
||||
let direct_part = kind.is_file() && super::is_erasure_part_name(&text);
|
||||
if kind.is_dir() {
|
||||
if let Some(abandoned) = &mut listing.abandoned
|
||||
&& !abandoned.is_empty()
|
||||
{
|
||||
abandoned.remove(&super::path_join_buf(&[&self.directory_name, &text]));
|
||||
}
|
||||
listing.work.names.add(text.into_owned());
|
||||
} else if direct_part {
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.part.get_or_insert_with(|| text.to_string());
|
||||
}
|
||||
listing.markers[0].get_or_insert_with(|| native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
|
||||
} else if text == super::STORAGE_FORMAT_FILE {
|
||||
if let Some(state) = &mut listing.work.recorder {
|
||||
state.metadata = true;
|
||||
}
|
||||
listing.markers[1] = Some(native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
|
||||
}
|
||||
}
|
||||
listing.work.maybe_checkpoint();
|
||||
}
|
||||
|
||||
async fn next_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
|
||||
let erasure_candidate = self
|
||||
.scope
|
||||
.as_ref()
|
||||
.is_some_and(|(scope, _)| scope.mode != FolderScanMode::Namespace);
|
||||
let indexed = self.indexed();
|
||||
let listing = self
|
||||
.listing
|
||||
.as_mut()
|
||||
.ok_or_else(|| io::Error::other("scanner directory already returned"))?;
|
||||
self.enumeration_only = false;
|
||||
if let Some(reader) = listing.reader.as_mut() {
|
||||
if !indexed {
|
||||
return reader.next_entry().await;
|
||||
}
|
||||
listing.work.names.flush_full().await?;
|
||||
if listing.pending.is_none() {
|
||||
listing.pending = match reader.next_entry().await {
|
||||
Ok(entry) => entry,
|
||||
Err(error) => {
|
||||
listing.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
}
|
||||
if let Some(entry) = &listing.pending {
|
||||
let kind = match entry.file_type().await {
|
||||
Ok(kind) => Some(kind),
|
||||
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::TooManyLinks) => None,
|
||||
Err(error) => {
|
||||
listing.failed = true;
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
let name = entry.file_name();
|
||||
let text = name.to_string_lossy();
|
||||
self.enumeration_only = kind.is_none_or(|kind| {
|
||||
(!kind.is_symlink() || text == super::STORAGE_FORMAT_FILE)
|
||||
&& !(erasure_candidate && kind.is_file() && super::is_erasure_part_name(&text))
|
||||
});
|
||||
listing.classified = Some(kind);
|
||||
return Ok(listing.pending.take());
|
||||
}
|
||||
listing.reader = None;
|
||||
listing.work.names.finish();
|
||||
listing.work.maybe_checkpoint();
|
||||
}
|
||||
while self.marker < listing.markers.len() {
|
||||
let marker = self.marker;
|
||||
self.marker += 1;
|
||||
if let Some(entry) = &listing.markers[marker] {
|
||||
return Ok(Some(entry.clone()));
|
||||
}
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScannerDirectory {
|
||||
fn drop(&mut self) {
|
||||
let Some((scope, stamp)) = self.scope.take() else { return };
|
||||
let Some(listing) = self.listing.take() else { return };
|
||||
if listing.failed || self.budget.reason().is_none() {
|
||||
if let Some(recorder) = &listing.work.recorder {
|
||||
*recorder.slot.lock().unwrap_or_else(|p| p.into_inner()) = None;
|
||||
}
|
||||
return;
|
||||
}
|
||||
listing.work.checkpoint();
|
||||
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let expired = expired_slots(&mut slots);
|
||||
let key = self.path.to_path_buf();
|
||||
let replaced = slots
|
||||
.get(&key)
|
||||
.is_some_and(|p| !p.scope.disk.ptr_eq(&scope.disk) || p.scope.owner < scope.owner)
|
||||
.then(|| slots.remove(&key));
|
||||
if slots.values().filter(|p| p.scope.disk.ptr_eq(&scope.disk)).count() < PARKED_DIRECTORIES_PER_DISK
|
||||
&& let std::collections::hash_map::Entry::Vacant(slot) = slots.entry(key)
|
||||
{
|
||||
static NEXT_SERIAL: AtomicU64 = AtomicU64::new(0);
|
||||
slot.insert(Parked {
|
||||
scope,
|
||||
parked_at: Instant::now(),
|
||||
serial: NEXT_SERIAL.fetch_add(1, Ordering::Relaxed),
|
||||
stamp,
|
||||
listing,
|
||||
});
|
||||
}
|
||||
drop(slots);
|
||||
drop((expired, replaced));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,364 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use std::collections::HashMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Instant, SystemTime};
|
||||
|
||||
use super::{DirectoryScope, DirectoryStamp, DirectoryWork, FolderScanMode, native_io, pages::NamePage};
|
||||
use crate::data_usage_define::{
|
||||
DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageScanIdentity, DataUsageScanProgress,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(super) struct Scope {
|
||||
bucket: String,
|
||||
source: DataUsageCacheSource,
|
||||
identity: DataUsageScanIdentity,
|
||||
progress: DataUsageScanProgress,
|
||||
mode: u8,
|
||||
prefixes: Option<[u8; 32]>,
|
||||
watch: Option<[u8; 32]>,
|
||||
}
|
||||
|
||||
impl Scope {
|
||||
fn may_adopt(&self, current: &Self) -> bool {
|
||||
self.bucket == current.bucket
|
||||
&& self.source == current.source
|
||||
&& self.identity == current.identity
|
||||
&& self.mode == current.mode
|
||||
&& self.prefixes == current.prefixes
|
||||
&& self.progress.started_plan == current.progress.started_plan
|
||||
&& ((self.progress == current.progress && self.watch == current.watch)
|
||||
|| current.progress.started_plan != current.progress.requested_plan)
|
||||
}
|
||||
pub(super) fn new(owner: &DirectoryScope) -> Self {
|
||||
let mode = match owner.mode {
|
||||
FolderScanMode::Namespace => 0,
|
||||
FolderScanMode::ErasureCandidate {
|
||||
reuse_namespace,
|
||||
parent_has_metadata,
|
||||
} => 1 + u8::from(reuse_namespace) + 2 * u8::from(parent_has_metadata),
|
||||
};
|
||||
let prefixes = owner.prefix_scope.as_ref().map(|scope| {
|
||||
let mut entries: Vec<_> = scope.selected_top_level_entries.iter().collect();
|
||||
entries.sort_unstable();
|
||||
let mut hash = Sha256::new();
|
||||
for entry in entries {
|
||||
hash.update(u64::try_from(entry.len()).unwrap_or(u64::MAX).to_le_bytes());
|
||||
hash.update(entry.as_bytes());
|
||||
}
|
||||
hash.finalize().into()
|
||||
});
|
||||
Self {
|
||||
bucket: owner.bucket.to_string(),
|
||||
source: owner.source,
|
||||
identity: owner.identity,
|
||||
progress: owner.progress,
|
||||
mode,
|
||||
prefixes,
|
||||
watch: owner.watch,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||
pub(crate) struct Physical {
|
||||
pub(crate) endpoint: String,
|
||||
device: u64,
|
||||
inode: u64,
|
||||
changed: (i64, i64),
|
||||
modified: SystemTime,
|
||||
}
|
||||
|
||||
impl Physical {
|
||||
#[cfg(unix)]
|
||||
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
|
||||
Some(Self {
|
||||
endpoint: endpoint.to_owned(),
|
||||
device: stamp.device,
|
||||
inode: stamp.inode,
|
||||
changed: stamp.changed,
|
||||
modified: stamp.modified,
|
||||
})
|
||||
}
|
||||
#[cfg(windows)]
|
||||
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
|
||||
let (device, inode, created) = stamp.physical?;
|
||||
Some(Self {
|
||||
endpoint: endpoint.to_owned(),
|
||||
device,
|
||||
inode,
|
||||
changed: (created, 0),
|
||||
modified: stamp.modified,
|
||||
})
|
||||
}
|
||||
#[cfg(not(any(unix, windows)))]
|
||||
pub(super) fn of(_endpoint: &str, _stamp: &DirectoryStamp) -> Option<Self> {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub(super) struct Aggregate {
|
||||
pub(super) through: String,
|
||||
pub(super) value: DataUsageEntry,
|
||||
}
|
||||
|
||||
/// Immutable directory membership and the last fully counted compacted child.
|
||||
/// The active child's provisional contribution is never part of this receipt.
|
||||
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||
pub struct DirectoryCheckpoint {
|
||||
version: u8,
|
||||
pub(super) parent: String,
|
||||
scope: Scope,
|
||||
pub(super) physical: Option<Physical>,
|
||||
pub(super) pages: Vec<NamePage>,
|
||||
pub(super) anchor: Option<native_io::DirectoryAnchor>,
|
||||
pub(super) entries_read: u64,
|
||||
pub(super) sealed: bool,
|
||||
pub(super) part: Option<String>,
|
||||
pub(super) metadata: bool,
|
||||
pub(super) aggregate: Option<Aggregate>,
|
||||
endpoint: String,
|
||||
digest: [u8; 32],
|
||||
}
|
||||
|
||||
/// Physical progress orders only callbacks from the same directory generation.
|
||||
/// The coordinator uses these receipts for fair scheduling, never for coverage.
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DirectoryReadProgress {
|
||||
pub(crate) parent: String,
|
||||
pub(crate) physical: Physical,
|
||||
position: (u64, bool),
|
||||
}
|
||||
impl DirectoryReadProgress {
|
||||
pub(crate) fn same_source(&self, other: &Self) -> bool {
|
||||
self.parent == other.parent && self.physical == other.physical
|
||||
}
|
||||
pub(crate) fn extends(&self, older: &Self) -> bool {
|
||||
self.same_source(older) && self.position >= older.position
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug)]
|
||||
pub(crate) struct DirectoryProgress {
|
||||
pub(crate) parent: String,
|
||||
pub(crate) physical: Physical,
|
||||
manifest: [u8; 32],
|
||||
through: String,
|
||||
}
|
||||
impl DirectoryProgress {
|
||||
pub(crate) fn same_source(&self, previous: &Self) -> bool {
|
||||
self.parent == previous.parent && self.physical == previous.physical && self.manifest == previous.manifest
|
||||
}
|
||||
pub(crate) fn extends(&self, previous: &Self) -> bool {
|
||||
self.same_source(previous) && self.through >= previous.through
|
||||
}
|
||||
pub(crate) fn covers(&self, path: &str) -> bool {
|
||||
path.strip_prefix(&self.parent)
|
||||
.and_then(|tail| tail.strip_prefix('/'))
|
||||
.and_then(|tail| tail.split('/').next())
|
||||
.is_some_and(|child| child <= self.through.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl DirectoryCheckpoint {
|
||||
/// Deserialize each replica's independent receipt, then intern equal
|
||||
/// verified immutable name pages across the bucket's active directories.
|
||||
pub(crate) fn verified_work(work: Vec<Self>, info: &DataUsageCacheInfo) -> Vec<Self> {
|
||||
let mut shared = HashMap::new();
|
||||
work.into_iter()
|
||||
.filter(|receipt| receipt.is_bound_to(info))
|
||||
.map(|mut receipt| {
|
||||
for page in &mut receipt.pages {
|
||||
let canonical = shared.entry(*page.digest()).or_insert_with(|| page.clone());
|
||||
*page = canonical.clone();
|
||||
}
|
||||
receipt
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
pub(crate) fn parent(&self) -> &str {
|
||||
&self.parent
|
||||
}
|
||||
|
||||
pub(super) fn manifest(&self) -> [u8; 32] {
|
||||
let mut hash = Sha256::new();
|
||||
for p in &self.pages {
|
||||
hash.update(p.digest());
|
||||
}
|
||||
hash.finalize().into()
|
||||
}
|
||||
fn checksum(&self) -> Option<[u8; 32]> {
|
||||
// Nested usage distributions can contain maps. Canonicalize those
|
||||
// map keys before hashing so restart hash randomization is irrelevant.
|
||||
let mut aggregate = serde_json::to_value(&self.aggregate).ok()?;
|
||||
aggregate.sort_all_objects();
|
||||
let header = rmp_serde::to_vec_named(&(
|
||||
self.version,
|
||||
&self.parent,
|
||||
&self.scope,
|
||||
&self.physical,
|
||||
&self.anchor,
|
||||
self.entries_read,
|
||||
self.sealed,
|
||||
&self.part,
|
||||
self.metadata,
|
||||
self.manifest(),
|
||||
aggregate,
|
||||
&self.endpoint,
|
||||
))
|
||||
.ok()?;
|
||||
Some(Sha256::digest(header).into())
|
||||
}
|
||||
pub(crate) fn is_bound_to(&self, info: &DataUsageCacheInfo) -> bool {
|
||||
self.version == 1
|
||||
&& self.scope.bucket == info.name
|
||||
&& self.scope.identity.is_valid()
|
||||
&& Some(self.scope.identity) == info.scan_identity
|
||||
&& Some(self.scope.source) == info.source
|
||||
&& info
|
||||
.scan_progress
|
||||
.is_some_and(|p| p.started_plan == self.scope.progress.started_plan)
|
||||
&& (self.parent == info.name
|
||||
|| self
|
||||
.parent
|
||||
.strip_prefix(&info.name)
|
||||
.and_then(|p| p.strip_prefix('/'))
|
||||
.is_some_and(|tail| tail.split('/').all(super::valid_component)))
|
||||
&& self.valid_payload()
|
||||
}
|
||||
pub(crate) fn read_progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryReadProgress> {
|
||||
let physical = self.physical.as_ref()?;
|
||||
(self.is_bound_to(info) && (self.sealed || self.anchor.is_some()) && self.entries_read > 0).then(|| {
|
||||
DirectoryReadProgress {
|
||||
parent: self.parent.clone(),
|
||||
physical: physical.clone(),
|
||||
position: (self.entries_read, self.sealed),
|
||||
}
|
||||
})
|
||||
}
|
||||
pub(crate) fn progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryProgress> {
|
||||
let done = self.aggregate.as_ref()?;
|
||||
let physical = self.physical.as_ref()?;
|
||||
(self.sealed && self.is_bound_to(info)).then(|| DirectoryProgress {
|
||||
parent: self.parent.clone(),
|
||||
physical: physical.clone(),
|
||||
manifest: self.manifest(),
|
||||
through: done.through.clone(),
|
||||
})
|
||||
}
|
||||
pub(crate) fn endpoint(&self) -> &str {
|
||||
&self.endpoint
|
||||
}
|
||||
fn valid_for(&self, parent: &str, owner: &Scope) -> bool {
|
||||
self.version == 1 && self.parent == parent && self.scope.may_adopt(owner) && self.valid_payload()
|
||||
}
|
||||
fn valid_payload(&self) -> bool {
|
||||
self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
|
||||
&& (self.sealed || self.anchor.is_some())
|
||||
&& self.checksum() == Some(self.digest)
|
||||
&& self.pages.iter().all(NamePage::valid)
|
||||
&& self.part.as_deref().is_none_or(super::super::is_erasure_part_name)
|
||||
&& self.aggregate.as_ref().is_none_or(|done| {
|
||||
self.sealed
|
||||
&& super::valid_component(&done.through)
|
||||
&& done.value.children.is_empty()
|
||||
&& done.value.failed_objects == 0
|
||||
&& self.pages.iter().any(|page| page.contains(&done.through))
|
||||
})
|
||||
}
|
||||
pub(super) fn restore_allowed(&self, parent: &str, owner: &Scope, physical: &Option<Physical>) -> bool {
|
||||
self.valid_for(parent, owner)
|
||||
&& self.physical.is_some()
|
||||
&& &self.physical == physical
|
||||
&& (self.sealed || self.anchor.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
pub(in crate::scanner_folder) type Saved = Arc<Mutex<Option<DirectoryCheckpoint>>>;
|
||||
|
||||
pub(super) struct Recorder {
|
||||
pub(super) slot: Saved,
|
||||
scope: Scope,
|
||||
parent: String,
|
||||
physical: Option<Physical>,
|
||||
pub(super) anchor: Option<native_io::DirectoryAnchor>,
|
||||
pub(super) entries_read: u64,
|
||||
pub(super) part: Option<String>,
|
||||
pub(super) metadata: bool,
|
||||
pub(super) valid_names: bool,
|
||||
endpoint: String,
|
||||
pub(super) last_update: Instant,
|
||||
}
|
||||
|
||||
impl Recorder {
|
||||
pub(super) fn new(
|
||||
slot: Saved,
|
||||
parent: &str,
|
||||
scope: Scope,
|
||||
endpoint: String,
|
||||
physical: Option<Physical>,
|
||||
previous: Option<&DirectoryCheckpoint>,
|
||||
) -> Self {
|
||||
Self {
|
||||
slot,
|
||||
scope,
|
||||
parent: parent.to_owned(),
|
||||
physical,
|
||||
endpoint,
|
||||
anchor: previous.and_then(|p| p.anchor.clone()),
|
||||
entries_read: previous.map_or(0, |p| p.entries_read),
|
||||
part: previous.and_then(|p| p.part.clone()),
|
||||
metadata: previous.is_some_and(|p| p.metadata),
|
||||
valid_names: true,
|
||||
last_update: Instant::now(),
|
||||
}
|
||||
}
|
||||
pub(super) fn save(&self, work: &DirectoryWork) {
|
||||
// An ordinary object directory is cheaper to reopen. Only a reader
|
||||
// which can actually resume or a closed directory can be checkpointed.
|
||||
let sealed = work.names.ready();
|
||||
let aggregate = if work.compacted && !work.gap {
|
||||
work.names.through().map(|through| Aggregate {
|
||||
through: through.to_owned(),
|
||||
value: work.compacted_total.clone(),
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let snapshot = if self.valid_names
|
||||
&& self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
|
||||
&& (sealed || self.anchor.is_some())
|
||||
{
|
||||
let mut checkpoint = DirectoryCheckpoint {
|
||||
version: 1,
|
||||
parent: self.parent.clone(),
|
||||
scope: self.scope.clone(),
|
||||
physical: self.physical.clone(),
|
||||
anchor: (!sealed).then(|| self.anchor.clone()).flatten(),
|
||||
entries_read: self.entries_read,
|
||||
pages: work.names.snapshot(),
|
||||
sealed,
|
||||
part: self.part.clone(),
|
||||
metadata: self.metadata,
|
||||
aggregate,
|
||||
endpoint: self.endpoint.clone(),
|
||||
digest: [0; 32],
|
||||
};
|
||||
checkpoint.checksum().map(|digest| {
|
||||
checkpoint.digest = digest;
|
||||
checkpoint
|
||||
})
|
||||
} else {
|
||||
None
|
||||
};
|
||||
*self.slot.lock().unwrap_or_else(|p| p.into_inner()) = snapshot;
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod storage_fixture;
|
||||
@@ -0,0 +1,48 @@
|
||||
// Copyright 2026 RustFS Team
|
||||
// Licensed under the Apache License, Version 2.0.
|
||||
|
||||
use super::*;
|
||||
|
||||
impl DirectoryCheckpoint {
|
||||
pub(crate) async fn wide_storage_fixture(info: &DataUsageCacheInfo) -> [Self; 2] {
|
||||
let scope = Scope {
|
||||
bucket: info.name.clone(),
|
||||
source: info.source.expect("fixture source"),
|
||||
identity: info.scan_identity.expect("fixture identity"),
|
||||
progress: info.scan_progress.expect("fixture coverage plan"),
|
||||
mode: 0,
|
||||
prefixes: None,
|
||||
watch: None,
|
||||
};
|
||||
let slot = Saved::default();
|
||||
let endpoint = info.scan_directory_source.clone().expect("fixture physical source");
|
||||
let physical = Physical {
|
||||
endpoint: endpoint.clone(),
|
||||
device: 1,
|
||||
inode: 2,
|
||||
changed: (123, 0),
|
||||
modified: SystemTime::UNIX_EPOCH,
|
||||
};
|
||||
let mut recorder = Recorder::new(slot.clone(), &info.name, scope, endpoint, Some(physical), None);
|
||||
recorder.entries_read = 300_000;
|
||||
let mut work = DirectoryWork {
|
||||
compacted: true,
|
||||
..Default::default()
|
||||
};
|
||||
for i in 0..300_000 {
|
||||
work.names.add(format!("obj-{i:06}"));
|
||||
work.names.flush_full().await.expect("record real immutable name page");
|
||||
}
|
||||
work.names.finish();
|
||||
[121_034, 121_035].map(|objects| {
|
||||
work.names.seek_after(Some(&format!("obj-{:06}", objects - 1)));
|
||||
work.compacted_total.objects = objects;
|
||||
work.compacted_total.size = objects * 1024;
|
||||
recorder.save(&work);
|
||||
slot.lock()
|
||||
.expect("fixture slot")
|
||||
.take()
|
||||
.expect("sealed directory checkpoint")
|
||||
})
|
||||
}
|
||||
}
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user