gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-30 10:08:58 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into feat/sse-ecstore-decoupling

Browse Source
This commit is contained in:
Zhengchao An
2026-07-30 07:26:55 +08:00
committed by GitHub
parent 3422ec791a 422e0ad768
commit 45edacf8f1
102 changed files with 10411 additions and 1215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/ci.yml
+34
View File
@@ -173,8 +173,39 @@ jobs:
run: cargo clippy --all-targets -- -D warnings
- name: Run nextest tests
env:
# Three concurrent workspace test links saturate the self-hosted
# runner's overlay I/O and can wedge Cargo until the 75m timeout.
CARGO_BUILD_JOBS: "2"
run: |
mkdir -p artifacts/test-and-lint
# Evidence sampler for issue #5394: the post-mortem pgrep below runs
# only after `timeout` has already TERM'd the whole cargo process
# group, so it cannot name a wedged process. Sample system and
# process state every 60s instead; the last samples before the
# timeout show what was stuck (rustc, linker, build script, memory
# pressure, ...). The log rides along in the existing artifact.
(
while true; do
{
echo "=== $(date --utc --iso-8601=seconds)"
echo "--- load"; cat /proc/loadavg
echo "--- psi"; grep -H . /proc/pressure/* 2>/dev/null || true
echo "--- mem"; free -m
echo "--- disk"; df -h / /home/runner 2>/dev/null || df -h /
echo "--- top-rss"
ps -eo pid,ppid,stat,etime,rss,pcpu,args --sort=-rss | head -15
echo "--- build/test processes"
ps -eo pid,ppid,stat,etime,rss,pcpu,args | grep -E '[c]argo|[r]ustc|[n]extest|[c]ollect2|rust-ll[d]|[b]uild-script|deps[/]' || true
echo "--- d-state (uninterruptible IO)"
ps -eo pid,stat,etime,args | awk 'NR > 1 && $2 ~ /D/' || true
echo
} >> artifacts/test-and-lint/sampler.log 2>&1 || true
sleep 60
done
) &
sampler_pid=$!
trap 'kill "${sampler_pid}" 2>/dev/null || true' EXIT
set +e
NEXTEST_HIDE_PROGRESS_BAR=1 timeout --verbose --signal=TERM --kill-after=30s 75m \
cargo nextest run --profile ci --all --exclude e2e_test \
@@ -188,6 +219,9 @@ jobs:
echo
echo "Remaining test-related processes:"
pgrep -af 'cargo|nextest|target/.*/deps/' || true
echo
echo "Kernel OOM / kill events:"
dmesg -T 2>/dev/null | grep -iE 'oom|out of memory|killed process' | tail -20 || true
} > artifacts/test-and-lint/nextest-diagnostics.txt
exit "${status}"
Cargo.lock
Generated
+8 -11
View File
@@ -5645,9 +5645,9 @@ dependencies = [
[[package]]
name = "lazy-regex"
version = "3.6.0"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6bae91019476d3ec7147de9aa291cadb6d870abf2f3015d2da73a90325ac1496"
checksum = "4994ba703f78b083e2f7946dac9251abd83fd43a0365f030e99b69be5b4b9ef9"
dependencies = [
"lazy-regex-proc_macros",
"once_cell",
@@ -5656,9 +5656,9 @@ dependencies = [
[[package]]
name = "lazy-regex-proc_macros"
version = "3.6.0"
version = "3.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4de9c1e1439d8b7b3061b2d209809f447ca33241733d9a3c01eabf2dc8d94358"
checksum = "fd97232314824e6dbef1918a871bb93f51070455e3715bf26e19a6d01aa977a0"
dependencies = [
"proc-macro2",
"quote",
@@ -7237,12 +7237,6 @@ dependencies = [
"path-dedot",
]
[[package]]
name = "path-clean"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17359afc20d7ab31fdb42bb844c8b3bb1dabd7dcf7e68428492da7f16966fcef"
[[package]]
name = "path-dedot"
version = "4.0.1"
@@ -9096,7 +9090,6 @@ name = "rustfs-data-usage"
version = "1.0.0-beta.11"
dependencies = [
"async-trait",
"path-clean",
"rmp-serde",
"rustfs-filemeta",
"serde",
@@ -9209,11 +9202,14 @@ dependencies = [
"tonic",
"tower",
"tracing",
"tracing-core",
"tracing-opentelemetry",
"tracing-subscriber",
"url",
"urlencoding",
"uuid",
"winapi-util",
"windows-sys 0.61.2",
"xxhash-rust",
]
@@ -10128,6 +10124,7 @@ dependencies = [
"bytes",
"convert_case 0.11.0",
"crc-fast",
"criterion",
"flate2",
"futures",
"hex-simd",
Cargo.toml
+3
View File
@@ -303,6 +303,7 @@ test-case = "3.3.1"
thiserror = "2.0.19"
tracing = { version = "0.1.44" }
tracing-appender = "0.2.5"
tracing-core = "0.1.36"
tracing-error = "0.2.1"
tracing-opentelemetry = { version = "0.33" }
tracing-subscriber = { version = "0.3.23" }
@@ -313,7 +314,9 @@ uuid = { version = "1.24.0" }
vaultrs = { version = "0.8.0" }
tar = "0.4.46"
walkdir = "2.5.0"
winapi-util = "0.1.11"
windows = { version = "0.62.2" }
windows-sys = "0.61.2"
xxhash-rust = { version = "0.8.18" }
zip = "8.6.0"
zstd = "0.13.3"
crates/common/src/metrics.rs
+145 -33
View File
@@ -1114,6 +1114,8 @@ pub struct ScannerLastMinute {
pub struct ScannerMetricsReport {
pub collected_at: DateTime<Utc>,
pub current_cycle: u64,
#[serde(default)]
pub current_cycle_active: bool,
pub current_started: DateTime<Utc>,
pub cycles_completed_at: Vec<DateTime<Utc>>,
pub ongoing_buckets: usize,
@@ -2051,7 +2053,7 @@ impl Metrics {
pub fn record_scanner_transition_failed(&self, count: u64) {
self.scanner_transition_failed.fetch_add(count, Ordering::Relaxed);
self.record_scanner_source_failed(ScannerWorkSource::Lifecycle, count);
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
self.record_last_cycle_scanner_source_work(ScannerWorkSource::Lifecycle, ScannerSourceWorkUpdate::failed(count));
}
}
@@ -2336,6 +2338,21 @@ impl Metrics {
*self.cycle_info.write().await = cycle;
}
/// Publish a scanner cycle and its work-accounting baseline as one state transition.
pub async fn start_scan_cycle_work_with_cycle(&self, cycle: CurrentCycle) -> ScanCycleWorkSnapshot {
let mut current_cycle = self.cycle_info.write().await;
let snapshot = self.start_scan_cycle_work();
*current_cycle = Some(cycle);
snapshot
}
/// Publish the completed work snapshot and idle cycle state as one state transition.
pub async fn finish_scan_cycle_work_with_cycle(&self, start: ScanCycleWorkSnapshot, cycle: CurrentCycle) {
let mut current_cycle = self.cycle_info.write().await;
self.finish_scan_cycle_work(start);
*current_cycle = Some(cycle);
}
/// Read the current cycle record.
pub async fn get_cycle(&self) -> Option<CurrentCycle> {
self.cycle_info.read().await.clone()
@@ -2464,7 +2481,7 @@ impl Metrics {
&self.current_scan_cycle_replication_repair_work_start,
&replication_repair_snapshot,
);
self.current_scan_cycle_work_active.store(true, Ordering::Relaxed);
self.current_scan_cycle_work_active.store(true, Ordering::Release);
snapshot
}
@@ -2476,11 +2493,11 @@ impl Metrics {
self.record_scan_cycle_work(work);
self.record_scan_cycle_source_work(&source_work);
self.record_scan_cycle_replication_repair_work(&replication_repair_work);
self.current_scan_cycle_work_active.store(false, Ordering::Relaxed);
self.current_scan_cycle_work_active.store(false, Ordering::Release);
}
pub fn current_scan_cycle_has_unresolved_heal_work(&self) -> bool {
if !self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
if !self.current_scan_cycle_work_active.load(Ordering::Acquire) {
return false;
}
@@ -2746,13 +2763,41 @@ impl Metrics {
pub async fn report(&self) -> ScannerMetricsReport {
let mut m = ScannerMetricsReport::default();
let has_cycle = if let Some(cycle) = self.get_cycle().await {
m.current_cycle = cycle.current;
m.cycles_completed_at = cycle.cycle_completed;
m.current_started = cycle.started;
true
} else {
false
let has_cycle = {
let cycle = self.cycle_info.read().await;
let has_cycle = if let Some(cycle) = cycle.as_ref() {
m.current_cycle = cycle.current;
m.cycles_completed_at = cycle.cycle_completed.clone();
m.current_started = cycle.started;
true
} else {
false
};
m.current_cycle_active = self.current_scan_cycle_work_active.load(Ordering::Acquire);
if m.current_cycle_active {
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
let current_replication_repair_work =
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
m.current_cycle_objects_scanned = current_work.objects_scanned;
m.current_cycle_directories_scanned = current_work.directories_scanned;
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
m.current_cycle_yield_events = current_work.yield_events;
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
m.current_cycle_ilm_actions = current_work.ilm_actions;
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
m.current_cycle_heal_objects = current_work.heal_objects;
m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves;
m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
m.current_cycle_replication_repair =
self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
}
has_cycle
};
if !has_cycle && let Some(init_time) = crate::get_global_init_time().await {
@@ -2793,28 +2838,6 @@ impl Metrics {
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
if self.current_scan_cycle_work_active.load(Ordering::Relaxed) {
let current_work = self.scan_cycle_work_since(self.current_scan_cycle_work_start());
let current_source_work = self.scanner_source_work_since(&self.current_scan_cycle_source_work_start_values());
let current_replication_repair_work =
self.scanner_replication_repair_work_since(&self.current_scan_cycle_replication_repair_work_start_values());
m.current_cycle_objects_scanned = current_work.objects_scanned;
m.current_cycle_directories_scanned = current_work.directories_scanned;
m.current_cycle_bucket_drive_scans = current_work.bucket_drive_scans;
m.current_cycle_bucket_drive_failures = current_work.bucket_drive_failures;
m.current_cycle_yield_events = current_work.yield_events;
m.current_cycle_yield_duration_seconds = current_work.yield_duration_millis as f64 / 1000.0;
m.current_cycle_throttle_sleep_events = current_work.throttle_sleep_events;
m.current_cycle_throttle_sleep_duration_seconds = current_work.throttle_sleep_duration_millis as f64 / 1000.0;
m.current_cycle_ilm_actions = current_work.ilm_actions;
m.current_cycle_lifecycle_expiry_actions = current_work.lifecycle_expiry_actions;
m.current_cycle_lifecycle_transition_actions = current_work.lifecycle_transition_actions;
m.current_cycle_heal_objects = current_work.heal_objects;
m.current_cycle_replication_checks = current_work.replication_checks;
m.current_cycle_usage_saves = current_work.usage_saves;
m.current_cycle_source_work = self.scanner_source_work_snapshots(&current_source_work);
m.current_cycle_replication_repair = self.scanner_replication_repair_work_snapshots(&current_replication_repair_work);
}
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
m.last_cycle_result_code = last_cycle_result as u64;
@@ -4142,6 +4165,8 @@ mod tests {
let report = metrics.report().await;
assert!(report.current_cycle_active);
assert_eq!(report.current_cycle, 0);
assert_eq!(report.current_cycle_objects_scanned, 7);
assert_eq!(report.current_cycle_directories_scanned, 3);
assert_eq!(report.current_cycle_bucket_drive_scans, 2);
@@ -4158,6 +4183,8 @@ mod tests {
metrics.finish_scan_cycle_work(start);
let report = metrics.report().await;
assert!(!report.current_cycle_active);
assert_eq!(report.current_cycle, 0);
assert_eq!(report.current_cycle_objects_scanned, 0);
assert_eq!(report.current_cycle_directories_scanned, 0);
assert_eq!(report.current_cycle_bucket_drive_scans, 0);
@@ -4184,6 +4211,91 @@ mod tests {
assert_eq!(report.last_cycle_usage_saves, 2);
}
#[tokio::test]
async fn scan_cycle_activity_and_cycle_state_publish_together() {
let metrics = Metrics::new();
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
let active_cycle = CurrentCycle {
current: 12,
next: 13,
started: cycle_started,
..Default::default()
};
let cycle_state = metrics.cycle_info.read().await;
let mut start_transition = Box::pin(metrics.start_scan_cycle_work_with_cycle(active_cycle));
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
assert!(start_transition.as_mut().poll(&mut context).is_pending());
assert!(!metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
drop(cycle_state);
let start = start_transition.await;
let active = metrics.report().await;
assert!(active.current_cycle_active);
assert_eq!(active.current_cycle, 12);
assert_eq!(active.current_started, cycle_started);
let idle_cycle = CurrentCycle {
current: 0,
next: 13,
started: cycle_started,
..Default::default()
};
let cycle_state = metrics.cycle_info.read().await;
let mut finish_transition = Box::pin(metrics.finish_scan_cycle_work_with_cycle(start, idle_cycle));
assert!(finish_transition.as_mut().poll(&mut context).is_pending());
assert!(metrics.current_scan_cycle_work_active.load(Ordering::Acquire));
drop(cycle_state);
finish_transition.await;
let idle = metrics.report().await;
assert!(!idle.current_cycle_active);
assert_eq!(idle.current_cycle, 0);
}
#[tokio::test]
async fn report_keeps_cycle_identity_and_work_in_one_snapshot() {
let metrics = Metrics::new();
let cycle_ten = CurrentCycle {
current: 10,
next: 11,
started: Utc::now() - chrono::Duration::seconds(10),
..Default::default()
};
let cycle_ten_start = metrics.start_scan_cycle_work_with_cycle(cycle_ten.clone()).await;
metrics.operations[Metric::ScanObject as usize].store(1, Ordering::Relaxed);
let paths = metrics.current_paths.write().await;
let mut report = Box::pin(metrics.report());
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
assert!(report.as_mut().poll(&mut context).is_pending());
metrics
.finish_scan_cycle_work_with_cycle(cycle_ten_start, CurrentCycle { current: 0, ..cycle_ten })
.await;
let cycle_eleven_start = metrics
.start_scan_cycle_work_with_cycle(CurrentCycle {
current: 11,
next: 12,
started: Utc::now(),
..Default::default()
})
.await;
metrics.operations[Metric::ScanObject as usize].store(101, Ordering::Relaxed);
drop(paths);
let snapshot = report.await;
assert_eq!(snapshot.current_cycle, 10);
assert_eq!(snapshot.current_cycle_objects_scanned, 1);
metrics
.finish_scan_cycle_work_with_cycle(cycle_eleven_start, CurrentCycle::default())
.await;
}
#[tokio::test]
async fn scanner_cycle_ilm_actions_ignore_global_ilm_work() {
let metrics = Metrics::new();
crates/config/src/constants/internode.rs
+31 -9
View File
@@ -158,17 +158,33 @@ pub const DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT: bool = false;
// rolling upgrades until the fleet-wide body-digest fallback counter reads zero.
const _: () = assert!(!DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT);
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
/// one-time consumption of body-bound v2 signatures.
/// Require the replay-scoped internode RPC signature after the fleet has converged on it.
///
/// The cache retains each nonce for the ~10-minute signature freshness envelope, so the steady
/// state holds roughly `mutating RPS x 601s` entries; the default sustains ~1,700 body-bound
/// mutating RPCs per second (about 120 MiB worst case, allocated only under sustained load).
/// Overflow fails closed — legitimate signed traffic is the only thing that can fill the cache
/// (replays are rejected before insertion, and an attacker cannot mint valid nonces without the
/// shared secret) — and increments
/// The default keeps v1/v2 peers available during a rolling upgrade. Operators may set this only
/// after `rustfs_system_network_internode_replay_scope_fallback_total` remains zero for a full
/// release window. The node still accepts a v2-authenticated `Ping` carrying an epoch challenge:
/// that narrowly scoped bootstrap lets an upgraded client learn the receiving process epoch and
/// immediately retry with the replay-scoped signature after a peer restart.
pub const ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT: &str = "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT";
pub const DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT: bool = false;
// Compile-time invariant: mixed-version clusters must remain available until operators make the
// observed fallback counter an explicit strictness decision.
const _: () = assert!(!DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT);
/// Capacity (distinct nonces) of the process-local internode RPC replay cache that enforces
/// one-time consumption of authenticated RPC signatures.
///
/// The cache retains each nonce for the ~10-minute signature freshness envelope. Once peers use
/// replay-scoped v3 authentication, every authenticated RPC consumes one entry, so the steady
/// state holds roughly `authenticated RPC RPS x 601s` entries. The default sustains about 1,700
/// authenticated RPCs per second (about 120 MiB worst case, allocated only under sustained load);
/// operators must size it for the node's aggregate peak RPC rate before enabling strict replay
/// scope. Overflow fails closed — legitimate signed traffic is the only thing that can fill the
/// cache (replays are rejected before insertion, and an attacker cannot mint valid nonces without
/// the shared secret) — and increments
/// `rustfs_system_network_internode_replay_cache_overflow_total`, so a sustained non-zero overflow
/// counter means this capacity is undersized for the node's peak mutation rate.
/// counter means this capacity is undersized for the node's peak authenticated RPC rate.
pub const ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: &str = "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY";
pub const DEFAULT_INTERNODE_RPC_REPLAY_CACHE_CAPACITY: usize = 1_048_576;
@@ -354,6 +370,12 @@ mod tests {
assert_eq!(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "RUSTFS_INTERNODE_RPC_BODY_DIGEST_STRICT");
}
#[test]
fn internode_replay_scope_strict_env_name_is_stable() {
// The fail-open default invariant is asserted at compile time next to the definition.
assert_eq!(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT");
}
#[test]
fn internode_replay_cache_capacity_defaults_and_env_name() {
assert_eq!(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY");
crates/data-usage/Cargo.toml
-1
View File
@@ -29,7 +29,6 @@ workspace = true
[dependencies]
serde = { workspace = true, features = ["derive"] }
path-clean = { workspace = true }
rmp-serde = { workspace = true }
async-trait = { workspace = true }
rustfs-filemeta = { workspace = true }
crates/data-usage/src/data_usage.rs
+94 -6
View File
@@ -12,12 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use path_clean::PathClean;
use serde::{Deserialize, Serialize};
use std::{
collections::{HashMap, HashSet},
hash::{DefaultHasher, Hash, Hasher},
path::Path,
time::{Duration, SystemTime},
};
@@ -334,7 +332,7 @@ impl<'de> Deserialize<'de> for SizeHistogram {
impl SizeHistogram {
pub fn add(&mut self, size: u64) {
let intervals = [
(0, 1024), // LESS_THAN_1024_B
(0, 1024 - 1), // LESS_THAN_1024_B
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
@@ -362,7 +360,7 @@ impl SizeHistogram {
// the sub-ranges in [1 KiB, 512 KiB).
const ONE_MIB: u64 = 1024 * 1024;
let intervals = [
(0, 1024), // LESS_THAN_1024_B
(0, 1024 - 1), // LESS_THAN_1024_B
(1024, 64 * 1024 - 1), // BETWEEN_1024_B_AND_64_KB
(64 * 1024, 256 * 1024 - 1), // BETWEEN_64_KB_AND_256_KB
(256 * 1024, 512 * 1024 - 1), // BETWEEN_256_KB_AND_512_KB
@@ -1110,9 +1108,39 @@ fn mark(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut HashSet<String
}
}
/// Hash a path for data usage caching
fn clean_data_usage_path(data: &str) -> String {
let rooted = data.starts_with('/');
let mut parts = Vec::new();
for part in data.split('/') {
match part {
"" | "." => {}
".." => {
if parts.last().is_some_and(|last| *last != "..") {
parts.pop();
} else if !rooted {
parts.push(part);
}
}
_ => parts.push(part),
}
}
let clean = parts.join("/");
match (rooted, clean.is_empty()) {
(true, true) => "/".to_string(),
(true, false) => format!("/{clean}"),
(false, true) => ".".to_string(),
(false, false) => clean,
}
}
/// Hash a slash-separated path for data usage caching.
///
/// Cache identifiers are persisted and exchanged across nodes, so their
/// normalization must not depend on the host operating system.
pub fn hash_path(data: &str) -> DataUsageHash {
DataUsageHash(Path::new(&data).clean().to_string_lossy().to_string())
DataUsageHash(clean_data_usage_path(data))
}
impl DataUsageInfo {
@@ -1497,6 +1525,23 @@ mod tests {
buckets_count: u64,
}
#[test]
fn hash_path_uses_portable_slash_semantics() {
for (input, expected) in [
("", "."),
(".", "."),
("/", "/"),
("//bucket///prefix/", "/bucket/prefix"),
("bucket/./prefix//object", "bucket/prefix/object"),
("bucket/a/../b", "bucket/b"),
("../bucket/..", ".."),
("/../../bucket", "/bucket"),
("bucket\\prefix/object", "bucket\\prefix/object"),
] {
assert_eq!(hash_path(input).key(), expected, "unexpected portable cache key for {input:?}");
}
}
#[test]
fn completeness_marker_is_additive_for_legacy_named_readers() {
let current = DataUsageInfo {
@@ -1601,6 +1646,49 @@ mod tests {
assert_eq!(map["BETWEEN_512_KB_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_classifies_adjacent_boundaries_once() {
let cases = [
(1023, 0),
(1024, 1),
(64 * 1024 - 1, 1),
(64 * 1024, 2),
(256 * 1024 - 1, 2),
(256 * 1024, 3),
(512 * 1024 - 1, 3),
(512 * 1024, 4),
(1024 * 1024 - 1, 4),
(1024 * 1024, 6),
(10 * 1024 * 1024 - 1, 6),
(10 * 1024 * 1024, 7),
(64 * 1024 * 1024 - 1, 7),
(64 * 1024 * 1024, 8),
(128 * 1024 * 1024 - 1, 8),
(128 * 1024 * 1024, 9),
(512 * 1024 * 1024 - 1, 9),
(512 * 1024 * 1024, 10),
];
for (size, expected_bucket) in cases {
let mut hist = SizeHistogram::default();
hist.add(size);
assert_eq!(hist.0.iter().sum::<u64>(), 1, "size {size} must have exactly one physical bucket");
assert_eq!(hist.0[expected_bucket], 1, "size {size} must select the expected bucket");
}
}
#[test]
fn test_size_histogram_1024_bytes_contributes_to_compat_rollup() {
let mut hist = SizeHistogram::default();
hist.add(1024);
let map = hist.to_map();
assert_eq!(map["LESS_THAN_1024_B"], 0);
assert_eq!(map["BETWEEN_1024_B_AND_64_KB"], 1);
assert_eq!(map["BETWEEN_1024B_AND_1_MB"], 1);
}
#[test]
fn test_size_histogram_compat_rollup_saturates_on_corrupt_counts() {
let mut hist = SizeHistogram::default();
crates/e2e_test/src/internode_rpc_signature_e2e_test.rs
+217 -14
View File
@@ -13,8 +13,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
//! Cross-process replay / tamper acceptance for the internode NodeService v2 RPC
//! signature (<https://github.com/rustfs/backlog/issues/1327>).
//! Cross-process replay / tamper acceptance for internode NodeService RPC
//! signatures (<https://github.com/rustfs/backlog/issues/1327>,
//! <https://github.com/rustfs/backlog/issues/1542>).
//!
//! # Why this exists on top of the in-process tests
//!
@@ -78,6 +79,8 @@
//! | mixed version: legacy-only still served, not blocked | [`legacy_only_signature_is_accepted_in_default_posture`] |
//! | strict flip closes the signature downgrade | [`signature_strict_rejects_legacy_only_downgrade`] |
//! | strict flip closes the body-digest downgrade, incl. v1 | [`body_digest_strict_rejects_digestless_mutation`] |
//! | replay scope binds every RPC and rejects restart replay | [`replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e`] |
//! | strict replay scope allows only Ping bootstrap before v3 | [`replay_scope_strict_requires_v3_after_ping_bootstrap_e2e`] |
//!
//! Two acceptance items are deliberately left to the in-process tests. A stale
//! timestamp cannot be forged from outside — it is inside the HMAC — so
@@ -88,18 +91,20 @@
use crate::common::{RustFSTestEnvironment, init_logging};
use crate::storage_api::internode_rpc_signature::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
};
use http::{HeaderMap, Method};
use rustfs_config::{
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_SIGNATURE_STRICT,
ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
ENV_INTERNODE_RPC_SIGNATURE_STRICT,
};
use rustfs_protos::canonical_make_volume_request_body;
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse};
use rustfs_protos::proto_gen::node_service::{MakeVolumeRequest, MakeVolumeResponse, PingRequest, PingResponse};
use serial_test::serial;
use sha2::{Digest, Sha256};
use std::error::Error;
use tonic::{Code, Request, Status};
use tonic::{Code, Request, Response, Status};
use uuid::Uuid;
type TestResult = Result<(), Box<dyn Error + Send + Sync>>;
@@ -115,12 +120,14 @@ const TEST_RPC_SECRET: &str = "rustfs-internode-signature-e2e-secret";
/// clears authentication stops harmlessly at `find_disk`.
const ABSENT_DISK: &str = "/nonexistent/rustfs-signature-e2e-disk";
/// Wire names of the two v2 headers these tests edit. They are `pub(crate)` in
/// ecstore, so they are repeated here rather than imported — [`overwrite_header`]
/// asserts the header it replaces was actually present, which turns a rename
/// into a loud failure instead of silently reducing an attack to a no-op.
/// Wire names of the v2 and replay-scope headers these black-box tests edit. They are
/// `pub(crate)` in ecstore, so they are repeated here rather than imported.
/// [`overwrite_header`] asserts the header it replaces was actually present, which turns a
/// rename into a loud failure instead of silently reducing an attack to a no-op.
const CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
const NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
const BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
/// gRPC service name carried in the signed scope, i.e. `TONIC_RPC_PREFIX`
/// without its leading `/`.
@@ -155,19 +162,28 @@ fn align_rpc_secret_with_server() {
///
/// Uses the no-cleanup spawn so a `pkill` pattern cannot reap servers belonging
/// to other tests running in the same binary.
async fn start_server(extra_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let mut env = RustFSTestEnvironment::new().await?;
fn server_env(extra_env: &[(&'static str, &'static str)]) -> Vec<(&'static str, &'static str)> {
let mut child_env = vec![
("RUSTFS_RPC_SECRET", TEST_RPC_SECRET),
(ENV_INTERNODE_RPC_SIGNATURE_STRICT, "false"),
(ENV_INTERNODE_RPC_BODY_DIGEST_STRICT, "false"),
(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "false"),
(ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY, "1048576"),
];
child_env.extend_from_slice(extra_env);
env.start_rustfs_server_without_cleanup_with_env(&child_env).await?;
child_env
}
async fn start_server_with_env(child_env: &[(&str, &str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
let mut env = RustFSTestEnvironment::new().await?;
env.start_rustfs_server_without_cleanup_with_env(child_env).await?;
Ok(env)
}
async fn start_server(extra_env: &[(&'static str, &'static str)]) -> Result<RustFSTestEnvironment, Box<dyn Error + Send + Sync>> {
start_server_with_env(&server_env(extra_env)).await
}
/// Stop the child and drop the cached gRPC channel for its address.
///
/// `node_service_time_out_client_no_auth` memoises channels in a process-global
@@ -238,12 +254,83 @@ fn overwrite_header(headers: &mut HeaderMap, name: &'static str, value: &str) {
/// and nothing else — no interceptor adds or rewrites auth metadata, so the
/// bytes on the wire are the ones the test chose.
async fn call_make_volume(url: &str, request: MakeVolumeRequest, headers: HeaderMap) -> Result<MakeVolumeResponse, Status> {
call_make_volume_response(url, request, headers)
.await
.map(Response::into_inner)
}
async fn call_make_volume_response(
url: &str,
request: MakeVolumeRequest,
headers: HeaderMap,
) -> Result<Response<MakeVolumeResponse>, Status> {
let mut client = node_service_time_out_client_no_auth(&url.to_string())
.await
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
let mut rpc_request = Request::new(request);
rpc_request.metadata_mut().as_mut().extend(headers);
client.make_volume(rpc_request).await.map(|response| response.into_inner())
client.make_volume(rpc_request).await
}
async fn call_ping_response(url: &str, headers: HeaderMap) -> Result<Response<PingResponse>, Status> {
let mut client = node_service_time_out_client_no_auth(&url.to_string())
.await
.map_err(|err| Status::unavailable(format!("cannot reach the node service: {err}")))?;
let mut rpc_request = Request::new(PingRequest {
version: 1,
body: bytes::Bytes::new(),
});
rpc_request.metadata_mut().as_mut().extend(headers);
client.ping(rpc_request).await
}
fn attach_boot_epoch_challenge(headers: &mut HeaderMap) -> Uuid {
let challenge = Uuid::new_v4();
headers.insert(
BOOT_EPOCH_CHALLENGE_HEADER,
challenge.to_string().parse().expect("UUID must be a valid header value"),
);
challenge
}
fn mint_replay_scope_headers(audience: &str, path: &str, content_sha256: &str, boot_epoch: Uuid) -> HeaderMap {
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(content_sha256));
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 headers must carry a timestamp")
.to_string();
headers.extend(
gen_tonic_replay_scope_headers(audience, path, &timestamp, content_sha256, boot_epoch)
.expect("replay-scope headers must mint with the aligned RPC secret"),
);
headers
}
async fn learn_boot_epoch_from_make_volume(url: &str, audience: &str) -> Uuid {
let request = make_volume_request("signature-e2e-epoch-bootstrap");
let mut headers = mint_v2_headers(audience, "MakeVolume", Some(&canonical_digest(&request)));
let challenge = attach_boot_epoch_challenge(&mut headers);
let response = call_make_volume_response(url, request, headers)
.await
.expect("v2 request with epoch challenge must clear default authentication");
let boot_epoch = verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
.expect("server must HMAC-authenticate the advertised boot epoch");
assert_authenticated(
Ok(response.into_inner()),
"a v2 epoch-challenge request in the default replay-scope posture",
);
boot_epoch
}
async fn learn_boot_epoch_from_ping(url: &str, audience: &str) -> Uuid {
let mut headers = mint_v2_headers(audience, "Ping", None);
let challenge = attach_boot_epoch_challenge(&mut headers);
let response = call_ping_response(url, headers)
.await
.expect("v2 Ping with an epoch challenge must bootstrap strict replay scope");
verify_tonic_boot_epoch_response(audience, challenge, response.metadata().as_ref())
.expect("strict replay-scope Ping must return a valid boot epoch proof")
}
/// Assert a call cleared authentication.
@@ -332,6 +419,122 @@ async fn internode_rpc_signature_default_posture_e2e() -> TestResult {
Ok(())
}
/// A replay-scoped signature is usable exactly once against the exact gRPC path and the server
/// process epoch that minted it. This crosses the child-process boundary twice: the HMAC-protected
/// epoch is learned from a real response, then the same server is restarted in place to prove its
/// replacement epoch rejects the captured request even though the nonce cache is necessarily new.
#[tokio::test]
#[serial]
async fn replay_scope_rejects_replay_path_transplant_and_stale_epoch_e2e() -> TestResult {
init_logging();
align_rpc_secret_with_server();
let child_env = server_env(&[]);
let mut env = start_server_with_env(&child_env).await?;
let url = env.url.clone();
let audience = audience_of(&env);
let boot_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
let request = make_volume_request("replay-scope-e2e-once");
let captured = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
assert_authenticated(
call_make_volume(&url, request.clone(), captured.clone()).await,
"the first replay-scoped mutation delivery",
);
assert_rejected(
call_make_volume(&url, request.clone(), captured).await,
Code::Unauthenticated,
None,
"the same replay-scoped mutation delivered twice",
);
let transplanted =
mint_replay_scope_headers(&audience, &format!("{TONIC_RPC_PREFIX}/Ping"), &canonical_digest(&request), boot_epoch);
assert_rejected(
call_make_volume(&url, request.clone(), transplanted).await,
Code::Unauthenticated,
None,
"a replay-scoped Ping signature transplanted onto MakeVolume",
);
let stale_epoch = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
env.restart_server_preserving_data(Vec::new(), &child_env).await?;
rustfs_protos::evict_failed_connection(&url).await;
assert_rejected(
call_make_volume(&url, request.clone(), stale_epoch).await,
Code::Unauthenticated,
None,
"a replay-scoped signature captured before the receiving process restart",
);
let restarted_epoch = learn_boot_epoch_from_make_volume(&url, &audience).await;
assert_ne!(boot_epoch, restarted_epoch, "a restarted child process must advertise a new boot epoch");
let fresh_epoch = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
restarted_epoch,
);
assert_authenticated(
call_make_volume(&url, request, fresh_epoch).await,
"a replay-scoped mutation signed with the replacement process epoch",
);
stop_server(env, &url).await;
Ok(())
}
/// Strict replay scope leaves one authenticated v2 bootstrap: `Ping` carrying a fresh challenge.
/// A mutating v2 request cannot use that lane; once the epoch proof is returned, the first v3
/// mutation succeeds. This protects a server restart without reopening a general downgrade path.
#[tokio::test]
#[serial]
async fn replay_scope_strict_requires_v3_after_ping_bootstrap_e2e() -> TestResult {
init_logging();
align_rpc_secret_with_server();
let env = start_server(&[(ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT, "true")]).await?;
let url = env.url.clone();
let audience = audience_of(&env);
let v2_request = make_volume_request("replay-scope-e2e-strict-v2");
assert_rejected(
call_make_volume(
&url,
v2_request.clone(),
mint_v2_headers(&audience, "MakeVolume", Some(&canonical_digest(&v2_request))),
)
.await,
Code::Unauthenticated,
None,
"a v2 mutation after replay-scope strictness is enabled",
);
let boot_epoch = learn_boot_epoch_from_ping(&url, &audience).await;
let request = make_volume_request("replay-scope-e2e-strict-v3");
let replay_scoped = mint_replay_scope_headers(
&audience,
&format!("{TONIC_RPC_PREFIX}/MakeVolume"),
&canonical_digest(&request),
boot_epoch,
);
assert_authenticated(
call_make_volume(&url, request, replay_scoped).await,
"a replay-scoped mutation after Ping bootstrap under strict replay scope",
);
stop_server(env, &url).await;
Ok(())
}
/// Baseline: correctly signed mutations are accepted, both with and without a
/// body digest.
///
crates/e2e_test/src/notification_webhook_test.rs
+106 -4
View File
@@ -21,18 +21,22 @@
//! function, never as an S3 event sink.
//!
//! Coverage:
//! * PUT / multipart-complete / DELETE each deliver one event with the correct
//! * PUT / multipart-complete / DeleteObject / DeleteObjects each deliver one event with the correct
//! eventName, bucket, key, versionId and eTag.
//! * prefix/suffix filters drop non-matching keys (rule-engine gate).
//! * an event queued while the target endpoint is unreachable is redelivered
//! from the on-disk store once the endpoint recovers (store-and-forward).
//! * responseElements and the S3 response use the canonical request ID while
//! requestParameters preserve a conflicting client-supplied value.
use crate::common::{RustFSTestEnvironment, init_logging};
use aws_sdk_s3::Client;
use aws_sdk_s3::operation::RequestId;
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::types::{
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Event, FilterRule, FilterRuleName,
NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter, VersioningConfiguration,
BucketVersioningStatus, CompletedMultipartUpload, CompletedPart, Delete, Event, FilterRule, FilterRuleName,
NotificationConfiguration, NotificationConfigurationFilter, ObjectIdentifier, QueueConfiguration, S3KeyFilter,
VersioningConfiguration,
};
use http::header::{CONTENT_TYPE, HOST};
use local_ip_address::local_ip;
@@ -40,10 +44,12 @@ use reqwest::StatusCode;
use rustfs_signer::constants::UNSIGNED_PAYLOAD;
use rustfs_signer::sign_v4;
use rustfs_utils::egress::ENV_OUTBOUND_ALLOW_ORIGINS;
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
use s3s::Body;
use serde_json::Value;
use serial_test::serial;
use std::error::Error;
use std::io::Cursor;
use std::path::Path;
use std::sync::{
Arc, Once,
@@ -63,6 +69,8 @@ type BoxError = Box<dyn Error + Send + Sync>;
/// for target lookup (see `process_queue_configurations`), so the region is
/// nominal, but it must be present for `ARN::parse` to succeed.
const NOTIFY_REGION: &str = "us-east-1";
const CLIENT_REQUEST_ID: &str = "client-supplied-request-id";
const CLIENT_AMZ_REQUEST_ID: &str = "client-supplied-amz-request-id";
/// Webhook targets are registered as `TargetID { id: <name>, name: "webhook" }`,
/// so the ARN a notification rule references is
@@ -579,6 +587,36 @@ fn trimmed_etag(value: Option<&str>) -> Option<String> {
value.map(|e| e.trim_matches('"').to_string())
}
fn assert_conflicting_request_id_correlation(record: &Value, server_request_id: &str) {
assert_eq!(
record["requestParameters"][REQUEST_ID_HEADER].as_str(),
Some(CLIENT_REQUEST_ID),
"notification request parameters should retain the actual client header: {record}"
);
assert_eq!(
record["requestParameters"][AMZ_REQUEST_ID].as_str(),
Some(CLIENT_AMZ_REQUEST_ID),
"notification request parameters should retain the actual client header: {record}"
);
assert_eq!(
record["responseElements"][AMZ_REQUEST_ID].as_str(),
Some(server_request_id),
"notification response elements should use the canonical request ID: {record}"
);
}
fn assert_generated_request_id_correlation(record: &Value, request_id: &str) {
assert!(
record["requestParameters"][AMZ_REQUEST_ID].is_null(),
"notification request parameters must not invent a client request header: {record}"
);
assert_eq!(
record["responseElements"][AMZ_REQUEST_ID].as_str(),
Some(request_id),
"notification response elements should match the S3 response request ID: {record}"
);
}
// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------
@@ -671,8 +709,17 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
.bucket(bucket)
.key(put_key)
.body(ByteStream::from_static(b"peri-1 put body"))
.customize()
.mutate_request(|request| {
request.headers_mut().insert(REQUEST_ID_HEADER, CLIENT_REQUEST_ID);
request.headers_mut().insert(AMZ_REQUEST_ID, CLIENT_AMZ_REQUEST_ID);
})
.send()
.await?;
let put_request_id = put.request_id().ok_or("PUT response missing request ID")?.to_owned();
assert!(uuid::Uuid::parse_str(&put_request_id).is_ok());
assert_ne!(put_request_id, CLIENT_REQUEST_ID);
assert_ne!(put_request_id, CLIENT_AMZ_REQUEST_ID);
let put_version = put
.version_id()
.ok_or("PUT response missing versionId (versioning not enabled?)")?;
@@ -692,6 +739,7 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
"record eventName: {record}"
);
assert_eq!(record["s3"]["bucket"]["name"].as_str(), Some(bucket), "bucket in event: {record}");
assert_conflicting_request_id_correlation(record, &put_request_id);
assert_eq!(object["versionId"].as_str(), Some(put_version), "versionId in event: {object}");
assert_eq!(
trimmed_etag(object["eTag"].as_str()),
@@ -744,6 +792,35 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
"multipart eTag in event: {mp_record}"
);
// --- Snowball extract: direct notification path keeps response correlation
let snowball_key = "uploads/snowball.dat";
let snowball_body = b"snowball notification body";
let mut archive_builder = tokio_tar::Builder::new(Cursor::new(Vec::new()));
let mut archive_header = tokio_tar::Header::new_gnu();
archive_header.set_size(u64::try_from(snowball_body.len()).expect("snowball fixture length should fit in u64"));
archive_header.set_mode(0o644);
archive_header.set_cksum();
archive_builder
.append_data(&mut archive_header, snowball_key, Cursor::new(snowball_body))
.await?;
let archive = archive_builder.into_inner().await?.into_inner();
let snowball = client
.put_object()
.bucket(bucket)
.key("snowball-fixture.tar")
.body(ByteStream::from(archive))
.customize()
.mutate_request(|request| {
request.headers_mut().insert("x-amz-meta-snowball-auto-extract", "true");
})
.send()
.await?;
let snowball_request_id = snowball.request_id().ok_or("Snowball response missing request ID")?;
let snowball_event = wait_for_event(&mut rx, snowball_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
let snowball_record = &snowball_event["Records"][0];
assert_generated_request_id_correlation(snowball_record, snowball_request_id);
// --- Filter: non-matching prefix and suffix must never be delivered ------
let wrong_prefix = "logs/report.dat"; // right suffix, wrong prefix
let wrong_suffix = "uploads/report.txt"; // right prefix, wrong suffix
@@ -772,7 +849,32 @@ async fn test_webhook_event_delivery_and_filtering() -> TestResult {
"wrong-suffix key {wrong_suffix} bypassed the filter; saw {seen:?}"
);
// --- DELETE on a versioned bucket: ObjectRemoved:* with delete-marker version
// --- DeleteObjects: direct notification path keeps response correlation --
let delete_many_key = "uploads/delete-many.dat";
client
.put_object()
.bucket(bucket)
.key(delete_many_key)
.body(ByteStream::from_static(b"delete objects notification body"))
.send()
.await?;
wait_for_event(&mut rx, delete_many_key, "s3:ObjectCreated:", Duration::from_secs(20)).await?;
let delete_many = client
.delete_objects()
.bucket(bucket)
.delete(
Delete::builder()
.objects(ObjectIdentifier::builder().key(delete_many_key).build()?)
.build()?,
)
.send()
.await?;
let delete_many_request_id = delete_many.request_id().ok_or("DeleteObjects response missing request ID")?;
let delete_many_event = wait_for_event(&mut rx, delete_many_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
assert_generated_request_id_correlation(&delete_many_event["Records"][0], delete_many_request_id);
// --- DeleteObject on a versioned bucket: delete-marker version ----------
let delete = client.delete_object().bucket(bucket).key(put_key).send().await?;
let removed = wait_for_event(&mut rx, put_key, "s3:ObjectRemoved:", Duration::from_secs(20)).await?;
let removed_record = &removed["Records"][0];
crates/e2e_test/src/reliant/grpc_lock_client.rs
+2 -2
View File
@@ -24,7 +24,7 @@ use rustfs_protos::proto_gen::node_service::{BatchGenerallyLockRequest, Generall
use tonic::Request;
use tracing::{info, warn};
use crate::storage_api::grpc_lock::{TonicInterceptor, node_service_time_out_client_no_auth};
use crate::storage_api::grpc_lock::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
/// gRPC lock client without authentication for testing
/// Similar to RemoteClient but uses no_auth client
@@ -42,7 +42,7 @@ impl GrpcLockClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClient<
tonic::service::interceptor::InterceptedService<tonic::transport::Channel, TonicInterceptor>,
tonic::service::interceptor::InterceptedService<AuthenticatedChannel, TonicInterceptor>,
>,
> {
node_service_time_out_client_no_auth(&self.addr)
crates/e2e_test/src/storage_api.rs
+8 -4
View File
@@ -16,9 +16,12 @@
pub(crate) use rustfs_ecstore::api::bucket::bucket_target_sys::BucketTargetSys;
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::disk::{VolumeInfo, WalkDirOptions};
pub(crate) use rustfs_ecstore::api::rpc::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers};
pub(crate) use rustfs_ecstore::api::rpc::{TonicInterceptor, node_service_time_out_client_no_auth};
pub(crate) use rustfs_ecstore::api::rpc::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
verify_tonic_boot_epoch_response,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{gen_tonic_signature_interceptor, node_service_time_out_client};
@@ -30,7 +33,7 @@ pub(crate) mod node_interact {
}
pub(crate) mod grpc_lock {
pub(crate) use super::{TonicInterceptor, node_service_time_out_client_no_auth};
pub(crate) use super::{AuthenticatedChannel, TonicInterceptor, node_service_time_out_client_no_auth};
}
/// Signing/transport surface used by the cross-process internode RPC signature
@@ -40,7 +43,8 @@ pub(crate) mod grpc_lock {
#[cfg(test)]
pub(crate) mod internode_rpc_signature {
pub(crate) use super::{
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_signature_headers, node_service_time_out_client_no_auth,
TONIC_RPC_PREFIX, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
node_service_time_out_client_no_auth, verify_tonic_boot_epoch_response,
};
}
crates/ecstore/Cargo.toml
+7
View File
@@ -150,11 +150,18 @@ metrics = { workspace = true }
[target.'cfg(target_os = "linux")'.dependencies]
rustfs-uring = "0.2.1"
[target.'cfg(windows)'.dependencies]
winapi-util.workspace = true
windows-sys = { workspace = true, features = ["Win32_Foundation", "Win32_Storage_FileSystem"] }
[dev-dependencies]
tokio = { workspace = true, features = ["rt-multi-thread", "macros", "test-util", "fs"] }
criterion = { workspace = true, features = ["html_reports"] }
temp-env = { workspace = true, features = ["async_closure"] }
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
# Only for `pin_callsite_interest_for_test`, which registers a `NoSubscriber`
# dispatcher to keep tracing's process-global callsite-interest cache honest.
tracing-core = { workspace = true }
serial_test = { workspace = true }
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
proptest = "1"
crates/ecstore/src/api/mod.rs
+20 -9
View File
@@ -67,6 +67,14 @@ pub mod bucket {
};
}
pub mod transition_transaction {
pub use crate::bucket::lifecycle::transition_transaction::{
TransitionOperatorDeleteResult, TransitionOperatorError, TransitionOperatorProbe, TransitionOperatorStatus,
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator,
};
}
pub mod evaluator {
pub use crate::bucket::lifecycle::evaluator::Evaluator;
}
@@ -122,13 +130,13 @@ pub mod bucket {
pub mod metadata_sys {
pub use crate::bucket::metadata_sys::{
BucketMetadataSys, acquire_bucket_targets_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
BucketMetadataSys, acquire_bucket_metadata_transaction_lock, delete, get, get_accelerate_config, get_bucket_policy,
get_bucket_policy_raw, get_bucket_targets_config, get_config_from_disk, get_cors_config, get_durability_config,
get_global_bucket_metadata_sys, get_lifecycle_config, get_logging_config, get_notification_config,
get_object_lock_config, get_public_access_block_config, get_quota_config, get_replication_config,
get_request_payment_config, get_sse_config, get_tagging_config, get_versioning_config, get_website_config,
init_bucket_metadata_sys, list_bucket_targets, reload_bucket_metadata, remove_bucket_metadata, set_bucket_metadata,
update, update_bucket_targets_under_transaction_lock, update_config_with,
update, update_bucket_targets_under_transaction_lock, update_config_with, update_under_transaction_lock,
};
}
@@ -377,6 +385,7 @@ pub mod metrics {
pub mod notification {
pub use crate::services::notification_sys::{
NotificationPeerErr, NotificationSys, get_global_notification_sys, new_global_notification_sys,
start_remote_version_state_fleet_probe,
};
}
@@ -409,13 +418,15 @@ pub mod rio {
pub mod rpc {
pub use crate::cluster::rpc::{
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client, S3PeerSys,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing, ScannerPeerActivity,
TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_signature_headers, gen_tonic_signature_interceptor,
node_service_time_out_client, node_service_time_out_client_no_auth, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature,
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof,
verify_tonic_rpc_signature,
AuthenticatedChannel, LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient,
PeerS3Client, S3PeerSys, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, ScannerBucketListing,
ScannerPeerActivity, TONIC_RPC_PREFIX, TonicInterceptor, gen_signature_headers, gen_tonic_replay_scope_headers,
gen_tonic_signature_headers, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth, normalize_tonic_rpc_audience, set_tonic_canonical_body_digest,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
}
crates/ecstore/src/bucket/lifecycle/transition_transaction.rs
+216 -1
View File
@@ -23,6 +23,7 @@ use uuid::Uuid;
use crate::bucket::lifecycle::config_boundary;
use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
use crate::bucket::lifecycle::tier_sweeper::{
delete_confirmed_transition_candidate_exact_with_lease_idempotent,
delete_confirmed_transition_candidate_exact_with_manager_and_identity,
delete_object_from_remote_tier_idempotent_with_manager_and_identity,
};
@@ -616,6 +617,199 @@ pub enum TransitionTransactionRecoveryOutcome {
Retained,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum TransitionOperatorProbe {
Missing,
UnversionedPresent,
VersionedPresent(String),
Ambiguous,
Unsupported,
}
impl From<TransitionCandidateProbe> for TransitionOperatorProbe {
fn from(value: TransitionCandidateProbe) -> Self {
match value {
TransitionCandidateProbe::Missing => Self::Missing,
TransitionCandidateProbe::UnversionedPresent => Self::UnversionedPresent,
TransitionCandidateProbe::VersionedPresent(version_id) => Self::VersionedPresent(version_id),
TransitionCandidateProbe::Ambiguous => Self::Ambiguous,
TransitionCandidateProbe::Unsupported => Self::Unsupported,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct TransitionOperatorStatus {
pub transaction_id: Uuid,
pub state: TransitionTransactionState,
pub tier_name: String,
pub remote_object: String,
pub not_after_unix_nanos: i64,
pub probe: TransitionOperatorProbe,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct TransitionOperatorDeleteResult {
pub status: TransitionOperatorStatus,
pub journal_observed_after_delete: bool,
}
#[derive(Debug, thiserror::Error)]
pub enum TransitionOperatorError {
#[error("transition transaction was not found")]
NotFound,
#[error("transition transaction is still inside its active ownership window")]
NotExpired,
#[error("transition transaction state is not eligible for operator reconciliation: {0:?}")]
InvalidState(TransitionTransactionState),
#[error("an exact non-empty remote version is required")]
RemoteVersionRequired,
#[error("remote candidate is not proven missing: {0:?}")]
CandidateNotMissing(TransitionOperatorProbe),
#[error("remote candidate version does not match requested exact version: expected {expected}, observed {actual:?}")]
CandidateVersionMismatch {
expected: String,
actual: TransitionOperatorProbe,
},
#[error("transition transaction store failed: {0}")]
Store(#[source] Error),
#[error("remote tier reconciliation failed: {0}")]
Remote(#[source] std::io::Error),
}
type TransitionOperatorResult<T> = std::result::Result<T, TransitionOperatorError>;
fn validate_operator_reconcile_transaction(
transaction: &TransitionTransaction,
now_unix_nanos: i128,
) -> TransitionOperatorResult<()> {
transaction
.validate()
.map_err(|err| TransitionOperatorError::Store(Error::other(err)))?;
if transaction.state != TransitionTransactionState::UploadOutcomeUnknown {
return Err(TransitionOperatorError::InvalidState(transaction.state));
}
if now_unix_nanos < i128::from(transaction.not_after_unix_nanos) {
return Err(TransitionOperatorError::NotExpired);
}
Ok(())
}
async fn load_operator_reconcile_transaction(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<TransitionTransaction> {
match load_transition_transaction_record(api, transaction_id).await {
Ok(transaction) => Ok(transaction),
Err(Error::ConfigNotFound) => Err(TransitionOperatorError::NotFound),
Err(err) => Err(TransitionOperatorError::Store(err)),
}
}
async fn operator_probe_transition_candidate(
api: Arc<ECStore>,
transaction: &TransitionTransaction,
) -> TransitionOperatorResult<TransitionOperatorProbe> {
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
&api.tier_config_mgr(),
&transaction.tier_name,
transaction.backend_fingerprint,
)
.await
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
lease
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map(TransitionOperatorProbe::from)
.map_err(TransitionOperatorError::Remote)
}
pub async fn inspect_transition_transaction_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<TransitionOperatorStatus> {
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let probe = operator_probe_transition_candidate(api, &transaction).await?;
Ok(TransitionOperatorStatus {
transaction_id,
state: transaction.state,
tier_name: transaction.tier_name,
remote_object: transaction.remote_object,
not_after_unix_nanos: transaction.not_after_unix_nanos,
probe,
})
}
pub async fn delete_transition_candidate_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
remote_version_id: &str,
) -> TransitionOperatorResult<TransitionOperatorDeleteResult> {
if remote_version_id.is_empty() {
return Err(TransitionOperatorError::RemoteVersionRequired);
}
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let lease = TierConfigMgr::acquire_operation_lease_for_backend_identity(
&api.tier_config_mgr(),
&transaction.tier_name,
transaction.backend_fingerprint,
)
.await
.map_err(|err| TransitionOperatorError::Remote(std::io::Error::other(err)))?;
lease
.validate_remote_version_id(remote_version_id)
.map_err(TransitionOperatorError::Remote)?;
let before_delete_probe = lease
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map(TransitionOperatorProbe::from)
.map_err(TransitionOperatorError::Remote)?;
if !matches!(&before_delete_probe, TransitionOperatorProbe::VersionedPresent(version_id) if version_id == remote_version_id) {
return Err(TransitionOperatorError::CandidateVersionMismatch {
expected: remote_version_id.to_string(),
actual: before_delete_probe,
});
}
delete_confirmed_transition_candidate_exact_with_lease_idempotent(&transaction.remote_object, remote_version_id, &lease)
.await
.map_err(TransitionOperatorError::Remote)?;
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
let journal_observed_after_delete = match load_transition_transaction_record(api, transaction_id).await {
Ok(_) => true,
Err(Error::ConfigNotFound) => false,
Err(err) => return Err(TransitionOperatorError::Store(err)),
};
Ok(TransitionOperatorDeleteResult {
status: TransitionOperatorStatus {
transaction_id,
state: transaction.state,
tier_name: transaction.tier_name,
remote_object: transaction.remote_object,
not_after_unix_nanos: transaction.not_after_unix_nanos,
probe,
},
journal_observed_after_delete,
})
}
pub async fn finalize_missing_transition_transaction_for_operator(
api: Arc<ECStore>,
transaction_id: Uuid,
) -> TransitionOperatorResult<()> {
let transaction = load_operator_reconcile_transaction(api.clone(), transaction_id).await?;
validate_operator_reconcile_transaction(&transaction, time::OffsetDateTime::now_utc().unix_timestamp_nanos())?;
let probe = operator_probe_transition_candidate(api.clone(), &transaction).await?;
if probe != TransitionOperatorProbe::Missing {
return Err(TransitionOperatorError::CandidateNotMissing(probe));
}
delete_transition_transaction_record(api, transaction_id)
.await
.map_err(TransitionOperatorError::Store)
}
pub(crate) fn decode_transition_transaction_record(object: &str, data: &[u8]) -> Result<TransitionTransaction> {
let transaction_id = transition_transaction_id_from_record_object_name(object)?;
TransitionTransaction::decode(transaction_id, data)
@@ -700,7 +894,7 @@ async fn recover_unknown_upload_outcome(
.map_err(Error::other)?;
match lease
.probe_transition_candidate(&transaction.remote_object)
.probe_transition_candidate_for(&transaction.remote_object, transaction.transaction_id)
.await
.map_err(Error::other)?
{
@@ -1097,6 +1291,27 @@ mod tests {
.expect("upload state change should succeed")
}
#[test]
fn operator_reconcile_requires_expired_unknown_upload_outcome() {
let mut transaction = new_transaction();
let active_deadline = transaction.not_after_unix_nanos;
assert!(matches!(
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) + 1),
Err(TransitionOperatorError::InvalidState(TransitionTransactionState::UploadStarted))
));
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("unknown upload outcome should be recorded");
assert!(matches!(
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline) - 1),
Err(TransitionOperatorError::NotExpired)
));
validate_operator_reconcile_transaction(&transaction, i128::from(active_deadline))
.expect("expired unknown upload outcome should be eligible");
}
fn cleanup_proof(transaction: &TransitionTransaction, decision: TransitionCleanupDecision) -> TransitionCleanupProof {
TransitionCleanupProof {
transaction_id: transaction.transaction_id,
crates/ecstore/src/bucket/metadata_sys.rs
+656 -169
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/client/provider_versions.rs
+1 -1
View File
@@ -172,7 +172,7 @@ impl ProviderVersionCapabilities {
}
}
fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
pub(crate) fn validate_remote_version_id(version_id: &str) -> Result<(), Error> {
if version_id.is_empty() {
return Err(Error::new(
ErrorKind::InvalidData,
crates/ecstore/src/cluster/rpc/client.rs
+214 -9
View File
@@ -12,11 +12,20 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::http_auth::RPC_CONTENT_SHA256_HEADER;
use crate::cluster::rpc::{gen_tonic_signature_headers, normalize_tonic_rpc_audience};
#[cfg(test)]
use crate::cluster::rpc::http_auth::RPC_REPLAY_SCOPE_VERSION_HEADER;
use crate::cluster::rpc::http_auth::{
RPC_AUTH_VERSION_HEADER, RPC_AUTH_VERSION_V2, RPC_BOOT_EPOCH_CHALLENGE_HEADER, RPC_BOOT_EPOCH_HEADER,
RPC_BOOT_EPOCH_PROOF_HEADER, RPC_CONTENT_SHA256_HEADER, TIMESTAMP_HEADER,
};
use crate::cluster::rpc::{
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience, verify_tonic_boot_epoch_response,
};
#[cfg(test)]
use crate::cluster::rpc::{tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers};
use crate::disk::error::{DiskError, Error as DiskErrorType, RpcStatusError};
use crate::runtime::sources as runtime_sources;
use http::Uri;
use http::{Request as HttpRequest, Response as HttpResponse, Uri};
use rustfs_protos::{
ChannelClass, create_new_channel, get_channel_for_class,
proto_gen::node_service::{
@@ -24,9 +33,19 @@ use rustfs_protos::{
tier_mutation_control_service_client::TierMutationControlServiceClient,
},
};
use std::{error::Error, io::ErrorKind};
use std::{
collections::HashMap,
error::Error,
future::Future,
io::ErrorKind,
pin::Pin,
sync::{LazyLock, Mutex},
task::{Context, Poll},
};
use tonic::{service::interceptor::InterceptedService, transport::Channel};
use tower::Service;
use tracing::debug;
use uuid::Uuid;
use super::context_propagation::{inject_request_id_into_metadata, inject_trace_context_into_metadata};
@@ -35,7 +54,7 @@ use super::context_propagation::{inject_request_id_into_metadata, inject_trace_c
pub async fn node_service_time_out_client(
addr: &String,
interceptor: TonicInterceptor,
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
// Default to the latency-sensitive control channel; bulk `bytes` RPCs opt in via the
// `_for_class` variant below (grpc-optimization P1).
node_service_time_out_client_for_class(addr, interceptor, ChannelClass::Control).await
@@ -44,13 +63,14 @@ pub async fn node_service_time_out_client(
pub async fn heal_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<HealControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<HealControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(HealControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -59,13 +79,14 @@ pub async fn heal_control_time_out_client(
pub async fn tier_mutation_control_time_out_client(
addr: &str,
interceptor: TonicInterceptor,
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match runtime_sources::cached_node_channel(addr).await {
Some(channel) => channel,
None => create_new_channel(addr).await?,
};
let max_message_size = rustfs_protos::TIER_MUTATION_RPC_MAX_MESSAGE_SIZE;
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(TierMutationControlServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -81,7 +102,7 @@ pub async fn node_service_time_out_client_for_class(
addr: &String,
interceptor: TonicInterceptor,
class: ChannelClass,
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
let interceptor = interceptor.with_rpc_audience(addr)?;
let channel = match class {
ChannelClass::Control => match runtime_sources::cached_node_channel(addr).await {
@@ -96,6 +117,7 @@ pub async fn node_service_time_out_client_for_class(
};
let max_message_size = rustfs_protos::internode_rpc_max_message_size();
let channel = ReplayScopeChannel::new(channel, interceptor.replay_scope_audience());
Ok(NodeServiceClient::with_interceptor(channel, interceptor)
.max_decoding_message_size(max_message_size)
.max_encoding_message_size(max_message_size))
@@ -103,7 +125,7 @@ pub async fn node_service_time_out_client_for_class(
pub async fn node_service_time_out_client_no_auth(
addr: &String,
) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>, Box<dyn Error>> {
) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>, Box<dyn Error>> {
node_service_time_out_client(addr, TonicInterceptor::NoOp(NoOpInterceptor)).await
}
@@ -199,6 +221,104 @@ pub(crate) fn is_network_like_disk_error(err: &DiskErrorType) -> bool {
}
}
/// The transport service that learns an authenticated peer boot epoch and adds the replay-scoped
/// signature only after one has been observed. The v1/v2 interceptor stays inside this wrapper so
/// old servers continue receiving precisely the metadata they understand.
#[derive(Clone, Debug)]
pub struct ReplayScopeChannel<S> {
inner: S,
audience: Option<String>,
}
/// The channel type used by internode clients after v2 authentication and replay-scope handling.
pub type AuthenticatedChannel = ReplayScopeChannel<Channel>;
static PEER_BOOT_EPOCHS: LazyLock<Mutex<HashMap<String, Uuid>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
impl<S> ReplayScopeChannel<S> {
fn new(inner: S, audience: Option<String>) -> Self {
Self { inner, audience }
}
}
fn cached_peer_boot_epoch(audience: &str) -> Option<Uuid> {
PEER_BOOT_EPOCHS.lock().ok().and_then(|epochs| epochs.get(audience).copied())
}
fn remember_peer_boot_epoch(audience: String, epoch: Uuid) {
if let Ok(mut epochs) = PEER_BOOT_EPOCHS.lock() {
epochs.insert(audience, epoch);
}
}
impl<S, ReqBody, ResBody> Service<HttpRequest<ReqBody>> for ReplayScopeChannel<S>
where
S: Service<HttpRequest<ReqBody>, Response = HttpResponse<ResBody>>,
S::Error: Send + 'static,
S::Future: Send + 'static,
ReqBody: Send + 'static,
ResBody: Send + 'static,
{
type Response = HttpResponse<ResBody>;
type Error = S::Error;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut request: HttpRequest<ReqBody>) -> Self::Future {
let authenticated = self.audience.as_ref().is_some_and(|_| {
request
.headers()
.get(RPC_AUTH_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
== Some(RPC_AUTH_VERSION_V2)
});
let challenge = authenticated.then(Uuid::new_v4);
if let (Some(audience), Some(challenge)) = (self.audience.as_deref(), challenge) {
// The challenge is independently HMAC-authenticated by the response proof. It is not
// part of v2 so old peers ignore it, while a new peer can safely advertise its epoch.
request.headers_mut().insert(
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
challenge.to_string().parse().expect("UUID must be a valid header value"),
);
if let (Some(boot_epoch), Some(timestamp), Some(content_sha256)) = (
cached_peer_boot_epoch(audience),
request.headers().get(TIMESTAMP_HEADER).and_then(|value| value.to_str().ok()),
request
.headers()
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok()),
) {
match gen_tonic_replay_scope_headers(audience, request.uri().path(), timestamp, content_sha256, boot_epoch) {
Ok(headers) => request.headers_mut().extend(headers),
Err(error) => debug!(error = %error, "could not attach replay-scoped RPC signature"),
}
}
}
let audience = self.audience.clone();
let future = self.inner.call(request);
Box::pin(async move {
let response = future.await?;
if let (Some(audience), Some(challenge)) = (audience, challenge) {
match verify_tonic_boot_epoch_response(&audience, challenge, response.headers()) {
Ok(epoch) => remember_peer_boot_epoch(audience, epoch),
Err(error)
if response.headers().contains_key(RPC_BOOT_EPOCH_HEADER)
|| response.headers().contains_key(RPC_BOOT_EPOCH_PROOF_HEADER) =>
{
debug!(error = %error, "peer boot epoch response proof was rejected")
}
Err(_) => {}
}
}
Ok(response)
})
}
}
pub struct TonicSignatureInterceptor {
audience: Option<String>,
}
@@ -257,6 +377,13 @@ impl TonicInterceptor {
}
Ok(self)
}
fn replay_scope_audience(&self) -> Option<String> {
match self {
Self::Signature(interceptor) => interceptor.audience.clone(),
Self::NoOp(_) => None,
}
}
}
impl tonic::service::Interceptor for TonicInterceptor {
@@ -279,6 +406,38 @@ mod tests {
use tracing_opentelemetry::OpenTelemetrySpanExt;
use tracing_subscriber::{Registry, layer::SubscriberExt};
#[derive(Clone)]
struct EpochProofService {
audience: String,
seen_headers: std::sync::Arc<Mutex<Vec<http::HeaderMap>>>,
}
impl Service<HttpRequest<()>> for EpochProofService {
type Response = HttpResponse<()>;
type Error = std::convert::Infallible;
type Future = std::future::Ready<Result<Self::Response, Self::Error>>;
fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
Poll::Ready(Ok(()))
}
fn call(&mut self, request: HttpRequest<()>) -> Self::Future {
self.seen_headers
.lock()
.expect("test header capture lock must not be poisoned")
.push(request.headers().clone());
let challenge = tonic_boot_epoch_challenge(request.headers())
.expect("client challenge must be syntactically valid")
.expect("authenticated client request must carry a boot epoch challenge");
let mut response = HttpResponse::new(());
response.headers_mut().extend(
tonic_boot_epoch_response_headers(&self.audience, challenge)
.expect("test server must be able to sign an epoch proof"),
);
std::future::ready(Ok(response))
}
}
fn ensure_test_rpc_secret() {
runtime_sources::ensure_test_rpc_secret();
}
@@ -420,6 +579,52 @@ mod tests {
assert_eq!(interceptor.audience.as_deref(), Some("node-a:9000"));
}
#[test]
fn replay_scope_channel_uses_epoch_proof_before_sending_v3() {
ensure_test_rpc_secret();
let audience = "replay-scope-client-test:9000";
PEER_BOOT_EPOCHS
.lock()
.expect("peer epoch cache lock must not be poisoned")
.remove(audience);
let seen_headers = std::sync::Arc::new(Mutex::new(Vec::new()));
let service = EpochProofService {
audience: audience.to_string(),
seen_headers: seen_headers.clone(),
};
let mut channel = ReplayScopeChannel::new(service, Some(audience.to_string()));
let make_request = || {
let mut request = HttpRequest::builder()
.uri("/node_service.NodeService/Ping")
.body(())
.expect("test RPC request must build");
request.headers_mut().extend(
gen_tonic_signature_headers(audience, "node_service.NodeService", "Ping", None)
.expect("v2 test headers must mint"),
);
request
};
futures::executor::block_on(channel.call(make_request())).expect("first request must complete");
futures::executor::block_on(channel.call(make_request())).expect("second request must complete");
let headers = seen_headers.lock().expect("test header capture lock must not be poisoned");
assert_eq!(headers.len(), 2);
assert!(headers[0].contains_key(RPC_BOOT_EPOCH_CHALLENGE_HEADER));
assert!(
!headers[0].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the first request must remain v2-compatible until the peer proves its epoch"
);
assert!(
headers[1].contains_key(RPC_REPLAY_SCOPE_VERSION_HEADER),
"the second request must carry the replay-scoped v3 signature"
);
PEER_BOOT_EPOCHS
.lock()
.expect("peer epoch cache lock must not be poisoned")
.remove(audience);
}
#[test]
fn test_signature_interceptor_requires_generated_method_metadata() {
ensure_test_rpc_secret();
crates/ecstore/src/cluster/rpc/http_auth.rs
+435 -9
View File
@@ -20,8 +20,8 @@
//! rustfs/rustfs#4402) is anchored by the `ghsa_r5qv_*` tests in the module
//! below, plus the broader negative-signature suite. The advisory class is: a
//! node must never accept an RPC whose auth is missing, malformed, or signed
//! with the default/empty shared secret. Body-bound v2 requests additionally
//! receive process-local replay protection. See
//! with the default/empty shared secret. Body-bound v2 requests and all replay-scoped v3
//! requests additionally receive process-local replay protection. See
//! `docs/testing/security-regressions.md` for the full advisory -> test map.
//!
//! Advisory: <https://github.com/rustfs/rustfs/security/advisories/GHSA-r5qv-rc46-hv8q>
@@ -50,13 +50,22 @@ use uuid::Uuid;
type HmacSha256 = Hmac<Sha256>;
const SIGNATURE_HEADER: &str = "x-rustfs-signature";
const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
pub(crate) const TIMESTAMP_HEADER: &str = "x-rustfs-timestamp";
pub(crate) const RPC_AUTH_VERSION_HEADER: &str = "x-rustfs-rpc-auth-version";
const RPC_SIGNATURE_V2_HEADER: &str = "x-rustfs-rpc-signature-v2";
const RPC_NONCE_HEADER: &str = "x-rustfs-rpc-nonce";
pub(crate) const RPC_CONTENT_SHA256_HEADER: &str = "x-rustfs-content-sha256";
const RPC_AUTH_VERSION_V2: &str = "2";
pub(crate) const RPC_AUTH_VERSION_V2: &str = "2";
pub const RPC_REPLAY_SCOPE_VERSION_HEADER: &str = "x-rustfs-rpc-replay-scope-version";
pub const RPC_REPLAY_SCOPE_SIGNATURE_HEADER: &str = "x-rustfs-rpc-signature-v3";
pub const RPC_REPLAY_SCOPE_NONCE_HEADER: &str = "x-rustfs-rpc-replay-nonce";
pub const RPC_BOOT_EPOCH_HEADER: &str = "x-rustfs-rpc-boot-epoch";
pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-challenge";
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
const UNSIGNED_PAYLOAD_NONCE: &str = "unsigned";
const SIGNATURE_VALID_DURATION: i64 = 300; // 5 minutes
@@ -75,9 +84,15 @@ static INTERNODE_RPC_BODY_DIGEST_STRICT: LazyLock<bool> = LazyLock::new(|| {
rustfs_config::DEFAULT_INTERNODE_RPC_BODY_DIGEST_STRICT,
)
});
// Sized for peak legitimate body-bound mutation RPS x the retention window; overflow fails closed
// and increments the replay-cache overflow counter. Clamped to at least 1 so a misconfigured zero
// cannot disable replay protection by rejecting every body-bound request.
static INTERNODE_RPC_REPLAY_SCOPE_STRICT: LazyLock<bool> = LazyLock::new(|| {
get_env_bool(
rustfs_config::ENV_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
rustfs_config::DEFAULT_INTERNODE_RPC_REPLAY_SCOPE_STRICT,
)
});
// Sized for peak legitimate authenticated RPC RPS x the retention window once replay scope is
// active; overflow fails closed and increments the replay-cache overflow counter. Clamped to at
// least 1 so a misconfigured zero cannot disable replay protection by rejecting every request.
static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
rustfs_utils::get_env_usize(
rustfs_config::ENV_INTERNODE_RPC_REPLAY_CACHE_CAPACITY,
@@ -86,6 +101,7 @@ static REPLAY_CACHE_CAPACITY: LazyLock<usize> = LazyLock::new(|| {
.max(1)
});
static RPC_SECRET_RESOLUTION_LOG_ONCE: Once = Once::new();
static RPC_BOOT_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
#[derive(Default)]
struct RpcNonceCache {
@@ -313,6 +329,189 @@ fn verify_signature_v2(secret: &str, scope: SignatureV2Scope<'_>, signature: &st
mac.verify_slice(&signature).is_ok()
}
#[derive(Clone, Copy)]
struct ReplayScope<'a> {
audience: &'a str,
path: &'a str,
timestamp: &'a str,
nonce: Uuid,
content_sha256: &'a str,
boot_epoch: Uuid,
}
fn update_replay_scope(mac: &mut HmacSha256, scope: ReplayScope<'_>) {
mac.update(RPC_REPLAY_SCOPE_DOMAIN);
for part in [
scope.audience.as_bytes(),
b"|",
scope.path.as_bytes(),
b"|POST|",
scope.timestamp.as_bytes(),
b"|",
scope.nonce.as_bytes(),
b"|",
scope.content_sha256.as_bytes(),
b"|",
scope.boot_epoch.as_bytes(),
] {
mac.update(part);
}
}
fn generate_replay_scope_signature(secret: &str, scope: ReplayScope<'_>) -> std::io::Result<String> {
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_replay_scope(&mut mac, scope);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
}
fn verify_replay_scope_signature(secret: &str, scope: ReplayScope<'_>, signature: &str) -> bool {
let Ok(signature) = general_purpose::STANDARD.decode(signature) else {
return false;
};
let Ok(mut mac) = <HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()) else {
return false;
};
update_replay_scope(&mut mac, scope);
mac.verify_slice(&signature).is_ok()
}
fn update_boot_epoch_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
mac.update(RPC_BOOT_EPOCH_PROOF_DOMAIN);
mac.update(audience.as_bytes());
mac.update(b"|");
mac.update(challenge.as_bytes());
mac.update(boot_epoch.as_bytes());
}
fn generate_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid) -> std::io::Result<String> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
Ok(general_purpose::STANDARD.encode(mac.finalize().into_bytes()))
}
fn verify_boot_epoch_proof(secret: &str, audience: &str, challenge: Uuid, boot_epoch: Uuid, proof: &str) -> std::io::Result<()> {
if audience.is_empty() || challenge.is_nil() || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid RPC boot epoch proof scope"));
}
let proof = general_purpose::STANDARD
.decode(proof)
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch proof"))?;
let mut mac =
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
update_boot_epoch_proof(&mut mac, audience, challenge, boot_epoch);
mac.verify_slice(&proof)
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC boot epoch proof"))
}
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
(!value.is_nil())
.then_some(value)
.ok_or_else(|| std::io::Error::other(format!("Invalid {name}")))
}
fn parse_tonic_rpc_path(path: &str) -> std::io::Result<(&str, &str)> {
path.strip_prefix('/')
.and_then(|path| path.split_once('/'))
.filter(|(service, rpc_method)| !service.is_empty() && !rpc_method.is_empty() && !rpc_method.contains('/'))
.ok_or_else(|| std::io::Error::other("Invalid RPC request path"))
}
/// The process-unique epoch included in every replay-scoped server verification.
///
/// A fresh process gets a fresh value, so a signature captured before a server restart cannot be
/// admitted even though the bounded in-memory nonce cache necessarily starts empty again.
pub fn tonic_rpc_boot_epoch() -> Uuid {
*RPC_BOOT_EPOCH
}
/// Build the additive replay-scope headers for a request that already carries rolling-upgrade-safe
/// v1/v2 metadata. `timestamp` and `content_sha256` are deliberately reused from the v2 scope so
/// old servers can continue validating the same request unchanged.
pub fn gen_tonic_replay_scope_headers(
audience: &str,
path: &str,
timestamp: &str,
content_sha256: &str,
boot_epoch: Uuid,
) -> std::io::Result<HeaderMap> {
if audience.is_empty() || !path.starts_with('/') || !valid_content_sha256(content_sha256) || boot_epoch.is_nil() {
return Err(std::io::Error::other("Invalid replay-scoped RPC signing scope"));
}
parse_tonic_rpc_path(path)?;
timestamp
.parse::<i64>()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
let nonce = Uuid::new_v4();
let signature = generate_replay_scope_signature(
&get_shared_secret()?,
ReplayScope {
audience,
path,
timestamp,
nonce,
content_sha256,
boot_epoch,
},
)?;
let mut headers = HeaderMap::new();
headers.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
headers.insert(
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
header_value(&signature, RPC_REPLAY_SCOPE_SIGNATURE_HEADER)?,
);
headers.insert(
RPC_REPLAY_SCOPE_NONCE_HEADER,
header_value(&nonce.to_string(), RPC_REPLAY_SCOPE_NONCE_HEADER)?,
);
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
Ok(headers)
}
/// Parse the optional client challenge used to authenticate a server boot-epoch advertisement.
pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option<Uuid>> {
headers
.get(RPC_BOOT_EPOCH_CHALLENGE_HEADER)
.map(|value| {
value
.to_str()
.map_err(|_| std::io::Error::other("Invalid RPC boot epoch challenge"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch challenge"))
})
.transpose()
}
/// Build the authenticated response headers for a client boot-epoch challenge.
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
let boot_epoch = tonic_rpc_boot_epoch();
let proof = generate_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch)?;
let mut headers = HeaderMap::new();
headers.insert(RPC_BOOT_EPOCH_HEADER, header_value(&boot_epoch.to_string(), RPC_BOOT_EPOCH_HEADER)?);
headers.insert(RPC_BOOT_EPOCH_PROOF_HEADER, header_value(&proof, RPC_BOOT_EPOCH_PROOF_HEADER)?);
Ok(headers)
}
/// Verify the server boot-epoch response for a challenge generated by this client.
pub fn verify_tonic_boot_epoch_response(audience: &str, challenge: Uuid, headers: &HeaderMap) -> std::io::Result<Uuid> {
let boot_epoch = headers
.get(RPC_BOOT_EPOCH_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
let proof = headers
.get(RPC_BOOT_EPOCH_PROOF_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch proof"))?;
verify_boot_epoch_proof(&get_shared_secret()?, audience, challenge, boot_epoch, proof)?;
Ok(boot_epoch)
}
fn valid_content_sha256(value: &str) -> bool {
value == UNSIGNED_PAYLOAD
|| (value.len() == 64
@@ -531,6 +730,17 @@ fn has_v2_auth_headers(headers: &HeaderMap) -> bool {
.any(|name| headers.contains_key(*name))
}
fn has_replay_scope_headers(headers: &HeaderMap) -> bool {
[
RPC_REPLAY_SCOPE_VERSION_HEADER,
RPC_REPLAY_SCOPE_SIGNATURE_HEADER,
RPC_REPLAY_SCOPE_NONCE_HEADER,
RPC_BOOT_EPOCH_HEADER,
]
.iter()
.any(|name| headers.contains_key(*name))
}
/// Whether the server requires target-bound v2 authentication on every internode gRPC request,
/// rejecting the legacy constant-target fallback instead of accepting it. Default-off rollout
/// lever gated on the v1-fallback counter reading zero fleet-wide; see
@@ -540,9 +750,127 @@ fn internode_rpc_signature_strict() -> bool {
*INTERNODE_RPC_SIGNATURE_STRICT
}
fn internode_rpc_replay_scope_strict() -> bool {
*INTERNODE_RPC_REPLAY_SCOPE_STRICT
}
fn verify_tonic_replay_scope_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
if audience.is_empty() {
return Err(std::io::Error::other("Missing RPC audience"));
}
parse_tonic_rpc_path(path)?;
let version = headers
.get(RPC_REPLAY_SCOPE_VERSION_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope version"))?;
if version != RPC_REPLAY_SCOPE_VERSION_V3 {
return Err(std::io::Error::other("Unsupported RPC replay scope version"));
}
let signature = headers
.get(RPC_REPLAY_SCOPE_SIGNATURE_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope signature"))?;
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing timestamp header"))?;
let signed_at = timestamp
.parse::<i64>()
.map_err(|_| std::io::Error::other("Invalid timestamp format"))?;
check_timestamp(signed_at)?;
let nonce = headers
.get(RPC_REPLAY_SCOPE_NONCE_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC replay scope nonce"))
.and_then(|value| non_nil_uuid(value, "RPC replay scope nonce"))?;
let content_sha256 = headers
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC content SHA-256"))?;
if !valid_content_sha256(content_sha256) {
return Err(std::io::Error::other("Invalid RPC content SHA-256"));
}
let boot_epoch = headers
.get(RPC_BOOT_EPOCH_HEADER)
.and_then(|value| value.to_str().ok())
.ok_or_else(|| std::io::Error::other("Missing RPC boot epoch"))
.and_then(|value| non_nil_uuid(value, "RPC boot epoch"))?;
let secret = get_shared_secret()?;
if !verify_replay_scope_signature(
&secret,
ReplayScope {
audience,
path,
timestamp,
nonce,
content_sha256,
boot_epoch,
},
signature,
) {
return Err(std::io::Error::other("Invalid RPC replay scope signature"));
}
if boot_epoch != tonic_rpc_boot_epoch() {
return Err(std::io::Error::other("RPC boot epoch is stale"));
}
check_and_record_nonce(nonce, signed_at)
}
/// Verify gRPC authentication, preferring v2 without downgrade on malformed v2 metadata.
pub fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &HeaderMap) -> std::io::Result<()> {
verify_tonic_rpc_signature_with_strictness(audience, path, headers, internode_rpc_signature_strict())
verify_tonic_rpc_signature_with_policy(
audience,
path,
headers,
internode_rpc_signature_strict(),
internode_rpc_replay_scope_strict(),
false,
)
}
/// Verify gRPC authentication while allowing the narrowly scoped v2 `Ping` bootstrap used to
/// obtain an authenticated server boot epoch when replay-scope strictness is enabled.
pub fn verify_tonic_rpc_signature_with_bootstrap(
audience: &str,
path: &str,
headers: &HeaderMap,
allow_replay_scope_bootstrap: bool,
) -> std::io::Result<()> {
verify_tonic_rpc_signature_with_policy(
audience,
path,
headers,
internode_rpc_signature_strict(),
internode_rpc_replay_scope_strict(),
allow_replay_scope_bootstrap,
)
}
fn verify_tonic_rpc_signature_with_policy(
audience: &str,
path: &str,
headers: &HeaderMap,
signature_strict: bool,
replay_scope_strict: bool,
allow_replay_scope_bootstrap: bool,
) -> std::io::Result<()> {
if has_replay_scope_headers(headers) {
return verify_tonic_replay_scope_signature(audience, path, headers);
}
// Only a method-bound v2 Ping with a syntactically valid challenge may bootstrap a strict
// client after its peer restarts. Legacy metadata never gets this exception.
let bootstrap = allow_replay_scope_bootstrap
&& has_v2_auth_headers(headers)
&& tonic_boot_epoch_challenge(headers).is_ok_and(|challenge| challenge.is_some());
if replay_scope_strict && !bootstrap {
return Err(std::io::Error::other("RPC replay-scoped authentication required"));
}
verify_tonic_rpc_signature_with_strictness(audience, path, headers, signature_strict)?;
global_internode_metrics().record_replay_scope_fallback();
Ok(())
}
/// [`verify_tonic_rpc_signature`] with the strict gate injected as a parameter, so both rollout
@@ -1273,6 +1601,104 @@ mod tests {
assert_eq!(error.to_string(), "Invalid RPC v2 signature");
}
#[test]
fn replay_scope_binds_path_epoch_and_random_nonce() {
ensure_test_rpc_secret();
let path = "/node_service.NodeService/Ping";
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
let timestamp = headers
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 timestamp")
.to_string();
let content_sha256 = headers
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 content digest")
.to_string();
headers.extend(
gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, tonic_rpc_boot_epoch())
.expect("replay-scope headers should build"),
);
assert!(
verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false).is_ok(),
"the first replay-scoped request must be accepted"
);
let replay = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &headers, false, false, false)
.expect_err("the random replay-scope nonce must be single-use");
assert_eq!(replay.to_string(), "RPC request replay detected");
let path_error = verify_tonic_replay_scope_signature("node-a:9000", "/node_service.NodeService/SignalService", &headers)
.expect_err("a replay-scoped signature must not move to another method");
assert_eq!(path_error.to_string(), "Invalid RPC replay scope signature");
}
#[test]
fn replay_scope_rejects_partial_metadata_and_stale_epoch_without_fallback() {
ensure_test_rpc_secret();
let path = "/node_service.NodeService/Ping";
let mut partial = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
partial.insert(RPC_REPLAY_SCOPE_VERSION_HEADER, HeaderValue::from_static(RPC_REPLAY_SCOPE_VERSION_V3));
let error = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
.expect_err("partial replay-scope metadata must never downgrade to v2");
assert_eq!(error.to_string(), "Missing RPC replay scope signature");
let timestamp = partial
.get(TIMESTAMP_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 timestamp")
.to_string();
let content_sha256 = partial
.get(RPC_CONTENT_SHA256_HEADER)
.and_then(|value| value.to_str().ok())
.expect("v2 content digest")
.to_string();
let stale_epoch = Uuid::new_v4();
partial.extend(
gen_tonic_replay_scope_headers("node-a:9000", path, &timestamp, &content_sha256, stale_epoch)
.expect("replay-scope headers should build"),
);
let stale = verify_tonic_rpc_signature_with_policy("node-a:9000", path, &partial, false, false, false)
.expect_err("a signature from a prior server boot epoch must be rejected");
assert_eq!(stale.to_string(), "RPC boot epoch is stale");
}
#[test]
fn replay_scope_strictness_allows_only_authenticated_ping_bootstrap() {
ensure_test_rpc_secret();
let mut headers = gen_tonic_signature_headers("node-a:9000", "node_service.NodeService", "Ping", None)
.expect("v2 compatibility headers should build");
let rejected =
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, false)
.expect_err("strict replay scope must reject stripped new metadata");
assert_eq!(rejected.to_string(), "RPC replay-scoped authentication required");
headers.insert(
RPC_BOOT_EPOCH_CHALLENGE_HEADER,
HeaderValue::from_str(&Uuid::new_v4().to_string()).expect("UUID header"),
);
assert!(
verify_tonic_rpc_signature_with_policy("node-a:9000", "/node_service.NodeService/Ping", &headers, false, true, true,)
.is_ok(),
"only the signed Ping bootstrap may obtain a new server epoch in strict mode"
);
}
#[test]
fn boot_epoch_response_proof_binds_audience_challenge_and_epoch() {
ensure_test_rpc_secret();
let challenge = Uuid::new_v4();
let headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("proof headers should build");
let epoch =
verify_tonic_boot_epoch_response("node-a:9000", challenge, &headers).expect("matching proof headers should verify");
assert_eq!(epoch, tonic_rpc_boot_epoch());
assert!(verify_tonic_boot_epoch_response("node-b:9000", challenge, &headers).is_err());
assert!(verify_tonic_boot_epoch_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
}
#[test]
fn malformed_v2_auth_does_not_downgrade_to_valid_legacy_signature() {
ensure_test_rpc_secret();
crates/ecstore/src/cluster/rpc/mod.rs
+10 -5
View File
@@ -26,13 +26,18 @@ pub(crate) mod runtime_sources;
pub use background_monitor::shutdown_background_monitors;
pub(crate) use background_monitor::spawn_background_monitor;
pub use client::{
TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client, node_service_time_out_client_no_auth,
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
node_service_time_out_client_no_auth,
};
// Re-exported through `api::rpc`; not every item is consumed inside this crate.
#[allow(unused_imports)]
pub use http_auth::{
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_signature_headers, normalize_tonic_rpc_audience,
set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability, sign_tonic_rpc_response_proof,
verify_ns_scanner_capability, verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
TONIC_RPC_PREFIX, build_auth_headers, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
normalize_tonic_rpc_audience, set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, sign_ns_scanner_capability,
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, verify_ns_scanner_capability,
verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use internode_data_transport::TcpHttpInternodeDataTransport;
crates/ecstore/src/cluster/rpc/peer_rest_client.rs
+57 -6
View File
@@ -13,7 +13,7 @@
// limitations under the License.
use crate::cluster::rpc::client::{
TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
AuthenticatedChannel, TonicInterceptor, embedded_tonic_status, gen_tonic_signature_interceptor, heal_control_time_out_client,
is_network_like_status, message_has_network_needle, node_service_time_out_client, tier_mutation_control_time_out_client,
};
use crate::cluster::rpc::{set_tonic_canonical_body_digest, set_tonic_mutation_body_digest, verify_tonic_rpc_response_proof};
@@ -66,7 +66,6 @@ use std::{
use tokio::{net::TcpStream, time::Duration};
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
use uuid::Uuid;
@@ -222,6 +221,21 @@ fn validate_heal_control_response_proof(canonical_response: &[u8], proof: &[u8])
.map_err(|_| Error::other("peer returned an invalid heal control response proof"))
}
fn decode_remote_version_state_capability(expected_member: &str, result: &[u8]) -> Result<Uuid> {
let (topology_member, process_epoch) = rustfs_protos::decode_remote_version_state_capability(result).map_err(Error::other)?;
if topology_member != expected_member {
return Err(Error::other(
"peer returned a remote version state capability for a different topology member",
));
}
let server_epoch =
Uuid::from_slice(process_epoch).map_err(|_| Error::other("peer returned an invalid remote version state epoch"))?;
if server_epoch.is_nil() {
return Err(Error::other("peer returned a nil remote version state epoch"));
}
Ok(server_epoch)
}
#[derive(Clone, Debug)]
pub struct PeerLiveEventsBatch {
pub events: Vec<u8>,
@@ -233,6 +247,7 @@ pub struct PeerLiveEventsBatch {
pub struct PeerRestClient {
pub host: XHost,
pub grid_host: String,
topology_member: String,
offline: Arc<AtomicBool>,
recovery_running: Arc<AtomicBool>,
}
@@ -325,9 +340,11 @@ impl PeerRestClient {
}
pub fn new(host: XHost, grid_host: String) -> Self {
let topology_member = host.to_string();
Self {
host,
grid_host,
topology_member,
offline: Arc::new(AtomicBool::new(false)),
recovery_running: Arc::new(AtomicBool::new(false)),
}
@@ -347,7 +364,11 @@ impl PeerRestClient {
let client = match grid_host {
Some(grid_host) => match XHost::try_from(peer_host_port.clone()) {
Ok(host) => Some(PeerRestClient::new(host, grid_host)),
Ok(host) => {
let mut client = PeerRestClient::new(host, grid_host);
client.topology_member = peer_host_port.clone();
Some(client)
}
Err(err) => {
warn!(peer = %peer_host_port, "Xhost parse failed while constructing peer client: {err:?}");
None
@@ -390,7 +411,7 @@ impl PeerRestClient {
(remote, all, remote_topology_hosts)
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if self.offline.load(Ordering::Acquire) {
self.mark_offline_and_spawn_recovery();
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -411,7 +432,7 @@ impl PeerRestClient {
&self,
) -> Result<
rustfs_protos::proto_gen::node_service::heal_control_service_client::HealControlServiceClient<
InterceptedService<Channel, TonicInterceptor>,
InterceptedService<AuthenticatedChannel, TonicInterceptor>,
>,
> {
if self.offline.load(Ordering::Acquire) {
@@ -432,7 +453,7 @@ impl PeerRestClient {
async fn get_tier_mutation_control_client(
&self,
) -> Result<TierMutationControlServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
) -> Result<TierMutationControlServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if self.offline.load(Ordering::Acquire) {
self.mark_offline_and_spawn_recovery();
return Err(Error::other(format!("peer {} is temporarily offline", self.grid_host)));
@@ -1154,6 +1175,15 @@ impl PeerRestClient {
validate_heal_control_capability_proof(&canonical_ack, &proof)
}
pub async fn probe_remote_version_state(&self, topology_fingerprint: String) -> Result<(String, Uuid)> {
let probe = rustfs_protos::remote_version_state_capability_probe(Uuid::new_v4().as_bytes());
let result = self
.heal_control(rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION, topology_fingerprint, probe)
.await?;
let epoch = decode_remote_version_state_capability(&self.topology_member, &result)?;
Ok((self.topology_member.clone(), epoch))
}
pub async fn load_bucket_metadata(&self, bucket: &str, scanner_maintenance_change: bool) -> Result<()> {
self.finalize_result(
async {
@@ -2470,6 +2500,22 @@ mod tests {
}
}
#[test]
fn remote_version_state_capability_decoder_fails_closed() {
let epoch = Uuid::new_v4();
let result = rustfs_protos::encode_remote_version_state_capability("node-a:9000", epoch.as_bytes())
.expect("small capability response should encode");
assert_eq!(
decode_remote_version_state_capability("node-a:9000", &result).expect("valid epoch should decode"),
epoch
);
assert!(decode_remote_version_state_capability("node-b:9000", &result).is_err());
assert!(decode_remote_version_state_capability("node-a:9000", &result[..result.len() - 1]).is_err());
let nil = rustfs_protos::encode_remote_version_state_capability("node-a:9000", Uuid::nil().as_bytes())
.expect("small capability response should encode");
assert!(decode_remote_version_state_capability("node-a:9000", &nil).is_err());
}
struct TierMutationResponseFixture<'a> {
version: u32,
phase: TierMutationRpcPhase,
@@ -2741,6 +2787,11 @@ mod tests {
.with_span_list(true),
);
let _guard = tracing::subscriber::set_default(subscriber);
// The `recovery-monitor` callsite is shared with the production
// `mark_offline_and_spawn_recovery` path that sibling tests exercise from
// subscriber-less threads; without this the span can be cached as
// `Interest::never()` and silently degrade to `Span::none()`.
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let client = test_peer_client();
let span = tracing::info_span!("request-span", request_id = "req-peer-rest");
crates/ecstore/src/cluster/rpc/peer_s3_client.rs
+180 -13
View File
@@ -14,7 +14,8 @@
use crate::bucket::metadata_sys;
use crate::cluster::rpc::client::{
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use crate::disk::error::DiskError;
@@ -40,16 +41,84 @@ use rustfs_protos::proto_gen::node_service::node_service_client::NodeServiceClie
use rustfs_protos::proto_gen::node_service::{
DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest,
};
#[cfg(test)]
use std::sync::{
Mutex as StdMutex,
atomic::{AtomicBool, Ordering},
};
use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration};
#[cfg(test)]
use tokio::sync::Notify;
use tokio::{net::TcpStream, sync::RwLock, time};
use tokio_util::sync::CancellationToken;
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
type Client = Arc<Box<dyn PeerS3Client>>;
#[cfg(test)]
#[derive(Default)]
pub(crate) struct DeleteBucketEmptyScanBarrier {
arrived: AtomicBool,
arrived_notify: Notify,
released: AtomicBool,
release_notify: Notify,
}
#[cfg(test)]
impl DeleteBucketEmptyScanBarrier {
pub(crate) async fn wait_until_paused(&self) {
loop {
let notified = self.arrived_notify.notified();
if self.arrived.load(Ordering::Acquire) {
return;
}
notified.await;
}
}
pub(crate) fn release(&self) {
self.released.store(true, Ordering::Release);
self.release_notify.notify_waiters();
}
async fn pause(&self) {
self.arrived.store(true, Ordering::Release);
self.arrived_notify.notify_waiters();
loop {
let notified = self.release_notify.notified();
if self.released.load(Ordering::Acquire) {
return;
}
notified.await;
}
}
}
#[cfg(test)]
static DELETE_BUCKET_EMPTY_SCAN_BARRIER: StdMutex<Option<Arc<DeleteBucketEmptyScanBarrier>>> = StdMutex::new(None);
#[cfg(test)]
pub(crate) fn install_delete_bucket_empty_scan_barrier() -> Arc<DeleteBucketEmptyScanBarrier> {
let barrier = Arc::new(DeleteBucketEmptyScanBarrier::default());
*DELETE_BUCKET_EMPTY_SCAN_BARRIER
.lock()
.expect("empty scan barrier lock should not be poisoned") = Some(barrier.clone());
barrier
}
#[cfg(test)]
async fn pause_after_delete_bucket_empty_scan() {
let barrier = DELETE_BUCKET_EMPTY_SCAN_BARRIER
.lock()
.expect("empty scan barrier lock should not be poisoned")
.take();
if let Some(barrier) = barrier {
barrier.pause().await;
}
}
#[derive(Clone, Debug)]
pub struct ScannerBucketListing {
pub buckets: Vec<BucketInfo>,
@@ -650,24 +719,22 @@ impl PeerS3Client for LocalPeerS3Client {
return Err(Error::ErasureWriteQuorum);
}
let force = if opts.force_if_empty && !opts.force {
if opts.force_if_empty && !opts.force {
for disk in local_disks.iter() {
if has_xlmeta_files(&disk.path().join(bucket)).await.map_err(Error::Io)? {
return Err(Error::VolumeNotEmpty);
}
}
true
} else {
opts.force
};
#[cfg(test)]
pause_after_delete_bucket_empty_scan().await;
}
let mut futures = Vec::with_capacity(local_disks.len());
for disk in local_disks.iter() {
// Non-force delete refuses a non-empty bucket (VolumeNotEmpty), which
// the recreate loop below turns into BucketNotEmpty; only an explicit
// force delete removes recursively (backlog#799 B1).
futures.push(disk.delete_volume(bucket, force));
// `force_if_empty` is validation-only. Passing it as force would let
// a PutObject committed after the scan be removed recursively.
futures.push(disk.delete_volume(bucket, opts.force));
}
let results = join_all(futures).await;
@@ -730,6 +797,15 @@ pub struct RemotePeerS3Client {
}
impl RemotePeerS3Client {
fn encode_delete_bucket_options(opts: &DeleteBucketOptions) -> Result<String> {
let mut remote_opts = opts.clone();
// Older peers promote `force_if_empty` to recursive force after their
// metadata scan. Keep this coordinator-only hint off the wire so a
// mixed-version delete fails closed on non-empty directory remnants.
remote_opts.force_if_empty = false;
serde_json::to_string(&remote_opts).map_err(Into::into)
}
fn recovery_monitor_span(addr: &str) -> tracing::Span {
tracing::info_span!(
"recovery-monitor",
@@ -756,7 +832,7 @@ impl RemotePeerS3Client {
client
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
node_service_time_out_client(&self.addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))
@@ -1040,7 +1116,7 @@ impl PeerS3Client for RemotePeerS3Client {
async fn delete_bucket(&self, bucket: &str, opts: &DeleteBucketOptions) -> Result<()> {
self.execute_with_timeout(
|| async {
let options = serde_json::to_string(opts)?;
let options = Self::encode_delete_bucket_options(opts)?;
let mut client = self.get_client().await?;
let mut request = Request::new(DeleteBucketRequest {
@@ -1414,6 +1490,49 @@ mod tests {
}
}
#[test]
fn remote_delete_bucket_options_fail_closed_for_legacy_peers() {
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
no_lock: true,
no_recreate: true,
force_if_empty: true,
..Default::default()
})
.expect("remote delete options should serialize");
let legacy_opts: DeleteBucketOptions =
serde_json::from_str(&encoded).expect("legacy peer should decode remote delete options");
assert!(legacy_opts.no_lock);
assert!(legacy_opts.no_recreate);
assert!(!legacy_opts.force);
assert!(!legacy_opts.force_if_empty);
let legacy_recursive_force = if legacy_opts.force_if_empty && !legacy_opts.force {
true
} else {
legacy_opts.force
};
assert!(
!legacy_recursive_force,
"legacy peer must not upgrade empty-only delete to recursive force"
);
}
#[test]
fn remote_delete_bucket_options_preserve_explicit_force() {
let encoded = RemotePeerS3Client::encode_delete_bucket_options(&DeleteBucketOptions {
force: true,
force_if_empty: true,
..Default::default()
})
.expect("remote force-delete options should serialize");
let remote_opts: DeleteBucketOptions =
serde_json::from_str(&encoded).expect("remote peer should decode force-delete options");
assert!(remote_opts.force);
assert!(!remote_opts.force_if_empty);
}
#[tokio::test]
async fn test_execute_with_timeout_marks_remote_peer_faulty_on_network_like_error() {
let client = test_remote_peer("http://peer-network-error:9000");
@@ -1571,6 +1690,54 @@ mod tests {
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn local_peer_force_if_empty_preserves_unclassified_file_in_selected_pool() {
reset_local_disk_test_state().await;
let temp_dir = TempDir::new().expect("create temp dir for empty-only delete regression");
let disks = init_test_local_disks_for_pools(
&temp_dir,
&[(0, 1), (1, 1)],
"local-peer-force-if-empty-preserves-unclassified-file",
)
.await;
let bucket = "empty-only-delete-bucket";
let marker = "object/commit-marker";
let data = bytes::Bytes::from_static(b"committed object data");
disks[1]
.make_volume(bucket)
.await
.expect("bucket should be created in the selected pool");
disks[1]
.write_all(bucket, marker, data.clone())
.await
.expect("unclassified committed file should be written");
let err = LocalPeerS3Client::new_with_local_disks(None, Some(vec![1]), disks.clone())
.delete_bucket(
bucket,
&DeleteBucketOptions {
force_if_empty: true,
..Default::default()
},
)
.await
.expect_err("empty-only delete must not recursively remove an unclassified file");
assert_eq!(err, Error::VolumeNotEmpty);
assert_eq!(
disks[1]
.read_all(bucket, marker)
.await
.expect("unclassified committed file should be preserved"),
data
);
reset_local_disk_test_state().await;
}
#[tokio::test]
#[serial]
async fn heal_bucket_local_recreates_missing_bucket_volumes() {
crates/ecstore/src/cluster/rpc/remote_disk.rs
+41 -5
View File
@@ -13,8 +13,8 @@
// limitations under the License.
use crate::cluster::rpc::client::{
TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error, node_service_time_out_client,
node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, is_network_like_disk_error,
node_service_time_out_client, node_service_time_out_client_for_class, node_service_time_out_client_no_auth,
};
use crate::cluster::rpc::http_auth::set_tonic_canonical_body_digest;
use crate::cluster::rpc::internode_data_transport::{
@@ -71,7 +71,7 @@ use tokio::{
time::timeout,
};
use tokio_util::sync::CancellationToken;
use tonic::{Code, Request, service::interceptor::InterceptedService, transport::Channel};
use tonic::{Code, Request, service::interceptor::InterceptedService};
use tracing::{debug, trace, warn};
use uuid::Uuid;
@@ -1083,7 +1083,7 @@ impl RemoteDisk {
internode_offline_bypass_reason(&self.addr).map(Error::other)
}
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
@@ -1096,7 +1096,7 @@ impl RemoteDisk {
/// Routes onto the isolated bulk channel pool so large transfers cannot head-of-line block
/// lock/health RPCs (grpc-optimization P1). Falls back to the control channel when isolation
/// is disabled.
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
async fn get_bulk_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
if let Some(err) = self.offline_bypass_error() {
return Err(err);
}
@@ -3005,6 +3005,7 @@ mod tests {
use crate::cluster::rpc::internode_data_transport::{InternodeDataTransportCapabilities, TcpHttpInternodeDataTransport};
use crate::runtime::sources as runtime_sources;
use serde_json::Value;
use serial_test::serial;
use std::io::{self as std_io, Write};
use std::pin::Pin;
use std::sync::{Arc, Mutex, Mutex as StdMutex, Once};
@@ -3018,6 +3019,20 @@ mod tests {
static INIT: Once = Once::new();
// `#[serial(internode_metrics)]` marks every test that observes
// `global_internode_metrics()`. Those counters are a process-wide singleton:
// some of these tests snapshot a counter, run one decode, and assert on the
// delta, while others deliberately record decode errors or call
// `reset_internode_metrics_for_test()`. Run concurrently in one process they
// corrupt each other's deltas — a sibling's error bumps the "no decode error"
// assertion off zero, and a sibling's reset can drive an `after > before`
// assertion backwards.
//
// The marker only takes effect under the `cargo test` fallback; nextest
// already isolates each test in its own process, so every test there gets its
// own copy of the counters (see `docs/testing/README.md`). Any new test that
// reads or mutates the global internode metrics belongs in this group.
#[test]
fn snapshot_lease_response_requires_current_protocol_and_valid_token() {
let token = SnapshotLeaseToken::new();
@@ -3306,6 +3321,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_read_multiple_resp("msgpack", b"binary");
@@ -3328,6 +3344,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_falls_back_to_json_payloads() {
let endpoint = sample_remote_endpoint();
let json_resp = sample_read_multiple_resp("json", b"fallback");
@@ -3349,6 +3366,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn rename_data_response_accepts_legacy_json_without_decode_error() {
crate::cluster::rpc::runtime_sources::reset_internode_metrics_for_test();
let response = RenameDataResp {
@@ -3500,6 +3518,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = ReadMultipleResponse {
@@ -3525,6 +3544,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn read_multiple_response_decode_reports_corrupt_json_item() {
let endpoint = sample_remote_endpoint();
let response = ReadMultipleResponse {
@@ -3565,6 +3585,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_decode_prefers_msgpack_payloads() {
let endpoint = sample_remote_endpoint();
let msgpack_resp = sample_batch_read_version_resp(7, "msgpack-object", true);
@@ -3585,6 +3606,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_rejects_invalid_success_metadata() {
let endpoint = sample_remote_endpoint();
let mut response_item = sample_batch_read_version_resp(0, "invalid-object", true);
@@ -3604,6 +3626,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_decode_reports_corrupt_msgpack_item() {
let endpoint = sample_remote_endpoint();
let response = BatchReadVersionResponse {
@@ -3630,6 +3653,7 @@ mod tests {
}
#[test]
#[serial(internode_metrics)]
fn batch_read_version_response_decode_reports_corrupt_json_item() {
let endpoint = sample_remote_endpoint();
let response = BatchReadVersionResponse {
@@ -4419,6 +4443,7 @@ mod tests {
}
#[tokio::test]
#[serial(internode_metrics)]
async fn test_remote_disk_create_file_retries_once_on_retryable_open_write_error() {
let transport = RetryingOpenWriteInternodeDataTransport::with_steps(vec![
OpenWriteTestStep::Error(DiskError::from(rustfs_rio::new_test_internode_http_io_error(
@@ -4457,6 +4482,7 @@ mod tests {
}
#[tokio::test]
#[serial(internode_metrics)]
async fn test_remote_disk_read_file_stream_retries_once_on_retryable_open_read_error() {
// A transient reset-by-peer on a shard read during the read-after-write window must be
// absorbed by one re-dial rather than eroding read quorum (issue #2761).
@@ -5306,6 +5332,11 @@ mod tests {
.with_span_list(true),
);
let _guard = tracing::subscriber::set_default(subscriber);
// The `recovery-monitor` span and the monitor's own log events are
// production callsites that sibling tests exercise from subscriber-less
// threads; without this they can be cached as `Interest::never()` and go
// silently missing here.
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let endpoint = Endpoint {
url: url::Url::parse("http://127.0.0.1:59996/data").expect("endpoint URL should parse"),
@@ -5360,6 +5391,11 @@ mod tests {
.with_span_list(true),
);
let _guard = tracing::subscriber::set_default(subscriber);
// The `recovery-monitor` span and the monitor's own log events are
// production callsites that sibling tests exercise from subscriber-less
// threads; without this they can be cached as `Interest::never()` and go
// silently missing here.
let _callsite_pin = crate::test_tracing::pin_callsite_interest_for_test();
let addr = "http://127.0.0.1:59997".to_string();
let endpoint = Endpoint {
crates/ecstore/src/cluster/rpc/remote_locker.rs
+4 -3
View File
@@ -12,7 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::cluster::rpc::client::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
use crate::cluster::rpc::client::{
AuthenticatedChannel, TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client,
};
use crate::cluster::rpc::set_tonic_mutation_body_digest;
use async_trait::async_trait;
use bytes::Bytes;
@@ -29,7 +31,6 @@ use std::time::Duration;
use tokio::time::timeout;
use tonic::Request;
use tonic::service::interceptor::InterceptedService;
use tonic::transport::Channel;
use tracing::{debug, info, warn};
/// Remote lock client implementation
@@ -78,7 +79,7 @@ impl RemoteClient {
}
}
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<Channel, TonicInterceptor>>> {
pub async fn get_client(&self) -> Result<NodeServiceClient<InterceptedService<AuthenticatedChannel, TonicInterceptor>>> {
// P3-2 offline bypass (now covering the lock path too): fast-fail a peer already marked
// offline instead of paying the connect timeout, so dsync reaches quorum sooner. Does not
// change quorum; the self-healing re-probe keeps the peer recoverable.
crates/ecstore/src/disk/local.rs
+1203 -27
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/disk/os.rs
+321 -9
View File
@@ -25,7 +25,7 @@ use std::{
sync::{Arc, LazyLock, Weak},
};
use tokio::fs;
use tokio::sync::{OwnedSemaphorePermit, Semaphore, SemaphorePermit};
use tokio::sync::{OwnedSemaphorePermit, RwLock, Semaphore, SemaphorePermit};
use tracing::warn;
/// Check path length according to OS limits.
@@ -123,6 +123,8 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new()));
fn default_global_file_sync_limit(cpu_count: usize, max_blocking_threads: usize) -> usize {
let cpu_scaled = cpu_count
@@ -157,6 +159,24 @@ pub(crate) fn disk_file_sync_limiter(root: &Path) -> Arc<Semaphore> {
limiter
}
/// Serialize a bucket's local metadata commits with physical bucket removal.
///
/// The key includes the canonical disk root, so independently reconnected
/// [`LocalDisk`](super::local::LocalDisk) instances share the same lock while
/// disconnected disks do not keep the registry alive.
pub(crate) fn disk_volume_mutation_lock(root: &Path, volume: &str) -> Arc<RwLock<()>> {
let key = root.join(volume);
let mut locks = DISK_VOLUME_MUTATION_LOCKS.lock();
locks.retain(|_, lock| lock.strong_count() > 0);
if let Some(lock) = locks.get(&key).and_then(Weak::upgrade) {
return lock;
}
let lock = Arc::new(RwLock::new(()));
locks.insert(key, Arc::downgrade(&lock));
lock
}
/// Always acquire the per-disk permit before the process-wide permit. Keeping
/// this order uniform prevents one slow disk from reserving global capacity
/// while it waits for its own concurrency slot.
@@ -572,15 +592,14 @@ async fn reliable_rename_inner(
base_dir: impl AsRef<Path>,
warn_on_missing_source: bool,
) -> io::Result<()> {
if let Some(parent) = dst_file_path.as_ref().parent()
&& !file_exists(parent)
{
reliable_mkdir_all(parent, base_dir.as_ref()).await?;
}
let parent_guard = match dst_file_path.as_ref().parent() {
Some(parent) => Some(mkdir_all_below_existing_base(parent, base_dir.as_ref()).await?),
None => None,
};
let mut i = 0;
loop {
if let Err(e) = super::fs::rename_std(src_file_path.as_ref(), dst_file_path.as_ref()) {
if let Err(e) = rename_into_existing_parent(src_file_path.as_ref(), dst_file_path.as_ref(), parent_guard.as_ref()) {
if should_retry_rename(&e, i) {
i += 1;
continue;
@@ -597,6 +616,159 @@ async fn reliable_rename_inner(
Ok(())
}
#[cfg(unix)]
fn rename_into_existing_parent(
src_file_path: &Path,
dst_file_path: &Path,
parent_guard: Option<&ExistingBaseDirectoryGuard>,
) -> io::Result<()> {
use rustix::fs::{Mode, OFlags, open, renameat};
let Some(parent_guard) = parent_guard else {
return super::fs::rename_std(src_file_path, dst_file_path);
};
let src_parent = src_file_path
.parent()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a parent directory"))?;
let src_name = src_file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename source must have a file name"))?;
let dst_name = dst_file_path
.file_name()
.ok_or_else(|| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must have a file name"))?;
let src_parent = open(
src_parent,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC,
Mode::empty(),
)
.map_err(io::Error::from)?;
let dst_parent = parent_guard
.last()
.ok_or_else(|| io::Error::other("rename destination parent guard is empty"))?;
renameat(&src_parent, src_name, dst_parent, dst_name).map_err(io::Error::from)
}
#[cfg(not(unix))]
fn rename_into_existing_parent(
src_file_path: &Path,
dst_file_path: &Path,
_parent_guard: Option<&ExistingBaseDirectoryGuard>,
) -> io::Result<()> {
super::fs::rename_std(src_file_path, dst_file_path)
}
async fn mkdir_all_below_existing_base(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
let dir_path = dir_path.to_path_buf();
let base_dir = base_dir.to_path_buf();
tokio::task::spawn_blocking(move || mkdir_all_below_existing_base_std(&dir_path, &base_dir)).await?
}
#[cfg(windows)]
pub(crate) type ExistingBaseDirectoryGuard = Vec<winapi_util::Handle>;
#[cfg(unix)]
pub(crate) type ExistingBaseDirectoryGuard = Vec<std::os::fd::OwnedFd>;
#[cfg(all(not(unix), not(windows)))]
pub(crate) type ExistingBaseDirectoryGuard = ();
#[cfg(windows)]
fn lock_windows_directory(path: &Path) -> io::Result<winapi_util::Handle> {
use std::os::windows::fs::OpenOptionsExt;
const FILE_ATTRIBUTE_DIRECTORY: u64 = 0x10;
const FILE_ATTRIBUTE_REPARSE_POINT: u32 = 0x400;
const FILE_FLAG_BACKUP_SEMANTICS: u32 = 0x0200_0000;
const FILE_FLAG_OPEN_REPARSE_POINT: u32 = 0x0020_0000;
const FILE_SHARE_READ: u32 = 0x1;
let file = std::fs::OpenOptions::new()
.read(true)
.share_mode(FILE_SHARE_READ)
.custom_flags(FILE_FLAG_BACKUP_SEMANTICS | FILE_FLAG_OPEN_REPARSE_POINT)
.open(path)?;
let handle = winapi_util::Handle::from_file(file);
let info = winapi_util::file::information(&handle)?;
if info.file_attributes() & FILE_ATTRIBUTE_DIRECTORY == 0
|| info.file_attributes() & u64::from(FILE_ATTRIBUTE_REPARSE_POINT) != 0
{
return Err(io::Error::from(io::ErrorKind::NotADirectory));
}
Ok(handle)
}
pub(crate) fn mkdir_all_below_existing_base_std(dir_path: &Path, base_dir: &Path) -> io::Result<ExistingBaseDirectoryGuard> {
let relative = dir_path
.strip_prefix(base_dir)
.map_err(|_| io::Error::new(io::ErrorKind::InvalidInput, "rename destination must remain below its base directory"))?;
for component in relative.components() {
if !matches!(component, Component::Normal(_) | Component::CurDir) {
return Err(io::Error::new(
io::ErrorKind::InvalidInput,
"rename destination contains an invalid path component",
));
}
}
#[cfg(unix)]
{
use rustix::fs::{Mode, OFlags, mkdirat, open, openat};
use rustix::io::Errno;
let flags = OFlags::RDONLY | OFlags::DIRECTORY | OFlags::NOFOLLOW | OFlags::CLOEXEC;
let mode = Mode::RWXU | Mode::RWXG | Mode::RWXO;
let mut parents = vec![open(base_dir, flags, Mode::empty()).map_err(io::Error::from)?];
for component in relative.components() {
let Component::Normal(component) = component else {
continue;
};
let parent = parents
.last()
.expect("base directory guard should contain the base directory");
match mkdirat(parent, component, mode) {
Ok(()) => {}
Err(Errno::EXIST) => {}
Err(err) => return Err(err.into()),
}
parents.push(openat(parent, component, flags, Mode::empty()).map_err(io::Error::from)?);
}
Ok(parents)
}
#[cfg(windows)]
{
let mut handles = vec![lock_windows_directory(base_dir)?];
let mut current = base_dir.to_path_buf();
for component in relative.components() {
let Component::Normal(component) = component else {
continue;
};
current.push(component);
match std::fs::create_dir(&current) {
Ok(()) => {}
Err(err) if err.kind() == io::ErrorKind::AlreadyExists => {}
Err(err) => return Err(err),
}
handles.push(lock_windows_directory(&current)?);
}
Ok(handles)
}
#[cfg(all(not(unix), not(windows)))]
{
let _ = relative;
Err(io::Error::new(
io::ErrorKind::Unsupported,
"safe recursive directory creation is unavailable on this platform",
))
}
}
fn warn_reliable_rename_failure(src_file_path: &Path, dst_file_path: &Path, base_dir: &Path, err: &io::Error) {
warn!(
"reliable_rename failed. src_file_path: {:?}, dst_file_path: {:?}, base_dir: {:?}, err: {:?}",
@@ -727,6 +899,20 @@ mod tests {
Arc::new(Semaphore::new(MAX_PARALLEL_FILE_SYNCS))
}
#[tokio::test]
async fn disk_volume_mutation_lock_is_shared_per_root_and_volume() {
let temp_dir = tempdir().expect("create temp dir");
let first = disk_volume_mutation_lock(temp_dir.path(), "bucket");
let second = disk_volume_mutation_lock(temp_dir.path(), "bucket");
let other = disk_volume_mutation_lock(temp_dir.path(), "other-bucket");
assert!(Arc::ptr_eq(&first, &second), "reconnected disks must share a bucket mutation lock");
assert!(!Arc::ptr_eq(&first, &other), "different buckets must not serialize each other");
let _write_guard = first.write().await;
assert!(second.try_read().is_err(), "a bucket delete lock must exclude local commits");
}
#[derive(Clone, Default)]
struct CapturedLogs {
buffer: Arc<Mutex<Vec<u8>>>,
@@ -771,9 +957,26 @@ mod tests {
}
}
/// Holds a `warn_capture()` capture alive: the thread-local subscriber, plus
/// the pin that keeps tracing's process-global callsite-interest cache from
/// being decided by some other test's thread.
struct WarnCaptureGuard {
_subscriber: tracing::subscriber::DefaultGuard,
_callsite_pin: tracing::Dispatch,
}
/// Capture WARN-level output on the current thread; tokio tests here run on
/// the current-thread runtime, so the guard covers the whole test body.
fn warn_capture() -> (CapturedLogs, tracing::subscriber::DefaultGuard) {
///
/// The callsite pin matters because `warn_reliable_rename_failure` is a
/// single production callsite shared with tests that call `rename_all`
/// *without* installing a subscriber — `rename_all_missing_source_returns_file_not_found`
/// is one. Whichever thread reaches it first fixes its `Interest`
/// process-wide, so without the pin that sibling can cache
/// `Interest::never()` and the WARN never fires here at all, leaving the
/// "must keep the WARN" assertions staring at empty output. See
/// [`crate::test_tracing::pin_callsite_interest_for_test`].
fn warn_capture() -> (CapturedLogs, WarnCaptureGuard) {
let logs = CapturedLogs::default();
let subscriber = tracing_subscriber::fmt()
.with_max_level(tracing::Level::WARN)
@@ -781,7 +984,10 @@ mod tests {
.with_ansi(false)
.without_time()
.finish();
let guard = tracing::subscriber::set_default(subscriber);
let guard = WarnCaptureGuard {
_subscriber: tracing::subscriber::set_default(subscriber),
_callsite_pin: crate::test_tracing::pin_callsite_interest_for_test(),
};
(logs, guard)
}
@@ -968,6 +1174,112 @@ mod tests {
assert_eq!(std::fs::read(dst.join("nested").join("part.1")).expect("read moved part"), b"payload");
}
#[tokio::test]
async fn rename_all_does_not_recreate_missing_base_directory() {
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
std::fs::create_dir(&base).expect("create destination base");
let src = temp_dir.path().join("staged-object");
std::fs::write(&src, b"payload").expect("write staged object");
let dst = base.join("object").join("xl.meta");
std::fs::remove_dir(&base).expect("delete destination base before commit");
let err = rename_all(&src, &dst, &base)
.await
.expect_err("rename must not recreate a deleted destination base");
assert!(matches!(err, DiskError::FileNotFound));
assert!(src.exists(), "failed commit must preserve the staged source");
assert!(!base.exists(), "failed commit must not recreate the deleted bucket");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_all_rejects_a_replaced_base_with_an_existing_parent() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&base).expect("create destination base");
std::fs::create_dir_all(outside.join("object")).expect("create outside destination parent");
let src = temp_dir.path().join("staged-object");
std::fs::write(&src, b"payload").expect("write staged object");
let dst = base.join("object").join("xl.meta");
std::fs::remove_dir(&base).expect("remove destination base before replacement");
symlink(&outside, &base).expect("replace destination base with a symlink");
rename_all(&src, &dst, &base)
.await
.expect_err("rename must reject an existing destination parent below a replaced base");
assert!(src.exists(), "rejected rename must preserve the staged source");
assert!(
!outside.join("object/xl.meta").exists(),
"rename must not publish through the replacement symlink"
);
}
#[cfg(windows)]
#[test]
fn windows_parent_guard_blocks_base_and_intermediate_replacement() {
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
std::fs::create_dir(&base).expect("create destination base");
let parent = base.join("object").join("nested");
let guard = mkdir_all_below_existing_base_std(&parent, &base).expect("create and lock destination parents");
std::fs::rename(&base, temp_dir.path().join("replacement-base"))
.expect_err("the locked base must not be replaceable before commit");
std::fs::rename(base.join("object"), base.join("replacement-object"))
.expect_err("a locked intermediate directory must not be replaceable before commit");
drop(guard);
std::fs::rename(base.join("object"), base.join("replacement-object"))
.expect("replacement should succeed after the commit guard is released");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_parent_creation_rejects_symlinked_base() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&outside).expect("create outside directory");
let base = temp_dir.path().join("bucket");
symlink(&outside, &base).expect("create symlinked base");
mkdir_all_below_existing_base(&base.join("object"), &base)
.await
.expect_err("symlinked base must be rejected");
assert!(!outside.join("object").exists(), "parent creation must remain confined to the base");
}
#[cfg(unix)]
#[tokio::test]
async fn rename_parent_creation_rejects_symlink_below_base() {
use std::os::unix::fs::symlink;
let temp_dir = tempdir().expect("create temp dir");
let base = temp_dir.path().join("bucket");
let outside = temp_dir.path().join("outside");
std::fs::create_dir(&base).expect("create destination base");
std::fs::create_dir(&outside).expect("create outside directory");
symlink(&outside, base.join("linked")).expect("create symlink below base");
mkdir_all_below_existing_base(&base.join("linked/object"), &base)
.await
.expect_err("symlink below base must be rejected");
assert!(
!outside.join("object").exists(),
"parent creation must not follow a symlink outside the base"
);
}
#[tokio::test]
async fn fsync_dir_succeeds_on_directory() {
let temp_dir = tempdir().expect("create temp dir");
crates/ecstore/src/lib.rs
+3
View File
@@ -93,3 +93,6 @@ pub(crate) mod ecstore_validation_blackbox;
#[cfg(test)]
pub(crate) mod test_metrics;
#[cfg(test)]
pub(crate) mod test_tracing;
crates/ecstore/src/runtime/sources.rs
-4
View File
@@ -202,10 +202,6 @@ pub(crate) async fn ensure_boot_time() {
GLOBAL_BOOT_TIME.get_or_init(|| async { SystemTime::now() }).await;
}
pub(crate) async fn scanner_init_time() -> Option<chrono::DateTime<chrono::Utc>> {
rustfs_common::get_global_init_time().await
}
pub(crate) async fn root_disk_threshold_for_erasure_disk() -> Option<u64> {
if is_erasure_sd().await {
None
crates/ecstore/src/services/metrics_realtime.rs
+42 -4
View File
@@ -71,6 +71,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_common::metrics::ScannerMetricsRepo
MadminScannerMetrics {
collected_at: metrics.collected_at,
current_cycle: metrics.current_cycle,
current_cycle_active: Some(metrics.current_cycle_active),
current_started: metrics.current_started,
cycles_completed_at: metrics.cycles_completed_at,
ongoing_buckets: metrics.ongoing_buckets,
@@ -398,10 +399,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
if types.contains(&MetricType::SCANNER) {
debug!("start get scanner metrics");
let mut metrics = global_metrics().report().await;
if let Some(init_time) = runtime_sources::scanner_init_time().await {
metrics.current_started = init_time;
}
let metrics = global_metrics().report().await;
real_time_metrics.aggregated.scanner = Some(to_madmin_scanner_metrics(metrics));
}
@@ -540,7 +538,9 @@ async fn collect_local_disks_metrics(disks: &HashSet<String>) -> HashMap<String,
#[cfg(test)]
mod test {
use super::*;
use rustfs_common::metrics::CurrentCycle;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use serial_test::serial;
use std::time::Duration;
#[test]
@@ -588,7 +588,10 @@ mod test {
#[test]
fn scanner_metrics_mapping_preserves_partial_source_status() {
let current_started = Utc::now() - chrono::Duration::seconds(5);
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
current_cycle_active: true,
current_started,
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
partial_cycles_by_source: vec![rustfs_common::metrics::ScannerSourceCycleSnapshot {
@@ -598,6 +601,8 @@ mod test {
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_started, current_started);
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
let usage = scanner
@@ -608,6 +613,39 @@ mod test {
assert_eq!(usage.cycles, 2);
}
#[tokio::test]
#[serial]
async fn collect_local_metrics_preserves_scanner_cycle_started_time() {
let previous_init_time = *rustfs_common::globals::GLOBAL_INIT_TIME.read().await;
let previous_cycle = global_metrics().get_cycle().await;
let init_time = Utc::now() - chrono::Duration::hours(1);
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = Some(init_time);
let cycle = CurrentCycle {
current: 0,
next: 1,
started: cycle_started,
..Default::default()
};
let cycle_start = global_metrics().start_scan_cycle_work_with_cycle(cycle).await;
let realtime = collect_local_metrics(MetricType::SCANNER, &CollectMetricsOpts::default()).await;
global_metrics()
.finish_scan_cycle_work_with_cycle(cycle_start, previous_cycle.clone().unwrap_or_default())
.await;
global_metrics().set_cycle(previous_cycle).await;
*rustfs_common::globals::GLOBAL_INIT_TIME.write().await = previous_init_time;
let encoded = rmp_serde::to_vec_named(&realtime).expect("realtime metrics should encode");
let decoded: RealtimeMetrics = rmp_serde::from_slice(&encoded).expect("realtime metrics should decode");
let mut aggregated = RealtimeMetrics::default();
aggregated.merge(decoded);
let scanner = aggregated.aggregated.scanner.expect("scanner metrics");
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_started, cycle_started);
}
#[test]
fn scanner_metrics_mapping_preserves_pacing_pressure() {
let scanner = to_madmin_scanner_metrics(rustfs_common::metrics::ScannerMetricsReport {
crates/ecstore/src/services/notification_sys.rs
+384 -2
View File
@@ -29,11 +29,11 @@ use rustfs_madmin::metrics::RealtimeMetrics;
use rustfs_madmin::net::NetInfo;
use rustfs_madmin::{ItemState, ServerProperties, StorageInfo};
use rustfs_utils::XHost;
use std::collections::{HashMap, hash_map::DefaultHasher};
use std::collections::{BTreeMap, HashMap, hash_map::DefaultHasher};
use std::future::Future;
use std::hash::{Hash, Hasher};
use std::sync::{Arc, Mutex, OnceLock};
use std::time::{Duration, SystemTime};
use std::time::{Duration, Instant, SystemTime};
use tokio::time::{sleep, timeout};
use tokio_util::sync::CancellationToken;
use tracing::{debug, error, info, warn};
@@ -47,6 +47,9 @@ const EVENT_NOTIFICATION_PEER_PROPAGATION: &str = "notification_peer_propagation
const SCANNER_ACTIVITY_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const TIER_CONFIG_RELOAD_RETRY_BASE: Duration = Duration::from_millis(100);
const TIER_CONFIG_RELOAD_RETRY_CAP: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROBE_INTERVAL: Duration = Duration::from_secs(10);
const REMOTE_VERSION_STATE_PROBE_TIMEOUT: Duration = Duration::from_secs(5);
const REMOTE_VERSION_STATE_PROOF_TTL: Duration = Duration::from_secs(30);
/// Cached result from the last successful admin call to a peer.
struct PeerAdminCache {
@@ -91,6 +94,180 @@ lazy_static! {
pub static ref GLOBAL_NOTIFICATION_SYS: OnceLock<Arc<NotificationSys>> = OnceLock::new();
}
#[derive(Clone)]
struct RemoteVersionStateFleetProof {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
expires_at: Instant,
}
impl RemoteVersionStateFleetProof {
fn token(&self) -> RemoteVersionStateFleetProofToken {
RemoteVersionStateFleetProofToken {
topology_fingerprint: self.topology_fingerprint.clone(),
peer_epochs: self.peer_epochs.clone(),
}
}
}
#[derive(Clone, PartialEq, Eq)]
pub(crate) struct RemoteVersionStateFleetProofToken {
topology_fingerprint: String,
peer_epochs: Arc<BTreeMap<String, Uuid>>,
}
#[derive(Default)]
struct RemoteVersionStateFleetProofState {
proof: Option<RemoteVersionStateFleetProof>,
topology_conflict: bool,
}
static REMOTE_VERSION_STATE_FLEET_PROOF: OnceLock<std::sync::RwLock<RemoteVersionStateFleetProofState>> = OnceLock::new();
static REMOTE_VERSION_STATE_PROBE_TOPOLOGY: OnceLock<String> = OnceLock::new();
fn remote_version_state_fleet_proof_slot() -> &'static std::sync::RwLock<RemoteVersionStateFleetProofState> {
REMOTE_VERSION_STATE_FLEET_PROOF.get_or_init(|| std::sync::RwLock::new(RemoteVersionStateFleetProofState::default()))
}
fn replace_remote_version_state_fleet_proof(proof: Option<RemoteVersionStateFleetProof>) {
replace_remote_version_state_fleet_proof_in(remote_version_state_fleet_proof_slot(), proof);
}
fn replace_remote_version_state_fleet_proof_in(
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
proof: Option<RemoteVersionStateFleetProof>,
) {
slot.write().unwrap_or_else(std::sync::PoisonError::into_inner).proof = proof;
}
fn publish_remote_version_state_probe_result(
slot: &std::sync::RwLock<RemoteVersionStateFleetProofState>,
topology_fingerprint: &str,
result: Result<BTreeMap<String, Uuid>>,
observed_at: Instant,
) -> Option<Error> {
match result {
Ok(peer_epochs) => {
let mut state = slot.write().unwrap_or_else(std::sync::PoisonError::into_inner);
let peer_epochs = state
.proof
.as_ref()
.filter(|proof| proof.topology_fingerprint == topology_fingerprint && proof.peer_epochs.as_ref() == &peer_epochs)
.map(|proof| Arc::clone(&proof.peer_epochs))
.unwrap_or_else(|| Arc::new(peer_epochs));
state.proof = Some(RemoteVersionStateFleetProof {
topology_fingerprint: topology_fingerprint.to_string(),
peer_epochs,
expires_at: observed_at + REMOTE_VERSION_STATE_PROOF_TTL,
});
None
}
Err(err) => {
replace_remote_version_state_fleet_proof_in(slot, None);
Some(err)
}
}
}
pub(crate) fn acquire_remote_version_state_fleet_proof() -> Option<RemoteVersionStateFleetProofToken> {
let expected_topology = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get()?;
let state = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
acquire_remote_version_state_fleet_proof_from(&state, expected_topology, Instant::now())
}
fn acquire_remote_version_state_fleet_proof_from(
state: &RemoteVersionStateFleetProofState,
expected_topology: &str,
now: Instant,
) -> Option<RemoteVersionStateFleetProofToken> {
if state.topology_conflict || !remote_version_state_fleet_proof_valid_at(state.proof.as_ref(), expected_topology, now) {
return None;
}
state.proof.as_ref().map(RemoteVersionStateFleetProof::token)
}
pub(crate) fn remote_version_state_fleet_proof_matches(proof: &RemoteVersionStateFleetProofToken) -> bool {
let Some(expected_topology) = REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() else {
return false;
};
let state = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.topology_conflict {
return false;
}
state.proof.as_ref().is_some_and(|current| {
current.topology_fingerprint == *expected_topology
&& current.topology_fingerprint == proof.topology_fingerprint
&& Arc::ptr_eq(&current.peer_epochs, &proof.peer_epochs)
&& Instant::now() < current.expires_at
})
}
fn remote_version_state_fleet_proof_valid_at(
proof: Option<&RemoteVersionStateFleetProof>,
expected_topology: &str,
now: Instant,
) -> bool {
proof.is_some_and(|proof| proof.topology_fingerprint == expected_topology && now < proof.expires_at)
}
fn insert_remote_version_state_peer(peer_epochs: &mut BTreeMap<String, Uuid>, peer: String, epoch: Uuid) -> Result<()> {
if epoch.is_nil() || peer_epochs.values().any(|existing| *existing == epoch) || peer_epochs.insert(peer, epoch).is_some() {
return Err(Error::other("remote version state capability peer identity is invalid"));
}
Ok(())
}
pub fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.set(topology_fingerprint.clone()).is_err() {
if REMOTE_VERSION_STATE_PROBE_TOPOLOGY.get() != Some(&topology_fingerprint) {
let mut state = remote_version_state_fleet_proof_slot()
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.topology_conflict = true;
state.proof = None;
}
return;
}
tokio::spawn(async move {
loop {
let result = match get_global_notification_sys() {
Some(notification_sys) => {
match timeout(
REMOTE_VERSION_STATE_PROBE_TIMEOUT,
notification_sys.probe_remote_version_state_fleet(&topology_fingerprint),
)
.await
{
Ok(result) => result,
Err(_) => Err(Error::other("remote version state fleet capability probe timed out")),
}
}
None => Err(Error::other("remote version state fleet capability notification system is unavailable")),
};
let topology_conflict = remote_version_state_fleet_proof_slot()
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.topology_conflict;
if topology_conflict {
replace_remote_version_state_fleet_proof(None);
} else if let Some(err) = publish_remote_version_state_probe_result(
remote_version_state_fleet_proof_slot(),
&topology_fingerprint,
result,
Instant::now(),
) {
debug!(error = %err, "remote version state fleet capability probe failed closed");
}
sleep(REMOTE_VERSION_STATE_PROBE_INTERVAL).await;
}
});
}
pub async fn new_global_notification_sys(eps: EndpointServerPools) -> Result<()> {
let _ = GLOBAL_NOTIFICATION_SYS
.set(Arc::new(NotificationSys::new(eps).await))
@@ -115,7 +292,17 @@ pub struct NotificationSys {
impl NotificationSys {
pub async fn new(eps: EndpointServerPools) -> Self {
let expected_remote_hosts = eps
.peer_grid_host_slots_sorted()
.into_iter()
.filter_map(|(peer, _, is_local)| (!is_local).then_some(peer))
.collect::<Vec<_>>();
let (peer_clients, all_peer_clients, peer_topology_hosts) = PeerRestClient::new_clients_with_topology(eps).await;
let peer_topology_hosts = if peer_topology_hosts.is_empty() {
expected_remote_hosts
} else {
peer_topology_hosts
};
let peer_admin_caches = (0..peer_clients.len()).map(|_| Mutex::new(PeerAdminCache::new())).collect();
Self {
peer_clients,
@@ -125,6 +312,24 @@ impl NotificationSys {
tier_config_reload_workers: Default::default(),
}
}
async fn probe_remote_version_state_fleet(&self, topology_fingerprint: &str) -> Result<BTreeMap<String, Uuid>> {
if self.peer_clients.len() != self.peer_topology_hosts.len() {
return Err(Error::other("remote version state capability fleet membership is incomplete"));
}
let probes = self.peer_clients.iter().map(|client| async {
let client = client
.as_ref()
.ok_or_else(|| Error::other("remote version state capability peer is unreachable"))?;
client.probe_remote_version_state(topology_fingerprint.to_string()).await
});
let mut peer_epochs = BTreeMap::new();
for result in join_all(probes).await {
let (peer, epoch) = result?;
insert_remote_version_state_peer(&mut peer_epochs, peer, epoch)?;
}
Ok(peer_epochs)
}
}
pub struct NotificationPeerErr {
@@ -1890,6 +2095,183 @@ fn aggregate_scanner_dirty_usage_acknowledgement_results(
mod tests {
use super::*;
#[test]
fn remote_version_state_fleet_proof_rejects_stale_or_mismatched_membership() {
let now = Instant::now();
let mut peer_epochs = BTreeMap::new();
peer_epochs.insert("peer-a".to_string(), Uuid::new_v4());
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(peer_epochs),
expires_at: now + Duration::from_secs(1),
};
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-b", now));
assert!(!remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", proof.expires_at));
assert!(!remote_version_state_fleet_proof_valid_at(None, "topology-a", now));
}
#[test]
fn remote_version_state_fleet_proof_rejects_nil_process_epoch() {
let mut peer_epochs = BTreeMap::new();
assert!(insert_remote_version_state_peer(&mut peer_epochs, "peer-a".to_string(), Uuid::nil()).is_err());
assert!(peer_epochs.is_empty());
}
#[test]
fn remote_version_state_fleet_proof_accepts_single_node_membership() {
let now = Instant::now();
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
};
assert!(remote_version_state_fleet_proof_valid_at(Some(&proof), "topology-a", now));
}
#[test]
fn remote_version_state_fleet_proof_token_changes_with_process_epoch() {
let now = Instant::now();
let proof = RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: now + Duration::from_secs(1),
};
let captured = proof.token();
let restarted = RemoteVersionStateFleetProof {
topology_fingerprint: proof.topology_fingerprint.clone(),
peer_epochs: Arc::new(BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())])),
expires_at: proof.expires_at,
};
assert!(captured != restarted.token());
}
#[test]
fn remote_version_state_fleet_proof_renewal_preserves_only_same_epoch_token() {
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now();
let epoch = Uuid::new_v4();
let peers = BTreeMap::from([("peer-a".to_string(), epoch)]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers.clone()), now).is_none());
let original = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("successful probe should publish proof")
.token();
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peers), now + Duration::from_millis(1)).is_none()
);
let renewed = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("renewal should retain proof")
.token();
assert!(Arc::ptr_eq(&original.peer_epochs, &renewed.peer_epochs));
let restarted = BTreeMap::from([("peer-a".to_string(), Uuid::new_v4())]);
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Ok(restarted), now + Duration::from_millis(2))
.is_none()
);
let replaced = slot
.read()
.expect("proof slot should not poison")
.proof
.as_ref()
.expect("restarted peer should publish a new proof")
.token();
assert!(!Arc::ptr_eq(&original.peer_epochs, &replaced.peer_epochs));
}
#[test]
fn remote_version_state_fleet_proof_conflict_revokes_atomic_snapshot() {
let now = Instant::now();
let mut state = RemoteVersionStateFleetProofState {
proof: Some(RemoteVersionStateFleetProof {
topology_fingerprint: "topology-a".to_string(),
peer_epochs: Arc::new(BTreeMap::new()),
expires_at: now + Duration::from_secs(1),
}),
topology_conflict: false,
};
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_some());
state.topology_conflict = true;
assert!(acquire_remote_version_state_fleet_proof_from(&state, "topology-a", now).is_none());
}
#[test]
fn remote_version_state_fleet_probe_rejects_duplicate_member_or_process_epoch() {
let epoch = Uuid::new_v4();
let mut peer_epochs = BTreeMap::new();
insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), epoch)
.expect("first member should be admitted");
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-b:9000".to_string(), epoch).is_err());
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-a:9000".to_string(), Uuid::new_v4()).is_err());
assert!(insert_remote_version_state_peer(&mut peer_epochs, "node-c:9000".to_string(), Uuid::nil()).is_err());
}
#[test]
fn remote_version_state_fleet_probe_failure_revokes_previous_proof() {
let slot = std::sync::RwLock::new(RemoteVersionStateFleetProofState::default());
let now = Instant::now();
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
assert!(
publish_remote_version_state_probe_result(&slot, "topology-a", Err(Error::other("peer unavailable")), now,).is_some()
);
assert!(slot.read().expect("proof slot should not poison").proof.is_none());
let peer_epochs = BTreeMap::from([("node-a:9000".to_string(), Uuid::new_v4())]);
assert!(publish_remote_version_state_probe_result(&slot, "topology-a", Ok(peer_epochs), now).is_none());
assert!(slot.read().expect("proof slot should not poison").proof.is_some());
}
#[tokio::test]
async fn remote_version_state_fleet_probe_rejects_unreachable_member() {
let notification_sys = NotificationSys {
peer_clients: vec![None],
all_peer_clients: vec![None, None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: vec![Mutex::new(PeerAdminCache::new())],
tier_config_reload_workers: Default::default(),
};
let err = notification_sys
.probe_remote_version_state_fleet("topology-a")
.await
.expect_err("an unreachable configured member must fail the fleet proof");
assert!(err.to_string().contains("unreachable"));
}
#[tokio::test]
async fn remote_version_state_fleet_probe_rejects_missing_member_slot() {
let notification_sys = NotificationSys {
peer_clients: Vec::new(),
all_peer_clients: vec![None],
peer_topology_hosts: vec!["peer-a".to_string()],
peer_admin_caches: Vec::new(),
tier_config_reload_workers: Default::default(),
};
let err = notification_sys
.probe_remote_version_state_fleet("topology-a")
.await
.expect_err("a missing configured member slot must fail the fleet proof");
assert!(err.to_string().contains("incomplete"));
}
fn build_props(endpoint: &str) -> ServerProperties {
ServerProperties {
endpoint: endpoint.to_string(),
crates/ecstore/src/services/tier/tier.rs
+39
View File
@@ -236,6 +236,7 @@ struct TierPublishTransition {
struct PreparedTierDriver {
tier_name: String,
tier_config: TierConfig,
config_fingerprint: TierDriverFingerprint,
backend_identity: TierDestinationId,
exact_get_delete: bool,
@@ -1342,6 +1343,7 @@ fn tier_exact_get_delete(config: &TierConfig) -> bool {
struct TierDriverGeneration {
tier_name: Arc<str>,
tier_config: TierConfig,
generation: DriverRevision,
// Process-local only: this may reflect credential changes and must never be persisted or logged.
config_fingerprint: TierDriverFingerprint,
@@ -1349,6 +1351,9 @@ struct TierDriverGeneration {
backend_identity: TierDestinationId,
exact_get_delete: bool,
driver: SharedWarmBackend,
reconciler: tokio::sync::OnceCell<
Option<Arc<dyn crate::services::tier::warm_backend::TransitionCandidateReconciler + Send + Sync + 'static>>,
>,
accepting: AtomicBool,
active_leases: AtomicUsize,
drained: tokio::sync::Notify,
@@ -1473,6 +1478,35 @@ impl TierOperationLease {
self.inner.backend_identity
}
pub(crate) async fn probe_transition_candidate_for(
&self,
object: &str,
transaction_id: uuid::Uuid,
) -> io::Result<TransitionCandidateProbe> {
let Some(reconciler) = self
.inner
.reconciler
.get_or_try_init(|| async {
crate::services::tier::warm_backend::new_transition_candidate_reconciler(&self.inner.tier_config)
.await
.map(|reconciler| reconciler.map(Arc::from))
})
.await
.map_err(|err| io::Error::other(err.message))?
else {
return self.inner.driver.probe_transition_candidate(object).await;
};
reconciler
.probe_transition_candidate_for(
object,
crate::services::tier::warm_backend::TransitionCandidateIdentity {
transaction_id,
destination_id: self.backend_identity(),
},
)
.await
}
pub(crate) fn validate_remote_version_id(&self, remote_version_id: &str) -> io::Result<()> {
self.inner.driver.validate_remote_version_id(remote_version_id)?;
if !remote_version_id.is_empty() && !self.inner.exact_get_delete {
@@ -2833,6 +2867,7 @@ impl TierConfigMgr {
let exact_get_delete = tier_exact_get_delete(config);
Some(PreparedTierDriver {
tier_name: tier_name.to_string(),
tier_config: config.clone(),
config_fingerprint,
backend_identity,
exact_get_delete,
@@ -2861,11 +2896,13 @@ impl TierConfigMgr {
})?;
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(prepared.tier_name.as_str()),
tier_config: prepared.tier_config.clone(),
generation,
config_fingerprint: prepared.config_fingerprint,
backend_identity: prepared.backend_identity,
exact_get_delete: prepared.exact_get_delete,
driver: prepared.driver.clone(),
reconciler: tokio::sync::OnceCell::new(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
@@ -3609,11 +3646,13 @@ impl TierConfigMgr {
let driver: SharedWarmBackend = Arc::from(driver);
let entry = Arc::new(TierDriverGeneration {
tier_name: Arc::from(tier_name),
tier_config: config.clone(),
generation,
config_fingerprint,
backend_identity,
exact_get_delete,
driver: driver.clone(),
reconciler: tokio::sync::OnceCell::new(),
accepting: AtomicBool::new(true),
active_leases: AtomicUsize::new(0),
drained: tokio::sync::Notify::new(),
crates/ecstore/src/services/tier/warm_backend.rs
+105
View File
@@ -73,6 +73,21 @@ pub enum TransitionCandidateProbe {
Unsupported,
}
#[derive(Clone, Copy)]
pub(crate) struct TransitionCandidateIdentity {
pub transaction_id: uuid::Uuid,
pub destination_id: [u8; 32],
}
#[async_trait::async_trait]
pub(crate) trait TransitionCandidateReconciler {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error>;
}
#[async_trait::async_trait]
pub trait WarmBackend {
async fn validate(&self) -> Result<(), std::io::Error> {
@@ -189,6 +204,20 @@ pub fn build_transition_put_options(storage_class: String, mut metadata: HashMap
metadata.remove(key);
}
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
] {
for key in [
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix),
format!("{}{}", rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX, suffix),
] {
if let Some(value) = metadata.remove(&key) {
metadata.insert(format!("x-amz-meta-{key}"), value);
}
}
}
opts.user_metadata = metadata;
opts
}
@@ -446,6 +475,51 @@ pub async fn new_warm_backend(tier: &TierConfig, probe: bool) -> Result<WarmBack
Ok(d)
}
pub(crate) async fn new_transition_candidate_reconciler(
tier: &TierConfig,
) -> Result<Option<Box<dyn TransitionCandidateReconciler + Send + Sync + 'static>>, AdminError> {
let reconciler: Box<dyn TransitionCandidateReconciler + Send + Sync + 'static> = match tier.tier_type {
TierType::S3 => Box::new(
WarmBackendS3::new(tier.s3.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::MinIO => Box::new(
WarmBackendMinIO::new(tier.minio.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::RustFS => Box::new(
WarmBackendRustFS::new(tier.rustfs.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
TierType::R2 => Box::new(
WarmBackendR2::new(tier.r2.as_ref().ok_or_else(|| ERR_TIER_INVALID_CONFIG.clone())?, &tier.name)
.await
.map_err(|err| {
let mut admin_err = ERR_TIER_INVALID_CONFIG.clone();
admin_err.message = err.to_string();
admin_err
})?,
),
_ => return Ok(None),
};
Ok(Some(reconciler))
}
#[cfg(test)]
mod tests {
use super::*;
@@ -775,6 +849,37 @@ mod tests {
assert!(!opts.user_metadata.contains_key(X_AMZ_REPLICATION_STATUS.as_str()));
}
#[test]
fn build_transition_put_options_persists_both_candidate_identity_keys_as_s3_metadata() {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
"aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa".to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
"5a".repeat(32),
);
let opts = build_transition_put_options("COLD".to_string(), metadata);
for suffix in [
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
] {
assert!(opts.user_metadata.contains_key(&format!(
"x-amz-meta-{}",
rustfs_utils::http::metadata_compat::internal_key_rustfs(suffix)
)));
assert!(opts.user_metadata.contains_key(&format!(
"x-amz-meta-{}{suffix}",
rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX
)));
}
}
#[test]
fn build_transition_put_options_requests_no_checksum_and_content_md5() {
// Regression for rustfs/rustfs#4811: transition uploads must leave the
crates/ecstore/src/services/tier/warm_backend_gcs.rs
+52 -5
View File
@@ -19,6 +19,7 @@
#![allow(clippy::all)]
use std::collections::HashMap;
use std::io::{Error, ErrorKind};
use std::sync::Arc;
use bytes::Bytes;
@@ -45,6 +46,19 @@ const MAX_PARTS_COUNT: i64 = 10000;
const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5;
const MIN_PART_SIZE: i64 = 1024 * 1024 * 128;
fn parse_generation(remote_version: &str) -> Result<Option<i64>, Error> {
if remote_version.is_empty() {
return Ok(None);
}
let generation = remote_version
.parse::<i64>()
.map_err(|_| Error::new(ErrorKind::InvalidData, "GCS remote version is not a valid generation"))?;
if generation <= 0 {
return Err(Error::new(ErrorKind::InvalidData, "GCS remote version generation must be positive"));
}
Ok(Some(generation))
}
pub struct WarmBackendGCS {
pub client: Arc<Storage>,
pub control: Arc<StorageControl>,
@@ -105,6 +119,10 @@ impl WarmBackendGCS {
#[async_trait::async_trait]
impl WarmBackend for WarmBackendGCS {
fn validate_remote_version_id(&self, remote_version_id: &str) -> Result<(), std::io::Error> {
parse_generation(remote_version_id).map(|_| ())
}
async fn put_with_meta(
&self,
object: &str,
@@ -135,6 +153,9 @@ impl WarmBackend for WarmBackendGCS {
async fn get(&self, object: &str, rv: &str, opts: WarmBackendGetOpts) -> Result<ReadCloser, std::io::Error> {
let mut req = self.client.read_object(&self.bucket, &self.get_dest(object));
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
}
// Honor the requested byte range so Range GETs on tiered objects return the exact
// interval instead of the whole object (matches the s3/s3sdk/rustfs warm backends).
@@ -164,13 +185,15 @@ impl WarmBackend for WarmBackendGCS {
// gRPC v2 DeleteObject requires the bucket in resource-name form. Without this the
// deleted tiered object was never removed from GCS (empty impl returned Ok), leaking
// remote data forever.
self.control
let mut req = self
.control
.delete_object()
.set_bucket(format!("projects/_/buckets/{}", self.bucket))
.set_object(self.get_dest(object))
.send()
.await
.map_err(|e| std::io::Error::other(e.to_string()))?;
.set_object(self.get_dest(object));
if let Some(generation) = parse_generation(rv)? {
req = req.set_generation(generation);
}
req.send().await.map_err(|e| std::io::Error::other(e.to_string()))?;
Ok(())
}
@@ -191,6 +214,30 @@ impl WarmBackend for WarmBackendGCS {
}
}
#[cfg(test)]
mod tests {
use super::parse_generation;
use std::io::ErrorKind;
#[test]
fn generation_parser_preserves_exact_numeric_versions() {
assert_eq!(parse_generation("").expect("empty generation means no version condition"), None);
assert_eq!(parse_generation("1").expect("minimum generation should parse"), Some(1));
assert_eq!(
parse_generation(&i64::MAX.to_string()).expect("maximum generation should parse"),
Some(i64::MAX)
);
}
#[test]
fn generation_parser_rejects_unknown_or_non_positive_versions() {
for value in ["unknown", "1.0", "-1", "0", "9223372036854775808"] {
let err = parse_generation(value).expect_err("unknown generation must fail closed");
assert_eq!(err.kind(), ErrorKind::InvalidData, "{value}");
}
}
}
/*fn gcs_to_object_error(err: Error, params: Vec<String>) -> Option<Error> {
if err == nil {
return nil
crates/ecstore/src/services/tier/warm_backend_minio.rs
+14
View File
@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendMinIO {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendMinIO {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
crates/ecstore/src/services/tier/warm_backend_r2.rs
+14
View File
@@ -135,6 +135,20 @@ impl WarmBackend for WarmBackendR2 {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendR2 {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
crates/ecstore/src/services/tier/warm_backend_rustfs.rs
+14
View File
@@ -132,6 +132,20 @@ impl WarmBackend for WarmBackendRustFS {
}
}
#[async_trait::async_trait]
impl crate::services::tier::warm_backend::TransitionCandidateReconciler for WarmBackendRustFS {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: crate::services::tier::warm_backend::TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
crate::services::tier::warm_backend::TransitionCandidateReconciler::probe_transition_candidate_for(
&self.0, object, identity,
)
.await
}
}
fn optimal_part_size(object_size: i64) -> Result<i64, std::io::Error> {
let mut object_size = object_size;
if object_size == -1 {
crates/ecstore/src/services/tier/warm_backend_s3.rs
+316 -71
View File
@@ -29,6 +29,7 @@ use crate::client::{
api_remove::{RemoveObjectOptions, RemoveObjectResult},
api_s3_datatypes::ListVersionsResult,
credentials::{Credentials, SignatureType, Static, Value},
provider_versions::validate_remote_version_id,
transition_api::{BucketLookupType, Options, TransitionClient, TransitionCore},
transition_api::{ReadCloser, ReaderImpl},
};
@@ -36,7 +37,10 @@ use crate::error::ErrorResponse;
use crate::error::error_resp_to_object_err;
use crate::services::tier::{
tier_config::TierS3,
warm_backend::{TransitionCandidateProbe, WarmBackend, WarmBackendGetOpts, build_transition_put_options},
warm_backend::{
TransitionCandidateIdentity, TransitionCandidateProbe, TransitionCandidateReconciler, WarmBackend, WarmBackendGetOpts,
build_transition_put_options,
},
};
use http::HeaderMap;
use rustfs_utils::egress::validate_outbound_url;
@@ -189,44 +193,188 @@ impl WarmBackendS3 {
remote_bucket_versioning_from_status(config.status.as_ref().map(|status| status.as_str()))
}
async fn list_transition_candidate_versions(&self, object: &str) -> Result<ListVersionsResult, std::io::Error> {
async fn probe_transition_candidate_versions(
&self,
object: &str,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
let mut opts = ListObjectsOptions::default();
opts.set("prefix", &self.get_dest(object));
opts.set("max-keys", "2");
self.client.list_object_versions_query(&self.bucket, &opts, "", "", "").await
opts.set("prefix", &remote_object);
opts.set("max-keys", "1000");
let mut key_marker = String::new();
let mut version_id_marker = String::new();
let mut candidates = TransitionCandidateVersions::default();
loop {
let versions = self
.client
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
.await?;
candidates.extend(&remote_object, &versions);
if candidates.is_ambiguous() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
if !versions.is_truncated {
return classify_transition_candidates(candidates, bucket_versioning);
}
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
}
async fn probe_transition_candidate_identity(
&self,
object: &str,
identity: TransitionCandidateIdentity,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let remote_object = self.get_dest(object);
let mut opts = ListObjectsOptions::default();
opts.set("prefix", &remote_object);
opts.set("max-keys", "1000");
let mut key_marker = String::new();
let mut version_id_marker = String::new();
let mut matched_version = None;
let mut saw_unproven_candidate = false;
loop {
let versions = self
.client
.list_object_versions_query(&self.bucket, &opts, &key_marker, &version_id_marker, "")
.await?;
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
let mut stat_opts = GetObjectOptions::default();
stat_opts.version_id.clone_from(&version.version_id);
let info = self.client.stat_object(&self.bucket, &remote_object, &stat_opts).await?;
let mut metadata = info.user_metadata;
for (name, value) in &info.metadata {
if (name
.as_str()
.starts_with(rustfs_utils::http::metadata_compat::RUSTFS_INTERNAL_PREFIX)
|| name
.as_str()
.starts_with(rustfs_utils::http::metadata_compat::MINIO_INTERNAL_PREFIX))
&& let Ok(value) = value.to_str()
{
metadata.insert(name.as_str().to_string(), value.to_string());
}
}
if transition_candidate_metadata_matches(&metadata, identity)? {
if matched_version.is_some() {
return Ok(TransitionCandidateProbe::Ambiguous);
}
matched_version = Some(version.version_id.clone());
} else {
saw_unproven_candidate = true;
}
}
if !versions.is_truncated {
if matched_version.is_none() && saw_unproven_candidate {
return Ok(TransitionCandidateProbe::Unsupported);
}
let candidates = TransitionCandidateVersions {
version_id: matched_version,
ambiguous: false,
};
return classify_transition_candidates(candidates, bucket_versioning);
}
advance_version_markers(&mut key_marker, &mut version_id_marker, &versions)?;
}
}
}
fn classify_transition_candidate_versions(
remote_object: &str,
bucket_versioning: RemoteBucketVersioning,
versions: &ListVersionsResult,
) -> TransitionCandidateProbe {
if versions.is_truncated {
return TransitionCandidateProbe::Ambiguous;
}
if versions.delete_markers.iter().any(|marker| marker.key == remote_object) {
return TransitionCandidateProbe::Ambiguous;
}
let mut exact_versions = versions.versions.iter().filter(|version| version.key == remote_object);
let Some(version) = exact_versions.next() else {
return TransitionCandidateProbe::Missing;
fn transition_candidate_metadata_matches(
metadata: &HashMap<String, String>,
identity: TransitionCandidateIdentity,
) -> Result<bool, std::io::Error> {
use rustfs_utils::http::metadata_compat::{
SUFFIX_TRANSITION_TIER_DESTINATION_ID, SUFFIX_TRANSITION_TRANSACTION_ID, contains_key_str, get_consistent_str,
};
if exact_versions.next().is_some() {
return TransitionCandidateProbe::Ambiguous;
let transaction_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID);
let destination_id = get_consistent_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID);
if transaction_id.is_none() || destination_id.is_none() {
if contains_key_str(metadata, SUFFIX_TRANSITION_TRANSACTION_ID)
|| contains_key_str(metadata, SUFFIX_TRANSITION_TIER_DESTINATION_ID)
{
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"transition candidate identity metadata is empty or conflicting",
));
}
return Ok(false);
}
let expected_transaction_id = identity.transaction_id.to_string();
let expected_destination_id = rustfs_utils::crypto::hex(identity.destination_id);
Ok(transaction_id == Some(expected_transaction_id.as_str()) && destination_id == Some(expected_destination_id.as_str()))
}
fn classify_transition_candidates(
candidates: TransitionCandidateVersions,
bucket_versioning: RemoteBucketVersioning,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let probe = candidates.classify(bucket_versioning);
if let TransitionCandidateProbe::VersionedPresent(version_id) = &probe {
validate_remote_version_id(version_id)?;
}
Ok(probe)
}
fn advance_version_markers(
key_marker: &mut String,
version_id_marker: &mut String,
versions: &ListVersionsResult,
) -> Result<(), std::io::Error> {
let next_markers = (&versions.next_key_marker, &versions.next_version_id_marker);
if next_markers == (&*key_marker, &*version_id_marker) {
return Err(std::io::Error::new(
std::io::ErrorKind::InvalidData,
"ListObjectVersions pagination markers did not advance",
));
}
key_marker.clone_from(&versions.next_key_marker);
version_id_marker.clone_from(&versions.next_version_id_marker);
Ok(())
}
#[derive(Default)]
struct TransitionCandidateVersions {
version_id: Option<String>,
ambiguous: bool,
}
impl TransitionCandidateVersions {
fn extend(&mut self, remote_object: &str, versions: &ListVersionsResult) {
for version in versions.versions.iter().filter(|version| version.key == remote_object) {
if self.version_id.is_some() {
self.ambiguous = true;
return;
}
self.version_id = Some(version.version_id.clone());
}
}
match bucket_versioning {
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
RemoteBucketVersioning::Suspended if version.version_id == "null" => {
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
fn is_ambiguous(&self) -> bool {
self.ambiguous
}
fn classify(self, bucket_versioning: RemoteBucketVersioning) -> TransitionCandidateProbe {
if self.ambiguous {
return TransitionCandidateProbe::Ambiguous;
}
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version.version_id.is_empty() => {
TransitionCandidateProbe::VersionedPresent(version.version_id.clone())
let Some(version_id) = self.version_id else {
return TransitionCandidateProbe::Missing;
};
match bucket_versioning {
RemoteBucketVersioning::Disabled => TransitionCandidateProbe::UnversionedPresent,
RemoteBucketVersioning::Suspended if version_id == "null" => TransitionCandidateProbe::VersionedPresent(version_id),
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled if !version_id.is_empty() => {
TransitionCandidateProbe::VersionedPresent(version_id)
}
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
}
RemoteBucketVersioning::Suspended | RemoteBucketVersioning::Enabled => TransitionCandidateProbe::Ambiguous,
}
}
@@ -275,74 +423,163 @@ mod tests {
}
}
fn classify_pages(bucket_versioning: RemoteBucketVersioning, pages: &[ListVersionsResult]) -> TransitionCandidateProbe {
let mut candidates = TransitionCandidateVersions::default();
for page in pages {
candidates.extend("archive/object", page);
}
candidates.classify(bucket_versioning)
}
fn candidate_identity() -> TransitionCandidateIdentity {
TransitionCandidateIdentity {
transaction_id: uuid::Uuid::parse_str("aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa").unwrap(),
destination_id: [0x5a; 32],
}
}
fn candidate_metadata(identity: TransitionCandidateIdentity) -> HashMap<String, String> {
let mut metadata = HashMap::new();
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
identity.transaction_id.to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut metadata,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(identity.destination_id),
);
metadata
}
#[test]
fn transition_candidate_identity_requires_exact_compatible_metadata() {
let identity = candidate_identity();
let metadata = candidate_metadata(identity);
assert!(transition_candidate_metadata_matches(&metadata, identity).unwrap());
let mut adjacent = metadata;
rustfs_utils::http::metadata_compat::insert_str(
&mut adjacent,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
uuid::Uuid::parse_str("bbbbbbbb-bbbb-4bbb-8bbb-bbbbbbbbbbbb")
.unwrap()
.to_string(),
);
assert!(!transition_candidate_metadata_matches(&adjacent, identity).unwrap());
assert!(!transition_candidate_metadata_matches(&HashMap::new(), identity).unwrap());
}
#[test]
fn transition_candidate_identity_rejects_conflicting_compatibility_keys() {
let identity = candidate_identity();
let mut metadata = candidate_metadata(identity);
metadata.insert(
rustfs_utils::http::metadata_compat::internal_key_rustfs(
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
),
uuid::Uuid::new_v4().to_string(),
);
assert!(transition_candidate_metadata_matches(&metadata, identity).is_err());
}
#[test]
fn transition_candidate_probe_classifier_is_fail_closed() {
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Disabled,
&list_versions(&[], &[], false),
),
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[], &[], false)],),
TransitionCandidateProbe::Missing
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Disabled,
&list_versions(&[("archive/object", "")], &[], false),
),
classify_pages(RemoteBucketVersioning::Disabled, &[list_versions(&[("archive/object", "")], &[], false)],),
TransitionCandidateProbe::UnversionedPresent
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
classify_pages(
RemoteBucketVersioning::Enabled,
&list_versions(&[("archive/object", "version-a")], &[], false),
&[list_versions(&[("archive/object", "version-a")], &[], false)],
),
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
classify_pages(
RemoteBucketVersioning::Suspended,
&list_versions(&[("archive/object", "null")], &[], false),
&[list_versions(&[("archive/object", "null")], &[], false)],
),
TransitionCandidateProbe::VersionedPresent("null".to_string())
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Enabled,
&list_versions(&[("archive/object", "")], &[], false),
),
classify_pages(RemoteBucketVersioning::Enabled, &[list_versions(&[("archive/object", "")], &[], false)],),
TransitionCandidateProbe::Ambiguous
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
classify_pages(
RemoteBucketVersioning::Enabled,
&list_versions(&[("archive/object", "version-a"), ("archive/object", "version-b")], &[], false),
&[list_versions(
&[("archive/object", "version-a"), ("archive/object", "version-b")],
&[],
false,
)],
),
TransitionCandidateProbe::Ambiguous
);
}
#[test]
fn transition_candidate_probe_reconciles_all_pages_and_ignores_delete_markers() {
assert_eq!(
classify_pages(
RemoteBucketVersioning::Enabled,
&[
list_versions(&[], &[("archive/object", "marker-a")], true),
list_versions(&[("archive/object", "version-a"), ("archive/object-adjacent", "unrelated"),], &[], false,),
],
),
TransitionCandidateProbe::VersionedPresent("version-a".to_string())
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
classify_pages(
RemoteBucketVersioning::Enabled,
&list_versions(&[("archive/object", "version-a")], &[("archive/object", "marker-a")], false),
),
TransitionCandidateProbe::Ambiguous
);
assert_eq!(
classify_transition_candidate_versions(
"archive/object",
RemoteBucketVersioning::Enabled,
&list_versions(&[("archive/object", "version-a")], &[], true),
&[
list_versions(&[("archive/object", "version-a")], &[], true),
list_versions(&[("archive/object", "version-b")], &[], false),
],
),
TransitionCandidateProbe::Ambiguous
);
}
#[test]
fn transition_candidate_pagination_advances_both_markers() {
let mut key_marker = "old-key".to_string();
let mut version_id_marker = "old-version".to_string();
let page = ListVersionsResult {
next_key_marker: "next-key".to_string(),
next_version_id_marker: "next-version".to_string(),
..Default::default()
};
advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
.expect("new ListObjectVersions markers should advance pagination");
assert_eq!(key_marker, "next-key");
assert_eq!(version_id_marker, "next-version");
let err = advance_version_markers(&mut key_marker, &mut version_id_marker, &page)
.expect_err("repeated ListObjectVersions markers must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
#[test]
fn transition_candidate_probe_rejects_untrusted_version_ids() {
let mut candidates = TransitionCandidateVersions::default();
candidates.extend("archive/object", &list_versions(&[("archive/object", "version\ninjection")], &[], false));
let err = classify_transition_candidates(candidates, RemoteBucketVersioning::Enabled)
.expect_err("control characters in listed version IDs must fail closed");
assert_eq!(err.kind(), std::io::ErrorKind::InvalidData);
}
#[test]
fn remote_bucket_versioning_status_parser_fails_closed() {
assert_eq!(
@@ -397,12 +634,7 @@ impl WarmBackend for WarmBackendS3 {
async fn probe_transition_candidate(&self, object: &str) -> Result<TransitionCandidateProbe, std::io::Error> {
let bucket_versioning = self.remote_bucket_versioning().await?;
let versions = self.list_transition_candidate_versions(object).await?;
Ok(classify_transition_candidate_versions(
&self.get_dest(object),
bucket_versioning,
&versions,
))
self.probe_transition_candidate_versions(object, bucket_versioning).await
}
async fn in_use(&self) -> Result<bool, std::io::Error> {
@@ -414,3 +646,16 @@ impl WarmBackend for WarmBackendS3 {
Ok(result.common_prefixes.len() > 0 || result.contents.len() > 0)
}
}
#[async_trait::async_trait]
impl TransitionCandidateReconciler for WarmBackendS3 {
async fn probe_transition_candidate_for(
&self,
object: &str,
identity: TransitionCandidateIdentity,
) -> Result<TransitionCandidateProbe, std::io::Error> {
let bucket_versioning = self.remote_bucket_versioning().await?;
self.probe_transition_candidate_identity(object, identity, bucket_versioning)
.await
}
}
crates/ecstore/src/set_disk/core/io_primitives.rs
+24 -13
View File
@@ -46,6 +46,7 @@ use crate::diagnostics::get::{
GetObjectFailureReason, classify_disk_error, get_stage_timer_if_enabled, record_get_object_pipeline_failure,
record_get_object_pipeline_failure_for_path, record_get_stage_duration_if_enabled,
};
use crate::disk::local::DELETE_DATA_DIR_MARKER_PREFIX;
use crate::disk::{
DataDirDeleteStatus, OldCurrentSize, PART_TRANSACTION_NEW_META, PART_TRANSACTION_OLD_META, PART_TRANSACTION_ROLLBACK,
PartTransactionAction, part_transaction_path,
@@ -3952,9 +3953,9 @@ impl SetDisks {
/// * The set of referenced data dirs is the UNION of `get_data_dirs()` across
/// every online disk's `xl.meta`, so a dir named by *any* replica is kept.
/// * If a disk holds the object directory but its `xl.meta` is missing or
/// unparsable, the object is treated as degraded and NOTHING is removed —
/// the unreadable copy could be the only one naming a live data dir, and a
/// heal must run first.
/// unparsable, the object is treated as degraded and unmarked data dirs are
/// never removed. A data dir carrying a committed delete-transaction marker
/// remains reclaimable after a downgrade/re-upgrade cleanup interruption.
/// * Only subdirectories whose names parse as a UUID are ever considered;
/// removal is non-recursive-safe via a recursive delete of the full stray
/// data-dir path only.
@@ -3969,7 +3970,7 @@ impl SetDisks {
// physical UUID subdirectories present on each disk. Abort on any degraded
// copy so a healable object is never stripped of a referenced data dir.
let mut referenced: HashSet<Uuid> = HashSet::new();
let mut per_disk_dirs: Vec<(usize, Vec<Uuid>)> = Vec::new();
let mut per_disk_dirs: Vec<(usize, Vec<(Uuid, bool)>)> = Vec::new();
let mut healthy_metas = 0usize;
for (i, disk) in disks.iter().enumerate() {
@@ -4005,6 +4006,22 @@ impl SetDisks {
// to the orphan-dir / dangling-object heal paths.
continue;
}
let mut committed = Vec::with_capacity(physical.len());
for dir in physical {
let data_dir = format!("{object}/{dir}");
let committed_delete = disk.list_dir("", bucket, &data_dir, 0).await.is_ok_and(|entries| {
entries.iter().any(|entry| {
entry
.strip_prefix(DELETE_DATA_DIR_MARKER_PREFIX)
.is_some_and(|transaction| Uuid::parse_str(transaction).is_ok())
})
});
committed.push((dir, committed_delete));
}
if committed.iter().all(|(_, committed_delete)| *committed_delete) {
per_disk_dirs.push((i, committed));
continue;
}
warn!(
target: "rustfs_ecstore::set_disk",
bucket, object,
@@ -4041,22 +4058,16 @@ impl SetDisks {
healthy_metas += 1;
if !physical.is_empty() {
per_disk_dirs.push((i, physical));
per_disk_dirs.push((i, physical.into_iter().map(|dir| (dir, false)).collect()));
}
}
// No healthy metadata anywhere: this is not a live object, so surplus dirs
// (if any) belong to the dangling-object heal path, not here.
if healthy_metas == 0 {
return Ok(0);
}
// Phase 2: delete every physical data dir not referenced by the union.
let mut removed = 0usize;
for (i, physical) in per_disk_dirs {
let Some(disk) = disks[i].as_ref() else { continue };
for dir in physical {
if referenced.contains(&dir) {
for (dir, committed_delete) in physical {
if referenced.contains(&dir) || (healthy_metas == 0 && !committed_delete) {
continue;
}
let stray = format!("{object}/{dir}");
crates/ecstore/src/set_disk/mod.rs
+933 -6
View File
File diff suppressed because it is too large Load Diff
crates/ecstore/src/set_disk/ops/multipart.rs
+88 -75
View File
@@ -3379,85 +3379,98 @@ mod tests {
#[tokio::test(flavor = "multi_thread")]
#[serial]
async fn complete_holds_object_then_upload_lock_through_commit() {
temp_env::async_with_vars([(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true"))], async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-layout-lock-order-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let create_opts = ObjectOptions {
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
..Default::default()
};
let (upload_id, parts) = stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
signaling.set_target(upload_resource.clone());
signaling.clear_observed();
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
temp_env::async_with_vars(
[
(crate::object_api::ENV_RUSTFS_ENCRYPTED_RANGE_SEEK, Some("true")),
(rustfs_config::ENV_OBJECT_LOCK_ACQUIRE_TIMEOUT, Some("60")),
],
async {
let manager = Arc::new(rustfs_lock::GlobalLockManager::new());
let signaling = Arc::new(SignalingLockClient::new(Arc::new(LocalClient::with_manager(manager))));
let lockers: Vec<Arc<dyn LockClient>> = vec![signaling.clone()];
let (_temp_dirs, disk_stores, set_disks) = hermetic_set_disks_with_lockers(4, 0, 2, lockers).await;
let bucket = "multipart-layout-lock-order-bucket";
let object = "object";
make_bucket_on_all(&disk_stores, bucket).await;
let create_opts = ObjectOptions {
user_defined: HashMap::from([(SSEC_ALGORITHM_HEADER.to_string(), "AES256".to_string())]),
..Default::default()
};
let (upload_id, parts) =
stage_upload_with_create_opts(&set_disks, bucket, object, &[0x43; 4096], &create_opts).await;
let upload_id_path = SetDisks::get_upload_id_dir(bucket, object, &upload_id);
let upload_resource = rustfs_lock::ObjectKey::new(RUSTFS_META_MULTIPART_BUCKET, upload_id_path);
let object_resource = rustfs_lock::ObjectKey::new(bucket, object);
signaling.set_target(upload_resource.clone());
signaling.clear_observed();
let _setup_type_guard = SetupTypeGuard::switch_to(SetupType::DistErasure).await;
let barrier = MultipartCommitBarrier::install(bucket, object, MultipartCommitPause::AfterRename);
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
let complete_store = set_disks.clone();
let complete_upload_id = upload_id.clone();
let complete = tokio::spawn(async move {
complete_store
.complete_multipart_upload(bucket, object, &complete_upload_id, parts, &ObjectOptions::default())
.await
});
barrier.wait_until_paused().await;
let observed = signaling.observed();
let object_position = observed
.iter()
.position(|resource| resource == &object_resource)
.expect("completion should acquire the object lock");
let upload_position = observed
.iter()
.position(|resource| resource == &upload_resource)
.expect("completion should acquire the upload lock");
assert!(object_position < upload_position, "completion must acquire object before upload");
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(2).await;
tokio::task::yield_now().await;
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
let list_store = set_disks.clone();
let list_upload_id = upload_id.clone();
let list = tokio::spawn(async move {
list_store
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.await
});
signaling.wait_for_attempts(3).await;
tokio::task::yield_now().await;
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
barrier.release();
complete
.await
});
barrier.wait_until_paused().await;
let observed = signaling.observed();
let object_position = observed
.iter()
.position(|resource| resource == &object_resource)
.expect("completion should acquire the object lock");
let upload_position = observed
.iter()
.position(|resource| resource == &upload_resource)
.expect("completion should acquire the upload lock");
assert!(object_position < upload_position, "completion must acquire object before upload");
let abort_store = set_disks.clone();
let abort_upload_id = upload_id.clone();
let abort = tokio::spawn(async move {
abort_store
.abort_multipart_upload(bucket, object, &abort_upload_id, &ObjectOptions::default())
.expect("completion task should not panic")
.expect("completion should commit after the barrier is released");
let abort_err = abort
.await
});
signaling.wait_for_attempts(2).await;
tokio::task::yield_now().await;
assert!(!abort.is_finished(), "abort must wait until completion releases the upload lock");
let list_store = set_disks.clone();
let list_upload_id = upload_id.clone();
let list = tokio::spawn(async move {
list_store
.list_object_parts(bucket, object, &list_upload_id, None, MAX_PARTS_COUNT, &ObjectOptions::default())
.expect("abort task should not panic")
.expect_err("the committed upload should no longer exist when abort acquires the lock");
assert!(
matches!(abort_err, StorageError::InvalidUploadID(..)),
"abort should return InvalidUploadID after completion, got {abort_err:?}"
);
let list_err = list
.await
});
signaling.wait_for_attempts(3).await;
tokio::task::yield_now().await;
assert!(!list.is_finished(), "ListParts must wait until completion releases the upload lock");
barrier.release();
complete
.await
.expect("completion task should not panic")
.expect("completion should commit after the barrier is released");
let abort_err = abort
.await
.expect("abort task should not panic")
.expect_err("the committed upload should no longer exist when abort acquires the lock");
assert!(matches!(abort_err, StorageError::InvalidUploadID(..)));
let list_err = list
.await
.expect("ListParts task should not panic")
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
assert!(matches!(list_err, StorageError::InvalidUploadID(..)));
})
.expect("ListParts task should not panic")
.expect_err("the committed upload should no longer exist when ListParts acquires the lock");
assert!(
matches!(list_err, StorageError::InvalidUploadID(..)),
"ListParts should return InvalidUploadID after completion, got {list_err:?}"
);
},
)
.await;
}
crates/ecstore/src/set_disk/ops/object.rs
+125 -43
View File
@@ -562,13 +562,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
&self.ctx.tier_config_mgr(),
)
.await?;
return Ok(finish_set_disk_read_lock(
gr,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
return Ok(finish_set_disk_read_lock(gr, read_lock_guard.take(), None, bucket, object));
}
// App-layer object data cache probe: metadata (etag/size) is resolved
@@ -695,6 +689,18 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
return Ok(reader);
}
let snapshot_lease = if lock_optimization_enabled {
match fi.data_dir.filter(|data_dir| !data_dir.is_nil()) {
Some(data_dir) => {
let data_dir_path = format!("{object}/{data_dir}");
acquire_snapshot_leases(&disks, bucket, &data_dir_path, fi.erasure.data_blocks).await
}
None => None,
}
} else {
None
};
match codec_streaming_gate.decision {
GetCodecStreamingDecision::Use => {
match Self::get_object_decode_reader_with_fileinfo(
@@ -724,13 +730,7 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
// Carry the hook probe result so the app layer skips its
// now-redundant lookup on the streaming miss path (ODC-16).
reader.body_source = body_source;
return Ok(finish_set_disk_read_lock(
reader,
read_lock_guard.take(),
lock_optimization_enabled,
bucket,
object,
));
return Ok(finish_set_disk_read_lock(reader, read_lock_guard.take(), snapshot_lease, bucket, object));
}
core::io_primitives::GetCodecStreamingReaderBuildOutcome::Fallback(reason) => {
record_get_codec_streaming_gate_decision(
@@ -770,15 +770,22 @@ impl crate::storage_api_contracts::object::ObjectIO for SetDisks {
let set_index = self.set_index;
let pool_index = self.pool_index;
let skip_verify = opts.skip_verify_bitrot;
if lock_optimization_enabled {
let producer_snapshot_lease = snapshot_lease.clone();
if let Some(lease) = snapshot_lease {
release_materialized_read_lock(&bucket, &object, read_lock_guard.take());
debug!(bucket, object, "Lock optimization: released read lock before streaming read");
reader.stream = Box::new(SnapshotLeaseReader {
inner: reader.stream,
lease: Some(lease),
terminal_error: false,
});
debug!(bucket, object, "Lock optimization: replaced read lock with snapshot leases");
}
// When lock optimization is disabled, keep the read-lock guard in the
// task so it lives for the duration of the streaming read.
// The producer shares the lease lifetime with the body so cancellation
// cannot release the snapshot while the duplex task is still unwinding.
tokio::spawn(async move {
let _guard = read_lock_guard;
let _snapshot_lease = producer_snapshot_lease;
let mut writer = GetObjectDownstreamWriter::new(wd);
// Do not wrap the entire read+write pipeline in `disk_read_timeout`.
// `get_object_with_fileinfo` also waits on `writer`, so an outer timeout
@@ -2327,13 +2334,25 @@ async fn pause_transition_commit(bucket: &str, object: &str, pause: TransitionCo
}
}
#[cfg(test)]
fn persisted_transition_version(
remote_version: &str,
) -> std::io::Result<(Option<String>, rustfs_filemeta::TransitionVersionState)> {
persisted_transition_version_with_gate(remote_version, remote_version_state_writer_enabled())
}
#[cfg(test)]
fn remote_version_state_writer_enabled() -> bool {
remote_version_state_writer_fleet_proof().is_some()
}
fn remote_version_state_writer_fleet_proof() -> Option<crate::services::notification_sys::RemoteVersionStateFleetProofToken> {
remote_version_state_writer_requested()
.then(crate::services::notification_sys::acquire_remote_version_state_fleet_proof)
.flatten()
}
fn remote_version_state_writer_requested() -> bool {
remote_version_state_writer_enabled_for(
rustfs_utils::get_env_bool(
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_WRITE,
@@ -2343,11 +2362,25 @@ fn remote_version_state_writer_enabled() -> bool {
rustfs_config::ENV_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
rustfs_config::DEFAULT_TIER_REMOTE_VERSION_STATE_FLEET_CONFIRMED,
),
true,
)
}
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool) -> bool {
requested && fleet_confirmed
fn remote_version_state_writer_fleet_proof_matches(
proof: &crate::services::notification_sys::RemoteVersionStateFleetProofToken,
) -> bool {
remote_version_state_writer_fleet_proof_matches_for(
remote_version_state_writer_requested(),
crate::services::notification_sys::remote_version_state_fleet_proof_matches(proof),
)
}
fn remote_version_state_writer_fleet_proof_matches_for(requested: bool, fleet_proof_matches: bool) -> bool {
requested && fleet_proof_matches
}
fn remote_version_state_writer_enabled_for(requested: bool, fleet_confirmed: bool, fleet_proof_valid: bool) -> bool {
requested && fleet_confirmed && fleet_proof_valid
}
fn persisted_transition_version_with_gate(
@@ -2366,7 +2399,7 @@ fn persisted_transition_version_with_gate(
Ok(_) => Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::Exact)),
Err(_) if !remote_version_state_writer_enabled => Err(std::io::Error::new(
std::io::ErrorKind::Unsupported,
"opaque remote tier versions require the operator-attested fleet gate",
"opaque remote tier versions require the operator-attested live fleet capability gate",
)),
Err(_) if remote_version == "null" => {
Ok((Some(remote_version.to_string()), rustfs_filemeta::TransitionVersionState::SuspendedNull))
@@ -2612,7 +2645,7 @@ mod transition_upload_completion_tests {
mod transition_version_id_tests {
use super::{
TransitionUploadCandidate, persisted_transition_version, persisted_transition_version_with_gate,
remote_version_state_writer_enabled_for,
remote_version_state_writer_enabled_for, remote_version_state_writer_fleet_proof_matches_for,
};
use rustfs_filemeta::TransitionVersionState;
use uuid::Uuid;
@@ -2655,15 +2688,35 @@ mod transition_version_id_tests {
#[test]
fn remote_version_state_writer_requires_request_and_fleet_confirmation() {
for (case, requested, fleet_confirmed, expected) in [
("old defaults", false, false, false),
("missing fleet confirmation", true, false, false),
("missing local opt-in", false, true, false),
("explicitly unconfirmed fleet", true, false, false),
("rolled-back writer", false, true, false),
("fully upgraded fleet", true, true, true),
for (case, requested, fleet_confirmed, fleet_proof_valid, expected) in [
("old defaults", false, false, false, false),
("missing fleet confirmation", true, false, true, false),
("missing local opt-in", false, true, true, false),
("missing fleet proof", true, true, false, false),
("explicitly unconfirmed fleet", true, false, true, false),
("rolled-back writer", false, true, true, false),
("fully upgraded fleet", true, true, true, true),
] {
assert_eq!(remote_version_state_writer_enabled_for(requested, fleet_confirmed), expected, "{case}");
assert_eq!(
remote_version_state_writer_enabled_for(requested, fleet_confirmed, fleet_proof_valid),
expected,
"{case}"
);
}
}
#[test]
fn remote_version_state_commit_rechecks_operator_gate_and_live_proof() {
for (case, requested, fleet_proof_matches, expected) in [
("operator gate closed", false, true, false),
("fleet proof changed", true, false, false),
("current authorization", true, true, true),
] {
assert_eq!(
remote_version_state_writer_fleet_proof_matches_for(requested, fleet_proof_matches),
expected,
"{case}"
);
}
}
@@ -3848,6 +3901,16 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
let dest_obj = transaction.remote_object.clone();
let mut transition_meta = (*oi.user_defined).clone();
transition_meta.insert("name".to_string(), object.to_string());
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TRANSACTION_ID,
transaction.transaction_id.to_string(),
);
rustfs_utils::http::metadata_compat::insert_str(
&mut transition_meta,
rustfs_utils::http::metadata_compat::SUFFIX_TRANSITION_TIER_DESTINATION_ID,
rustfs_utils::crypto::hex(transaction.backend_fingerprint),
);
if let Some(content_type) = oi.content_type.as_ref().filter(|value| !value.is_empty()) {
transition_meta.insert(CONTENT_TYPE.to_ascii_lowercase(), content_type.clone());
@@ -3958,20 +4021,24 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object).await;
return Err(err.into());
}
let (transition_version_id, transition_version_state) = match persisted_transition_version(candidate.remote_version()) {
Ok(version) => version,
Err(err) => {
let cleanup_api = transition_cleanup_store(&self.ctx).await;
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
return Err(StorageError::Io(std::io::Error::other(format!(
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
))));
let fleet_proof = remote_version_state_writer_fleet_proof();
let remote_version_requires_fleet_proof =
!candidate.remote_version().is_empty() && Uuid::parse_str(candidate.remote_version()).is_err();
let (transition_version_id, transition_version_state) =
match persisted_transition_version_with_gate(candidate.remote_version(), fleet_proof.is_some()) {
Ok(version) => version,
Err(err) => {
let cleanup_api = transition_cleanup_store(&self.ctx).await;
if let Err(cleanup_err) = upload_cleanup.cleanup_rejected_upload(cleanup_api).await {
return Err(StorageError::Io(std::io::Error::other(format!(
"{err}; rejected remote upload cleanup failed: {cleanup_err}"
))));
}
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
return Err(err.into());
}
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
return Err(err.into());
}
};
};
if let Err(err) = advance_and_save_transition_transaction(
transaction_api.as_ref(),
&mut transaction,
@@ -4087,6 +4154,21 @@ impl crate::storage_api_contracts::object::ObjectOperations for SetDisks {
}
#[cfg(test)]
pause_transition_commit(bucket, object, TransitionCommitPause::AfterLeaseValidation).await;
// This check is the fleet-proof lease linearization point. Revocation
// blocks later commits; an already-authorized local quorum commit is
// allowed to finish without holding a synchronous lock across I/O.
if remote_version_requires_fleet_proof
&& !fleet_proof
.as_ref()
.is_some_and(remote_version_state_writer_fleet_proof_matches)
{
drop(transition_lock_guard);
if upload_cleanup.cleanup().await.is_ok() {
delete_transition_transaction_after_remote_cleanup(transaction_api.as_ref(), transaction_id, bucket, object)
.await;
}
return Err(Error::other("remote version state fleet capability changed during transition"));
}
if let Err(err) = advance_and_save_transition_transaction(
transaction_api.as_ref(),
&mut transaction,
crates/ecstore/src/store/bucket.rs
+52 -1
View File
@@ -568,6 +568,7 @@ mod tests {
};
use crate::bucket::metadata::table_bucket_catalog_metadata_prefix;
use crate::bucket::metadata_sys;
use crate::cluster::rpc::peer_s3_client::install_delete_bucket_empty_scan_barrier;
use crate::disk::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
use crate::error::StorageError;
use crate::object_api::{ObjectOptions, PutObjReader};
@@ -589,6 +590,7 @@ mod tests {
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::{Duration, SystemTime};
use time::OffsetDateTime;
use tokio::io::AsyncReadExt;
use tokio::sync::{Notify, OnceCell};
use tokio_util::sync::CancellationToken;
use uuid::Uuid;
@@ -1048,7 +1050,7 @@ mod tests {
.await
.expect("bucket should be created");
for disk_path in &disk_paths {
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory"))
tokio::fs::create_dir_all(disk_path.join(&bucket).join("empty-directory/nested/leaf"))
.await
.expect("empty directory remnant should be created");
}
@@ -1257,6 +1259,55 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn bucket_delete_preserves_put_committed_after_empty_scan() {
let (_, ecstore) = setup_bucket_delete_test_env().await;
let bucket = format!("bucket-delete-empty-scan-race-{}", Uuid::new_v4().simple());
let object = "committed-after-empty-scan";
let payload = b"object committed after DeleteBucket empty scan".to_vec();
ecstore
.make_bucket(&bucket, &MakeBucketOptions::default())
.await
.expect("bucket should be created");
let barrier = install_delete_bucket_empty_scan_barrier();
let delete_store = ecstore.clone();
let delete_bucket = bucket.clone();
let delete = tokio::spawn(async move {
delete_store
.delete_bucket(&delete_bucket, &DeleteBucketOptions::default())
.await
});
barrier.wait_until_paused().await;
let mut put_reader = PutObjReader::from_vec(payload.clone());
ecstore
.put_object(&bucket, object, &mut put_reader, &ObjectOptions::default())
.await
.expect("PUT should commit while DeleteBucket is paused after its empty scan");
barrier.release();
let err = delete
.await
.expect("DeleteBucket task should join")
.expect_err("DeleteBucket must reject a PUT committed after its empty scan");
assert!(matches!(err, StorageError::BucketNotEmpty(name) if name == bucket));
let mut reader = ecstore
.get_object_reader(&bucket, object, None, http::HeaderMap::new(), &ObjectOptions::default())
.await
.expect("committed object should remain readable after DeleteBucket fails");
let mut restored = Vec::new();
reader
.stream
.read_to_end(&mut restored)
.await
.expect("object body should remain readable");
assert_eq!(restored, payload);
}
#[tokio::test]
#[serial]
async fn bucket_recreation_does_not_publish_unverified_usage() {
crates/ecstore/src/store/init.rs
+191 -6
View File
@@ -562,10 +562,12 @@ mod tests {
},
tier_sweeper::Jentry,
transition_transaction::{
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionRemoteVersion,
TransitionSourceIdentity, TransitionSourceVersionMode, TransitionTransaction, TransitionTransactionInit,
TransitionTransactionState, load_transition_transaction_record, recover_transition_transaction_records,
save_transition_transaction_record,
TRANSITION_TRANSACTION_RECORD_PREFIX, TransitionCleanupDecision, TransitionCleanupProof, TransitionOperatorError,
TransitionOperatorProbe, TransitionRemoteVersion, TransitionSourceIdentity, TransitionSourceVersionMode,
TransitionTransaction, TransitionTransactionInit, TransitionTransactionState,
delete_transition_candidate_for_operator, finalize_missing_transition_transaction_for_operator,
inspect_transition_transaction_for_operator, load_transition_transaction_record,
recover_transition_transaction_records, save_transition_transaction_record,
},
},
client::transition_api::ReaderImpl,
@@ -575,6 +577,7 @@ mod tests {
services::tier::{
test_util::{MockWarmBackend, MockWarmOp, TransitionCleanupStoreBarrier, register_mock_tier},
tier::{TIER_CONFIG_FILE, TierConfigMgr},
tier_config::{TierConfig, TierType, TierWasabi},
tier_mutation_intent::{
TIER_MUTATION_INTENT_RECORD_PREFIX, TierMutationIntent, TierMutationIntentKind, TierMutationIntentState,
TierMutationIntentTarget, advance_tier_mutation_intent_record_idempotent, delete_tier_mutation_intent_record,
@@ -1163,6 +1166,37 @@ mod tests {
.len()
}
#[cfg(feature = "test-util")]
async fn register_transition_reconcile_test_tier(
handle: &Arc<tokio::sync::RwLock<TierConfigMgr>>,
tier_name: &str,
) -> MockWarmBackend {
let backend = MockWarmBackend::new();
let mut manager = handle.write().await;
manager.tiers.insert(
tier_name.to_string(),
TierConfig {
version: "v1".to_string(),
tier_type: TierType::Wasabi,
name: tier_name.to_string(),
wasabi: Some(TierWasabi {
name: tier_name.to_string(),
endpoint: "https://s3.wasabisys.com".to_string(),
access_key: "test-access-key".to_string(),
secret_key: "test-secret-key".to_string(),
bucket: "mock-tier".to_string(),
prefix: format!("mock/{}/", uuid::Uuid::new_v4()),
region: "us-east-1".to_string(),
}),
..Default::default()
},
);
manager
.install_test_driver(tier_name, Box::new(backend.clone()))
.expect("mock fallback tier driver should install");
backend
}
#[cfg(feature = "test-util")]
async fn wait_for_tier_delete_journal_recovery(
store: Arc<crate::store::ECStore>,
@@ -2738,7 +2772,7 @@ mod tests {
.await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
@@ -2809,6 +2843,157 @@ mod tests {
}
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn operator_reconcile_deletes_exact_candidate_before_finalizing_record() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-reconcile", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXOPERATOR";
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
.backend_identity();
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: None,
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Unversioned,
},
tier_name: tier_name.to_string(),
backend_fingerprint: backend_identity,
not_after_unix_nanos: 1,
})
.expect("transaction should build");
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("transaction should enter unknown upload outcome state");
let remote_version = uuid::Uuid::new_v4().to_string();
backend.set_put_remote_version(Some(remote_version.clone())).await;
let candidate = bytes::Bytes::from_static(b"operator-confirmed transition candidate");
backend
.put(
&transaction.remote_object,
ReaderImpl::Body(candidate.clone()),
i64::try_from(candidate.len()).expect("test candidate length should fit i64"),
)
.await
.expect("mock backend should accept candidate");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
let status = inspect_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
.await
.expect("operator inspection should probe the candidate");
assert_eq!(status.probe, TransitionOperatorProbe::VersionedPresent(remote_version.clone()));
let wrong_version = uuid::Uuid::new_v4().to_string();
let err = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &wrong_version)
.await
.expect_err("a mismatched exact version must fail before deleting a candidate");
assert!(matches!(
err,
TransitionOperatorError::CandidateVersionMismatch {
expected,
actual: TransitionOperatorProbe::VersionedPresent(ref observed),
} if expected == wrong_version && observed == &remote_version
));
assert!(backend.contains(&transaction.remote_object).await);
assert_eq!(backend.exact_remove_count(), 0);
load_transition_transaction_record(store.clone(), transaction.transaction_id)
.await
.expect("an incorrect exact version must retain the transaction journal");
let result = delete_transition_candidate_for_operator(store.clone(), transaction.transaction_id, &remote_version)
.await
.expect("operator-confirmed exact candidate should be deleted");
assert_eq!(result.status.probe, TransitionOperatorProbe::Missing);
assert!(result.journal_observed_after_delete);
assert_eq!(backend.exact_remove_count(), 1);
assert_eq!(backend.remove_versions().await, vec![(transaction.remote_object.clone(), remote_version)]);
load_transition_transaction_record(store.clone(), transaction.transaction_id)
.await
.expect("candidate deletion must retain the transaction journal");
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id)
.await
.expect("a separately confirmed missing candidate should permit finalization");
assert!(matches!(
load_transition_transaction_record(store, transaction.transaction_id).await,
Err(Error::ConfigNotFound)
));
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
async fn operator_finalize_retains_record_without_missing_proof() {
let temp_dir = tempfile::tempdir().expect("create temp store dir");
let (ctx, store, _shutdown) =
without_storage_class_env(build_isolated_test_store(temp_dir.path(), "transition-operator-fail-closed", &[4])).await;
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXOPERATORFAIL";
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend_identity = TierConfigMgr::acquire_operation_lease(&ctx.tier_config_mgr(), tier_name)
.await
.expect("tier lease should resolve")
.backend_identity();
let mut transaction = TransitionTransaction::new(TransitionTransactionInit {
deployment_id: ctx.deployment_id().expect("test store should initialize deployment id"),
transaction_id: uuid::Uuid::new_v4(),
owner_epoch: uuid::Uuid::new_v4(),
write_id: uuid::Uuid::new_v4(),
source: TransitionSourceIdentity {
bucket: "source-bucket".to_string(),
object: "source-object".to_string(),
version_id: None,
data_dir: uuid::Uuid::new_v4(),
mod_time_unix_nanos: 1_770_000_000_000_000_000,
size: 42,
etag: "source-etag".to_string(),
version_mode: TransitionSourceVersionMode::Unversioned,
},
tier_name: tier_name.to_string(),
backend_fingerprint: backend_identity,
not_after_unix_nanos: 1,
})
.expect("transaction should build");
transaction
.advance(transaction.fence(), TransitionTransactionState::UploadOutcomeUnknown, None)
.expect("transaction should enter unknown upload outcome state");
save_transition_transaction_record(store.clone(), &transaction)
.await
.expect("transaction record should persist");
backend
.set_transition_candidate_probe_override(Some(TransitionCandidateProbe::Unsupported))
.await;
assert!(matches!(
finalize_missing_transition_transaction_for_operator(store.clone(), transaction.transaction_id).await,
Err(TransitionOperatorError::CandidateNotMissing(TransitionOperatorProbe::Unsupported))
));
load_transition_transaction_record(store, transaction.transaction_id)
.await
.expect("an unsupported probe must retain the transaction journal");
assert_eq!(backend.remove_count().await, 0);
}
#[cfg(feature = "test-util")]
#[tokio::test]
#[serial_test::serial(storage_class_env)]
@@ -2819,7 +3004,7 @@ mod tests {
crate::bucket::metadata_sys::init_bucket_metadata_sys(store.clone(), Vec::new()).await;
let tier_name = "TXRESPONSELOSS";
let backend = register_mock_tier(&ctx.tier_config_mgr(), tier_name).await;
let backend = register_transition_reconcile_test_tier(&ctx.tier_config_mgr(), tier_name).await;
let bucket = "transition-response-loss-bucket";
let object = "source.bin";
store
crates/ecstore/src/store/list_objects.rs
+1
View File
@@ -6895,6 +6895,7 @@ mod test {
)
.await
.expect("local disk should be created");
disk.make_volume("bucket").await.expect("test bucket should be created");
let mut fi = FileInfo::new("object", 1, 1);
fi.volume = "bucket".to_owned();
fi.name = "object".to_owned();
crates/ecstore/src/test_tracing.rs
+43
View File
@@ -0,0 +1,43 @@
//! Test-only guard against tracing's process-global callsite-interest cache.
//!
//! Any test that installs its own subscriber and then asserts on span or event
//! context is asserting on process-global state. See
//! [`pin_callsite_interest_for_test`] for what goes wrong and why holding its
//! guard fixes it.
/// Stop a sibling test thread from poisoning tracing's process-global callsite
/// interest cache while this test asserts on span or event context.
///
/// `tracing` caches every callsite's `Interest` in **process-global** state, and
/// the first thread to reach a callsite fixes that value. While at most one
/// dispatcher is registered, tracing-core takes a fast path
/// (`Dispatchers::rebuilder` -> `Rebuilder::JustOne`) that derives a
/// newly-registered callsite's interest from whichever subscriber is current
/// *on the registering thread*, and registration happens exactly once (guarded
/// by a CAS in `DefaultCallsite::register`).
///
/// In a multi-threaded `cargo test` binary a sibling test therefore routinely
/// reaches a production callsite first, from a thread with no subscriber
/// installed: the interest is derived from that thread's `NoSubscriber` and
/// cached as `Interest::never()` for the whole process. From then on the
/// callsite is dead for *every* later caller, including a test that installed
/// its own subscriber — an `info_span!` silently evaluates to `Span::none()`, and
/// a `warn!`/`info!` event never fires at all.
///
/// Registering a second, inert dispatcher closes both halves of that race:
///
/// * constructing it rebuilds every *already-registered* callsite's interest
/// against the live dispatcher set — which includes the caller's subscriber —
/// repairing whatever a sibling may already have poisoned; and
/// * holding it alive keeps tracing-core off the single-dispatcher fast path, so
/// a callsite registered *later* by any thread is resolved against that live
/// set instead of the registering thread's `NoSubscriber`.
///
/// Call this **after** installing the test's subscriber, and hold the returned
/// guard for the rest of the test. Only the `cargo test` fallback needs it:
/// nextest runs each test in its own process, where there is no sibling thread
/// to lose the race to (see `docs/testing/README.md`).
#[must_use = "callsite interest is only pinned while the returned guard is held"]
pub(crate) fn pin_callsite_interest_for_test() -> tracing::Dispatch {
tracing::Dispatch::new(tracing_core::subscriber::NoSubscriber::default())
}
crates/io-metrics/src/internode_metrics.rs
+25
View File
@@ -67,6 +67,7 @@ const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_inter
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
const ERASURE_WRITE_QUORUM_FAILURES_TOTAL: &str = "rustfs_system_storage_erasure_write_quorum_failures_total";
@@ -159,6 +160,7 @@ pub struct InternodeMetricsSnapshot {
pub operation_write_shutdown_errors_total: u64,
pub signature_v1_fallback_total: u64,
pub body_digest_fallback_total: u64,
pub replay_scope_fallback_total: u64,
pub replay_cache_overflow_total: u64,
}
@@ -180,6 +182,7 @@ pub struct InternodeMetrics {
msgpack_json_decode_error_total: AtomicU64,
signature_v1_fallback_total: AtomicU64,
body_digest_fallback_total: AtomicU64,
replay_scope_fallback_total: AtomicU64,
replay_cache_overflow_total: AtomicU64,
}
@@ -431,6 +434,13 @@ impl InternodeMetrics {
counter!(INTERNODE_BODY_DIGEST_FALLBACK_TOTAL).increment(1);
}
/// Count an accepted v1/v2 request that does not carry the replay-scoped signature. This is
/// the convergence signal for `RUSTFS_INTERNODE_RPC_REPLAY_SCOPE_STRICT`.
pub fn record_replay_scope_fallback(&self) {
self.replay_scope_fallback_total.fetch_add(1, Ordering::Relaxed);
counter!(INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL).increment(1);
}
/// Count a body-bound internode RPC rejected because the replay-protection nonce cache was
/// full. Overflow fails closed, so a sustained non-zero rate means
/// `RUSTFS_INTERNODE_RPC_REPLAY_CACHE_CAPACITY` is undersized for this node's peak legitimate
@@ -488,6 +498,7 @@ impl InternodeMetrics {
operation_write_shutdown_errors_total: self.operation_write_shutdown_errors_total.load(Ordering::Relaxed),
signature_v1_fallback_total: self.signature_v1_fallback_total.load(Ordering::Relaxed),
body_digest_fallback_total: self.body_digest_fallback_total.load(Ordering::Relaxed),
replay_scope_fallback_total: self.replay_scope_fallback_total.load(Ordering::Relaxed),
replay_cache_overflow_total: self.replay_cache_overflow_total.load(Ordering::Relaxed),
}
}
@@ -510,6 +521,7 @@ impl InternodeMetrics {
self.msgpack_json_decode_error_total.store(0, Ordering::Relaxed);
self.signature_v1_fallback_total.store(0, Ordering::Relaxed);
self.body_digest_fallback_total.store(0, Ordering::Relaxed);
self.replay_scope_fallback_total.store(0, Ordering::Relaxed);
self.replay_cache_overflow_total.store(0, Ordering::Relaxed);
}
}
@@ -887,6 +899,19 @@ mod tests {
assert_eq!(metrics.snapshot().signature_v1_fallback_total, 0);
}
#[test]
fn replay_scope_fallback_counter_updates_snapshot_and_resets() {
let metrics = InternodeMetrics::default();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
metrics.record_replay_scope_fallback();
metrics.record_replay_scope_fallback();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 2);
metrics.reset_for_test();
assert_eq!(metrics.snapshot().replay_scope_fallback_total, 0);
}
#[test]
fn cluster_peer_flips_offline_after_threshold_and_back_online() {
// Unique addr keeps this independent of the process-global registry / other tests.
crates/keystone/Cargo.toml
+3
View File
@@ -25,6 +25,9 @@ keywords = ["rustfs", "openstack", "keystone", "authentication", "s3"]
categories = ["authentication", "web-programming"]
authors.workspace = true
[lints]
workspace = true
[dependencies]
tokio = { workspace = true, features = ["rt", "sync"] }
reqwest = { workspace = true, features = ["json"] }
crates/madmin/src/metrics.rs
+309 -6
View File
@@ -545,6 +545,8 @@ pub struct ScannerMetrics {
pub collected_at: DateTime<Utc>,
#[serde(rename = "current_cycle")]
pub current_cycle: u64,
#[serde(rename = "current_cycle_active", default, skip_serializing_if = "Option::is_none")]
pub current_cycle_active: Option<bool>,
#[serde(rename = "current_started")]
pub current_started: DateTime<Utc>,
#[serde(rename = "cycle_complete_times")]
@@ -718,7 +720,31 @@ pub struct ScannerMetrics {
}
impl ScannerMetrics {
pub fn is_current_cycle_active(&self) -> bool {
self.current_cycle_active.unwrap_or(self.current_cycle > 0)
}
pub fn merge(&mut self, other: &Self) {
// Legacy nodes omit the activity field and use a non-zero cycle as
// their active signal. New nodes publish explicit first-cycle and idle
// states, including cycle zero.
let self_cycle_active = self.is_current_cycle_active();
let other_cycle_active = other.is_current_cycle_active();
let self_cycle_authority = (
self_cycle_active,
self.current_cycle,
self.cycles_completed_at.len(),
self.cycles_completed_at.as_slice(),
self.current_started,
);
let other_cycle_authority = (
other_cycle_active,
other.current_cycle,
other.cycles_completed_at.len(),
other.cycles_completed_at.as_slice(),
other.current_started,
);
let other_cycle_is_authoritative = self_cycle_authority < other_cycle_authority;
let other_is_newer = self.collected_at < other.collected_at;
if other_is_newer {
self.collected_at = other.collected_at;
@@ -854,15 +880,12 @@ impl ScannerMetrics {
self.ongoing_buckets = other.ongoing_buckets;
}
if self.current_cycle < other.current_cycle {
if other_cycle_is_authoritative {
self.current_cycle = other.current_cycle;
self.cycles_completed_at = other.cycles_completed_at.clone();
self.current_started = other.current_started;
}
if other.cycles_completed_at.len() > self.cycles_completed_at.len() {
self.cycles_completed_at = other.cycles_completed_at.clone();
}
self.current_cycle_active = Some(self_cycle_active || other_cycle_active);
if !other.life_time_ops.is_empty() && self.life_time_ops.is_empty() {
self.life_time_ops = other.life_time_ops.clone();
@@ -931,7 +954,13 @@ impl Metrics {
if let Some(scanner) = other.scanner.as_ref() {
match self.scanner {
Some(ref mut s_scanner) => s_scanner.merge(scanner),
None => self.scanner = Some(scanner.clone()),
None => {
let mut scanner = scanner.clone();
if scanner.current_cycle_active.is_none() {
scanner.current_cycle_active = Some(scanner.is_current_cycle_active());
}
self.scanner = Some(scanner);
}
}
}
@@ -1389,6 +1418,280 @@ pub struct Operations {
mod tests {
use super::*;
#[test]
fn scanner_metrics_serializes_cycle_active_presence() {
let missing_value = serde_json::to_value(ScannerMetrics::default()).expect("scanner metrics should serialize");
assert!(missing_value.get("current_cycle_active").is_none());
let missing: ScannerMetrics =
serde_json::from_value(missing_value).expect("older scanner metrics without cycle-active should decode");
assert_eq!(missing.current_cycle_active, None);
let explicit_false_value = serde_json::to_value(ScannerMetrics {
current_cycle_active: Some(false),
..Default::default()
})
.expect("scanner metrics with explicit cycle-active should serialize");
assert_eq!(explicit_false_value["current_cycle_active"], serde_json::Value::Bool(false));
let explicit_false: ScannerMetrics =
serde_json::from_value(explicit_false_value).expect("explicit cycle-active should decode");
assert_eq!(explicit_false.current_cycle_active, Some(false));
}
#[test]
fn scanner_metrics_merge_prefers_an_active_first_cycle() {
let collected_at = Utc::now();
let idle_started = collected_at - chrono::Duration::hours(1);
let active_started = collected_at - chrono::Duration::seconds(5);
let mut scanner = ScannerMetrics {
collected_at,
current_cycle: 0,
current_cycle_active: Some(false),
current_started: idle_started,
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 0,
current_cycle_active: Some(true),
current_started: active_started,
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 0);
assert_eq!(scanner.current_started, active_started);
}
#[test]
fn metrics_merge_preserves_explicit_active_first_cycle() {
let mut aggregated = Metrics::default();
aggregated.merge(&Metrics {
scanner: Some(ScannerMetrics {
current_cycle_active: Some(true),
..Default::default()
}),
..Default::default()
});
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 0);
}
#[test]
fn scanner_metrics_merge_preserves_legacy_nonzero_active_signal() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics {
collected_at,
..Default::default()
};
scanner.merge(&ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 7,
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 7);
}
#[test]
fn metrics_merge_normalizes_first_legacy_scanner_snapshot() {
let legacy = Metrics {
scanner: Some(ScannerMetrics {
current_cycle: 7,
..Default::default()
}),
..Default::default()
};
let mut aggregated = Metrics::default();
aggregated.merge(&legacy);
let scanner = aggregated.scanner.expect("aggregated scanner metrics");
assert_eq!(scanner.current_cycle_active, Some(true));
assert_eq!(scanner.current_cycle, 7);
}
#[test]
fn scanner_metrics_merge_preserves_explicit_inactive_nonzero_cycle() {
let collected_at = Utc::now();
let mut scanner = ScannerMetrics::default();
scanner.merge(&ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(false),
..Default::default()
});
assert_eq!(scanner.current_cycle_active, Some(false));
assert_eq!(scanner.current_cycle, 7);
}
#[test]
fn scanner_metrics_merge_cycle_active_is_order_independent() {
let collected_at = Utc::now();
let legacy_active = ScannerMetrics {
collected_at,
current_cycle: 7,
current_started: collected_at - chrono::Duration::seconds(10),
..Default::default()
};
let explicit_idle = ScannerMetrics {
collected_at,
current_cycle: 0,
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::hours(1),
..Default::default()
};
let mut legacy_first = legacy_active.clone();
legacy_first.merge(&explicit_idle);
let mut legacy_second = explicit_idle.clone();
legacy_second.merge(&legacy_active);
assert_eq!(legacy_first.current_cycle_active, Some(true));
assert_eq!(legacy_second.current_cycle_active, Some(true));
assert_eq!(legacy_first.current_cycle, 7);
assert_eq!(legacy_second.current_cycle, 7);
let explicit_inactive_nonzero = ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(false),
..Default::default()
};
let mut inactive_first = explicit_inactive_nonzero.clone();
inactive_first.merge(&explicit_idle);
let mut inactive_second = explicit_idle;
inactive_second.merge(&explicit_inactive_nonzero);
assert_eq!(inactive_first.current_cycle_active, Some(false));
assert_eq!(inactive_second.current_cycle_active, Some(false));
assert_eq!(inactive_first.current_cycle, 7);
assert_eq!(inactive_second.current_cycle, 7);
}
#[test]
fn scanner_metrics_merge_cycle_authority_is_order_independent() {
let collected_at = Utc::now();
let completion = collected_at - chrono::Duration::minutes(1);
let earlier_active = ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(10),
cycles_completed_at: vec![completion],
..Default::default()
};
let later_active = ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 7,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(5),
cycles_completed_at: vec![completion],
..Default::default()
};
let mut earlier_first = earlier_active.clone();
earlier_first.merge(&later_active);
let mut later_first = later_active.clone();
later_first.merge(&earlier_active);
assert_eq!(earlier_first.current_started, later_active.current_started);
assert_eq!(later_first.current_started, later_active.current_started);
assert_eq!(earlier_first.cycles_completed_at, later_active.cycles_completed_at);
assert_eq!(later_first.cycles_completed_at, later_active.cycles_completed_at);
let stale_idle = ScannerMetrics {
collected_at,
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::hours(1),
..Default::default()
};
let completed_idle = ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::seconds(5),
cycles_completed_at: vec![completion],
..Default::default()
};
let mut stale_first = stale_idle.clone();
stale_first.merge(&completed_idle);
let mut completed_first = completed_idle.clone();
completed_first.merge(&stale_idle);
assert_eq!(stale_first.current_started, completed_idle.current_started);
assert_eq!(completed_first.current_started, completed_idle.current_started);
assert_eq!(stale_first.cycles_completed_at, completed_idle.cycles_completed_at);
assert_eq!(completed_first.cycles_completed_at, completed_idle.cycles_completed_at);
}
#[test]
fn scanner_metrics_merge_cycle_authority_is_associative() {
let collected_at = Utc::now();
let older_completion = collected_at - chrono::Duration::minutes(3);
let last_completion = collected_at - chrono::Duration::minutes(1);
let cycle_seven = ScannerMetrics {
collected_at,
current_cycle: 7,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(10),
cycles_completed_at: vec![older_completion, last_completion],
..Default::default()
};
let cycle_eight = ScannerMetrics {
collected_at: collected_at + chrono::Duration::seconds(1),
current_cycle: 8,
current_cycle_active: Some(true),
current_started: collected_at - chrono::Duration::seconds(5),
cycles_completed_at: vec![last_completion],
..Default::default()
};
let newer_idle = ScannerMetrics {
collected_at: collected_at + chrono::Duration::hours(1),
current_cycle_active: Some(false),
current_started: collected_at - chrono::Duration::hours(1),
..Default::default()
};
let mut left_associative = cycle_seven.clone();
left_associative.merge(&cycle_eight);
left_associative.merge(&newer_idle);
let mut right_group = cycle_eight.clone();
right_group.merge(&newer_idle);
let mut right_associative = cycle_seven.clone();
right_associative.merge(&right_group);
assert_eq!(left_associative.current_cycle, 8);
assert_eq!(right_associative.current_cycle, 8);
assert_eq!(left_associative.current_started, cycle_eight.current_started);
assert_eq!(right_associative.current_started, cycle_eight.current_started);
assert_eq!(left_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
assert_eq!(right_associative.cycles_completed_at, cycle_eight.cycles_completed_at);
for order in [
[&cycle_seven, &cycle_eight, &newer_idle],
[&cycle_seven, &newer_idle, &cycle_eight],
[&cycle_eight, &cycle_seven, &newer_idle],
[&cycle_eight, &newer_idle, &cycle_seven],
[&newer_idle, &cycle_seven, &cycle_eight],
[&newer_idle, &cycle_eight, &cycle_seven],
] {
let mut merged = ScannerMetrics::default();
for scanner in order {
merged.merge(scanner);
}
assert_eq!(merged.current_cycle_active, Some(true));
assert_eq!(merged.current_cycle, 8);
assert_eq!(merged.current_started, cycle_eight.current_started);
assert_eq!(merged.cycles_completed_at, cycle_eight.cycles_completed_at);
}
}
#[test]
fn scanner_metrics_merge_aggregates_partial_cycles_by_source() {
let collected_at = Utc::now();
crates/obs/src/metrics/stats_collector.rs
+7 -7
View File
@@ -262,11 +262,11 @@ async fn obs_site_replication_stats() -> ReplicationStats {
}
fn current_scanner_cycle_age_seconds(
current_cycle: u64,
current_cycle_active: bool,
current_started: chrono::DateTime<Utc>,
now: chrono::DateTime<Utc>,
) -> u64 {
if current_cycle == 0 {
if !current_cycle_active {
0
} else {
now.signed_duration_since(current_started).num_seconds().max(0) as u64
@@ -1090,7 +1090,7 @@ pub async fn collect_scanner_metric_stats() -> Option<ScannerStats> {
let reference_time = metrics.cycles_completed_at.last().copied().unwrap_or(metrics.current_started);
let last_activity_seconds = now.signed_duration_since(reference_time).num_seconds().max(0) as u64;
let active_paths = metrics.active_scan_paths as u64;
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle, metrics.current_started, now);
let current_cycle_age_seconds = current_scanner_cycle_age_seconds(metrics.current_cycle_active, metrics.current_started, now);
let current_scan_mode = scanner_scan_mode_code(&metrics.current_scan_mode);
let current_cycle_age = current_cycle_age_seconds as f64;
let last_cycle_duration = metrics.last_cycle_duration_seconds;
@@ -1464,21 +1464,21 @@ mod tests {
fn current_scanner_cycle_age_seconds_returns_zero_when_idle() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(0, now - chrono::Duration::seconds(30), now), 0);
assert_eq!(current_scanner_cycle_age_seconds(false, now - chrono::Duration::seconds(30), now), 0);
}
#[test]
fn current_scanner_cycle_age_seconds_clamps_future_start() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(4, now + chrono::Duration::seconds(30), now), 0);
assert_eq!(current_scanner_cycle_age_seconds(true, now + chrono::Duration::seconds(30), now), 0);
}
#[test]
fn current_scanner_cycle_age_seconds_reports_active_elapsed_time() {
fn current_scanner_cycle_age_seconds_reports_active_first_cycle_elapsed_time() {
let now = Utc::now();
assert_eq!(current_scanner_cycle_age_seconds(4, now - chrono::Duration::seconds(45), now), 45);
assert_eq!(current_scanner_cycle_age_seconds(true, now - chrono::Duration::seconds(45), now), 45);
}
#[test]
crates/protocols/Cargo.toml
+3
View File
@@ -25,6 +25,9 @@ description = "Protocol implementations for RustFS (FTPS, SFTP, etc.)"
keywords = ["ftp", "sftp", "protocol", "storage", "rustfs"]
categories = ["network-programming", "filesystem"]
[lints]
workspace = true
[features]
default = []
ftps = ["dep:libunftp", "dep:unftp-core", "dep:rustls", "dep:rustfs-tls-runtime", "dep:subtle"]
crates/protocols/src/common/dummy_storage.rs
+5 -5
View File
@@ -608,7 +608,7 @@ impl StorageBackend for DummyBackend {
inner.put_object_calls.push(PutObjectCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
metadata: input.metadata.clone(),
metadata: input.metadata,
});
let stall = inner.stall_put_object;
let entered = inner.put_object_entered.clone();
@@ -731,7 +731,7 @@ impl StorageBackend for DummyBackend {
inner.create_multipart_calls.push(CreateMultipartCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
metadata: input.metadata.clone(),
metadata: input.metadata,
});
}
match self.inner.lock().expect("lock").create_multipart_upload.pop_front() {
@@ -749,7 +749,7 @@ impl StorageBackend for DummyBackend {
inner.upload_part_calls.push(UploadPartCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
upload_id: input.upload_id.to_string(),
upload_id: input.upload_id,
part_number: input.part_number,
content_length: input.content_length,
});
@@ -787,7 +787,7 @@ impl StorageBackend for DummyBackend {
inner.complete_multipart_calls.push(CompleteCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
upload_id: input.upload_id.to_string(),
upload_id: input.upload_id,
part_count,
});
}
@@ -808,7 +808,7 @@ impl StorageBackend for DummyBackend {
inner.abort_multipart_calls.push(AbortCall {
bucket: input.bucket.to_string(),
key: input.key.to_string(),
upload_id: input.upload_id.to_string(),
upload_id: input.upload_id,
});
}
match self.inner.lock().expect("lock").abort_multipart_upload.pop_front() {
crates/protocols/src/ftps/driver.rs
+1 -3
View File
@@ -379,9 +379,7 @@ where
.await
.map_err(|_| Error::new(ErrorKind::PermanentFileNotAvailable, "Access denied"))?;
let prefix_with_slash = prefix
.clone()
.map(|p| if p.ends_with('/') { p.to_string() } else { format!("{}/", p) });
let prefix_with_slash = prefix.clone().map(|p| if p.ends_with('/') { p } else { format!("{}/", p) });
let list_input = ListObjectsV2Input::builder()
.bucket(bucket)
crates/protocols/src/ftps/server.rs
+1 -1
View File
@@ -371,7 +371,7 @@ impl UserDetailProvider for FtpsUserDetailProvider {
let ftps_user = FtpsUser {
username: principal.username.clone(),
name: identity.credentials.name.clone(),
name: identity.credentials.name,
session_context,
};
crates/protocols/src/sftp/write.rs
+2 -2
View File
@@ -1495,12 +1495,12 @@ mod tests {
let buffer_len_u64 = part_buffer_len as u64;
let phase = match phase_variant {
0 => WritePhase::Buffering {
part_buffer: part_buffer.clone(),
part_buffer,
},
1 => WritePhase::Streaming {
upload_id: "UP-proptest".to_string(),
abort_authorized: true,
part_buffer: part_buffer.clone(),
part_buffer,
uploaded_parts: Vec::new(),
next_part_number,
},
crates/protocols/src/swift/bulk.rs
+1 -3
View File
@@ -627,9 +627,7 @@ mod tests {
#[test]
fn test_parse_paths_with_empty_lines() {
let body = "/container1/file1.txt\n\n/container2/file2.txt\n \n/container1/file3.txt";
let paths: Vec<&str> = body.lines().filter(|line| !line.trim().is_empty()).collect();
assert_eq!(paths.len(), 3);
assert_eq!(body.lines().filter(|line| !line.trim().is_empty()).count(), 3);
}
/// Tests for the `extract_tar_entries` async function.
crates/protocols/src/swift/object.rs
+1 -2
View File
@@ -206,10 +206,9 @@ impl ObjectKeyMapper {
#[allow(dead_code)] // Used in: object operations
pub fn normalize_path(object: &str) -> String {
// Split by '/', filter out empty segments (except if it's the end)
let segments: Vec<&str> = object.split('/').collect();
let has_trailing_slash = object.ends_with('/');
let normalized_segments: Vec<&str> = segments.into_iter().filter(|s| !s.is_empty()).collect();
let normalized_segments: Vec<&str> = object.split('/').filter(|s| !s.is_empty()).collect();
let mut result = normalized_segments.join("/");
crates/protocols/src/swift/router.rs
+58 -12
View File
@@ -84,20 +84,19 @@ impl SwiftRouter {
Self { enabled, url_prefix }
}
/// Return whether a URI matches the Swift route shape without allocating
/// decoded route components.
pub fn matches(&self, uri: &Uri) -> bool {
let Some(path) = self.route_path(uri) else {
return false;
};
let mut segments = path.trim_start_matches('/').split('/');
segments.next() == Some("v1") && segments.next().is_some_and(Self::is_valid_account)
}
/// Parse a URI and return a SwiftRoute if it matches Swift URL pattern
pub fn route(&self, uri: &Uri, method: Method) -> Option<SwiftRoute> {
if !self.enabled {
return None;
}
let path = uri.path();
// Strip optional prefix
let path = if let Some(prefix) = &self.url_prefix {
path.strip_prefix(&format!("/{}/", prefix))?
} else {
path
};
let path = self.route_path(uri)?;
// Split path into segments - preserve empty segments to maintain object key fidelity
// Swift allows trailing slashes and consecutive slashes in object names (e.g., "dir/" or "a//b")
@@ -175,6 +174,17 @@ impl SwiftRouter {
fn is_valid_account(account: &str) -> bool {
ACCOUNT_PATTERN.is_match(account)
}
fn route_path<'a>(&self, uri: &'a Uri) -> Option<&'a str> {
if !self.enabled {
return None;
}
let path = uri.path();
let Some(prefix) = &self.url_prefix else {
return Some(path);
};
path.strip_prefix('/')?.strip_prefix(prefix)?.strip_prefix('/')
}
}
#[cfg(test)]
@@ -281,6 +291,42 @@ mod tests {
assert_eq!(route, None);
}
#[test]
fn test_matches_agrees_with_route_without_decoding_components() {
let router = SwiftRouter::new(true, None);
for path in [
"/v1/AUTH_project",
"/v1/AUTH_project/",
"/v1/AUTH_project/container",
"/v1/AUTH_project/container/a%20long/object",
"//v1/AUTH_project/container/object",
"/v1/not-a-swift-account/object",
"/v1/AUTH_/object",
"/bucket/object",
] {
let uri = path.parse().expect("Swift route test URI");
assert_eq!(
router.matches(&uri),
router.route(&uri, Method::GET).is_some(),
"classification must match full routing for {path}"
);
}
let prefixed_router = SwiftRouter::new(true, Some("swift".to_string()));
for path in [
"/swift/v1/AUTH_project/container",
"/swiftish/v1/AUTH_project/container",
"/v1/AUTH_project/container",
] {
let uri = path.parse().expect("prefixed Swift route test URI");
assert_eq!(
prefixed_router.matches(&uri),
prefixed_router.route(&uri, Method::GET).is_some(),
"prefixed classification must match full routing for {path}"
);
}
}
#[test]
fn test_project_id_extraction() {
let route = SwiftRoute::Account {
crates/protocols/src/webdav/driver.rs
+4 -4
View File
@@ -222,7 +222,7 @@ where
created: modified,
is_dir: false,
etag: output.e_tag.as_ref().map(etag_to_string),
content_type: output.content_type.map(|c| c.to_string()),
content_type: output.content_type,
}) as Box<dyn DavMetaData>)
}
Err(e) => {
@@ -1185,7 +1185,7 @@ where
created: modified,
is_dir: false,
etag: output.e_tag.as_ref().map(etag_to_string),
content_type: output.content_type.map(|c| c.to_string()),
content_type: output.content_type,
}) as Box<dyn DavMetaData>)
}
ResolvedPath::Directory { metadata, .. } => {
@@ -1204,7 +1204,7 @@ where
created: modified,
is_dir: true,
etag: metadata.as_ref().and_then(|output| output.e_tag.as_ref().map(etag_to_string)),
content_type: metadata.and_then(|output| output.content_type.map(|c| c.to_string())),
content_type: metadata.and_then(|output| output.content_type),
}) as Box<dyn DavMetaData>)
}
};
@@ -2035,7 +2035,7 @@ mod tests {
_access_key: &str,
_secret_key: &str,
) -> Result<ListObjectsV2Output, Self::Error> {
let prefix = input.prefix.map(|p| p.to_string()).unwrap_or_default();
let prefix = input.prefix.unwrap_or_default();
let mut keys: Vec<String> = self
.state
.lock()
crates/protocols/tests/swift_listing_symlink_tests.rs
+2 -4
View File
@@ -436,12 +436,10 @@ fn test_symlink_to_nested_object() {
fn test_listing_empty_container() {
let objects: Vec<&str> = vec![];
let filtered: Vec<_> = objects.iter().collect();
assert_eq!(filtered.len(), 0);
assert!(objects.is_empty());
// With prefix
let with_prefix: Vec<_> = objects.iter().filter(|o| o.starts_with("prefix/")).collect();
assert_eq!(with_prefix.len(), 0);
assert!(!objects.iter().any(|o| o.starts_with("prefix/")));
}
/// Test listing lexicographic ordering
crates/protos/src/lib.rs
+72 -3
View File
@@ -171,6 +171,7 @@ pub const HEAL_CONTROL_RPC_MAX_MESSAGE_SIZE: usize = heal_control::RESULT_MAX_SI
pub const HEAL_CONTROL_PROTOCOL_VERSION: u32 = 2;
pub const DYNAMIC_CONFIG_PROTOCOL_VERSION: u32 = 1;
pub const HEAL_CONTROL_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-heal-control-capability-v2\0";
pub const REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX: &[u8] = b"rustfs-tier-remote-version-state-capability-v1\0";
pub const TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE: usize = 64 * 1024;
pub const TIER_MUTATION_RPC_MAX_COMMIT_PAYLOAD_SIZE: usize = 1024;
pub const TIER_MUTATION_RPC_MAX_MESSAGE_SIZE: usize = TIER_MUTATION_RPC_MAX_PREPARE_PAYLOAD_SIZE + 4096;
@@ -197,6 +198,49 @@ pub fn is_heal_control_capability_probe(command: &[u8]) -> bool {
command.len() == HEAL_CONTROL_CAPABILITY_PROBE_PREFIX.len() + 16 && command.starts_with(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX)
}
pub fn remote_version_state_capability_probe(nonce: &[u8; 16]) -> Vec<u8> {
let mut probe = Vec::with_capacity(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + nonce.len());
probe.extend_from_slice(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX);
probe.extend_from_slice(nonce);
probe
}
pub fn is_remote_version_state_capability_probe(command: &[u8]) -> bool {
command.len() == REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX.len() + 16
&& command.starts_with(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX)
}
pub fn encode_remote_version_state_capability(
topology_member: &str,
process_epoch: &[u8; 16],
) -> Result<Vec<u8>, std::num::TryFromIntError> {
let topology_member = topology_member.as_bytes();
let mut result = Vec::with_capacity(8 + topology_member.len() + process_epoch.len());
result.extend_from_slice(&u64::try_from(topology_member.len())?.to_be_bytes());
result.extend_from_slice(topology_member);
result.extend_from_slice(process_epoch);
Ok(result)
}
pub fn decode_remote_version_state_capability(result: &[u8]) -> Result<(&str, &[u8; 16]), &'static str> {
let member_len = result
.get(..8)
.and_then(|value| value.try_into().ok())
.map(u64::from_be_bytes)
.ok_or("remote version state capability is truncated")?;
let member_len = usize::try_from(member_len).map_err(|_| "remote version state member length cannot be represented")?;
let member_end = 8_usize
.checked_add(member_len)
.ok_or("remote version state member length overflow")?;
let topology_member = std::str::from_utf8(result.get(8..member_end).ok_or("remote version state member is truncated")?)
.map_err(|_| "remote version state member is not UTF-8")?;
let process_epoch = result
.get(member_end..)
.and_then(|value| value.try_into().ok())
.ok_or("remote version state process epoch has an invalid length")?;
Ok((topology_member, process_epoch))
}
/// Builds the stable byte representation authenticated for a heal-control request.
///
/// This deliberately does not reuse protobuf encoding: mixed-version peers may
@@ -1656,10 +1700,12 @@ mod scanner_activity_tests {
#[cfg(test)]
mod heal_control_tests {
use super::{
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, canonical_heal_control_capability_ack,
canonical_heal_control_request_body, canonical_heal_control_response_body, heal_control_capability_probe,
HEAL_CONTROL_CAPABILITY_PROBE_PREFIX, HEAL_CONTROL_PROTOCOL_VERSION, REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX,
canonical_heal_control_capability_ack, canonical_heal_control_request_body, canonical_heal_control_response_body,
decode_remote_version_state_capability, encode_remote_version_state_capability, heal_control_capability_probe,
heal_control_coordinator_epoch, heal_control_execution_timeout, heal_control_execution_timeout_for,
internode_rpc_timeout, is_heal_control_capability_probe, normalize_internode_rpc_timeout,
internode_rpc_timeout, is_heal_control_capability_probe, is_remote_version_state_capability_probe,
normalize_internode_rpc_timeout, remote_version_state_capability_probe,
};
use crate::heal_control;
use std::time::Duration;
@@ -1716,6 +1762,29 @@ mod heal_control_tests {
assert!(!is_heal_control_capability_probe(HEAL_CONTROL_CAPABILITY_PROBE_PREFIX));
}
#[test]
fn remote_version_state_capability_probe_requires_exact_nonce() {
let probe = remote_version_state_capability_probe(&[7; 16]);
assert!(is_remote_version_state_capability_probe(&probe));
assert!(!is_remote_version_state_capability_probe(REMOTE_VERSION_STATE_CAPABILITY_PROBE_PREFIX));
}
#[test]
fn remote_version_state_capability_binds_member_and_process_epoch() {
let encoded =
encode_remote_version_state_capability("node-a:9000", &[7; 16]).expect("small capability response should encode");
assert_eq!(
decode_remote_version_state_capability(&encoded).expect("capability response should decode"),
("node-a:9000", &[7; 16])
);
assert!(decode_remote_version_state_capability(&encoded[..encoded.len() - 1]).is_err());
let mut invalid_utf8 =
encode_remote_version_state_capability("node-a", &[7; 16]).expect("small capability response should encode");
invalid_utf8[8] = 0xff;
assert!(decode_remote_version_state_capability(&invalid_utf8).is_err());
}
#[test]
fn canonical_response_binds_request_and_result() {
let baseline = canonical_heal_control_response_body(2, "abcdef", b"query", b"result").unwrap();
crates/s3select-api/Cargo.toml
+3
View File
@@ -25,6 +25,9 @@ keywords = ["s3-select", "api", "rustfs", "Minio", "object-store"]
categories = ["web-programming", "development-tools", "asynchronous"]
documentation = "https://docs.rs/rustfs-s3select-api/latest/rustfs_s3select_api/"
[lints]
workspace = true
[dependencies]
metrics = { workspace = true }
async-trait.workspace = true
crates/s3select-query/Cargo.toml
+3
View File
@@ -25,6 +25,9 @@ keywords = ["s3-select", "query-engine", "rustfs", "Minio", "data-retrieval"]
categories = ["web-programming", "development-tools", "data-structures"]
documentation = "https://docs.rs/rustfs-s3select-query/latest/rustfs_s3select_query/"
[lints]
workspace = true
[dependencies]
rustfs-s3select-api = { workspace = true }
async-recursion = { workspace = true }
crates/s3select-query/src/metadata/mod.rs
+1 -1
View File
@@ -76,7 +76,7 @@ impl ContextProviderExtension for MetadataProvider {
let table_handle = self.build_table_handle()?;
Ok(Arc::new(TableSourceAdapter::try_new(table_ref.clone(), table_name, table_handle)?))
Ok(Arc::new(TableSourceAdapter::try_new(table_ref, table_name, table_handle)?))
}
}
crates/scanner/Cargo.toml
+1 -1
View File
@@ -47,7 +47,7 @@ rmp-serde = { workspace = true }
hmac = { workspace = true }
sha2 = { workspace = true }
rustfs-filemeta = { workspace = true }
tokio-util = { workspace = true, features = ["io", "compat"] }
tokio-util = { workspace = true, features = ["io", "compat", "rt"] }
rustfs-ecstore = { workspace = true }
rustfs-storage-api = { workspace = true }
http = { workspace = true }
crates/scanner/src/data_usage_define.rs
+417 -17
View File
@@ -48,6 +48,7 @@ pub const DATA_USAGE_ROOT: &str = SLASH_SEPARATOR;
const DATA_USAGE_BLOOM_NAME: &str = ".bloomcycle.bin";
pub const DATA_USAGE_CACHE_NAME: &str = ".usage-cache.bin";
pub(crate) const DATA_USAGE_CACHE_KEY_FORMAT: u16 = 1;
const DATA_USAGE_CACHE_SAVE_RETRIES: u32 = 2;
const DATA_USAGE_CACHE_BACKUP_SAVE_TIMEOUT_SECS_MAX: u64 = 5;
const DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES: u32 = 0;
@@ -437,6 +438,8 @@ pub struct DataUsageCacheInfo {
pub snapshot_complete: bool,
#[serde(default)]
pub scan_plan_digest: Option<DataUsageScanPlanDigest>,
#[serde(default)]
pub cache_key_format: u16,
}
impl Serialize for DataUsageCacheInfo {
@@ -446,7 +449,7 @@ impl Serialize for DataUsageCacheInfo {
{
// Keep this metadata map-encoded so older readers can ignore fields
// appended by newer scanner versions during rolling upgrades.
let mut state = serializer.serialize_map(Some(15))?;
let mut state = serializer.serialize_map(Some(16))?;
state.serialize_entry("name", &self.name)?;
state.serialize_entry("next_cycle", &self.next_cycle)?;
state.serialize_entry("leader_epoch", &self.leader_epoch)?;
@@ -462,6 +465,7 @@ impl Serialize for DataUsageCacheInfo {
state.serialize_entry("source", &self.source)?;
state.serialize_entry("snapshot_complete", &self.snapshot_complete)?;
state.serialize_entry("scan_plan_digest", &self.scan_plan_digest)?;
state.serialize_entry("cache_key_format", &self.cache_key_format)?;
state.end()
}
}
@@ -500,19 +504,34 @@ impl DataUsageCache {
let source_matches = self.info.source == Some(source);
let plan_matches = self.info.scan_plan_digest == Some(scan_plan_digest);
let reusable = self.info.name == name
let metadata_is_reusable = self.info.name == name
&& self.info.leader_epoch == leader_epoch
&& plan_matches
&& (source_matches || (!require_source && self.info.source.is_none()));
&& (source_matches || (!require_source && self.info.source.is_none()))
&& self.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
let reusable = metadata_is_reusable
&& (self.cache.is_empty()
|| if name == DATA_USAGE_ROOT || self.info.snapshot_complete {
self.checked_flatten_complete_scope(name).is_some()
} else {
self.checked_flatten(name).is_some()
});
if !reusable {
let pending_heals = if self.info.name == name {
std::mem::take(&mut self.info.pending_heals)
} else {
Vec::new()
};
*self = Self::default();
self.info.name = name.to_string();
self.info.pending_heals = pending_heals;
}
self.info.next_cycle = next_cycle;
self.info.leader_epoch = leader_epoch;
self.info.source = Some(source);
self.info.scan_plan_digest = Some(scan_plan_digest);
self.info.cache_key_format = DATA_USAGE_CACHE_KEY_FORMAT;
self.info.snapshot_complete = false;
if reusable {
DataUsageCachePrepareOutcome::Reused
@@ -576,36 +595,60 @@ impl DataUsageCache {
}
pub(crate) fn checked_flatten(&self, path: &str) -> Option<DataUsageEntry> {
self.checked_flatten_inner(path).map(|(entry, _)| entry)
}
pub(crate) fn checked_flatten_complete(&self, path: &str) -> Option<DataUsageEntry> {
self.checked_flatten_inner(path)
.filter(|(_, visited)| *visited == self.cache.len())
.map(|(entry, _)| entry)
}
pub(crate) fn checked_flatten_complete_scope(&self, path: &str) -> Option<DataUsageEntry> {
if path == DATA_USAGE_ROOT {
return self.checked_flatten_complete(path);
}
let (entry, visited) = self.checked_flatten_inner(path)?;
let root_parent_only = {
let path_key = hash_path(path).key();
self.cache
.get(DATA_USAGE_ROOT)
.is_some_and(|root| root_is_parent_only(root, &path_key))
};
let expected_entries = self.cache.len().saturating_sub(usize::from(root_parent_only));
(visited == expected_entries).then_some(entry)
}
fn checked_flatten_inner(&self, path: &str) -> Option<(DataUsageEntry, usize)> {
let root_key = hash_path(path).key();
let root = self.cache.get(&root_key)?;
let mut visited = HashSet::from([root_key]);
let mut pending = root.children.iter().map(|child| (child.clone(), 1usize)).collect::<Vec<_>>();
let (root_key, root) = self.cache.get_key_value(&root_key)?;
if root.compacted && !root.children.is_empty() {
return None;
}
let mut visited = HashSet::from([root_key.as_str()]);
let mut pending = root.children.iter().map(|child| (child.as_str(), 1usize)).collect::<Vec<_>>();
let mut flattened = DataUsageEntry::default();
let mut root_entry = root.clone();
root_entry.children.clear();
if !flattened.checked_merge(&root_entry) {
if !flattened.checked_merge(root) {
return None;
}
flattened.compacted = root.compacted;
while let Some((key, depth)) = pending.pop() {
if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key.clone()) {
if depth > MAX_DATA_USAGE_CACHE_DEPTH || !visited.insert(key) {
return None;
}
let entry = self.cache.get(&key)?;
if depth == MAX_DATA_USAGE_CACHE_DEPTH && !entry.children.is_empty() {
let entry = self.cache.get(key)?;
if (entry.compacted || depth == MAX_DATA_USAGE_CACHE_DEPTH) && !entry.children.is_empty() {
return None;
}
pending.extend(entry.children.iter().map(|child| (child.clone(), depth + 1)));
pending.extend(entry.children.iter().map(|child| (child.as_str(), depth + 1)));
let mut child_entry = entry.clone();
child_entry.children.clear();
if !flattened.checked_merge(&child_entry) {
if !flattened.checked_merge(entry) {
return None;
}
}
Some(flattened)
Some((flattened, visited.len()))
}
fn flatten_with_guard(&self, root: &DataUsageEntry, visited: &mut HashSet<String>, depth: usize) -> DataUsageEntry {
@@ -1524,6 +1567,18 @@ fn mark_with_depth(duc: &DataUsageCache, entry: &DataUsageEntry, found: &mut Has
}
}
fn root_is_parent_only(root: &DataUsageEntry, child: &str) -> bool {
root.children.len() == 1
&& root.children.contains(child)
&& root.size == 0
&& root.objects == 0
&& root.versions == 0
&& root.delete_markers == 0
&& root.replication_stats.is_none()
&& !root.compacted
&& root.failed_objects == 0
}
/// Trait for storage-specific operations on DataUsageCache
#[async_trait::async_trait]
pub trait DataUsageCacheStorage {
@@ -2287,6 +2342,7 @@ mod tests {
assert!(decoded.source.is_none());
assert!(!decoded.snapshot_complete);
assert!(decoded.scan_plan_digest.is_none());
assert_eq!(decoded.cache_key_format, 0);
}
#[test]
@@ -2328,6 +2384,7 @@ mod tests {
assert!(decoded.source.is_none());
assert!(!decoded.snapshot_complete);
assert!(decoded.scan_plan_digest.is_none());
assert_eq!(decoded.cache_key_format, 0);
}
#[test]
@@ -2363,6 +2420,7 @@ mod tests {
source: Some(DataUsageCacheSource::new(1, 2)),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -2381,6 +2439,7 @@ mod tests {
assert_eq!(current.info.source, Some(DataUsageCacheSource::new(1, 2)));
assert!(current.info.snapshot_complete);
assert_eq!(current.info.scan_plan_digest, Some(TEST_PLAN_DIGEST));
assert_eq!(current.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
assert_eq!(current.find("bucket").map(|entry| entry.objects), Some(3));
let decoded: OldDataUsageCache = rmp_serde::from_slice(&buf).expect("Old reader failed to deserialize new cache");
@@ -2442,6 +2501,7 @@ mod tests {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -2466,6 +2526,254 @@ mod tests {
assert!(!cache.info.snapshot_complete);
}
#[test]
fn data_usage_cache_prepare_for_scan_preserves_pending_heal_only_progress() {
let source = DataUsageCacheSource::new(1, 0);
let pending_heal = PendingScannerHeal {
kind: PendingScannerHealKind::Object,
bucket: "bucket".to_string(),
object: Some("prefix/object".to_string()),
version_id: Some("version-a".to_string()),
scan_mode: HealScanMode::Deep,
first_seen: 1,
last_attempt: 2,
attempts: 3,
last_admission_result: "full".to_string(),
last_admission_reason: "queue_full".to_string(),
};
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
pending_heals: vec![pending_heal.clone()],
..Default::default()
},
..Default::default()
};
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reused);
assert_eq!(cache.info.pending_heals, vec![pending_heal]);
assert!(cache.cache.is_empty());
assert!(!cache.info.snapshot_complete);
}
#[test]
fn data_usage_cache_prepare_for_scan_preserves_namespace_pending_heals_during_key_format_rebuild() {
let source = DataUsageCacheSource::new(1, 0);
let pending_heal = PendingScannerHeal {
kind: PendingScannerHealKind::Object,
bucket: "bucket".to_string(),
object: Some("prefix/object".to_string()),
version_id: Some("version-a".to_string()),
scan_mode: HealScanMode::Deep,
first_seen: 1,
last_attempt: 2,
attempts: 3,
last_admission_result: "full".to_string(),
last_admission_reason: "queue_full".to_string(),
};
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
pending_heals: vec![pending_heal.clone()],
..Default::default()
},
..Default::default()
};
cache.replace(DATA_USAGE_ROOT, "", DataUsageEntry::default());
let outcome = cache.prepare_for_scan(DATA_USAGE_ROOT, 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert_eq!(cache.info.pending_heals, vec![pending_heal]);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_drops_pending_heals_from_a_different_scope() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "old-bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
pending_heals: vec![PendingScannerHeal {
kind: PendingScannerHealKind::Object,
bucket: "old-bucket".to_string(),
object: Some("prefix/object".to_string()),
version_id: None,
scan_mode: HealScanMode::Normal,
first_seen: 1,
last_attempt: 2,
attempts: 3,
last_admission_result: "full".to_string(),
last_admission_reason: "queue_full".to_string(),
}],
..Default::default()
},
..Default::default()
};
let outcome = cache.prepare_for_scan("new-bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert_eq!(cache.info.name, "new-bucket");
assert!(cache.info.pending_heals.is_empty());
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_unknown_key_format() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT + 1,
..Default::default()
},
..Default::default()
};
cache.replace(
"bucket",
"",
DataUsageEntry {
objects: 3,
..Default::default()
},
);
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_persisted_windows_key_mismatch() {
let source = DataUsageCacheSource::new(1, 0);
let root_key = hash_path("bucket").key();
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
cache.cache.insert(
root_key,
DataUsageEntry {
children: HashSet::from(["bucket/prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
let encoded = cache.marshal_msg().expect("legacy Windows cache should serialize");
let mut decoded = DataUsageCache::unmarshal(&encoded).expect("legacy Windows cache should deserialize");
let outcome = decoded.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(decoded.cache.is_empty());
assert_eq!(decoded.info.name, "bucket");
assert_eq!(decoded.info.next_cycle, 8);
assert_eq!(decoded.info.source, Some(source));
assert!(!decoded.info.snapshot_complete);
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_current_cache_with_dangling_child() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
hash_path("bucket").key(),
DataUsageEntry {
children: HashSet::from([hash_path("bucket/missing").key()]),
..Default::default()
},
);
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_resets_complete_bucket_cache_with_detached_entry() {
let source = DataUsageCacheSource::new(1, 0);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
hash_path("bucket").key(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
cache.cache.insert(
hash_path("bucket/detached").key(),
DataUsageEntry {
objects: 2,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
assert!(
cache.checked_flatten_complete("bucket").is_none(),
"complete bucket cache reuse must reject detached entries"
);
let outcome = cache.prepare_for_scan("bucket", 8, 0, source, TEST_PLAN_DIGEST, true);
assert_eq!(outcome, DataUsageCachePrepareOutcome::Reset);
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn data_usage_cache_prepare_for_scan_rejects_legacy_cache_without_a_bucket_plan() {
let source = DataUsageCacheSource::new(0, 0);
@@ -2836,6 +3144,98 @@ mod tests {
);
}
#[test]
fn checked_flatten_complete_rejects_detached_entries() {
let mut cache = DataUsageCache::default();
cache.cache.insert(
hash_path("bucket").key(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
cache.cache.insert(
hash_path("bucket/detached").key(),
DataUsageEntry {
objects: 2,
..Default::default()
},
);
assert_eq!(
cache.checked_flatten("bucket").map(|entry| entry.objects),
Some(1),
"subtree flattening may ignore entries outside the requested subtree"
);
assert!(
cache.checked_flatten_complete("bucket").is_none(),
"an authoritative cache root must reach every persisted entry"
);
}
#[test]
fn checked_flatten_rejects_compacted_entries_with_children() {
let root_key = hash_path("bucket").key();
let child_key = hash_path("bucket/prefix").key();
let mut cache = DataUsageCache::default();
cache.cache.insert(
root_key,
DataUsageEntry {
children: HashSet::from([child_key.clone()]),
compacted: true,
..Default::default()
},
);
cache.cache.insert(
child_key,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(
cache.checked_flatten_complete("bucket").is_none(),
"a compacted entry cannot retain child links without double-counting"
);
}
#[test]
fn checked_flatten_rejects_compacted_descendants_with_children() {
let root_key = hash_path("bucket").key();
let child_key = hash_path("bucket/prefix").key();
let grandchild_key = hash_path("bucket/prefix/object").key();
let mut cache = DataUsageCache::default();
cache.cache.insert(
root_key,
DataUsageEntry {
children: HashSet::from([child_key.clone()]),
..Default::default()
},
);
cache.cache.insert(
child_key,
DataUsageEntry {
objects: 1,
children: HashSet::from([grandchild_key.clone()]),
compacted: true,
..Default::default()
},
);
cache.cache.insert(
grandchild_key,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(
cache.checked_flatten("bucket").is_none(),
"a compacted descendant cannot retain child links without double-counting"
);
}
#[test]
fn checked_flatten_accepts_depth_limit_and_rejects_deeper_tree() {
let root_key = hash_path("bucket").key();
crates/scanner/src/remote_scanner.rs
+34 -30
View File
@@ -14,8 +14,8 @@
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig};
use crate::scanner_io::{
ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, cache_snapshot_is_current, scanner_cache_lock_resource,
scanner_cache_lock_timeout, scanner_set_disk_inventory,
DataUsageCacheScanState, ScannerDiskScanOutcome, ScannerIODisk, cache_root_entry_info, current_cache_root_or_prepare,
scanner_cache_lock_resource, scanner_cache_lock_timeout, scanner_set_disk_inventory,
};
use crate::storage_api::owner::NS_SCANNER_PROTOCOL_VERSION;
use crate::storage_api::scan::NamespaceLocking as _;
@@ -966,32 +966,34 @@ async fn scan_and_persist_local_bucket(
let revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
})?;
if cache_snapshot_is_current(&cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest) {
if guard.is_lock_lost() {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner cache lock was lost before reusing the current snapshot",
));
let scan_state = current_cache_root_or_prepare(&mut cache, &bucket, source, next_cycle, leader_epoch, scan_plan_digest, true);
match scan_state {
DataUsageCacheScanState::Current(usage) => {
if guard.is_lock_lost() {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner cache lock was lost before reusing the current snapshot",
));
}
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
source,
scan_plan_digest,
usage: *usage,
pending_maintenance_work: !cache.info.pending_heals.is_empty(),
})));
}
return Ok(RemoteScannerFrameResult::Complete(Box::new(RemoteScannerComplete {
source,
scan_plan_digest,
usage: cache_root_entry_info(&cache)
.map_err(|err| RemoteScannerServerError::worker(format!("remote namespace scanner cache is corrupt: {err}")))?,
pending_maintenance_work: !cache.info.pending_heals.is_empty(),
})));
}
match cache.prepare_for_scan(&bucket, next_cycle, leader_epoch, source, scan_plan_digest, true) {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
return Ok(RemoteScannerFrameResult::CycleAhead {
required_cycle: cache.info.next_cycle,
});
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner rejected work from an older leader epoch",
));
}
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
DataUsageCacheScanState::Prepared { outcome, .. } => match outcome {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
return Ok(RemoteScannerFrameResult::CycleAhead {
required_cycle: cache.info.next_cycle,
});
}
DataUsageCachePrepareOutcome::RejectedNewerLeader => {
return Err(RemoteScannerServerError::worker(
"remote namespace scanner rejected work from an older leader epoch",
));
}
DataUsageCachePrepareOutcome::Reused | DataUsageCachePrepareOutcome::Reset => {}
},
}
cache.info.skip_healing = skip_healing;
@@ -1971,6 +1973,7 @@ mod tests {
#[test]
fn request_rejects_empty_truncated_oversized_and_wrong_version_payloads() {
assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
assert!(decode_remote_scanner_request(&[]).is_err());
let mut body = rmp_serde::to_vec_named(&test_request(Uuid::new_v4())).expect("request should encode");
@@ -1980,7 +1983,7 @@ mod tests {
let oversized = vec![0_u8; NS_SCANNER_MAX_REQUEST_BODY_SIZE + 1];
assert!(decode_remote_scanner_request(&oversized).is_err());
for version in [NS_SCANNER_PROTOCOL_VERSION - 1, NS_SCANNER_PROTOCOL_VERSION + 1] {
for version in [2, 4] {
let mut wrong_version = test_request(Uuid::new_v4());
wrong_version.version = version;
let body = rmp_serde::to_vec_named(&wrong_version).expect("request should encode");
@@ -2885,14 +2888,15 @@ mod tests {
#[tokio::test]
async fn wrong_frame_version_and_sequence_are_rejected() {
assert_eq!(NS_SCANNER_PROTOCOL_VERSION, 3);
let request_id = Uuid::new_v4();
let auth = FrameAuthenticator::for_test(request_id);
let frame = RemoteScannerFrame::progress(RemoteScannerProgress::default());
let payload = rmp_serde::to_vec_named(&frame).expect("frame should encode");
for (version, sequence, expected_error) in [
(NS_SCANNER_PROTOCOL_VERSION - 1, 0, "unsupported remote namespace scanner frame version"),
(NS_SCANNER_PROTOCOL_VERSION + 1, 0, "unsupported remote namespace scanner frame version"),
(2, 0, "unsupported remote namespace scanner frame version"),
(4, 0, "unsupported remote namespace scanner frame version"),
(NS_SCANNER_PROTOCOL_VERSION, 1, "frame sequence is invalid"),
] {
let envelope = RemoteScannerFrameEnvelope {
crates/scanner/src/scanner.rs
+342 -47
View File
@@ -14,6 +14,8 @@
use std::collections::BTreeMap;
use std::future::Future;
#[cfg(test)]
use std::sync::Mutex as StdMutex;
use std::sync::{Arc, LazyLock, RwLock};
use crate::ScannerObjectIO;
@@ -38,8 +40,8 @@ use bytes::Bytes;
use chrono::{DateTime, Utc};
use rustfs_common::heal_channel::HealScanMode;
use rustfs_common::metrics::{
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
CurrentCycle, Metric, Metrics, ScanCyclePartialReason, ScanCycleWorkSnapshot, ScannerUsageSaveResult, ScannerWorkSource,
emit_scan_cycle_complete, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
};
use rustfs_config::ScannerSpeed;
#[cfg(test)]
@@ -50,9 +52,12 @@ use rustfs_config::{
use rustfs_config::{ENV_SCANNER_CYCLE, ENV_SCANNER_SPEED, ENV_SCANNER_START_DELAY_SECS};
use serde::{Deserialize, Serialize};
use sha2::{Digest as _, Sha256};
#[cfg(test)]
use tokio::sync::Notify;
use tokio::sync::mpsc;
use tokio::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;
use tokio_util::task::AbortOnDropHandle;
use tracing::{debug, error, info, instrument, warn};
use crate::storage_api::scan::{
@@ -93,6 +98,44 @@ const SCANNER_CYCLE_STATE_MAGIC: &[u8; 8] = b"RSCYC001";
const SCANNER_CYCLE_STATE_HEADER_LEN: usize = 24;
#[cfg(test)]
const ENV_SCANNER_START_DELAY_SECS_DEPRECATED: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS";
#[cfg(test)]
type ScannerCycleStatePersistTestHook = (u64, Arc<Notify>);
#[cfg(test)]
static SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK: LazyLock<StdMutex<Option<ScannerCycleStatePersistTestHook>>> =
LazyLock::new(|| StdMutex::new(None));
#[cfg(test)]
struct ScannerCycleStatePersistTestHookGuard;
#[cfg(test)]
impl Drop for ScannerCycleStatePersistTestHookGuard {
fn drop(&mut self) {
*SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = None;
}
}
#[cfg(test)]
fn set_scanner_cycle_state_persist_test_hook(leader_epoch: u64, reached: Arc<Notify>) -> ScannerCycleStatePersistTestHookGuard {
*SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner()) = Some((leader_epoch, reached));
ScannerCycleStatePersistTestHookGuard
}
#[cfg(test)]
fn notify_scanner_cycle_state_persist_test_hook(leader_epoch: u64) {
let reached = SCANNER_CYCLE_STATE_PERSIST_TEST_HOOK
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
.as_ref()
.filter(|(expected_epoch, _)| *expected_epoch == leader_epoch)
.map(|(_, reached)| reached.clone());
if let Some(reached) = reached {
reached.notify_one();
}
}
#[derive(Debug, thiserror::Error)]
enum ScannerCycleStateError {
@@ -1691,10 +1734,10 @@ fn data_usage_persist_timeout() -> Duration {
DataUsageCache::persistence_timeout()
}
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle) {
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
global_metrics().set_cycle(Some(cycle_info.clone())).await;
cycle_metrics_guard.finish(cycle_info.clone()).await;
}
fn encode_scanner_cycle_state(cycle_info: &CurrentCycle, leader_epoch: u64) -> Result<Vec<u8>, ScannerCycleStateError> {
@@ -2218,6 +2261,8 @@ async fn persist_scanner_cycle_state(
return false;
}
#[cfg(test)]
notify_scanner_cycle_state_persist_test_hook(leader_epoch);
match save_config_with_preconditions(storeapi.clone(), &DATA_USAGE_BLOOM_NAME_PATH, buf.clone(), revision.preconditions())
.await
{
@@ -2340,7 +2385,6 @@ async fn persist_scanner_cycle_state(
if persisted_cycle.next >= cycle_info.next {
*cycle_info = persisted_cycle;
global_metrics().set_cycle(Some(cycle_info.clone())).await;
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_PERSIST_STATE,
@@ -2406,6 +2450,7 @@ async fn finalize_partial_scan_cycle(
cycle_info: &mut CurrentCycle,
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool {
// A budget-limited cycle is deliberate pacing, not a failure. The cycle counter
// must still advance (and persist) because per-bucket next_cycle is stamped from
@@ -2422,11 +2467,14 @@ async fn finalize_partial_scan_cycle(
error = %err,
"Scanner partial cycle could not advance"
);
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await;
return false;
}
mark_scan_cycle_idle(cycle_info).await;
persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted
}
async fn persist_required_scanner_cycle_floor(
@@ -2436,6 +2484,7 @@ async fn persist_required_scanner_cycle_floor(
revision: &mut DataUsageCacheRevision,
leader_epoch: u64,
required_cycle: u64,
cycle_metrics_guard: &mut ScannerCycleMetricsGuard,
) -> bool {
if required_cycle <= cycle_info.current || required_cycle == u64::MAX {
error!(
@@ -2448,13 +2497,16 @@ async fn persist_required_scanner_cycle_floor(
state = "invalid_cache_cycle_floor",
"Scanner cache cycle floor is invalid"
);
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, cycle_metrics_guard).await;
return false;
}
cycle_info.next = cycle_info.next.max(required_cycle);
mark_scan_cycle_idle(cycle_info).await;
persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await
cycle_info.current = 0;
global_metrics().clear_current_scan_mode();
let persisted = persist_scanner_cycle_state(ctx, storeapi, cycle_info, revision, leader_epoch).await;
cycle_metrics_guard.finish(cycle_info.clone()).await;
persisted
}
async fn await_scanner_cycle_with_lock_fence<Cycle, LockLost>(
@@ -2513,7 +2565,7 @@ async fn run_data_scanner_cycle(
let now = Instant::now();
cycle_info.started = Utc::now();
global_metrics().set_cycle(Some(cycle_info.clone())).await;
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
let mut background_heal_info = read_background_heal_info(storeapi.clone()).await;
@@ -2564,14 +2616,14 @@ async fn run_data_scanner_cycle(
"Scanner cycle could not capture the data usage persistence baseline"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
}
};
let (sender, receiver) = mpsc::channel::<DataUsageInfo>(1);
let storeapi_clone = storeapi.clone();
let ctx_clone = ctx.clone();
let mut usage_persist_task = tokio::spawn(async move {
let mut usage_persist_task = AbortOnDropHandle::new(tokio::spawn(async move {
store_data_usage_in_backend_with_outcome_for_epoch_and_baseline(
ctx_clone,
storeapi_clone,
@@ -2580,10 +2632,9 @@ async fn run_data_scanner_cycle(
Some(usage_persist_baseline),
)
.await
});
}));
let done_cycle = Metrics::time(Metric::ScanCycle);
let cycle_work_start = global_metrics().start_scan_cycle_work();
let cycle_budget = ScannerCycleBudget::new(ctx, cycle_budget_config);
let scan_result = storeapi
.clone()
@@ -2640,7 +2691,6 @@ async fn run_data_scanner_cycle(
}
};
let unresolved_heal_work = global_metrics().current_scan_cycle_has_unresolved_heal_work();
global_metrics().finish_scan_cycle_work(cycle_work_start);
let scan_cycle_result = match scan_result {
Ok(result) => result,
@@ -2663,7 +2713,7 @@ async fn run_data_scanner_cycle(
{
save_background_heal_info(storeapi.clone(), new_heal_info).await;
}
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
}
};
@@ -2678,7 +2728,7 @@ async fn run_data_scanner_cycle(
"Scanner cycle stopped before committing cycle state"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
}
if let Some(required_cycle) = scan_cycle_result.required_cycle_floor() {
@@ -2700,6 +2750,7 @@ async fn run_data_scanner_cycle(
cycle_revision,
leader_epoch,
required_cycle,
&mut cycle_metrics_guard,
)
.await
{
@@ -2719,7 +2770,7 @@ async fn run_data_scanner_cycle(
"Scanner cycle completed without a durable data usage snapshot"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
}
if budget_elapsed {
@@ -2743,7 +2794,16 @@ async fn run_data_scanner_cycle(
scan_cycle_partial_reason(budget_reason),
scan_cycle_partial_source(budget_reason),
);
return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
return if finalize_partial_scan_cycle(
ctx,
storeapi.clone(),
cycle_info,
cycle_revision,
leader_epoch,
&mut cycle_metrics_guard,
)
.await
{
ScannerCycleOutcome::Partial
} else {
ScannerCycleOutcome::Failed
@@ -2792,7 +2852,7 @@ async fn run_data_scanner_cycle(
"Scanner cycle completed without a durable data usage snapshot"
);
emit_scan_cycle_complete(false, cycle_start.elapsed());
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
return ScannerCycleOutcome::Failed;
}
ScannerCycleOutcome::Partial => {
@@ -2818,7 +2878,16 @@ async fn run_data_scanner_cycle(
);
}
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
return if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
return if finalize_partial_scan_cycle(
ctx,
storeapi.clone(),
cycle_info,
cycle_revision,
leader_epoch,
&mut cycle_metrics_guard,
)
.await
{
ScannerCycleOutcome::Partial
} else {
ScannerCycleOutcome::Failed
@@ -2834,7 +2903,16 @@ async fn run_data_scanner_cycle(
state = "superseded",
"Scanner cycle usage snapshot was superseded by concurrent namespace activity"
);
if finalize_partial_scan_cycle(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
if finalize_partial_scan_cycle(
ctx,
storeapi.clone(),
cycle_info,
cycle_revision,
leader_epoch,
&mut cycle_metrics_guard,
)
.await
{
emit_scan_cycle_superseded(cycle_start.elapsed());
return ScannerCycleOutcome::Superseded;
}
@@ -2853,7 +2931,7 @@ async fn run_data_scanner_cycle(
error = %err,
"Scanner completed cycle could not advance"
);
mark_scan_cycle_idle(cycle_info).await;
mark_scan_cycle_idle(cycle_info, &mut cycle_metrics_guard).await;
emit_scan_cycle_complete(false, cycle_start.elapsed());
return ScannerCycleOutcome::Failed;
}
@@ -2862,9 +2940,8 @@ async fn run_data_scanner_cycle(
global_metrics().clear_current_scan_mode();
retain_recent_cycle_completions(&mut cycle_info.cycle_completed);
global_metrics().set_cycle(Some(cycle_info.clone())).await;
if !persist_scanner_cycle_state(ctx, storeapi.clone(), cycle_info, cycle_revision, leader_epoch).await {
mark_scan_cycle_idle(cycle_info).await;
cycle_metrics_guard.finish(cycle_info.clone()).await;
emit_scan_cycle_complete(false, cycle_start.elapsed());
return ScannerCycleOutcome::Failed;
}
@@ -2888,9 +2965,37 @@ async fn run_data_scanner_cycle(
"Scanner cycle completed"
);
cycle_metrics_guard.finish(cycle_info.clone()).await;
scanner_cycle_outcome_with_pending_maintenance(ScannerCycleOutcome::Completed, pending_maintenance_work)
}
struct ScannerCycleMetricsGuard {
start: Option<ScanCycleWorkSnapshot>,
}
impl ScannerCycleMetricsGuard {
async fn new(cycle: CurrentCycle) -> Self {
Self {
start: Some(global_metrics().start_scan_cycle_work_with_cycle(cycle).await),
}
}
async fn finish(&mut self, cycle: CurrentCycle) {
if let Some(start) = self.start {
global_metrics().finish_scan_cycle_work_with_cycle(start, cycle).await;
self.start = None;
}
}
}
impl Drop for ScannerCycleMetricsGuard {
fn drop(&mut self) {
if let Some(start) = self.start.take() {
global_metrics().finish_scan_cycle_work(start);
}
}
}
async fn record_scanner_leader_lock_lost(message: &'static str) {
reset_scanner_cycle_schedule();
record_scanner_leader_lock_state("lost");
@@ -3443,7 +3548,7 @@ enum DataUsagePersistTaskResult {
async fn wait_for_data_usage_persist_task(
ctx: &CancellationToken,
task: &mut tokio::task::JoinHandle<DataUsagePersistOutcome>,
task: &mut AbortOnDropHandle<DataUsagePersistOutcome>,
timeout: Duration,
) -> DataUsagePersistTaskResult {
tokio::select! {
@@ -3840,8 +3945,9 @@ mod tests {
use super::*;
use crate::EcstoreResult;
use crate::{
ScannerGetObjectReader as GetObjectReader, ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions,
ScannerPutObjReader as PutObjReader,
Endpoint, EndpointServerPools, Endpoints, InstanceContext, PoolEndpoints, ScannerGetObjectReader as GetObjectReader,
ScannerObjectInfo as ObjectInfo, ScannerObjectOptions as ObjectOptions, ScannerPutObjReader as PutObjReader,
init_bucket_metadata_sys_for_scanner_tests, init_ecstore_config_for_scanner_tests, init_local_disks_with_instance_ctx,
};
use serial_test::serial;
use std::collections::HashMap;
@@ -3853,6 +3959,47 @@ mod tests {
const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
async fn setup_scanner_cycle_store() -> (tempfile::TempDir, Arc<ECStore>) {
init_ecstore_config_for_scanner_tests();
let temp_dir = tempfile::tempdir().expect("scanner cycle test directory should be created");
let mut endpoints = Vec::new();
for disk_index in 0..4 {
let disk_path = temp_dir.path().join(format!("disk{disk_index}"));
tokio::fs::create_dir_all(&disk_path)
.await
.expect("scanner cycle test disk should be created");
let mut endpoint =
Endpoint::try_from(disk_path.to_str().expect("disk path should be utf8")).expect("endpoint should parse");
endpoint.set_pool_index(0);
endpoint.set_set_index(0);
endpoint.set_disk_index(disk_index);
endpoints.push(endpoint);
}
let endpoint_pools = EndpointServerPools::from(vec![PoolEndpoints {
legacy: false,
set_count: 1,
drives_per_set: 4,
endpoints: Endpoints::from(endpoints),
cmd_line: "scanner-cycle-metrics".to_string(),
platform: format!("OS: {} | Arch: {}", std::env::consts::OS, std::env::consts::ARCH),
}]);
let instance_ctx = Arc::new(InstanceContext::new());
init_local_disks_with_instance_ctx(&instance_ctx, endpoint_pools.clone())
.await
.expect("scanner cycle test disks should initialize");
let store = ECStore::new_with_instance_ctx(
"127.0.0.1:0".parse().expect("test address should parse"),
endpoint_pools,
CancellationToken::new(),
instance_ctx,
)
.await
.expect("scanner cycle test ECStore should initialize");
init_bucket_metadata_sys_for_scanner_tests(store.clone()).await;
(temp_dir, store)
}
fn assert_run_data_scanner_signature<F, Fut>(_run: F)
where
F: Fn(CancellationToken, Arc<ECStore>) -> Fut,
@@ -4312,9 +4459,9 @@ mod tests {
};
global_metrics().set_current_scan_mode(HealScanMode::Deep);
global_metrics().set_cycle(Some(cycle_info.clone())).await;
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
mark_scan_cycle_idle(&mut cycle_info).await;
mark_scan_cycle_idle(&mut cycle_info, &mut cycle_metrics_guard).await;
let published = global_metrics()
.get_cycle()
@@ -4330,6 +4477,123 @@ mod tests {
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn scanner_cycle_metrics_guard_covers_published_first_cycle_lifetime() {
let cycle_started = Utc::now() - chrono::Duration::seconds(5);
let mut cycle_info = CurrentCycle {
current: 0,
next: 1,
started: cycle_started,
..Default::default()
};
let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
let setup_report = global_metrics().report().await;
assert!(setup_report.current_cycle_active);
assert_eq!(setup_report.current_cycle, 0);
assert_eq!(setup_report.current_started, cycle_started);
mark_scan_cycle_idle(&mut cycle_info, &mut guard).await;
let idle_report = global_metrics().report().await;
assert!(!idle_report.current_cycle_active);
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn scanner_cycle_metrics_guard_keeps_active_cycle_published_during_finalization() {
let mut cycle_info = CurrentCycle {
current: 12,
next: 13,
started: Utc::now(),
..Default::default()
};
let mut guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
cycle_info.current = 0;
tokio::task::yield_now().await;
let finalizing_report = global_metrics().report().await;
assert!(finalizing_report.current_cycle_active);
assert_eq!(finalizing_report.current_cycle, 12);
guard.finish(cycle_info).await;
let idle_report = global_metrics().report().await;
assert!(!idle_report.current_cycle_active);
assert_eq!(idle_report.current_cycle, 0);
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn scanner_cycle_metrics_guard_drop_clears_activity() {
let guard = ScannerCycleMetricsGuard::new(CurrentCycle {
current: 12,
next: 13,
started: Utc::now(),
..Default::default()
})
.await;
assert!(global_metrics().report().await.current_cycle_active);
drop(guard);
assert!(!global_metrics().report().await.current_cycle_active);
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn run_data_scanner_cycle_publishes_activity_for_owner_lifetime() {
let (_temp_dir, store) = setup_scanner_cycle_store().await;
let ctx = CancellationToken::new();
let mut cycle_info = CurrentCycle::default();
let mut revision = DataUsageCacheRevision::Missing;
let leader_epoch = u64::MAX - 1;
let state_persist_reached = Arc::new(Notify::new());
let _state_persist_hook = set_scanner_cycle_state_persist_test_hook(leader_epoch, state_persist_reached.clone());
let state_lock = store
.new_ns_lock(RUSTFS_META_BUCKET, DATA_USAGE_BLOOM_NAME_PATH.as_str())
.await
.expect("scanner cycle state lock should be created");
let state_guard = state_lock
.get_write_lock(Duration::from_secs(1))
.await
.expect("scanner cycle state lock should be acquired");
let mut cycle = Box::pin(run_data_scanner_cycle(&ctx, &store, &mut cycle_info, &mut revision, leader_epoch));
let waker = std::task::Waker::noop();
let mut context = std::task::Context::from_waker(waker);
assert!(cycle.as_mut().poll(&mut context).is_pending());
let active = global_metrics().report().await;
assert!(active.current_cycle_active);
assert_eq!(active.current_cycle, 0);
tokio::time::timeout(Duration::from_secs(30), async {
tokio::select! {
outcome = &mut cycle => panic!("scanner cycle finished before state persistence was released: {outcome:?}"),
_ = state_persist_reached.notified() => {}
}
})
.await
.expect("scanner cycle should reach state persistence");
let finalizing = global_metrics().report().await;
assert!(finalizing.current_cycle_active);
drop(state_guard);
let outcome = tokio::time::timeout(Duration::from_secs(30), cycle)
.await
.expect("scanner cycle should finish");
assert!(matches!(
outcome,
ScannerCycleOutcome::Completed | ScannerCycleOutcome::CompletedWithPendingMaintenance
));
assert!(!global_metrics().report().await.current_cycle_active);
global_metrics().set_cycle(None).await;
}
#[tokio::test]
#[serial]
async fn test_finalize_partial_scan_cycle_advances_and_persists_counter() {
@@ -4342,8 +4606,11 @@ mod tests {
cycle_completed: vec![],
started: Utc::now(),
};
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
assert!(finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1).await);
assert!(
finalize_partial_scan_cycle(&ctx, store.clone(), &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await
);
assert_eq!(cycle_info.next, 13);
assert_eq!(cycle_info.current, 0);
@@ -4377,8 +4644,20 @@ mod tests {
cycle_completed: vec![],
started: Utc::now(),
};
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
assert!(persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 19).await);
assert!(
persist_required_scanner_cycle_floor(
&ctx,
store.clone(),
&mut cycle_info,
&mut revision,
7,
19,
&mut cycle_metrics_guard,
)
.await
);
assert_eq!(cycle_info.current, 0);
assert_eq!(cycle_info.next, 19);
@@ -4405,22 +4684,37 @@ mod tests {
cycle_completed: vec![],
started: Utc::now(),
};
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
assert!(!persist_required_scanner_cycle_floor(&ctx, store.clone(), &mut cycle_info, &mut revision, 7, 12).await);
assert_eq!(cycle_info.next, 12);
assert_eq!(revision, DataUsageCacheRevision::Missing);
assert!(
!persist_required_scanner_cycle_floor(
&ctx,
store.clone(),
&mut CurrentCycle {
current: 12,
next: 12,
..Default::default()
},
&mut cycle_info,
&mut revision,
7,
12,
&mut cycle_metrics_guard,
)
.await
);
assert_eq!(cycle_info.next, 12);
assert_eq!(revision, DataUsageCacheRevision::Missing);
let mut max_cycle_info = CurrentCycle {
current: 12,
next: 12,
..Default::default()
};
let mut max_cycle_metrics_guard = ScannerCycleMetricsGuard::new(max_cycle_info.clone()).await;
assert!(
!persist_required_scanner_cycle_floor(
&ctx,
store.clone(),
&mut max_cycle_info,
&mut revision,
7,
u64::MAX,
&mut max_cycle_metrics_guard,
)
.await
);
@@ -4685,8 +4979,9 @@ mod tests {
cycle_completed: vec![],
started: Utc::now(),
};
let mut cycle_metrics_guard = ScannerCycleMetricsGuard::new(cycle_info.clone()).await;
assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1).await);
assert!(!finalize_partial_scan_cycle(&ctx, store, &mut cycle_info, &mut revision, 1, &mut cycle_metrics_guard,).await);
assert_eq!(cycle_info.next, 13);
assert_eq!(cycle_info.current, 0);
assert_eq!(revision, DataUsageCacheRevision::Missing);
@@ -5943,10 +6238,10 @@ mod tests {
#[tokio::test]
async fn data_usage_persist_wait_aborts_when_scanner_is_cancelled() {
let ctx = CancellationToken::new();
let mut task = tokio::spawn(async {
let mut task = AbortOnDropHandle::new(tokio::spawn(async {
std::future::pending::<()>().await;
DataUsagePersistOutcome::Saved
});
}));
ctx.cancel();
let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(60)).await;
@@ -5958,10 +6253,10 @@ mod tests {
#[tokio::test(start_paused = true)]
async fn data_usage_persist_wait_aborts_after_timeout() {
let ctx = CancellationToken::new();
let mut task = tokio::spawn(async {
let mut task = AbortOnDropHandle::new(tokio::spawn(async {
std::future::pending::<()>().await;
DataUsagePersistOutcome::Saved
});
}));
let result = wait_for_data_usage_persist_task(&ctx, &mut task, Duration::from_secs(30)).await;
crates/scanner/src/scanner_folder.rs
+137 -26
View File
@@ -1508,6 +1508,9 @@ impl FolderScanner {
if entry.children.contains(&child) {
continue;
}
if !self.old_cache.cache.contains_key(&child) {
continue;
}
let child_hash = DataUsageHash(child.clone());
self.new_cache
@@ -2320,7 +2323,6 @@ impl FolderScanner {
}
tokio::task::yield_now().await;
let h = DataUsageHash(folder_item.name.clone());
into.add_child(&h);
self.record_scan_resume_hint(&folder_item.name);
// We scanned a folder, optionally send update.
@@ -2646,7 +2648,7 @@ impl FolderScanner {
tokio::task::yield_now().await;
} else {
let mut dst = DataUsageEntry::default();
let h = DataUsageHash(folder_item.name.clone());
let h = hash_path(&folder_item.name);
// Use Box::pin for recursive async call
let fut = Box::pin(self.scan_folder(ctx.clone(), folder_item.clone(), &mut dst));
@@ -2911,7 +2913,7 @@ mod tests {
use serial_test::serial;
#[cfg(unix)]
use std::os::unix::fs::{PermissionsExt, symlink};
use std::sync::atomic::AtomicBool;
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering};
use temp_env::{with_var, with_var_unset};
use uuid::Uuid;
@@ -3864,11 +3866,15 @@ mod tests {
}
async fn write_test_object_metadata(root: &std::path::Path, bucket: &str, object: &str) {
write_test_object_metadata_bytes(root, bucket, object, &metadata_for_object(bucket, object)).await;
}
async fn write_test_object_metadata_bytes(root: &std::path::Path, bucket: &str, object: &str, metadata: &[u8]) {
let object_dir = root.join(bucket).join(object);
tokio::fs::create_dir_all(&object_dir)
.await
.expect("failed to create test object directory");
tokio::fs::write(object_dir.join("xl.meta"), metadata_for_object(bucket, object))
tokio::fs::write(object_dir.join("xl.meta"), metadata)
.await
.expect("failed to write test object metadata");
}
@@ -4165,27 +4171,28 @@ mod tests {
async fn test_scan_folder_exits_when_abandoned_child_listing_finishes() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard::new(60, 100, &mut scanner, temp_dir.clone());
let _heal_responder = rustfs_common::heal_channel::init_heal_channel().ok().map(|mut heal_rx| {
tokio::spawn(async move {
while let Some(command) = heal_rx.recv().await {
if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
}
let heal_starts = Arc::new(AtomicUsize::new(0));
let heal_starts_clone = heal_starts.clone();
let mut heal_rx =
rustfs_common::heal_channel::init_heal_channel().expect("heal channel should initialize once for scanner tests");
let _heal_responder = tokio::spawn(async move {
while let Some(command) = heal_rx.recv().await {
if let rustfs_common::heal_channel::HealChannelCommand::Start { response_tx, .. } = command {
heal_starts_clone.fetch_add(1, Ordering::Relaxed);
let _ = response_tx.send(Ok(HealAdmissionResult::Accepted));
}
})
}
});
let bucket = "src-archive";
tokio::fs::create_dir_all(temp_dir.join(bucket))
.await
.expect("failed to create bucket directory");
let object = "snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4";
let metadata = metadata_for_object(bucket, object);
write_test_object_metadata_bytes(&temp_dir, bucket, object, &metadata).await;
let mut disks = vec![scanner.local_disk.clone()];
for disk_name in ["disk2", "disk3", "disk4"] {
let disk_root = temp_dir.join(disk_name);
tokio::fs::create_dir_all(disk_root.join(bucket))
.await
.expect("failed to create extra disk bucket directory");
write_test_object_metadata_bytes(&disk_root, bucket, object, &metadata).await;
let endpoint =
Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("failed to create extra disk endpoint");
let disk = new_disk(
@@ -4204,7 +4211,7 @@ mod tests {
scanner.disks = disks;
scanner.disks_quorum = 2;
scanner.old_cache.replace(
"src-archive/snapshots/37b3f20d941e2f5e6d99114d9bb2f3e67a8a2e5c9c4c5a1b0d6e7f8091a2b3c4",
&format!("{bucket}/{object}"),
bucket,
DataUsageEntry {
objects: 1,
@@ -4219,13 +4226,17 @@ mod tests {
object_heal_prob_div: 1,
};
tokio::time::timeout(
Duration::from_millis(200),
scanner.scan_folder(CancellationToken::new(), folder, &mut into),
)
.await
.expect("scan_folder should not hang after list_path_raw finishes")
.expect("scan_folder should finish successfully");
tokio::time::timeout(Duration::from_secs(2), scanner.scan_folder(CancellationToken::new(), folder, &mut into))
.await
.expect("scan_folder should not hang after list_path_raw finishes")
.expect("scan_folder should finish successfully");
let root = scanner
.new_cache
.checked_flatten(bucket)
.expect("healed cache must contain canonical child links");
assert_eq!(root.objects, 1);
assert!(heal_starts.load(Ordering::Relaxed) > 0, "test must execute the heal child-link path");
}
#[tokio::test]
@@ -4774,7 +4785,7 @@ mod tests {
.expect("unbounded scan should finish after partial progress");
let root = result
.size_recursive("bucket")
.checked_flatten("bucket")
.expect("completed cache should retain bucket usage");
assert_eq!(root.objects, 5);
assert!(result.info.snapshot_complete);
@@ -4828,6 +4839,106 @@ mod tests {
);
}
#[tokio::test]
#[serial]
async fn test_partial_entry_does_not_carry_missing_old_child() {
let (mut scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir),
};
let root_hash = hash_path("bucket");
scanner.old_cache.cache.insert(
root_hash.key(),
DataUsageEntry {
children: HashSet::from([hash_path("bucket/missing").key()]),
..Default::default()
},
);
let mut partial = DataUsageEntry {
objects: 2,
size: 2,
..Default::default()
};
scanner.carry_forward_old_children(&root_hash, &mut partial);
scanner.new_cache.replace_hashed(&root_hash, &None, &partial);
assert!(partial.children.is_empty());
let flattened = scanner
.new_cache
.checked_flatten("bucket")
.expect("a partial cache must not retain dangling child links");
assert_eq!(flattened.objects, 2);
assert_eq!(flattened.size, 2);
}
#[tokio::test]
#[serial]
async fn test_legacy_windows_cache_rebuilds_and_round_trips_portable_keys() {
let (scanner, temp_dir) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(temp_dir.clone()),
};
write_test_object_metadata(&temp_dir, "bucket", "prefix/object").await;
let source = crate::data_usage_define::DataUsageCacheSource::new(0, 0);
let scan_plan_digest = crate::data_usage_define::DataUsageScanPlanDigest([9; 32]);
let mut legacy = DataUsageCache {
info: crate::data_usage_define::DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 7,
source: Some(source),
scan_plan_digest: Some(scan_plan_digest),
..Default::default()
},
..Default::default()
};
legacy.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket\\prefix".to_string()]),
..Default::default()
},
);
legacy.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
let encoded = legacy.marshal_msg().expect("legacy cache should serialize");
let mut migrated = DataUsageCache::unmarshal(&encoded).expect("legacy cache should deserialize");
assert_eq!(
migrated.prepare_for_scan("bucket", 8, 0, source, scan_plan_digest, true),
crate::data_usage_define::DataUsageCachePrepareOutcome::Reset
);
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new(&parent, Default::default());
let rebuilt = scan_data_folder(
budget.token(),
budget,
vec![scanner.local_disk.clone()],
scanner.local_disk,
migrated,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await
.expect("portable cache rebuild should complete");
let persisted = rebuilt.marshal_msg().expect("rebuilt cache should serialize");
let decoded = DataUsageCache::unmarshal(&persisted).expect("rebuilt cache should deserialize");
assert_eq!(decoded.info.cache_key_format, crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT);
assert!(decoded.cache.keys().all(|key| !key.contains('\\')));
let root = decoded
.checked_flatten("bucket")
.expect("rebuilt persisted cache should have a complete root");
assert_eq!(root.objects, 1);
}
#[tokio::test]
#[serial]
async fn test_scan_data_folder_success_clears_resume_hint() {
crates/scanner/src/scanner_io.rs
+337 -59
View File
@@ -12,6 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
use crate::data_usage_define::DATA_USAGE_CACHE_KEY_FORMAT;
use crate::scanner_budget::ScannerCycleBudget;
use crate::scanner_folder::{ScannerItem, scan_data_folder};
use crate::sleeper::SCANNER_SLEEPER;
@@ -830,7 +831,7 @@ pub(crate) fn cache_root_entry_info(cache: &DataUsageCache) -> std::result::Resu
return Err(ScannerError::Other("scanner cache root name is empty".to_string()));
}
let entry = cache
.checked_flatten(&cache.info.name)
.checked_flatten_complete_scope(&cache.info.name)
.ok_or_else(|| ScannerError::Other(format!("scanner cache root is missing or corrupt: {}", cache.info.name)))?;
Ok(DataUsageEntryInfo {
@@ -852,7 +853,7 @@ fn should_publish_completed_snapshot(completed_count: usize, total_count: usize,
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum NamespaceScannerWorkerMode {
Coordinator,
RemoteV3(uuid::Uuid),
RemoteV4(uuid::Uuid),
}
fn namespace_scanner_workers<T>(
@@ -868,7 +869,7 @@ fn namespace_scanner_workers<T>(
workers.extend(
remote_disks
.into_iter()
.map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV3(server_epoch))),
.map(|(disk, server_epoch)| (disk, NamespaceScannerWorkerMode::RemoteV4(server_epoch))),
);
workers
}
@@ -958,7 +959,57 @@ where
let _ = tokio::time::timeout(SCANNER_CACHE_LOCK_LOSS_SHUTDOWN_TIMEOUT, scan).await;
}
pub(crate) fn cache_snapshot_is_current(
pub(crate) fn current_cache_root_entry(
cache: &DataUsageCache,
name: &str,
source: DataUsageCacheSource,
next_cycle: u64,
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
) -> std::result::Result<Option<DataUsageEntryInfo>, ScannerError> {
let metadata_is_current = cache.info.name == name
&& cache.info.source == Some(source)
&& cache.info.snapshot_complete
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
&& cache.info.last_update.is_some()
&& cache.info.next_cycle == next_cycle
&& cache.info.leader_epoch == leader_epoch
&& cache.info.cache_key_format == DATA_USAGE_CACHE_KEY_FORMAT;
if !metadata_is_current {
return Ok(None);
}
cache_root_entry_info(cache).map(Some)
}
pub(crate) enum DataUsageCacheScanState {
Current(Box<DataUsageEntryInfo>),
Prepared {
outcome: DataUsageCachePrepareOutcome,
invalid_current: Option<ScannerError>,
},
}
pub(crate) fn current_cache_root_or_prepare(
cache: &mut DataUsageCache,
name: &str,
source: DataUsageCacheSource,
next_cycle: u64,
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
require_source: bool,
) -> DataUsageCacheScanState {
match current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest) {
Ok(Some(root)) => DataUsageCacheScanState::Current(Box::new(root)),
current => DataUsageCacheScanState::Prepared {
invalid_current: current.err(),
outcome: cache.prepare_for_scan(name, next_cycle, leader_epoch, source, scan_plan_digest, require_source),
},
}
}
#[cfg(test)]
fn cache_snapshot_is_current(
cache: &DataUsageCache,
name: &str,
source: DataUsageCacheSource,
@@ -966,13 +1017,10 @@ pub(crate) fn cache_snapshot_is_current(
leader_epoch: u64,
scan_plan_digest: DataUsageScanPlanDigest,
) -> bool {
cache.info.name == name
&& cache.info.source == Some(source)
&& cache.info.snapshot_complete
&& cache.info.scan_plan_digest == Some(scan_plan_digest)
&& cache.info.last_update.is_some()
&& cache.info.next_cycle == next_cycle
&& cache.info.leader_epoch == leader_epoch
matches!(
current_cache_root_entry(cache, name, source, next_cycle, leader_epoch, scan_plan_digest),
Ok(Some(_))
)
}
fn completed_data_usage_info(
@@ -1061,6 +1109,7 @@ mod publish_gate_tests {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -1102,6 +1151,7 @@ mod publish_gate_tests {
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -1423,17 +1473,145 @@ mod publish_gate_tests {
assert!(!cache_snapshot_is_current(&cache, DATA_USAGE_ROOT, source, 10, 2, TEST_PLAN_DIGEST));
}
#[test]
fn current_cache_snapshot_rejects_persisted_windows_key_mismatch() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket/prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
assert!(!cache_snapshot_is_current(&cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: Some(_),
} => {}
_ => panic!("an invalid current cache must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn current_cache_snapshot_rejects_structurally_valid_legacy_key_format() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
children: HashSet::from(["bucket\\prefix".to_string()]),
..Default::default()
},
);
cache.cache.insert(
"bucket\\prefix".to_string(),
DataUsageEntry {
objects: 3,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(3));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: None,
} => {}
_ => panic!("a legacy key format must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn current_cache_snapshot_rejects_current_bucket_cache_with_detached_entry() {
let source = DataUsageCacheSource::new(1, 2);
let mut cache = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
next_cycle: 10,
last_update: Some(SystemTime::UNIX_EPOCH),
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(TEST_PLAN_DIGEST),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
};
cache.cache.insert(
"bucket".to_string(),
DataUsageEntry {
objects: 1,
..Default::default()
},
);
cache.cache.insert(
"bucket/detached".to_string(),
DataUsageEntry {
objects: 2,
..Default::default()
},
);
assert_eq!(cache.checked_flatten("bucket").map(|entry| entry.objects), Some(1));
match current_cache_root_or_prepare(&mut cache, "bucket", source, 10, 0, TEST_PLAN_DIGEST, true) {
DataUsageCacheScanState::Prepared {
outcome: DataUsageCachePrepareOutcome::Reset,
invalid_current: Some(_),
} => {}
_ => panic!("a detached complete bucket cache must enter the rebuild path"),
}
assert!(cache.cache.is_empty());
assert_eq!(cache.info.cache_key_format, DATA_USAGE_CACHE_KEY_FORMAT);
}
#[test]
fn namespace_scanner_worker_selection_keeps_coordinator_fallback_disks() {
let server_epoch = uuid::Uuid::new_v4();
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v3", server_epoch)]);
let workers = namespace_scanner_workers(vec!["local", "legacy-remote"], vec![("v4", server_epoch)]);
assert_eq!(
workers,
vec![
("local", NamespaceScannerWorkerMode::Coordinator),
("legacy-remote", NamespaceScannerWorkerMode::Coordinator),
("v3", NamespaceScannerWorkerMode::RemoteV3(server_epoch)),
("v4", NamespaceScannerWorkerMode::RemoteV4(server_epoch)),
]
);
assert!(namespace_scanner_workers::<()>(Vec::new(), Vec::new()).is_empty());
@@ -1507,6 +1685,7 @@ mod publish_gate_tests {
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(first),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
..Default::default()
@@ -1595,6 +1774,15 @@ async fn send_cache_root_entry_info(
pending_maintenance_work: &AtomicBool,
) -> std::result::Result<(), ScannerError> {
let root = cache_root_entry_info(cache)?;
send_cache_root_entry(bucket_result_tx, root, cache, pending_maintenance_work).await
}
async fn send_cache_root_entry(
bucket_result_tx: &mpsc::Sender<DataUsageEntryInfo>,
root: DataUsageEntryInfo,
cache: &DataUsageCache,
pending_maintenance_work: &AtomicBool,
) -> std::result::Result<(), ScannerError> {
record_bucket_pending_maintenance_work(cache, pending_maintenance_work);
bucket_result_tx
.send(root)
@@ -1690,13 +1878,16 @@ async fn persist_and_publish_cache_snapshot(
);
return None;
}
if cache_snapshot_is_current(
&persisted,
DATA_USAGE_ROOT,
source,
cache_snapshot.info.next_cycle,
cache_snapshot.info.leader_epoch,
scan_plan_digest,
if matches!(
current_cache_root_entry(
&persisted,
DATA_USAGE_ROOT,
source,
cache_snapshot.info.next_cycle,
cache_snapshot.info.leader_epoch,
scan_plan_digest,
),
Ok(Some(_))
) {
cache_snapshot = persisted;
} else {
@@ -2329,6 +2520,7 @@ impl ScannerIOCache for SetDisks {
source: Some(source),
snapshot_complete: true,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
@@ -2450,7 +2642,7 @@ impl ScannerIOCache for SetDisks {
subsystem = LOG_SUBSYSTEM_IO,
pool = self.pool_index,
set = self.set_index,
v3_disks = remote_disk_count,
v4_disks = remote_disk_count,
unsupported_remote_disks,
state = "unsupported_remote_disks_using_coordinator",
"Scanner set assigned remote disks without namespace scanner support to coordinator-driven workers"
@@ -2547,6 +2739,7 @@ impl ScannerIOCache for SetDisks {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
@@ -2638,7 +2831,7 @@ impl ScannerIOCache for SetDisks {
let dirty_usage_buckets_clone = dirty_usage_buckets.clone();
let cache_cycle_floor_clone = cache_cycle_floor.clone();
let remote_server_epoch = match worker_mode {
NamespaceScannerWorkerMode::RemoteV3(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::RemoteV4(server_epoch) => Some(server_epoch),
NamespaceScannerWorkerMode::Coordinator => None,
};
futs.push(tokio::spawn(async move {
@@ -2948,48 +3141,71 @@ impl ScannerIOCache for SetDisks {
continue;
}
};
if cache_snapshot_is_current(&cache, &bucket.name, source, want_cycle, leader_epoch, bucket_scan_plan_digest)
{
if cache_guard.is_lock_lost() {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_lost_before_reuse",
"Current scanner bucket cache root publish skipped after lock loss"
);
continue;
}
if let Err(e) =
send_cache_root_entry_info(&bucket_result_tx_clone, &cache, &pending_maintenance_work_clone).await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_current_root_failed",
error = %e,
"Current scanner bucket cache root entry publish failed"
);
}
continue;
}
match cache.prepare_for_scan(
let scan_state = current_cache_root_or_prepare(
&mut cache,
&bucket.name,
source,
want_cycle,
leader_epoch,
source,
bucket_scan_plan_digest,
require_cache_source,
) {
);
let outcome = match scan_state {
DataUsageCacheScanState::Current(root) => {
if cache_guard.is_lock_lost() {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "lock_lost_before_reuse",
"Current scanner bucket cache root publish skipped after lock loss"
);
continue;
}
if let Err(e) =
send_cache_root_entry(&bucket_result_tx_clone, *root, &cache, &pending_maintenance_work_clone)
.await
{
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_DATA_USAGE_STREAM,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
state = "send_current_root_failed",
error = %e,
"Current scanner bucket cache root entry publish failed"
);
}
continue;
}
DataUsageCacheScanState::Prepared {
outcome,
invalid_current,
} => {
if let Some(e) = invalid_current {
warn!(
target: "rustfs::scanner::io",
event = EVENT_SCANNER_CACHE_PERSIST_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
bucket = %bucket.name,
cache_name = %cache_name,
state = "current_cache_invalid",
error = %e,
"Current scanner bucket cache is invalid; rebuilding"
);
}
outcome
}
};
match outcome {
DataUsageCachePrepareOutcome::RejectedNewerCycle => {
cache_cycle_floor_clone.fetch_max(cache.info.next_cycle, Ordering::AcqRel);
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
@@ -3361,6 +3577,7 @@ impl ScannerIOCache for SetDisks {
source: Some(source),
snapshot_complete: false,
scan_plan_digest: Some(scan_plan_digest),
cache_key_format: DATA_USAGE_CACHE_KEY_FORMAT,
..Default::default()
},
cache: HashMap::new(),
@@ -4668,6 +4885,67 @@ mod tests {
root.add_child(&crate::hash_path("bucket/missing"));
dangling.replace("bucket", DATA_USAGE_ROOT, root);
assert!(cache_root_entry_info(&dangling).is_err());
let mut detached = DataUsageCache {
info: DataUsageCacheInfo {
name: DATA_USAGE_ROOT.to_string(),
..Default::default()
},
..Default::default()
};
detached.replace("bucket", DATA_USAGE_ROOT, DataUsageEntry::default());
detached.replace(
"bucket/detached",
"",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(cache_root_entry_info(&detached).is_err());
let mut detached_bucket = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
detached_bucket.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
objects: 1,
..Default::default()
},
);
detached_bucket.replace(
"bucket/detached",
"",
DataUsageEntry {
objects: 1,
..Default::default()
},
);
assert!(cache_root_entry_info(&detached_bucket).is_err());
let mut compacted_with_child = DataUsageCache {
info: DataUsageCacheInfo {
name: "bucket".to_string(),
..Default::default()
},
..Default::default()
};
compacted_with_child.replace(
"bucket",
DATA_USAGE_ROOT,
DataUsageEntry {
compacted: true,
..Default::default()
},
);
compacted_with_child.replace("bucket/prefix", "bucket", DataUsageEntry::default());
assert!(cache_root_entry_info(&compacted_with_child).is_err());
}
#[test]
crates/storage-api/src/bucket.rs
+1 -1
View File
@@ -53,7 +53,7 @@ pub struct DeleteBucketOptions {
pub no_recreate: bool,
/// Force deletion even if bucket is not empty.
pub force: bool,
/// Force deletion only after the local peer verifies the bucket is empty.
/// Delete empty directory remnants after the local peer verifies no object metadata exists.
pub force_if_empty: bool,
/// Site replication delete operation.
pub srdelete_op: SRBucketDeleteOp,
crates/utils/Cargo.toml
+5
View File
@@ -58,11 +58,16 @@ url = { workspace = true, optional = true }
zstd = { workspace = true, optional = true }
[dev-dependencies]
criterion = { workspace = true, features = ["html_reports"] }
tempfile = { workspace = true }
tokio = { workspace = true, features = ["macros", "rt"] }
temp-env = { workspace = true }
proptest = "1"
[[bench]]
name = "hash_hotpath_benchmark"
harness = false
[target.'cfg(windows)'.dependencies]
windows = { workspace = true, optional = true, features = ["Win32_Storage_FileSystem"] }
crates/utils/benches/hash_hotpath_benchmark.rs
+55
View File
@@ -0,0 +1,55 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use criterion::{BenchmarkId, Criterion, Throughput, criterion_group, criterion_main};
use rustfs_utils::HashAlgorithm;
use std::hint::black_box;
fn generate_payload(size: usize) -> Vec<u8> {
(0..size)
.map(|i| u8::try_from(i % 251).expect("modulo output fits in u8"))
.collect()
}
fn bench_hash_hotpaths(c: &mut Criterion) {
let payloads = [
("64KiB", generate_payload(64 * 1024)),
("1MiB", generate_payload(1024 * 1024)),
];
let algorithms = [
("md5", HashAlgorithm::Md5),
("sha256", HashAlgorithm::SHA256),
("highwayhash256s", HashAlgorithm::HighwayHash256S),
("highwayhash256s_legacy", HashAlgorithm::HighwayHash256SLegacy),
];
let mut group = c.benchmark_group("hash_hotpath");
for (payload_name, payload) in &payloads {
let payload_len = u64::try_from(payload.len()).expect("benchmark payload length fits in u64");
group.throughput(Throughput::Bytes(payload_len));
for (algo_name, algorithm) in &algorithms {
let algorithm = algorithm.clone();
group.bench_with_input(BenchmarkId::new(*algo_name, payload_name), payload.as_slice(), move |b, payload| {
b.iter(|| {
let hash = algorithm.hash_encode(black_box(payload));
black_box(hash.as_ref()[0]);
});
});
}
}
group.finish();
}
criterion_group!(benches, bench_hash_hotpaths);
criterion_main!(benches);
crates/utils/src/hash.rs
+20 -12
View File
@@ -29,14 +29,24 @@ const LEGACY_HIGHWAY_HASH256_KEY: [u8; 32] = [
3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0,
];
fn highway_key_from_bytes(bytes: &[u8; 32]) -> [u64; 4] {
let mut key = [0u64; 4];
for (i, chunk) in bytes.chunks_exact(8).enumerate() {
key[i] = u64::from_le_bytes(chunk.try_into().unwrap());
}
key
const fn highway_key_from_bytes(bytes: &[u8; 32]) -> [u64; 4] {
[
u64::from_le_bytes([bytes[0], bytes[1], bytes[2], bytes[3], bytes[4], bytes[5], bytes[6], bytes[7]]),
u64::from_le_bytes([
bytes[8], bytes[9], bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15],
]),
u64::from_le_bytes([
bytes[16], bytes[17], bytes[18], bytes[19], bytes[20], bytes[21], bytes[22], bytes[23],
]),
u64::from_le_bytes([
bytes[24], bytes[25], bytes[26], bytes[27], bytes[28], bytes[29], bytes[30], bytes[31],
]),
]
}
const MAGIC_HIGHWAY_HASH256_PARSED_KEY: Key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
const LEGACY_HIGHWAY_HASH256_PARSED_KEY: Key = Key(highway_key_from_bytes(&LEGACY_HIGHWAY_HASH256_KEY));
#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)]
/// Supported hash algorithms for bitrot protection.
pub enum HashAlgorithm {
@@ -100,25 +110,23 @@ impl HashAlgorithm {
/// # Returns
/// A byte slice containing the hash of the input data
///
#[inline]
pub fn hash_encode(&self, data: &[u8]) -> impl AsRef<[u8]> {
match self {
HashAlgorithm::Md5 => HashEncoded::Md5(Md5::digest(data).into()),
HashAlgorithm::HighwayHash256 => {
let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
let mut hasher = HighwayHasher::new(key);
let mut hasher = HighwayHasher::new(MAGIC_HIGHWAY_HASH256_PARSED_KEY);
hasher.append(data);
HashEncoded::HighwayHash256(u8x32_from_u64x4(hasher.finalize256()))
}
HashAlgorithm::SHA256 => HashEncoded::Sha256(Sha256::digest(data).into()),
HashAlgorithm::HighwayHash256S => {
let key = Key(highway_key_from_bytes(&MAGIC_HIGHWAY_HASH256_KEY));
let mut hasher = HighwayHasher::new(key);
let mut hasher = HighwayHasher::new(MAGIC_HIGHWAY_HASH256_PARSED_KEY);
hasher.append(data);
HashEncoded::HighwayHash256S(u8x32_from_u64x4(hasher.finalize256()))
}
HashAlgorithm::HighwayHash256SLegacy => {
let key = Key(highway_key_from_bytes(&LEGACY_HIGHWAY_HASH256_KEY));
let mut hasher = HighwayHasher::new(key);
let mut hasher = HighwayHasher::new(LEGACY_HIGHWAY_HASH256_PARSED_KEY);
hasher.append(data);
HashEncoded::HighwayHash256SLegacy(u8x32_from_u64x4(hasher.finalize256()))
}
crates/utils/src/http/metadata_compat.rs
+1
View File
@@ -40,6 +40,7 @@ pub const SUFFIX_TRANSITIONED_VERSION_ID: &str = "transitioned-versionID";
pub const SUFFIX_TRANSITIONED_VERSION_STATE: &str = "transitioned-version-state";
pub const SUFFIX_TRANSITION_TIER: &str = "transition-tier";
pub const SUFFIX_TRANSITION_TIER_DESTINATION_ID: &str = "transition-tier-destination-id";
pub const SUFFIX_TRANSITION_TRANSACTION_ID: &str = "transition-transaction-id";
pub const SUFFIX_RESTORE_OPERATION_ID: &str = "restore-operation-id";
pub const SUFFIX_FREE_VERSION: &str = "free-version";
pub const SUFFIX_PURGESTATUS: &str = "purgestatus";
docs/operations/tier-ilm-debugging.md
+33
View File
@@ -98,6 +98,39 @@ rc admin ilm transition run local/mybucket --prefix logs/ --tier cold --max-obje
Inspect the aggregate counters before widening scope. Full object-key lists are intentionally not returned by the admin response. If `RUSTFS_RPC_SECRET` or other credentials were pasted into an issue, chat, log, or ticket while debugging tiering, rotate them on every node, restart the cluster with the new value, and redact the exposed copy before sharing more diagnostics.
## Reconcile an unknown transition upload
Historical transition transactions in `upload_outcome_unknown` state can use an explicit two-stage operator workflow when the tier probe is ambiguous and the provider supports exact version deletion. The endpoint refuses transactions that are still inside their ownership window or are in any other state.
First inspect the transaction without changing it:
```text
GET /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
```
If independent provider evidence identifies the exact remote version to remove, submit that opaque version identifier with explicit confirmation:
```json
POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
{
"action": "delete_candidate",
"confirm": true,
"remote_version_id": "<exact-provider-version>"
}
```
This operation performs only an exact version delete. Its response reports whether the transaction journal was still observed after the delete; background recovery may have finalized the same transaction concurrently. If the journal remains, inspect the transaction again and finalize it only after the live provider probe proves that the candidate is missing:
```json
POST /rustfs/admin/v3/ilm/transition/reconcile/<transaction-id>
{
"action": "finalize_missing",
"confirm": true
}
```
`finalize_missing` re-runs the provider probe and fails closed for `unversioned_present`, `versioned_present`, `ambiguous`, `unsupported`, or probe errors. It never accepts an operator assertion in place of a live `missing` result. Providers without an authoritative probe or exact version deletion remain pending; this endpoint does not infer provider capabilities, accept external absence assertions, or select a candidate automatically.
## Historical fixes (for context, already merged)
- Expire/GET race (`NoSuchVersion` during expiry of a tiered object):
rustfs/src/admin/handlers/ilm_transition.rs
+266 -11
View File
@@ -20,8 +20,10 @@ use crate::admin::storage_api::error::StorageError;
use crate::admin::storage_api::lifecycle::{
ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink,
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission,
ManualTransitionScopeAdmissionClaim, claim_manual_transition_scope_admission,
delete_manual_transition_scope_admission_if_current, enqueue_transition_for_existing_objects_scoped,
ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError,
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
persist_manual_transition_job_progress, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
@@ -33,6 +35,7 @@ use crate::server::{ADMIN_PREFIX, RemoteAddr};
use http::{HeaderMap, HeaderValue};
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
use rustfs_policy::policy::action::{Action, AdminAction};
use rustfs_utils::{
MaskedAccessKey,
@@ -56,6 +59,7 @@ const MAX_MANUAL_TRANSITION_DURATION_SECONDS: u64 = 3600;
const LOG_COMPONENT_ADMIN: &str = "admin";
const LOG_SUBSYSTEM_ILM_TRANSITION: &str = "ilm_transition";
const EVENT_ADMIN_ILM_TRANSITION_STATE: &str = "admin_ilm_transition_state";
const EVENT_ADMIN_ILM_TRANSITION_RECONCILE: &str = "admin_ilm_transition_reconcile";
static ACTIVE_MANUAL_TRANSITION_SCOPES: OnceLock<Mutex<Vec<ManualTransitionRunScope>>> = OnceLock::new();
#[cfg(feature = "e2e-test-hooks")]
@@ -218,6 +222,16 @@ pub fn register_ilm_transition_route(r: &mut S3Router<AdminOperation>) -> std::i
format!("{ADMIN_PREFIX}/v3/ilm/transition/jobs/{{job_id}}").as_str(),
AdminOperation(&ManualTransitionJobCancelHandler {}),
)?;
r.insert(
Method::GET,
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
AdminOperation(&TransitionReconcileInspectHandler {}),
)?;
r.insert(
Method::POST,
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
AdminOperation(&TransitionReconcileApplyHandler {}),
)?;
Ok(())
}
@@ -389,6 +403,10 @@ fn log_manual_transition_completed(
}
async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<String> {
authorize_transition_admin_request(req, AdminAction::SetTierAction).await
}
async fn authorize_transition_admin_request(req: &S3Request<Body>, action: AdminAction) -> S3Result<String> {
let Some(input_cred) = req.credentials.as_ref() else {
return Err(s3_error!(InvalidRequest, "authentication required"));
};
@@ -401,19 +419,122 @@ async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<
.get::<Option<RemoteAddr>>()
.and_then(|opt| opt.map(|addr| addr.0));
validate_admin_request(
&req.headers,
&cred,
owner,
false,
vec![Action::AdminAction(AdminAction::SetTierAction)],
remote_addr,
)
.await?;
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await?;
Ok(actor)
}
fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result<Uuid> {
Uuid::parse_str(params.get("transaction_id").unwrap_or(""))
.map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id"))
}
fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error {
match err {
TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"),
TransitionOperatorError::NotExpired => {
s3_error!(OperationAborted, "transition transaction is still inside its active ownership window")
}
TransitionOperatorError::InvalidState(_) => {
s3_error!(OperationAborted, "transition transaction is not eligible for operator reconciliation")
}
TransitionOperatorError::RemoteVersionRequired => {
s3_error!(InvalidArgument, "an exact non-empty remote version is required")
}
TransitionOperatorError::CandidateNotMissing(_) => {
s3_error!(OperationAborted, "remote candidate is not proven missing")
}
TransitionOperatorError::CandidateVersionMismatch { .. } => {
s3_error!(OperationAborted, "remote candidate version does not match requested exact version")
}
TransitionOperatorError::Store(_) | TransitionOperatorError::Remote(_) => {
s3_error!(InternalError, "transition reconciliation failed")
}
}
}
#[derive(Debug, Deserialize)]
#[serde(rename_all = "snake_case")]
enum TransitionReconcileAction {
DeleteCandidate,
FinalizeMissing,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct TransitionReconcileRequest {
action: TransitionReconcileAction,
confirm: bool,
#[serde(default)]
remote_version_id: Option<String>,
}
enum ValidatedTransitionReconcileAction<'a> {
DeleteCandidate(&'a str),
FinalizeMissing,
}
fn validate_transition_reconcile_request(
request: &TransitionReconcileRequest,
) -> S3Result<ValidatedTransitionReconcileAction<'_>> {
if !request.confirm {
return Err(s3_error!(
InvalidRequest,
"transition reconciliation requires confirm=true; use GET to inspect without changes"
));
}
match request.action {
TransitionReconcileAction::DeleteCandidate => request
.remote_version_id
.as_deref()
.filter(|version_id| !version_id.is_empty())
.map(ValidatedTransitionReconcileAction::DeleteCandidate)
.ok_or_else(|| s3_error!(InvalidArgument, "delete_candidate requires remote_version_id")),
TransitionReconcileAction::FinalizeMissing if request.remote_version_id.is_none() => {
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
}
TransitionReconcileAction::FinalizeMissing => {
Err(s3_error!(InvalidArgument, "finalize_missing must not include remote_version_id"))
}
}
}
#[derive(Debug, Serialize)]
struct TransitionCandidateDeleteResponse {
outcome: &'static str,
result: TransitionOperatorDeleteResult,
}
#[derive(Debug, Serialize)]
struct TransitionFinalizeMissingResponse {
outcome: &'static str,
journal_retained: bool,
transaction_id: Uuid,
}
fn log_transition_reconcile_applied(
transaction_id: Uuid,
action: &str,
outcome: &str,
request_id: &str,
actor: &str,
remote_addr: &str,
) {
info!(
event = EVENT_ADMIN_ILM_TRANSITION_RECONCILE,
component = LOG_COMPONENT_ADMIN,
subsystem = LOG_SUBSYSTEM_ILM_TRANSITION,
operation = "transition_operator_reconcile",
transaction_id = %transaction_id,
action,
outcome,
request_id = %request_id,
actor = %actor,
remote_addr = %remote_addr,
"admin transition reconciliation applied"
);
}
fn response_state(report: &ManualTransitionRunReport) -> &'static str {
if report.was_truncated() || report.has_partial_enqueue() || report.tier_failure > 0 || report.transition_failed > 0 {
"partial"
@@ -902,6 +1023,81 @@ impl Operation for ManualTransitionJobCancelHandler {
}
}
pub struct TransitionReconcileInspectHandler {}
#[async_trait::async_trait]
impl Operation for TransitionReconcileInspectHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
let transaction_id = transition_transaction_id_from_params(&params)?;
let Some(store) = object_store_from_extensions(&req.extensions) else {
return Err(s3_error!(InternalError, "object store is not initialized"));
};
let status = inspect_transition_transaction_for_operator(store, transaction_id)
.await
.map_err(map_transition_operator_error)?;
json_response(&status, StatusCode::OK)
}
}
pub struct TransitionReconcileApplyHandler {}
#[async_trait::async_trait]
impl Operation for TransitionReconcileApplyHandler {
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
let remote_addr = admin_remote_addr(&req).unwrap_or_default();
let actor = authorize_transition_admin_request(&req, AdminAction::SetTierAction).await?;
let transaction_id = transition_transaction_id_from_params(&params)?;
let Some(store) = object_store_from_extensions(&req.extensions) else {
return Err(s3_error!(InternalError, "object store is not initialized"));
};
let mut input = req.input;
let body = input
.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
.await
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation body is too large or unreadable"))?;
let request: TransitionReconcileRequest = serde_json::from_slice(&body)
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation request must be valid JSON"))?;
match validate_transition_reconcile_request(&request)? {
ValidatedTransitionReconcileAction::DeleteCandidate(remote_version_id) => {
let result = delete_transition_candidate_for_operator(store, transaction_id, remote_version_id)
.await
.map_err(map_transition_operator_error)?;
let outcome = if result.journal_observed_after_delete {
"exact_delete_completed_journal_observed"
} else {
"exact_delete_completed_journal_already_finalized"
};
log_transition_reconcile_applied(transaction_id, "delete_candidate", outcome, &request_id, &actor, &remote_addr);
json_response(&TransitionCandidateDeleteResponse { outcome, result }, StatusCode::OK)
}
ValidatedTransitionReconcileAction::FinalizeMissing => {
finalize_missing_transition_transaction_for_operator(store, transaction_id)
.await
.map_err(map_transition_operator_error)?;
log_transition_reconcile_applied(
transaction_id,
"finalize_missing",
"journal_deleted_after_missing_probe",
&request_id,
&actor,
&remote_addr,
);
json_response(
&TransitionFinalizeMissingResponse {
outcome: "journal_finalized",
journal_retained: false,
transaction_id,
},
StatusCode::OK,
)
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -931,6 +1127,65 @@ mod tests {
}
}
#[test]
fn transition_reconcile_request_is_explicit_and_fail_closed() {
let unconfirmed: TransitionReconcileRequest =
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":false,"remote_version_id":"v1"}"#)
.expect("request should decode");
assert!(validate_transition_reconcile_request(&unconfirmed).is_err());
let missing_version: TransitionReconcileRequest =
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true}"#).expect("request should decode");
assert!(validate_transition_reconcile_request(&missing_version).is_err());
let unsafe_finalize: TransitionReconcileRequest =
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true,"remote_version_id":"v1"}"#)
.expect("request should decode");
assert!(validate_transition_reconcile_request(&unsafe_finalize).is_err());
let delete: TransitionReconcileRequest =
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true,"remote_version_id":"opaque-v1"}"#)
.expect("request should decode");
assert!(matches!(
validate_transition_reconcile_request(&delete),
Ok(ValidatedTransitionReconcileAction::DeleteCandidate("opaque-v1"))
));
let finalize: TransitionReconcileRequest =
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true}"#).expect("request should decode");
assert!(matches!(
validate_transition_reconcile_request(&finalize),
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
));
assert!(
serde_json::from_slice::<TransitionReconcileRequest>(
br#"{"action":"finalize_missing","confirm":true,"unexpected":true}"#
)
.is_err()
);
}
#[test]
fn transition_reconcile_routes_use_read_and_write_tier_actions() {
let src = include_str!("ilm_transition.rs");
let inspect = src
.split("impl Operation for TransitionReconcileInspectHandler")
.nth(1)
.and_then(|block| block.split("impl Operation for TransitionReconcileApplyHandler").next())
.expect("inspect handler block");
assert!(inspect.contains("AdminAction::ListTierAction"));
assert!(!inspect.contains("AdminAction::SetTierAction"));
let apply = src
.split("impl Operation for TransitionReconcileApplyHandler")
.nth(1)
.and_then(|block| block.split("#[cfg(test)]").next())
.expect("apply handler block");
assert!(apply.contains("AdminAction::SetTierAction"));
assert!(!apply.contains("AdminAction::ListTierAction"));
}
#[test]
fn manual_transition_query_defaults_to_bounded_run() {
let (bucket, options, run_mode) =
rustfs/src/admin/handlers/site_replication.rs
+1 -1
View File
@@ -7205,7 +7205,7 @@ async fn start_site_bucket_resync(bucket: &str, target_arn: &str, resync_id: &st
return bucket_status;
};
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
let _transaction_guard = match metadata_sys::acquire_bucket_targets_transaction_lock(bucket).await {
let _transaction_guard = match metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await {
Ok(guard) => guard,
Err(_) => {
bucket_status.status = "failed".to_string();
rustfs/src/admin/route_policy.rs
+17 -1
View File
@@ -432,6 +432,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
SET_TIER,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
LIST_TIER,
RouteRiskLevel::High,
),
admin(
HttpMethod::Post,
"/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
SET_TIER,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/audit/target/list",
@@ -1825,13 +1837,17 @@ mod tests {
}
#[test]
fn route_policy_requires_set_tier_for_manual_transition_routes() {
fn route_policy_uses_tier_actions_for_transition_routes() {
assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER);
assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
assert_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SERVER_INFO);
assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
assert_not_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
}
#[test]
rustfs/src/admin/route_registration_test.rs
+20
View File
@@ -202,6 +202,16 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
"/v3/ilm/transition/jobs/{job_id}",
"/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111",
),
admin_route_sample(
Method::GET,
"/v3/ilm/transition/reconcile/{transaction_id}",
"/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
),
admin_route_sample(
Method::POST,
"/v3/ilm/transition/reconcile/{transaction_id}",
"/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
),
admin_route_sample(
Method::DELETE,
"/v3/ilm/transition/jobs/{job_id}",
@@ -856,6 +866,16 @@ fn test_register_routes_cover_representative_admin_paths() {
Method::DELETE,
&admin_path("/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111"),
);
assert_route(
&router,
Method::GET,
&admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
);
assert_route(
&router,
Method::POST,
&admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
);
assert_route(&router, Method::GET, &table_catalog_path("/config"));
assert_route(&router, Method::PUT, &table_catalog_path("/buckets/analytics"));
rustfs/src/admin/router.rs
+11 -1
View File
@@ -2478,7 +2478,7 @@ async fn start_replication_resync(bucket: &str, reset: &ReplicationResetStartReq
};
let _targets_guard = lock_bucket_targets_metadata(bucket).await;
let _transaction_guard = metadata_sys::acquire_bucket_targets_transaction_lock(bucket)
let _transaction_guard = metadata_sys::acquire_bucket_metadata_transaction_lock(bucket)
.await
.map_err(ApiError::from)?;
let (config, _) = metadata_sys::get_replication_config(bucket).await.map_err(ApiError::from)?;
@@ -2655,6 +2655,10 @@ fn is_public_health_path(path: &str) -> bool {
path == HEALTH_PREFIX || path == HEALTH_READY_PATH
}
fn server_context_not_ready_error() -> S3Error {
s3_error!(ServiceUnavailable, "server context is not ready")
}
fn is_object_zip_download_token_path(method: &Method, uri: &Uri) -> bool {
if method != Method::GET {
return false;
@@ -2783,6 +2787,9 @@ where
async fn check_access(&self, req: &mut S3Request<Body>) -> S3Result<()> {
if let Some(server_ctx) = &self.server_ctx {
req.extensions.insert(server_ctx.clone());
if !is_public_health_path(req.uri.path()) && server_ctx.installed_app_context().is_none() {
return Err(server_context_not_ready_error());
}
}
if parse_replication_extension_request(&req.method, &req.uri).is_some()
|| parse_misc_extension_request(&req.method, &req.uri).is_some()
@@ -2836,6 +2843,9 @@ where
async fn call(&self, mut req: S3Request<Body>) -> S3Result<S3Response<Body>> {
if let Some(server_ctx) = &self.server_ctx {
req.extensions.insert(server_ctx.clone());
if !is_public_health_path(req.uri.path()) && server_ctx.installed_app_context().is_none() {
return Err(server_context_not_ready_error());
}
}
if let Some(ext_req) = parse_replication_extension_request(&req.method, &req.uri) {
return handle_replication_extension_request(&mut req, &ext_req).await;
rustfs/src/admin/runtime_sources.rs
+166 -8
View File
@@ -76,19 +76,29 @@ pub(crate) fn object_store_from_req<B>(req: &s3s::S3Request<B>) -> Option<Arc<EC
}
pub(crate) fn app_context_from_req<B>(req: &s3s::S3Request<B>) -> Option<Arc<AppContext>> {
req.extensions
.get::<Arc<ServerContextSlot>>()
.and_then(|slot| slot.app_context())
.or_else(current_app_context)
app_context_from_extensions(&req.extensions)
}
/// Resolve an application context from request extensions.
///
/// An injected slot identifies the server that owns the request. If that
/// server has not finished startup yet, returning its ambient global context
/// could select a different server, so this fails closed. Only requests with
/// no slot retain the legacy ambient fallback.
pub(crate) fn app_context_from_extensions(extensions: &http::Extensions) -> Option<Arc<AppContext>> {
match extensions.get::<Arc<ServerContextSlot>>() {
Some(slot) => slot.installed_app_context(),
None => current_app_context(),
}
}
/// Field-borrow form of [`object_store_from_req`] for handlers that have
/// already moved other request fields (body, credentials) out of the request.
pub(crate) fn object_store_from_extensions(extensions: &http::Extensions) -> Option<Arc<ECStore>> {
extensions
.get::<Arc<ServerContextSlot>>()
.and_then(|slot| slot.object_store())
.or_else(current_object_store_handle)
match extensions.get::<Arc<ServerContextSlot>>() {
Some(slot) => slot.installed_object_store(),
None => current_object_store_handle(),
}
}
pub(crate) fn current_notification_system() -> Option<Arc<NotificationSys>> {
@@ -168,3 +178,151 @@ pub(crate) fn publish_storage_class_config(config: StorageClassConfig) {
root_runtime_sources::fallback_storage_class_interface().set(config);
}
}
#[cfg(test)]
mod tests {
use super::{
AppContext, IamInterface, KmsInterface, ServerContextSlot, app_context_from_extensions, app_context_from_req,
current_app_context, object_store_from_extensions, publish_test_app_context,
};
use crate::admin::router::{Operation, S3Router};
use hyper::{Method, StatusCode};
use matchit::Params;
use rustfs_iam::{store::object::ObjectStore, sys::IamSys};
use rustfs_kms::KmsServiceManager;
use s3s::route::S3Route;
use s3s::{Body, S3ErrorCode, S3Request, S3Response, s3_error};
use std::sync::Arc;
struct UnreadyIam;
impl IamInterface for UnreadyIam {
fn handle(&self) -> Arc<IamSys<ObjectStore>> {
panic!("test context does not resolve IAM")
}
fn is_ready(&self) -> bool {
false
}
}
struct TestKms;
impl KmsInterface for TestKms {
fn handle(&self) -> Arc<KmsServiceManager> {
Arc::new(KmsServiceManager::new())
}
}
async fn ambient_context() -> Arc<AppContext> {
if let Some(context) = current_app_context() {
return context;
}
let env = rustfs_test_utils::TestECStoreEnv::builder()
.prefix("server_context_slot")
.disk_count(1)
.init_bucket_metadata(false)
.build()
.await;
let context = Arc::new(AppContext::new(env.ecstore, Arc::new(UnreadyIam), Arc::new(TestKms)));
publish_test_app_context(context);
current_app_context().expect("test context must be globally published")
}
fn distinct_context(context: &AppContext) -> Arc<AppContext> {
Arc::new(AppContext::new(context.object_store(), Arc::new(UnreadyIam), Arc::new(TestKms)))
}
fn request(extensions: http::Extensions) -> S3Request<Body> {
S3Request {
input: Body::empty(),
method: Method::GET,
uri: "/context-probe".parse().expect("test URI"),
headers: http::HeaderMap::new(),
extensions,
credentials: None,
region: None,
service: None,
trailing_headers: None,
}
}
#[tokio::test]
#[serial_test::serial]
async fn request_with_empty_slot_never_uses_ambient_context() {
let ambient = ambient_context().await;
let slot = ServerContextSlot::new();
let mut extensions = http::Extensions::new();
extensions.insert(slot);
assert!(app_context_from_extensions(&extensions).is_none());
assert!(object_store_from_extensions(&extensions).is_none());
assert!(app_context_from_req(&request(extensions)).is_none());
assert!(Arc::ptr_eq(
&current_app_context().expect("ambient context must remain available"),
&ambient
));
}
#[tokio::test]
#[serial_test::serial]
async fn request_with_installed_slot_resolves_only_its_context() {
let ambient = ambient_context().await;
let installed = distinct_context(&ambient);
let slot = ServerContextSlot::new();
assert!(slot.install(installed.clone()));
let mut extensions = http::Extensions::new();
extensions.insert(slot);
let resolved = app_context_from_extensions(&extensions).expect("installed slot must resolve");
assert!(Arc::ptr_eq(&resolved, &installed));
assert!(!Arc::ptr_eq(&resolved, &ambient));
}
#[tokio::test]
#[serial_test::serial]
async fn request_without_slot_keeps_legacy_ambient_resolution() {
let ambient = ambient_context().await;
let extensions = http::Extensions::new();
let resolved = app_context_from_extensions(&extensions).expect("legacy request must use ambient context");
assert!(Arc::ptr_eq(&resolved, &ambient));
assert!(object_store_from_extensions(&extensions).is_some());
}
struct ContextDependentAdminRoute;
#[async_trait::async_trait]
impl Operation for ContextDependentAdminRoute {
async fn call(&self, req: S3Request<Body>, _params: Params<'_, '_>) -> s3s::S3Result<S3Response<(StatusCode, Body)>> {
app_context_from_req(&req).ok_or_else(|| s3_error!(ServiceUnavailable, "server context is not ready"))?;
Ok(S3Response::new((StatusCode::NO_CONTENT, Body::empty())))
}
}
#[tokio::test]
#[serial_test::serial]
async fn context_dependent_admin_route_fails_closed_until_slot_installation() {
let ambient = ambient_context().await;
let slot = ServerContextSlot::new();
let mut router = S3Router::new(false);
router.set_server_ctx(slot.clone());
router
.insert(Method::GET, "/context-probe", ContextDependentAdminRoute)
.expect("register test route");
let err = router
.call(request(http::Extensions::new()))
.await
.expect_err("empty slot must fail closed");
assert_eq!(err.code(), &S3ErrorCode::ServiceUnavailable);
assert!(slot.install(distinct_context(&ambient)));
let response = router
.call(request(http::Extensions::new()))
.await
.expect("installed slot must serve request");
assert_eq!(response.status, Some(StatusCode::NO_CONTENT));
}
}
rustfs/src/admin/storage_api.rs
+6 -2
View File
@@ -210,6 +210,10 @@ pub(crate) mod lifecycle {
pub(crate) type ManualTransitionRunOptions =
super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions;
pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport;
pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{
TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator,
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
};
pub(crate) async fn enqueue_transition_for_existing_objects_scoped(
api: std::sync::Arc<super::ECStore>,
@@ -282,8 +286,8 @@ pub(crate) mod metadata_sys {
crate::storage::storage_api::update_bucket_metadata_config(bucket, config_file, data).await
}
pub(crate) async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
crate::storage::storage_api::acquire_bucket_targets_transaction_lock(bucket).await
pub(crate) async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
crate::storage::storage_api::acquire_bucket_metadata_transaction_lock(bucket).await
}
pub(crate) async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec<u8>) -> Result<OffsetDateTime> {
rustfs/src/app/context/server_slot.rs
+23 -8
View File
@@ -20,11 +20,10 @@
//! request-path owners (the `FS` service), and installs the `AppContext` into
//! it once IAM bootstrap completes.
//!
//! Until every app subsystem is per-server (backlog#1052 S3), resolution falls
//! back to the process-global `AppContext` singleton when the slot has not
//! been installed — byte-for-byte the ambient resolution the request path used
//! before this seam existed. The fallback is removed when multi-instance flips
//! on (backlog#1052 S5).
//! Some legacy, non-request call sites still use the ambient compatibility
//! resolver. Request dispatch must instead use the strict installed-context
//! accessors: once a request carries a server slot, an empty slot means that
//! server is not ready rather than that another server's global context applies.
use super::global::{AppContext, get_global_app_context};
use crate::app::storage_api::context::ECStore;
@@ -62,10 +61,26 @@ impl ServerContextSlot {
self.app_context.set(context).is_ok()
}
/// This server's installed application context, if startup has completed.
///
/// Request-scoped resolution must use this accessor so an empty slot never
/// resolves through another server's ambient context.
pub fn installed_app_context(&self) -> Option<Arc<AppContext>> {
self.app_context.get().cloned()
}
/// This server's installed object store, if startup has completed.
pub fn installed_object_store(&self) -> Option<Arc<ECStore>> {
self.installed_app_context().map(|context| context.object_store())
}
/// This server's application context: the installed one, or the
/// process-global singleton as the single-instance legacy default.
/// process-global singleton as the legacy compatibility default.
///
/// This accessor is for callers without request-slot semantics. Request
/// dispatch must use [`Self::installed_app_context`] instead.
pub fn app_context(&self) -> Option<Arc<AppContext>> {
self.app_context.get().cloned().or_else(get_global_app_context)
self.installed_app_context().or_else(get_global_app_context)
}
/// This server's object store, resolved through [`Self::app_context`].
@@ -88,7 +103,7 @@ mod tests {
// nothing: the same "not ready yet" answer the ambient path gives before
// IAM bootstrap completes.
#[test]
fn empty_slot_resolves_like_the_ambient_path() {
fn empty_slot_retains_ambient_compatibility() {
let slot = ServerContextSlot::new();
assert_eq!(slot.app_context().is_some(), get_global_app_context().is_some());
assert_eq!(slot.object_store().is_some(), get_global_app_context().is_some());
rustfs/src/app/object_usecase.rs
+13 -12
View File
@@ -72,7 +72,7 @@ use super::storage_api::object_usecase::error::{
is_err_version_not_found,
};
use super::storage_api::object_usecase::head_prefix::{head_prefix_not_found_message, probe_prefix_has_children};
use super::storage_api::object_usecase::helper::{OperationHelper, spawn_background_with_context};
use super::storage_api::object_usecase::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context};
use super::storage_api::object_usecase::io::{DynReader, HashReader, WritePlan, compression_metadata_value, wrap_reader};
#[cfg(test)]
use super::storage_api::object_usecase::object_cache::GetObjectBodySource;
@@ -130,9 +130,7 @@ use rustfs_object_capacity::capacity_manager::get_capacity_manager;
use rustfs_policy::policy::action::{Action, S3Action};
use rustfs_s3_ops::{S3Operation, delete_event_name_for_marker, put_event_name_for_post_object};
use rustfs_s3select_api::object_store::bytes_stream;
use rustfs_targets::{
EventName, extract_params_header, extract_resp_elements, get_request_host, get_request_port, get_request_user_agent,
};
use rustfs_targets::{EventName, get_request_host, get_request_port, get_request_user_agent};
use rustfs_utils::CompressionAlgorithm;
use rustfs_utils::http::{
AMZ_BUCKET_REPLICATION_STATUS, AMZ_CHECKSUM_MODE, AMZ_CHECKSUM_TYPE, AMZ_WEBSITE_REDIRECT_LOCATION, CONTENT_TYPE,
@@ -6540,6 +6538,7 @@ impl DefaultObjectUsecase {
}
let helper = OperationHelper::new(&req, EventName::ObjectRemovedDelete, S3Operation::DeleteObjects).suppress_event();
let request_context = helper.request_context_or_from_request(&req);
let (bucket, delete) = {
let bucket = req.input.bucket.clone();
let delete = req.input.delete.clone();
@@ -6948,10 +6947,12 @@ impl DefaultObjectUsecase {
let req_headers = req.headers.clone();
let notify = current_notify_interface_for_context(self.context.as_deref());
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
let req_params = rustfs_targets::extract_params_header(&req_headers);
let resp_elements =
build_event_resp_elements(&S3Response::new(DeleteObjectsOutput::default()), &request_context.request_id);
let deleted_any = delete_results.iter().any(|result| result.delete_object.is_some());
let notify_bucket = bucket.clone();
spawn_background_with_context(request_context, async move {
spawn_background_with_context(Some(request_context), async move {
let _activity_guard = DeleteTailActivityGuard::new(DeleteTailStage::Notify);
for res in delete_results {
if let Some(dobj) = res.delete_object {
@@ -6966,8 +6967,8 @@ impl DefaultObjectUsecase {
}),
)
.version_id(dobj.version_id.map(|v| v.to_string()).unwrap_or_default())
.req_params(extract_params_header(&req_headers))
.resp_elements(extract_resp_elements(&S3Response::new(DeleteObjectsOutput::default())))
.req_params(req_params.clone())
.resp_elements(resp_elements.clone())
.host(get_request_host(&req_headers))
.user_agent(get_request_user_agent(&req_headers))
.build();
@@ -7849,6 +7850,7 @@ impl DefaultObjectUsecase {
#[instrument(level = "debug", skip(self, req))]
pub async fn execute_put_object_extract(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
let helper = OperationHelper::new(&req, EventName::ObjectCreatedPut, S3Operation::PutObject).suppress_event();
let request_context = helper.request_context_or_from_request(&req);
let auth_method = req.method.clone();
let auth_uri = req.uri.clone();
let auth_headers = req.headers.clone();
@@ -8025,7 +8027,7 @@ impl DefaultObjectUsecase {
};
let notify = current_notify_interface_for_context(self.context.as_deref());
let req_params = extract_params_header(&req.headers);
let req_params = rustfs_targets::extract_params_header(&req.headers);
let host = get_request_host(&req.headers);
let port = get_request_port(&req.headers);
let user_agent = get_request_user_agent(&req.headers);
@@ -8254,7 +8256,7 @@ impl DefaultObjectUsecase {
bucket_name: bucket.clone(),
object: convert_ecstore_object_info(obj_info.clone()),
req_params: req_params.clone(),
resp_elements: extract_resp_elements(&S3Response::new(output.clone())),
resp_elements: build_event_resp_elements(&S3Response::new(output.clone()), &request_context.request_id),
version_id: version_id.clone(),
host: host.clone(),
port,
@@ -8262,8 +8264,7 @@ impl DefaultObjectUsecase {
};
let notify = notify.clone();
let request_context = req.extensions.get::<request_context::RequestContext>().cloned();
spawn_background_with_context(request_context, async move {
spawn_background_with_context(Some(request_context.clone()), async move {
notify.notify(event_args).await;
});
}
rustfs/src/app/storage_api.rs
+3 -1
View File
@@ -853,7 +853,9 @@ pub(crate) mod head_prefix {
}
pub(crate) mod helper {
pub(crate) use crate::storage::storage_api::helper_consumer::{OperationHelper, spawn_background_with_context};
pub(crate) use crate::storage::storage_api::helper_consumer::{
OperationHelper, build_event_resp_elements, spawn_background_with_context,
};
}
pub(crate) mod object_utils {
rustfs/src/embedded.rs
+88
View File
@@ -60,6 +60,94 @@ use std::net::SocketAddr;
use std::path::PathBuf;
use tokio_util::sync::CancellationToken;
#[cfg(feature = "e2e-test-hooks")]
use std::sync::{Mutex, OnceLock};
#[cfg(feature = "e2e-test-hooks")]
use tokio::sync::oneshot;
#[cfg(feature = "e2e-test-hooks")]
struct EmbeddedStartupHook {
port: u16,
http_bound: oneshot::Sender<()>,
release: oneshot::Receiver<()>,
}
#[cfg(feature = "e2e-test-hooks")]
static EMBEDDED_STARTUP_HOOK: OnceLock<Mutex<Option<EmbeddedStartupHook>>> = OnceLock::new();
/// Test-only gate for the point after an embedded HTTP listener binds and
/// before its application context is installed.
#[cfg(feature = "e2e-test-hooks")]
#[doc(hidden)]
pub struct EmbeddedStartupBarrier {
http_bound: oneshot::Receiver<()>,
release: Option<oneshot::Sender<()>>,
}
#[cfg(feature = "e2e-test-hooks")]
impl EmbeddedStartupBarrier {
/// Wait until the next embedded server has bound its HTTP listener.
pub async fn wait_until_http_bound(&mut self) {
(&mut self.http_bound)
.await
.expect("embedded startup hook must signal after binding HTTP");
}
/// Allow the paused embedded startup to install its application context.
pub fn release(mut self) {
if let Some(release) = self.release.take() {
let _ = release.send(());
}
}
}
#[cfg(feature = "e2e-test-hooks")]
impl Drop for EmbeddedStartupBarrier {
fn drop(&mut self) {
if let Some(release) = self.release.take() {
let _ = release.send(());
}
}
}
/// Pause the embedded startup for `port` after its HTTP listener binds.
#[cfg(feature = "e2e-test-hooks")]
#[doc(hidden)]
pub fn pause_embedded_startup_after_http_bind(port: u16) -> EmbeddedStartupBarrier {
let (http_bound_tx, http_bound) = oneshot::channel();
let (release, release_rx) = oneshot::channel();
let hook = EmbeddedStartupHook {
port,
http_bound: http_bound_tx,
release: release_rx,
};
let mut active_hook = EMBEDDED_STARTUP_HOOK
.get_or_init(|| Mutex::new(None))
.lock()
.expect("embedded startup hook lock must not be poisoned");
assert!(active_hook.replace(hook).is_none(), "only one embedded startup hook may be active");
EmbeddedStartupBarrier {
http_bound,
release: Some(release),
}
}
#[cfg(feature = "e2e-test-hooks")]
pub(crate) async fn wait_for_embedded_startup_hook(port: u16) {
let hook = {
let mut active_hook = EMBEDDED_STARTUP_HOOK
.get_or_init(|| Mutex::new(None))
.lock()
.expect("embedded startup hook lock must not be poisoned");
active_hook.take_if(|hook| hook.port == port)
};
if let Some(hook) = hook {
let _ = hook.http_bound.send(());
let _ = hook.release.await;
}
}
/// Error type for embedded server operations.
#[derive(Debug)]
pub enum ServerError {
rustfs/src/init.rs
+165 -105
View File
@@ -15,6 +15,7 @@
use crate::runtime_sources::current_region;
use crate::server::ShutdownHandle;
use crate::server::runtime_sources::current_notify_interface;
use crate::storage_api::startup::bucket_metadata::contract::bucket::{BucketOperations, BucketOptions};
use crate::storage_api::startup::init::{
get_bucket_notification_config, process_lambda_configurations, process_queue_configurations, process_topic_configurations,
};
@@ -23,11 +24,14 @@ use rustfs_config::{
DEFAULT_BUFFER_MAX_SIZE, DEFAULT_BUFFER_MIN_SIZE, DEFAULT_BUFFER_PROFILE, DEFAULT_BUFFER_UNKNOWN_SIZE, DEFAULT_UPDATE_CHECK,
ENV_RUSTFS_BUFFER_DEFAULT_SIZE, ENV_RUSTFS_BUFFER_MAX_SIZE, ENV_RUSTFS_BUFFER_MIN_SIZE, ENV_UPDATE_CHECK, RUSTFS_REGION,
};
use rustfs_targets::arn::{ARN, TargetIDError};
use rustfs_notify::NotificationError;
use rustfs_s3_types::EventName;
use rustfs_targets::arn::{ARN, TargetID, TargetIDError};
use rustfs_utils::get_env_usize;
use s3s::s3_error;
use std::env;
use std::io::Error;
use std::sync::Arc;
use tracing::{debug, error, info, instrument, warn};
const LOG_COMPONENT_INIT: &str = "init";
@@ -41,6 +45,8 @@ const LOG_SUBSYSTEM_PROTOCOL: &str = "protocol";
const EVENT_PROTOCOL_RUNTIME_STATE: &str = "protocol_runtime_state";
const EVENT_PROTOCOL_SERVER_STATE: &str = "protocol_server_state";
type NotificationEventRule = (Vec<EventName>, String, String, Vec<TargetID>);
#[instrument]
pub fn print_server_info() {
let current_year = jiff::Zoned::now().year();
@@ -151,6 +157,57 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
.map_err(|e| TargetIDError::InvalidFormat(e.to_string()))
}
fn notification_config_to_event_rules(
cfg: &s3s::dto::NotificationConfiguration,
) -> Result<Vec<NotificationEventRule>, TargetIDError> {
let mut event_rules = Vec::new();
process_queue_configurations(&mut event_rules, cfg.queue_configurations.clone(), arn_to_target_id)?;
process_topic_configurations(&mut event_rules, cfg.topic_configurations.clone(), arn_to_target_id)?;
process_lambda_configurations(&mut event_rules, cfg.lambda_function_configurations.clone(), arn_to_target_id)?;
Ok(event_rules)
}
async fn apply_bucket_notification_configuration(bucket: &str, region: &str) -> Result<bool, NotificationError> {
let has_notification_config = get_bucket_notification_config(bucket)
.await
.map_err(|err| NotificationError::StorageNotAvailable(format!("load bucket notification config for {bucket}: {err}")))?;
match has_notification_config {
Some(cfg) => {
info!(
target: "rustfs::init",
event = "notification_config_loaded",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
queue_configuration_count = cfg.queue_configurations.as_ref().map_or(0, Vec::len),
topic_configuration_count = cfg.topic_configurations.as_ref().map_or(0, Vec::len),
lambda_configuration_count = cfg.lambda_function_configurations.as_ref().map_or(0, Vec::len),
"Loaded bucket notification configuration"
);
let event_rules =
notification_config_to_event_rules(&cfg).map_err(|err| NotificationError::BucketNotification(err.to_string()))?;
current_notify_interface()
.add_event_specific_rules(bucket, region, &event_rules)
.await?;
Ok(true)
}
None => {
info!(
target: "rustfs::init",
event = "notification_config_missing",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
"Bucket notification configuration not found"
);
current_notify_interface().clear_bucket_notification_rules(bucket).await?;
Ok(false)
}
}
}
/// Add existing bucket notification configurations to the global notifier system.
/// This function retrieves notification configurations for each bucket
/// and registers the corresponding event rules with the notifier system.
@@ -159,11 +216,52 @@ fn arn_to_target_id(arn_str: &str) -> Result<rustfs_targets::arn::TargetID, Targ
/// * `buckets` - A vector of bucket names to process
#[instrument(skip_all)]
pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
let region = notification_region();
for bucket in buckets.iter() {
if let Err(err) = apply_bucket_notification_configuration(bucket, region.as_str()).await {
let err = s3_error!(InternalError, "Failed to add rules: {err}");
error!(
target: "rustfs::init",
event = "notification_rules_registration_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
region,
error = ?err,
"Failed to register bucket notification rules"
);
}
}
}
pub(crate) async fn reconcile_persisted_bucket_notification_configurations(
store: Arc<rustfs_notify::NotifyStore>,
) -> Result<usize, NotificationError> {
let bucket_infos = store
.list_bucket(&BucketOptions {
no_metadata: true,
..Default::default()
})
.await
.map_err(|err| NotificationError::StorageNotAvailable(format!("list buckets for notification reconciliation: {err}")))?;
let region = notification_region();
let mut configured_bucket_count = 0;
for bucket in bucket_infos {
if apply_bucket_notification_configuration(&bucket.name, region.as_str()).await? {
configured_bucket_count += 1;
}
}
Ok(configured_bucket_count)
}
fn notification_region() -> String {
let global_region = current_region();
let region = global_region
global_region
.as_ref()
.filter(|r| !r.as_str().is_empty())
.map(|r| r.as_str())
.map(|r| r.to_string())
.unwrap_or_else(|| {
warn!(
target: "rustfs::init",
@@ -173,107 +271,8 @@ pub async fn add_bucket_notification_configuration(buckets: Vec<String>) {
fallback_region = RUSTFS_REGION,
"Notification configuration falling back to default region"
);
RUSTFS_REGION
});
for bucket in buckets.iter() {
let has_notification_config = get_bucket_notification_config(bucket).await.unwrap_or_else(|err| {
warn!(
target: "rustfs::init",
event = "notification_config_load_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
error = ?err,
"Failed to load bucket notification configuration"
);
None
});
match has_notification_config {
Some(cfg) => {
info!(
target: "rustfs::init",
event = "notification_config_loaded",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
queue_configuration_count = cfg.queue_configurations.as_ref().map_or(0, Vec::len),
topic_configuration_count = cfg.topic_configurations.as_ref().map_or(0, Vec::len),
lambda_configuration_count = cfg.lambda_function_configurations.as_ref().map_or(0, Vec::len),
"Loaded bucket notification configuration"
);
let mut event_rules = Vec::new();
if let Err(e) = process_queue_configurations(&mut event_rules, cfg.queue_configurations.clone(), arn_to_target_id)
{
error!(
target: "rustfs::init",
event = "notification_config_parse_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
config_type = "queue",
error = ?e,
"Failed to parse bucket notification configuration"
);
}
if let Err(e) = process_topic_configurations(&mut event_rules, cfg.topic_configurations.clone(), arn_to_target_id)
{
error!(
target: "rustfs::init",
event = "notification_config_parse_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
config_type = "topic",
error = ?e,
"Failed to parse bucket notification configuration"
);
}
if let Err(e) =
process_lambda_configurations(&mut event_rules, cfg.lambda_function_configurations.clone(), arn_to_target_id)
{
error!(
target: "rustfs::init",
event = "notification_config_parse_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
config_type = "lambda",
error = ?e,
"Failed to parse bucket notification configuration"
);
}
if let Err(e) = current_notify_interface()
.add_event_specific_rules(bucket, region, &event_rules)
.await
.map_err(|e| s3_error!(InternalError, "Failed to add rules: {e}"))
{
error!(
target: "rustfs::init",
event = "notification_rules_registration_failed",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
region,
error = ?e,
"Failed to register bucket notification rules"
);
}
}
None => {
info!(
target: "rustfs::init",
event = "notification_config_missing",
component = LOG_COMPONENT_INIT,
subsystem = LOG_SUBSYSTEM_NOTIFICATION,
bucket = %bucket,
"Bucket notification configuration not found"
);
}
}
}
RUSTFS_REGION.to_string()
})
}
/// Build KMS configuration for local backend
@@ -1354,9 +1353,13 @@ pub async fn init_sftp_system() -> Result<Option<ShutdownHandle>, Box<dyn std::e
#[cfg(test)]
mod tests {
use super::resolve_buffer_profile_config;
use super::{notification_config_to_event_rules, resolve_buffer_profile_config};
use crate::config::{BufferConfig, WorkloadProfile};
use rustfs_config::KI_B;
use rustfs_s3_types::EventName;
use s3s::dto::{
FilterRule, FilterRuleName, NotificationConfiguration, NotificationConfigurationFilter, QueueConfiguration, S3KeyFilter,
};
#[test]
fn resolve_buffer_profile_config_returns_fallback_when_primary_is_invalid() {
@@ -1386,4 +1389,61 @@ mod tests {
assert!(resolved.is_none());
}
#[test]
fn notification_config_to_event_rules_preserves_target_and_filters() {
let cfg = NotificationConfiguration {
queue_configurations: Some(vec![QueueConfiguration {
events: vec!["s3:ObjectCreated:Put".to_string().into()],
queue_arn: "arn:rustfs:sqs:us-east-1:rustfs_to_activemq:mqtt".to_string(),
filter: Some(NotificationConfigurationFilter {
key: Some(S3KeyFilter {
filter_rules: Some(vec![
FilterRule {
name: Some(FilterRuleName::from_static(FilterRuleName::PREFIX)),
value: Some("uploads/".to_string()),
},
FilterRule {
name: Some(FilterRuleName::from_static(FilterRuleName::SUFFIX)),
value: Some(".json".to_string()),
},
]),
}),
}),
id: Some("primary".to_string()),
}]),
topic_configurations: None,
lambda_function_configurations: None,
event_bridge_configuration: None,
};
let rules = notification_config_to_event_rules(&cfg).expect("valid notification config should map to event rules");
assert_eq!(rules.len(), 1);
assert_eq!(rules[0].0, vec![EventName::ObjectCreatedPut]);
assert_eq!(rules[0].1, "uploads/");
assert_eq!(rules[0].2, ".json");
assert_eq!(rules[0].3.len(), 1);
assert_eq!(rules[0].3[0].id, "rustfs_to_activemq");
assert_eq!(rules[0].3[0].name, "mqtt");
}
#[test]
fn notification_config_to_event_rules_rejects_invalid_arn() {
let cfg = NotificationConfiguration {
queue_configurations: Some(vec![QueueConfiguration {
events: vec!["s3:ObjectCreated:Put".to_string().into()],
queue_arn: "arn:aws:sqs:us-east-1:rustfs_to_activemq:mqtt".to_string(),
filter: None,
id: None,
}]),
topic_configurations: None,
lambda_function_configurations: None,
event_bridge_configuration: None,
};
let err = notification_config_to_event_rules(&cfg).expect_err("invalid ARN partition must fail");
assert!(err.to_string().contains("Invalid ARN"), "unexpected error: {err}");
}
}
rustfs/src/server/event.rs
+57 -3
View File
@@ -16,6 +16,7 @@ use super::{
module_switch::{resolve_notify_module_state, validate_notify_module_env, with_refreshed_notify_module_state_from},
refresh_persisted_module_switches_from, runtime_sources,
};
use crate::init::reconcile_persisted_bucket_notification_configurations;
use crate::storage_api::server::event::{
EventArgs as EcstoreEventArgs, StorageObjectInfo, read_existing_server_config_no_lock, register_event_dispatch_hook,
with_server_config_read_lock,
@@ -38,6 +39,7 @@ use tracing::{info, instrument, warn};
static NOTIFY_MODULE_ENABLED: AtomicBool = AtomicBool::new(rustfs_config::DEFAULT_NOTIFY_ENABLE);
static NOTIFY_RUNTIME_RECONCILED: AtomicBool = AtomicBool::new(false);
static NOTIFY_BUCKET_RULES_RECONCILED: AtomicBool = AtomicBool::new(false);
static ECSTORE_EVENT_DISPATCH_HOOK: OnceLock<()> = OnceLock::new();
const EVENT_NOTIFIER_RECONCILE_INTERVAL: Duration = Duration::from_secs(5);
@@ -54,6 +56,19 @@ pub(crate) fn mark_event_notifier_reconciled() {
pub(crate) fn mark_event_notifier_unreconciled() {
NOTIFY_RUNTIME_RECONCILED.store(false, Ordering::Release);
NOTIFY_BUCKET_RULES_RECONCILED.store(false, Ordering::Release);
}
fn are_bucket_notification_rules_reconciled() -> bool {
NOTIFY_BUCKET_RULES_RECONCILED.load(Ordering::Acquire)
}
fn mark_bucket_notification_rules_reconciled() {
NOTIFY_BUCKET_RULES_RECONCILED.store(true, Ordering::Release);
}
fn should_reconcile_bucket_notification_rules(runtime_changed: bool, notify_enabled: bool) -> bool {
notify_enabled && (runtime_changed || !are_bucket_notification_rules_reconciled())
}
pub fn refresh_notify_module_enabled() -> bool {
@@ -180,7 +195,7 @@ pub(crate) async fn reconcile_event_notifier_from_store(
let system = ensure_live_events_initialized();
let transition_system = system.clone();
let transition_store = store.clone();
let transition = with_refreshed_notify_module_state_from(store, move |resolution| async move {
let transition = with_refreshed_notify_module_state_from(store.clone(), move |resolution| async move {
NOTIFY_MODULE_ENABLED.store(resolution.enabled, Ordering::Relaxed);
let read_store = transition_store.clone();
let config_system = transition_system.clone();
@@ -208,11 +223,25 @@ pub(crate) async fn reconcile_event_notifier_from_store(
.await
.map_err(|err| NotificationError::Initialization(format!("failed to refresh notify module switch: {err}")))??;
let runtime_changed = transition.is_some();
if let Some(transition) = transition {
transition.wait().await?;
}
ensure_event_notifier_converged(&system)
ensure_event_notifier_converged(&system)?;
if should_reconcile_bucket_notification_rules(runtime_changed, is_notify_module_enabled()) {
let configured_bucket_count = reconcile_persisted_bucket_notification_configurations(store).await?;
mark_bucket_notification_rules_reconciled();
info!(
event = EVENT_NOTIFY_RUNTIME_RECONCILE,
component = "notify",
subsystem = "bucket_rules",
configured_bucket_count,
"Persisted bucket notification rules reconciled"
);
}
Ok(())
}
.await;
@@ -405,7 +434,8 @@ mod tests {
set_persisted_module_switches,
};
use super::{
convert_ecstore_object_info, init_event_notifier_with_store, parse_host_and_port, run_persisted_event_notifier_reconciler,
convert_ecstore_object_info, init_event_notifier_with_store, mark_bucket_notification_rules_reconciled,
parse_host_and_port, run_persisted_event_notifier_reconciler, should_reconcile_bucket_notification_rules,
};
use crate::server::is_event_notifier_reconciled;
use crate::storage_api::server::event::StorageObjectInfo;
@@ -533,6 +563,30 @@ mod tests {
assert_eq!(converted.transitioned_tier.as_deref(), Some("DEEP_ARCHIVE"));
}
#[test]
fn bucket_rule_reconcile_runs_once_and_after_runtime_change() {
super::mark_event_notifier_unreconciled();
assert!(
should_reconcile_bucket_notification_rules(false, true),
"enabled notify must restore bucket rules until the first successful replay"
);
mark_bucket_notification_rules_reconciled();
assert!(
!should_reconcile_bucket_notification_rules(false, true),
"steady-state reconcile must not rescan buckets every tick"
);
assert!(
should_reconcile_bucket_notification_rules(true, true),
"target runtime changes must replay persisted bucket rules"
);
assert!(
!should_reconcile_bucket_notification_rules(true, false),
"disabled notify must not load external target rules"
);
}
#[tokio::test(start_paused = true)]
async fn persisted_reconciler_converges_after_one_injected_tick() {
let persisted_generation = Arc::new(AtomicUsize::new(1));
rustfs/src/server/http.rs
+60 -82
View File
@@ -23,9 +23,9 @@ use crate::server::{
hybrid::hybrid,
layer::{
BodylessStatusFixLayer, ConditionalCorsLayer, DoubleSlashListBucketsCompatLayer, EmptyBodyContentLengthCompatLayer,
HeadRequestBodyFixLayer, IcebergRestErrorCompatLayer, ObjectAttributesEtagFixLayer, PublicHealthEndpointLayer,
RedirectLayer, RequestContextLayer, RequestLoggingLayer, S3ErrorMessageCompatLayer, StsQueryApiCompatLayer,
VirtualHostStyleHintLayer, redact_sensitive_uri_query,
ExternalRequestContextLayer, HeadRequestBodyFixLayer, IcebergRestErrorCompatLayer, ObjectAttributesEtagFixLayer,
PublicHealthEndpointLayer, RedirectLayer, RequestContextLayer, RequestLoggingLayer, S3ErrorMessageCompatLayer,
StsQueryApiCompatLayer, VirtualHostStyleHintLayer, redact_sensitive_uri_query,
},
rate_limit::{RateLimitLayer, api_rate_limit_layer_from_env},
tls_material::{
@@ -34,11 +34,11 @@ use crate::server::{
},
};
use crate::storage_api::server::http as storage;
use crate::storage_api::server::http::request_context::{RequestContext, extract_request_id_from_headers};
use crate::storage_api::server::http::rpc::InternodeRpcService;
use crate::storage_api::server::http::tonic_service::make_server;
use crate::storage_api::server::http::{
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
};
use bytes::Bytes;
use http::{HeaderMap, Method, Request as HttpRequest, Response, Uri};
@@ -153,13 +153,17 @@ impl<S> RpcRequestPathService<S> {
}
}
impl<S, B> Service<HttpRequest<B>> for RpcRequestPathService<S>
impl<S, B, ResBody> Service<HttpRequest<B>> for RpcRequestPathService<S>
where
S: Service<HttpRequest<B>>,
S: Service<HttpRequest<B>, Response = Response<ResBody>>,
S::Error: Send + 'static,
S::Future: Send + 'static,
B: Send + 'static,
ResBody: Send + 'static,
{
type Response = S::Response;
type Response = Response<ResBody>;
type Error = S::Error;
type Future = S::Future;
type Future = Pin<Box<dyn Future<Output = std::result::Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
self.inner.poll_ready(cx)
@@ -171,7 +175,22 @@ where
method: req.method().clone(),
};
req.extensions_mut().insert(target);
self.inner.call(req)
let response_headers = tonic_boot_epoch_challenge(req.headers())
.ok()
.flatten()
.and_then(|challenge| {
storage::try_current_local_node_name()
.and_then(|node| normalize_tonic_rpc_audience(&node).ok())
.and_then(|audience| tonic_boot_epoch_response_headers(&audience, challenge).ok())
});
let future = self.inner.call(req);
Box::pin(async move {
let mut response = future.await?;
if let Some(headers) = response_headers {
response.headers_mut().extend(headers);
}
Ok(response)
})
}
}
@@ -1135,47 +1154,6 @@ struct PathDispatchService<A, B> {
internode: B,
}
#[derive(Clone, Default)]
struct InternodeRequestContextLiteLayer;
impl<S> tower::Layer<S> for InternodeRequestContextLiteLayer {
type Service = InternodeRequestContextLiteService<S>;
fn layer(&self, inner: S) -> Self::Service {
InternodeRequestContextLiteService { inner }
}
}
#[derive(Clone)]
struct InternodeRequestContextLiteService<S> {
inner: S,
}
impl<S, B> Service<HttpRequest<B>> for InternodeRequestContextLiteService<S>
where
S: Service<HttpRequest<B>> + Clone,
{
type Response = S::Response;
type Error = S::Error;
type Future = S::Future;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<std::result::Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut req: HttpRequest<B>) -> Self::Future {
let request_id = extract_request_id_from_headers(req.headers());
req.extensions_mut().insert(RequestContext {
x_amz_request_id: request_id.clone(),
request_id,
trace_id: None,
span_id: None,
start_time: std::time::Instant::now(),
});
self.inner.call(req)
}
}
impl<A, B> PathDispatchService<A, B> {
fn new(external: A, internode: B) -> Self {
Self { external, internode }
@@ -1388,34 +1366,28 @@ fn process_connection(
// 1. AddExtensionLayer<RemoteAddr> — per-connection peer address
// 2. AddExtensionLayer<SocketAddr> — per-connection raw socket addr (TrustedProxy)
// 3. TrustedProxyLayer — conditional, parses X-Forwarded-For
// 4. SetRequestIdLayer — generates X-Request-ID
// 5. RequestContextLayer — creates RequestContext in extensions
// 6. StsQueryApiCompatLayer — route-scoped STS envelopes, including outer short-circuit errors
// 7. EmptyBodyContentLengthCompatLayer — adds Content-Length: 0 for known empty-body API routes
// 8. CatchPanicLayer — panic → 500
// 9. RateLimitLayer — conditional (external stack only), per-client 429 throttling
// 10. ReadinessGateLayer — blocks until ready
// 11. KeystoneAuthLayer — X-Auth-Token validation
// 12. TraceLayer — request span creation + metrics
// 13. RequestLoggingLayer — single completion event per request
// 14. PropagateRequestIdLayer — X-Request-ID → response
// 15. CompressionLayer — response compression (whitelist, path-aware)
// 16. PathCategoryInjectionLayer — injects path category for compression predicate
// 17. S3ErrorMessageCompatLayer — missing S3 error message compatibility
// 18. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
// 19. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
// 20. ConditionalCorsLayer — S3 API CORS
// 21. RedirectLayer — console redirect (conditional)
// 22. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
// 23. HeadRequestBodyFixLayer — strips actual body bytes from HEAD responses
// 24. PublicHealthEndpointLayer — handles public health before s3s host parsing
// 25. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
// 26. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
// 4. ExternalRequestContextLayer — S3 canonical ID / control-plane propagated ID
// 5. StsQueryApiCompatLayer — route-scoped STS envelopes, including outer short-circuit errors
// 6. EmptyBodyContentLengthCompatLayer — adds Content-Length: 0 for known empty-body API routes
// 7. CatchPanicLayer — panic → 500
// 8. RateLimitLayer — conditional (external stack only), per-client 429 throttling
// 9. ReadinessGateLayer — blocks until ready
// 10. KeystoneAuthLayer — X-Auth-Token validation
// 11. TraceLayer — request span creation + metrics
// 12. RequestLoggingLayer — single completion event per request
// 13. CompressionLayer — response compression (whitelist, path-aware)
// 14. PathCategoryInjectionLayer — injects path category for compression predicate
// 15. S3ErrorMessageCompatLayer — missing S3 error message compatibility
// 16. IcebergRestErrorCompatLayer — Iceberg REST JSON error compatibility
// 17. ObjectAttributesEtagFixLayer — ETag fix for GetObjectAttributes
// 18. ConditionalCorsLayer — S3 API CORS
// 19. RedirectLayer — console redirect (conditional)
// 20. BodylessStatusFixLayer — clears body for 1xx/204/205/304 responses
// 21. HeadRequestBodyFixLayer — strips actual body bytes from HEAD responses
// 22. PublicHealthEndpointLayer — handles public health before s3s host parsing
// 23. VirtualHostStyleHintLayer — actionable error for unroutable virtual-hosted-style (conditional)
// 24. DoubleSlashListBucketsCompatLayer — rewrites `GET //` to `GET /` for ListBuckets (MinIO browser compat)
// ─────────────────────────────────────────────────────────────
// Batch 1 intentionally keeps the external and internode stacks behaviorally
// identical while giving each path family a named construction boundary.
// Later batches will trim internode-only middleware without risking drift in
// the public HTTP stack.
let build_external_stack = |service| {
ServiceBuilder::new()
// NOTE: Both extension types are intentionally inserted to maintain compatibility:
@@ -1430,8 +1402,7 @@ fn process_connection(
// This should be placed before TraceLayer so that logs reflect the real client IP
// Pre-computed in ConnectionContext to avoid per-connection is_enabled() check.
.option_layer(trusted_proxy_layer.clone())
.layer(SetRequestIdLayer::x_request_id(MakeRequestUuid))
.layer(InternodeRequestContextLiteLayer)
.layer(ExternalRequestContextLayer::new(is_console))
.layer(StsQueryApiCompatLayer)
.layer(EmptyBodyContentLengthCompatLayer)
.layer(CatchPanicLayer::new())
@@ -1584,7 +1555,6 @@ fn process_connection(
}),
)
.layer(RequestLoggingLayer)
.layer(PropagateRequestIdLayer::x_request_id())
.layer(CompressionLayer::new().compress_when(PathAwareHttpCompressionPredicate::new(compression_config.clone())))
.layer(PathCategoryInjectionLayer)
.layer(S3ErrorMessageCompatLayer)
@@ -1912,7 +1882,15 @@ fn check_auth(req: Request<()>) -> std::result::Result<Request<()>, Status> {
.filter(|method| !method.is_empty() && !method.contains('/'))
.ok_or_else(|| Status::unauthenticated("Invalid RPC request path"))?;
debug_assert!(!rpc_method.is_empty());
verify_tonic_rpc_signature(&audience, target.uri.path(), req.metadata().as_ref()).map_err(|e| {
let allow_replay_scope_bootstrap = target.uri.path() == "/node_service.NodeService/Ping"
&& tonic_boot_epoch_challenge(req.metadata().as_ref()).is_ok_and(|challenge| challenge.is_some());
verify_tonic_rpc_signature_with_bootstrap(
&audience,
target.uri.path(),
req.metadata().as_ref(),
allow_replay_scope_bootstrap,
)
.map_err(|e| {
error!(
event = EVENT_RPC_SIGNATURE_VERIFICATION_FAILED,
component = LOG_COMPONENT_SERVER,
rustfs/src/server/hybrid.rs
+13 -6
View File
@@ -28,6 +28,13 @@ pub(crate) fn hybrid<MakeRest, Grpc>(make_rest: MakeRest, grpc: Grpc) -> HybridS
HybridService { rest: make_rest, grpc }
}
pub(crate) fn is_grpc_request<B>(req: &Request<B>) -> bool {
matches!(
(req.version(), req.headers().get(hyper::header::CONTENT_TYPE)),
(hyper::Version::HTTP_2, Some(value)) if value.as_bytes().starts_with(b"application/grpc")
)
}
/// The service that can serve both gRPC and REST HTTP Requests
#[derive(Clone)]
pub struct HybridService<Rest, Grpc> {
@@ -63,14 +70,14 @@ where
/// and if the Content-Type is "application/grpc"; otherwise, the request is served
/// as a REST request
fn call(&mut self, req: Request<Incoming>) -> Self::Future {
match (req.version(), req.headers().get(hyper::header::CONTENT_TYPE)) {
(hyper::Version::HTTP_2, Some(hv)) if hv.as_bytes().starts_with(b"application/grpc") => HybridFuture::Grpc {
if is_grpc_request(&req) {
HybridFuture::Grpc {
grpc_future: self.grpc.call(req),
},
_ => HybridFuture::Rest {
}
} else {
HybridFuture::Rest {
rest_future: self.rest.call(req),
},
}
}
}
}
rustfs/src/server/layer.rs
+570 -50
View File
@@ -17,27 +17,28 @@ use crate::admin::console::is_console_path;
use crate::error::ApiError;
use crate::server::RemoteAddr;
use crate::server::cors;
use crate::server::hybrid::HybridBody;
use crate::server::hybrid::{HybridBody, is_grpc_request};
use crate::server::{
ADMIN_PREFIX, CONSOLE_PREFIX, HEALTH_COMPAT_LIVE_PATH, HEALTH_PREFIX, HEALTH_READY_PATH, HealthProbe, MINIO_ADMIN_PREFIX,
MINIO_ADMIN_V3_PREFIX, MINIO_HEALTH_CLUSTER_PATH, MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH,
MINIO_HEALTH_READY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests, build_health_response_parts,
collect_probe_readiness, has_path_prefix, is_admin_path, is_table_catalog_path,
MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH, RPC_PREFIX, RUSTFS_ADMIN_PREFIX, active_http_requests,
build_health_response_parts, collect_probe_readiness, has_path_prefix, is_admin_path, is_table_catalog_path,
};
use crate::storage_api::server::layer::apply_cors_headers;
use crate::storage_api::server::layer::request_context::{
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
};
use crate::storage_api::server::layer::request_context::{RequestContext, extract_request_id_from_headers, spawn_traced};
use bytes::{Bytes, BytesMut};
use futures::future::Either;
use http::{HeaderMap, HeaderValue, Method, Request as HttpRequest, Response, StatusCode, Uri};
use http_body::Body;
use http_body_util::{BodyExt, Full};
use hyper::body::Incoming;
use pin_project_lite::pin_project;
use quick_xml::events::Event;
#[cfg(feature = "swift")]
use rustfs_protocols::swift::SwiftRouter;
use rustfs_trusted_proxies::ClientInfo;
use rustfs_utils::get_env_opt_str;
use rustfs_utils::http::headers::AMZ_REQUEST_ID;
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
use s3s::S3ErrorCode;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
@@ -62,6 +63,8 @@ const HTTP_REQUEST_INFLIGHT_WARN_THRESHOLD: Duration = Duration::from_secs(5);
const STS_RESPONSE_METADATA_TAG: &str = "ResponseMetadata";
const STS_REQUEST_ID_TAG: &str = "RequestId";
const STS_SUCCESS_RESPONSE_TAGS: [&str; 2] = ["AssumeRoleResponse", "AssumeRoleWithWebIdentityResponse"];
#[cfg(feature = "swift")]
const SWIFT_API_PATH_PREFIX: &str = "/v1/";
pub(crate) fn redact_sensitive_uri_query(uri: &http::Uri) -> String {
let path = uri.path();
@@ -117,9 +120,6 @@ fn is_object_zip_download_path(path: &str) -> bool {
///
/// This layer must be placed after `SetRequestIdLayer` in the middleware stack,
/// as it reads the `x-request-id` header that `SetRequestIdLayer` generates.
///
/// Additionally, it preserves any upstream `x-amz-request-id` in the separate
/// `RequestContext.x_amz_request_id` field without mutating signed request headers.
#[derive(Clone, Default)]
pub struct RequestContextLayer;
@@ -150,35 +150,163 @@ where
}
fn call(&mut self, mut req: HttpRequest<B>) -> Self::Future {
let request_id = extract_request_id_from_headers(req.headers());
let (trace_id, span_id) = extract_trace_context_ids_from_headers(req.headers())
.map(|(trace_id, span_id)| (Some(trace_id), Some(span_id)))
.unwrap_or((None, None));
// Preserve the upstream x-amz-request-id if present as the S3 compatibility alias;
// otherwise mirror the canonical internal request_id.
let x_amz_request_id = req
.headers()
.get(AMZ_REQUEST_ID)
.and_then(|v| v.to_str().ok())
.map(String::from)
.unwrap_or_else(|| request_id.clone());
let ctx = RequestContext {
request_id,
x_amz_request_id,
trace_id,
span_id,
start_time: Instant::now(),
};
req.extensions_mut().insert(ctx);
let request_context = RequestContext::from_headers(req.headers());
req.extensions_mut().insert(request_context);
self.inner.call(req)
}
}
fn uses_server_owned_s3_request_id<B>(req: &HttpRequest<B>, console_redirect_enabled: bool) -> bool {
if is_grpc_request(req)
|| req.uri().path().starts_with(RPC_PREFIX)
|| is_sts_query_request(req.method(), req.uri(), req.headers())
|| (console_redirect_enabled && is_console_redirect_request(req))
{
return false;
}
#[cfg(feature = "swift")]
if req.uri().path().starts_with(SWIFT_API_PATH_PREFIX) && SwiftRouter::new(true, None).matches(req.uri()) {
return false;
}
let path = req.uri().path();
let method = req.method();
let is_admin_health_request =
(method == Method::GET || method == Method::HEAD) && matches!(path, HEALTH_PREFIX | HEALTH_READY_PATH);
let is_profile_request = method == Method::GET && matches!(path, PROFILE_CPU_PATH | PROFILE_MEMORY_PATH);
let is_public_health_alias_request = (method == Method::GET || method == Method::HEAD)
&& matches!(
path,
HEALTH_COMPAT_LIVE_PATH
| MINIO_HEALTH_LIVE_PATH
| MINIO_HEALTH_READY_PATH
| MINIO_HEALTH_CLUSTER_PATH
| MINIO_HEALTH_CLUSTER_READ_PATH
)
&& is_public_health_endpoint_request(method, path);
!(is_admin_path(path)
|| is_console_path(path)
|| is_admin_health_request
|| is_profile_request
|| is_public_health_alias_request)
}
/// Creates a server-owned context and response ID for S3 requests while
/// preserving the existing request-ID propagation contract for non-S3 routes.
#[derive(Clone, Default)]
pub struct ExternalRequestContextLayer {
console_redirect_enabled: bool,
}
impl ExternalRequestContextLayer {
pub(crate) fn new(console_redirect_enabled: bool) -> Self {
Self {
console_redirect_enabled,
}
}
}
impl<S> Layer<S> for ExternalRequestContextLayer {
type Service = ExternalRequestContextService<S>;
fn layer(&self, inner: S) -> Self::Service {
ExternalRequestContextService {
inner,
console_redirect_enabled: self.console_redirect_enabled,
}
}
}
#[derive(Clone)]
pub struct ExternalRequestContextService<S> {
inner: S,
console_redirect_enabled: bool,
}
impl<S, B, ResBody> Service<HttpRequest<B>> for ExternalRequestContextService<S>
where
S: Service<HttpRequest<B>, Response = Response<ResBody>>,
{
type Response = Response<ResBody>;
type Error = S::Error;
type Future = ExternalRequestContextFuture<S::Future>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, mut req: HttpRequest<B>) -> Self::Future {
let is_s3 = uses_server_owned_s3_request_id(&req, self.console_redirect_enabled);
let has_request_id = req.headers().contains_key(REQUEST_ID_HEADER);
let request_context = if is_s3 {
let request_context = RequestContext::from_external_headers(req.headers());
if !has_request_id && let Ok(request_id) = HeaderValue::from_str(&request_context.request_id) {
req.headers_mut().insert(REQUEST_ID_HEADER, request_id);
}
request_context
} else {
if !has_request_id {
let request_id = uuid::Uuid::new_v4().to_string();
if let Ok(request_id) = HeaderValue::from_str(&request_id) {
req.headers_mut().insert(REQUEST_ID_HEADER, request_id);
}
}
RequestContext::from_propagated_headers(req.headers())
};
let request_id = if is_s3 {
HeaderValue::from_str(&request_context.request_id).ok()
} else {
req.headers().get(REQUEST_ID_HEADER).cloned()
};
req.extensions_mut().insert(request_context);
ExternalRequestContextFuture {
inner: self.inner.call(req),
request_id,
is_s3,
}
}
}
pin_project! {
pub struct ExternalRequestContextFuture<F> {
#[pin]
inner: F,
request_id: Option<HeaderValue>,
is_s3: bool,
}
}
impl<F, ResBody, E> Future for ExternalRequestContextFuture<F>
where
F: Future<Output = Result<Response<ResBody>, E>>,
{
type Output = Result<Response<ResBody>, E>;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let this = self.project();
let mut response = match this.inner.poll(cx) {
Poll::Ready(Ok(response)) => response,
Poll::Ready(Err(error)) => return Poll::Ready(Err(error)),
Poll::Pending => return Poll::Pending,
};
if let Some(request_id) = this.request_id.take() {
if *this.is_s3 {
response.headers_mut().insert(REQUEST_ID_HEADER, request_id.clone());
response.headers_mut().insert(AMZ_REQUEST_ID, request_id);
} else if !response.headers().contains_key(REQUEST_ID_HEADER) {
response.headers_mut().insert(REQUEST_ID_HEADER, request_id);
}
}
Poll::Ready(Ok(response))
}
}
#[derive(Clone, Default)]
pub struct RequestLoggingLayer;
@@ -395,6 +523,18 @@ pub struct RedirectService<S> {
inner: S,
}
fn is_console_redirect_request<B>(req: &HttpRequest<B>) -> bool {
let path = req.uri().path().trim_end_matches('/');
req.method() == http::Method::GET
&& !req.headers().contains_key(http::header::AUTHORIZATION)
&& req
.headers()
.get(http::header::USER_AGENT)
.and_then(|value| value.to_str().ok())
.is_some_and(|user_agent| user_agent.contains("Mozilla"))
&& (path.is_empty() || path == "/rustfs" || path == "/index.html")
}
impl<S, RestBody, GrpcBody> Service<HttpRequest<Incoming>> for RedirectService<S>
where
S: Service<HttpRequest<Incoming>, Response = Response<HybridBody<RestBody, GrpcBody>>> + Clone + Send + 'static,
@@ -412,20 +552,8 @@ where
}
fn call(&mut self, req: HttpRequest<Incoming>) -> Self::Future {
// Check if this is a GET request without Authorization header and User-Agent contains Mozilla
// and the path is either "/" or "/index.html"
let path = req.uri().path().trim_end_matches('/');
let should_redirect = req.method() == http::Method::GET
&& !req.headers().contains_key(http::header::AUTHORIZATION)
&& req
.headers()
.get(http::header::USER_AGENT)
.and_then(|v| v.to_str().ok())
.map(|ua| ua.contains("Mozilla"))
.unwrap_or(false)
&& (path.is_empty() || path == "/rustfs" || path == "/index.html");
if should_redirect {
if is_console_redirect_request(&req) {
debug!("Redirecting browser request from {} to console", path);
// Create redirect response
@@ -1733,7 +1861,7 @@ impl ConditionalCorsLayer {
// Expose common headers
response_headers.insert(
cors::response::ACCESS_CONTROL_EXPOSE_HEADERS,
HeaderValue::from_static("x-request-id, content-type, content-length, etag"),
HeaderValue::from_static("x-request-id, x-amz-request-id, content-type, content-length, etag"),
);
// Credentials are only safe for origins matched from an explicit allow-list.
@@ -2046,12 +2174,25 @@ mod tests {
#[derive(Clone, Default)]
struct HeaderCaptureService {
headers: Arc<Mutex<Option<HeaderMap>>>,
request_context: Arc<Mutex<Option<RequestContext>>>,
response_request_id: Option<HeaderValue>,
}
impl HeaderCaptureService {
fn with_response_request_id(request_id: &'static str) -> Self {
Self {
response_request_id: Some(HeaderValue::from_static(request_id)),
..Self::default()
}
}
fn headers(&self) -> Arc<Mutex<Option<HeaderMap>>> {
Arc::clone(&self.headers)
}
fn request_context(&self) -> Arc<Mutex<Option<RequestContext>>> {
Arc::clone(&self.request_context)
}
}
impl<B: Send + 'static> Service<Request<B>> for HeaderCaptureService {
@@ -2065,10 +2206,383 @@ mod tests {
fn call(&mut self, req: Request<B>) -> Self::Future {
*self.headers.lock().expect("capture headers") = Some(req.headers().clone());
ready(Ok(Response::new(Full::from(Bytes::new()))))
*self.request_context.lock().expect("capture request context") = req.extensions().get::<RequestContext>().cloned();
let mut response = Response::new(Full::from(Bytes::new()));
if let Some(request_id) = self.response_request_id.clone() {
response.headers_mut().insert(REQUEST_ID_HEADER, request_id);
}
ready(Ok(response))
}
}
async fn assert_non_s3_request_id_contract(mut request: Request<()>, route: &str) {
let capture = HeaderCaptureService::default();
let captured_context = capture.request_context();
let mut service = ExternalRequestContextLayer::default().layer(capture);
request
.headers_mut()
.insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
request
.headers_mut()
.insert(AMZ_REQUEST_ID, HeaderValue::from_static("client-amz-request-id"));
let response = service.call(request).await.expect("non-S3 response");
assert_eq!(
response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok()),
Some("client-request-id"),
"non-S3 x-request-id contract changed for {route}"
);
assert!(
!response.headers().contains_key(AMZ_REQUEST_ID),
"non-S3 response unexpectedly gained x-amz-request-id for {route}"
);
let context = captured_context
.lock()
.expect("captured request context")
.clone()
.expect("non-S3 request context");
assert_eq!(context.request_id, "client-request-id", "non-S3 context changed for {route}");
assert_eq!(context.x_amz_request_id, "client-request-id");
assert!(context.trace_id.is_none());
assert!(context.span_id.is_none());
}
#[tokio::test]
async fn external_request_context_rejects_client_request_id_as_canonical() {
global::set_text_map_propagator(TraceContextPropagator::new());
let capture = HeaderCaptureService::default();
let captured_headers = capture.headers();
let captured_context = capture.request_context();
let mut service = ExternalRequestContextLayer::default().layer(capture);
let mut request = Request::builder().uri("/bucket/object").body(()).expect("build S3 request");
request
.headers_mut()
.insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-supplied-request-id"));
request
.headers_mut()
.insert(AMZ_REQUEST_ID, HeaderValue::from_static("client-supplied-amz-request-id"));
request.headers_mut().insert(
"traceparent",
HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
);
let response = service.call(request).await.expect("response");
let request_id = response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("canonical request ID");
assert!(uuid::Uuid::parse_str(request_id).is_ok());
assert_ne!(request_id, "client-supplied-request-id");
assert_eq!(
response.headers().get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
Some(request_id)
);
let headers = captured_headers.lock().expect("captured headers");
let headers = headers.as_ref().expect("inner request headers");
assert_eq!(headers.get(REQUEST_ID_HEADER).expect("client x-request-id"), "client-supplied-request-id");
assert_eq!(
headers.get(AMZ_REQUEST_ID).expect("client x-amz-request-id"),
"client-supplied-amz-request-id"
);
assert_eq!(
headers.get("traceparent").expect("client traceparent"),
"00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"
);
let context = captured_context
.lock()
.expect("captured request context")
.clone()
.expect("S3 request context");
assert_eq!(context.request_id, request_id);
assert_eq!(context.trace_id.as_deref(), Some("4bf92f3577b34da6a3ce929d0e0e4736"));
assert_eq!(context.span_id.as_deref(), Some("00f067aa0ba902b7"));
}
#[tokio::test]
async fn external_request_context_replaces_empty_response_id() {
let capture = HeaderCaptureService::default();
let captured_headers = capture.headers();
let mut service = ExternalRequestContextLayer::default().layer(capture);
let mut request = Request::builder().uri("/bucket/object").body(()).expect("build S3 request");
request.headers_mut().insert(REQUEST_ID_HEADER, HeaderValue::from_static(""));
request.headers_mut().insert(AMZ_REQUEST_ID, HeaderValue::from_static(" "));
let response = service.call(request).await.expect("response");
let request_id = response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("canonical request ID");
assert!(uuid::Uuid::parse_str(request_id).is_ok());
assert_eq!(
response.headers().get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
Some(request_id)
);
let headers = captured_headers.lock().expect("captured headers");
let headers = headers.as_ref().expect("inner request headers");
assert_eq!(headers.get(REQUEST_ID_HEADER).expect("empty client x-request-id"), "");
assert_eq!(headers.get(AMZ_REQUEST_ID).expect("blank client x-amz-request-id"), " ");
}
#[tokio::test]
async fn external_request_context_inserts_generated_id_when_header_is_absent() {
let capture = HeaderCaptureService::default();
let captured_headers = capture.headers();
let mut service = ExternalRequestContextLayer::default().layer(capture);
let request = Request::builder().uri("/bucket/object").body(()).expect("build S3 request");
let response = service.call(request).await.expect("response");
let response_request_id = response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("canonical request ID");
let headers = captured_headers.lock().expect("captured headers");
let headers = headers.as_ref().expect("inner request headers");
assert_eq!(
headers.get(REQUEST_ID_HEADER).and_then(|value| value.to_str().ok()),
Some(response_request_id)
);
}
#[tokio::test]
async fn non_s3_request_id_contract_preserves_client_correlation() {
for path in [
"/rustfs/admin/v3/info",
"/minio/admin/v3/info",
"/rustfs/console/",
HEALTH_PREFIX,
"/iceberg/v1/config",
"/rustfs/rpc/v1/read-file",
"/rustfs/rpcx",
] {
let request = Request::builder().uri(path).body(()).expect("build non-S3 request");
assert_non_s3_request_id_contract(request, path).await;
}
}
#[tokio::test]
async fn non_s3_request_context_preserves_propagated_trace_context() {
global::set_text_map_propagator(TraceContextPropagator::new());
let capture = HeaderCaptureService::default();
let captured_context = capture.request_context();
let mut service = ExternalRequestContextLayer::default().layer(capture);
let request = Request::builder()
.uri("/rustfs/admin/v3/info")
.header(REQUEST_ID_HEADER, "client-request-id")
.header("traceparent", "00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01")
.body(())
.expect("build admin request");
let response = service.call(request).await.expect("admin response");
let context = captured_context
.lock()
.expect("captured request context")
.clone()
.expect("admin request context");
assert_eq!(
response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok()),
Some("client-request-id")
);
assert_eq!(context.request_id, "client-request-id");
assert_eq!(context.x_amz_request_id, "client-request-id");
assert_eq!(context.trace_id.as_deref(), Some("4bf92f3577b34da6a3ce929d0e0e4736"));
assert_eq!(context.span_id.as_deref(), Some("00f067aa0ba902b7"));
}
#[test]
fn console_redirect_request_id_contract_follows_redirect_enablement() {
for path in ["/", "/rustfs", "/index.html"] {
let request = Request::builder()
.method(Method::GET)
.uri(path)
.header(http::header::USER_AGENT, "Mozilla/5.0")
.body(())
.expect("build console redirect request");
assert!(!uses_server_owned_s3_request_id(&request, true));
assert!(uses_server_owned_s3_request_id(&request, false));
}
}
#[test]
fn method_scoped_control_routes_preserve_request_id_contract() {
for path in [HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH] {
let control_request = Request::builder()
.method(Method::GET)
.uri(path)
.body(())
.expect("build control-plane request");
assert!(!uses_server_owned_s3_request_id(&control_request, false));
let s3_request = Request::builder()
.method(Method::POST)
.uri(path)
.body(())
.expect("build S3 request");
assert!(uses_server_owned_s3_request_id(&s3_request, false));
}
}
#[test]
#[serial]
fn public_health_alias_request_id_contract_follows_enablement() {
let request = Request::builder()
.method(Method::GET)
.uri(MINIO_HEALTH_LIVE_PATH)
.body(())
.expect("build health request");
with_var(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("true"), || {
assert!(!uses_server_owned_s3_request_id(&request, false));
});
with_var(rustfs_config::ENV_HEALTH_ENDPOINT_ENABLE, Some("false"), || {
assert!(uses_server_owned_s3_request_id(&request, false));
});
}
#[tokio::test]
async fn grpc_request_id_contract_preserves_client_correlation() {
for path in [
"/node_service.NodeService/GetMetrics",
"/node_service.HealControlService/HealControl",
"/node_service.TierMutationControlService/PrepareTierMutation",
] {
let request = Request::builder()
.version(http::Version::HTTP_2)
.uri(path)
.header(http::header::CONTENT_TYPE, "application/grpc")
.body(())
.expect("build gRPC request");
assert_non_s3_request_id_contract(request, path).await;
}
}
#[tokio::test]
async fn sts_query_request_id_contract_preserves_client_correlation() {
let request = Request::builder()
.method(Method::POST)
.uri("/")
.header(http::header::CONTENT_TYPE, "application/x-www-form-urlencoded")
.body(())
.expect("build STS Query request");
assert_non_s3_request_id_contract(request, "STS Query").await;
}
#[cfg(feature = "swift")]
#[tokio::test]
async fn swift_request_id_contract_preserves_client_correlation() {
let path = "/v1/AUTH_project/container/object";
let request = Request::builder().uri(path).body(()).expect("build Swift request");
assert_non_s3_request_id_contract(request, path).await;
}
#[cfg(feature = "swift")]
#[test]
fn swift_request_id_classification_preserves_v1_prefix_boundary() {
let swift_request = Request::builder()
.uri("/v1/AUTH_project/container/object")
.body(())
.expect("build Swift request");
assert!(!uses_server_owned_s3_request_id(&swift_request, false));
let double_slash_request = Request::builder()
.uri("//v1/AUTH_project/container/object")
.body(())
.expect("build double-slash request");
assert!(uses_server_owned_s3_request_id(&double_slash_request, false));
}
#[cfg(feature = "swift")]
#[tokio::test]
async fn non_swift_v1_path_keeps_s3_request_id_contract() {
let capture = HeaderCaptureService::default();
let mut service = ExternalRequestContextLayer::default().layer(capture);
let mut request = Request::builder()
.uri("/v1/not-a-swift-account/object")
.body(())
.expect("build S3 request");
request
.headers_mut()
.insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
let response = service.call(request).await.expect("S3 response");
let response_request_id = response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("S3 response request ID");
assert!(uuid::Uuid::parse_str(response_request_id).is_ok());
assert_eq!(
response.headers().get(AMZ_REQUEST_ID).and_then(|value| value.to_str().ok()),
Some(response_request_id)
);
}
#[tokio::test]
async fn non_s3_response_preserves_handler_request_id() {
let capture = HeaderCaptureService::with_response_request_id("handler-request-id");
let mut service = ExternalRequestContextLayer::default().layer(capture);
let mut request = Request::builder()
.uri("/rustfs/admin/v3/info")
.body(())
.expect("build admin request");
request
.headers_mut()
.insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
let response = service.call(request).await.expect("admin response");
assert_eq!(
response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok()),
Some("handler-request-id")
);
assert!(!response.headers().contains_key(AMZ_REQUEST_ID));
}
#[tokio::test]
async fn non_s3_request_without_id_generates_only_x_request_id() {
let capture = HeaderCaptureService::default();
let captured_context = capture.request_context();
let mut service = ExternalRequestContextLayer::default().layer(capture);
let request = Request::builder()
.uri("/rustfs/admin/v3/info")
.body(())
.expect("build admin request");
let response = service.call(request).await.expect("admin response");
let response_request_id = response
.headers()
.get(REQUEST_ID_HEADER)
.and_then(|value| value.to_str().ok())
.expect("generated admin request ID");
assert!(uuid::Uuid::parse_str(response_request_id).is_ok());
assert!(!response.headers().contains_key(AMZ_REQUEST_ID));
let context = captured_context
.lock()
.expect("captured request context")
.clone()
.expect("admin request context");
assert_eq!(context.request_id, response_request_id);
}
#[derive(Clone, Default)]
struct CountingHybridService {
calls: Arc<AtomicUsize>,
@@ -3571,6 +4085,12 @@ mod tests {
"https://allowed.com"
);
assert_eq!(resp_headers.get(cors::response::ACCESS_CONTROL_ALLOW_CREDENTIALS).unwrap(), "true");
let exposed = resp_headers
.get(cors::response::ACCESS_CONTROL_EXPOSE_HEADERS)
.and_then(|value| value.to_str().ok())
.expect("exposed response headers");
assert!(exposed.split(',').any(|header| header.trim() == "x-request-id"));
assert!(exposed.split(',').any(|header| header.trim() == "x-amz-request-id"));
}
#[test]
@@ -3619,7 +4139,7 @@ mod tests {
}
#[test]
fn request_context_layer_preserves_upstream_s3_request_id() {
fn request_context_layer_does_not_mutate_upstream_s3_request_id() {
let mut service = RequestContextLayer.layer(CaptureService);
let request = Request::builder()
.uri("/bucket/object")
rustfs/src/server/rate_limit.rs
+31 -14
View File
@@ -57,6 +57,7 @@ use crate::server::{
MINIO_HEALTH_CLUSTER_READ_PATH, MINIO_HEALTH_LIVE_PATH, MINIO_HEALTH_READY_PATH, PROFILE_CPU_PATH, PROFILE_MEMORY_PATH,
RPC_PREFIX, RemoteAddr, TONIC_PREFIX, has_path_prefix, is_admin_path, is_table_catalog_path,
};
use crate::storage_api::server::layer::request_context::RequestContext;
use bytes::Bytes;
use futures::future::{Either, Ready, ready};
use http::{HeaderMap, HeaderValue, Request, Response, StatusCode};
@@ -408,15 +409,10 @@ fn u64_header(value: u64) -> HeaderValue {
type BoxError = Box<dyn std::error::Error + Send + Sync>;
type BoxBody = http_body_util::combinators::UnsyncBoxBody<Bytes, BoxError>;
/// Build the S3-style `429` rejection. The request id (already generated by
/// `SetRequestIdLayer`, which sits outside this layer) is echoed manually
/// because the rejection short-circuits below `PropagateRequestIdLayer`.
fn s3_too_many_requests_response(request_id: Option<&HeaderValue>, limit_rpm: u32, throttle: &ThrottleInfo) -> Response<BoxBody> {
// The header may be client-supplied (SetRequestIdLayer only fills it when
// absent), so gate the XML interpolation on a UUID-safe charset instead of
// reflecting arbitrary bytes into the body.
/// Build the S3-style `429` rejection. The server-owned request ID is echoed
/// manually because the rejection short-circuits the inner response stack.
fn s3_too_many_requests_response(request_id: Option<&str>, limit_rpm: u32, throttle: &ThrottleInfo) -> Response<BoxBody> {
let request_id_xml = request_id
.and_then(|value| value.to_str().ok())
.filter(|id| !id.is_empty() && id.bytes().all(|b| b.is_ascii_alphanumeric() || b == b'-'))
.map(|id| format!("<RequestId>{id}</RequestId>"))
.unwrap_or_default();
@@ -436,8 +432,8 @@ fn s3_too_many_requests_response(request_id: Option<&HeaderValue>, limit_rpm: u3
.headers_mut()
.insert(http::header::CONTENT_TYPE, HeaderValue::from_static("application/xml"));
apply_throttle_headers(response.headers_mut(), limit_rpm, throttle);
if let Some(id) = request_id {
response.headers_mut().insert(X_REQUEST_ID, id.clone());
if let Some(id) = request_id.and_then(|id| HeaderValue::from_str(id).ok()) {
response.headers_mut().insert(X_REQUEST_ID, id);
}
response
}
@@ -559,7 +555,9 @@ fn rejected_response<F, E, ReqBody>(
"Request rejected by API rate limit"
);
Either::Right(ready(Ok(s3_too_many_requests_response(
req.headers().get(X_REQUEST_ID),
req.extensions()
.get::<RequestContext>()
.map(|context| context.request_id.as_str()),
limit_rpm,
throttle,
))))
@@ -871,7 +869,15 @@ mod tests {
}
let mut req = request_from(ip(1), "/bucket/object");
req.headers_mut().insert(X_REQUEST_ID, HeaderValue::from_static("req-123"));
req.headers_mut()
.insert(X_REQUEST_ID, HeaderValue::from_static("client-request-id"));
req.extensions_mut().insert(RequestContext {
request_id: "req-123".to_string(),
x_amz_request_id: "req-123".to_string(),
trace_id: None,
span_id: None,
start_time: Instant::now(),
});
let resp = service.call(req).await.expect("ok");
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
assert_eq!(resp.headers().get(http::header::RETRY_AFTER).and_then(|v| v.to_str().ok()), Some("1"));
@@ -891,20 +897,31 @@ mod tests {
}
#[tokio::test]
async fn hostile_request_id_is_not_reflected_into_the_429_body() {
async fn hostile_request_id_does_not_override_the_429_request_id() {
let mut service = service_with_quota(60, 1);
let _ = service.call(request_from(ip(3), "/bucket/object")).await.expect("ok");
let mut req = request_from(ip(3), "/bucket/object");
req.headers_mut()
.insert(X_REQUEST_ID, HeaderValue::from_static("<Code>evil</Code>"));
req.extensions_mut().insert(RequestContext {
request_id: "server-request-id".to_string(),
x_amz_request_id: "server-request-id".to_string(),
trace_id: None,
span_id: None,
start_time: Instant::now(),
});
let resp = service.call(req).await.expect("ok");
assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS);
assert_eq!(
resp.headers().get(X_REQUEST_ID).and_then(|value| value.to_str().ok()),
Some("server-request-id")
);
let body = resp.into_body().collect().await.expect("body").to_bytes();
let body = String::from_utf8_lossy(&body);
assert!(!body.contains("evil"), "client-controlled request id must not be reflected: {body}");
assert!(!body.contains("<RequestId>"), "malformed id must be omitted entirely: {body}");
assert!(body.contains("<RequestId>server-request-id</RequestId>"), "body: {body}");
}
#[tokio::test]
rustfs/src/startup_embedded.rs
+3
View File
@@ -156,6 +156,9 @@ pub(crate) async fn run_embedded_startup(args: EmbeddedStartupArgs) -> Result<Em
let bound_addr = http_server.bound_addr;
let cancel_token = CancellationToken::new();
#[cfg(feature = "e2e-test-hooks")]
crate::embedded::wait_for_embedded_startup_hook(bound_addr.port()).await;
let storage_runtime = match init_embedded_startup_storage_runtime(
listen_context.server_addr,
&endpoint_pools,
rustfs/src/storage/helper.rs
+79 -42
View File
@@ -14,7 +14,7 @@
use crate::server::{convert_ecstore_object_info, is_audit_module_enabled, is_notify_module_enabled};
use crate::storage::access::{ReqInfo, request_context_from_req};
use crate::storage::request_context::{RequestContext, extract_request_id_from_headers};
use crate::storage::request_context::RequestContext;
use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use hashbrown::HashMap;
use http::StatusCode;
@@ -73,6 +73,13 @@ where
}
}
/// Builds S3-compatible notification response elements with the canonical request ID.
pub(crate) fn build_event_resp_elements<T>(response: &S3Response<T>, request_id: &str) -> HashMap<String, String> {
let mut resp_elements = extract_resp_elements(response);
resp_elements.insert(AMZ_REQUEST_ID.to_string(), request_id.to_string());
resp_elements
}
/// A unified helper structure for building and distributing audit logs and event notifications via RAII mode at the end of an S3 operation scope.
pub enum OperationHelper {
Disabled,
@@ -86,7 +93,7 @@ pub struct EnabledOperationHelper {
api_builder: ApiDetailsBuilder,
event_builder: Option<EventArgsBuilder>,
start_time: std::time::Instant,
request_context: Option<RequestContext>,
request_context: RequestContext,
}
impl OperationHelper {
@@ -157,16 +164,13 @@ impl OperationHelper {
api_builder = api_builder.object(&object_key);
}
// Audit builder
// Resolve canonical request context and request_id in a single pass:
// RequestContext.request_id > extract_request_id_from_headers() > generated fallback id
// Resolve the canonical request context once for both output chains.
let request_context = request_context_from_req(req);
if request_context.is_none() {
counter!("rustfs_log_chain_orphan_total", "component" => "operation_helper").increment(1);
}
let request_id = request_context
.as_ref()
.map(|ctx| ctx.request_id.clone())
.unwrap_or_else(|| extract_request_id_from_headers(&req.headers));
let request_context = request_context.unwrap_or_else(|| RequestContext::from_external_headers(&req.headers));
let request_id = request_context.request_id.clone();
let audit_builder = if audit_enabled {
Some(
@@ -189,12 +193,6 @@ impl OperationHelper {
};
let mut req_params = extract_params_header(&req.headers);
// Inject x-amz-request-id from RequestContext into req_params for event correlation
if let Some(ref ctx) = request_context {
req_params
.entry(AMZ_REQUEST_ID.to_string())
.or_insert_with(|| ctx.x_amz_request_id.clone());
}
if let Some(principal_id) = req_info
.and_then(|info| info.cred.as_ref())
.map(|cred| cred.access_key.clone())
@@ -228,10 +226,7 @@ impl OperationHelper {
audit_builder,
api_builder,
event_builder,
start_time: request_context
.as_ref()
.map(|ctx| ctx.start_time)
.unwrap_or_else(std::time::Instant::now),
start_time: request_context.start_time,
request_context,
}))
}
@@ -331,14 +326,12 @@ impl OperationHelper {
}
// Inject OpenTelemetry trace context into audit tags for distributed tracing correlation
if let Some(ref ctx) = state.request_context
&& (ctx.trace_id.is_some() || ctx.span_id.is_some())
{
if state.request_context.trace_id.is_some() || state.request_context.span_id.is_some() {
let mut tags = HashMap::new();
if let Some(ref tid) = ctx.trace_id {
if let Some(ref tid) = state.request_context.trace_id {
tags.insert("traceId".to_string(), Value::String(tid.clone()));
}
if let Some(ref sid) = ctx.span_id {
if let Some(ref sid) = state.request_context.span_id {
tags.insert("spanId".to_string(), Value::String(sid.clone()));
}
final_builder = final_builder.tags(tags);
@@ -352,7 +345,7 @@ impl OperationHelper {
if state.notify_enabled
&& let (Some(builder), Ok(res)) = (state.event_builder.take(), result)
{
state.event_builder = Some(builder.resp_elements(extract_resp_elements(res)));
state.event_builder = Some(builder.resp_elements(build_event_resp_elements(res, &state.request_context.request_id)));
}
self
@@ -365,6 +358,17 @@ impl OperationHelper {
}
self
}
/// Returns the operation's canonical context, reading the ingress extension
/// when the disabled fast path holds no request state.
pub(crate) fn request_context_or_from_request<T>(&self, req: &S3Request<T>) -> RequestContext {
match self {
Self::Enabled(state) => state.request_context.clone(),
Self::Disabled => {
request_context_from_req(req).unwrap_or_else(|| RequestContext::from_external_headers(&req.headers))
}
}
}
}
fn should_build_notification_event(notify_module_enabled: bool) -> bool {
@@ -381,7 +385,7 @@ impl Drop for OperationHelper {
if state.audit_enabled
&& let Some(builder) = state.audit_builder.take()
{
let ctx = state.request_context.clone();
let ctx = Some(state.request_context.clone());
spawn_background_with_context(ctx, async move {
AuditLogger::log(builder.build()).await;
});
@@ -395,7 +399,7 @@ impl Drop for OperationHelper {
let event_args = builder.build();
// Avoid generating notifications for copy requests
if !event_args.is_replication_request() {
let ctx = state.request_context.clone();
let ctx = Some(state.request_context.clone());
spawn_background_with_context(ctx, async move {
runtime_sources::current_notify_interface().notify(event_args).await;
});
@@ -416,8 +420,9 @@ mod tests {
use rustfs_credentials::Credentials;
use rustfs_s3_ops::S3Operation;
use rustfs_s3_types::EventName;
use s3s::S3Request;
use s3s::dto::DeleteObjectTaggingInput;
use rustfs_utils::http::headers::{AMZ_REQUEST_ID, REQUEST_ID_HEADER};
use s3s::dto::{DeleteObjectTaggingInput, DeleteObjectTaggingOutput};
use s3s::{S3Request, S3Response};
use std::sync::{Arc, Mutex};
use temp_env::with_vars;
@@ -554,11 +559,14 @@ mod tests {
let mut req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
req.headers.insert("host", HeaderValue::from_static("example.com"));
req.headers.insert("user-agent", HeaderValue::from_static("rustfs-test"));
req.headers
.insert(REQUEST_ID_HEADER, HeaderValue::from_static("ingress-canonical-uuid"));
req.headers
.insert("x-amz-request-id", HeaderValue::from_static("client-supplied-request-id"));
// Insert RequestContext (set by ingress layer) with a specific request_id
req.extensions.insert(RequestContext {
request_id: "ingress-canonical-uuid".to_string(),
x_amz_request_id: "ingress-canonical-uuid".to_string(),
x_amz_request_id: "client-supplied-request-id".to_string(),
trace_id: None,
span_id: None,
start_time: std::time::Instant::now(),
@@ -570,20 +578,32 @@ mod tests {
..Default::default()
});
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
let result = Ok(S3Response::new(DeleteObjectTaggingOutput::default()));
let mut helper =
OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject).complete(&result);
// Verify the helper stored the RequestContext
let OperationHelper::Enabled(state) = &helper else {
let OperationHelper::Enabled(state) = &mut helper else {
panic!("helper should be enabled when notify/audit switches are on");
};
assert!(state.request_context.is_some());
assert_eq!(state.request_context.as_ref().unwrap().request_id, "ingress-canonical-uuid");
assert_eq!(state.request_context.request_id, "ingress-canonical-uuid");
let audit_entry = state.audit_builder.take().expect("audit builder should exist").build();
assert_eq!(audit_entry.request_id.as_deref(), Some("ingress-canonical-uuid"));
let event_args = state.event_builder.clone().expect("event builder should exist").build();
assert_eq!(
event_args.req_params.get(AMZ_REQUEST_ID).map(String::as_str),
Some("client-supplied-request-id")
);
assert_eq!(
event_args.resp_elements.get(AMZ_REQUEST_ID).map(String::as_str),
Some("ingress-canonical-uuid")
);
},
);
}
#[test]
fn operation_helper_no_request_context_when_absent() {
fn operation_helper_reuses_generated_context_when_headers_absent() {
with_vars(
[
(rustfs_config::ENV_NOTIFY_ENABLE, Some("true")),
@@ -601,8 +621,6 @@ mod tests {
let mut req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
req.headers.insert("host", HeaderValue::from_static("example.com"));
req.headers.insert("user-agent", HeaderValue::from_static("rustfs-test"));
req.headers
.insert("x-amz-request-id", HeaderValue::from_static("amz-header-uuid"));
// No RequestContext inserted
req.extensions.insert(ReqInfo {
@@ -612,12 +630,21 @@ mod tests {
});
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
// Verify the helper has no RequestContext
let request_context = helper.request_context_or_from_request(&req);
let request_id = request_context.request_id;
assert!(uuid::Uuid::parse_str(&request_id).is_ok());
let result = Ok(S3Response::new(DeleteObjectTaggingOutput::default()));
let helper = helper.complete(&result);
let OperationHelper::Enabled(state) = &helper else {
panic!("helper should be enabled when notify/audit switches are on");
};
assert!(state.request_context.is_none());
assert_eq!(state.request_context.request_id, request_id);
let event_args = state.event_builder.clone().expect("event builder should exist").build();
assert!(!event_args.req_params.contains_key(AMZ_REQUEST_ID));
assert_eq!(
event_args.resp_elements.get(AMZ_REQUEST_ID).map(String::as_str),
Some(request_id.as_str())
);
},
);
}
@@ -638,10 +665,20 @@ mod tests {
.key("test-key".to_string())
.build()
.unwrap();
let req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
let mut req = build_request(input, Method::DELETE, Uri::from_static("/test-bucket/test-key"));
req.headers
.insert(REQUEST_ID_HEADER, HeaderValue::from_static("client-request-id"));
req.extensions.insert(RequestContext {
request_id: "server-request-id".to_string(),
x_amz_request_id: "server-request-id".to_string(),
trace_id: None,
span_id: None,
start_time: std::time::Instant::now(),
});
let helper = OperationHelper::new(&req, EventName::ObjectAccessedGet, S3Operation::GetObject);
assert!(matches!(helper, OperationHelper::Disabled));
assert!(matches!(&helper, OperationHelper::Disabled));
assert_eq!(helper.request_context_or_from_request(&req).request_id, "server-request-id");
},
);
}
rustfs/src/storage/request_context.rs
+128 -10
View File
@@ -17,11 +17,10 @@
//! # Architecture
//!
//! ```text
//! HTTP Ingress (SetRequestIdLayer)
//! → generates x-request-id UUID
//! → RequestContextLayer creates RequestContext
//! External S3 HTTP ingress
//! → generates a server-owned request ID without mutating signed headers
//! → ExternalRequestContextLayer creates RequestContext
//! → stores in request.extensions()
//! → stores the S3-compatible request-id alias without changing signed headers
//! Auth (FS::check)
//! → copies RequestContext into ReqInfo.request_context
//! Storage (FS methods)
@@ -40,10 +39,11 @@
//! # Frozen Rules (T00 Guardrails)
//!
//! ## request-id contract
//! - Canonical wire header: `x-request-id` (set by `SetRequestIdLayer`)
//! - External S3 response headers: `x-request-id` and `x-amz-request-id`
//! - Non-S3 response header: propagated `x-request-id`
//! - Compatibility wire header: `x-amz-request-id`
//! - Canonical internal field: `RequestContext.request_id`
//! - S3 compatibility internal alias field: `RequestContext.x_amz_request_id`
//! - Client-provided request ID headers are never canonical on external S3 requests
//! - Internal modules MUST NOT generate a second request id under the field name `request_id`
//! except for orphan/non-ingress fallback paths where no canonical request-id exists.
//! - Internal identifiers for sub-operations should use `operation_id` or `subtask_id`
@@ -72,10 +72,14 @@ use tracing_opentelemetry::OpenTelemetrySpanExt;
/// Created exactly once at HTTP ingress. Cloned by value; never mutated after creation.
#[derive(Clone, Debug)]
pub struct RequestContext {
/// Canonical request ID (from `x-request-id` header, set by `SetRequestIdLayer`).
/// Canonical request ID: server-owned for external S3 requests and
/// propagated for non-S3 and trusted internal requests.
pub request_id: String,
/// S3-compatible request ID alias (preserves upstream `x-amz-request-id` if present,
/// otherwise equals `request_id`).
/// Compatibility-only alias that preserves an incoming
/// `x-amz-request-id`, or mirrors [`Self::request_id`] when absent.
///
/// Internal correlation must use [`Self::request_id`]. This field remains
/// for compatibility with existing in-crate consumers.
pub x_amz_request_id: String,
/// OpenTelemetry trace ID (if present from upstream propagation).
pub trace_id: Option<String>,
@@ -86,6 +90,57 @@ pub struct RequestContext {
}
impl RequestContext {
/// Create a context for a trusted internal request that may propagate its
/// canonical ID through headers.
pub(crate) fn from_headers(headers: &HeaderMap) -> Self {
Self::new(
extract_request_id_from_headers(headers),
headers,
extract_trace_context_ids_from_headers(headers),
)
}
/// Create a context for a non-S3 request while mirroring the propagated
/// canonical request ID into the compatibility alias.
pub(crate) fn from_propagated_headers(headers: &HeaderMap) -> Self {
let request_id = extract_request_id_from_headers(headers);
let (trace_id, span_id) = extract_trace_context_ids_from_headers(headers)
.map(|(trace_id, span_id)| (Some(trace_id), Some(span_id)))
.unwrap_or((None, None));
Self {
x_amz_request_id: request_id.clone(),
request_id,
trace_id,
span_id,
start_time: Instant::now(),
}
}
/// Create an external request context with a server-owned ID while keeping
/// client headers unchanged for signature verification.
pub(crate) fn from_external_headers(headers: &HeaderMap) -> Self {
Self::new(uuid::Uuid::new_v4().to_string(), headers, extract_trace_context_ids_from_headers(headers))
}
fn new(request_id: String, headers: &HeaderMap, trace_context: Option<(String, String)>) -> Self {
let x_amz_request_id = headers
.get(AMZ_REQUEST_ID)
.and_then(|value| value.to_str().ok())
.map(String::from)
.unwrap_or_else(|| request_id.clone());
let (trace_id, span_id) = trace_context
.map(|(trace_id, span_id)| (Some(trace_id), Some(span_id)))
.unwrap_or((None, None));
Self {
request_id,
x_amz_request_id,
trace_id,
span_id,
start_time: Instant::now(),
}
}
/// Create a fallback `RequestContext` for paths that bypass HTTP ingress.
/// Generates a canonical internal `request_id` in `trace-{trace_id}` or `req-{uuid}` format.
pub fn fallback() -> Self {
@@ -206,7 +261,7 @@ where
#[allow(unused_imports)]
mod tests {
use super::{RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers};
use http::HeaderMap;
use http::{HeaderMap, HeaderValue};
use opentelemetry::global;
use opentelemetry::trace::{SpanContext, TraceContextExt, TraceFlags, TraceId, TraceState, TracerProvider as _};
use opentelemetry_sdk::propagation::TraceContextPropagator;
@@ -252,6 +307,69 @@ mod tests {
assert!(ctx.span_id.is_none());
}
#[test]
fn test_request_context_from_headers_prioritizes_canonical_request_id() {
let mut headers = HeaderMap::new();
headers.insert("x-request-id", HeaderValue::from_static("canonical-request-id"));
headers.insert("x-amz-request-id", HeaderValue::from_static("client-request-id"));
let ctx = RequestContext::from_headers(&headers);
assert_eq!(ctx.request_id, "canonical-request-id");
assert_eq!(ctx.x_amz_request_id, "client-request-id");
}
#[test]
fn test_request_context_from_headers_preserves_empty_amz_alias() {
let mut headers = HeaderMap::new();
headers.insert("x-request-id", HeaderValue::from_static("canonical-request-id"));
headers.insert("x-amz-request-id", HeaderValue::from_static(""));
let ctx = RequestContext::from_headers(&headers);
assert_eq!(ctx.request_id, "canonical-request-id");
assert_eq!(ctx.x_amz_request_id, "");
}
#[test]
fn test_propagated_request_context_mirrors_canonical_id_and_preserves_trace_context() {
global::set_text_map_propagator(TraceContextPropagator::new());
let mut headers = HeaderMap::new();
headers.insert("x-request-id", HeaderValue::from_static("canonical-request-id"));
headers.insert("x-amz-request-id", HeaderValue::from_static("untrusted-amz-request-id"));
headers.insert(
"traceparent",
HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
);
let ctx = RequestContext::from_propagated_headers(&headers);
assert_eq!(ctx.request_id, "canonical-request-id");
assert_eq!(ctx.x_amz_request_id, "canonical-request-id");
assert_eq!(ctx.trace_id.as_deref(), Some("4bf92f3577b34da6a3ce929d0e0e4736"));
assert_eq!(ctx.span_id.as_deref(), Some("00f067aa0ba902b7"));
}
#[test]
fn test_external_request_context_owns_id_and_preserves_trace_context() {
global::set_text_map_propagator(TraceContextPropagator::new());
let mut headers = HeaderMap::new();
headers.insert("x-request-id", HeaderValue::from_static("client-request-id"));
headers.insert("x-amz-request-id", HeaderValue::from_static("client-amz-request-id"));
headers.insert(
"traceparent",
HeaderValue::from_static("00-4bf92f3577b34da6a3ce929d0e0e4736-00f067aa0ba902b7-01"),
);
let ctx = RequestContext::from_external_headers(&headers);
assert_ne!(ctx.request_id, "client-request-id");
assert!(uuid::Uuid::parse_str(&ctx.request_id).is_ok());
assert_eq!(ctx.x_amz_request_id, "client-amz-request-id");
assert_eq!(ctx.trace_id.as_deref(), Some("4bf92f3577b34da6a3ce929d0e0e4736"));
assert_eq!(ctx.span_id.as_deref(), Some("00f067aa0ba902b7"));
}
#[test]
fn test_request_context_fallback_uses_trace_prefix_when_span_context_valid() {
let trace_id = "70f5f77e2f0a4f24be343b59f8b66f8f";
rustfs/src/storage/rpc/node_service.rs
+113 -9
View File
@@ -52,7 +52,7 @@ use std::{
collections::HashMap,
io::Cursor,
pin::Pin,
sync::{Arc, OnceLock},
sync::{Arc, LazyLock, OnceLock},
};
use time::OffsetDateTime;
use tokio::spawn;
@@ -128,6 +128,7 @@ fn remove_heal_control_replay(
}
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
fn admit_heal_control_replay(
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
@@ -465,6 +466,19 @@ impl HealControlRpcService {
pub(crate) async fn initialize_heal_topology_fingerprint(
cache: Arc<tokio::sync::OnceCell<String>>,
endpoint_pools: EndpointServerPools,
) -> Result<(), String> {
initialize_heal_topology_fingerprint_with_probe(
cache,
endpoint_pools,
crate::storage::storage_api::start_remote_version_state_fleet_probe,
)
.await
}
async fn initialize_heal_topology_fingerprint_with_probe(
cache: Arc<tokio::sync::OnceCell<String>>,
endpoint_pools: EndpointServerPools,
start_probe: impl FnOnce(String),
) -> Result<(), String> {
if cache.get().is_some() {
return Ok(());
@@ -472,7 +486,8 @@ pub(crate) async fn initialize_heal_topology_fingerprint(
let fingerprint = tokio::task::spawn_blocking(move || heal::heal_topology_fingerprint(&endpoint_pools))
.await
.map_err(|_| "heal control topology calculation task failed".to_string())??;
let _ = cache.set(fingerprint);
let _ = cache.set(fingerprint.clone());
start_probe(fingerprint);
Ok(())
}
@@ -806,6 +821,35 @@ impl heal_control_service_server::HealControlService for HealControlRpcService {
response_proof: Bytes::new(),
}));
}
if rustfs_protos::is_remote_version_state_capability_probe(&request.get_ref().command) {
let topology_member = self
.endpoint_pools()
.await
.ok_or_else(|| Status::failed_precondition("heal control topology is not initialized"))?
.peers()
.1;
if topology_member.is_empty() {
return Err(Status::failed_precondition("local topology member identity is unavailable"));
}
let result =
rustfs_protos::encode_remote_version_state_capability(&topology_member, NODE_CAPABILITY_SERVER_EPOCH.as_bytes())
.map_err(|_| Status::internal("remote version state capability length cannot be represented"))?;
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
request.get_ref().version,
&request.get_ref().topology_fingerprint,
&request.get_ref().command,
&result,
)
.map_err(|_| Status::internal("heal control response length cannot be represented"))?;
let response_proof = sign_tonic_rpc_response_proof(&canonical_response)
.map_err(|_| Status::internal("heal control response proof is unavailable"))?;
return Ok(Response::new(HealControlResponse {
success: true,
result: result.into(),
error_info: None,
response_proof: response_proof.into(),
}));
}
let endpoints = self
.endpoint_pools()
.await
@@ -2090,10 +2134,10 @@ mod tests {
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response,
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
scanner_activity_response, stop_rebalance_response,
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint,
initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, make_heal_control_server,
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
};
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
@@ -2147,6 +2191,7 @@ mod tests {
use tokio::time::Duration;
use tokio_stream::wrappers::TcpListenerStream;
use tonic::{Request, Response, Status};
use uuid::Uuid;
const DISK_MUTATION_RPC_METHODS: [&str; 18] = [
"renamedata",
@@ -3132,6 +3177,60 @@ mod tests {
assert_eq!(non_coordinator.code(), tonic::Code::FailedPrecondition);
}
#[tokio::test]
async fn remote_version_state_probe_authenticates_topology_challenge_and_process_epoch() {
let _ = rustfs_credentials::set_global_rpc_secret("remote-version-state-node-service-test-secret".to_string());
let endpoints = heal_control_test_endpoints_with_coordinator("node-d", true);
let fingerprint = heal_topology_fingerprint(&endpoints).expect("test topology should hash");
let (service, source) = super::make_heal_control_server_for_source();
*source.write().await = Some(endpoints);
let probe_command = rustfs_protos::remote_version_state_capability_probe(&[7; 16]);
let mut request = Request::new(HealControlRequest {
version: rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
topology_fingerprint: fingerprint.clone(),
command: Bytes::from(probe_command.clone()),
});
let body = rustfs_protos::canonical_heal_control_request_body(
request.get_ref().version,
&request.get_ref().topology_fingerprint,
&request.get_ref().command,
)
.expect("probe should encode");
set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
mark_v2_authenticated(&mut request);
let response = service
.heal_control(request)
.await
.expect("matching topology should be acknowledged")
.into_inner();
let (topology_member, process_epoch) =
rustfs_protos::decode_remote_version_state_capability(&response.result).expect("capability response should decode");
assert_eq!(topology_member, "node-a:9000");
let server_epoch = Uuid::from_slice(process_epoch).expect("server epoch should be a UUID");
assert!(!server_epoch.is_nil());
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
&fingerprint,
&probe_command,
&response.result,
)
.expect("response should encode");
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
.expect("outer proof should bind the response to the request");
let different_probe = rustfs_protos::remote_version_state_capability_probe(&[8; 16]);
let different_response = rustfs_protos::canonical_heal_control_response_body(
rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
&fingerprint,
&different_probe,
&response.result,
)
.expect("different response should encode");
crate::storage::storage_api::verify_tonic_rpc_response_proof(&different_response, &response.response_proof)
.expect_err("proof from one challenge must not be reusable");
}
#[tokio::test]
async fn heal_control_coordinator_rejects_expired_and_non_admin_starts() {
let _ = rustfs_credentials::set_global_rpc_secret("heal-control-node-service-test-secret".to_string());
@@ -3194,10 +3293,15 @@ mod tests {
let topology = heal_control_test_endpoints("node-d");
let expected = heal_topology_fingerprint(&topology).expect("test topology should hash");
let cache = Arc::new(tokio::sync::OnceCell::new());
initialize_heal_topology_fingerprint(Arc::clone(&cache), topology)
.await
.expect("valid topology should initialize");
let started_probe = Arc::new(std::sync::Mutex::new(None));
let started_probe_capture = Arc::clone(&started_probe);
initialize_heal_topology_fingerprint_with_probe(Arc::clone(&cache), topology, move |fingerprint| {
*started_probe_capture.lock().expect("probe capture should not poison") = Some(fingerprint);
})
.await
.expect("valid topology should initialize");
assert_eq!(cache.get(), Some(&expected));
assert_eq!(started_probe.lock().expect("probe capture should not poison").as_ref(), Some(&expected));
let mut invalid = heal_control_test_endpoints("node-d");
invalid.as_mut()[0].endpoints.as_mut()[0].pool_idx = -1;
rustfs/src/storage/rpc/node_service/bucket.rs
+2 -2
View File
@@ -66,14 +66,14 @@ impl NodeService {
}));
}
let Some(_store) = self.resolve_object_store() else {
let Some(store) = self.resolve_object_store() else {
return Ok(Response::new(LoadBucketMetadataResponse {
success: false,
error_info: Some("errServerNotInitialized".to_string()),
}));
};
match reload_bucket_metadata(&bucket).await {
match reload_bucket_metadata(store, &bucket).await {
Ok(()) => {
if scanner_maintenance_change {
rustfs_scanner::record_scanner_maintenance_change(&bucket);
rustfs/src/storage/storage_api.rs
+30 -14
View File
@@ -174,7 +174,7 @@ pub(crate) mod head_prefix_consumer {
}
pub(crate) mod helper_consumer {
pub(crate) use super::super::helper::{OperationHelper, spawn_background_with_context};
pub(crate) use super::super::helper::{OperationHelper, build_event_resp_elements, spawn_background_with_context};
pub(crate) type StorageObjectInfo = super::StorageObjectInfo;
}
@@ -203,9 +203,7 @@ pub(crate) mod options_consumer {
}
pub(crate) mod request_context_consumer {
pub(crate) use super::super::request_context::{
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
};
pub(crate) use super::super::request_context::{RequestContext, extract_request_id_from_headers, spawn_traced};
}
pub(crate) mod rpc_consumer {
@@ -472,7 +470,7 @@ pub(crate) mod ecstore_metrics {
#[allow(unused_imports)]
pub(crate) mod ecstore_notification {
pub(crate) use rustfs_ecstore::api::notification::{
NotificationSys, get_global_notification_sys, new_global_notification_sys,
NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe,
};
}
@@ -497,8 +495,9 @@ pub(crate) mod ecstore_rpc {
pub(crate) use rustfs_ecstore::api::rpc::{
LocalPeerS3Client, PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, PeerRestClient, PeerS3Client,
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, verify_rpc_signature, verify_tonic_canonical_body_digest,
verify_tonic_mutation_body_digest, verify_tonic_rpc_signature,
sign_ns_scanner_capability, sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
verify_tonic_rpc_signature_with_bootstrap,
};
#[cfg(test)]
pub(crate) use rustfs_ecstore::api::rpc::{
@@ -845,7 +844,7 @@ pub(crate) async fn reconcile_bucket_resync_target_intents(buckets: &[String]) -
};
for bucket in buckets {
let _transaction_guard = ecstore_bucket::metadata_sys::acquire_bucket_targets_transaction_lock(bucket).await?;
let _transaction_guard = ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await?;
let status = pool.get_bucket_resync_status(bucket).await?;
if status.targets_map.is_empty() {
continue;
@@ -980,6 +979,10 @@ pub(crate) async fn new_global_notification_sys(endpoint_pools: EndpointServerPo
ecstore_notification::new_global_notification_sys(endpoint_pools).await
}
pub(crate) fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
ecstore_notification::start_remote_version_state_fleet_probe(topology_fingerprint);
}
pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>> {
ecstore_config::com::read_config(api, file).await
}
@@ -1459,8 +1462,8 @@ pub(crate) async fn set_bucket_metadata(bucket: String, bm: BucketMetadata) -> R
ecstore_bucket::metadata_sys::set_bucket_metadata(bucket, bm).await
}
pub(crate) async fn reload_bucket_metadata(bucket: &str) -> Result<()> {
ecstore_bucket::metadata_sys::reload_bucket_metadata(bucket).await
pub(crate) async fn reload_bucket_metadata(api: Arc<ECStore>, bucket: &str) -> Result<()> {
ecstore_bucket::metadata_sys::reload_bucket_metadata(api, bucket).await
}
pub(crate) async fn remove_bucket_metadata(bucket: &str) -> Result<bool> {
@@ -1477,8 +1480,8 @@ pub(crate) async fn update_bucket_metadata_config(
Ok(updated_at)
}
pub(crate) async fn acquire_bucket_targets_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
ecstore_bucket::metadata_sys::acquire_bucket_targets_transaction_lock(bucket).await
pub(crate) async fn acquire_bucket_metadata_transaction_lock(bucket: &str) -> Result<rustfs_lock::NamespaceLockGuard> {
ecstore_bucket::metadata_sys::acquire_bucket_metadata_transaction_lock(bucket).await
}
pub(crate) async fn update_bucket_targets_under_transaction_lock(bucket: &str, data: Vec<u8>) -> Result<time::OffsetDateTime> {
@@ -1620,8 +1623,21 @@ pub(crate) fn sign_ns_scanner_capability(challenge: uuid::Uuid, server_epoch: uu
ecstore_rpc::sign_ns_scanner_capability(challenge, server_epoch)
}
pub(crate) fn verify_tonic_rpc_signature(audience: &str, path: &str, headers: &http::HeaderMap) -> std::io::Result<()> {
ecstore_rpc::verify_tonic_rpc_signature(audience, path, headers)
pub(crate) fn verify_tonic_rpc_signature_with_bootstrap(
audience: &str,
path: &str,
headers: &http::HeaderMap,
allow_replay_scope_bootstrap: bool,
) -> std::io::Result<()> {
ecstore_rpc::verify_tonic_rpc_signature_with_bootstrap(audience, path, headers, allow_replay_scope_bootstrap)
}
pub(crate) fn tonic_boot_epoch_challenge(headers: &http::HeaderMap) -> std::io::Result<Option<uuid::Uuid>> {
ecstore_rpc::tonic_boot_epoch_challenge(headers)
}
pub(crate) fn tonic_boot_epoch_response_headers(audience: &str, challenge: uuid::Uuid) -> std::io::Result<http::HeaderMap> {
ecstore_rpc::tonic_boot_epoch_response_headers(audience, challenge)
}
pub(crate) fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
rustfs/src/storage_api.rs
+4 -5
View File
@@ -97,7 +97,8 @@ pub(crate) mod server {
pub(crate) mod http {
pub(crate) use crate::storage::storage_api::{
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, verify_tonic_rpc_signature,
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
};
pub(crate) fn try_current_local_node_name() -> Option<String> {
@@ -124,9 +125,7 @@ pub(crate) mod server {
}
pub(crate) mod request_context {
pub(crate) use crate::storage::storage_api::request_context_consumer::{
RequestContext, extract_request_id_from_headers,
};
pub(crate) use crate::storage::storage_api::request_context_consumer::RequestContext;
}
pub(crate) mod rpc {
@@ -149,7 +148,7 @@ pub(crate) mod server {
pub(crate) mod request_context {
pub(crate) use crate::storage::storage_api::request_context_consumer::{
RequestContext, extract_request_id_from_headers, extract_trace_context_ids_from_headers, spawn_traced,
RequestContext, extract_request_id_from_headers, spawn_traced,
};
}
}
rustfs/tests/admin_diagnostic_capability_e2e.rs
+19
View File
@@ -84,6 +84,22 @@ async fn signed_bytes_request(
.expect("signed admin request")
}
async fn wait_for_ready(client: &Client, endpoint: &str) {
let ready_url = format!("{endpoint}/health/ready");
tokio::time::timeout(Duration::from_secs(10), async {
loop {
if let Ok(response) = client.get(&ready_url).send().await
&& response.status().is_success()
{
return;
}
tokio::time::sleep(Duration::from_millis(100)).await;
}
})
.await
.expect("embedded server should become ready");
}
#[tokio::test]
async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
let port = match find_available_port() {
@@ -100,9 +116,11 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
.expect("start embedded server");
let endpoint = server.endpoint();
let client = Client::builder()
.no_proxy()
.timeout(Duration::from_secs(5))
.build()
.expect("HTTP client");
wait_for_ready(&client, &endpoint).await;
let response = signed_bytes_request(
&client,
@@ -144,6 +162,7 @@ async fn diagnostic_handlers_enforce_advertised_runtime_contract() {
);
let stalled_client = Client::builder()
.no_proxy()
.timeout(Duration::from_secs(60))
.build()
.expect("stalled HTTP client");
rustfs/tests/embedded_multi_instance_test.rs
+180
View File
@@ -19,7 +19,22 @@
use aws_sdk_s3::config::{Credentials, Region};
use aws_sdk_s3::primitives::ByteStream;
use aws_sdk_s3::{Client, Config};
#[cfg(feature = "e2e-test-hooks")]
use chrono::Utc;
#[cfg(feature = "e2e-test-hooks")]
use hmac::{Hmac, KeyInit, Mac};
#[cfg(feature = "e2e-test-hooks")]
use reqwest::StatusCode;
#[cfg(feature = "e2e-test-hooks")]
use rustfs::embedded::pause_embedded_startup_after_http_bind;
use rustfs::embedded::{RustFSServerBuilder, find_available_port};
#[cfg(feature = "e2e-test-hooks")]
use sha2::{Digest, Sha256};
#[cfg(feature = "e2e-test-hooks")]
use std::time::Duration;
#[cfg(feature = "e2e-test-hooks")]
type HmacSha256 = Hmac<Sha256>;
fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
let creds = Credentials::new(access_key, secret_key, None, None, "test");
@@ -33,6 +48,62 @@ fn s3_client(endpoint: &str, access_key: &str, secret_key: &str) -> Client {
Client::from_conf(config)
}
#[cfg(feature = "e2e-test-hooks")]
fn hex(bytes: impl AsRef<[u8]>) -> String {
bytes.as_ref().iter().map(|byte| format!("{byte:02x}")).collect()
}
#[cfg(feature = "e2e-test-hooks")]
fn sha256_hex(bytes: &[u8]) -> String {
hex(Sha256::digest(bytes))
}
#[cfg(feature = "e2e-test-hooks")]
fn hmac(key: &[u8], value: &str) -> Vec<u8> {
let mut mac = HmacSha256::new_from_slice(key).expect("HMAC accepts arbitrary key lengths");
mac.update(value.as_bytes());
mac.finalize().into_bytes().to_vec()
}
#[cfg(feature = "e2e-test-hooks")]
fn signed_admin_request(
client: &reqwest::Client,
endpoint: &str,
request_path: &str,
access_key: &str,
secret_key: &str,
) -> reqwest::RequestBuilder {
let host = endpoint
.strip_prefix("http://")
.or_else(|| endpoint.strip_prefix("https://"))
.expect("embedded endpoint scheme");
let payload_hash = sha256_hex(b"");
let now = Utc::now();
let amz_date = now.format("%Y%m%dT%H%M%SZ").to_string();
let date = now.format("%Y%m%d").to_string();
let canonical_headers = format!("host:{host}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{amz_date}\n");
let signed_headers = "host;x-amz-content-sha256;x-amz-date";
let (path, query) = request_path.split_once('?').unwrap_or((request_path, ""));
let canonical_request = format!("GET\n{path}\n{query}\n{canonical_headers}\n{signed_headers}\n{payload_hash}");
let scope = format!("{date}/us-east-1/s3/aws4_request");
let string_to_sign = format!("AWS4-HMAC-SHA256\n{amz_date}\n{scope}\n{}", sha256_hex(canonical_request.as_bytes()));
let date_key = hmac(format!("AWS4{secret_key}").as_bytes(), &date);
let region_key = hmac(&date_key, "us-east-1");
let service_key = hmac(&region_key, "s3");
let signing_key = hmac(&service_key, "aws4_request");
let authorization = format!(
"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, SignedHeaders={signed_headers}, Signature={}",
hex(hmac(&signing_key, &string_to_sign))
);
client
.get(format!("{endpoint}{request_path}"))
.header("host", host)
.header("x-amz-content-sha256", payload_hash)
.header("x-amz-date", amz_date)
.header("authorization", authorization)
}
// backlog#1052 acceptance: a second embedded server in the same process no
// longer aborts on write-once startup state — before this change,
// `RustFSServer::build()` returned AlreadyStarted (guard) or panicked on
@@ -246,3 +317,112 @@ async fn two_embedded_servers_isolate_auth_and_data_planes() {
server_a.shutdown().await;
server_b.shutdown().await;
}
#[cfg(feature = "e2e-test-hooks")]
#[tokio::test]
async fn second_embedded_server_fails_closed_until_its_context_slot_is_installed() {
let port_a = match find_available_port() {
Ok(port) => port,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => return,
Err(err) => panic!("find free port for server A: {err}"),
};
let server_a = RustFSServerBuilder::new()
.address(format!("127.0.0.1:{port_a}"))
.access_key("startup-window-access-a")
.secret_key("startup-window-secret-a")
.build()
.await
.expect("start embedded server A");
let client_a = s3_client(&server_a.endpoint(), server_a.access_key(), server_a.secret_key());
client_a
.create_bucket()
.bucket("startup-window")
.send()
.await
.expect("server A creates the shared-name bucket");
client_a
.put_object()
.bucket("startup-window")
.key("marker.txt")
.body(ByteStream::from_static(b"from A"))
.send()
.await
.expect("server A writes its marker");
let port_b = match find_available_port() {
Ok(port) => port,
Err(err) if err.kind() == std::io::ErrorKind::PermissionDenied => {
server_a.shutdown().await;
return;
}
Err(err) => {
server_a.shutdown().await;
panic!("find free port for server B: {err}");
}
};
let endpoint_b = format!("http://127.0.0.1:{port_b}");
let b_access_key = "startup-window-access-b";
let b_secret_key = "startup-window-secret-b";
let mut barrier = pause_embedded_startup_after_http_bind(port_b);
let startup_b = tokio::spawn(async move {
RustFSServerBuilder::new()
.address(format!("127.0.0.1:{port_b}"))
.access_key(b_access_key)
.secret_key(b_secret_key)
.build()
.await
});
tokio::time::timeout(Duration::from_secs(10), barrier.wait_until_http_bound())
.await
.expect("server B must bind HTTP before installing its context slot");
let http = reqwest::Client::builder()
.no_proxy()
.timeout(Duration::from_secs(5))
.build()
.expect("build local admin client without proxy");
let inspect_path = "/rustfs/admin/v3/inspect-data?file=marker.txt&volume=startup-window";
let before_install = signed_admin_request(&http, &endpoint_b, inspect_path, b_access_key, b_secret_key)
.send()
.await
.expect("server B HTTP listener must accept the paused request");
let before_install_status = before_install.status();
let before_install_body = before_install.text().await.expect("read paused response body");
assert_eq!(before_install_status, StatusCode::SERVICE_UNAVAILABLE, "{before_install_body}");
assert!(
before_install_body.contains("server context is not ready"),
"paused request must not resolve server A: {before_install_body}"
);
barrier.release();
let server_b = tokio::time::timeout(Duration::from_secs(20), startup_b)
.await
.expect("server B startup must complete after releasing the barrier")
.expect("server B startup task must not panic")
.expect("start embedded server B");
let client_b = s3_client(&server_b.endpoint(), server_b.access_key(), server_b.secret_key());
client_b
.create_bucket()
.bucket("startup-window")
.send()
.await
.expect("server B creates its isolated shared-name bucket");
client_b
.put_object()
.bucket("startup-window")
.key("marker.txt")
.body(ByteStream::from_static(b"from B"))
.send()
.await
.expect("server B writes its marker");
let after_install = signed_admin_request(&http, &server_b.endpoint(), inspect_path, b_access_key, b_secret_key)
.send()
.await
.expect("server B admin request after context installation");
assert_eq!(after_install.status(), StatusCode::OK);
assert_eq!(after_install.bytes().await.expect("read server B marker"), b"from B".as_slice());
server_b.shutdown().await;
server_a.shutdown().await;
}

Some files were not shown because too many files have changed in this diff Show More