Compare commits

..

7 Commits

134 changed files with 6630 additions and 24247 deletions
-4
View File
@@ -34,10 +34,6 @@ If you want the Kafka-backed HA Tempo path, use `docker-compose-example-for-rust
- **High Performance**: Optimized configurations for batching, compression, and memory management.
- **Standardized Protocols**: Built entirely on OpenTelemetry standards.
## HTTP Request Latency Percentiles
The main RustFS dashboard shows request latency percentiles in milliseconds. The HTTP server request, response-body chunk, and response-body stream histograms export bucket boundaries in seconds. During a rolling upgrade or rollback that changes the bucket layout, compute percentiles from nodes that share one layout and from a query window entirely on that layout. The bundled dashboard and recording rules use a five-minute window. The mean from `_sum` divided by `_count` remains usable across the change.
## GET Performance Optimization Dashboards
Three pre-built Grafana dashboards are included for monitoring RustFS GET performance optimization rollout:
-4
View File
@@ -34,10 +34,6 @@
- **高性能**: 针对批处理、压缩和内存管理进行了优化配置。
- **标准化协议**: 完全基于 OpenTelemetry 标准构建。
## HTTP 请求延迟分位数
RustFS 主仪表盘以毫秒显示请求延迟分位数。HTTP 服务的请求耗时、响应体分块延迟和响应体流传输耗时直方图以秒为单位导出桶边界。滚动升级或回滚导致桶边界变化时,只对采用同一套桶边界的节点和时间窗口计算分位数。自带仪表盘和记录规则使用五分钟窗口。变更期间仍可用 `_sum` 除以 `_count` 计算平均值。
## GET 性能优化仪表盘
包含三个预构建的 Grafana 仪表盘,用于监控 RustFS GET 性能优化发布:
Generated
+5 -1
View File
@@ -5573,6 +5573,8 @@ dependencies = [
[[package]]
name = "hyper"
version = "1.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c"
dependencies = [
"atomic-waker",
"bytes",
@@ -9865,7 +9867,6 @@ dependencies = [
"futures-lite",
"futures-util",
"google-cloud-auth",
"h2",
"h3",
"h3-quinn",
"hashbrown 0.17.1",
@@ -11376,7 +11377,9 @@ dependencies = [
"rustfs-storage-api",
"rustfs-test-utils",
"rustfs-utils",
"rustix",
"s3s",
"same-file",
"serde",
"serde_json",
"serial_test",
@@ -11390,6 +11393,7 @@ dependencies = [
"tracing",
"tracing-subscriber",
"uuid",
"winapi-util",
]
[[package]]
-3
View File
@@ -71,7 +71,6 @@ members = [
"crates/zip", # ZIP file handling and compression
"crates/zstd-compat", # Fixed codec for dependencies requiring zstd 0.13
]
exclude = ["vendor/hyper"]
resolver = "3"
[workspace.package]
@@ -457,6 +456,4 @@ strip = "none"
# Pin upstream fixes until compatible crates.io releases are available.
[patch.crates-io]
# Remove after Hyper releases HTTP/2 Expect support (hyperium/hyper#4217).
hyper = { path = "vendor/hyper" }
zstd = { path = "crates/zstd-compat" }
+72 -3
View File
@@ -1235,23 +1235,66 @@ impl Serialize for DataUsageEntry {
where
S: serde::Serializer,
{
self.serialize_fields(serializer, &self.children, &self.replication_stats, &self.all_tier_stats)
}
}
struct OrderedUsageEntry<'a>(&'a DataUsageEntry);
impl Serialize for OrderedUsageEntry<'_> {
fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
let mut children: Vec<_> = self.0.children.iter().collect();
children.sort_unstable();
// MessagePack decoders accept struct fields as positional sequences.
// Include the nested target and tier maps in stable order as well.
let replication = self.0.replication_stats.as_ref().map(|stats| {
let ReplicationAllStats {
targets,
replica_size,
replica_count,
} = stats;
let targets: std::collections::BTreeMap<_, _> = targets.iter().collect();
(targets, replica_size, replica_count)
});
let tiers = self.0.all_tier_stats.as_ref().map(|stats| {
let AllTierStats { tiers } = stats;
let tiers: std::collections::BTreeMap<_, _> = tiers.iter().collect();
(tiers,)
});
self.0.serialize_fields(serializer, &children, &replication, &tiers)
}
}
impl DataUsageEntry {
/// Stable MessagePack encoding for independently reusable scanner checkpoint pages.
pub fn ordered_encoding(&self) -> impl Serialize + '_ {
OrderedUsageEntry(self)
}
fn serialize_fields<S: serde::Serializer, C: Serialize, R: Serialize, T: Serialize>(
&self,
serializer: S,
children: &C,
replication: &R,
tiers: &T,
) -> Result<S::Ok, S::Error> {
// Keep entries map-encoded so older readers can ignore fields appended
// by newer scanner versions during rolling upgrades. The derived
// (array) encoding made any appended field a decode error for them.
let mut state = serializer.serialize_map(Some(
11 + usize::from(self.unknown_tier_stats.is_some()) + usize::from(self.tier_accounting_proof.is_some()),
))?;
state.serialize_entry("children", &self.children)?;
state.serialize_entry("children", children)?;
state.serialize_entry("size", &self.size)?;
state.serialize_entry("objects", &self.objects)?;
state.serialize_entry("versions", &self.versions)?;
state.serialize_entry("delete_markers", &self.delete_markers)?;
state.serialize_entry("obj_sizes", &self.obj_sizes)?;
state.serialize_entry("obj_versions", &self.obj_versions)?;
state.serialize_entry("replication_stats", &self.replication_stats)?;
state.serialize_entry("replication_stats", replication)?;
state.serialize_entry("compacted", &self.compacted)?;
state.serialize_entry("failed_objects", &self.failed_objects)?;
state.serialize_entry("all_tier_stats", &self.all_tier_stats)?;
state.serialize_entry("all_tier_stats", tiers)?;
// Keep the legacy no-unknown shape byte-for-byte stable. Once unknown
// accounting exists, append its map field before the optional proof.
if let Some(unknown_tier_stats) = self.unknown_tier_stats.as_ref() {
@@ -2534,6 +2577,32 @@ mod tests {
entry
}
#[test]
fn ordered_checkpoint_entry_round_trips_and_stays_identical_after_reload() {
let mut entry = DataUsageEntry::default();
let mut replication = ReplicationAllStats {
replica_size: 998,
replica_count: 3,
..Default::default()
};
let mut tiers = AllTierStats::default();
for index in 1..=32 {
let key = format!("source-{index}");
entry.children.insert(key.clone());
replication.targets.insert(key.clone(), ReplicationTargetUsage::default());
tiers.tiers.insert(key, TierStats::default());
}
entry.replication_stats = Some(replication);
entry.all_tier_stats = Some(tiers);
let bytes = rmp_serde::to_vec(&entry.ordered_encoding()).expect("encode ordered checkpoint");
let reloaded: DataUsageEntry = rmp_serde::from_slice(&bytes).expect("read with the existing codec");
assert_eq!(reloaded.children, entry.children);
let replicas = reloaded.replication_stats.as_ref().expect("replication stats");
assert_eq!((replicas.targets.len(), replicas.replica_size, replicas.replica_count), (32, 998, 3));
assert_eq!(reloaded.all_tier_stats.as_ref().expect("tier stats").tiers.len(), 32);
assert_eq!(rmp_serde::to_vec(&reloaded.ordered_encoding()).expect("encode reloaded entry"), bytes);
}
#[test]
fn tier_stats_survive_entry_merge() {
let mut left = tier_entry(
+3
View File
@@ -102,6 +102,9 @@ mod replication_target_matrix_test;
#[cfg(test)]
mod data_usage_test;
#[cfg(test)]
mod scanner_wide_directory_test;
// KMS-specific test modules
#[cfg(test)]
mod kms;
@@ -0,0 +1,208 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use std::collections::BTreeMap;
use std::path::Path;
use std::time::{Duration, SystemTime};
use aws_sdk_s3::{Client, primitives::ByteStream};
use futures::{StreamExt, TryStreamExt, stream};
use rustfs_data_usage::{BucketUsageValueState, DataUsageInfo};
use serde::{Deserialize, de::DeserializeOwned};
use tokio::time::{Instant, sleep, timeout};
use crate::common::{RustFSTestEnvironment, signed_request};
use crate::storage_api::scanner_peer_bucket::{BUCKET_META_PREFIX, RUSTFS_META_BUCKET};
type TestResult<T = ()> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
const BUCKET: &str = "wide-directory-usage";
const WIDTH: usize = 512;
// The per-disk token bucket allows an initial burst of one second. Even the
// first pass can read at most (1 + 3) * 96 = 384 of these 512 root entries.
const SCAN_ENV: &[(&str, &str)] = &[
("RUST_LOG", "warn,rustfs::scanner::io=debug,rustfs::scanner::data_usage=debug"),
("RUSTFS_SCANNER_ENABLED", "true"),
("RUSTFS_SCANNER_START_DELAY_SECS", "0"),
("RUSTFS_SCANNER_CYCLE", "1"),
("RUSTFS_SCANNER_CYCLE_MAX_DURATION_SECS", "3"),
("RUSTFS_SCANNER_SPEED", "fastest"),
("RUSTFS_SCANNER_DISK_IO_BUDGET_IOPS", "96"),
("RUSTFS_SCANNER_DISK_IO_BUDGET_BYTES_PER_SEC", "off"),
];
#[derive(Debug, Deserialize)]
struct Status {
metrics: Metrics,
disk_io_budget: Vec<DiskBudget>,
}
#[derive(Debug, Deserialize)]
struct Metrics {
current_cycle_active: bool,
active_paths: Vec<String>,
cycle_max_duration_seconds: f64,
partial_cycles_runtime: u64,
}
#[derive(Debug, Deserialize)]
struct DiskBudget {
iops: Option<u64>,
}
async fn admin<T: DeserializeOwned>(env: &RustFSTestEnvironment, resource: &str) -> TestResult<T> {
let url = format!("{}/rustfs/admin/v3/{resource}", env.url);
let response = signed_request(http::Method::GET, &url, &env.access_key, &env.secret_key, None, None).await?;
Ok(response.error_for_status()?.json().await?)
}
async fn inventory(client: &Client) -> TestResult<BTreeMap<String, u64>> {
let mut pages = client.list_objects_v2().bucket(BUCKET).max_keys(37).into_paginator().send();
let mut sizes = BTreeMap::new();
while let Some(page) = pages.next().await {
for object in page?.contents() {
let key = object.key().ok_or("S3 inventory omitted object key")?;
let size = u64::try_from(object.size().ok_or("S3 inventory omitted object size")?)?;
assert!(sizes.insert(key.to_owned(), size).is_none(), "duplicate inventory key {key}");
}
}
Ok(sizes)
}
async fn scan_status(env: &RustFSTestEnvironment) -> TestResult<Status> {
let status: Status = admin(env, "scanner/status").await?;
assert_eq!(status.metrics.cycle_max_duration_seconds, 3.0, "effective cycle budget");
assert!(status.disk_io_budget.iter().all(|disk| disk.iops == Some(96)), "effective disk budget");
Ok(status)
}
#[tokio::test(flavor = "multi_thread")]
async fn certifies_wide_directory_after_mid_scan_process_restart() -> TestResult {
let mut env = RustFSTestEnvironment::new().await?;
let temporary_log = tempfile::NamedTempFile::new()?;
let server_log = env
.capture_log_path
.get_or_insert_with(|| temporary_log.path().display().to_string())
.clone();
let bucket_metadata = Path::new(&env.temp_dir)
.join(RUSTFS_META_BUCKET)
.join(BUCKET_META_PREFIX)
.join(BUCKET);
// Seed a cold bucket without allowing any earlier, larger-budget sweep to count it.
env.start_rustfs_server_without_cleanup_with_env(&[("RUSTFS_SCANNER_ENABLED", "false")])
.await?;
let client = env.create_s3_client();
client.create_bucket().bucket(BUCKET).send().await?;
stream::iter(0..WIDTH)
.map(Ok)
.try_for_each_concurrent(12, |i| {
let client = &client;
async move {
client
.put_object()
.bucket(BUCKET)
.key(format!("object-{i:064x}"))
.body(ByteStream::from(vec![42; 1 + i % 67]))
.send()
.await
.map(|_| ())
}
})
.await?;
let expected = inventory(&client).await?;
assert_eq!(expected.len(), WIDTH);
env.restart_server_preserving_data(vec![], SCAN_ENV).await?;
let first_pid = env.process.as_ref().ok_or("missing scanner process")?.id();
timeout(Duration::from_secs(60), async {
loop {
let status = scan_status(&env).await?;
if status.metrics.partial_cycles_runtime >= 1
&& bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir()
&& status.metrics.current_cycle_active
&& status
.metrics
.active_paths
.iter()
.any(|path| path.split('/').any(|part| part == BUCKET))
{
let log = tokio::fs::read_to_string(&server_log).await?;
if !log
.lines()
.any(|line| line.contains("partial_saved_not_published") && line.contains(BUCKET))
{
sleep(Duration::from_millis(200)).await;
continue;
}
assert!(!status.disk_io_budget.is_empty(), "scanner never admitted disk I/O");
break Ok::<_, Box<dyn std::error::Error + Send + Sync>>(());
}
sleep(Duration::from_millis(200)).await;
}
})
.await
.map_err(|_| "scanner did not re-enter the bucket after exhausting the fixed runtime budget")??;
let partial: DataUsageInfo = admin(&env, "datausageinfo").await?;
assert!(
partial
.buckets_usage
.get(BUCKET)
.is_none_or(|value| value.usage_certification.is_none()),
"the first process certified the bucket before the injected crash"
);
env.stop_server();
// The terminal save has returned and the old process can no longer write.
assert!(
bucket_metadata.join(".usage-cache.bin").join("xl.meta").is_file()
&& bucket_metadata.join(".usage-cache.bin.pages").is_dir(),
"the interrupted process must have committed a paged checkpoint"
);
let restart_log_offset = usize::try_from(tokio::fs::metadata(&server_log).await?.len())?;
let restarted_at = SystemTime::now();
env.start_rustfs_server_without_cleanup_with_env(SCAN_ENV).await?;
let second_pid = env.process.as_ref().ok_or("missing restarted scanner process")?.id();
assert_ne!(first_pid, second_pid);
let deadline = Instant::now() + Duration::from_secs(180);
loop {
let usage: DataUsageInfo = admin(&env, "datausageinfo").await?;
let status = scan_status(&env).await?;
if let Some(bucket) = usage.buckets_usage.get(BUCKET)
&& bucket.usage_value_state == Some(BucketUsageValueState::Certified)
{
let log = tokio::fs::read_to_string(&server_log).await?;
let first_admission = log
.get(restart_log_offset..)
.ok_or("server log truncated across restart")?
.lines()
.filter_map(|line| serde_json::from_str::<serde_json::Value>(line).ok())
.find(|record| record["state"] == "bucket_checkpoint_prepared" && record["bucket"] == BUCKET)
.ok_or("no checkpoint admission in the restarted process")?;
assert_eq!(first_admission["resume_position"], true, "restarted scanner discarded the checkpoint");
assert!(usage.is_complete_bucket_usage_snapshot());
let certificate = bucket
.usage_certification
.ok_or("missing certificate for the measured bucket")?;
assert!(certificate.certified_at >= restarted_at, "stale pre-restart certificate");
assert!(status.metrics.partial_cycles_runtime > 0, "new process must cross a cycle deadline");
assert!(!status.disk_io_budget.is_empty(), "restarted scanner never admitted disk I/O");
assert_eq!(inventory(&client).await?, expected, "durable S3 inventory changed across restart");
let count = u64::try_from(expected.len())?;
let bytes = expected.values().sum::<u64>();
assert_eq!((bucket.objects_count, bucket.size), (count, bytes));
assert_eq!((usage.objects_total_count, usage.objects_total_size), (count, bytes));
eprintln!(
"PIDs {first_pid}/{second_pid}, runtime-limited cycles since restart {}, certificate epoch {} cycle {}, S3 and admin {} objects/{} bytes",
status.metrics.partial_cycles_runtime, certificate.epoch, certificate.cycle, count, bytes
);
return Ok(());
}
assert!(
Instant::now() < deadline,
"no certified bucket after restart: {status:?}, value={:?}",
usage.buckets_usage.get(BUCKET)
);
sleep(Duration::from_millis(500)).await;
}
}
@@ -1071,8 +1071,7 @@ mod tests {
}
fn signing_time(headers: &[(String, String)]) -> jiff::Timestamp {
let value = recorded_header(headers, "x-amz-date").trim_end_matches('Z');
jiff::Timestamp::strptime("%Y%m%dT%H%M%S%z", format!("{value}+0000"))
jiff::Timestamp::strptime("%Y%m%dT%H%M%SZ", recorded_header(headers, "x-amz-date"))
.expect("SDK signing timestamp must use the SigV4 format")
}
+7 -20
View File
@@ -14854,9 +14854,13 @@ async fn batch_shard_pread_blocking(requests: Vec<(std::path::PathBuf, usize, us
metrics.as_deref(),
LocalBatchReadStage::Read,
|| {
// Preload the requested window before reading it. Kernel
// readahead is advisory and may still extend past the
// window, so the tail is reclaimed after the read below.
// Announce the exact window before reading it. Kernel
// readahead otherwise sizes its first request from the
// read size: a small window at offset 0 pulls several
// times its size, and a multi-MiB read is split into
// serial readahead chunks. WILLNEED submits exactly the
// window at once. It is advisory, so an unsupported hint
// must not fail the read.
#[cfg(target_os = "linux")]
if let (Ok(start), Ok(len)) = (u64::try_from(offset), u64::try_from(length)) {
let _ = rustix::fs::fadvise(
@@ -14881,23 +14885,6 @@ async fn batch_shard_pread_blocking(requests: Vec<(std::path::PathBuf, usize, us
}
total += nbytes;
}
// Keep a cold shard window from evicting unrelated data
// when the kernel over-reads. The requested tail page is
// retained because its final bytes may share a page with
// the readahead range.
#[cfg(target_os = "linux")]
if let Ok(page) = mmap_page_size() {
let aligned_end = end
.checked_add(page.saturating_sub(1))
.map(|value| value / page * page)
.unwrap_or(end);
if aligned_end < meta.len()
&& let Some(reclaim_len) = std::num::NonZeroU64::new(meta.len() - aligned_end)
{
let _ =
rustix::fs::fadvise(&file, aligned_end, Some(reclaim_len), rustix::fs::Advice::DontNeed);
}
}
Ok::<_, DiskError>(())
}
))?;
@@ -158,6 +158,7 @@ fn to_madmin_scanner_metrics(metrics: rustfs_scanner_metrics::metrics::ScannerMe
last_cycle_lifecycle_transition_actions: metrics.last_cycle_lifecycle_transition_actions,
last_cycle_partial_source: metrics.last_cycle_partial_source,
last_cycle_partial_source_code: metrics.last_cycle_partial_source_code,
last_cycle_partial_stage: metrics.last_cycle_partial_stage,
last_cycle_duration_seconds: metrics.last_cycle_duration_seconds,
last_cycle_objects_scanned: metrics.last_cycle_objects_scanned,
last_cycle_directories_scanned: metrics.last_cycle_directories_scanned,
@@ -685,6 +686,7 @@ mod test {
current_started,
last_cycle_partial_source: "usage".to_string(),
last_cycle_partial_source_code: 1,
last_cycle_partial_stage: Some("object_budget".into()),
cycle_timeout_total: 3,
cycle_recovery_required_total: 2,
cycle_last_progress_age: 17,
@@ -700,6 +702,7 @@ mod test {
assert_eq!(scanner.current_started, current_started);
assert_eq!(scanner.last_cycle_partial_source, "usage");
assert_eq!(scanner.last_cycle_partial_source_code, 1);
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
assert_eq!(scanner.cycle_timeout_total, 3);
assert_eq!(scanner.cycle_recovery_required_total, 2);
assert_eq!(scanner.cycle_last_progress_age, 17);
@@ -118,16 +118,6 @@ pub(crate) async fn test_two_pool_stores(
test_pool_stores_with_contexts(rebalance_meta, false, 2, 2).await
}
/// Two pools whose sets have three drives each: EC:1, quorum 2 of 3.
#[cfg(test)]
pub(crate) async fn test_two_pool_stores_with_three_drive_sets() -> (
Vec<tempfile::TempDir>,
std::sync::Arc<crate::store::ECStore>,
std::sync::Arc<crate::store::ECStore>,
) {
test_pool_stores_with_contexts(None, false, 2, 3).await
}
#[cfg(all(test, feature = "test-util"))]
pub(crate) async fn test_three_pool_stores(
rebalance_meta: Option<RebalanceMeta>,
+1 -10
View File
@@ -17380,17 +17380,8 @@ mod metadata_mutation_generation_tests {
version_id: old.version_id.map(|id| id.to_string()),
versioned: true,
mod_time: Some(old_time),
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..Default::default()
};
let dst_opts = ObjectOptions {
object_lock_config_snapshot: Some(Arc::new(ObjectLockConfigSnapshot::new(
ObjectLockConfigState::ConfirmedAbsent,
))),
..opts.clone()
};
let mut source = set.get_object_info(bucket, object, &opts).await.expect("read old metadata");
assert_eq!(source.user_tags.as_str(), tags);
assert_eq!(source.expires, Some(expires));
@@ -17399,7 +17390,7 @@ mod metadata_mutation_generation_tests {
source.metadata_only = true;
Arc::make_mut(&mut source.user_defined)
.insert(metadata_keys::RESTORE.to_string(), "ongoing-request=\"true\"".to_string());
set.copy_object(bucket, object, bucket, object, &mut source, &opts, &dst_opts)
set.copy_object(bucket, object, bucket, object, &mut source, &opts, &opts)
.await
.expect("restore metadata update");
let restored = set
+1 -96
View File
@@ -111,14 +111,6 @@ fn is_pool_meta_object(bucket: &str, object: &str) -> bool {
bucket == RUSTFS_META_BUCKET && object == POOL_META_NAME
}
/// Republish only for a confirmed missing, stale, or corrupt replica. A failed
/// probe (including a timeout) can leave the replica intact. Republishing to
/// the other drives in that case can leave no quorum after the next drive loss.
fn pool_meta_drive_needs_repair(state: &str) -> bool {
use rustfs_heal_contracts::heal_channel::DriveState;
state == DriveState::Missing.to_str() || state == DriveState::Corrupt.to_str()
}
#[derive(Debug, Clone, Copy)]
enum HealFormatPoolSkip {
Completed,
@@ -1503,11 +1495,7 @@ impl ECStore {
.await?;
let target = pool.get_disks_for_heal_object(POOL_META_NAME, opts)?;
let owns_replica = Arc::ptr_eq(&target, &pool.get_disks_by_key(POOL_META_NAME));
let needs_repair = before
.after
.drives
.iter()
.any(|drive| pool_meta_drive_needs_repair(&drive.state));
let needs_repair = !before.after.drives.is_empty() && before.after.drives.iter().any(|drive| drive.state != "ok");
if opts.dry_run || error.is_some() || !owns_replica || !needs_repair {
return Ok((before, error, absence));
}
@@ -1588,7 +1576,6 @@ mod tests {
use crate::runtime::instance::InstanceContext;
use crate::services::rebalance::{
RebalanceInfo, RebalanceStats, test_three_pool_stores_with_isolated_node_contexts, test_two_pool_stores,
test_two_pool_stores_with_three_drive_sets,
};
use crate::storage_api_contracts::bucket::{
BucketInfo, BucketOperations, BucketOptions, DeleteBucketOptions, MakeBucketOptions,
@@ -3814,88 +3801,6 @@ mod tests {
);
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_read_repair_does_not_republish_for_an_unreachable_drive() {
// Three drives at EC:1. A republish while one drive is unreachable would
// give the two others a version the unreachable drive lacks; losing one
// of them later would leave pool.bin without read quorum.
let (_temp_dirs, store, _other_store) = test_two_pool_stores_with_three_drive_sets().await;
let disks: Vec<DiskStore> = store.pools[0]
.get_disks_by_key(POOL_META_NAME)
.disks
.read()
.await
.iter()
.map(|disk| disk.clone().expect("pool metadata fixture disk should be online"))
.collect();
assert_eq!(disks.len(), 3);
let unreachable = &disks[0];
unreachable.close().await.expect("stop disk monitoring");
unreachable.force_offline_for_test();
let mut replicas_before = Vec::new();
for disk in &disks[1..] {
let raw = disk
.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false)
.await
.expect("reachable pool metadata replica");
replicas_before.push(raw.buf);
}
let (result, error) = tokio::time::timeout(
std::time::Duration::from_secs(30),
store.handle_heal_object(
RUSTFS_META_BUCKET,
POOL_META_NAME,
"",
&HealOpts {
read_repair: true,
pool: Some(0),
..Default::default()
},
),
)
.await
.expect("pool metadata read repair should not hang on an unreachable drive")
.expect("pool metadata read repair should report its outcome");
assert!(error.is_none(), "pool metadata inspection should have read quorum: {error:?}");
assert!(!result.metadata_repair_verified, "skipping a write must not report a verified repair");
assert!(
result
.after
.drives
.iter()
.any(|drive| [DriveState::Offline.to_string(), DriveState::Faulty.to_string()].contains(&drive.state)),
"the fixture drive should be reported unreachable: {:?}",
result.after.drives
);
for (disk, before) in disks[1..].iter().zip(replicas_before) {
let after = disk
.read_xl(RUSTFS_META_BUCKET, POOL_META_NAME, false)
.await
.expect("reachable pool metadata replica");
assert_eq!(after.buf, before, "pool.bin must keep the version the unreachable drive holds");
}
}
#[test]
fn pool_meta_drive_needs_repair_only_for_confirmed_replica_damage() {
for state in [
DriveState::Ok,
DriveState::Offline,
DriveState::Faulty,
DriveState::Unknown(DiskError::Timeout.to_string()),
DriveState::Unknown(DiskError::from(tonic::Status::deadline_exceeded("metadata read deadline")).to_string()),
DriveState::Unknown(DiskError::from(tonic::Status::permission_denied("metadata read denied")).to_string()),
] {
assert!(!pool_meta_drive_needs_repair(state.to_str()), "{state:?}");
}
for state in [DriveState::Missing, DriveState::Corrupt] {
assert!(pool_meta_drive_needs_repair(state.to_str()), "{state:?}");
}
}
#[tokio::test]
#[serial_test::serial]
async fn pool_meta_heal_dry_run_preserves_missing_shard() {
-3
View File
@@ -2854,9 +2854,6 @@ async fn evaluate_prepared_session_policy(policy: &PreparedSessionPolicy, args:
PreparedSessionPolicy::Policy(p) => {
let mut session_policy_args = args.clone();
session_policy_args.is_owner = false;
// Self-service checks ask whether an explicit session-policy Deny
// applies. They must not require an unrelated Allow statement.
session_policy_args.deny_only = args.deny_only;
Some(p.is_allowed(&session_policy_args).await)
}
}
+5
View File
@@ -709,6 +709,8 @@ pub struct ScannerMetrics {
pub last_cycle_partial_source: String,
#[serde(rename = "last_cycle_partial_source_code", default)]
pub last_cycle_partial_source_code: u64,
#[serde(rename = "last_cycle_partial_stage", default, skip_serializing_if = "Option::is_none")]
pub last_cycle_partial_stage: Option<String>,
#[serde(rename = "last_cycle_duration_seconds", default)]
pub last_cycle_duration_seconds: f64,
#[serde(rename = "last_cycle_objects_scanned", default)]
@@ -853,6 +855,7 @@ impl ScannerMetrics {
self.last_cycle_partial_reason_code = other.last_cycle_partial_reason_code;
self.last_cycle_partial_source = other.last_cycle_partial_source.clone();
self.last_cycle_partial_source_code = other.last_cycle_partial_source_code;
self.last_cycle_partial_stage.clone_from(&other.last_cycle_partial_stage);
self.last_cycle_duration_seconds = other.last_cycle_duration_seconds;
self.throttle_idle_mode_enabled = other.throttle_idle_mode_enabled;
self.throttle_sleep_factor = other.throttle_sleep_factor;
@@ -2138,6 +2141,7 @@ mod tests {
collected_at: collected_at + SignedDuration::from_secs(1),
last_cycle_partial_source: "lifecycle".to_string(),
last_cycle_partial_source_code: 2,
last_cycle_partial_stage: Some("object_budget".into()),
pacing_pressure: ScannerPacingPressureSnapshot {
primary_pressure: "queued_scans".to_string(),
current_queued_scans: 4,
@@ -2162,6 +2166,7 @@ mod tests {
assert_eq!(scanner.last_cycle_partial_source, "lifecycle");
assert_eq!(scanner.last_cycle_partial_source_code, 2);
assert_eq!(scanner.last_cycle_partial_stage.as_deref(), Some("object_budget"));
assert_eq!(scanner.pacing_pressure.primary_pressure, "queued_scans");
assert_eq!(scanner.pacing_pressure.current_queued_scans, 4);
assert_eq!(scanner.pacing_pressure.current_active_scans, 3);
+10 -82
View File
@@ -83,11 +83,6 @@ const GET_OBJECT_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
];
// Keep millisecond resolution for fast requests and coarser buckets for long streams.
const HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
0.0001, 0.001, 0.002, 0.003, 0.005, 0.01, 0.025, 0.05, 0.25, 1.0, 5.0, 25.0, 120.0, 600.0, 2500.0, 10000.0,
];
const LOCAL_BATCH_DURATION_HISTOGRAM_METRICS: &[&str] = &[
"rustfs_io_get_object_local_batch_prepare_seconds",
"rustfs_io_get_object_local_batch_stage_seconds",
@@ -527,15 +522,16 @@ fn duration_histogram_view(instrument: &Instrument) -> Option<Stream> {
}
fn duration_histogram_buckets(name: &str) -> Option<&'static [f64]> {
match name {
"rustfs_http_server_request_duration_seconds"
| "rustfs_http_server_response_body_chunk_latency_seconds"
| "rustfs_http_server_response_body_stream_duration_seconds" => Some(HTTP_SERVER_DURATION_HISTOGRAM_BUCKETS),
_ if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS),
_ if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS),
_ if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS),
_ if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) => Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS),
_ => None,
if LOCAL_BATCH_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(LOCAL_BATCH_DURATION_HISTOGRAM_BUCKETS)
} else if GET_OBJECT_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(GET_OBJECT_DURATION_HISTOGRAM_BUCKETS)
} else if OBJECT_LOCK_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS)
} else if INTERNODE_DURATION_HISTOGRAM_METRICS.contains(&name) {
Some(INTERNODE_DURATION_HISTOGRAM_BUCKETS)
} else {
None
}
}
@@ -1160,79 +1156,11 @@ mod tests {
"rustfs_io_get_object_local_batch_shards",
"rustfs_object_lock_diag_slow_acquire_total",
"rustfs_object_lock_diag_acquire_duration_seconds_bucket",
"rustfs_http_server_response_body_chunk_size_bytes",
] {
assert!(duration_histogram_buckets(name).is_none());
}
}
#[test]
fn http_server_duration_histograms_export_seconds_buckets() {
use opentelemetry_sdk::metrics::{
InMemoryMetricExporter,
data::{AggregatedMetrics, MetricData},
};
let exporter = InMemoryMetricExporter::default();
let reader = PeriodicReader::builder(exporter.clone())
.with_interval(Duration::from_secs(60))
.build();
let (provider, recorder) = Recorder::builder("http-server-duration-test")
.with_meter_provider(|builder| builder.with_reader(reader).with_view(duration_histogram_view))
.build();
let instruments = [
"rustfs_http_server_request_duration_seconds",
"rustfs_http_server_response_body_chunk_latency_seconds",
"rustfs_http_server_response_body_stream_duration_seconds",
];
let observations = [
(0.00008, 0.0001),
(0.0024, 0.003),
(0.037, 0.05),
(42.0, 120.0),
(4000.0, 10000.0),
(12000.0, f64::INFINITY),
];
metrics::with_local_recorder(&recorder, || {
for name in instruments {
let histogram = metrics::histogram!(name);
for (seconds, _) in observations {
histogram.record(seconds);
}
}
});
provider.force_flush().expect("export HTTP duration observations");
let exported = exporter.get_finished_metrics().expect("read exported HTTP histograms");
let expected_sum = observations.iter().map(|(seconds, _)| seconds).sum::<f64>();
let expected_buckets = observations.map(|(_, bound)| (bound, 1));
for name in instruments {
let metric = exported
.iter()
.flat_map(|resource| resource.scope_metrics())
.flat_map(|scope| scope.metrics())
.find(|metric| metric.name() == name)
.unwrap_or_else(|| panic!("{name} must be exported"));
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
panic!("{name} must export as an f64 histogram");
};
let mut points = histogram.data_points();
let point = points.next().unwrap_or_else(|| panic!("{name} must export a point"));
assert!(points.next().is_none(), "{name} must export one point");
assert_eq!(point.count(), 6, "{name} must retain every observation");
assert!((point.sum() - expected_sum).abs() < 1e-9, "{name} must retain the sum in seconds");
assert_eq!(point.bucket_counts().count(), 17, "{name} must have 16 finite buckets and overflow");
let occupied: Vec<_> = point
.bounds()
.chain([f64::INFINITY])
.zip(point.bucket_counts())
.filter(|(_, count)| *count != 0)
.collect();
assert_eq!(occupied, expected_buckets, "{name}");
}
provider.shutdown().expect("stop HTTP histogram test provider");
}
#[test]
fn object_lock_histograms_export_millisecond_and_long_hold_buckets() {
use metrics::Recorder as _;
+163 -69
View File
@@ -282,19 +282,6 @@ impl ScannerWorkSource {
}
}
fn from_code(code: u8) -> Option<Self> {
match code {
1 => Some(Self::Usage),
2 => Some(Self::Lifecycle),
3 => Some(Self::BucketReplication),
4 => Some(Self::SiteReplication),
5 => Some(Self::Heal),
6 => Some(Self::Bitrot),
7 => Some(Self::Alerts),
_ => None,
}
}
fn all() -> &'static [Self] {
&Self::ALL
}
@@ -975,6 +962,34 @@ impl ScannerUsagePublicationObservation {
}
}
#[derive(Clone, Copy, Default)]
struct LastScanCycle {
sequence: u64,
result: u8,
partial_reason: ScanCyclePartialReason,
partial_source: Option<ScannerWorkSource>,
partial_stage: Option<&'static str>,
duration_millis: u64,
}
/// Updates only this observation if a later state write fails. A supervisor
/// cancellation leaves the already recorded partial result and budget visible.
pub struct PartialCycleObservation<'a> {
latest: &'a Mutex<LastScanCycle>,
sequence: u64,
}
impl PartialCycleObservation<'_> {
pub fn observe_state_write(self, persisted: bool) {
if !persisted {
let mut last = self.latest.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
if last.sequence == self.sequence {
last.partial_stage = Some("cycle_state_not_durable");
}
}
}
}
pub struct Metrics {
startup: crate::startup::ScannerStartupObservation,
operations: Vec<AtomicU64>,
@@ -1020,11 +1035,8 @@ pub struct Metrics {
scanner_leader_lock_held: AtomicBool,
scanner_leader_lock_last_error: RwLock<String>,
scanner_leader_lock_last_update_unix_secs: AtomicU64,
last_scan_cycle_result: AtomicU8,
last_scan_cycle_partial_reason: AtomicU8,
last_scan_cycle_partial_source: AtomicU8,
last_scan_cycle: Mutex<LastScanCycle>,
last_scan_cycle_end_unix_secs: AtomicU64,
last_scan_cycle_duration_millis: AtomicU64,
last_scan_cycle_objects_scanned: AtomicU64,
last_scan_cycle_directories_scanned: AtomicU64,
/// Lifetime count of object versions walked by the data scanner, counted
@@ -1140,7 +1152,6 @@ pub struct Metrics {
partial_scan_cycles: AtomicU64,
}
const SCAN_CYCLE_RESULT_UNKNOWN: u8 = 0;
const SCAN_CYCLE_RESULT_SUCCESS: u8 = 1;
const SCAN_CYCLE_RESULT_ERROR: u8 = 2;
const SCAN_CYCLE_RESULT_PARTIAL: u8 = 3;
@@ -1172,12 +1183,12 @@ impl ScanCyclePartialReason {
}
}
fn from_code(code: u8) -> Self {
match code {
1 => Self::Runtime,
2 => Self::Objects,
3 => Self::Directories,
_ => Self::Unknown,
fn budget_stage(self) -> Option<&'static str> {
match self {
Self::Runtime => Some("runtime_budget"),
Self::Objects => Some("object_budget"),
Self::Directories => Some("directory_budget"),
Self::Unknown => None,
}
}
}
@@ -1577,6 +1588,9 @@ pub struct ScannerMetricsReport {
pub last_cycle_partial_source: String,
#[serde(default)]
pub last_cycle_partial_source_code: u64,
/// First observed blocker for this partial cycle, including the exact budget when known.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub last_cycle_partial_stage: Option<String>,
pub last_cycle_duration_seconds: f64,
#[serde(default)]
pub last_cycle_objects_scanned: u64,
@@ -2071,8 +2085,18 @@ pub fn emit_scan_cycle_partial_with_source(
reason: ScanCyclePartialReason,
source: Option<ScannerWorkSource>,
) {
global_metrics().record_scan_cycle_partial_with_source(duration, reason, source);
emit_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
}
pub fn emit_scan_cycle_partial_with_stage(
duration: Duration,
reason: ScanCyclePartialReason,
source: Option<ScannerWorkSource>,
stage: Option<&'static str>,
) -> PartialCycleObservation<'static> {
let observation = global_metrics().record_scan_cycle_partial_with_stage(duration, reason, source, stage);
metrics::counter!(OTEL_SCANNER_CYCLES, "result" => SCAN_CYCLE_RESULT_PARTIAL_LABEL).increment(1);
observation
}
pub fn emit_scan_cycle_superseded(duration: Duration) {
@@ -2173,11 +2197,8 @@ impl Metrics {
scanner_leader_lock_held: AtomicBool::new(false),
scanner_leader_lock_last_error: RwLock::new(String::new()),
scanner_leader_lock_last_update_unix_secs: AtomicU64::new(0),
last_scan_cycle_result: AtomicU8::new(SCAN_CYCLE_RESULT_UNKNOWN),
last_scan_cycle_partial_reason: AtomicU8::new(ScanCyclePartialReason::Unknown as u8),
last_scan_cycle_partial_source: AtomicU8::new(0),
last_scan_cycle: Mutex::new(LastScanCycle::default()),
last_scan_cycle_end_unix_secs: AtomicU64::new(0),
last_scan_cycle_duration_millis: AtomicU64::new(0),
last_scan_cycle_objects_scanned: AtomicU64::new(0),
last_scan_cycle_directories_scanned: AtomicU64::new(0),
lifetime_versions_scanned: AtomicU64::new(0),
@@ -3143,36 +3164,43 @@ impl Metrics {
self.failed_scan_cycles.fetch_add(1, Ordering::Relaxed);
SCAN_CYCLE_RESULT_ERROR
};
self.record_scanner_cycle_end_time();
self.last_scan_cycle_result.store(result, Ordering::Relaxed);
self.last_scan_cycle_partial_reason
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
self.record_last_cycle(
duration,
LastScanCycle {
result,
..Default::default()
},
);
}
pub fn record_scan_cycle_superseded(&self, duration: Duration) {
self.record_scanner_cycle_end_time();
self.superseded_scan_cycles.fetch_add(1, Ordering::Relaxed);
self.last_scan_cycle_result
.store(SCAN_CYCLE_RESULT_SUPERSEDED, Ordering::Relaxed);
self.last_scan_cycle_partial_reason
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
self.record_last_cycle(
duration,
LastScanCycle {
result: SCAN_CYCLE_RESULT_SUPERSEDED,
..Default::default()
},
);
}
pub fn record_scan_cycle_deferred(&self, duration: Duration) {
self.record_last_cycle(
duration,
LastScanCycle {
result: SCAN_CYCLE_RESULT_DEFERRED,
..Default::default()
},
);
}
fn record_last_cycle(&self, duration: Duration, mut cycle: LastScanCycle) -> u64 {
self.record_scanner_cycle_end_time();
self.last_scan_cycle_result
.store(SCAN_CYCLE_RESULT_DEFERRED, Ordering::Relaxed);
self.last_scan_cycle_partial_reason
.store(ScanCyclePartialReason::Unknown as u8, Ordering::Relaxed);
self.last_scan_cycle_partial_source.store(0, Ordering::Relaxed);
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
cycle.duration_millis = duration_millis_saturated(duration);
let mut latest = self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
cycle.sequence = latest.sequence.wrapping_add(1);
*latest = cycle;
cycle.sequence
}
pub fn record_scan_cycle_partial(&self, duration: Duration, reason: ScanCyclePartialReason) {
@@ -3185,7 +3213,16 @@ impl Metrics {
reason: ScanCyclePartialReason,
source: Option<ScannerWorkSource>,
) {
self.record_scanner_cycle_end_time();
self.record_scan_cycle_partial_with_stage(duration, reason, source, reason.budget_stage());
}
fn record_scan_cycle_partial_with_stage(
&self,
duration: Duration,
reason: ScanCyclePartialReason,
source: Option<ScannerWorkSource>,
stage: Option<&'static str>,
) -> PartialCycleObservation<'_> {
self.partial_scan_cycles.fetch_add(1, Ordering::Relaxed);
match reason {
ScanCyclePartialReason::Unknown => &self.partial_scan_cycles_unknown,
@@ -3194,18 +3231,25 @@ impl Metrics {
ScanCyclePartialReason::Directories => &self.partial_scan_cycles_directories,
}
.fetch_add(1, Ordering::Relaxed);
self.last_scan_cycle_result
.store(SCAN_CYCLE_RESULT_PARTIAL, Ordering::Relaxed);
self.last_scan_cycle_partial_reason.store(reason as u8, Ordering::Relaxed);
self.last_scan_cycle_partial_source
.store(source.map(ScannerWorkSource::code).unwrap_or_default(), Ordering::Relaxed);
if let Some(source) = source
&& let Some(cycles) = self.partial_scan_cycles_by_source.get(source.index())
{
cycles.fetch_add(1, Ordering::Relaxed);
}
self.last_scan_cycle_duration_millis
.store(duration_millis_saturated(duration), Ordering::Relaxed);
let sequence = self.record_last_cycle(
duration,
LastScanCycle {
result: SCAN_CYCLE_RESULT_PARTIAL,
partial_reason: reason,
partial_source: source,
partial_stage: stage,
..Default::default()
},
);
PartialCycleObservation {
latest: &self.last_scan_cycle,
sequence,
}
}
pub fn start_scan_cycle_work(&self) -> ScanCycleWorkSnapshot {
@@ -3794,19 +3838,19 @@ impl Metrics {
m.current_disk_scan_concurrency_limit = disk_scan_concurrency_limit;
m.current_disk_bucket_scans_queued = disk_bucket_scans_queued;
m.current_disk_bucket_scans_active = disk_bucket_scans_active;
let last_cycle_result = self.last_scan_cycle_result.load(Ordering::Relaxed);
m.last_cycle_result = scan_cycle_result_label(last_cycle_result).to_string();
m.last_cycle_result_code = last_cycle_result as u64;
let partial_reason = ScanCyclePartialReason::from_code(self.last_scan_cycle_partial_reason.load(Ordering::Relaxed));
m.last_cycle_partial_reason = partial_reason.as_str().to_string();
m.last_cycle_partial_reason_code = partial_reason as u8 as u64;
let partial_source = ScannerWorkSource::from_code(self.last_scan_cycle_partial_source.load(Ordering::Relaxed));
m.last_cycle_partial_source = partial_source
let last = *self.last_scan_cycle.lock().unwrap_or_else(|poisoned| poisoned.into_inner());
m.last_cycle_result = scan_cycle_result_label(last.result).to_string();
m.last_cycle_result_code = last.result.into();
m.last_cycle_partial_reason = last.partial_reason.as_str().to_string();
m.last_cycle_partial_reason_code = last.partial_reason as u64;
m.last_cycle_partial_source = last
.partial_source
.map(ScannerWorkSource::as_str)
.unwrap_or(SCAN_CYCLE_RESULT_UNKNOWN_LABEL)
.to_string();
m.last_cycle_partial_source_code = partial_source.map(|source| source.code().into()).unwrap_or_default();
m.last_cycle_duration_seconds = self.last_scan_cycle_duration_millis.load(Ordering::Relaxed) as f64 / 1000.0;
m.last_cycle_partial_source_code = last.partial_source.map(|source| source.code().into()).unwrap_or_default();
m.last_cycle_partial_stage = last.partial_stage.map(str::to_owned);
m.last_cycle_duration_seconds = last.duration_millis as f64 / 1000.0;
m.last_cycle_objects_scanned = self.last_scan_cycle_objects_scanned.load(Ordering::Relaxed);
m.last_cycle_directories_scanned = self.last_scan_cycle_directories_scanned.load(Ordering::Relaxed);
m.last_cycle_bucket_drive_scans = self.last_scan_cycle_bucket_drive_scans.load(Ordering::Relaxed);
@@ -5223,6 +5267,56 @@ mod tests {
assert_eq!(report.partial_cycles, 1);
}
#[tokio::test]
async fn delayed_partial_state_write_can_only_update_its_own_observation() {
let metrics = Metrics::new();
let record = || {
metrics.record_scan_cycle_partial_with_stage(
Duration::from_secs(1),
ScanCyclePartialReason::Runtime,
None,
Some("runtime_budget"),
)
};
let old = record();
let current = record();
assert_eq!(
metrics.report().await.partial_cycles,
2,
"both stops are visible before either state write finishes"
);
old.observe_state_write(false);
assert_eq!(metrics.report().await.last_cycle_partial_stage.as_deref(), Some("runtime_budget"));
current.observe_state_write(false);
let failed = metrics.report().await;
assert_eq!(failed.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
assert_eq!(failed.partial_cycles, 2);
assert_eq!(failed.last_cycle_partial_reason, "runtime");
let superseded = record();
metrics.record_scan_cycle_complete(true, Duration::ZERO);
superseded.observe_state_write(false);
assert!(metrics.report().await.last_cycle_partial_stage.is_none());
}
#[tokio::test]
async fn partial_stage_is_reported_with_the_last_result_and_cleared_on_completion() {
let metrics = Metrics::new();
metrics.record_scan_cycle_partial(Duration::from_secs(1), ScanCyclePartialReason::Directories);
let partial = serde_json::to_value(metrics.report().await).unwrap();
assert_eq!(partial["last_cycle_partial_reason"], "directories");
assert_eq!(partial["last_cycle_partial_stage"], "directory_budget");
metrics.record_scan_cycle_complete(true, Duration::from_secs(1));
let finished = serde_json::to_value(metrics.report().await).unwrap();
assert!(
!finished
.as_object()
.expect("serialized report")
.contains_key("last_cycle_partial_stage")
);
let legacy: ScannerMetricsReport = serde_json::from_value(finished).expect("older reports omit the stage");
assert_eq!(legacy.last_cycle_partial_stage, None);
}
#[tokio::test]
async fn report_tracks_partial_scan_cycle_without_failed_increment() {
let metrics = Metrics::new();
+8 -1
View File
@@ -73,6 +73,7 @@ hotpath-cpu = [
]
[dependencies]
tempfile.workspace = true
hotpath.workspace = true
rustfs-config = { workspace = true, features = ["server-config-model"] }
rustfs-common = { workspace = true }
@@ -108,6 +109,13 @@ uuid = { workspace = true, features = ["v4", "serde", "fast-rng"] }
bytes.workspace = true
hex-simd.workspace = true
[target.'cfg(any(target_os = "linux", target_os = "macos"))'.dependencies]
rustix = { workspace = true, features = ["fs"] }
[target.'cfg(windows)'.dependencies]
same-file.workspace = true
winapi-util.workspace = true
[dev-dependencies]
s3s = { workspace = true, features = ["minio"] }
opentelemetry-proto.workspace = true
@@ -119,7 +127,6 @@ rustfs-test-utils.workspace = true
tracing-subscriber = { workspace = true, features = ["json", "env-filter", "time"] }
serial_test = { workspace = true }
temp-env = { workspace = true, features = ["async_closure"] }
tempfile = { workspace = true }
uuid = { workspace = true, features = ["v4", "serde", "fast-rng", "macro-diagnostics"] }
tokio = { workspace = true, features = ["test-util", "fs", "rt-multi-thread"] }
# Test-only: pins the emitted scanner alert wire names against the canonical
+145 -23
View File
@@ -200,10 +200,25 @@ pub(crate) async fn read_config_revision<S: ScannerObjectIO>(store: Arc<S>, path
}
}
#[derive(Clone, Debug, PartialEq, Eq)]
#[derive(Clone, Debug, Eq)]
pub(crate) struct DataUsageCacheRevisions {
main: DataUsageCacheRevision,
backup: Option<DataUsageCacheRevision>,
damaged_pages: DamagedPages,
}
// Admissions compare CAS tokens. A reader's local repair hint does not
// indicate that either committed object changed while its locks were released.
impl PartialEq for DataUsageCacheRevisions {
fn eq(&self, other: &Self) -> bool {
self.main == other.main && self.backup == other.backup
}
}
#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
struct DamagedPages {
main: bool,
backup: bool,
}
pub static DATA_USAGE_BUCKET: LazyLock<String> =
@@ -576,10 +591,31 @@ pub(crate) struct DataUsageDurableProgress {
identity: Option<DataUsageScanIdentity>,
started_plan: Option<DataUsageScanPlanDigest>,
frontier: Option<String>,
dormant: HashMap<String, String>,
raw: HashMap<String, crate::raw_page_index::RawEnumerationDurableProgress>,
compacted: Vec<crate::scanner_folder::DirectoryProgress>,
listings: Vec<crate::scanner_folder::DirectoryReadProgress>,
active_source: Option<String>,
}
impl DataUsageDurableProgress {
fn source_frontier(&self, source: Option<&str>) -> Option<&str> {
if source == self.active_source.as_deref() {
self.frontier.as_deref()
} else {
source.and_then(|source| self.dormant.get(source)).map(String::as_str)
}
}
fn covers_on(&self, source: Option<&str>, path: &str) -> bool {
self.source_frontier(source)
.is_some_and(|frontier| scan_frontier_covers_path(path, frontier))
|| self
.compacted
.iter()
.any(|done| Some(done.physical.endpoint.as_str()) == source && done.covers(path))
}
pub(crate) fn advanced_from(&self, previous: &Self) -> bool {
if self.bucket != previous.bucket
|| self.source.is_none()
@@ -591,31 +627,69 @@ impl DataUsageDurableProgress {
{
return false;
}
let mut advanced = match (&previous.frontier, &self.frontier) {
(Some(before), Some(after)) => {
if !scan_frontier_covers_path(before, after) {
return false;
let active = self.active_source.as_deref();
let old = previous.active_source.as_deref();
let switched = old.is_some() && active != old;
// A frontier can move from active to dormant without earning a turn.
// Retention and forward movement are compared only on the same source.
if previous.frontier.as_deref().is_some_and(|path| !self.covers_on(old, path))
|| previous
.dormant
.iter()
.any(|(source, path)| !self.covers_on(Some(source), path))
{
return false;
}
let mut advanced = self.frontier.as_deref().is_some_and(|now| {
previous
.source_frontier(active)
.is_none_or(|before| before != now && scan_frontier_covers_path(before, now))
});
if switched {
// Legacy raw-name pages are local to their writer. The physical
// root position must still exist for that writer across rotation.
if !previous.raw.is_empty()
&& !previous.listings.iter().any(|before| {
before.parent == self.bucket
&& Some(before.physical.endpoint.as_str()) == old
&& self.listings.iter().any(|after| after.extends(before))
})
{
return false;
}
} else {
for (parent, before) in &previous.raw {
match self.raw.get(parent) {
Some(after) if after.extends(before) => {}
None if self.covers_on(active, parent) => {}
_ => return false,
}
before != after
}
(Some(_), None) => return false,
(None, Some(_)) => true,
(None, None) => false,
};
for (parent, before) in &previous.raw {
match self.raw.get(parent) {
Some(after) if after.extends(before) => {}
None if self
.frontier
.as_ref()
.is_some_and(|frontier| scan_frontier_covers_path(parent, frontier)) => {}
_ => return false,
advanced |= self.raw.iter().any(|(parent, after)| {
after.committed_entries() > previous.raw.get(parent).map_or(0, |before| before.committed_entries())
});
}
for before in &previous.listings {
if !self.listings.iter().any(|after| after.extends(before))
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
{
return false;
}
}
for (parent, after) in &self.raw {
let before = previous.raw.get(parent).map_or(0, |value| value.committed_entries());
advanced |= after.committed_entries() > before;
for before in &previous.compacted {
if !self.compacted.iter().any(|after| after.extends(before))
&& !self.covers_on(Some(&before.physical.endpoint), &before.parent)
{
return false;
}
}
advanced |= self.listings.iter().any(|after| {
Some(after.physical.endpoint.as_str()) == active && !previous.listings.iter().any(|before| before.extends(after))
});
advanced |= self.compacted.iter().any(|after| {
Some(after.physical.endpoint.as_str()) == active && !previous.compacted.iter().any(|before| before.extends(after))
});
advanced
}
}
@@ -905,6 +979,14 @@ pub struct DataUsageCacheInfo {
pub scan_raw_enumeration_cursor: Option<DataUsageRawEnumerationCursor>,
#[serde(default, skip_serializing_if = "Option::is_none")]
pub scan_raw_enumeration_page_index: Option<RawEnumerationPageIndex>,
/// Source-bound native listing positions and fully committed compacted work.
#[serde(default)]
pub scan_directory_work: Vec<crate::scanner_folder::DirectoryCheckpoint>,
/// Physical source of the regular linked tree and its frontier.
#[serde(default)]
pub scan_directory_source: Option<String>,
#[serde(default)]
pub scan_source_trees: std::collections::BTreeMap<String, Arc<source_tree::SourceTree>>,
/// Indexes of the other directories on the active walk path. A restart
/// must replay every wide ancestor before it reaches the selected one, and
/// recharging those replays would stop a fixed budget from ever advancing.
@@ -994,6 +1076,9 @@ impl Serialize for DataUsageCacheInfo {
+ usize::from(self.scan_raw_enumeration_cursor.is_some())
+ usize::from(self.scan_raw_enumeration_page_index.is_some())
+ usize::from(!self.scan_raw_enumeration_path_page_indexes.is_empty())
+ usize::from(!self.scan_directory_work.is_empty())
+ usize::from(self.scan_directory_source.is_some())
+ usize::from(!self.scan_source_trees.is_empty())
+ usize::from(self.scan_identity.is_some())
+ usize::from(self.scan_progress.is_some())
+ usize::from(self.scan_usage_watch.is_some())
@@ -1029,6 +1114,15 @@ impl Serialize for DataUsageCacheInfo {
if let Some(index) = &self.scan_raw_enumeration_page_index {
state.serialize_entry("scan_raw_enumeration_page_index", index)?;
}
if !self.scan_source_trees.is_empty() {
state.serialize_entry("scan_source_trees", &self.scan_source_trees)?;
}
if let Some(source) = &self.scan_directory_source {
state.serialize_entry("scan_directory_source", source)?;
}
if !self.scan_directory_work.is_empty() {
state.serialize_entry("scan_directory_work", &self.scan_directory_work)?;
}
if !self.scan_raw_enumeration_path_page_indexes.is_empty() {
state.serialize_entry("scan_raw_enumeration_path_page_indexes", &self.scan_raw_enumeration_path_page_indexes)?;
}
@@ -1236,6 +1330,25 @@ impl DataUsageCache {
identity: self.info.scan_identity,
started_plan: self.info.scan_progress.map(|progress| progress.started_plan),
frontier: self.validated_scan_frontier().map(str::to_owned),
dormant: self
.info
.scan_source_trees
.iter()
.filter_map(|(source, tree)| tree.frontier(&self.info).map(|last| (source.clone(), last.to_owned())))
.collect(),
active_source: self.info.scan_directory_source.clone(),
listings: self
.info
.scan_directory_work
.iter()
.filter_map(|d| d.read_progress(&self.info))
.collect(),
compacted: self
.info
.scan_directory_work
.iter()
.filter_map(|d| d.progress(&self.info))
.collect(),
raw: self
.info
.scan_raw_enumeration_page_index
@@ -1249,7 +1362,8 @@ impl DataUsageCache {
}
pub(crate) fn has_scan_position(&self) -> bool {
self.info.scan_resume_after.is_some()
!self.info.scan_directory_work.is_empty()
|| self.info.scan_resume_after.is_some()
|| self.info.scan_checkpoint.is_some()
|| self.info.scan_raw_enumeration_cursor.is_some()
|| self.info.scan_raw_enumeration_page_index.is_some()
@@ -1319,7 +1433,8 @@ impl DataUsageCache {
&& self.info.scan_identity == Some(identity)
&& (self.validated_scan_frontier().is_some()
|| self.validated_raw_enumeration_cursor().is_some()
|| self.validated_raw_enumeration_page_index().is_some());
|| self.validated_raw_enumeration_page_index().is_some()
|| self.info.scan_directory_work.iter().any(|d| d.is_bound_to(&self.info)));
let handoff_frontier = cross_epoch_checkpoint
.then(|| self.validated_scan_frontier().map(clone_cache_value))
.flatten();
@@ -1404,7 +1519,10 @@ impl DataUsageCache {
.take(MAX_DATA_USAGE_CACHE_DEPTH)
.filter(|index| self.raw_enumeration_page_index_is_valid(index))
.collect();
let directory_work = std::mem::take(&mut self.info.scan_directory_work);
self.info.scan_directory_work = directory_work.into_iter().filter(|d| d.is_bound_to(&self.info)).collect();
let cursor_is_valid = !self.has_scan_position()
|| !self.info.scan_directory_work.is_empty()
|| self.validated_scan_frontier().is_some()
|| self.info.scan_raw_enumeration_cursor.is_some()
|| self.info.scan_raw_enumeration_page_index.is_some();
@@ -1447,6 +1565,9 @@ impl DataUsageCache {
self.info.scan_raw_enumeration_cursor = None;
self.info.scan_raw_enumeration_page_index = None;
self.info.scan_raw_enumeration_path_page_indexes.clear();
self.info.scan_directory_work.clear();
self.info.scan_directory_source = None;
self.info.scan_source_trees.clear();
self.info.scan_coverage_receipt = None;
}
// Old readers do not understand coverage sweeps. An absent plan makes
@@ -2192,6 +2313,7 @@ impl DataUsageCache {
}
mod persistence;
mod source_tree;
#[derive(Default, Clone)]
struct Inner {
@@ -12,6 +12,9 @@
// See the License for the specific language governing permissions and
// limitations under the License.
mod pages;
mod scan_load;
use super::*;
use bytes::Bytes;
use std::sync::Arc;
@@ -70,6 +73,7 @@ struct DataUsageCacheLoadResult {
cache: DataUsageCache,
main_revision: DataUsageCacheRevision,
backup_revision: Option<DataUsageCacheRevision>,
damaged_pages: DamagedPages,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
@@ -106,34 +110,42 @@ impl DataUsageCache {
store: Arc<S>,
name: &str,
) -> StorageResult<DataUsageCacheRevisions> {
let backup_name = format!("{name}.bkp");
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
let loaded = Self::load_cache(store.clone(), name).await?;
let backup = match loaded.backup_revision {
Some(revision) => Some(revision),
None => match read_config_revision(store, &backup_path).await {
Ok(revision) => Some(revision),
Err(err) => {
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
debug!(
target: "rustfs::scanner::data_usage",
event = EVENT_SCANNER_CACHE_LOAD_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_CACHE,
cache_name = %name,
backup_path = %backup_path,
state = "backup_revision_unavailable",
error = %err,
"Scanner cache backup revision lookup failed"
);
None
}
},
None => Self::backup_revision(store, name).await,
};
let main = loaded.main_revision;
*self = loaded.cache;
Ok(DataUsageCacheRevisions { main, backup })
Ok(DataUsageCacheRevisions {
main,
backup,
damaged_pages: loaded.damaged_pages,
})
}
async fn backup_revision<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> Option<DataUsageCacheRevision> {
let backup_name = format!("{name}.bkp");
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
match read_config_revision(store, &backup_path).await {
Ok(revision) => Some(revision),
Err(err) => {
counter!(METRIC_CACHE_BACKUP_REVISION_FAILURE_TOTAL).increment(1);
debug!(
target: "rustfs::scanner::data_usage",
event = EVENT_SCANNER_CACHE_LOAD_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_CACHE,
cache_name = %name,
backup_path = %backup_path,
state = "backup_revision_unavailable",
error = %err,
"Scanner cache backup revision lookup failed"
);
None
}
}
}
pub(crate) async fn read_revisions<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheRevisions> {
@@ -142,7 +154,11 @@ impl DataUsageCache {
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
let main = read_config_revision(store.clone(), &main_path).await?;
let backup = Some(read_config_revision(store, &backup_path).await?);
Ok(DataUsageCacheRevisions { main, backup })
Ok(DataUsageCacheRevisions {
main,
backup,
damaged_pages: DamagedPages::default(),
})
}
async fn load_cache<S: ScannerObjectIO>(store: Arc<S>, name: &str) -> StorageResult<DataUsageCacheLoadResult> {
@@ -160,6 +176,7 @@ impl DataUsageCache {
cache: *cache,
main_revision,
backup_revision: None,
damaged_pages: DamagedPages::default(),
});
}
DataUsageCacheLoadAttempt::Loaded { revision: None, .. } => {
@@ -184,6 +201,11 @@ impl DataUsageCache {
let backup_name = format!("{name}.bkp");
let backup_attempt = Self::try_load_inner(store.clone(), &backup_name, Duration::from_secs(30)).await?;
let backup_revision = backup_attempt.revision();
if backup_revision.is_some()
&& let Some(DataUsageCacheRevision::Etag(etag)) = &main_revision
{
scan_load::PendingBackup::finish(&store, name, etag);
}
let backup_state = match backup_attempt {
DataUsageCacheLoadAttempt::Loaded {
cache,
@@ -197,6 +219,10 @@ impl DataUsageCache {
cache: *cache,
main_revision,
backup_revision: Some(backup_revision),
damaged_pages: DamagedPages {
main: main_state == DataUsageCacheLoadState::Corrupt,
backup: false,
},
});
}
DataUsageCacheLoadState::Loaded
@@ -220,6 +246,10 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache missing state has no revision: {name}")))?,
backup_revision,
damaged_pages: DamagedPages {
main: main_state == DataUsageCacheLoadState::Corrupt,
backup: backup_state == DataUsageCacheLoadState::Corrupt,
},
});
}
(DataUsageCacheLoadState::Corrupt, DataUsageCacheLoadState::Missing)
@@ -239,6 +269,10 @@ impl DataUsageCache {
main_revision: main_revision
.ok_or_else(|| Error::other(format!("scanner cache corrupt state has no revision: {name}")))?,
backup_revision,
damaged_pages: DamagedPages {
main: main_state == DataUsageCacheLoadState::Corrupt,
backup: backup_state == DataUsageCacheLoadState::Corrupt,
},
});
}
_ => {}
@@ -294,10 +328,35 @@ impl DataUsageCache {
.filter(|etag| !etag.is_empty())
.cloned()
.map(DataUsageCacheRevision::Etag);
if let Some(DataUsageCacheRevision::Etag(etag)) = &revision
&& scan_load::PendingBackup::needs_backup(&store, load_name, etag)
{
return Ok((None, revision, true));
}
match reader.read_all().await {
Ok(data) => match DataUsageCache::unmarshal(&data) {
Ok(data) => match Box::pin(pages::decode(store.clone(), load_name, &data)).await {
Ok(cache) => Ok((Some(cache), revision, false)),
Err(_) => Ok((None, revision, true)),
Err(err)
if matches!(err, StorageError::Io(ref io) if io.kind() == std::io::ErrorKind::InvalidData)
|| matches!(err, StorageError::FileNotFound | StorageError::ObjectNotFound(..))
|| (pages::is_manifest(&data) && pages::unreadable_page(&err)) =>
{
// A concurrent manifest commit can retire the generation
// we were reading. Retry the current root before treating
// missing pages as corruption or falling back to a backup.
if pages::is_manifest(&data) {
if Some(read_config_revision(store.clone(), &path).await?) != revision {
return Err(StorageError::PreconditionFailed);
}
if !load_name.ends_with(".bkp")
&& let Some(DataUsageCacheRevision::Etag(etag)) = &revision
{
scan_load::PendingBackup::begin(&store, load_name, etag);
}
}
Ok((None, revision, true))
}
Err(err) => Err(err),
},
Err(e) => {
// Read error
@@ -643,20 +702,44 @@ impl DataUsageCache {
expected_epoch: Option<u64>,
) -> StorageResult<()> {
let serialize_started = Instant::now();
let mut buf = Vec::new();
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
// Primary, backup, and retries read the same immutable encoded body.
let buf = Bytes::from(buf);
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(u64::try_from(buf.len()).unwrap_or(u64::MAX));
let paged = matches!((revisions, expected_epoch), (Some(_), Some(_))) && pages::eligible(self, name);
let encoded = if paged { Some(pages::encode(self)?) } else { None };
let inline = if paged {
Bytes::new()
} else {
let mut buf = Vec::new();
self.serialize(&mut rmp_serde::Serializer::new(&mut buf))?;
Bytes::from(buf)
};
counter!("rustfs_scanner_cache_serialized_bytes_total").increment(
encoded
.as_ref()
.map_or_else(|| u64::try_from(inline.len()).unwrap_or(u64::MAX), pages::EncodedPages::length),
);
histogram!("rustfs_scanner_cache_serialize_seconds").record(serialize_started.elapsed().as_secs_f64());
let timeout_duration = Self::cache_save_timeout();
let (primary_buf, primary_retired) = match (&encoded, revisions, expected_epoch) {
(Some(encoded), Some(revisions), Some(epoch)) => {
Box::pin(pages::prepare(
self,
store.clone(),
name,
&revisions.main,
encoded,
epoch,
revisions.damaged_pages.main,
))
.await?
}
_ => (inline.clone(), None),
};
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
let primary_save_started = Instant::now();
Self::save_path_with_retry(
store.clone(),
&path,
&buf,
&primary_buf,
timeout_duration,
DATA_USAGE_CACHE_SAVE_RETRIES,
revisions.map(|revisions| revisions.main.clone()),
@@ -664,6 +747,9 @@ impl DataUsageCache {
)
.await?;
if let (Some(retired), Some(epoch)) = (primary_retired, expected_epoch) {
let _ = pages::reclaim(store.clone(), name, retired, epoch).await;
}
let backup_name = format!("{name}.bkp");
let backup_path = path_join_buf(&[BUCKET_META_PREFIX, &backup_name]);
let backup_timeout_duration = Self::backup_cache_save_timeout(timeout_duration, primary_save_started.elapsed());
@@ -671,17 +757,38 @@ impl DataUsageCache {
if revisions.is_some() && backup_revision.is_none() {
return Ok(());
}
if let Err(e) = Self::save_path_with_retry(
store,
&backup_path,
&buf,
backup_timeout_duration,
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
backup_revision,
expected_epoch,
)
.await
{
let save_backup = async {
let (backup_buf, retired) = match (&encoded, &backup_revision, expected_epoch) {
(Some(encoded), Some(revision), Some(epoch)) => {
Box::pin(pages::prepare(
self,
store.clone(),
&backup_name,
revision,
encoded,
epoch,
revisions.is_some_and(|revisions| revisions.damaged_pages.backup),
))
.await?
}
_ => (inline, None),
};
Self::save_path_with_retry(
store.clone(),
&backup_path,
&backup_buf,
backup_timeout_duration,
DATA_USAGE_CACHE_BACKUP_SAVE_RETRIES,
backup_revision,
expected_epoch,
)
.await?;
if let (Some(bundle), Some(epoch)) = (retired, expected_epoch) {
let _ = pages::reclaim(store, &backup_name, bundle, epoch).await;
}
Ok::<_, StorageError>(())
};
if let Err(e) = save_backup.await {
warn!(
target: "rustfs::scanner::data_usage",
event = EVENT_SCANNER_CACHE_SAVE_STATE,
@@ -0,0 +1,579 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
mod restore;
mod save;
mod staging;
use super::*;
use futures::future::try_join_all;
use sha2::Digest as _;
use std::collections::{BTreeMap, HashSet};
use std::io::Write;
use tokio::io::AsyncReadExt;
const PAGE_MIN: usize = 8 * 1024;
const PAGE_MAX: usize = 64 * 1024;
const FANOUT: usize = 128;
const IO_CONCURRENCY: usize = 4;
const PREFIX: &[u8] = b"\x83\xa7pagesV1";
type Digest = [u8; 32];
/// Older readers see the committed compact bucket or an incomplete empty tree.
#[derive(Deserialize)]
struct PageEnvelope {
#[serde(rename = "pagesV1")]
header: Header,
info: DataUsageCacheInfo,
#[serde(rename = "cache")]
_cache: serde::de::IgnoredAny,
}
#[derive(Debug, Serialize, Deserialize)]
struct Header {
body: Payload,
retired: Option<uuid::Uuid>,
}
#[derive(Debug, Serialize, Deserialize)]
enum Payload {
Inline,
Pages(Manifest),
}
impl Header {
fn manifest(&self) -> Option<&Manifest> {
match &self.body {
Payload::Pages(manifest) => Some(manifest),
Payload::Inline => None,
}
}
}
#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
struct Manifest {
bundle: uuid::Uuid,
finished: bool,
cycle: u64,
length: u64,
level: u8,
root: Digest,
#[serde(default)]
bundle_bytes: u64,
#[serde(default)]
bundle_commits: u16,
}
#[derive(Debug, Default)]
struct Archive {
blocks: BTreeMap<Digest, Bytes>,
leaves: Vec<Digest>,
buffer: Vec<u8>,
rolling: u64,
length: u64,
}
fn invalid(message: &str) -> Error {
Error::Io(std::io::Error::new(std::io::ErrorKind::InvalidData, message.to_owned()))
}
pub(super) fn unreadable_page(error: &StorageError) -> bool {
matches!(error, StorageError::FileCorrupt)
|| matches!(error, StorageError::Io(io) if matches!(io.kind(), std::io::ErrorKind::InvalidData | std::io::ErrorKind::UnexpectedEof))
}
fn digest(data: &[u8]) -> Digest {
Sha256::digest(data).into()
}
fn block_path(name: &str, bundle: uuid::Uuid, block: &Digest) -> String {
let hash = hex_simd::encode_to_string(block, hex_simd::AsciiCase::Lower);
format!("{BUCKET_META_PREFIX}/{name}.pages/{bundle}/{hash}")
}
fn bucket_cache_name(name: &str) -> Option<&str> {
let (bucket, leaf) = name.split_once('/')?;
(!crate::is_reserved_or_invalid_bucket(bucket, true) && leaf == DATA_USAGE_CACHE_NAME).then_some(name)
}
// The high bits of a multiplicative hash of each byte give a rolling
// content-defined boundary. Insertion before a sealed run need not shift the
// offsets of every following page. Page length is bounded for every input.
const fn byte_gear(byte: u8) -> u64 {
let mut x = (byte as u64).wrapping_add(0x9e37_79b9_7f4a_7c15);
x = (x ^ (x >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9);
x = (x ^ (x >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb);
x ^ (x >> 31)
}
const GEAR: [u64; 256] = {
let mut table = [0; 256];
let mut byte = 0u8;
loop {
table[byte as usize] = byte_gear(byte);
if byte == u8::MAX {
break;
}
byte += 1;
}
table
};
impl Archive {
fn intern(&mut self, bytes: Vec<u8>) -> Digest {
let key = digest(&bytes);
self.blocks.entry(key).or_insert_with(|| Bytes::from(bytes));
key
}
fn seal(&mut self) {
if !self.buffer.is_empty() {
let bytes = std::mem::take(&mut self.buffer);
let key = self.intern(bytes);
self.leaves.push(key);
self.rolling = 0;
}
}
fn finish(&mut self, bundle: uuid::Uuid) -> StorageResult<Manifest> {
self.seal();
let mut children = std::mem::take(&mut self.leaves);
let mut level = 0;
while children.len() > 1 {
children = children
.chunks(FANOUT)
.map(|chunk| {
let bytes = chunk.iter().flatten().copied().collect();
self.intern(bytes)
})
.collect();
level += 1;
}
Ok(Manifest {
bundle,
finished: false,
cycle: 0,
length: self.length,
level,
root: children.pop().ok_or_else(|| invalid("empty scanner checkpoint"))?,
bundle_bytes: 0,
bundle_commits: 0,
})
}
}
impl Write for Archive {
fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
self.length = self
.length
.checked_add(u64::try_from(bytes.len()).map_err(std::io::Error::other)?)
.ok_or_else(|| std::io::Error::other("scanner checkpoint is too large"))?;
let mut start = 0;
for (offset, byte) in bytes.iter().enumerate() {
self.rolling = self.rolling.wrapping_shl(1).wrapping_add(GEAR[usize::from(*byte)]);
let end = offset + 1;
let len = self.buffer.len() + (end - start);
if len == PAGE_MAX || (len >= PAGE_MIN && self.rolling & 0x3fff == 0) {
self.buffer.extend_from_slice(&bytes[start..end]);
self.seal();
start = end;
}
}
self.buffer.extend_from_slice(&bytes[start..]);
Ok(bytes.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
pub(super) struct EncodedPages {
archive: Archive,
root: Manifest,
inline: Option<Bytes>,
}
impl EncodedPages {
pub(super) fn length(&self) -> u64 {
self.archive.length
}
}
pub(super) fn encode(cache: &DataUsageCache) -> StorageResult<EncodedPages> {
let mut archive = Archive::default();
archive.write_all(&[0x92])?;
cache.info.serialize(&mut rmp_serde::Serializer::new(&mut archive))?;
archive.seal();
let count = u32::try_from(cache.cache.len()).map_err(|_| invalid("too many scanner checkpoint entries"))?;
// A wide root's growing child set must not shift every subtree page.
let mut entries: Vec<_> = cache.cache.iter().collect();
entries.sort_unstable_by(|a, b| a.0.cmp(b.0));
let mut header = Vec::new();
if count < 16 {
header.push(0x80 | u8::try_from(count).map_err(|_| invalid("invalid fixed map length"))?);
} else if let Ok(n) = u16::try_from(count) {
header.push(0xde);
header.extend_from_slice(&n.to_be_bytes());
} else {
header.push(0xdf);
header.extend_from_slice(&count.to_be_bytes());
}
archive.write_all(&header)?;
for (name, entry) in entries {
let mut serializer = rmp_serde::Serializer::new(&mut archive);
name.serialize(&mut serializer)?;
entry.ordered_encoding().serialize(&mut serializer)?;
if name == &cache.info.name {
archive.seal();
}
}
archive.seal();
let inline = if archive.length <= PAGE_MIN as u64 {
let mut buf = Vec::new();
for key in &archive.leaves {
buf.extend_from_slice(
archive
.blocks
.get(key)
.ok_or_else(|| invalid("missing encoded scanner page"))?,
);
}
Some(Bytes::from(buf))
} else {
None
};
let root = archive.finish(uuid::Uuid::nil())?;
Ok(EncodedPages { archive, root, inline })
}
pub(super) fn is_manifest(bytes: &[u8]) -> bool {
bytes.starts_with(PREFIX)
}
fn read_envelope(bytes: &[u8], bucket: &str) -> StorageResult<PageEnvelope> {
let head: PageEnvelope = rmp_serde::from_slice(bytes).map_err(|_| invalid("invalid scanner checkpoint manifest"))?;
if head.info.name != bucket
|| head.header.retired.is_some_and(|id| id.is_nil())
|| head.header.manifest().is_some_and(|manifest| {
manifest.bundle.is_nil() || manifest.length == 0 || manifest.level > 9 || head.header.retired == Some(manifest.bundle)
})
{
return Err(invalid("invalid scanner checkpoint identity or size"));
}
Ok(head)
}
async fn read_block<S: ScannerObjectIO>(store: Arc<S>, name: &str, bundle: uuid::Uuid, key: Digest) -> StorageResult<Bytes> {
let path = block_path(name, bundle, &key);
let mut reader = store
.get_object_reader(
RUSTFS_META_BUCKET,
&path,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await?;
if reader.object_info.size < 1 || usize::try_from(reader.object_info.size).map_or(true, |size| size > PAGE_MAX) {
return Err(invalid("invalid scanner checkpoint page length"));
}
let bytes = Bytes::from(reader.read_all().await?);
if bytes.len() > PAGE_MAX || digest(&bytes) != key {
return Err(invalid("scanner checkpoint page checksum mismatch"));
}
Ok(bytes)
}
async fn committed_keys<S: ScannerObjectIO>(
store: Arc<S>,
name: &str,
manifest: &Manifest,
candidate: &Archive,
) -> StorageResult<HashSet<Digest>> {
let mut keys = vec![manifest.root];
let mut known = HashSet::from([manifest.root]);
for _ in 0..manifest.level {
let parents = std::mem::take(&mut keys);
for batch in parents.chunks(IO_CONCURRENCY) {
let mut reads = Vec::new();
let mut nodes = Vec::with_capacity(batch.len());
for &key in batch {
// The committed parent authenticates the table digest.
// Unchanged tables are already available in the candidate.
if let Some(node) = candidate.blocks.get(&key) {
nodes.push(node.clone());
} else {
reads.push(read_block(store.clone(), name, manifest.bundle, key));
}
}
nodes.extend(try_join_all(reads).await?);
for node in nodes {
if node.is_empty() || node.len() > FANOUT * 32 || node.len() % 32 != 0 {
return Err(invalid("invalid scanner checkpoint page table"));
}
for &key in node.as_chunks::<32>().0 {
keys.push(key);
known.insert(key);
}
if u64::try_from(keys.len()).unwrap_or(u64::MAX) > manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3) {
return Err(invalid("scanner checkpoint contains excess page references"));
}
}
}
}
Ok(known)
}
pub(super) async fn decode<S: ScannerObjectIO>(store: Arc<S>, load_name: &str, bytes: &[u8]) -> StorageResult<DataUsageCache> {
if !is_manifest(bytes) {
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid scanner cache"));
}
let name = load_name.strip_suffix(".bkp").unwrap_or(load_name);
let bucket = bucket_cache_name(name)
.and_then(|name| name.split_once('/'))
.ok_or_else(|| invalid("misplaced scanner checkpoint"))?
.0;
let head = read_envelope(bytes, bucket)?;
let Some(manifest) = head.header.manifest() else {
return DataUsageCache::unmarshal(bytes).map_err(|_| invalid("invalid inline scanner checkpoint"));
};
let cache = restore::restore(store, load_name, manifest).await?;
if cache.info.name != bucket
|| cache.info.snapshot_complete != manifest.finished
|| cache.info.next_cycle != manifest.cycle
|| cache.info.scan_identity != head.info.scan_identity
|| cache.info.source != head.info.source
|| cache.info.leader_epoch != head.info.leader_epoch
|| cache.info.next_cycle != head.info.next_cycle
{
return Err(invalid("scanner checkpoint scope changed"));
}
Ok(cache)
}
async fn previous_header<S: ScannerObjectIO>(
store: Arc<S>,
name: &str,
revision: &DataUsageCacheRevision,
) -> StorageResult<Option<Header>> {
if *revision == DataUsageCacheRevision::Missing {
return Ok(None);
}
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
let mut reader = store
.get_object_reader(
RUSTFS_META_BUCKET,
&path,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await?;
if reader
.object_info
.etag
.as_ref()
.map(|etag| DataUsageCacheRevision::Etag(etag.clone()))
.as_ref()
!= Some(revision)
{
return Err(StorageError::PreconditionFailed);
}
// Header is the first value in the envelope. A large compatibility
// projection or legacy cache does not require reading its body here.
let mut prefix = Vec::new();
(&mut reader).take(1024).read_to_end(&mut prefix).await?;
if !is_manifest(&prefix) {
return Ok(None);
}
let mut deserializer = rmp_serde::Deserializer::new(&prefix[PREFIX.len()..]);
// A damaged old header cannot participate in reuse or reclamation.
// Rebuilding uses the CAS revision obtained by the normal loader.
Ok(Header::deserialize(&mut deserializer).ok().filter(|header| {
header.retired.is_none_or(|id| !id.is_nil())
&& header
.manifest()
.is_none_or(|m| !m.bundle.is_nil() && m.length != 0 && m.level <= 9 && header.retired != Some(m.bundle))
}))
}
fn build_bundle(name: &str, revision: &DataUsageCacheRevision, cache: &DataUsageCache) -> uuid::Uuid {
// Before the first root CAS, retries and competing builders populate the
// same staging namespace. Individual page names are immutable digests.
let mut hash = Sha256::new();
hash.update(b"rustfs-scanner-checkpoint-v1\0");
hash.update(name.as_bytes());
if let Some(identity) = cache.info.scan_identity {
hash.update(identity.bucket_incarnation.as_bytes());
}
if let DataUsageCacheRevision::Etag(etag) = revision {
hash.update(etag.as_bytes());
}
let mut id = [0; 16];
id.copy_from_slice(&hash.finalize()[..16]);
uuid::Uuid::from_bytes(id)
}
pub(super) fn eligible(cache: &DataUsageCache, name: &str) -> bool {
bucket_cache_name(name).is_some()
&& cache.info.scan_identity.is_some()
&& cache.info.name == name.split('/').next().unwrap_or("")
}
pub(super) async fn prepare<S: ScannerObjectIO + ScannerConfigObjectDelete>(
cache: &DataUsageCache,
store: Arc<S>,
name: &str,
revision: &DataUsageCacheRevision,
encoded: &EncodedPages,
epoch: u64,
damaged: bool,
) -> StorageResult<(Bytes, Option<uuid::Uuid>)> {
let previous_header = previous_header(store.clone(), name, revision).await?;
let previous = previous_header.as_ref().and_then(Header::manifest);
let mut retired = previous_header.as_ref().and_then(|header| header.retired);
if let Some(id) = retired
&& reclaim(store.clone(), name, id, epoch).await.is_ok()
{
retired = None;
}
// Each root has its own page namespace and can retire independently of
// the other copy after a successful conditional commit.
let allow_rotation = retired.is_none();
if let Some(inline) = &encoded.inline
&& (previous.is_none() || allow_rotation)
{
retired = retired.or_else(|| previous.map(|head| head.bundle));
let bytes = if retired.is_some() {
rmp_serde::to_vec_named(&CompatibilityEnvelope {
header: &Header {
body: Payload::Inline,
retired,
},
info: &cache.info,
cache: &cache.cache,
})
.map_err(|_| invalid("failed to encode inline scanner checkpoint"))?
} else {
return Ok((inline.clone(), None));
};
return Ok((Bytes::from(bytes), retired));
}
let live_bytes = save::block_bytes(encoded.archive.blocks.values());
let reuse = match previous {
Some(head) if !damaged && !(head.finished && head.cycle != cache.info.next_cycle) => {
save::reuse(store.clone(), name, head, &encoded.archive, live_bytes).await?
}
_ => None,
};
let verify_backup = name.ends_with(".bkp");
let (bundle, known, bundle_bytes, bundle_commits) = if let Some((head, known, total)) = reuse {
(head.bundle, known, total, head.bundle_commits + 1)
} else {
if let Some(head) = previous {
if !allow_rotation {
// Keep both readable generations and their cleanup obligation.
// Further staging into either bundle would defeat the space bound.
return Err(Error::other("cannot rotate scanner checkpoint before reclaiming its retired bundle"));
}
retired = Some(head.bundle);
}
(build_bundle(name, revision, cache), HashSet::new(), live_bytes, 1)
};
let mut manifest = encoded.root.clone();
manifest.bundle = bundle;
manifest.finished = cache.info.snapshot_complete;
manifest.cycle = cache.info.next_cycle;
manifest.bundle_bytes = bundle_bytes;
manifest.bundle_commits = bundle_commits;
if let Err(error) = save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &known).await {
// A corrupt XL header need not yield an ETag for a conditional repair.
// Rebuild this backup under a fresh namespace; its old root stays visible
// until the manifest CAS. Other I/O failures remain retryable in place.
if !(verify_backup && manifest.bundle_commits > 1 && allow_rotation && unreadable_page(&error)) {
return Err(error);
}
retired = Some(manifest.bundle);
manifest.bundle = build_bundle(name, revision, cache);
manifest.bundle_bytes = live_bytes;
manifest.bundle_commits = 1;
save::stage(store.clone(), name, revision, epoch, &manifest, &encoded.archive, &HashSet::new()).await?;
}
let header = Header {
body: Payload::Pages(manifest),
retired,
};
let projection = |info: &DataUsageCacheInfo, cache: &BTreeMap<String, DataUsageEntry>| {
rmp_serde::to_vec_named(&CompatibilityEnvelope {
header: &header,
info,
cache,
})
.map_err(|_| invalid("failed to encode scanner checkpoint root"))
};
if cache.info.snapshot_complete {
// Older readers can serve compact totals when the compatibility
// projection fits. The manifest remains bounded even with many
// pending repairs or a large lifecycle configuration.
let mut root = cache
.checked_flatten_complete_scope(&cache.info.name)
.ok_or_else(|| invalid("disconnected complete bucket checkpoint"))?;
root.compacted = true;
let bytes = projection(&cache.info, &BTreeMap::from([(cache.info.name.clone(), root)]))?;
if bytes.len() <= PAGE_MIN {
return Ok((Bytes::from(bytes), retired));
}
}
let info = DataUsageCacheInfo {
name: cache.info.name.clone(),
source: cache.info.source,
scan_identity: cache.info.scan_identity,
leader_epoch: cache.info.leader_epoch,
next_cycle: cache.info.next_cycle,
..Default::default()
};
let bytes = projection(&info, &BTreeMap::new())?;
Ok((Bytes::from(bytes), retired))
}
#[derive(Serialize)]
struct CompatibilityEnvelope<'a, T> {
#[serde(rename = "pagesV1")]
header: &'a Header,
info: &'a DataUsageCacheInfo,
cache: &'a T,
}
pub(super) async fn reclaim<S: ScannerObjectIO + ScannerConfigObjectDelete>(
store: Arc<S>,
name: &str,
bundle: uuid::Uuid,
epoch: u64,
) -> StorageResult<()> {
let _admission = scanner_publication_admission_for_epoch(store.clone(), epoch)
.await
.ok_or_else(cache_publication_epoch_changed)?;
let prefix = format!("{BUCKET_META_PREFIX}/{name}.pages/{bundle}/");
let opts = ObjectOptions {
delete_prefix: true,
..Default::default()
};
match timeout(
DataUsageCache::cache_save_timeout(),
store.delete_config_object(RUSTFS_META_BUCKET, &prefix, opts),
)
.await
{
Ok(Ok(_)) | Ok(Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..))) => Ok(()),
Ok(Err(error)) => Err(error),
Err(_) => Err(Error::other("retired scanner checkpoint cleanup timed out")),
}
}
@@ -0,0 +1,207 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::super::scan_load::park_restore;
use super::*;
use std::any::Any;
use std::fs::File;
use std::io::{Seek, SeekFrom};
use std::sync::{Mutex, Weak};
use tokio::time::Instant as WallInstant;
const MEMORY_LIMIT: u64 = 1024 * 1024;
const PARKED_LIMIT: usize = 32;
enum Buffer {
Memory(Vec<u8>),
File(Arc<Mutex<File>>),
}
struct Position {
pending: Vec<(Digest, u8)>,
length: u64,
leaves: u64,
nodes: u64,
buffer: Buffer,
writer: Option<tokio::task::JoinHandle<std::io::Result<()>>>,
decoder: Option<tokio::task::JoinHandle<StorageResult<DataUsageCache>>>,
}
struct Suspended {
owner: Weak<dyn Any + Send + Sync>,
name: String,
manifest: Manifest,
last_polled: WallInstant,
position: Position,
}
struct Recovery(Option<Suspended>);
static PARKED: LazyLock<Mutex<Vec<Suspended>>> = LazyLock::new(|| Mutex::new(Vec::new()));
impl Recovery {
async fn open<S: ScannerObjectIO>(store: &Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<Self> {
let erased: Arc<dyn Any + Send + Sync> = store.clone();
let owner = Arc::downgrade(&erased);
let resumed = {
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
parked.retain(|job| {
job.owner.strong_count() != 0 && (!job.owner.ptr_eq(&owner) || job.name != name || job.manifest == *manifest)
});
parked
.iter()
.position(|job| job.owner.ptr_eq(&owner) && job.name == name && job.manifest == *manifest)
.map(|index| parked.swap_remove(index))
};
if let Some(mut job) = resumed {
job.last_polled = WallInstant::now();
return Ok(Self(Some(job)));
}
let buffer = if manifest.length <= MEMORY_LIMIT {
Buffer::Memory(Vec::new())
} else {
let file = tokio::task::spawn_blocking(tempfile::tempfile)
.await
.map_err(Error::other)??;
Buffer::File(Arc::new(Mutex::new(file)))
};
Ok(Self(Some(Suspended {
owner,
name: name.to_owned(),
manifest: manifest.clone(),
last_polled: WallInstant::now(),
position: Position {
pending: vec![(manifest.root, manifest.level)],
length: 0,
leaves: 0,
nodes: 0,
buffer,
writer: None,
decoder: None,
},
})))
}
}
// Park only verified bytes and traversal state. Outstanding metadata reads
// and their admissions are dropped with the waiter. A local spool write or
// decode may finish after cancellation and is joined before advancing again.
impl Drop for Recovery {
fn drop(&mut self) {
let Some(mut job) = self.0.take() else { return };
if job.position.nodes == 0 && job.position.leaves == 0 {
return;
}
let mut parked = PARKED.lock().unwrap_or_else(|error| error.into_inner());
job.last_polled = WallInstant::now();
if let Some(previous) = parked
.iter_mut()
.find(|previous| previous.owner.ptr_eq(&job.owner) && previous.name == job.name && previous.manifest == job.manifest)
{
// Readers of an immutable root traverse the same ordered prefix.
// Keep the furthest admission regardless of cancellation order.
if job.position.verified_progress() > previous.position.verified_progress() {
std::mem::swap(previous, &mut job);
}
} else {
let displaced = park_restore(&mut parked, PARKED_LIMIT, job, |job| job.last_polled);
drop(parked);
drop(displaced);
}
}
}
impl Position {
fn verified_progress(&self) -> (u64, bool) {
(self.nodes + self.leaves, self.decoder.is_some())
}
async fn restore<S: ScannerObjectIO>(
&mut self,
store: Arc<S>,
name: &str,
manifest: &Manifest,
) -> StorageResult<DataUsageCache> {
let max_leaves = manifest.length.div_ceil(PAGE_MIN as u64).saturating_add(3);
loop {
// Retain the local write across cancellation. The verified page already
// owns its offset and must not consume another storage read next round.
if let Some(writer) = &mut self.writer {
writer.await.map_err(Error::other)??;
self.writer = None;
}
let Some(&(key, level)) = self.pending.last() else { break };
if level != 0 {
let node = read_block(store.clone(), name, manifest.bundle, key).await?;
if node.is_empty()
|| node.len() > FANOUT * 32
|| node.len() % 32 != 0
|| self.nodes >= max_leaves.saturating_mul(u64::from(manifest.level))
{
return Err(invalid("invalid scanner checkpoint restore table"));
}
self.pending.pop();
for &child in node.as_chunks::<32>().0.iter().rev() {
self.pending.push((child, level - 1));
}
self.nodes += 1;
continue;
}
let block = read_block(store.clone(), name, manifest.bundle, key).await?;
let next = self
.length
.checked_add(u64::try_from(block.len()).map_err(|_| invalid("invalid restore length"))?)
.filter(|len| *len <= manifest.length)
.ok_or_else(|| invalid("scanner checkpoint exceeds its committed length"))?;
if self.leaves >= max_leaves {
return Err(invalid("too many scanner checkpoint leaves"));
}
match &mut self.buffer {
Buffer::Memory(bytes) => bytes.extend_from_slice(&block),
Buffer::File(file) => {
let file = file.clone();
let offset = self.length;
self.writer = Some(tokio::task::spawn_blocking(move || {
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
file.seek(SeekFrom::Start(offset))?;
file.write_all(&block)
}));
}
}
self.pending.pop();
self.length = next;
self.leaves += 1;
}
if self.length != manifest.length {
return Err(invalid("incomplete scanner checkpoint"));
}
let job = self.decoder.get_or_insert_with(|| {
let buffer = std::mem::replace(&mut self.buffer, Buffer::Memory(Vec::new()));
tokio::task::spawn_blocking(move || {
let result = match buffer {
Buffer::Memory(bytes) => rmp_serde::from_slice(&bytes),
Buffer::File(file) => {
let mut file = file.lock().unwrap_or_else(|error| error.into_inner());
file.seek(SeekFrom::Start(0))?;
rmp_serde::from_read(std::io::BufReader::new(&mut *file))
}
};
result.map_err(|_| invalid("invalid scanner checkpoint payload"))
})
});
job.await.map_err(Error::other)?
}
}
pub(super) async fn restore<S: ScannerObjectIO>(store: Arc<S>, name: &str, manifest: &Manifest) -> StorageResult<DataUsageCache> {
let mut job = Recovery::open(&store, name, manifest).await?;
let result = job
.0
.as_mut()
.ok_or_else(|| invalid("missing checkpoint restore state"))?
.position
.restore(store, name, manifest)
.await;
job.0.take();
result
}
@@ -0,0 +1,138 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
// A losing writer can stage blocks without publishing its accounting. Even
// tiny or identical successful checkpoints periodically retire these orphans.
const MAX_BUNDLE_COMMITS: u16 = 256;
pub(super) fn block_bytes<'a>(blocks: impl Iterator<Item = &'a Bytes>) -> u64 {
blocks.fold(0u64, |sum, bytes| sum.saturating_add(u64::try_from(bytes.len()).unwrap_or(u64::MAX)))
}
pub(super) async fn reuse<'a, S: ScannerObjectIO>(
store: Arc<S>,
name: &str,
head: &'a Manifest,
candidate: &Archive,
live: u64,
) -> StorageResult<Option<(&'a Manifest, HashSet<Digest>, u64)>> {
// Legacy manifests did not account for unreachable blocks in this bundle.
if head.bundle_bytes == 0 || head.bundle_commits == 0 || head.bundle_commits >= MAX_BUNDLE_COMMITS {
return Ok(None);
}
let known = match committed_keys(store, name, head, candidate).await {
Ok(known) => known,
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(None),
Err(error) if unreadable_page(&error) => return Ok(None),
Err(error) => return Err(error),
};
// Count reachable allocation once at bundle creation and newly staged
// bytes thereafter. A small delta stays incremental even across cycles.
let added = block_bytes(
candidate
.blocks
.iter()
.filter_map(|(key, bytes)| (!known.contains(key)).then_some(bytes)),
);
let total = head.bundle_bytes.checked_add(added);
Ok(total
.filter(|&total| total <= live.saturating_mul(2))
.map(|total| (head, known, total)))
}
/// Returns the conditional write necessary to restore this content-addressed
/// backup page, or None when the bytes in storage already match the candidate.
async fn backup_revision<S: ScannerObjectIO>(
store: &S,
path: &str,
expected: &Bytes,
) -> StorageResult<Option<DataUsageCacheRevision>> {
let mut reader = match store
.get_object_reader(
RUSTFS_META_BUCKET,
path,
None,
HeaderMap::new(),
&ObjectOptions {
no_lock: true,
..Default::default()
},
)
.await
{
Ok(reader) => reader,
Err(StorageError::FileNotFound | StorageError::ObjectNotFound(..)) => return Ok(Some(DataUsageCacheRevision::Missing)),
Err(error) => return Err(error),
};
if usize::try_from(reader.object_info.size).ok() == Some(expected.len()) {
let mut bytes = Vec::with_capacity(expected.len());
(&mut reader).take(PAGE_MAX as u64 + 1).read_to_end(&mut bytes).await?;
if bytes == expected.as_ref() {
return Ok(None);
}
}
let etag = reader
.object_info
.etag
.ok_or_else(|| invalid("damaged scanner backup page has no revision"))?;
Ok(Some(DataUsageCacheRevision::Etag(etag)))
}
pub(super) async fn stage<S: ScannerObjectIO + ScannerConfigObjectDelete>(
store: Arc<S>,
name: &str,
revision: &DataUsageCacheRevision,
epoch: u64,
manifest: &Manifest,
archive: &Archive,
known: &HashSet<Digest>,
) -> StorageResult<()> {
let verify_backup = name.ends_with(".bkp");
let staging = staging::Staging::open(&store, name, manifest.bundle, epoch, revision);
let mut writes = Vec::with_capacity(IO_CONCURRENCY);
for (key, bytes) in &archive.blocks {
let staged = staging.contains(key);
if !verify_backup && (known.contains(key) || staged) {
continue;
}
let path = block_path(name, manifest.bundle, key);
let store = store.clone();
let bytes = bytes.clone();
let key = *key;
let receipts = &staging;
writes.push(async move {
let needed = if verify_backup && (manifest.bundle_commits > 1 || staged) {
// A healthy primary load does not establish the integrity of its
// independent backup. Check reused bytes and repair only damaged pages.
match timeout(DataUsageCache::cache_save_timeout(), backup_revision(store.as_ref(), &path, &bytes))
.await
.map_err(|_| Error::other("scanner backup page validation timed out"))??
{
Some(revision) => revision,
None => return Ok(()),
}
} else {
DataUsageCacheRevision::Missing
};
DataUsageCache::save_path_with_retry(
store,
&path,
&bytes,
DataUsageCache::cache_save_timeout(),
DATA_USAGE_CACHE_SAVE_RETRIES,
Some(needed),
Some(epoch),
)
.await?;
receipts.record(key);
Ok::<(), Error>(())
});
if writes.len() == IO_CONCURRENCY {
try_join_all(std::mem::take(&mut writes)).await?;
}
}
try_join_all(writes).await?;
Ok(())
}
@@ -0,0 +1,76 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use std::any::Any;
use std::sync::{Mutex, Weak};
use std::time::Instant as WallInstant;
/// A manifest commit can be interrupted after any group of immutable PUTs.
/// Reuse only receipts from the same store, epoch and predecessor revision.
struct Attempt {
store: Weak<dyn Any + Send + Sync>,
path: String,
bundle: uuid::Uuid,
epoch: u64,
revision: DataUsageCacheRevision,
seen: Arc<Mutex<HashSet<Digest>>>,
last_used: WallInstant,
}
const MAX_ATTEMPTS: usize = 32;
static ATTEMPTS: LazyLock<Mutex<Vec<Attempt>>> = LazyLock::new(|| Mutex::new(Vec::new()));
pub(super) struct Staging(Arc<Mutex<HashSet<Digest>>>);
impl Staging {
pub(super) fn open<S: ScannerObjectIO>(
store: &Arc<S>,
path: &str,
bundle: uuid::Uuid,
epoch: u64,
revision: &DataUsageCacheRevision,
) -> Self {
let store: Arc<dyn Any + Send + Sync> = store.clone();
let owner = Arc::downgrade(&store);
let mut attempts = ATTEMPTS.lock().unwrap_or_else(|p| p.into_inner());
attempts.retain(|job| job.store.strong_count() != 0 && job.last_used.elapsed() < Duration::from_secs(3600));
if let Some(attempt) = attempts.iter_mut().find(|attempt| {
attempt.store.ptr_eq(&owner)
&& attempt.path == path
&& attempt.bundle == bundle
&& attempt.epoch == epoch
&& &attempt.revision == revision
}) {
attempt.last_used = WallInstant::now();
return Self(attempt.seen.clone());
}
// A successfully committed predecessor naturally retires the receipts
// for its previous ETag. Concurrent older writers retain their own Arc.
attempts.retain(|attempt| !attempt.store.ptr_eq(&owner) || attempt.path != path);
if attempts.len() >= MAX_ATTEMPTS
&& let Some((index, _)) = attempts.iter().enumerate().min_by_key(|(_, job)| job.last_used)
{
attempts.swap_remove(index);
}
let seen = Arc::new(Mutex::new(HashSet::new()));
attempts.push(Attempt {
store: owner,
path: path.to_owned(),
bundle,
epoch,
revision: revision.clone(),
seen: seen.clone(),
last_used: WallInstant::now(),
});
Self(seen)
}
pub(super) fn contains(&self, key: &Digest) -> bool {
self.0.lock().unwrap_or_else(|p| p.into_inner()).contains(key)
}
pub(super) fn record(&self, key: Digest) {
self.0.lock().unwrap_or_else(|p| p.into_inner()).insert(key);
}
}
@@ -0,0 +1,205 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use crate::scanner_budget::ScannerCycleBudget;
use std::any::Any;
use std::sync::{Mutex, Weak};
use tokio_util::sync::CancellationToken;
// Keep only a failed root's identity while its independent backup is being
// recovered. A fresh primary reader still obtains the current ETag every time.
pub(super) struct PendingBackup {
owner: Weak<dyn Any + Send + Sync>,
name: String,
failed_etag: String,
last_polled: tokio::time::Instant,
}
static PENDING_BACKUPS: Mutex<Vec<PendingBackup>> = Mutex::new(Vec::new());
const MAX_PENDING_BACKUPS: usize = 32;
// Age permits reclamation under contention. A scanner whose configured
// interval exceeds this age can still resume while slots remain available.
pub(super) fn park_restore<T>(
slots: &mut Vec<T>,
limit: usize,
value: T,
last_polled: impl Fn(&T) -> tokio::time::Instant,
) -> Option<T> {
if slots.len() < limit {
slots.push(value);
return None;
}
if let Some(oldest) = slots.iter_mut().min_by_key(|slot| last_polled(slot))
&& last_polled(oldest).elapsed() >= Duration::from_secs(3600)
{
return Some(std::mem::replace(oldest, value));
}
Some(value)
}
impl PendingBackup {
fn owner<S: ScannerObjectIO>(store: &Arc<S>) -> Weak<dyn Any + Send + Sync> {
let erased: Arc<dyn Any + Send + Sync> = store.clone();
Arc::downgrade(&erased)
}
pub(super) fn needs_backup<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) -> bool {
let owner = Self::owner(store);
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
let Some(index) = pending
.iter()
.position(|phase| phase.owner.ptr_eq(&owner) && phase.name == name)
else {
return false;
};
if pending[index].failed_etag == etag {
pending[index].last_polled = tokio::time::Instant::now();
true
} else {
pending.swap_remove(index);
false
}
}
pub(super) fn begin<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
let owner = Self::owner(store);
let mut pending = PENDING_BACKUPS.lock().unwrap_or_else(|error| error.into_inner());
pending.retain(|phase| phase.owner.strong_count() > 0 && !(phase.owner.ptr_eq(&owner) && phase.name == name));
let phase = Self {
owner,
name: name.to_owned(),
failed_etag: etag.to_owned(),
last_polled: tokio::time::Instant::now(),
};
let _ = park_restore(&mut pending, MAX_PENDING_BACKUPS, phase, |phase| phase.last_polled);
}
pub(super) fn finish<S: ScannerObjectIO>(store: &Arc<S>, name: &str, etag: &str) {
let owner = Self::owner(store);
PENDING_BACKUPS
.lock()
.unwrap_or_else(|error| error.into_inner())
.retain(|phase| !(phase.owner.ptr_eq(&owner) && phase.name == name && phase.failed_etag == etag));
}
}
struct DecodedCheckpoint {
owner: Weak<dyn Any + Send + Sync>,
name: String,
parked_at: std::time::Instant,
cache: DataUsageCache,
revisions: DataUsageCacheRevisions,
}
// The scanner may finish a cold restore with too little time for the walk.
// Move that exact value to the next admission. Only one decoded bucket can
// occupy the handoff and it never owns storage, a permit, or a publication fence.
static DECODED: Mutex<Option<DecodedCheckpoint>> = Mutex::new(None);
impl DataUsageCache {
pub(crate) async fn load_for_scan<S: ScannerObjectIO>(
&mut self,
store: Arc<S>,
name: &str,
ctx: &CancellationToken,
budget: &ScannerCycleBudget,
) -> StorageResult<Option<DataUsageCacheRevisions>> {
tokio::select! {
biased;
_ = ctx.cancelled() => Ok(None),
result = self.load_for_scan_window(store, name, budget) => result,
}
}
async fn load_for_scan_window<S: ScannerObjectIO>(
&mut self,
store: Arc<S>,
name: &str,
budget: &ScannerCycleBudget,
) -> StorageResult<Option<DataUsageCacheRevisions>> {
let erased: Arc<dyn Any + Send + Sync> = store.clone();
let owner = Arc::downgrade(&erased);
let is_ours = |ready: &DecodedCheckpoint| ready.owner.ptr_eq(&owner) && ready.name == name;
let cached = {
let mut decoded = DECODED.lock().unwrap_or_else(|error| error.into_inner());
// Give a retired or deleted bucket's slot back to other active
// readers. A matching admission can consume it even after a long
// scheduled interval, subject to the revision checks below.
decoded.take_if(|ready| {
ready.owner.strong_count() == 0 || (!is_ours(ready) && ready.parked_at.elapsed() >= Duration::from_secs(3600))
});
decoded.as_ref().is_some_and(is_ours)
};
if cached {
let path = path_join_buf(&[BUCKET_META_PREFIX, name]);
let main = read_config_revision(store.clone(), &path).await;
let backup = Self::backup_revision(store.clone(), name).await;
let ready = DECODED
.lock()
.unwrap_or_else(|error| error.into_inner())
.take_if(|ready| is_ours(ready));
if let (Some(ready), Ok(main)) = (ready, main)
&& ready.revisions.main == main
&& ready.revisions.backup == backup
{
*self = ready.cache;
return Ok(Some(ready.revisions));
}
}
let started = tokio::time::Instant::now();
// Obtain the optional backup head before the expensive decode. A budget
// stop after the decoder finishes must not discard the materialized tree
// while waiting for an unrelated backup lookup.
let backup = Self::backup_revision(store.clone(), name).await;
let loaded = Box::pin(Self::load_cache(store, name)).await?;
let revisions = DataUsageCacheRevisions {
main: loaded.main_revision,
backup: loaded.backup_revision.or(backup),
damaged_pages: loaded.damaged_pages,
};
*self = loaded.cache;
if self.info.scan_identity.is_some()
&& !self.cache.is_empty()
&& budget
.remaining_config()
.max_duration
.is_some_and(|remaining| started.elapsed() > remaining)
{
let mut slot = DECODED.lock().unwrap_or_else(|error| error.into_inner());
if slot.as_ref().is_none_or(|ready| ready.owner.strong_count() == 0) {
*slot = Some(DecodedCheckpoint {
owner,
name: name.to_owned(),
parked_at: std::time::Instant::now(),
cache: std::mem::take(self),
revisions,
});
return Ok(None);
}
}
Ok(Some(revisions))
}
}
#[cfg(test)]
mod tests {
use super::*;
#[tokio::test(start_paused = true)]
async fn only_reclaim_idle_restore_slots_under_pressure() {
let first = tokio::time::Instant::now();
tokio::time::advance(Duration::from_secs(7200)).await;
let second = tokio::time::Instant::now();
let mut slots = vec![first];
assert_eq!(park_restore(&mut slots, 2, second, |&time| time), None);
assert_eq!(slots, [first, second]);
let new = second + Duration::from_secs(1);
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(first));
assert_eq!(slots, [new, second]);
assert_eq!(park_restore(&mut slots, 2, new, |&time| time), Some(new));
assert_eq!(slots, [new, second], "active recoveries retain both slots");
}
}
@@ -0,0 +1,133 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
/// A linked tree paused on a different physical replica. It contains only
/// traversal fields, so dormant sources cannot recursively contain each other.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SourceTree {
bucket: String,
identity: DataUsageScanIdentity,
started_plan: DataUsageScanPlanDigest,
#[serde(serialize_with = "ordered_entries")]
entries: HashMap<String, DataUsageEntry>,
resume_after: Option<String>,
checkpoint: Option<DataUsageScanCheckpoint>,
coverage: Option<DataUsageScanCoverageReceipt>,
cursor: Option<DataUsageRawEnumerationCursor>,
index: Option<RawEnumerationPageIndex>,
ancestors: Vec<RawEnumerationPageIndex>,
}
impl SourceTree {
/// An inactive receipt may preserve an existing source position. Only an
/// active, verified traversal can contribute new progress.
pub(super) fn frontier(&self, info: &DataUsageCacheInfo) -> Option<&str> {
if self.bucket != info.name
|| Some(self.identity) != info.scan_identity
|| Some(self.started_plan) != info.scan_progress.map(|p| p.started_plan)
{
return None;
}
self.coverage.as_ref().map(|receipt| receipt.through.as_str())
}
}
// Dormant trees live in the checkpoint header stream. A cold process has
// fresh HashMap seeds; preserve stable bytes for immutable Merkle pages.
fn ordered_entries<S: serde::Serializer>(
entries: &HashMap<String, DataUsageEntry>,
serializer: S,
) -> std::result::Result<S::Ok, S::Error> {
let mut sorted = entries.iter().collect::<Vec<_>>();
sorted.sort_unstable_by(|a, b| a.0.cmp(b.0));
serializer.collect_map(sorted.into_iter().map(|(name, entry)| (name, entry.ordered_encoding())))
}
impl DataUsageCache {
pub(crate) fn activate_directory_source(&mut self, source: String, endpoints: &HashSet<String>) {
let (Some(identity), Some(progress)) = (self.info.scan_identity, self.info.scan_progress) else {
return;
};
let previous = self.info.scan_directory_source.replace(source.clone());
// Every production endpoint has an erasure-set slot. A disk replacement
// cannot retain a backup under the same slot or under a retired endpoint.
let current_location = directory_source_location(&source);
self.info.scan_directory_work.retain(|work| {
work.endpoint() == source
|| endpoints.contains(directory_source_location(work.endpoint()))
&& directory_source_location(work.endpoint()) != current_location
});
self.info.scan_source_trees.retain(|key, tree| {
tree.bucket == self.info.name
&& tree.identity == identity
&& tree.started_plan == progress.started_plan
&& (key == &source
|| endpoints.contains(directory_source_location(key)) && directory_source_location(key) != current_location)
});
let Some(previous) = previous else {
// Pre-source checkpoints identify an erasure set but never recorded
// which disk supplied the partial tree. The same keys can have
// different metadata on another replica. Keep the certified usage,
// repair obligations and observations outside this traversal.
if self.has_scan_position() {
self.clear_active_directory_traversal();
}
return;
};
if previous == source {
return;
}
let entries = std::mem::take(&mut self.cache);
let linked = entries.get(&hash_path(&self.info.name).0).is_some_and(|root| !root.compacted);
let previous_tree = SourceTree {
bucket: self.info.name.clone(),
identity,
started_plan: progress.started_plan,
entries,
resume_after: self.info.scan_resume_after.take(),
checkpoint: self.info.scan_checkpoint.take(),
coverage: self.info.scan_coverage_receipt.take(),
cursor: self.info.scan_raw_enumeration_cursor.take(),
index: self.info.scan_raw_enumeration_page_index.take(),
ancestors: std::mem::take(&mut self.info.scan_raw_enumeration_path_page_indexes),
};
if linked
&& endpoints.contains(directory_source_location(&previous))
&& directory_source_location(&previous) != current_location
{
self.info.scan_source_trees.insert(previous, Arc::new(previous_tree));
}
if let Some(tree) = self.info.scan_source_trees.remove(&source) {
let tree = Arc::unwrap_or_clone(tree);
self.cache = tree.entries;
self.info.scan_resume_after = tree.resume_after;
self.info.scan_coverage_receipt = tree.coverage;
self.info.scan_checkpoint = tree.checkpoint;
self.info.scan_raw_enumeration_cursor = tree.cursor;
self.info.scan_raw_enumeration_page_index = tree.index;
self.info.scan_raw_enumeration_path_page_indexes = tree.ancestors;
// A dormant tree must provide the same coverage/graph guarantees
// required of an active checkpoint before its child links are used.
if self.checked_flatten_complete_scope(&self.info.name).is_none()
|| self.info.scan_coverage_receipt.is_some() && self.validated_scan_frontier().is_none()
{
self.clear_active_directory_traversal();
}
}
}
fn clear_active_directory_traversal(&mut self) {
self.cache.clear();
self.info.scan_resume_after = None;
self.info.scan_checkpoint = None;
self.info.scan_coverage_receipt = None;
self.info.scan_raw_enumeration_cursor = None;
self.info.scan_raw_enumeration_page_index = None;
self.info.scan_raw_enumeration_path_page_indexes.clear();
}
}
fn directory_source_location(source: &str) -> &str {
source.split_once('|').map_or(source, |(_, endpoint)| endpoint)
}
@@ -653,6 +653,7 @@ async fn cache_save_reconciles_an_ambiguous_committed_write() {
cache.replace("bucket", "", DataUsageEntry::default());
let revisions = DataUsageCacheRevisions {
main: DataUsageCacheRevision::Missing,
damaged_pages: DamagedPages::default(),
backup: None,
};
@@ -2880,6 +2881,7 @@ async fn test_save_backup_budget_tracks_primary_cost() {
let store = Arc::new(store);
let revisions = DataUsageCacheRevisions {
main: DataUsageCacheRevision::Missing,
damaged_pages: DamagedPages::default(),
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
};
DataUsageCache::default()
@@ -2932,6 +2934,7 @@ async fn test_checkpoint_budget_covers_retries_and_reconciliation() {
let store = Arc::new(store);
let revisions = DataUsageCacheRevisions {
main: DataUsageCacheRevision::Missing,
damaged_pages: DamagedPages::default(),
backup: Some(DataUsageCacheRevision::Etag("backup-etag".to_string())),
};
let started = Instant::now();
@@ -88,6 +88,7 @@ async fn check_save(store: BackupFallbackStore, revisions: Option<DataUsageCache
fn revisions(backup: bool) -> DataUsageCacheRevisions {
DataUsageCacheRevisions {
main: DataUsageCacheRevision::Missing,
damaged_pages: DamagedPages::default(),
backup: backup.then_some(DataUsageCacheRevision::Missing),
}
}
+11 -16
View File
@@ -15,6 +15,7 @@
use crate::data_usage_define::clone_cache_value;
use serde::{Deserialize, Serialize};
use sha2::{Digest as _, Sha256};
use std::sync::Arc;
use thiserror::Error;
mod writer;
@@ -58,7 +59,7 @@ pub struct RawEnumerationPage {
parent: String,
page_index: u64,
entries_start: u64,
entries: Vec<String>,
entries: Arc<[String]>,
terminal: bool,
digest: [u8; 32],
}
@@ -518,32 +519,26 @@ impl RawEnumerationPage {
}
fn new(parent: &str, building: RawEnumerationPageBuilder) -> Self {
let digest = raw_page_digest(parent, &building);
let digest = raw_page_digest(parent, building.page_index, building.entries_start, building.terminal, &building.entries);
Self {
version: RAW_PAGE_INDEX_VERSION,
parent: clone_cache_value(parent),
page_index: building.page_index,
entries_start: building.entries_start,
entries: building.entries,
entries: building.entries.into(),
terminal: building.terminal,
digest,
}
}
fn validate(&self, parent: &str, page_index: u64, entries_start: u64) -> Result<(), RawEnumerationPageIndexError> {
let builder = RawEnumerationPageBuilder {
page_index: self.page_index,
entries_start: self.entries_start,
entries: clone_cache_value(&self.entries),
terminal: self.terminal,
};
if self.version != RAW_PAGE_INDEX_VERSION
|| self.parent != parent
|| self.page_index != page_index
|| self.entries_start != entries_start
|| self.entries.is_empty()
|| !entries_are_normalized(&self.entries)
|| self.digest != raw_page_digest(parent, &builder)
|| self.digest != raw_page_digest(parent, self.page_index, self.entries_start, self.terminal, &self.entries)
{
return Err(RawEnumerationPageIndexError::CorruptIndex);
}
@@ -587,16 +582,16 @@ fn entry_sets_match(left: &[String], right: &[String]) -> bool {
left.len() == right.len() && entries_contain_all(left, right)
}
fn raw_page_digest(parent: &str, building: &RawEnumerationPageBuilder) -> [u8; 32] {
fn raw_page_digest(parent: &str, page_index: u64, entries_start: u64, terminal: bool, entries: &[String]) -> [u8; 32] {
#[cfg(test)]
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(building.entries.len())));
RAW_PAGE_DIGEST_ENTRIES.with(|count| count.set(count.get().saturating_add(entries.len())));
let mut digest = Sha256::new();
update_digest(&mut digest, b"version", &RAW_PAGE_INDEX_VERSION.to_le_bytes());
update_digest(&mut digest, b"parent", parent.as_bytes());
update_digest(&mut digest, b"page_index", &building.page_index.to_le_bytes());
update_digest(&mut digest, b"entries_start", &building.entries_start.to_le_bytes());
update_digest(&mut digest, b"terminal", &[u8::from(building.terminal)]);
for entry in &building.entries {
update_digest(&mut digest, b"page_index", &page_index.to_le_bytes());
update_digest(&mut digest, b"entries_start", &entries_start.to_le_bytes());
update_digest(&mut digest, b"terminal", &[u8::from(terminal)]);
for entry in entries {
update_digest(&mut digest, b"entry", entry.as_bytes());
}
digest.finalize().into()
@@ -64,6 +64,15 @@ impl RawEnumerationPageWriter {
})
}
/// A parked native stream delivers the unread tail first. Incomplete
/// membership can immediately append that tail; terminal page verification
/// still requires observing the entire current directory.
pub(crate) fn continue_native_stream(&mut self) {
if !self.inner.complete && !self.inner.building.as_ref().is_some_and(|page| page.terminal) {
self.observations = self.revalidate_after_entries;
}
}
pub(crate) fn indexed_entry_count(&self) -> usize {
self.indexed.len()
}
+8 -3
View File
@@ -956,9 +956,14 @@ async fn scan_and_persist_local_bucket_with_store(
}
})?;
let mut cache = DataUsageCache::default();
let mut revisions = cache.load_with_revisions(set.clone(), &cache_name).await.map_err(|err| {
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
})?;
let Some(mut revisions) = Box::pin(cache.load_for_scan(set.clone(), &cache_name, &ctx, &budget))
.await
.map_err(|err| {
RemoteScannerServerError::worker(format!("remote namespace scanner cache load or revision lookup failed: {err}"))
})?
else {
return Ok(RemoteScannerFrameResult::Partial);
};
let expected_incarnation = cache
.info
.scan_identity
+115 -32
View File
@@ -34,8 +34,9 @@ use crate::runtime_config::{
use crate::scanner_budget::{ScannerCycleBudget, ScannerCycleBudgetConfig, ScannerCycleBudgetReason};
use crate::scanner_folder::{data_usage_update_dir_cycles, heal_object_select_prob};
use crate::scanner_io::{
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, dirty_usage_bucket_notified, dirty_usage_buckets_pending,
dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed, scanner_maintenance_generation,
ScannerCycleDeferReason, ScannerCycleResult, ScannerCycleStatus, ScannerServiceCohort, dirty_usage_bucket_notified,
dirty_usage_buckets_pending, dirty_usage_generation, scanner_dirty_usage_state, scanner_maintenance_changed,
scanner_maintenance_generation,
};
use crate::sleeper::{SCANNER_SLEEPER, set_scanner_default_speed};
use crate::{DataUsageInfo, ScannerActivityGuard, ScannerError, ScannerRuntimeGuard};
@@ -55,7 +56,7 @@ use rustfs_lock::{NamespaceLockGuard, error::LockError};
use rustfs_scanner_metrics::metrics::{
CurrentCycle, Metric, Metrics, RemoteScanProgress, ScanCyclePartialReason, ScanCycleWorkSnapshot,
ScannerUsagePublicationObservation, ScannerUsageSaveResult, ScannerWorkSource, emit_scan_cycle_complete,
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_source, emit_scan_cycle_superseded, global_metrics,
emit_scan_cycle_deferred, emit_scan_cycle_partial_with_stage, emit_scan_cycle_superseded, global_metrics,
};
use rustfs_scanner_metrics::startup::ScannerStartupToken;
use serde::{Deserialize, Serialize};
@@ -1212,6 +1213,8 @@ where
}
let mut transient_backoff = ScannerRetryBackoff::default();
// Only this supervisor, for this storage instance, owns resumable service.
let mut service_continuation = None;
let mut recovery_retry_count = 0_u32;
loop {
if ctx_clone.is_cancelled() {
@@ -1226,6 +1229,7 @@ where
startup_features,
startup_maintenance_generation,
startup,
&mut service_continuation,
)
.await;
if let Err(e) = &run_result {
@@ -1773,11 +1777,26 @@ where
}
}
/// Scheduling and mutation witnesses retained across a clean local deadline.
/// Only an immediately succeeding leadership claim can resume this cohort.
struct ScannerServiceContinuation {
cohort: Arc<StdMutex<ScannerServiceCohort>>,
fenced_epoch: u64,
fenced_revision: DataUsageCacheRevision,
}
impl ScannerServiceContinuation {
fn matches_claim(&self, previous_revision: &DataUsageCacheRevision, claimed_epoch: u64) -> bool {
&self.fenced_revision == previous_revision && self.fenced_epoch.checked_add(1) == Some(claimed_epoch)
}
}
struct ScannerCycleDeadlineState<'a> {
cycle_info: &'a mut CurrentCycle,
cycle_revision: &'a mut DataUsageCacheRevision,
leader_epoch: &'a mut u64,
cycle_budget: &'a ScannerCycleBudget,
service_cohort: Arc<StdMutex<ScannerServiceCohort>>,
allow_bootstrap_pending: bool,
}
@@ -1791,7 +1810,8 @@ async fn handle_scanner_cycle_deadline<Store>(
state: ScannerCycleDeadlineState<'_>,
worker_stopped: bool,
guard: &mut NamespaceLockGuard,
) where
) -> Option<ScannerServiceContinuation>
where
Store: ScannerObjectIO + ScannerConfigObjectDelete,
{
let fenced = fence_scanner_epoch_after_cycle_timeout(
@@ -1821,8 +1841,14 @@ async fn handle_scanner_cycle_deadline<Store>(
global_metrics().record_scanner_cycle_timeout(recovery_required, state.cycle_budget.progress_age());
// Stop renewing before releasing the lease. A new leader can then claim the
// higher persisted generation instead of inheriting the expired worker.
let may_resume = !recovery_required && !guard.is_lock_lost() && !ctx.is_cancelled();
guard.release();
global_metrics().set_cycle(None).await;
may_resume.then(|| ScannerServiceContinuation {
cohort: state.service_cohort,
fenced_epoch: *state.leader_epoch,
fenced_revision: state.cycle_revision.clone(),
})
}
async fn mark_scan_cycle_idle(cycle_info: &mut CurrentCycle, cycle_metrics_guard: &mut ScannerCycleMetricsGuard) {
@@ -2460,13 +2486,20 @@ where
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
required_cycle,
state = "cache_cycle_ahead",
partial_stage = "cache_cycle_ahead",
cycle = cycle_info.current,
leader_epoch,
required_cycle,
partial_cause = "cache_cycle_ahead",
"Scanner cycle is recovering to a newer durable cache generation"
);
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
let partial = emit_scan_cycle_partial_with_stage(
cycle_start.elapsed(),
ScanCyclePartialReason::Unknown,
None,
Some("cache_cycle_ahead"),
);
let persisted = persist_required_scanner_cycle_floor_for_epoch(
ctx,
storeapi.clone(),
@@ -2480,6 +2513,7 @@ where
},
)
.await;
partial.observe_state_write(persisted);
return if persisted {
cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial
@@ -2530,18 +2564,7 @@ where
}
usage_publication_result.restrict_outcome(usage_persist_outcome);
let partial_cause = if scan_cycle_result.has_partial_bucket_progress() && !scan_cycle_result.has_usage_update() {
"saved_bucket_progress"
} else if scan_cycle_result.has_usage_update()
&& matches!(
scan_cycle_result.status,
ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
)
{
"activity_unverified_observation"
} else {
"incomplete_coverage"
};
let (partial_cause, partial_stage) = partial_cycle_diagnostic(&scan_cycle_result);
#[cfg(feature = "e2e-test-hooks")]
let durable_observation = scan_cycle_result
.publication_expectation()
@@ -2620,25 +2643,35 @@ where
);
}
if cycle_budget.budget_elapsed() {
let budget_reason = cycle_budget.reason();
let budget_stage = match budget_reason {
Some(ScannerCycleBudgetReason::Runtime) => Some("runtime_budget"),
Some(ScannerCycleBudgetReason::Objects) => Some("object_budget"),
Some(ScannerCycleBudgetReason::Directories) => Some("directory_budget"),
Some(ScannerCycleBudgetReason::Entries) => Some("entry_budget"),
None => None,
};
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "budget_reached",
partial_stage = budget_stage,
cycle = cycle_info.current,
leader_epoch,
duration = ?now.elapsed(),
reason = ?cycle_budget.reason(),
reason = ?budget_reason,
max_duration = ?cycle_budget.max_duration(),
max_objects = ?cycle_budget.max_objects(),
max_directories = ?cycle_budget.max_directories(),
state = "budget_reached",
"Scanner cycle budget reached"
);
let budget_reason = cycle_budget.reason();
emit_scan_cycle_partial_with_source(
let partial = emit_scan_cycle_partial_with_stage(
cycle_start.elapsed(),
scan_cycle_partial_reason(budget_reason),
scan_cycle_partial_source(budget_reason),
budget_stage,
);
let persisted = finalize_partial_scan_cycle_for_epoch(
ctx,
@@ -2650,6 +2683,7 @@ where
publication_epoch,
)
.await;
partial.observe_state_write(persisted);
return if persisted {
cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial
@@ -2683,29 +2717,56 @@ where
return ScannerCycleOutcome::Failed;
}
ScannerCycleOutcome::Partial => {
if ctx.is_cancelled() {
let owner_cancelled = ctx.is_cancelled();
let partial_stage = if owner_cancelled { "owner_cancelled" } else { partial_stage };
if owner_cancelled {
debug!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
state = "incomplete_cancelled",
cycle = cycle_info.current,
leader_epoch,
partial_stage,
"Scanner cycle stopped before a complete usage snapshot was produced"
);
} else {
} else if partial_cause == "saved_bucket_progress"
&& matches!(partial_stage, "bucket_checkpoint_saved" | "bucket_walk_incomplete")
{
warn!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
cycle = cycle_info.current,
state = "incomplete",
cycle = cycle_info.current,
leader_epoch,
partial_cause,
"Scanner cycle ended without a complete usage snapshot"
partial_stage,
"Scanner saved bucket progress before completing coverage"
);
} else {
error!(
target: "rustfs::scanner",
event = EVENT_SCANNER_CYCLE_STATE,
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_RUNTIME,
state = "incomplete",
cycle = cycle_info.current,
leader_epoch,
partial_cause,
partial_stage,
persist_outcome = ?usage_persist_outcome,
"Scanner cycle could not certify complete usage"
);
}
emit_scan_cycle_partial_with_source(cycle_start.elapsed(), ScanCyclePartialReason::Unknown, None);
let partial = emit_scan_cycle_partial_with_stage(
cycle_start.elapsed(),
ScanCyclePartialReason::Unknown,
None,
Some(partial_stage),
);
let persisted = finalize_partial_scan_cycle_for_epoch(
ctx,
storeapi.clone(),
@@ -2716,6 +2777,7 @@ where
publication_epoch,
)
.await;
partial.observe_state_write(persisted);
return if persisted {
cycle_budget.mark_cycle_state_persisted();
ScannerCycleOutcome::Partial
@@ -2909,6 +2971,7 @@ where
maintenance_features,
maintenance_generation,
runtime_guard.startup_token(),
&mut None,
)
.await
}
@@ -2995,10 +3058,12 @@ async fn run_data_scanner_with_maintenance_state<S>(
mut maintenance_features: ScannerMaintenanceFeatures,
mut maintenance_generation_seen: Option<u64>,
startup: ScannerStartupToken,
continuation: &mut Option<ScannerServiceContinuation>,
) -> Result<(), ScannerError>
where
S: ScannerStorage,
{
let previous = continuation.take();
reset_scanner_cycle_schedule();
// Normal scanner entry is an attempt, not evidence of leadership, coverage, or publication.
global_metrics().startup().record_attempt(startup);
@@ -3108,7 +3173,6 @@ where
let mut deferred_backoff = ScannerRetryBackoff::default();
let mut scope_fallback_backoff = ScannerRetryBackoff::default();
let scope_fallback_slot = Arc::new(ScannerScopeFallbackSlot::default());
let service_cohort = Arc::new(StdMutex::new(crate::scanner_io::ScannerServiceCohort::default()));
let initial_runtime_config = resolve_scanner_runtime_config();
if clean_idle_topology_supported && maintenance_generation_seen.is_none() {
let Some((features, generation)) = detect_stable_scanner_maintenance_features(&ctx, &storeapi).await else {
@@ -3269,6 +3333,7 @@ where
finish_scanner_leader_iteration(guard.is_lock_lost(), "stopped", String::new()).await;
return Ok(());
}
let previous_revision = cycle_revision.clone();
let claim_ctx = ctx.child_token();
let leadership_claimed = await_scanner_cycle_with_lock_fence(
&claim_ctx,
@@ -3318,6 +3383,10 @@ where
finish_scanner_leader_iteration(false, "epoch_claim_failed", "leadership epoch claim failed".to_string()).await;
return Ok(());
}
let service_cohort = previous
.filter(|previous| previous.matches_claim(&previous_revision, leader_epoch))
.map(|previous| previous.cohort)
.unwrap_or_default();
if rebuild_from_quarantine {
record_scanner_cycle_state_rebuilt_from_quarantine(cycle_info.next, leader_epoch);
}
@@ -3402,7 +3471,7 @@ where
return Ok(());
}
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
handle_scanner_cycle_deadline(
*continuation = handle_scanner_cycle_deadline(
&ctx,
storeapi.clone(),
ScannerCycleDeadlineState {
@@ -3410,6 +3479,7 @@ where
cycle_revision: &mut cycle_revision,
leader_epoch: &mut leader_epoch,
cycle_budget: &cycle_budget,
service_cohort,
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
},
worker_stopped,
@@ -3747,7 +3817,7 @@ where
return Ok(());
}
ScannerCycleWaitOutcome::Deadline { worker_stopped } => {
handle_scanner_cycle_deadline(
*continuation = handle_scanner_cycle_deadline(
&ctx,
storeapi.clone(),
ScannerCycleDeadlineState {
@@ -3755,6 +3825,7 @@ where
cycle_revision: &mut cycle_revision,
leader_epoch: &mut leader_epoch,
cycle_budget: &cycle_budget,
service_cohort,
allow_bootstrap_pending: allow_usage_floor_bootstrap_pending,
},
worker_stopped,
@@ -4334,6 +4405,18 @@ fn finalize_scanner_cycle_result(
)
}
fn partial_cycle_diagnostic(scan: &crate::scanner_io::ScannerCycleResult) -> (&'static str, &'static str) {
if scan.has_partial_bucket_progress() && !scan.has_usage_update() {
("saved_bucket_progress", scan.partial_stage.unwrap_or("bucket_checkpoint_saved"))
} else if scan.has_usage_update()
&& scan.status == ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable)
{
("activity_unverified_observation", "activity_baseline_unverified")
} else {
("incomplete_coverage", scan.partial_stage.unwrap_or("root_coverage_incomplete"))
}
}
fn scanner_cycle_completion_outcome_for_result(
scan_cycle_result: &crate::scanner_io::ScannerCycleResult,
usage_persist_outcome: DataUsagePersistOutcome,
+39 -2
View File
@@ -52,9 +52,11 @@ const TEST_DEFAULT_SCANNER_CYCLE_SECS: u64 = 24 * 60 * 60;
mod cycle_budget_publication;
pub(super) mod cycle_persist_failure;
mod gate_self_heal;
mod paged_checkpoint;
mod quota_reset_preservation;
mod recovery_control;
mod restore_deadline;
mod scoped_ack_publication;
mod wakeup_metrics;
@@ -773,7 +775,8 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
);
budget.mark_cycle_state_persisted();
handle_scanner_cycle_deadline(
let service_cohort = Arc::new(StdMutex::new(ScannerServiceCohort::default()));
let continuation = handle_scanner_cycle_deadline(
&ctx,
store.clone(),
ScannerCycleDeadlineState {
@@ -781,12 +784,24 @@ async fn cycle_budget_deadline_handler_fences_and_releases_guard() {
cycle_revision: &mut cycle_revision,
leader_epoch: &mut leader_epoch,
cycle_budget: &budget,
service_cohort: service_cohort.clone(),
allow_bootstrap_pending: false,
},
true,
&mut guard,
)
.await;
.await
.expect("a fully drained and fenced deadline retains the live cohort across supervisor iterations");
assert!(Arc::ptr_eq(&continuation.cohort, &service_cohort));
assert!(continuation.matches_claim(&cycle_revision, 3));
assert!(
!continuation.matches_claim(&cycle_revision, 4),
"an intervening owner ends the continuation"
);
assert!(
!continuation.matches_claim(&DataUsageCacheRevision::Missing, 3),
"a replaced cursor ends the continuation"
);
assert!(guard.is_released());
assert_eq!(leader_epoch, 2, "deadline handler should claim the next epoch");
@@ -989,6 +1004,7 @@ struct MemoryConfigStore {
insert_after_gets: Mutex<HashMap<String, Vec<u8>>>,
read_errors: Mutex<HashMap<String, EcstoreError>>,
delayed_gets: Mutex<HashMap<String, Duration>>,
checkpoint_page_delay_ms: AtomicUsize,
non_regular_objects: Mutex<HashSet<String>>,
fail_put_number: Mutex<HashMap<String, usize>>,
/// Fail every put whose number is at least the stored value.
@@ -1000,6 +1016,7 @@ struct MemoryConfigStore {
cancel_after_successful_puts: Mutex<HashMap<String, (usize, CancellationToken)>>,
replace_after_successful_puts: Mutex<HashMap<String, (usize, Vec<u8>)>>,
error_after_commit_deletes: Mutex<HashSet<String>>,
fail_prefix_deletes: AtomicBool,
cancel_after_deletes: Mutex<HashMap<String, CancellationToken>>,
pause_next_publication_admission: Mutex<Option<(Arc<tokio::sync::Notify>, Arc<tokio::sync::Notify>)>>,
put_counts: Mutex<HashMap<String, usize>>,
@@ -1046,6 +1063,17 @@ impl crate::storage_api::scanner_io::ObjectIO for MemoryConfigStore {
.await
.push(crate::storage_api::ecstore_diagnostic::current_read_version_entry_source());
let key = memory_config_key(bucket, object);
let _serial_page_read = if object.contains(".pages/") {
restore_deadline::admit_page(object).await
} else {
None
};
if object.contains(".pages/") {
let delay = self.checkpoint_page_delay_ms.load(Ordering::Relaxed);
if delay != 0 {
tokio::time::sleep(Duration::from_millis(delay as u64)).await;
}
}
if let Some(error) = self.read_errors.lock().await.get(&key).cloned() {
return Err(error);
}
@@ -5003,6 +5031,15 @@ impl crate::ScannerConfigObjectDelete for MemoryConfigStore {
async fn delete_config_object(&self, bucket: &str, object: &str, opts: ObjectOptions) -> EcstoreResult<ObjectInfo> {
let key = memory_config_key(bucket, object);
let mut objects = self.objects.lock().await;
if opts.delete_prefix && !opts.delete_prefix_object {
if self.fail_prefix_deletes.load(Ordering::Relaxed) {
return Err(EcstoreError::other("injected prefix deletion failure"));
}
assert!(opts.http_preconditions.is_none(), "fixture prefix deletion cannot condition one object");
objects.retain(|object, _| !object.starts_with(&key));
self.revisions.lock().await.retain(|object, _| !object.starts_with(&key));
return Ok(ObjectInfo::default());
}
if !objects.contains_key(&key) {
return Err(EcstoreError::FileNotFound);
}
@@ -253,6 +253,9 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
.await;
assert!(matches!(outcome, ScannerCycleOutcome::Partial));
assert_eq!(budget.reason(), Some(ScannerCycleBudgetReason::Objects));
let report = global_metrics().report().await;
assert_eq!(report.last_cycle_partial_reason, "objects");
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("object_budget"));
assert!(budget.progress().0 > 0);
assert!(budget.cycle_state_persisted());
assert_eq!(cycle.next, 353);
@@ -410,6 +413,16 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
super::cycle_persist_failure::fail_saves(1, false, SCANNER_PERSIST_CAS_RETRIES + 1)
}
});
use tracing::instrument::WithSubscriber as _;
let error_log = tempfile::NamedTempFile::new().expect("error log");
let subscriber = tracing_subscriber::fmt()
.json()
.without_time()
.with_ansi(false)
.with_max_level(tracing::Level::ERROR)
.with_writer(error_log.reopen().expect("error log writer"))
.finish();
let _interest_guard = tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default());
let tail_outcome = run_data_scanner_cycle_with_budget(
&ctx,
&store,
@@ -423,7 +436,23 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
scope_fallback: None,
},
)
.with_subscriber(subscriber)
.await;
let error_events = std::fs::read_to_string(error_log.path()).expect("read error log");
let event = error_events
.lines()
.map(|line| serde_json::from_str::<serde_json::Value>(line).expect("JSON error event"))
.find(|event| event["fields"]["state"] == "incomplete" && event["fields"]["cycle"] == 353)
.expect("unbudgeted incomplete observation should be visible at error level");
assert_eq!(event["fields"]["leader_epoch"], 1);
assert_eq!(event["fields"]["partial_stage"], "mixed_plan_recertification_pending");
let report = global_metrics().report().await;
assert_eq!(report.last_cycle_partial_reason, "unknown");
if fail_cycle_save == Some(false) {
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("cycle_state_not_durable"));
} else {
assert_eq!(report.last_cycle_partial_stage.as_deref(), Some("mixed_plan_recertification_pending"));
}
assert!(!tail_budget.budget_elapsed(), "this is not the budget-exhaustion fast path");
checkpoint.load(set, &cache_name).await.expect("reload the saved mixed tail");
assert!(!checkpoint.info.snapshot_complete);
@@ -527,6 +556,22 @@ async fn assert_coordinator_partial_progress(resume_mixed_plan: bool, fail_cycle
crate::scanner_io::clear_dirty_usage_buckets_for_tests();
}
#[test]
fn partial_diagnostics_keep_the_activity_fence_distinct_from_checkpoint_progress() {
let observation =
ScannerCycleResult::new(ScannerCycleStatus::Deferred(ScannerCycleDeferReason::ActivityBaselineUnavailable), None)
.with_usage_update_emitted(true);
assert_eq!(
partial_cycle_diagnostic(&observation),
("activity_unverified_observation", "activity_baseline_unverified")
);
let checkpoint = ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_partial_bucket_progress(true);
assert_eq!(
partial_cycle_diagnostic(&checkpoint),
("saved_bucket_progress", "bucket_checkpoint_saved")
);
}
#[test]
fn partial_progress_does_not_mask_missing_updates_or_persistence_failure() {
for (persisted, expected) in [
@@ -0,0 +1,473 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use crate::BUCKET_META_PREFIX;
mod save_review;
const NAME: &str = "bucket/.usage-cache.bin";
fn empty_bucket_checkpoint() -> DataUsageCache {
let mut cache = DataUsageCache::default();
cache.info.name = "bucket".into();
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
cache.info.scan_identity = Some(crate::data_usage_define::DataUsageScanIdentity {
version: crate::data_usage_define::DATA_USAGE_SCAN_CHECKPOINT_VERSION,
bucket_incarnation: uuid::Uuid::from_u128(42),
set_layout: DataUsageScanPlanDigest([7; 32]),
publication_epoch: 0,
tier_registry_generation: 1,
scan_mode: HealScanMode::Normal,
});
cache.replace("bucket", "", DataUsageEntry::default());
cache
}
fn seeded() -> DataUsageCache {
let mut cache = empty_bucket_checkpoint();
for index in 0..4096 {
cache.replace(
&format!("bucket/prefix-{index:05}"),
"bucket",
DataUsageEntry {
size: index + 1,
objects: 1,
versions: 1,
..Default::default()
},
);
}
cache
}
async fn layout(store: &MemoryConfigStore) -> HashMap<String, (usize, usize)> {
let puts = store.put_counts.lock().await.clone();
store
.objects
.lock()
.await
.iter()
.map(|(k, v)| (k.clone(), (puts[k], v.len())))
.collect()
}
fn page_bytes_after_save(before: &HashMap<String, (usize, usize)>, after: &HashMap<String, (usize, usize)>) -> (usize, usize) {
let mut existing = 0;
let mut uploaded = 0;
for (path, &(writes, size)) in after.iter().filter(|(path, _)| path.contains(".pages/")) {
if let Some(&(previous_writes, _)) = before.get(path) {
assert_eq!(writes, previous_writes, "a committed immutable page was resubmitted");
existing += size;
} else {
uploaded += size;
}
}
(existing, uploaded)
}
#[tokio::test]
async fn compacted_checkpoint_advances_one_child_without_rewriting_300k_names() {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = empty_bucket_checkpoint();
let plan = DataUsageScanPlanDigest([1; 32]);
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: plan,
requested_plan: plan,
});
cache.info.scan_directory_source = Some(format!("{}|disk-a", uuid::Uuid::from_u128(1)));
let [first, second] = crate::scanner_folder::DirectoryCheckpoint::wide_storage_fixture(&cache.info).await;
let mut before: Option<HashMap<String, (usize, usize)>> = None;
let mut prior_progress = None;
for (objects, receipt) in [(121_034, first), (121_035, second)] {
cache.info.scan_directory_work = vec![receipt];
cache.replace(
"bucket",
"",
DataUsageEntry {
compacted: true,
objects,
size: objects * 1024,
..Default::default()
},
);
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("current revisions");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("save compacted receipt");
let after = layout(&store).await;
if let Some(before) = before.as_ref() {
let (old_bytes, new_bytes) = page_bytes_after_save(before, &after);
eprintln!("300k-name checkpoint: {new_bytes} bytes in new pages, {old_bytes} bytes in existing pages");
assert!(old_bytes > 3_000_000, "fixture must contain the full physical listing");
assert!(
new_bytes > 0 && new_bytes < old_bytes / 10,
"one more completed child wrote {new_bytes} bytes alongside {old_bytes} reused bytes"
);
}
before = Some(after);
let mut restored = DataUsageCache::default();
restored
.load(store.clone(), NAME)
.await
.expect("cold read of compacted receipt");
assert!(restored.info.scan_directory_work[0].is_bound_to(&restored.info));
let progress = restored.durable_scan_progress();
if let Some(prior) = &prior_progress {
assert!(progress.advanced_from(prior), "the new child must earn another durable work turn");
}
prior_progress = Some(progress);
let total = restored.checked_flatten("bucket").expect("connected root after restore");
assert_eq!((total.objects, total.size), (objects, objects * 1024));
cache = restored;
}
}
#[tokio::test]
async fn dormant_linked_tree_pages_survive_source_switch_and_cold_reencoding() {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = seeded();
let plan = DataUsageScanPlanDigest([1; 32]);
cache.info.scan_progress = Some(crate::DataUsageScanProgress {
started_plan: plan,
requested_plan: plan,
});
let endpoints = std::collections::HashSet::from(["disk-a".to_owned(), "disk-b".to_owned()]);
let a = format!("{}|disk-a", uuid::Uuid::from_u128(1));
let b = format!("{}|disk-b", uuid::Uuid::from_u128(2));
cache.info.scan_directory_source = Some(a.clone());
cache.activate_directory_source(b.clone(), &endpoints);
cache.replace("bucket", "", DataUsageEntry::default());
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("before first checkpoint");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("persist dormant linked tree");
let previous = layout(&store).await;
let mut cold = DataUsageCache::default();
cold.load(store.clone(), NAME)
.await
.expect("restore checkpoint in a new process");
cold.activate_directory_source(a, &endpoints);
assert_eq!(cold.checked_flatten("bucket").expect("first source entries").objects, 4096);
cold.activate_directory_source(b, &endpoints);
assert_eq!(cold.checked_flatten("bucket").expect("second source root").objects, 0);
cold.replace(
"bucket/new-key",
"bucket",
DataUsageEntry {
objects: 1,
size: 99,
..Default::default()
},
);
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("before second checkpoint");
cold.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("save one new child");
let after = layout(&store).await;
let (prior_bytes, new_bytes) = page_bytes_after_save(&previous, &after);
assert!(prior_bytes > 128 * 1024);
assert!(
new_bytes > 0 && new_bytes < prior_bytes / 8,
"dormant tree rewritten: {new_bytes} new bytes for {prior_bytes} prior bytes"
);
}
#[tokio::test]
async fn paged_checkpoint_writes_only_changed_pages_and_commits_root_last() {
let store = Arc::new(MemoryConfigStore::default());
let cache = seeded();
let first = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("empty revisions");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
.await
.expect("initial checkpoint");
let before = layout(&store).await;
let encoded = cache.marshal_msg().expect("encode baseline").len();
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
assert!(before[&main].1 < 8192, "only a compact manifest belongs in the root");
assert!(before.keys().any(|key| key.contains(".pages/")), "large cache must be paged");
let mut restarted = DataUsageCache::default();
let revisions = restarted
.load_with_revisions(store.clone(), NAME)
.await
.expect("restart from durable pages");
let prior = restarted.checked_flatten("bucket").expect("all old entries");
restarted.replace(
"bucket/prefix-tail",
"bucket",
DataUsageEntry {
objects: 1,
versions: 1,
size: 4711,
..Default::default()
},
);
store.fail_puts_from.lock().await.insert(main.clone(), before[&main].0 + 1);
restarted
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect_err("commit interrupted after writing pages");
let interrupted = layout(&store).await;
let mut observer = DataUsageCache::default();
observer.load(store.clone(), NAME).await.expect("last root still readable");
assert_eq!(observer.checked_flatten("bucket").expect("previous coverage").size, prior.size);
store.fail_puts_from.lock().await.remove(&main);
restarted
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("retry interrupted commit");
observer.load(store.clone(), NAME).await.expect("load committed result");
let actual = observer.checked_flatten("bucket").expect("new root");
assert_eq!((actual.objects, actual.size), (4097, prior.size + 4711));
let after = layout(&store).await;
for (key, (writes, _)) in &interrupted {
if key.contains(".bin.pages/") {
assert_eq!(*writes, after[key].0, "retry restaged a previously successful immutable page");
}
}
let new_bytes: usize = after
.iter()
.filter(|(key, _)| !before.contains_key(*key))
.map(|(_, (_, size))| *size)
.sum();
assert!(
new_bytes < encoded / 3,
"one append staged {new_bytes} new bytes for a {encoded}-byte cache"
);
for (key, (writes, _)) in &before {
if key.contains(".pages/") {
assert_eq!(*writes, after[key].0, "a committed page was resubmitted");
}
}
restarted.info.next_cycle += 1;
let stale = restarted.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0).await;
assert!(
matches!(stale, Err(EcstoreError::PreconditionFailed)),
"stale writer cannot replace the new root"
);
}
#[tokio::test]
async fn paged_checkpoint_cold_restore_spans_fixed_duration_and_repairs_corrupt_primary() {
let store = Arc::new(MemoryConfigStore::default());
let cache = seeded();
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("read revisions");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("write both copies");
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
let mut loaded = DataUsageCache::default();
let mut completed_at = None;
for round in 1..=128 {
if let Ok(result) = tokio::time::timeout(Duration::from_millis(26), loaded.load(store.clone(), NAME)).await {
result.expect("resume exact committed manifest");
completed_at = Some(round);
break;
}
}
assert!(
completed_at.is_some_and(|round| round > 1),
"cold recovery must advance across fixed deadlines"
);
store.checkpoint_page_delay_ms.store(0, Ordering::Relaxed);
let prior = loaded.checked_flatten("bucket").expect("cold cache totals");
assert_eq!((prior.objects, prior.size), (4096, 4096 * 4097 / 2));
{
let mut objects = store.objects.lock().await;
let (_, block) = objects
.iter_mut()
.find(|(key, block)| key.contains(".bin.pages/") && block.len() > 8192)
.expect("primary data page");
block[0] ^= 0xff;
}
let revisions = loaded
.load_with_revisions(store.clone(), NAME)
.await
.expect("read unaffected backup");
assert_eq!(
revisions,
DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("re-enter admission on same heads")
);
loaded
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("replace damaged primary generation");
store.objects.lock().await.retain(|key, _| !key.contains(".bkp"));
let mut primary = DataUsageCache::default();
primary.load(store, NAME).await.expect("repaired independent primary");
assert_eq!(primary.checked_flatten("bucket").expect("primary totals").size, prior.size);
}
#[tokio::test(start_paused = true)]
async fn paged_checkpoint_bucket_admission_uses_the_scan_budget_during_cold_load() {
let store = Arc::new(MemoryConfigStore::default());
let cache = seeded();
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.unwrap();
store.checkpoint_page_delay_ms.store(16, Ordering::Relaxed);
let owner = CancellationToken::new();
let mut loaded = DataUsageCache::default();
for round in 0..128 {
let budget = ScannerCycleBudget::new_with_progress_tracking(
&owner,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_millis(45)),
..Default::default()
},
);
// Exercise the same admission used by local and remote workers. The
// worker may drain after the deadline; only the scan token interrupts
// recovery here, as it does in the daemon.
let revisions = loaded
.load_for_scan(store.clone(), NAME, &budget.token(), &budget)
.await
.unwrap();
if revisions.is_some() {
assert!(round > 0, "the first slice cannot finish this physical restore");
assert!(!budget.token().is_cancelled(), "return work only to an active scan");
let totals = loaded.checked_flatten("bucket").expect("restored full bucket");
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
return;
}
}
panic!("bucket admission repeatedly consumed the entire budget restoring the same root");
}
#[tokio::test]
async fn paged_checkpoint_retirement_preserves_independent_backup_and_legacy_projection() {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = seeded();
cache.info.snapshot_complete = true;
let first = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("initial revisions");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
.await
.expect("persist first generation");
let original = layout(&store).await;
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
let backup = format!("{main}.bkp");
let legacy =
DataUsageCache::unmarshal(&store.objects.lock().await[&main]).expect("old reader can read compact complete root");
let totals = legacy
.checked_flatten_complete_scope("bucket")
.expect("legacy compact totals");
assert_eq!((totals.objects, totals.size), (4096, 4096 * 4097 / 2));
let initial = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("first generation revisions");
cache.info.next_cycle += 1;
store
.fail_puts_from
.lock()
.await
.insert(backup.clone(), original[&backup].0 + 1);
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &initial, 0)
.await
.expect("commit primary despite failed backup");
let updated = layout(&store).await;
for key in original.keys().filter(|key| key.contains(".pages/")) {
assert_eq!(
updated.contains_key(key),
key.contains(".bkp.pages/"),
"unexpected reclamation scope: {key}"
);
}
let mut latest = DataUsageCache::default();
latest.load(store.clone(), NAME).await.expect("new main remains readable");
assert_eq!(latest.info.next_cycle, cache.info.next_cycle);
store.objects.lock().await.remove(&main);
latest
.load(store, NAME)
.await
.expect("previous independent backup remains readable");
assert_eq!(latest.info.next_cycle + 1, cache.info.next_cycle);
}
#[tokio::test(start_paused = true)]
async fn paged_checkpoint_scan_handoff_checks_the_latest_root_before_using_its_totals() {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = seeded();
cache.cache.clear();
let replace = |cache: &mut DataUsageCache, size| {
cache.replace(
"bucket",
"",
DataUsageEntry {
objects: 1,
size,
..Default::default()
},
);
};
replace(&mut cache, 17);
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.unwrap();
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
store.delayed_gets.lock().await.insert(main, Duration::from_secs(6));
let parent = CancellationToken::new();
let budget = || {
ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(Duration::from_secs(10)),
..Default::default()
},
)
};
let first = budget();
let mut loaded = DataUsageCache::default();
assert!(
loaded
.load_for_scan(store.clone(), NAME, &first.token(), &first)
.await
.unwrap()
.is_none()
);
assert!(!first.token().is_cancelled(), "the slow load moved its result to the next admission");
let revisions = DataUsageCache::read_revisions(store.clone(), NAME).await.unwrap();
replace(&mut cache, 99);
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.unwrap();
let next = budget();
assert!(
loaded
.load_for_scan(store, NAME, &next.token(), &next)
.await
.unwrap()
.is_some()
);
assert_eq!(loaded.checked_flatten("bucket").expect("new root").size, 99);
}
@@ -0,0 +1,192 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
#[tokio::test]
async fn indefinite_partial_checkpoints_keep_unreachable_bytes_bounded() {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = seeded();
let mut initial_bytes = 0;
for round in 0..10 {
for (name, entry) in &mut cache.cache {
if name != "bucket" {
entry.size += 17;
}
}
// A single long-running cycle can checkpoint many times without changing cycle number.
assert!(!cache.info.snapshot_complete);
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("current heads");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("commit partial");
let current: usize = layout(&store)
.await
.iter()
.filter(|(path, _)| path.contains(".pages/"))
.map(|(_, (_, size))| size)
.sum();
if round == 0 {
initial_bytes = current;
assert!(initial_bytes > 100_000);
}
assert!(
current <= initial_bytes * 4,
"after {} partials, {current} page bytes for {initial_bytes} initially",
round + 1
);
}
let loaded = &mut DataUsageCache::default();
loaded.load(store, NAME).await.expect("restore bounded checkpoint");
assert_eq!(
loaded.checked_flatten("bucket").expect("valid restored tree").size,
cache.checked_flatten("bucket").expect("fixture tree").size
);
}
enum BackupDamage {
Missing,
Corrupt,
UnreadableMetadata,
}
async fn survives_lost_primary_after_backup_page_damage(damage: BackupDamage) {
let store = Arc::new(MemoryConfigStore::default());
let cache = seeded();
let first = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("empty heads");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &first, 0)
.await
.expect("initial checkpoint");
let modify_tail = |cache: &mut DataUsageCache| {
cache.cache.get_mut("bucket/prefix-04095").expect("last leaf").size += 99;
};
let control = Arc::new(MemoryConfigStore::default());
let mut target = cache.clone();
modify_tail(&mut target);
let empty = DataUsageCache::read_revisions(control.clone(), NAME)
.await
.expect("empty control store");
target
.save_with_revisions_for_epoch(control.clone(), NAME, &empty, 0)
.await
.expect("encode target in an independent fresh bundle");
{
let control_objects = control.objects.lock().await;
let mut objects = store.objects.lock().await;
let victim = objects
.iter()
.find_map(|(path, data)| {
// A fresh target bundle only contains currently reachable objects.
(path.contains(".bkp.pages/") && data.len() > 8192 && control_objects.get(path) == Some(data))
.then(|| path.clone())
})
.expect("a large unchanged leaf is reachable in both manifests");
match damage {
BackupDamage::Corrupt => objects.get_mut(&victim).expect("existing backup leaf")[0] ^= 1,
BackupDamage::Missing => {
objects.remove(&victim);
}
BackupDamage::UnreadableMetadata => {
drop(objects);
store.read_errors.lock().await.insert(victim, EcstoreError::FileCorrupt);
}
}
}
let mut from_main = DataUsageCache::default();
let revisions = from_main
.load_with_revisions(store.clone(), NAME)
.await
.expect("primary remains healthy");
modify_tail(&mut from_main);
from_main
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("incremental commit while primary is healthy");
let main = memory_config_key(RUSTFS_META_BUCKET, &format!("{BUCKET_META_PREFIX}/{NAME}"));
store.objects.lock().await.remove(&main);
let mut recovered = DataUsageCache::default();
recovered.load(store, NAME).await.expect("standalone cold backup recovery");
let actual = recovered
.checked_flatten("bucket")
.expect("a connected backup after losing its primary");
assert_eq!((actual.objects, actual.size), (4096, 4096 * 4097 / 2 + 99));
}
#[tokio::test]
async fn incremental_save_repairs_missing_backup_leaf() {
survives_lost_primary_after_backup_page_damage(BackupDamage::Missing).await;
}
#[tokio::test]
async fn incremental_save_repairs_corrupt_backup_leaf() {
survives_lost_primary_after_backup_page_damage(BackupDamage::Corrupt).await;
}
#[tokio::test]
async fn incremental_save_rebuilds_backup_with_unreadable_page_metadata() {
survives_lost_primary_after_backup_page_damage(BackupDamage::UnreadableMetadata).await;
}
#[tokio::test]
async fn unreclaimed_partial_generation_bounds_staging_and_can_retry() {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = seeded();
cache.info.snapshot_complete = true;
let empty = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("empty store");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &empty, 0)
.await
.expect("committed predecessor");
store.fail_prefix_deletes.store(true, Ordering::Relaxed);
cache.info.snapshot_complete = false;
cache.info.next_cycle += 1;
for _ in 0..8 {
for (name, entry) in &mut cache.cache {
if name != "bucket" {
entry.size += 17;
}
}
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("current heads");
let before = layout(&store).await;
if let Err(error) = cache.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0).await {
assert!(error.to_string().contains("reclaim"), "unexpected save failure: {error}");
assert_eq!(before, layout(&store).await, "a blocked retirement must not stage further objects");
assert!(
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.is_err()
);
assert_eq!(
before,
layout(&store).await,
"retry while cleanup is unavailable must leave both generations readable"
);
let mut previous = DataUsageCache::default();
previous.load(store.clone(), NAME).await.expect("last committed checkpoint");
assert!(previous.checked_flatten("bucket").is_some());
store.fail_prefix_deletes.store(false, Ordering::Relaxed);
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("save after previous retirement is reclaimed");
previous.load(store.clone(), NAME).await.expect("read latest checkpoint");
assert_eq!(
cache.checked_flatten("bucket").expect("fixture").size,
previous.checked_flatten("bucket").expect("recovered root").size
);
return;
}
}
panic!("persistent retirement failure allowed unbounded staging");
}
@@ -160,6 +160,7 @@ async fn issue_8297_contended_follower_clears_only_transient_cycle_recovery() {
ScannerMaintenanceFeatures::default(),
None,
runtime.startup_token(),
&mut None,
),
)
.await
@@ -0,0 +1,275 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
use crate::data_usage_define::{DATA_USAGE_SCAN_CHECKPOINT_VERSION, DataUsageCacheRevisions, DataUsageScanIdentity};
use tokio::sync::{Notify, OwnedSemaphorePermit, Semaphore};
const NAME: &str = "bucket/.usage-cache.bin";
const COUNT: usize = 1536;
struct ReadWindow {
pages: usize,
device: Arc<Semaphore>,
completed: std::sync::Mutex<Vec<String>>,
deadline: Notify,
}
tokio::task_local! {
static READ_WINDOW: Arc<ReadWindow>;
}
// Model a serial device with one page of I/O time per admission. Completing
// a second read would exceed the deadline, so the caller drops that waiter.
pub(super) async fn admit_page(object: &str) -> Option<OwnedSemaphorePermit> {
let window = READ_WINDOW.try_with(Arc::clone).ok()?;
let permit = window.device.clone().acquire_owned().await.expect("device stays open");
{
let mut completed = window.completed.lock().expect("device history");
if completed.len() < window.pages {
completed.push(object.into());
return Some(permit);
}
}
drop(permit);
window.deadline.notify_one();
std::future::pending().await
}
async fn window(store: &Arc<MemoryConfigStore>, loaded: &mut DataUsageCache) -> (Option<DataUsageCacheRevisions>, Vec<String>) {
let window = Arc::new(ReadWindow {
pages: 1,
device: Arc::new(Semaphore::new(1)),
completed: Default::default(),
deadline: Notify::new(),
});
let result = READ_WINDOW
.scope(window.clone(), async {
tokio::select! {
biased;
value = loaded.load_with_revisions(store.clone(), NAME) => Some(value.expect("checkpoint load")),
() = window.deadline.notified() => None,
}
})
.await;
let pages = std::mem::take(&mut *window.completed.lock().expect("device history"));
(result, pages)
}
async fn fixture() -> (Arc<MemoryConfigStore>, DataUsageCache) {
let store = Arc::new(MemoryConfigStore::default());
let mut cache = DataUsageCache::default();
cache.info.name = "bucket".into();
cache.info.source = Some(DataUsageCacheSource::new(0, 0));
cache.info.scan_identity = Some(DataUsageScanIdentity {
version: DATA_USAGE_SCAN_CHECKPOINT_VERSION,
bucket_incarnation: uuid::Uuid::from_u128(9),
set_layout: DataUsageScanPlanDigest([7; 32]),
publication_epoch: 0,
tier_registry_generation: 1,
scan_mode: HealScanMode::Normal,
});
cache.replace("bucket", "", DataUsageEntry::default());
for key in 0..COUNT {
cache.replace(
&format!("bucket/key-{key:08}"),
"bucket",
DataUsageEntry {
objects: 1,
size: key + 1,
..Default::default()
},
);
}
let revisions = DataUsageCache::read_revisions(store.clone(), NAME)
.await
.expect("initial revisions");
cache
.save_with_revisions_for_epoch(store.clone(), NAME, &revisions, 0)
.await
.expect("seed both copies");
(store, cache)
}
async fn corrupt_primary(store: &MemoryConfigStore) {
let mut objects = store.objects.lock().await;
let mut pages = 0;
for (_, data) in objects.iter_mut().filter(|(path, _)| path.contains(".bin.pages/")) {
data[0] ^= 1;
pages += 1;
}
assert!(pages > 4, "fixture must span more than one physical read batch");
}
async fn finish(store: &Arc<MemoryConfigStore>, backup: bool) {
finish_across_windows(store, backup, None).await;
}
async fn finish_across_windows(store: &Arc<MemoryConfigStore>, backup: bool, competing: Option<&Arc<MemoryConfigStore>>) {
let mut loaded = DataUsageCache::default();
let mut main_reads = 0;
let mut backup_reads = 0;
for _ in 0..128 {
let (result, pages) = window(store, &mut loaded).await;
for page in pages {
if page.contains(".bkp.pages/") {
backup_reads += 1;
} else {
main_reads += 1;
}
}
if result.is_some() {
let total = loaded.checked_flatten("bucket").expect("restored bucket");
assert_eq!(total.objects, COUNT);
assert_eq!(total.size, COUNT * (COUNT + 1) / 2);
if backup {
assert_eq!(main_reads, 1, "read the corrupt primary once for this revision");
assert!(backup_reads > 4, "the independent backup must be restored across admissions");
} else {
assert!(main_reads > 4);
assert_eq!(backup_reads, 0);
}
return;
}
if let Some(other) = competing {
tokio::time::advance(Duration::from_secs(2 * 60 * 60)).await;
let _ = window(other, &mut DataUsageCache::default()).await;
}
}
panic!("checkpoint did not converge: {main_reads} primary and {backup_reads} backup page reads");
}
#[tokio::test]
async fn serial_backend_restores_one_verified_page_per_deadline() {
let (store, _) = fixture().await;
finish(&store, false).await;
}
#[tokio::test]
async fn concurrent_cancellations_keep_the_longer_verified_prefix() {
let (store, _) = fixture().await;
let longer = Arc::new(ReadWindow {
pages: 3,
device: Arc::new(Semaphore::new(1)),
completed: Default::default(),
deadline: Notify::new(),
});
let mut first = DataUsageCache::default();
let mut read = Box::pin(READ_WINDOW.scope(longer.clone(), first.load(store.clone(), NAME)));
tokio::select! {
result = &mut read => panic!("checkpoint unexpectedly finished within three reads: {result:?}"),
() = longer.deadline.notified() => {}
}
let mut concurrent = DataUsageCache::default();
assert!(window(&store, &mut concurrent).await.0.is_none());
drop(read);
let (_, next) = window(&store, &mut concurrent).await;
let previous = longer.completed.lock().expect("first reader history");
assert_eq!(previous.len(), 3);
assert!(
!previous.contains(&next[0]),
"a shorter concurrent cancellation displaced a verified prefix"
);
}
#[tokio::test]
async fn corrupt_primary_yields_future_admissions_to_the_healthy_backup() {
let (store, _) = fixture().await;
corrupt_primary(&store).await;
finish(&store, true).await;
}
#[tokio::test]
async fn new_primary_revision_supersedes_an_incomplete_backup_restore() {
let (store, _) = fixture().await;
corrupt_primary(&store).await;
let mut loaded = DataUsageCache::default();
assert!(window(&store, &mut loaded).await.0.is_none());
let (result, pages) = window(&store, &mut loaded).await;
assert!(result.is_none());
assert!(pages[0].contains(".bkp.pages/"), "second admission should advance backup");
let mut new_primary = DataUsageCache::default();
new_primary.info.name = "bucket".into();
new_primary.replace(
"bucket",
"",
DataUsageEntry {
objects: 1,
size: 19,
..Default::default()
},
);
let root = memory_config_key(RUSTFS_META_BUCKET, &format!("{}/{NAME}", crate::BUCKET_META_PREFIX));
let bytes = rmp_serde::to_vec_named(&new_primary).expect("replacement main root");
store.objects.lock().await.insert(root.clone(), bytes);
*store.revisions.lock().await.get_mut(&root).expect("primary revision") += 1;
let (result, pages) = window(&store, &mut loaded).await;
assert!(result.is_some(), "must prefer the replacement main head immediately");
assert!(pages.is_empty(), "must discard work on the older backup");
let total = loaded.checked_flatten("bucket").expect("new primary root");
assert_eq!((total.objects, total.size), (1, 19));
}
async fn inject_primary_page_error(store: &MemoryConfigStore, error: EcstoreError) {
let errors = store
.objects
.lock()
.await
.keys()
.filter(|key| key.contains(".bin.pages/"))
.map(|key| (key.clone(), error.clone()))
.collect();
*store.read_errors.lock().await = errors;
}
#[tokio::test(start_paused = true)]
async fn a_transient_primary_page_error_does_not_schedule_the_backup() {
let (store, _) = fixture().await;
inject_primary_page_error(&store, EcstoreError::SlowDown).await;
let mut loaded = DataUsageCache::default();
assert!(window(&store, &mut loaded).await.0.is_none());
store.read_errors.lock().await.clear();
finish(&store, false).await;
}
#[tokio::test]
async fn backup_phase_is_bound_to_its_storage_instance() {
let (damaged, _) = fixture().await;
corrupt_primary(&damaged).await;
assert!(window(&damaged, &mut DataUsageCache::default()).await.0.is_none());
let (healthy, _) = fixture().await;
let first = DataUsageCache::read_revisions(damaged.clone(), NAME)
.await
.expect("failed primary revision");
let second = DataUsageCache::read_revisions(healthy.clone(), NAME)
.await
.expect("other storage revision");
assert_eq!(first, second, "both stores must expose the same ETags");
finish(&healthy, false).await;
}
#[tokio::test(start_paused = true)]
async fn idle_restore_keeps_verified_pages_until_another_bucket_needs_its_slot() {
let (primary, _) = fixture().await;
let (other, _) = fixture().await;
finish_across_windows(&primary, false, Some(&other)).await;
}
#[tokio::test(start_paused = true)]
async fn idle_restore_keeps_the_failed_primary_revision_across_scheduled_rounds() {
let (primary, _) = fixture().await;
let (other, _) = fixture().await;
corrupt_primary(&primary).await;
corrupt_primary(&other).await;
finish_across_windows(&primary, true, Some(&other)).await;
}
#[tokio::test(start_paused = true)]
async fn physical_primary_page_corruption_falls_back_to_the_healthy_copy() {
let (store, _) = fixture().await;
inject_primary_page_error(&store, EcstoreError::FileCorrupt).await;
finish(&store, true).await;
}
+433 -52
View File
@@ -526,6 +526,19 @@ fn data_usage_entry_has_progress(entry: &DataUsageEntry) -> bool {
data_usage_root_has_progress(entry)
}
#[cfg(test)]
tokio::task_local! {
static TEST_FORCE_COMPACTION_WIDTH: usize;
}
fn force_compaction_width() -> usize {
#[cfg(test)]
if let Ok(width) = TEST_FORCE_COMPACTION_WIDTH.try_with(|width| *width) {
return width;
}
DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS
}
/// Root-fanout limit for `DataUsageCache::force_compact` at the end of a sweep.
///
/// An interrupted sweep still has to seal its resume frontier, so it keeps the
@@ -847,6 +860,7 @@ pub struct FolderScanner {
resume_failed_paths: Vec<String>,
coverage_gap: bool,
raw_enumeration_progress: Vec<RawEnumerationProgress>,
directory_checkpoints: HashMap<String, HashMap<String, directory_cursor::Saved>>,
pending_size_reconciliation_keys: HashSet<String>,
pending_size_reconciliation_scopes: HashSet<String>,
pending_size_reconciliation_truncated: bool,
@@ -866,6 +880,7 @@ struct RawEnumerationProgress {
entries_seen: u64,
digest: Sha256,
page_index: Option<RawEnumerationPageWriter>,
sealed_pages: Option<(RawEnumerationPageIndex, usize)>,
}
impl RawEnumerationProgress {
@@ -881,10 +896,34 @@ impl RawEnumerationProgress {
entries_seen: 0,
digest,
page_index,
sealed_pages: None,
}
}
fn from_sealed(parent: &str, index: &RawEnumerationPageIndex) -> Option<Self> {
use crate::raw_page_index::RawEnumerationPageOwnerStatus;
let count = match index.status() {
RawEnumerationPageOwnerStatus::Building { indexed_entries, .. }
| RawEnumerationPageOwnerStatus::Ready { indexed_entries, .. } => usize::try_from(indexed_entries).ok()?,
RawEnumerationPageOwnerStatus::Unsupported => return None,
};
if count == 0 {
return None;
}
Some(Self {
parent: parent.to_owned(),
last_entry: None,
entries_seen: 0,
digest: Sha256::new(),
page_index: None,
sealed_pages: Some((index.clone(), count)),
})
}
fn record_entry(&mut self, entry: &str) {
if let Some((index, _)) = self.sealed_pages.take() {
*self = Self::new(&self.parent, Some(index));
}
update_raw_enumeration_digest(&mut self.digest, b"entry", entry.as_bytes());
self.last_entry = Some(entry.to_string());
self.entries_seen = self.entries_seen.saturating_add(1);
@@ -908,6 +947,9 @@ impl RawEnumerationProgress {
}
fn page_index(&self) -> Option<RawEnumerationPageIndex> {
if let Some((index, _)) = &self.sealed_pages {
return Some(index.clone());
}
self.page_index
.as_ref()
.filter(|index| index.indexed_entry_count() > 0)
@@ -919,6 +961,9 @@ impl RawEnumerationProgress {
}
fn checkpointable_entry_count(&self) -> usize {
if let Some((_, count)) = &self.sealed_pages {
return *count;
}
self.page_index
.as_ref()
.map_or(0, RawEnumerationPageWriter::indexed_entry_count)
@@ -991,16 +1036,26 @@ async fn wait_for_scanner_io<T>(ctx: &CancellationToken, wait: impl std::future:
}
}
struct ScannerDirectory {
pub(crate) use directory_cursor::parked_bucket_disks;
use directory_cursor::{DirectoryScope, ScannerDirectory};
struct DirectoryStream {
reader: native_io::ReadDir,
batch_entries: usize,
#[cfg(test)]
ordered: Option<tests::erasure_probe::OrderedEntries>,
}
impl ScannerDirectory {
async fn open(path: &str) -> std::io::Result<Self> {
let reader = native_io::ReadDir::open(path).await?;
impl DirectoryStream {
async fn open_at(path: &str, anchor: &native_io::DirectoryAnchor) -> std::io::Result<Self> {
Ok(Self {
reader: native_io::ReadDir::open_at(path, anchor).await?,
#[cfg(test)]
ordered: None,
})
}
async fn open(path: &str, indexed: bool) -> std::io::Result<Self> {
let reader = native_io::ReadDir::open(path, indexed).await?;
#[cfg(test)]
let (reader, ordered) = {
let mut reader = reader;
@@ -1009,7 +1064,6 @@ impl ScannerDirectory {
};
Ok(Self {
reader,
batch_entries: 0,
#[cfg(test)]
ordered,
})
@@ -1034,11 +1088,14 @@ async fn next_scanner_dir_entry(
wait_for_scanner_io(ctx, tokio::task::yield_now()).await?;
entries.batch_entries = 0;
}
let entry = wait_for_scanner_io(ctx, run_scanner_node_operation(admission, ctx, entries.next_entry())).await???;
if entry.is_some() {
let entry = wait_for_scanner_io(ctx, run_scanner_node_operation(admission, ctx, entries.peek_entry())).await???;
if entry.as_ref().is_some_and(|item| !item.is_recorded()) {
// Tokio prefetches bounded chunks. Pacing consumption also paces the
// next refill, without charging an extra entry for EOF.
wait_for_scanner_io(ctx, sleeper.admit_disk_io(1, 0)).await?;
}
if entry.is_some() {
entries.accept_entry();
entries.batch_entries += 1;
}
Ok(entry)
@@ -1056,7 +1113,7 @@ impl FolderScanner {
&self,
folder: &CachedFolder,
into: &DataUsageEntry,
child: &CachedFolder,
child_name: &str,
child_hash: &DataUsageHash,
abandoned_children: &DataUsageHashMap,
) -> bool {
@@ -1076,8 +1133,7 @@ impl FolderScanner {
return false;
}
let Some(entry) = child
.name
let Some(entry) = child_name
.strip_prefix(folder.name.as_str())
.and_then(|entry| entry.strip_prefix('/'))
else {
@@ -1086,6 +1142,21 @@ impl FolderScanner {
!entry.is_empty() && !entry.contains('/') && !prefix_scan_scope.contains(entry)
}
fn can_reuse_scoped_child(&self, folder: &CachedFolder, into: &DataUsageEntry, name: &str, hash: &DataUsageHash) -> bool {
self.prefix_scan_scope.is_some()
&& self
.old_cache
.find(&folder.name)
.is_some_and(|parent| self.should_reuse_clean_root_child(folder, into, name, hash, &parent.children))
}
fn can_restore_linked_child(&self, folder: &CachedFolder, into: &DataUsageEntry, name: &str) -> bool {
self.can_reuse_covered_namespace(name, false)
|| (self.prefix_scan_scope.is_some()
&& self.old_cache.find(name).is_some()
&& self.can_reuse_scoped_child(folder, into, name, &hash_path(name)))
}
fn reuse_clean_root_child(&mut self, child: &CachedFolder, child_hash: &DataUsageHash, into: &mut DataUsageEntry) {
self.new_cache.copy_with_children(&self.old_cache, child_hash, &child.parent);
if let Some(updates) = &mut self.usage_updates {
@@ -1322,7 +1393,12 @@ impl FolderScanner {
/// Records one raw entry. Returns whether a durable index holds it, which
/// is what makes stopping at this entry recoverable.
#[cfg(test)]
fn record_raw_enumeration_entry(&mut self, parent: &str, entry: &str) -> bool {
self.record_raw_enumeration_entry_with_stream(parent, entry, false)
}
fn record_raw_enumeration_entry_with_stream(&mut self, parent: &str, entry: &str, continues_stream: bool) -> bool {
if self.old_cache.info.scan_progress.is_none() {
return false;
}
@@ -1334,8 +1410,11 @@ impl FolderScanner {
self.raw_enumeration_progress.truncate(position + 1);
} else {
let page_index = self.raw_enumeration_page_index_for_parent(parent).map(clone_cache_value);
self.raw_enumeration_progress
.push(RawEnumerationProgress::new(parent, page_index));
let mut progress = RawEnumerationProgress::new(parent, page_index);
if continues_stream && let Some(writer) = &mut progress.page_index {
writer.continue_native_stream();
}
self.raw_enumeration_progress.push(progress);
}
self.raw_enumeration_progress.last_mut().is_some_and(|progress| {
progress.record_entry(entry);
@@ -1402,6 +1481,17 @@ impl FolderScanner {
.map(|(index, _)| index)
}
fn saved_directories(&self) -> Vec<DirectoryCheckpoint> {
let mut snapshots: Vec<_> = self
.directory_checkpoints
.values()
.flat_map(HashMap::values)
.filter_map(|slot| slot.lock().unwrap_or_else(|p| p.into_inner()).clone())
.collect();
snapshots.sort_by(|left, right| (left.parent(), left.endpoint()).cmp(&(right.parent(), right.endpoint())));
snapshots
}
/// The walk path except the directory the resume state selects.
fn raw_enumeration_path_page_indexes(&self) -> Vec<RawEnumerationPageIndex> {
let selected = self.selected_raw_enumeration_progress();
@@ -1453,9 +1543,22 @@ impl FolderScanner {
}
fn maybe_send_checkpoint(&mut self, folder: &CachedFolder, hash: &DataUsageHash, into: &DataUsageEntry) {
self.maybe_send_owned_checkpoint(folder, hash, into, None);
}
// A compact accumulator is checkpointable only at the boundary where
// its owning frame has accepted a complete child. Descendant frames may
// temporarily share the same accumulator and cannot seal it themselves.
fn maybe_send_owned_checkpoint(
&mut self,
folder: &CachedFolder,
hash: &DataUsageHash,
into: &DataUsageEntry,
work: Option<&directory_cursor::DirectoryWork>,
) {
let Some(checkpoint_tx) = self.checkpoint_tx.clone() else { return };
let elapsed = self.last_checkpoint_at.elapsed();
if into.compacted
if (into.compacted && work.is_none())
|| self.new_cache.info.scan_progress.is_none()
|| elapsed < checkpoint_interval_for_build_time(self.last_checkpoint_build_time)
|| (self.checkpoint_objects.saturating_sub(self.last_checkpoint_objects) < SCANNER_CHECKPOINT_OBJECT_INTERVAL
@@ -1463,7 +1566,7 @@ impl FolderScanner {
{
return;
}
if self.new_cache.root().is_none() && self.raw_enumeration_progress.is_empty() {
if self.new_cache.root().is_none() && self.raw_enumeration_progress.is_empty() && work.is_none() {
return;
}
@@ -1479,6 +1582,9 @@ impl FolderScanner {
// Invalid partial graphs must not trigger a fresh O(N) clone on every object.
self.last_checkpoint_objects = self.checkpoint_objects;
self.last_checkpoint_at = Instant::now();
if let Some(work) = work {
work.checkpoint();
}
let snapshot = self.checkpoint_snapshot(folder, hash, into);
self.last_checkpoint_build_time = self.last_checkpoint_at.elapsed();
if let Some(snapshot) = snapshot {
@@ -1508,6 +1614,7 @@ impl FolderScanner {
snapshot.checked_flatten_complete_scope(&snapshot.info.name)?;
snapshot.info.last_update = Some(SystemTime::now());
snapshot.info.snapshot_complete = false;
snapshot.info.scan_directory_work = self.saved_directories();
let (cursor, page_index) = self.peek_raw_enumeration_resume_state();
snapshot.info.scan_raw_enumeration_path_page_indexes = self.raw_enumeration_path_page_indexes();
snapshot.info.scan_raw_enumeration_cursor = cursor;
@@ -1515,13 +1622,24 @@ impl FolderScanner {
snapshot.seal_scan_frontier(self.coverage_frontier.as_deref()).ok()?;
// A resumed readdir can run before covered subtrees are copied. Keep
// its durable receipt until this snapshot can represent that coverage.
if self.coverage_frontier.is_some() && snapshot.info.scan_coverage_receipt.is_none() {
let compacted_work = snapshot
.info
.scan_directory_work
.iter()
.filter_map(|directory| directory.progress(&snapshot.info));
if self
.coverage_frontier
.as_deref()
.is_some_and(|frontier| !compacted_work.clone().any(|proof| proof.covers(frontier)))
&& snapshot.info.scan_coverage_receipt.is_none()
{
return None;
}
// The receipt was just sealed from this snapshot; do not hash it twice.
if snapshot.info.scan_raw_enumeration_cursor.is_none()
&& snapshot.info.scan_raw_enumeration_page_index.is_none()
&& snapshot.info.scan_coverage_receipt.is_none()
&& snapshot.info.scan_directory_work.is_empty()
{
return None;
}
@@ -1865,7 +1983,15 @@ impl FolderScanner {
folder: CachedFolder,
into: &mut DataUsageEntry,
) -> Result<(), ScannerError> {
self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace, false)
if self.old_cache.info.scan_directory_source.is_none()
&& self.old_cache.info.scan_identity.is_some()
&& self.old_cache.info.scan_progress.is_some()
{
let source = wait_for_scanner_io(&ctx, directory_cursor::physical_source(&self.local_disk)).await??;
self.old_cache.info.scan_directory_source = Some(source.clone());
self.new_cache.info.scan_directory_source = Some(source);
}
Box::pin(self.scan_folder_inner(ctx, folder, into, FolderScanMode::Namespace, false))
.await
.map(|_| ())
}
@@ -1972,9 +2098,43 @@ impl FolderScanner {
// directory that vanished during iteration instead of failing the
// whole parent subtree. The bucket root (no parent) still propagates
// NotFound so namespace removal is detected by the caller.
// A small object directory normally finishes in one call. Allocate
// listing state at the bucket root or after an interruption has
// actually left this directory on the saved traversal path.
let info = &self.old_cache.info;
let previously_interrupted = info
.scan_raw_enumeration_cursor
.as_ref()
.is_some_and(|cursor| cursor.parent == folder.name)
|| info
.scan_raw_enumeration_page_index
.iter()
.chain(&info.scan_raw_enumeration_path_page_indexes)
.any(|index| index.parent() == Some(folder.name.as_str()));
let mut directory_scope = (folder.parent.is_none()
|| previously_interrupted
|| self.directory_checkpoints.contains_key(&folder.name)
|| directory_cursor::is_parked(&dir_path))
.then(|| DirectoryScope::new(&self.local_disk, &self.old_cache, mode, self.prefix_scan_scope.clone()))
.flatten();
if let Some(scope) = &mut directory_scope {
scope.bind_source(self.old_cache.info.scan_directory_source.as_deref());
}
let saved = directory_scope.as_ref().map(|owner| {
self.directory_checkpoints
.entry(folder.name.clone())
.or_default()
.entry(owner.endpoint().to_owned())
.or_default()
.clone()
});
let mut dir_reader = match wait_for_scanner_io(
&ctx,
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, ScannerDirectory::open(&dir_path)),
run_scanner_node_operation(
self.node_admission.as_ref(),
&ctx,
ScannerDirectory::open(&dir_path, &folder.name, directory_scope, self.budget.clone(), saved),
),
)
.await??
{
@@ -1997,16 +2157,48 @@ impl FolderScanner {
if mode != FolderScanMode::Namespace {
self.coverage_gap |= e.kind() == ErrorKind::NotADirectory && self.old_cache.info.scan_progress.is_some();
done_folder();
self.directory_checkpoints.remove(&folder.name);
return Ok(FolderScanOutcome::Vanished);
}
None
}
Err(e) => return Err(ScannerError::Io(e)),
};
if dir_reader.as_ref().is_some_and(|reader| reader.directory_replaced) && self.resume_frontier.is_some() {
// Names from a replaced directory cannot validate the saved
// child prefix. Finish this walk only as an observation.
self.coverage_gap = true;
self.resume_frontier = None;
}
let mut pending_entry_progress = 0_u64;
let mut last_entry_progress = Instant::now();
if let Some(reader) = &mut dir_reader {
reader.hold_abandoned(&mut abandoned_children);
}
let indexed = dir_reader.as_ref().is_some_and(ScannerDirectory::indexed);
let mut raw_enumeration_complete = false;
let raw_enumeration_committed_entries = self.raw_enumeration_committed_entry_oracle(&folder.name);
let continues_stream = dir_reader.as_ref().is_some_and(ScannerDirectory::continues_stream);
// A sealed work queue only emits the two classification markers;
// it never needs the raw-name membership table again.
let raw_enumeration_committed_entries = if dir_reader.as_ref().is_some_and(ScannerDirectory::finished_listing) {
// A later quantum can stop in a child. Its durable proof
// still needs every ancestor's prior immutable pages, even
// though this cursor need not enumerate those names again.
if let Some(progress) = self
.old_cache
.info
.scan_raw_enumeration_page_index
.iter()
.chain(&self.old_cache.info.scan_raw_enumeration_path_page_indexes)
.find(|index| index.parent() == Some(folder.name.as_str()))
.and_then(|index| RawEnumerationProgress::from_sealed(&folder.name, index))
{
self.raw_enumeration_progress.push(progress);
}
HashSet::new()
} else {
self.raw_enumeration_committed_entry_oracle(&folder.name)
};
let raw_replay_floor = u64::try_from(raw_enumeration_committed_entries.len()).unwrap_or(u64::MAX);
let mut raw_entries_observed = 0_u64;
let mut raw_entries_charged = 0_u64;
@@ -2017,12 +2209,11 @@ impl FolderScanner {
loop {
// Hold at most one metadata entry until a candidate reaches
// EOF: a later part must not leave object accounting or ILM work.
let (entry, replayed_metadata) = if let Some(reader) = dir_reader.as_mut() {
let (entry, replayed_metadata) = if let Some(reader) = dir_reader.as_mut().filter(|_| !raw_enumeration_complete) {
let entry = match next_scanner_dir_entry(&ctx, &self.sleeper, reader, self.node_admission.as_ref()).await {
Ok(Some(entry)) => entry,
Ok(None) => {
raw_enumeration_complete = true;
dir_reader = None;
continue;
}
Err(ScannerError::Io(e))
@@ -2042,7 +2233,6 @@ impl FolderScanner {
"Scanner folder state updated"
);
raw_enumeration_complete = true;
dir_reader = None;
continue;
}
Err(ScannerError::Io(e)) if e.kind() == ErrorKind::NotADirectory => {
@@ -2057,12 +2247,12 @@ impl FolderScanner {
"Scanner folder state updated"
);
raw_enumeration_complete = true;
dir_reader = None;
continue;
}
Err(e) => return Err(e),
};
(entry, false)
let marker = entry.is_recorded();
(entry, marker)
} else {
raw_enumeration_complete = true;
if candidate_unclassified {
@@ -2081,6 +2271,7 @@ impl FolderScanner {
self.budget.record_entries_visited(pending_entry_progress);
self.finish_raw_enumeration_parent(&folder.name);
done_folder();
self.directory_checkpoints.remove(&folder.name);
return Ok(FolderScanOutcome::Namespace);
}
candidate_unclassified = false;
@@ -2094,7 +2285,11 @@ impl FolderScanner {
..
}
) && deferred_metadata.is_none()
&& queued_folders.is_empty();
&& dir_reader
.as_ref()
.and_then(ScannerDirectory::folder_count)
.unwrap_or(queued_folders.len())
== 0;
if !empty_object_child {
if !self.budget.try_start_directory() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
@@ -2114,8 +2309,8 @@ impl FolderScanner {
// its old size, so charging only that tail keeps each
// budget unit durable even if the listing order changed.
raw_entries_observed = raw_entries_observed.saturating_add(1);
let raw_entry_is_new =
raw_entries_observed > raw_replay_floor && !raw_enumeration_committed_entries.contains(&file_name);
let raw_entry_is_new = (continues_stream || raw_entries_observed > raw_replay_floor)
&& !raw_enumeration_committed_entries.contains(&file_name);
if raw_entry_is_new {
#[cfg(test)]
tests::enumeration_restart::observe_raw_entry(&dir_path, &entry.file_name(), &self.budget);
@@ -2128,7 +2323,8 @@ impl FolderScanner {
last_entry_progress = Instant::now();
}
}
let raw_entry_is_durable = self.record_raw_enumeration_entry(&folder.name, &file_name);
let raw_entry_is_durable =
self.record_raw_enumeration_entry_with_stream(&folder.name, &file_name, continues_stream);
// Object metadata and the entries behind it are charged to
// the object budget; this one covers listing work.
let object_entry = file_name == STORAGE_FORMAT_FILE || deferred_metadata.is_some();
@@ -2142,6 +2338,9 @@ impl FolderScanner {
}
self.maybe_send_checkpoint(&folder, &this_hash, into);
}
if !replayed_metadata && dir_reader.as_ref().is_some_and(|reader| reader.enumeration_only) {
continue;
}
let is_storage_format_entry = file_name == STORAGE_FORMAT_FILE;
if candidate_unclassified && is_storage_format_entry {
deferred_metadata = Some(entry);
@@ -2160,12 +2359,15 @@ impl FolderScanner {
// Ignore entries that disappeared during traversal or hit symlink
// loops, but propagate other walk errors.
let mut entry_type = match wait_for_scanner_io(
&ctx,
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, entry.file_type()),
)
.await??
let file_type = if entry.is_recorded()
&& let Some(kind) = entry.cached_type()
{
Ok(kind)
} else {
wait_for_scanner_io(&ctx, run_scanner_node_operation(self.node_admission.as_ref(), &ctx, entry.file_type()))
.await??
};
let mut entry_type = match file_type {
Ok(entry_type) => entry_type,
Err(e) if e.kind() == ErrorKind::NotFound => {
debug!(
@@ -2202,6 +2404,7 @@ impl FolderScanner {
// namespace coverage. Discard its provisional resume owner.
self.finish_raw_enumeration_parent(&folder.name);
done_folder();
self.directory_checkpoints.remove(&folder.name);
return Ok(FolderScanOutcome::ErasureData);
}
@@ -2302,7 +2505,7 @@ impl FolderScanner {
object_heal_prob_div: folder.object_heal_prob_div,
};
if exists && self.should_reuse_clean_root_child(&folder, into, &this, &h, &abandoned_children) {
if exists && self.should_reuse_clean_root_child(&folder, into, &this.name, &h, &abandoned_children) {
abandoned_children.remove(&clone_cache_value(&h.0));
self.reuse_clean_root_child(&this, &h, into);
} else if exists {
@@ -2528,7 +2731,11 @@ impl FolderScanner {
item.transform_meta_dir();
abandoned_children.remove(&path_join_buf(&[&item.bucket, &item.object_path()]));
let resolved = path_join_buf(&[&item.bucket, &item.object_path()]);
abandoned_children.remove(&resolved);
if let Some(reader) = &mut dir_reader {
reader.resolve_abandoned(&resolved);
}
apply_scanner_size_summary(into, &sz);
if !sz.size_reconciliation.is_empty() {
@@ -2562,7 +2769,19 @@ impl FolderScanner {
self.budget.record_entries_visited(pending_entry_progress);
// A directory with queued children keeps its index until they end:
// a restart replays this listing to reach the unfinished child.
if raw_enumeration_complete && queued_folders.is_empty() {
if indexed && dir_reader.as_ref().is_some_and(|reader| !reader.finished_listing()) {
return Err(ScannerError::Io(std::io::Error::new(
ErrorKind::Interrupted,
"scanner directory changed before listing completed",
)));
}
if raw_enumeration_complete
&& dir_reader
.as_ref()
.and_then(ScannerDirectory::folder_count)
.unwrap_or(queued_folders.len())
== 0
{
self.finish_raw_enumeration_parent(&folder.name);
}
@@ -2576,8 +2795,18 @@ impl FolderScanner {
// the emergency 250k ceiling. Keep FORCE as the absolute backstop.
let is_scan_root = folder.name == self.old_cache.info.name;
let forward_sweep = self.old_cache.info.scan_progress.is_some();
let total_folders = queued_folders.len();
let force_compact = total_folders >= DATA_SCANNER_FORCE_COMPACT_AT_FOLDERS;
if let Some(reader) = &dir_reader {
reader.restore_abandoned(&mut abandoned_children);
}
let total_folders = dir_reader
.as_ref()
.and_then(ScannerDirectory::folder_count)
.unwrap_or(queued_folders.len());
// A completed root above the normal cache ceiling is compacted by
// the existing completion path. A resumable sorted listing can use
// that representation throughout processing without losing its place.
let emergency_compact = total_folders >= force_compaction_width();
let force_compact = emergency_compact || (indexed && total_folders > DATA_SCANNER_COMPACT_AT_CHILDREN);
// Reused root children still have links in the new cache.
let should_compact = force_compact
|| (!forward_sweep
@@ -2586,7 +2815,7 @@ impl FolderScanner {
// Preserve the ordinary namespace walk's child isolation when the
// root starts sharing one accumulator across its descendants.
let isolate_compacted_children =
isolate_compacted_namespace_errors || (is_scan_root && !into.compacted && should_compact && !force_compact);
isolate_compacted_namespace_errors || (is_scan_root && !into.compacted && should_compact && !emergency_compact);
self.alert_excessive_folders(&folder.name, total_folders);
@@ -2603,7 +2832,7 @@ impl FolderScanner {
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_FOLDER,
folder = %folder.name,
entry_count = queued_folders.len(),
entry_count = total_folders,
state = "preemptive_compaction",
"Scanner folder switched to compacted mode"
);
@@ -2632,7 +2861,7 @@ impl FolderScanner {
.and_then(|resume_after| folder_resume_match(&folder.name, resume_after))
.is_some_and(|resume_match| matches!(resume_match, FolderResumeMatch::Descendant));
let scan_resume_after = checkpoint_resume_after.or(self.old_cache.info.scan_resume_after.as_deref());
let has_queued_folders = !queued_folders.is_empty();
let has_queued_folders = total_folders > 0;
let resume_order = if forward_sweep {
order_queued_folders_for_resume(&mut queued_folders, None)
} else {
@@ -2646,14 +2875,95 @@ impl FolderScanner {
}
}
// Scan child folders in the combined resume order.
for queued_folder in queued_folders {
let mut work = dir_reader.as_mut().and_then(ScannerDirectory::work);
if let Some(work) = &mut work {
work.restore_mode(into.compacted);
// A saved prefix tree must actually contain the contributions
// the parked queue has consumed. A failed cache save can leave
// this process's classified names ahead of its durable values.
if !into.compacted
&& !work
.linked
.iter()
.all(|name| self.can_restore_linked_child(&folder, into, name))
{
work.rewind();
}
self.coverage_gap |= work.gap;
if into.compacted {
into.merge(&work.compacted_total);
} else {
for name in &work.linked {
let child = hash_path(name);
self.new_cache
.copy_with_children(&self.old_cache, &child, &Some(this_hash.clone()));
into.add_child(&child);
}
}
}
let mut queued_folders = queued_folders.into_iter();
let mut acknowledge_previous: Option<(String, bool)> = None;
loop {
if let Some(work) = &mut work
&& let Some((name, linked)) = acknowledge_previous.take()
{
work.gap |= self.coverage_gap;
work.acknowledge(&name, linked, into.compacted);
}
if ctx.is_cancelled() {
return Err(ScannerError::Other("Operation cancelled".to_string()));
}
let queued_folder = if let Some(work) = &mut work {
let Some(name) = wait_for_scanner_io(
&ctx,
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, work.next_name()),
)
.await???
else {
break;
};
let name = path_join_buf(&[&folder.name, &name]);
let source = if !force_compact && self.old_cache.find(&name).is_some() {
FolderScanSource::Existing
} else {
FolderScanSource::New
};
let h = hash_path(&name);
if source == FolderScanSource::Existing
&& let Some(updates) = &mut self.usage_updates
{
updates
.cache
.copy_with_children(&self.old_cache, &h, &Some(this_hash.clone()));
}
QueuedFolder {
folder: CachedFolder {
name,
parent: Some(this_hash.clone()),
object_heal_prob_div: folder.object_heal_prob_div,
},
source,
}
} else {
let Some(queued) = queued_folders.next() else { break };
queued
};
let mut folder_item = queued_folder.folder;
let h = hash_path(&folder_item.name);
if work.is_some() {
acknowledge_previous = Some((folder_item.name.clone(), false));
}
if work.is_some()
&& queued_folder.source == FolderScanSource::Existing
&& self.can_reuse_scoped_child(&folder, into, &folder_item.name, &h)
{
self.reuse_clean_root_child(&folder_item, &h, into);
if let Some((_, linked)) = &mut acknowledge_previous {
*linked = true;
}
continue;
}
let reuse_covered = self.can_reuse_covered_namespace(&folder_item.name, into.compacted);
let mut child_mode = if self.is_erasure_mode && is_erasure_candidate(&folder_item.name) {
FolderScanMode::ErasureCandidate {
@@ -2666,6 +2976,9 @@ impl FolderScanner {
if child_mode == FolderScanMode::Namespace && reuse_covered {
self.new_cache.copy_with_children(&self.old_cache, &h, &folder_item.parent);
into.add_child(&h);
if let Some((_, linked)) = &mut acknowledge_previous {
*linked = true;
}
continue;
}
@@ -2744,12 +3057,16 @@ impl FolderScanner {
// rolling back totals already committed by healthy siblings.
let isolate_compacted_child =
into.compacted && isolate_compacted_children && child_mode == FolderScanMode::Namespace;
let outcome = if into.compacted && !isolate_compacted_child {
// A parked queue owns only committed subtree totals. Run the
// active child in its own accumulator so cancellation cannot
// count a partial child twice at the next admission.
let child_summary = isolate_compacted_child || (into.compacted && work.is_some());
let outcome = if into.compacted && !child_summary {
// Keep the shared-accumulator path free of child summaries.
Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), into, child_mode, false)).await?
} else {
let mut dst = DataUsageEntry {
compacted: isolate_compacted_child,
compacted: child_summary,
..Default::default()
};
// Provisional child usage must not escape through progress
@@ -2759,8 +3076,15 @@ impl FolderScanner {
} else {
None
};
let result = if isolate_compacted_child {
Box::pin(self.scan_folder_inner(ctx.clone(), folder_item.clone(), &mut dst, child_mode, true)).await
let result = if child_summary {
Box::pin(self.scan_folder_inner(
ctx.clone(),
folder_item.clone(),
&mut dst,
child_mode,
isolate_compacted_child,
))
.await
} else {
self.scan_child_folder(ctx.clone(), folder_item.clone(), &mut this_hash, into, &mut dst, child_mode)
.await
@@ -2770,14 +3094,22 @@ impl FolderScanner {
}
match result {
Ok(outcome) => {
if isolate_compacted_child {
if child_summary {
into.merge(&dst);
if let Some(work) = &mut work {
if outcome == FolderScanOutcome::Namespace && !self.coverage_gap && dst.failed_objects == 0 {
work.compacted_total.merge(&dst);
work.acknowledge(&folder_item.name, false, true);
} else if dst.failed_objects != 0 {
self.coverage_gap = true;
}
}
}
outcome
}
Err(e) => {
if ctx.is_cancelled() || matches!(&e, ScannerError::ResourceDeferred) {
if isolate_compacted_child {
if child_summary {
// Cancellation and resource deferral keep completed
// work even though the child has not finished.
into.merge(&dst);
@@ -2787,7 +3119,7 @@ impl FolderScanner {
}
return Err(e);
}
if child_mode != FolderScanMode::Namespace {
if child_mode != FolderScanMode::Namespace || (into.compacted && !isolate_compacted_child) {
return Err(e);
}
warn!(
@@ -2844,10 +3176,16 @@ impl FolderScanner {
}
if into.compacted {
self.record_completed_child(&folder_item.name);
if !was_compacted && work.is_some() {
self.maybe_send_owned_checkpoint(&folder, &this_hash, into, work.as_deref());
}
self.send_update_for_entry(&this_hash, &folder.parent, into).await;
tokio::task::yield_now().await;
} else {
into.add_child(&h);
if let Some(work) = &mut work {
work.acknowledge(&folder_item.name, true, false);
}
self.record_completed_child(&folder_item.name);
tokio::task::yield_now().await;
self.maybe_send_checkpoint(&folder, &this_hash, into);
@@ -2856,6 +3194,14 @@ impl FolderScanner {
}
}
if let Some(reader) = &mut dir_reader {
wait_for_scanner_io(
&ctx,
run_scanner_node_operation(self.node_admission.as_ref(), &ctx, reader.verify_current()),
)
.await???;
}
// Scan for healing
if abandoned_children.is_empty() || !self.should_heal().await {
debug!(
@@ -3334,6 +3680,7 @@ impl FolderScanner {
let scanned_objects = u64::try_from(into.objects).unwrap_or(u64::MAX);
emit_scanner_folder_trace(&self.root, &folder.name, scanned_objects, trace_started_at, "completed");
self.directory_checkpoints.remove(&folder.name);
Ok(FolderScanOutcome::Namespace)
}
@@ -3367,7 +3714,7 @@ pub(crate) async fn scan_data_folder_scoped(
budget: Arc<ScannerCycleBudget>,
disks: Vec<Arc<Disk>>,
local_disk: Arc<Disk>,
cache: DataUsageCache,
mut cache: DataUsageCache,
updates: Option<mpsc::Sender<DataUsageEntry>>,
scan_mode: HealScanMode,
sleeper: DynamicSleeper,
@@ -3381,6 +3728,19 @@ pub(crate) async fn scan_data_folder_scoped(
return Err(ScannerError::Other("internal error: root scan attempted".to_string()));
}
// URL endpoints include the storage host. A formatted disk also contributes
// its persistent UUID, fencing replacements and clone-like device/inode IDs.
if cache.info.scan_progress.is_some() && cache.info.scan_identity.is_some() {
use crate::storage_api::owner::EcstoreDiskAPI;
let endpoint = local_disk.endpoint().to_string();
let source = wait_for_scanner_io(&ctx, directory_cursor::physical_source(&local_disk)).await??;
let mut endpoints: HashSet<_> = disks.iter().map(|disk| disk.endpoint().to_string()).collect();
endpoints.insert(endpoint);
cache.activate_directory_source(source, &endpoints);
}
let directory_work = std::mem::take(&mut cache.info.scan_directory_work);
cache.info.scan_directory_work = DirectoryCheckpoint::verified_work(directory_work, &cache.info);
// Get disk path
let base_path = local_disk.path().to_string_lossy().to_string();
@@ -3437,6 +3797,16 @@ pub(crate) async fn scan_data_folder_scoped(
Vec::new()
};
// Create folder scanner
let mut directory_checkpoints = HashMap::new();
for checkpoint in &cache.info.scan_directory_work {
directory_checkpoints
.entry(checkpoint.parent().to_owned())
.or_insert_with(HashMap::new)
.insert(
checkpoint.endpoint().to_owned(),
Arc::new(std::sync::Mutex::new(Some(checkpoint.clone()))),
);
}
let mut scanner = FolderScanner {
root: base_path,
old_cache: cache,
@@ -3477,6 +3847,7 @@ pub(crate) async fn scan_data_folder_scoped(
resume_failed_paths,
coverage_gap: false,
raw_enumeration_progress: Vec::new(),
directory_checkpoints,
pending_size_reconciliation_keys: HashSet::new(),
pending_size_reconciliation_scopes: HashSet::new(),
pending_size_reconciliation_truncated: false,
@@ -3514,6 +3885,7 @@ pub(crate) async fn scan_data_folder_scoped(
.await
{
Ok(()) => {
directory_cursor::forget_bucket(&scanner.local_disk, &scanner.old_cache.info);
// Get the new cache and finalize it
let coverage_gap = scanner.coverage_gap;
scanner.reconcile_failed_objects_after_full_scan();
@@ -3558,6 +3930,9 @@ pub(crate) async fn scan_data_folder_scoped(
new_cache.info.scan_raw_enumeration_cursor = None;
new_cache.info.scan_raw_enumeration_page_index = None;
new_cache.info.scan_raw_enumeration_path_page_indexes.clear();
new_cache.info.scan_directory_work.clear();
new_cache.info.scan_directory_source = None;
new_cache.info.scan_source_trees.clear();
new_cache.info.scan_coverage_receipt = None;
if had_scan_checkpoint {
global_metrics().record_scanner_checkpoint_cleared();
@@ -3606,7 +3981,9 @@ pub(crate) async fn scan_data_folder_scoped(
scanner.carry_forward_old_children(&root_hash, &mut root);
}
let coverage_frontier = clone_cache_value(&scanner.coverage_frontier);
let directories = scanner.saved_directories();
let new_cache = scanner.as_mut_new_cache();
new_cache.info.scan_directory_work = directories;
if root_has_progress {
new_cache.replace_hashed(&root_hash, &None, &root);
} else if let Some(cache) = carry_forward_cache {
@@ -3626,6 +4003,7 @@ pub(crate) async fn scan_data_folder_scoped(
new_cache.info.scan_coverage_receipt = None;
}
if partial_cache_is_useful(&root, pending_heals_changed)
|| !new_cache.info.scan_directory_work.is_empty()
|| new_cache.info.scan_raw_enumeration_cursor.is_some()
|| new_cache.info.scan_raw_enumeration_page_index.is_some()
|| !new_cache.info.size_reconciliation.is_empty()
@@ -3701,6 +4079,9 @@ pub(crate) async fn scan_data_folder_scoped(
}
}
mod directory_cursor;
pub use directory_cursor::DirectoryCheckpoint;
pub(crate) use directory_cursor::{DirectoryProgress, DirectoryReadProgress};
mod item_actions;
mod ledger;
mod native_io;
@@ -0,0 +1,714 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Cycle-local execution can borrow an interrupted physical directory stream.
//! Only the current scanner owns admission and classification; parked streams
//! have no task, budget or authority to update a bucket cache.
use std::collections::HashMap;
mod checkpoint;
mod pages;
pub use checkpoint::DirectoryCheckpoint;
pub(super) use checkpoint::Saved;
pub(crate) use checkpoint::{DirectoryProgress, DirectoryReadProgress};
use checkpoint::{Physical, Recorder, Scope};
use pages::NamePages;
use std::io;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, LazyLock, Mutex, Weak};
use std::time::{Duration, Instant, SystemTime};
use super::{DirectoryStream, Disk, FolderScanMode, ScannerBucketPrefixScanScope, native_io};
use crate::data_usage_define::{DataUsageCacheInfo, DataUsageCacheSource, DataUsageScanIdentity, DataUsageScanProgress};
use crate::scanner_budget::ScannerCycleBudget;
pub(super) const PARKED_DIRECTORIES_PER_DISK: usize = 32;
/// Only a positively identified disk can re-adopt a linked tree. A missing
/// format on an otherwise visible local drive uses its directory generation;
/// genuine format I/O errors propagate before any cached subtotals are used.
pub(super) async fn physical_source(disk: &Arc<Disk>) -> Result<String, super::ScannerError> {
use crate::storage_api::owner::EcstoreDiskAPI;
use sha2::{Digest, Sha256};
let endpoint = disk.endpoint().to_string();
let generation = match disk.get_disk_id().await {
Ok(Some(id)) if !id.is_nil() => id.to_string(),
Ok(None) | Err(super::DiskError::UnformattedDisk) => {
let path = disk.path();
let stamp = DirectoryStamp::at(path.to_str().ok_or_else(|| io::Error::from(io::ErrorKind::InvalidInput))?).await?;
// Bucket creation and scanner startup can change the volume root's
// mtime/ctime. Birth time distinguishes inode reuse without making
// ordinary directory content changes appear to replace the disk.
if let Some(generation) = stamp.volume_generation() {
let bytes = rmp_serde::to_vec_named(&(std::env::consts::OS, generation)).map_err(io::Error::other)?;
hex_simd::encode_to_string(Sha256::digest(bytes), hex_simd::AsciiCase::Lower)
} else {
// The platform cannot attest the generation across processes.
uuid::Uuid::new_v4().to_string()
}
}
Ok(Some(_)) => return Err(io::Error::new(io::ErrorKind::InvalidData, "scanner disk format contains a nil ID").into()),
Err(error) => return Err(io::Error::other(error).into()),
};
Ok(format!("{generation}|{endpoint}"))
}
#[derive(Clone)]
pub(super) struct DirectoryScope {
disk: Weak<Disk>,
bucket: Arc<str>,
source: DataUsageCacheSource,
identity: DataUsageScanIdentity,
progress: DataUsageScanProgress,
mode: FolderScanMode,
prefix_scope: Option<ScannerBucketPrefixScanScope>,
owner: (u64, u64),
endpoint: String,
watch: Option<[u8; 32]>,
continuing: bool,
}
impl DirectoryScope {
pub(super) fn new(
disk: &Arc<Disk>,
cache: &super::DataUsageCache,
mode: FolderScanMode,
prefix_scope: Option<ScannerBucketPrefixScanScope>,
) -> Option<Self> {
use crate::storage_api::owner::EcstoreDiskAPI;
let info = &cache.info;
Some(Self {
disk: Arc::downgrade(disk),
bucket: Arc::from(info.name.as_str()),
source: info.source?,
identity: info.scan_identity.filter(DataUsageScanIdentity::is_valid)?,
progress: info.scan_progress?,
mode,
prefix_scope,
owner: (info.leader_epoch, info.next_cycle),
endpoint: format!("|{}", disk.endpoint()),
watch: info
.scan_usage_watch
.as_ref()
.filter(|proof| proof.is_well_formed())
.map(|proof| proof.digest().0),
continuing: cache.has_scan_position(),
})
}
pub(super) fn bind_source(&mut self, source: Option<&str>) {
if let Some(source) = source {
self.endpoint = source.to_owned();
}
}
pub(super) fn endpoint(&self) -> &str {
&self.endpoint
}
fn matches(&self, other: &Self) -> bool {
self.endpoint == other.endpoint
&& self.disk.ptr_eq(&other.disk)
&& self.bucket == other.bucket
&& self.source == other.source
&& self.identity == other.identity
&& self.progress == other.progress
&& self.mode == other.mode
&& self.watch == other.watch
&& self.prefix_scope == other.prefix_scope
&& (other.continuing || self.owner == other.owner)
}
}
#[derive(PartialEq, Eq)]
struct DirectoryStamp {
modified: SystemTime,
birth: Option<SystemTime>,
#[cfg(unix)]
device: u64,
#[cfg(unix)]
inode: u64,
#[cfg(unix)]
changed: (i64, i64),
#[cfg(windows)]
identity: same_file::Handle,
#[cfg(windows)]
physical: Option<(u64, u64, i64)>,
}
impl DirectoryStamp {
fn volume_generation(&self) -> Option<(u64, u64, SystemTime)> {
#[cfg(unix)]
{
Some((self.device, self.inode, self.birth?))
}
#[cfg(windows)]
{
let (volume, file, _) = self.physical?;
Some((volume, file, self.birth?))
}
#[cfg(not(any(unix, windows)))]
{
None
}
}
async fn at(path: &str) -> io::Result<Self> {
let path = PathBuf::from(path);
crate::storage_api::ecstore_spawn_blocking_with_resource_ownership(move || {
let meta = std::fs::symlink_metadata(&path)?;
if !meta.is_dir() {
return Err(io::ErrorKind::NotADirectory.into());
}
#[cfg(unix)]
use std::os::unix::fs::MetadataExt;
#[cfg(windows)]
let identity = same_file::Handle::from_path(&path)?;
#[cfg(windows)]
let physical = winapi_util::file::information(identity.as_file()).ok().and_then(|info| {
let created = i64::try_from(info.creation_time()?).ok()?;
(info.file_index() != 0).then_some((info.volume_serial_number(), info.file_index(), created))
});
Ok(Self {
modified: meta.modified()?,
birth: meta.created().ok(),
#[cfg(unix)]
device: meta.dev(),
#[cfg(unix)]
inode: meta.ino(),
#[cfg(unix)]
changed: (meta.ctime(), meta.ctime_nsec()),
#[cfg(windows)]
identity,
#[cfg(windows)]
physical,
})
})
.await
.map_err(io::Error::other)?
}
}
#[derive(Default)]
pub(super) struct DirectoryWork {
names: NamePages,
// For a prefix tree the canonical child values live in the bucket cache.
// The final compaction threshold caps how many paths this list can hold.
pub(super) linked: Vec<String>,
pub(super) compacted_total: super::DataUsageEntry,
pub(super) gap: bool,
compacted: bool,
recorder: Option<Recorder>,
}
impl DirectoryWork {
pub(super) fn restore_mode(&mut self, compacted: bool) {
if self.compacted && !compacted {
self.rewind();
}
}
fn maybe_checkpoint(&mut self) {
if let Some(recorder) = &mut self.recorder
&& recorder.last_update.elapsed() >= Duration::from_secs(1)
{
recorder.last_update = Instant::now();
self.checkpoint();
}
}
pub(super) fn checkpoint(&self) {
if let Some(recorder) = &self.recorder {
recorder.save(self);
}
}
pub(super) async fn next_name(&mut self) -> io::Result<Option<String>> {
self.names.next().await
}
pub(super) fn acknowledge(&mut self, name: &str, linked: bool, compacted: bool) {
if self.names.acknowledge() {
self.compacted = compacted;
if linked && !compacted {
self.linked.push(name.to_owned());
}
self.maybe_checkpoint();
}
}
pub(super) fn rewind(&mut self) {
self.compacted = false;
self.names.rewind();
self.linked.clear();
self.compacted_total = super::DataUsageEntry::default();
self.gap = false;
}
}
struct Listing {
reader: Option<DirectoryStream>,
pending: Option<native_io::DirEntry>,
delivery: Option<(native_io::DirEntry, bool)>,
classified: Option<Option<std::fs::FileType>>,
markers: [Option<native_io::DirEntry>; 2],
abandoned: Option<super::DataUsageHashMap>,
work: DirectoryWork,
failed: bool,
}
struct Parked {
scope: DirectoryScope,
parked_at: Instant,
serial: u64,
stamp: DirectoryStamp,
listing: Box<Listing>,
}
static PARKED: LazyLock<Mutex<HashMap<PathBuf, Parked>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
fn expired_slots(slots: &mut HashMap<PathBuf, Parked>) -> Vec<(PathBuf, Parked)> {
slots
.extract_if(|_, parked| {
parked.scope.disk.strong_count() == 0 || parked.parked_at.elapsed() > Duration::from_secs(24 * 3600)
})
.collect()
}
/// A routing hint only. The admitted walker revalidates the physical source,
/// bucket instance, coverage plan and directory before using a parked stream.
pub(crate) fn parked_bucket_disks(bucket: &str) -> Vec<Weak<Disk>> {
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
let expired = expired_slots(&mut slots);
let matches = slots
.values()
.filter(|entry| entry.scope.bucket.as_ref() == bucket)
.map(|entry| entry.scope.disk.clone())
.collect();
drop(slots);
drop(expired);
matches
}
pub(super) fn is_parked(path: &str) -> bool {
PARKED.lock().unwrap_or_else(|p| p.into_inner()).contains_key(Path::new(path))
}
pub(super) fn forget_bucket(disk: &Arc<Disk>, info: &DataUsageCacheInfo) {
let owner = Arc::downgrade(disk);
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
let removed = slots
.extract_if(|_, parked| {
parked.scope.disk.ptr_eq(&owner)
&& parked.scope.bucket.as_ref() == info.name
&& parked.scope.owner <= (info.leader_epoch, info.next_cycle)
})
.collect::<Vec<_>>();
drop(slots);
drop(removed);
}
// Prefer existing active work over cyclic LRU eviction. Under pressure, a
// deleted or replaced directory must not monopolize one of the physical slots.
async fn reclaim_retired(scope: &DirectoryScope, path: &Path) {
static NEXT_PROBE: AtomicUsize = AtomicUsize::new(0);
let candidate = {
let slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
if slots.contains_key(path) {
return;
}
let disk_slots: Vec<_> = slots
.iter()
.filter(|(_, parked)| parked.scope.disk.ptr_eq(&scope.disk))
.collect();
if disk_slots.len() < PARKED_DIRECTORIES_PER_DISK {
return;
}
let (path, parked) = disk_slots[NEXT_PROBE.fetch_add(1, Ordering::Relaxed) % disk_slots.len()];
(path.clone(), parked.serial)
};
let Some(candidate_path) = candidate.0.to_str() else { return };
let current = DirectoryStamp::at(candidate_path).await;
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
let stale = slots.get(&candidate.0).is_some_and(|parked| {
parked.serial == candidate.1
&& match &current {
Ok(stamp) => stamp != &parked.stamp,
Err(error) => matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::NotADirectory),
}
});
let retired = stale.then(|| slots.remove(&candidate.0));
drop(slots);
drop(retired);
}
fn valid_component(name: &str) -> bool {
!name.is_empty()
&& name.len() <= 4096
&& name != "."
&& name != ".."
&& !name.as_bytes().contains(&0)
&& std::path::Path::new(name)
.file_name()
.is_some_and(|part| part == std::ffi::OsStr::new(name))
}
pub(super) struct ScannerDirectory {
path: Arc<Path>,
scope: Option<(DirectoryScope, DirectoryStamp)>,
directory_name: Arc<str>,
budget: Arc<ScannerCycleBudget>,
listing: Option<Box<Listing>>,
marker: usize,
pub(super) enumeration_only: bool,
resumed: bool,
pub(super) directory_replaced: bool,
pub(super) batch_entries: usize,
}
impl ScannerDirectory {
pub(super) async fn open(
path: &str,
directory_name: &str,
scope: Option<DirectoryScope>,
budget: Arc<ScannerCycleBudget>,
saved: Option<Saved>,
) -> io::Result<Self> {
// A different physical directory must never inherit old names. Directory
// timestamps supplement (and cannot replace) the scanner's plan fences.
let stamp = if scope.is_some() {
Some(DirectoryStamp::at(path).await?)
} else {
None
};
if let Some(scope) = &scope {
reclaim_retired(scope, Path::new(path)).await;
}
let parked = scope.as_ref().and_then(|current| {
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
let key = Path::new(path);
if slots
.get(key)
.is_some_and(|p| p.scope.disk.ptr_eq(&current.disk) && p.scope.owner > current.owner)
{
return None;
}
slots.remove(key)
});
let directory_replaced = parked.as_ref().is_some_and(|entry| Some(&entry.stamp) != stamp.as_ref());
let parked =
parked.filter(|entry| !directory_replaced && scope.as_ref().is_some_and(|current| entry.scope.matches(current)));
let mut resumed = parked.is_some();
let mut previous = saved
.as_ref()
.and_then(|slot| slot.lock().unwrap_or_else(|p| p.into_inner()).clone());
let durable_scope = scope.as_ref().map(Scope::new);
let endpoint = scope.as_ref().map(|s| s.endpoint.clone()).unwrap_or_default();
let physical = scope
.as_ref()
.zip(stamp.as_ref())
.and_then(|(scope, stamp)| Physical::of(&scope.endpoint, stamp));
let mut replaced = directory_replaced
|| previous
.as_ref()
.is_some_and(|old| old.endpoint() == endpoint && old.physical.is_some() && old.physical != physical);
previous = previous.filter(|old| {
durable_scope
.as_ref()
.is_some_and(|current| old.restore_allowed(directory_name, current, &physical))
});
let mut listing = if let Some(parked) = parked {
parked.listing
} else {
let mut reader = None;
if let Some(checkpoint) = &previous {
if checkpoint.sealed {
resumed = true;
} else if let Some(anchor) = &checkpoint.anchor {
match DirectoryStream::open_at(path, anchor).await {
Ok(restored) => {
reader = Some(restored);
resumed = true;
}
Err(error)
if matches!(
error.kind(),
io::ErrorKind::InvalidData
| io::ErrorKind::Unsupported
| io::ErrorKind::NotFound
| io::ErrorKind::InvalidInput
) =>
{
previous = None;
replaced = true;
}
Err(error) => return Err(error),
}
}
}
if !resumed {
reader = Some(DirectoryStream::open(path, scope.is_some()).await?);
}
let mut work = DirectoryWork::default();
let mut markers = [None, None];
if let Some(saved) = &previous {
if resumed && let Some(owner) = &scope {
tracing::debug!(
event = super::EVENT_SCANNER_FOLDER_STATE,
component = super::LOG_COMPONENT_SCANNER,
subsystem = super::LOG_SUBSYSTEM_FOLDER,
state = "directory_checkpoint_restored",
bucket = %owner.bucket,
directory = directory_name,
disk = %endpoint,
leader_epoch = owner.owner.0,
cycle = owner.owner.1,
restore_mode = if saved.sealed { "sealed_membership" } else { "native_position" },
entries_read = saved.entries_read,
completed_through = saved.aggregate.as_ref().map(|aggregate| aggregate.through.as_str()),
"Restored directory checkpoint"
);
}
work.names = NamePages::restore(&saved.pages, saved.sealed, saved.aggregate.as_ref().map(|a| a.through.as_str()));
if let Some(aggregate) = &saved.aggregate {
work.compacted = true;
work.compacted_total = aggregate.value.clone();
}
let physical_path: Arc<Path> = Arc::from(Path::new(path));
if let Some(part) = &saved.part {
markers[0] = Some(native_io::DirEntry::recorded(physical_path.clone(), part.into(), None));
}
if saved.metadata {
markers[1] = Some(native_io::DirEntry::recorded(physical_path, super::STORAGE_FORMAT_FILE.into(), None));
}
}
Box::new(Listing {
reader,
pending: None,
delivery: None,
classified: None,
markers,
abandoned: None,
work,
failed: false,
})
};
if let (Some(slot), Some(durable_scope)) = (saved, durable_scope) {
if let Some(recorder) = &mut listing.work.recorder {
recorder.slot = slot;
} else {
let recorder = Recorder::new(slot, directory_name, durable_scope, endpoint, physical, previous.as_ref());
listing.work.recorder = Some(recorder);
}
}
Ok(Self {
path: Arc::from(Path::new(path)),
scope: scope.zip(stamp),
directory_name: Arc::from(directory_name),
budget,
listing: Some(listing),
marker: 0,
enumeration_only: false,
resumed,
directory_replaced: replaced,
batch_entries: 0,
})
}
pub(super) fn continues_stream(&self) -> bool {
self.resumed
}
pub(super) fn indexed(&self) -> bool {
self.scope.is_some()
}
pub(super) async fn verify_current(&mut self) -> io::Result<()> {
let Some((_, original)) = &self.scope else { return Ok(()) };
let path = self.path.to_str().ok_or(io::ErrorKind::InvalidInput)?;
let current = DirectoryStamp::at(path).await?;
if *original == current {
return Ok(());
}
if let Some(listing) = &mut self.listing {
listing.failed = true;
}
Err(io::Error::new(io::ErrorKind::Interrupted, "scanner directory changed during traversal"))
}
pub(super) fn folder_count(&self) -> Option<usize> {
self.indexed()
.then(|| self.listing.as_ref().map_or(0, |l| l.work.names.len()))
}
pub(super) fn finished_listing(&self) -> bool {
self.listing.as_ref().is_some_and(|l| l.work.names.ready())
}
pub(super) fn work(&mut self) -> Option<&mut DirectoryWork> {
if !self.indexed() {
return None;
}
self.listing.as_mut().map(|listing| &mut listing.work)
}
pub(super) fn resolve_abandoned(&mut self, path: &str) {
if let Some(abandoned) = self.listing.as_mut().and_then(|l| l.abandoned.as_mut()) {
abandoned.remove(path);
}
}
pub(super) fn hold_abandoned(&mut self, baseline: &mut super::DataUsageHashMap) {
if self.indexed()
&& let Some(listing) = &mut self.listing
&& listing.abandoned.is_none()
{
if !baseline.is_empty() && self.resumed {
// Cold restore carries classified members, not the previous
// process's transient unresolved heal candidates.
for page in listing.work.names.snapshot() {
for name in page.names() {
baseline.remove(&super::path_join_buf(&[&self.directory_name, name]));
}
}
}
listing.abandoned = Some(std::mem::take(baseline));
}
}
pub(super) fn restore_abandoned(&self, baseline: &mut super::DataUsageHashMap) {
if let Some(abandoned) = self.listing.as_ref().and_then(|listing| listing.abandoned.as_ref()) {
baseline.clone_from(abandoned);
}
}
pub(super) async fn peek_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
if let Some((entry, enumeration_only)) = self.listing.as_ref().and_then(|listing| listing.delivery.as_ref()) {
self.enumeration_only = *enumeration_only;
return Ok(Some(entry.clone()));
}
let entry = self.next_entry().await?;
if let (Some(listing), Some(entry)) = (&mut self.listing, &entry) {
listing.delivery = Some((entry.clone(), self.enumeration_only));
}
Ok(entry)
}
pub(super) fn accept_entry(&mut self) {
let Some(listing) = &mut self.listing else { return };
let Some((entry, _)) = listing.delivery.take() else { return };
// Discovering a name can precede the I/O admission wait. Commit its
// classification and native position together only after admission;
// a cold retry then re-reads any name still pending on cancellation.
let Some(kind) = listing.classified.take() else { return };
let name = entry.file_name();
let text = name.to_string_lossy();
if let Some(state) = &mut listing.work.recorder {
state.anchor = entry.anchor().cloned();
state.entries_read = state.entries_read.saturating_add(1);
state.valid_names &= name.to_str().is_some_and(valid_component);
}
if let Some(kind) = kind {
let direct_part = kind.is_file() && super::is_erasure_part_name(&text);
if kind.is_dir() {
if let Some(abandoned) = &mut listing.abandoned
&& !abandoned.is_empty()
{
abandoned.remove(&super::path_join_buf(&[&self.directory_name, &text]));
}
listing.work.names.add(text.into_owned());
} else if direct_part {
if let Some(state) = &mut listing.work.recorder {
state.part.get_or_insert_with(|| text.to_string());
}
listing.markers[0].get_or_insert_with(|| native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
} else if text == super::STORAGE_FORMAT_FILE {
if let Some(state) = &mut listing.work.recorder {
state.metadata = true;
}
listing.markers[1] = Some(native_io::DirEntry::recorded(self.path.clone(), name, Some(kind)));
}
}
listing.work.maybe_checkpoint();
}
async fn next_entry(&mut self) -> io::Result<Option<native_io::DirEntry>> {
let erasure_candidate = self
.scope
.as_ref()
.is_some_and(|(scope, _)| scope.mode != FolderScanMode::Namespace);
let indexed = self.indexed();
let listing = self
.listing
.as_mut()
.ok_or_else(|| io::Error::other("scanner directory already returned"))?;
self.enumeration_only = false;
if let Some(reader) = listing.reader.as_mut() {
if !indexed {
return reader.next_entry().await;
}
listing.work.names.flush_full().await?;
if listing.pending.is_none() {
listing.pending = match reader.next_entry().await {
Ok(entry) => entry,
Err(error) => {
listing.failed = true;
return Err(error);
}
};
}
if let Some(entry) = &listing.pending {
let kind = match entry.file_type().await {
Ok(kind) => Some(kind),
Err(error) if matches!(error.kind(), io::ErrorKind::NotFound | io::ErrorKind::TooManyLinks) => None,
Err(error) => {
listing.failed = true;
return Err(error);
}
};
let name = entry.file_name();
let text = name.to_string_lossy();
self.enumeration_only = kind.is_none_or(|kind| {
(!kind.is_symlink() || text == super::STORAGE_FORMAT_FILE)
&& !(erasure_candidate && kind.is_file() && super::is_erasure_part_name(&text))
});
listing.classified = Some(kind);
return Ok(listing.pending.take());
}
listing.reader = None;
listing.work.names.finish();
listing.work.maybe_checkpoint();
}
while self.marker < listing.markers.len() {
let marker = self.marker;
self.marker += 1;
if let Some(entry) = &listing.markers[marker] {
return Ok(Some(entry.clone()));
}
}
Ok(None)
}
}
impl Drop for ScannerDirectory {
fn drop(&mut self) {
let Some((scope, stamp)) = self.scope.take() else { return };
let Some(listing) = self.listing.take() else { return };
if listing.failed || self.budget.reason().is_none() {
if let Some(recorder) = &listing.work.recorder {
*recorder.slot.lock().unwrap_or_else(|p| p.into_inner()) = None;
}
return;
}
listing.work.checkpoint();
let mut slots = PARKED.lock().unwrap_or_else(|p| p.into_inner());
let expired = expired_slots(&mut slots);
let key = self.path.to_path_buf();
let replaced = slots
.get(&key)
.is_some_and(|p| !p.scope.disk.ptr_eq(&scope.disk) || p.scope.owner < scope.owner)
.then(|| slots.remove(&key));
if slots.values().filter(|p| p.scope.disk.ptr_eq(&scope.disk)).count() < PARKED_DIRECTORIES_PER_DISK
&& let std::collections::hash_map::Entry::Vacant(slot) = slots.entry(key)
{
static NEXT_SERIAL: AtomicU64 = AtomicU64::new(0);
slot.insert(Parked {
scope,
parked_at: Instant::now(),
serial: NEXT_SERIAL.fetch_add(1, Ordering::Relaxed),
stamp,
listing,
});
}
drop(slots);
drop((expired, replaced));
}
}
@@ -0,0 +1,364 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use std::collections::HashMap;
use std::sync::{Arc, Mutex};
use std::time::{Instant, SystemTime};
use super::{DirectoryScope, DirectoryStamp, DirectoryWork, FolderScanMode, native_io, pages::NamePage};
use crate::data_usage_define::{
DataUsageCacheInfo, DataUsageCacheSource, DataUsageEntry, DataUsageScanIdentity, DataUsageScanProgress,
};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(super) struct Scope {
bucket: String,
source: DataUsageCacheSource,
identity: DataUsageScanIdentity,
progress: DataUsageScanProgress,
mode: u8,
prefixes: Option<[u8; 32]>,
watch: Option<[u8; 32]>,
}
impl Scope {
fn may_adopt(&self, current: &Self) -> bool {
self.bucket == current.bucket
&& self.source == current.source
&& self.identity == current.identity
&& self.mode == current.mode
&& self.prefixes == current.prefixes
&& self.progress.started_plan == current.progress.started_plan
&& ((self.progress == current.progress && self.watch == current.watch)
|| current.progress.started_plan != current.progress.requested_plan)
}
pub(super) fn new(owner: &DirectoryScope) -> Self {
let mode = match owner.mode {
FolderScanMode::Namespace => 0,
FolderScanMode::ErasureCandidate {
reuse_namespace,
parent_has_metadata,
} => 1 + u8::from(reuse_namespace) + 2 * u8::from(parent_has_metadata),
};
let prefixes = owner.prefix_scope.as_ref().map(|scope| {
let mut entries: Vec<_> = scope.selected_top_level_entries.iter().collect();
entries.sort_unstable();
let mut hash = Sha256::new();
for entry in entries {
hash.update(u64::try_from(entry.len()).unwrap_or(u64::MAX).to_le_bytes());
hash.update(entry.as_bytes());
}
hash.finalize().into()
});
Self {
bucket: owner.bucket.to_string(),
source: owner.source,
identity: owner.identity,
progress: owner.progress,
mode,
prefixes,
watch: owner.watch,
}
}
}
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
pub(crate) struct Physical {
pub(crate) endpoint: String,
device: u64,
inode: u64,
changed: (i64, i64),
modified: SystemTime,
}
impl Physical {
#[cfg(unix)]
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
Some(Self {
endpoint: endpoint.to_owned(),
device: stamp.device,
inode: stamp.inode,
changed: stamp.changed,
modified: stamp.modified,
})
}
#[cfg(windows)]
pub(super) fn of(endpoint: &str, stamp: &DirectoryStamp) -> Option<Self> {
let (device, inode, created) = stamp.physical?;
Some(Self {
endpoint: endpoint.to_owned(),
device,
inode,
changed: (created, 0),
modified: stamp.modified,
})
}
#[cfg(not(any(unix, windows)))]
pub(super) fn of(_endpoint: &str, _stamp: &DirectoryStamp) -> Option<Self> {
None
}
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub(super) struct Aggregate {
pub(super) through: String,
pub(super) value: DataUsageEntry,
}
/// Immutable directory membership and the last fully counted compacted child.
/// The active child's provisional contribution is never part of this receipt.
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct DirectoryCheckpoint {
version: u8,
pub(super) parent: String,
scope: Scope,
pub(super) physical: Option<Physical>,
pub(super) pages: Vec<NamePage>,
pub(super) anchor: Option<native_io::DirectoryAnchor>,
pub(super) entries_read: u64,
pub(super) sealed: bool,
pub(super) part: Option<String>,
pub(super) metadata: bool,
pub(super) aggregate: Option<Aggregate>,
endpoint: String,
digest: [u8; 32],
}
/// Physical progress orders only callbacks from the same directory generation.
/// The coordinator uses these receipts for fair scheduling, never for coverage.
#[derive(Clone, Debug)]
pub(crate) struct DirectoryReadProgress {
pub(crate) parent: String,
pub(crate) physical: Physical,
position: (u64, bool),
}
impl DirectoryReadProgress {
pub(crate) fn same_source(&self, other: &Self) -> bool {
self.parent == other.parent && self.physical == other.physical
}
pub(crate) fn extends(&self, older: &Self) -> bool {
self.same_source(older) && self.position >= older.position
}
}
#[derive(Clone, Debug)]
pub(crate) struct DirectoryProgress {
pub(crate) parent: String,
pub(crate) physical: Physical,
manifest: [u8; 32],
through: String,
}
impl DirectoryProgress {
pub(crate) fn same_source(&self, previous: &Self) -> bool {
self.parent == previous.parent && self.physical == previous.physical && self.manifest == previous.manifest
}
pub(crate) fn extends(&self, previous: &Self) -> bool {
self.same_source(previous) && self.through >= previous.through
}
pub(crate) fn covers(&self, path: &str) -> bool {
path.strip_prefix(&self.parent)
.and_then(|tail| tail.strip_prefix('/'))
.and_then(|tail| tail.split('/').next())
.is_some_and(|child| child <= self.through.as_str())
}
}
impl DirectoryCheckpoint {
/// Deserialize each replica's independent receipt, then intern equal
/// verified immutable name pages across the bucket's active directories.
pub(crate) fn verified_work(work: Vec<Self>, info: &DataUsageCacheInfo) -> Vec<Self> {
let mut shared = HashMap::new();
work.into_iter()
.filter(|receipt| receipt.is_bound_to(info))
.map(|mut receipt| {
for page in &mut receipt.pages {
let canonical = shared.entry(*page.digest()).or_insert_with(|| page.clone());
*page = canonical.clone();
}
receipt
})
.collect()
}
pub(crate) fn parent(&self) -> &str {
&self.parent
}
pub(super) fn manifest(&self) -> [u8; 32] {
let mut hash = Sha256::new();
for p in &self.pages {
hash.update(p.digest());
}
hash.finalize().into()
}
fn checksum(&self) -> Option<[u8; 32]> {
// Nested usage distributions can contain maps. Canonicalize those
// map keys before hashing so restart hash randomization is irrelevant.
let mut aggregate = serde_json::to_value(&self.aggregate).ok()?;
aggregate.sort_all_objects();
let header = rmp_serde::to_vec_named(&(
self.version,
&self.parent,
&self.scope,
&self.physical,
&self.anchor,
self.entries_read,
self.sealed,
&self.part,
self.metadata,
self.manifest(),
aggregate,
&self.endpoint,
))
.ok()?;
Some(Sha256::digest(header).into())
}
pub(crate) fn is_bound_to(&self, info: &DataUsageCacheInfo) -> bool {
self.version == 1
&& self.scope.bucket == info.name
&& self.scope.identity.is_valid()
&& Some(self.scope.identity) == info.scan_identity
&& Some(self.scope.source) == info.source
&& info
.scan_progress
.is_some_and(|p| p.started_plan == self.scope.progress.started_plan)
&& (self.parent == info.name
|| self
.parent
.strip_prefix(&info.name)
.and_then(|p| p.strip_prefix('/'))
.is_some_and(|tail| tail.split('/').all(super::valid_component)))
&& self.valid_payload()
}
pub(crate) fn read_progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryReadProgress> {
let physical = self.physical.as_ref()?;
(self.is_bound_to(info) && (self.sealed || self.anchor.is_some()) && self.entries_read > 0).then(|| {
DirectoryReadProgress {
parent: self.parent.clone(),
physical: physical.clone(),
position: (self.entries_read, self.sealed),
}
})
}
pub(crate) fn progress(&self, info: &DataUsageCacheInfo) -> Option<DirectoryProgress> {
let done = self.aggregate.as_ref()?;
let physical = self.physical.as_ref()?;
(self.sealed && self.is_bound_to(info)).then(|| DirectoryProgress {
parent: self.parent.clone(),
physical: physical.clone(),
manifest: self.manifest(),
through: done.through.clone(),
})
}
pub(crate) fn endpoint(&self) -> &str {
&self.endpoint
}
fn valid_for(&self, parent: &str, owner: &Scope) -> bool {
self.version == 1 && self.parent == parent && self.scope.may_adopt(owner) && self.valid_payload()
}
fn valid_payload(&self) -> bool {
self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
&& (self.sealed || self.anchor.is_some())
&& self.checksum() == Some(self.digest)
&& self.pages.iter().all(NamePage::valid)
&& self.part.as_deref().is_none_or(super::super::is_erasure_part_name)
&& self.aggregate.as_ref().is_none_or(|done| {
self.sealed
&& super::valid_component(&done.through)
&& done.value.children.is_empty()
&& done.value.failed_objects == 0
&& self.pages.iter().any(|page| page.contains(&done.through))
})
}
pub(super) fn restore_allowed(&self, parent: &str, owner: &Scope, physical: &Option<Physical>) -> bool {
self.valid_for(parent, owner)
&& self.physical.is_some()
&& &self.physical == physical
&& (self.sealed || self.anchor.is_some())
}
}
pub(in crate::scanner_folder) type Saved = Arc<Mutex<Option<DirectoryCheckpoint>>>;
pub(super) struct Recorder {
pub(super) slot: Saved,
scope: Scope,
parent: String,
physical: Option<Physical>,
pub(super) anchor: Option<native_io::DirectoryAnchor>,
pub(super) entries_read: u64,
pub(super) part: Option<String>,
pub(super) metadata: bool,
pub(super) valid_names: bool,
endpoint: String,
pub(super) last_update: Instant,
}
impl Recorder {
pub(super) fn new(
slot: Saved,
parent: &str,
scope: Scope,
endpoint: String,
physical: Option<Physical>,
previous: Option<&DirectoryCheckpoint>,
) -> Self {
Self {
slot,
scope,
parent: parent.to_owned(),
physical,
endpoint,
anchor: previous.and_then(|p| p.anchor.clone()),
entries_read: previous.map_or(0, |p| p.entries_read),
part: previous.and_then(|p| p.part.clone()),
metadata: previous.is_some_and(|p| p.metadata),
valid_names: true,
last_update: Instant::now(),
}
}
pub(super) fn save(&self, work: &DirectoryWork) {
// An ordinary object directory is cheaper to reopen. Only a reader
// which can actually resume or a closed directory can be checkpointed.
let sealed = work.names.ready();
let aggregate = if work.compacted && !work.gap {
work.names.through().map(|through| Aggregate {
through: through.to_owned(),
value: work.compacted_total.clone(),
})
} else {
None
};
let snapshot = if self.valid_names
&& self.physical.as_ref().is_some_and(|source| source.endpoint == self.endpoint)
&& (sealed || self.anchor.is_some())
{
let mut checkpoint = DirectoryCheckpoint {
version: 1,
parent: self.parent.clone(),
scope: self.scope.clone(),
physical: self.physical.clone(),
anchor: (!sealed).then(|| self.anchor.clone()).flatten(),
entries_read: self.entries_read,
pages: work.names.snapshot(),
sealed,
part: self.part.clone(),
metadata: self.metadata,
aggregate,
endpoint: self.endpoint.clone(),
digest: [0; 32],
};
checkpoint.checksum().map(|digest| {
checkpoint.digest = digest;
checkpoint
})
} else {
None
};
*self.slot.lock().unwrap_or_else(|p| p.into_inner()) = snapshot;
}
}
#[cfg(test)]
mod storage_fixture;
@@ -0,0 +1,48 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::*;
impl DirectoryCheckpoint {
pub(crate) async fn wide_storage_fixture(info: &DataUsageCacheInfo) -> [Self; 2] {
let scope = Scope {
bucket: info.name.clone(),
source: info.source.expect("fixture source"),
identity: info.scan_identity.expect("fixture identity"),
progress: info.scan_progress.expect("fixture coverage plan"),
mode: 0,
prefixes: None,
watch: None,
};
let slot = Saved::default();
let endpoint = info.scan_directory_source.clone().expect("fixture physical source");
let physical = Physical {
endpoint: endpoint.clone(),
device: 1,
inode: 2,
changed: (123, 0),
modified: SystemTime::UNIX_EPOCH,
};
let mut recorder = Recorder::new(slot.clone(), &info.name, scope, endpoint, Some(physical), None);
recorder.entries_read = 300_000;
let mut work = DirectoryWork {
compacted: true,
..Default::default()
};
for i in 0..300_000 {
work.names.add(format!("obj-{i:06}"));
work.names.flush_full().await.expect("record real immutable name page");
}
work.names.finish();
[121_034, 121_035].map(|objects| {
work.names.seek_after(Some(&format!("obj-{:06}", objects - 1)));
work.compacted_total.objects = objects;
work.compacted_total.size = objects * 1024;
recorder.save(&work);
slot.lock()
.expect("fixture slot")
.take()
.expect("sealed directory checkpoint")
})
}
}
@@ -0,0 +1,290 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use std::cmp::Reverse;
use std::collections::BinaryHeap;
use std::io;
use std::sync::{Arc, OnceLock};
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
const PAGE_NAMES: usize = 256;
const PAGE_NAME_BYTES: usize = 32 * 1024;
/// One contiguous UTF-8 string per run. Names from the filesystem cannot
/// contain NUL. Checkpoint clones share both the payload and its local index.
#[derive(Clone, Debug, Serialize, Deserialize)]
#[serde(transparent)]
pub(super) struct NamePage(Arc<PackedNames>);
#[derive(Debug, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct PackedNames {
text: Box<str>,
digest: [u8; 32],
#[serde(skip)]
starts: OnceLock<Box<[usize]>>,
#[serde(skip)]
valid: OnceLock<bool>,
}
impl NamePage {
fn hash(text: &str) -> [u8; 32] {
Sha256::new()
.chain_update(b"rustfs-scanner-directory-names-v1")
.chain_update(text.as_bytes())
.finalize()
.into()
}
fn new(mut names: Vec<String>) -> Self {
names.sort_unstable();
let mut text = String::with_capacity(names.iter().map(|n| n.len() + 1).sum());
for name in names {
text.push_str(&name);
text.push('\0');
}
Self(Arc::new(PackedNames {
digest: Self::hash(&text),
text: text.into_boxed_str(),
starts: OnceLock::new(),
valid: OnceLock::new(),
}))
}
fn starts(&self) -> &[usize] {
self.0.starts.get_or_init(|| {
self.names()
.scan(0, |offset, name| {
let start = *offset;
*offset += name.len() + 1;
Some(start)
})
.collect()
})
}
fn at(&self, offset: usize) -> &str {
self.0.text[offset..].split('\0').next().unwrap_or_default()
}
fn get(&self, index: usize) -> Option<&str> {
self.starts().get(index).map(|&offset| self.at(offset))
}
fn after(&self, last: &str) -> usize {
self.starts().partition_point(|&offset| self.at(offset) <= last)
}
pub(super) fn contains(&self, name: &str) -> bool {
self.starts().binary_search_by(|&offset| self.at(offset).cmp(name)).is_ok()
}
pub(super) fn digest(&self) -> &[u8; 32] {
&self.0.digest
}
pub(super) fn names(&self) -> impl Iterator<Item = &str> {
self.0.text.split_terminator('\0')
}
pub(super) fn valid(&self) -> bool {
*self.0.valid.get_or_init(|| {
let text = &self.0.text;
text.ends_with('\0')
&& text.len() <= PAGE_NAME_BYTES + PAGE_NAMES + 4096
&& self.names().take(PAGE_NAMES + 1).count() <= PAGE_NAMES
&& self.names().all(super::valid_component)
&& self.names().is_sorted()
&& self.0.digest == Self::hash(text)
})
}
}
/// Immutable sorted runs are shared with checkpoint snapshots. The merge
/// retains one head per run. At a cold resume each run seeks to the first
/// name after the authenticated completed child using binary search.
#[derive(Default)]
pub(super) struct NamePages {
pages: Vec<NamePage>,
building: Vec<String>,
name_bytes: usize,
count: usize,
heads: BinaryHeap<Reverse<(String, usize, usize)>>,
ready: bool,
pending: Option<String>,
through: Option<String>,
}
impl NamePages {
pub(super) fn len(&self) -> usize {
self.count
}
pub(super) fn ready(&self) -> bool {
self.ready
}
pub(super) fn through(&self) -> Option<&str> {
self.through.as_deref()
}
pub(super) fn add(&mut self, name: String) {
self.name_bytes = self.name_bytes.saturating_add(name.len());
self.count = self.count.saturating_add(1);
self.building.push(name);
}
fn flush(&mut self) {
if self.building.is_empty() {
return;
}
let page = NamePage::new(std::mem::take(&mut self.building));
if let Some(first) = page.get(0) {
self.heads.push(Reverse((first.to_owned(), self.pages.len(), 0)));
}
self.pages.push(page);
self.name_bytes = 0;
}
pub(super) async fn flush_full(&mut self) -> io::Result<()> {
if self.building.len() >= PAGE_NAMES || self.name_bytes >= PAGE_NAME_BYTES {
self.flush();
}
Ok(())
}
pub(super) fn finish(&mut self) {
self.flush();
self.ready = true;
}
pub(super) fn snapshot(&self) -> Vec<NamePage> {
let mut pages = self.pages.clone();
if !self.building.is_empty() {
pages.push(NamePage::new(self.building.clone()));
}
pages
}
pub(super) fn restore(pages: &[NamePage], ready: bool, through: Option<&str>) -> Self {
let mut names = Self {
pages: pages.to_vec(),
count: pages.iter().map(|p| p.starts().len()).sum(),
ready,
..Self::default()
};
if !ready
&& names
.pages
.last()
.is_some_and(|page| page.starts().len() < PAGE_NAMES && page.0.text.len() < PAGE_NAME_BYTES)
&& let Some(page) = names.pages.pop()
{
names.building = page.names().map(str::to_owned).collect();
names.name_bytes = names.building.iter().map(String::len).sum();
}
names.seek_after(through);
names
}
pub(super) fn seek_after(&mut self, through: Option<&str>) {
self.pending = None;
self.through = through.map(str::to_owned);
self.heads.clear();
for (page_index, page) in self.pages.iter().enumerate() {
let start = through.map_or(0, |last| page.after(last));
if let Some(next) = page.get(start) {
self.heads.push(Reverse((next.to_owned(), page_index, start)));
}
}
}
pub(super) fn rewind(&mut self) {
self.seek_after(None);
}
pub(super) async fn next(&mut self) -> io::Result<Option<String>> {
if self.pending.is_some() {
return Ok(self.pending.clone());
}
while let Some(Reverse((name, page, index))) = self.heads.pop() {
if let Some(next) = self.pages[page].get(index + 1) {
self.heads.push(Reverse((next.to_owned(), page, index + 1)));
}
if self.through.as_ref() == Some(&name) {
continue;
}
self.pending = Some(name.clone());
return Ok(Some(name));
}
Ok(None)
}
pub(super) fn acknowledge(&mut self) -> bool {
if let Some(name) = self.pending.take() {
self.through = Some(name);
return true;
}
false
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn packed_membership_shares_names_and_rejects_corrupt_payloads() {
let page = NamePage::new((0..PAGE_NAMES).rev().map(|i| format!("obj-{i:06}")).collect());
assert!(page.valid());
let shared = page.clone();
assert!(Arc::ptr_eq(&page.0, &shared.0));
let encoded = rmp_serde::to_vec_named(&page).expect("serialize packed names");
assert!(encoded.len() < PAGE_NAMES * 12, "checkpoint stores one delimited string per page");
let restored: NamePage = rmp_serde::from_slice(&encoded).expect("cold name page");
assert!(restored.valid());
assert!(restored.contains("obj-000137"));
assert_eq!(restored.get(restored.after("obj-000137")), Some("obj-000138"));
for text in ["", "obj", "a\0\0b\0", "b\0a\0", "../other\0"] {
let malformed = NamePage(Arc::new(PackedNames {
text: text.into(),
digest: NamePage::hash(text),
starts: OnceLock::new(),
valid: OnceLock::new(),
}));
assert!(!malformed.valid(), "invalid membership accepted: {text:?}");
}
let mut bytes = encoded;
let offset = bytes
.windows(10)
.position(|value| value == b"obj-000137")
.expect("encoded name");
bytes[offset] = b'x';
assert!(!rmp_serde::from_slice::<NamePage>(&bytes).expect("decodable payload").valid());
}
#[tokio::test]
async fn interleaved_pages_resume_at_each_unacknowledged_name() {
let mut pages = NamePages::default();
let mut expected = std::collections::BTreeSet::new();
// Each consecutive sorted run interleaves with every other run. One
// duplicate straddles two runs and a partial run remains in memory.
for run in 0..5 {
for index in (0..PAGE_NAMES).rev() {
let name = format!("item-{index:04}-{run:02}");
expected.insert(name.clone());
pages.add(name);
pages.flush_full().await.expect("append sorted run");
}
}
pages.add("item-0000-02".to_owned());
pages.finish();
for name in &expected {
assert_eq!(pages.next().await.expect("scheduled child").as_ref(), Some(name));
assert_eq!(pages.next().await.expect("retry after cancellation").as_ref(), Some(name));
assert!(pages.acknowledge());
}
assert_eq!(pages.next().await.expect("sealed EOF"), None);
pages.rewind();
assert_eq!(pages.next().await.expect("fresh coverage").as_ref(), expected.first());
}
}
+223 -20
View File
@@ -8,7 +8,7 @@ use std::ffi::OsString;
use std::fs::{FileType, Metadata};
use std::future::{Future, poll_fn};
use std::io;
use std::path::PathBuf;
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll, ready};
@@ -18,31 +18,156 @@ use crate::storage_api::ecstore_spawn_blocking_with_resource_ownership as spawn_
// Keep Tokio's existing bounded prefetch and one native job per refill.
const DIRECTORY_CHUNK_SIZE: usize = 32;
#[cfg(any(target_os = "linux", target_os = "macos"))]
mod positioned;
/// Identifies a verified prefix of a physical directory buffer. Only the
/// matching directory generation can use its platform-specific kernel cookie.
#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub(crate) struct DirectoryAnchor {
directory: [u8; 32],
cookie: u64,
entries: u32,
prefix: [u8; 32],
}
#[derive(Clone)]
pub(super) struct DirEntry {
entry: Arc<std::fs::DirEntry>,
source: EntrySource,
file_type: Option<FileType>,
}
#[derive(Clone)]
enum EntrySource {
Native(Arc<std::fs::DirEntry>),
#[cfg(any(target_os = "linux", target_os = "macos"))]
Positioned(Arc<PositionedEntry>),
Recorded {
parent: Arc<std::path::Path>,
name: OsString,
},
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[derive(Clone)]
struct PositionedEntry {
parent: Arc<Path>,
name: OsString,
anchor: Option<DirectoryAnchor>,
}
impl DirEntry {
pub(super) fn recorded(parent: Arc<std::path::Path>, name: OsString, file_type: Option<FileType>) -> Self {
Self {
source: EntrySource::Recorded { parent, name },
file_type,
}
}
pub(super) fn anchor(&self) -> Option<&DirectoryAnchor> {
match &self.source {
#[cfg(any(target_os = "linux", target_os = "macos"))]
EntrySource::Positioned(entry) => entry.anchor.as_ref(),
_ => None,
}
}
#[cfg(test)]
pub(super) fn without_anchor(self) -> Self {
#[cfg(any(target_os = "linux", target_os = "macos"))]
{
let mut entry = self;
if let EntrySource::Positioned(position) = &mut entry.source {
Arc::make_mut(position).anchor = None;
}
entry
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
self
}
pub(super) fn is_recorded(&self) -> bool {
matches!(&self.source, EntrySource::Recorded { .. })
}
pub(super) fn cached_type(&self) -> Option<FileType> {
self.file_type
}
pub(super) fn path(&self) -> PathBuf {
self.entry.path()
match &self.source {
EntrySource::Native(entry) => entry.path(),
#[cfg(any(target_os = "linux", target_os = "macos"))]
EntrySource::Positioned(entry) => entry.parent.join(&entry.name),
EntrySource::Recorded { parent, name } => parent.join(name),
}
}
pub(super) fn file_name(&self) -> OsString {
self.entry.file_name()
match &self.source {
EntrySource::Native(entry) => entry.file_name(),
#[cfg(any(target_os = "linux", target_os = "macos"))]
EntrySource::Positioned(entry) => entry.name.clone(),
EntrySource::Recorded { name, .. } => name.clone(),
}
}
pub(super) async fn file_type(&self) -> io::Result<FileType> {
if let Some(file_type) = self.file_type {
return Ok(file_type);
}
let entry = self.entry.clone();
spawn_native(move || entry.file_type()).await.map_err(io::Error::other)?
match &self.source {
EntrySource::Native(entry) => {
let entry = entry.clone();
spawn_native(move || entry.file_type()).await.map_err(io::Error::other)?
}
_ => {
let path = self.path();
spawn_native(move || std::fs::symlink_metadata(path).map(|meta| meta.file_type()))
.await
.map_err(io::Error::other)?
}
}
}
}
enum PhysicalReader {
Standard(std::fs::ReadDir),
#[cfg(any(target_os = "linux", target_os = "macos"))]
Positioned(Box<positioned::Reader>),
}
impl PhysicalReader {
fn open(path: &Path, indexed: bool, anchor: Option<&DirectoryAnchor>) -> io::Result<Self> {
#[cfg(any(target_os = "linux", target_os = "macos"))]
if indexed {
return positioned::Reader::open(path, anchor).map(Box::new).map(Self::Positioned);
}
#[cfg(not(any(target_os = "linux", target_os = "macos")))]
let _ = indexed;
if anchor.is_some() {
return Err(io::ErrorKind::Unsupported.into());
}
std::fs::read_dir(path).map(Self::Standard)
}
fn next(&mut self) -> Option<io::Result<DirEntry>> {
match self {
Self::Standard(reader) => reader.next().map(|entry| {
entry.map(|entry| DirEntry {
file_type: entry.file_type().ok(),
source: EntrySource::Native(Arc::new(entry)),
})
}),
#[cfg(any(target_os = "linux", target_os = "macos"))]
Self::Positioned(reader) => reader.next(),
}
}
}
struct DirectoryBuffer {
reader: std::fs::ReadDir,
reader: PhysicalReader,
entries: VecDeque<io::Result<DirEntry>>,
more: bool,
}
@@ -55,10 +180,7 @@ impl DirectoryBuffer {
break;
};
let success = entry.is_ok();
self.entries.push_back(entry.map(|entry| DirEntry {
file_type: entry.file_type().ok(),
entry: Arc::new(entry),
}));
self.entries.push_back(entry);
if !success {
break;
}
@@ -74,14 +196,26 @@ enum DirectoryState {
pub(super) struct ReadDir {
state: DirectoryState,
#[cfg(test)]
slow_directory: bool,
}
impl ReadDir {
pub(super) async fn open(path: &str) -> io::Result<Self> {
pub(super) async fn open(path: &str, indexed: bool) -> io::Result<Self> {
Self::open_directory(path, indexed, None).await
}
pub(super) async fn open_at(path: &str, anchor: &DirectoryAnchor) -> io::Result<Self> {
Self::open_directory(path, true, Some(anchor.clone())).await
}
async fn open_directory(path: &str, indexed: bool, anchor: Option<DirectoryAnchor>) -> io::Result<Self> {
#[cfg(test)]
let slow_directory = super::tests::fair_prefix_quantum::is_slow_directory(path);
let path = PathBuf::from(path);
let buffer = spawn_native(move || {
let mut buffer = DirectoryBuffer {
reader: std::fs::read_dir(path)?,
reader: PhysicalReader::open(&path, indexed, anchor.as_ref())?,
entries: VecDeque::with_capacity(DIRECTORY_CHUNK_SIZE),
more: true,
};
@@ -92,12 +226,18 @@ impl ReadDir {
.map_err(io::Error::other)??;
Ok(Self {
state: DirectoryState::Idle(Some(buffer)),
#[cfg(test)]
slow_directory,
})
}
pub(super) async fn next_entry(&mut self) -> io::Result<Option<DirEntry>> {
// Keep a pending refill on the reader, not in the borrowed next-entry
// future. Cancelling that future must not skip its prefetched entries.
#[cfg(test)]
if self.slow_directory {
super::tests::fair_prefix_quantum::charge_physical_read().await;
}
poll_fn(|cx| self.poll_next_entry(cx)).await
}
@@ -201,7 +341,7 @@ mod tests {
Resources::Layered(owner.capture(&units.iter().collect::<Vec<_>>()).expect("native ownership"))
};
drop(units);
let mut reader = ReadDir::open(path).await.expect("prepare reader");
let mut reader = ReadDir::open(path, false).await.expect("prepare reader");
let mut entry = reader.next_entry().await.expect("first entry").expect("file");
entry.file_type = None;
for _ in 1..DIRECTORY_CHUNK_SIZE {
@@ -210,7 +350,7 @@ mod tests {
let (release, blocker) = occupy_native_worker().await;
let work = async {
match operation {
"open" => ReadDir::open(path).await.map(|_| ()),
"open" => ReadDir::open(path, false).await.map(|_| ()),
"refill" => reader.next_entry().await.map(|_| ()),
"metadata" => metadata(entry.path().to_str().expect("entry path")).await.map(|_| ()),
"file_type" => entry.file_type().await.map(|_| ()),
@@ -253,7 +393,9 @@ mod tests {
runtime().block_on(async {
let count = DIRECTORY_CHUNK_SIZE * 3 + 7;
let root = directory(count);
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
.await
.expect("open");
let mut names = std::collections::BTreeSet::new();
for _ in 0..DIRECTORY_CHUNK_SIZE {
let entry = reader.next_entry().await.expect("entry").expect("file");
@@ -273,12 +415,71 @@ mod tests {
});
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[test]
fn reopened_directory_matches_original_suffix_and_rejects_stale_or_corrupt_positions() {
runtime().block_on(async {
#[cfg(target_os = "linux")]
use std::os::unix::ffi::OsStrExt;
let root = directory(900);
let path = root.path().to_str().expect("root path");
#[cfg(target_os = "linux")]
let raw_name = std::ffi::OsStr::from_bytes(b"object-\xff");
// APFS requires filenames that are valid Unicode.
#[cfg(target_os = "macos")]
let raw_name = std::ffi::OsStr::new("object-目录");
std::fs::write(root.path().join(raw_name), []).expect("platform-native directory entry");
let mut reader = ReadDir::open(path, true).await.expect("open indexed directory");
let mut names = Vec::new();
let mut anchors = Vec::new();
while let Some(entry) = reader.next_entry().await.expect("read indexed entry") {
assert!(!entry.is_recorded());
names.push(entry.file_name());
anchors.push(entry.anchor().expect("native directory position").clone());
}
drop(reader);
assert!(names.iter().any(|name| name == raw_name));
assert!(anchors.iter().any(|anchor| anchor.cookie != 0), "exercise a later kernel buffer");
for i in [0, 32, 145, 419, names.len() - 1] {
let mut resumed = ReadDir::open_at(path, &anchors[i])
.await
.expect("reopen directory at verified entry");
for expected in &names[i + 1..] {
assert_eq!(
&resumed
.next_entry()
.await
.expect("suffix read")
.expect("suffix entry")
.file_name(),
expected
);
}
assert!(resumed.next_entry().await.expect("resumed EOF").is_none());
}
let mut corrupt = anchors[0].clone();
corrupt.prefix[0] ^= 1;
assert!(matches!(ReadDir::open_at(path, &corrupt).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
corrupt.entries = u32::MAX;
assert!(matches!(ReadDir::open_at(path, &corrupt).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
let other = directory(1);
let another = other.path().to_str().expect("other directory");
assert!(matches!(ReadDir::open_at(another, &anchors[0]).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
// Removing the bound entry also invalidates its directory generation.
std::fs::remove_file(root.path().join(&names[0])).expect("remove anchor entry");
assert!(matches!(ReadDir::open_at(path, &anchors[0]).await, Err(err) if err.kind() == io::ErrorKind::InvalidData));
});
}
#[test]
fn directory_prefetch_preserves_empty_and_exact_chunk_boundaries() {
runtime().block_on(async {
for count in [0, 1, DIRECTORY_CHUNK_SIZE, DIRECTORY_CHUNK_SIZE + 1] {
let root = directory(count);
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
.await
.expect("open");
let mut names = std::collections::BTreeSet::new();
while let Some(entry) = reader.next_entry().await.expect("read entry") {
assert!(names.insert(entry.file_name()), "prefetch duplicated an entry");
@@ -293,7 +494,9 @@ mod tests {
fn failed_refill_is_an_error_after_partial_enumeration() {
runtime().block_on(async {
let root = directory(DIRECTORY_CHUNK_SIZE + 1);
let mut reader = ReadDir::open(root.path().to_str().expect("root path")).await.expect("open");
let mut reader = ReadDir::open(root.path().to_str().expect("root path"), false)
.await
.expect("open");
for _ in 0..DIRECTORY_CHUNK_SIZE {
assert!(reader.next_entry().await.expect("prefetched entry").is_some());
}
@@ -322,10 +525,10 @@ mod tests {
let root = directory(1);
let path = root.path().join("missing");
let missing = path.to_str().expect("missing path");
assert!(matches!(ReadDir::open(missing).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
assert!(matches!(ReadDir::open(missing, false).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
assert!(matches!(metadata(missing).await, Err(error) if error.kind() == io::ErrorKind::NotFound));
let file = root.path().join("object-0");
assert!(ReadDir::open(file.to_str().expect("file path")).await.is_err());
assert!(ReadDir::open(file.to_str().expect("file path"), false).await.is_err());
#[cfg(unix)]
{
let link = root.path().join("link");
@@ -0,0 +1,209 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
//! Reopen an indexed directory at the last entry committed by the walker.
use std::ffi::OsStr;
use std::fs::{File, FileType, Metadata};
use std::io;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::fs::MetadataExt;
use std::path::Path;
use std::sync::Arc;
use rustix::fs::{Dir, FileType as NativeType, Mode, OFlags, SeekFrom};
use sha2::{Digest, Sha256};
use super::{DirEntry, DirectoryAnchor, EntrySource, PositionedEntry};
// Darwin's dirent does not expose Linux d_off. Use the kernel buffer's start
// position and verify the consumed prefix of that buffer on a fresh handle.
#[cfg(target_os = "macos")]
const MAX_ANCHOR_PREFIX: u32 = 4096;
#[cfg(target_os = "linux")]
const MAX_ANCHOR_PREFIX: u32 = 1;
fn changed() -> io::Error {
io::Error::new(io::ErrorKind::InvalidData, "scanner directory position no longer matches its checkpoint")
}
fn resume_error(error: rustix::io::Errno) -> io::Error {
use rustix::io::Errno;
match error {
Errno::SPIPE | Errno::OPNOTSUPP => io::ErrorKind::Unsupported.into(),
Errno::INVAL | Errno::OVERFLOW | Errno::STALE => changed(),
other => other.into(),
}
}
fn hash_record(prefix: &mut Sha256, record: &rustix::fs::DirEntry) {
prefix.update(record.ino().to_le_bytes());
// Names cannot contain NUL. Include the terminator to delimit records.
prefix.update(record.file_name().to_bytes_with_nul());
}
fn identity(meta: &Metadata) -> [u8; 32] {
let mut digest = Sha256::new();
digest.update(std::env::consts::OS.as_bytes());
for value in [
meta.dev().to_le_bytes(),
meta.ino().to_le_bytes(),
meta.ctime().to_le_bytes(),
meta.ctime_nsec().to_le_bytes(),
meta.mtime().to_le_bytes(),
meta.mtime_nsec().to_le_bytes(),
] {
digest.update(value);
}
digest.finalize().into()
}
pub(super) struct Reader {
dir: Dir,
parent: Arc<Path>,
directory_type: FileType,
regular_type: Option<FileType>,
link_type: Option<FileType>,
identity: [u8; 32],
next_cookie: Option<u64>,
#[cfg(target_os = "macos")]
page: DarwinPage,
}
#[cfg(target_os = "macos")]
struct DarwinPage {
position: File,
start: Option<u64>,
count: u32,
prefix: Sha256,
}
impl Reader {
pub(super) fn open(path: &Path, resume: Option<&DirectoryAnchor>) -> io::Result<Self> {
let fd = File::from(rustix::fs::open(
path,
OFlags::RDONLY | OFlags::DIRECTORY | OFlags::CLOEXEC | OFlags::NOFOLLOW,
Mode::empty(),
)?);
let meta = fd.metadata()?;
let identity = identity(&meta);
let cookie = resume.map_or(0, |anchor| anchor.cookie);
if let Some(anchor) = resume {
if anchor.directory != identity || !(1..=MAX_ANCHOR_PREFIX).contains(&anchor.entries) {
return Err(changed());
}
if rustix::fs::seek(&fd, SeekFrom::Start(cookie)).map_err(resume_error)? != cookie {
return Err(changed());
}
}
#[cfg(target_os = "macos")]
let page = DarwinPage {
position: fd.try_clone()?,
start: Some(cookie),
count: 0,
prefix: Sha256::new(),
};
let mut reader = Self {
dir: Dir::new(fd)?,
parent: Arc::from(path),
directory_type: meta.file_type(),
regular_type: None,
link_type: None,
identity,
next_cookie: Some(cookie),
#[cfg(target_os = "macos")]
page,
};
if let Some(expected) = resume {
// A fresh libc stream can choose a different buffer size. Validate
// exactly the saved prefix, regardless of its new buffer boundaries.
let mut prefix = Sha256::new();
for _ in 0..expected.entries {
let record = reader.dir.read().transpose().map_err(resume_error)?.ok_or_else(changed)?;
hash_record(&mut prefix, &record);
reader.anchor(&record);
}
if prefix.finalize()[..] != expected.prefix {
return Err(changed());
}
}
Ok(reader)
}
fn anchor(&mut self, record: &rustix::fs::DirEntry) -> Option<DirectoryAnchor> {
#[cfg(target_os = "linux")]
let mut single = Sha256::new();
#[cfg(target_os = "linux")]
let (cookie, entries, prefix) = {
let cookie = self.next_cookie;
self.next_cookie = u64::try_from(record.offset()).ok();
(cookie?, 1, &mut single)
};
#[cfg(target_os = "macos")]
let (cookie, entries, prefix) = {
let after = rustix::fs::tell(&self.page.position).ok();
if after != self.next_cookie {
self.page.start = self.next_cookie;
self.next_cookie = after;
self.page.count = 0;
self.page.prefix = Sha256::new();
}
self.page.count = self.page.count.saturating_add(1);
if self.page.count > MAX_ANCHOR_PREFIX {
return None;
}
(self.page.start?, self.page.count, &mut self.page.prefix)
};
hash_record(prefix, record);
Some(DirectoryAnchor {
directory: self.identity,
cookie,
entries,
prefix: prefix.clone().finalize().into(),
})
}
fn file_type(&mut self, kind: NativeType, name: &OsStr) -> Option<FileType> {
// std::fs::FileType has no public constructor. Reuse the observed type
// only when the kernel dirent identified the same class. DT_UNKNOWN
// and special files retain the per-entry symlink_metadata fallback.
let cached = match kind {
NativeType::Directory => return Some(self.directory_type),
NativeType::RegularFile => &mut self.regular_type,
NativeType::Symlink => &mut self.link_type,
_ => return None,
};
if cached.is_none() {
let ty = std::fs::symlink_metadata(self.parent.join(name)).ok()?.file_type();
if (ty.is_file() && kind == NativeType::RegularFile) || (ty.is_symlink() && kind == NativeType::Symlink) {
*cached = Some(ty);
}
return Some(ty);
}
*cached
}
pub(super) fn next(&mut self) -> Option<io::Result<DirEntry>> {
loop {
let raw = match self.dir.read()? {
Ok(record) => record,
Err(error) => return Some(Err(error.into())),
};
let anchor = self.anchor(&raw);
let bytes = raw.file_name().to_bytes();
if matches!(bytes, b"." | b"..") {
continue;
}
let name = OsStr::from_bytes(bytes);
let file_type = self.file_type(raw.file_type(), name);
return Some(Ok(DirEntry {
source: EntrySource::Positioned(Arc::new(PositionedEntry {
parent: self.parent.clone(),
name: name.into(),
anchor,
})),
file_type,
}));
}
}
}
+8 -1
View File
@@ -30,7 +30,7 @@ use std::sync::Mutex;
pub(super) mod checkpoint_fixture;
pub(super) mod enumeration_restart;
mod fair_prefix_quantum;
pub(super) mod fair_prefix_quantum;
mod incremental_enumeration;
mod listing_completion;
mod metadata_visits;
@@ -403,6 +403,7 @@ async fn build_test_scanner() -> (FolderScanner, std::path::PathBuf) {
resume_failed_paths: Vec::new(),
coverage_gap: false,
raw_enumeration_progress: Vec::new(),
directory_checkpoints: HashMap::new(),
pending_size_reconciliation_keys: HashSet::new(),
pending_size_reconciliation_scopes: HashSet::new(),
pending_size_reconciliation_truncated: false,
@@ -3144,6 +3145,12 @@ async fn scoped_root_scan_reuses_clean_top_level_entries_and_rescans_dirty_entri
parent: None,
object_heal_prob_div: 1,
};
// The sorted queue can reach a budget stop after linking a clean prefix
// but before the first dirty child advances the durable sweep frontier.
assert!(!scanner.can_reuse_covered_namespace("bucket/clean", false));
assert!(scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/clean"));
assert!(!scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/dirty"));
assert!(!scanner.can_restore_linked_child(&folder, &DataUsageEntry::default(), "bucket/absent"));
let mut root = DataUsageEntry::default();
scanner
.scan_folder(CancellationToken::new(), folder, &mut root)
@@ -1129,7 +1129,7 @@ async fn checkpoint_admission_log_capture_survives_another_future_dispatcher() {
}
}
async fn write_checkpoint_object(root: &std::path::Path, object: &str, versions: &[(Option<Uuid>, i64)]) {
pub(super) async fn write_checkpoint_object(root: &std::path::Path, object: &str, versions: &[(Option<Uuid>, i64)]) {
let mut metadata = FileMeta::new();
for (index, (version_id, size)) in versions.iter().enumerate() {
let mut info = FileInfo::new(object, 4, 2);
@@ -820,6 +820,11 @@ async fn periodic_nested_snapshot_adoption_records_checkpoint_used_on_real_scan_
.await
.expect("create the real cached subtree directory");
let (mut cache, _) = bound_checkpoint();
cache.info.scan_directory_source = Some(
directory_cursor::physical_source(&scanner.local_disk)
.await
.expect("fixture disk source"),
);
cache.info.skip_healing = true;
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(
@@ -856,6 +861,80 @@ async fn periodic_nested_snapshot_adoption_records_checkpoint_used_on_real_scan_
assert_eq!(budget.reason(), None);
}
#[tokio::test]
#[serial]
async fn periodic_compacted_checkpoint_survives_a_crash_inside_the_first_quantum() {
use super::super::{enumeration_restart::seed_flat_bucket, fair_prefix_quantum::run_until_finished};
let (mut scanner, root) = build_test_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
};
// seed_flat_bucket creates its directory; retain the allocated native
// disk and initialize this already-created bucket in the same shape.
let template = tempfile::tempdir().expect("fixture directory");
let cache = seed_flat_bucket(&template.path().join("initial"), 0).await;
for index in 0..40 {
write_checkpoint_object(&root, &format!("object-{index:04}"), &[(Some(Uuid::from_u128(index + 1)), 3)]).await;
}
prepare_scanner(&mut scanner, cache);
let (tx, mut rx) = mpsc::channel(1);
scanner.checkpoint_tx = Some(tx);
scanner.checkpoint_objects = SCANNER_CHECKPOINT_OBJECT_INTERVAL - 1;
scanner.last_checkpoint_at = Instant::now()
.checked_sub(SCANNER_CHECKPOINT_MIN_INTERVAL)
.expect("checkpoint interval");
let ctx = CancellationToken::new();
scanner.budget = ScannerCycleBudget::new_with_progress_tracking(&ctx, Default::default());
let mut entry = DataUsageEntry::default();
super::super::super::TEST_FORCE_COMPACTION_WIDTH
.scope(16, async {
let checkpoint = {
let scan = scanner.scan_folder(
ctx.clone(),
CachedFolder {
name: "bucket".to_owned(),
parent: None,
object_heal_prob_div: 1,
},
&mut entry,
);
tokio::pin!(scan);
tokio::select! {
checkpoint=rx.recv()=>checkpoint.expect("compacted owner emitted a periodic checkpoint"),
result=&mut scan=>panic!("scan returned before it checkpointed a completed compacted child: {result:?}"),
}
};
assert!(!ctx.is_cancelled());
assert!(!checkpoint.info.snapshot_complete);
assert!(
checkpoint
.info
.scan_directory_work
.iter()
.any(|receipt| receipt.progress(&checkpoint.info).is_some())
);
let total = checkpoint.checked_flatten("bucket").expect("connected partial root");
assert!((1..40).contains(&total.objects));
assert_eq!(total.size, total.objects * 3);
let bytes = checkpoint.marshal_msg().expect("persist running checkpoint");
drop(scanner);
let durable = DataUsageCache::unmarshal(&bytes).expect("reload checkpoint after crash");
let resumed = run_until_finished(
&root,
durable,
ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
},
12,
)
.await;
let complete = resumed.checked_flatten("bucket").expect("restarted compacted root");
assert_eq!((complete.objects, complete.size, complete.versions), (40, 120, 40));
})
.await;
}
#[tokio::test]
#[serial]
async fn periodic_nested_snapshot_does_not_publish_shared_compacted_accumulator_under_child_path() {
@@ -74,7 +74,9 @@ pub(in crate::scanner_folder) async fn ordered_entries(
}
let mut entries = Vec::new();
while let Some(entry) = reader.next_entry().await? {
entries.push(entry);
// This fixture reorders native entries. Their kernel positions no
// longer describe the prefixes delivered to the walker.
entries.push(entry.without_anchor());
}
// Real entries and file types, with an explicit adversarial order rather
// than an assumption about the filesystem's enumeration order.
@@ -533,6 +535,32 @@ async fn erasure_candidate_defers_metadata_until_part_classification_finishes()
#[tokio::test]
#[serial]
async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassifies_on_resume() {
check_candidate_resume(CandidateResume::Legacy, 136).await;
}
#[tokio::test]
#[serial]
async fn erasure_candidate_warm_resume_avoids_rereading_sealed_root() {
check_candidate_resume(CandidateResume::Warm, 135).await;
}
#[cfg(any(unix, windows))]
#[tokio::test]
#[serial]
async fn erasure_candidate_cold_resume_uses_the_validated_sealed_root() {
check_candidate_resume(CandidateResume::Cold, 135).await;
}
enum CandidateResume {
Warm,
Legacy,
#[cfg(any(unix, windows))]
Cold,
}
async fn check_candidate_resume(kind: CandidateResume, expected_io: u64) {
use crate::storage_api::owner::EcstoreDiskAPI;
let (mut scanner, root, folder) = candidate_scanner().await;
let _guard = TestGuard {
temp_dir: Some(root.clone()),
@@ -596,7 +624,33 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
.expect("validated candidate index");
assert_eq!(index.indexed_entries().expect("index entries").len(), 64);
let bytes = serde_json::to_vec(&snapshot).expect("serialize partial checkpoint");
let reloaded: DataUsageCache = serde_json::from_slice(&bytes).expect("reload partial checkpoint");
let mut reloaded: DataUsageCache = serde_json::from_slice(&bytes).expect("reload partial checkpoint");
if !matches!(kind, CandidateResume::Warm) {
scanner.directory_checkpoints.clear();
if matches!(kind, CandidateResume::Legacy) {
// A pre-directory-checkpoint writer persisted raw names but no
// authenticated physical position or sealed membership manifest.
reloaded.info.scan_directory_source = None;
reloaded.info.scan_directory_work.clear();
} else {
for state in &reloaded.info.scan_directory_work {
scanner
.directory_checkpoints
.entry(state.parent().to_owned())
.or_default()
.insert(state.endpoint().to_owned(), Arc::new(std::sync::Mutex::new(Some(state.clone()))));
}
}
scanner.local_disk = new_disk(
&scanner.local_disk.endpoint(),
&DiskOption {
cleanup: false,
health_check: false,
},
)
.await
.expect("reopen cold scanner disk");
}
scanner.old_cache = reloaded.clone();
scanner.new_cache = reloaded;
scanner.raw_enumeration_progress.clear();
@@ -615,8 +669,8 @@ async fn erasure_candidate_cancelled_batch_keeps_partial_checkpoint_and_reclassi
);
assert_eq!(
charged_ops(&root),
136,
"resume recharges actual reads; the raw index is not a filesystem seek"
expected_io,
"resume charges every physical read, including cold reconstruction of the root"
);
}
@@ -4,6 +4,25 @@
use super::enumeration_restart::{install_raw_entry_observer, observed_raw_entries, seed_flat_bucket};
use super::*;
use crate::scanner_budget::ScannerCycleBudgetConfig;
use std::cell::Cell;
use std::path::PathBuf;
const PHYSICAL_READ_COST: Duration = Duration::from_secs(60);
tokio::task_local! {
static SLOW_DIRECTORY: (PathBuf, Cell<usize>);
}
pub(in crate::scanner_folder) fn is_slow_directory(path: &str) -> bool {
SLOW_DIRECTORY.try_with(|(directory, _)| directory.as_path() == std::path::Path::new(path)) == Ok(true)
}
/// Charge every read attempt (including replay and EOF) before entries leave
/// the native buffer, so cancelling a simulated slow read cannot skip a name.
pub(in crate::scanner_folder) async fn charge_physical_read() {
SLOW_DIRECTORY.with(|(_, reads)| reads.set(reads.get() + 1));
tokio::time::advance(PHYSICAL_READ_COST).await;
}
async fn open_fixture_disk(disk_root: &std::path::Path) -> Arc<Disk> {
let endpoint = Endpoint::try_from(disk_root.to_string_lossy().as_ref()).expect("fixture endpoint");
@@ -18,6 +37,73 @@ async fn open_fixture_disk(disk_root: &std::path::Path) -> Arc<Disk> {
.expect("open synthetic disk")
}
#[tokio::test(start_paused = true)]
#[serial]
async fn wide_directory_runtime_budget_reaches_eof() {
const WIDTH: usize = 48;
const ROUNDS: usize = 16;
let temp = tempfile::tempdir().expect("fixture directory");
let disk_root = temp.path().join("disk");
let mut cache = seed_flat_bucket(&disk_root, WIDTH).await;
let disk = open_fixture_disk(&disk_root).await;
let mut anchor = FileInfo::new("large", 4, 2);
anchor.volume = "bucket".to_owned();
anchor.version_id = Some(Uuid::from_u128(1000));
anchor.versioned = true;
anchor.mod_time = Some(OffsetDateTime::now_utc());
anchor.size = 4096;
let mut metadata = FileMeta::new();
metadata.add_version(anchor).expect("large fixture object");
write_test_object_metadata_bytes(&disk_root, "bucket", "large", &metadata.marshal_msg().expect("encode large object")).await;
let parent = CancellationToken::new();
let expected = (WIDTH + 1, WIDTH + 4096);
let scan = async {
for prior_timeouts in 0..ROUNDS {
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_duration: Some(8 * PHYSICAL_READ_COST),
..Default::default()
},
);
let result = scan_data_folder(
budget.token(),
budget.clone(),
vec![disk.clone()],
disk.clone(),
cache.clone(),
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
match result {
Ok(complete) => {
assert!(prior_timeouts >= 2, "the wide directory must span multiple actual runtime deadlines");
assert!(complete.info.snapshot_complete);
let totals = complete.checked_flatten("bucket").expect("complete bucket root");
assert_eq!((totals.objects, totals.size), expected);
return;
}
Err(ScannerError::PartialCache(partial)) => {
assert!(!partial.info.snapshot_complete);
cache = DataUsageCache::unmarshal(&partial.marshal_msg().expect("persist partial")).expect("reload partial");
}
Err(ScannerError::Other(message)) if budget.token().is_cancelled() && message == "Operation cancelled" => {}
Err(error) => panic!("unexpected scan failure: {error}"),
}
assert_eq!(budget.reason(), Some(crate::scanner_budget::ScannerCycleBudgetReason::Runtime));
}
let reads = SLOW_DIRECTORY.with(|(_, count)| count.get());
let totals = cache.checked_flatten("bucket").expect("last partial root");
panic!(
"wide directory still incomplete after {ROUNDS} fixed runtime deadlines and {reads} physical reads: {} objects, {} bytes",
totals.objects, totals.size
);
};
SLOW_DIRECTORY.scope((disk.path().join("bucket"), Cell::new(0)), scan).await;
}
/// A count budget is the operator's total work limit. It must also stop the
/// first listing of a directory far wider than that budget, instead of letting
/// one read_dir pass finish before the first object is charged.
@@ -75,15 +161,28 @@ async fn scan_one_restart_round(
disk_root: &std::path::Path,
cache: DataUsageCache,
config: ScannerCycleBudgetConfig,
) -> RestartRound {
scan_one_replica_round(disk_root, None, cache, config).await
}
async fn scan_one_replica_round(
disk_root: &std::path::Path,
peer: Option<&std::path::Path>,
cache: DataUsageCache,
config: ScannerCycleBudgetConfig,
) -> RestartRound {
let disk = open_fixture_disk(disk_root).await;
let mut disks = vec![disk.clone()];
if let Some(peer) = peer {
disks.push(open_fixture_disk(peer).await);
}
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
let _observation = install_raw_entry_observer(disk.path());
let result = scan_data_folder(
budget.token(),
budget.clone(),
vec![disk.clone()],
disks,
disk,
cache.clone(),
None,
@@ -105,7 +204,7 @@ async fn scan_one_restart_round(
}
}
async fn run_until_finished(
pub(super) async fn run_until_finished(
disk_root: &std::path::Path,
mut cache: DataUsageCache,
config: ScannerCycleBudgetConfig,
@@ -231,3 +330,476 @@ async fn fair_prefix_quantum_budget_stop_keeps_progress_without_certifying_it()
let expected = u64::try_from(OBJECTS).expect("fixture size");
assert_eq!((totals.objects as u64, totals.versions as u64), (expected, expected));
}
#[tokio::test]
#[serial]
async fn cold_compacted_directory_totals_progress_after_every_executor_restart() {
let temp = tempfile::tempdir().expect("fixture directory");
let root = temp.path().join("disk");
let mut cache = seed_flat_bucket(&root, 40).await;
let budget = ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
};
super::super::TEST_FORCE_COMPACTION_WIDTH
.scope(16, async {
let mut partial_counts = Vec::new();
for round in 0..16 {
let step = scan_one_restart_round(&root, cache.clone(), budget).await;
if !step.finished {
assert!(
step.outcome
.durable_scan_progress()
.advanced_from(&cache.durable_scan_progress()),
"each cold compacted round must earn a coordinator service turn, round {round}"
);
}
cache = step.outcome;
let total = cache.checked_flatten("bucket").expect("connected compact bucket");
if step.finished {
assert!(round > 1);
assert!(cache.info.snapshot_complete);
assert_eq!((total.objects, total.size, total.versions), (40, 40, 40));
return;
}
assert!(!cache.info.snapshot_complete);
partial_counts.push(total.objects);
}
panic!("cold compacted scan did not finish in 16 rounds; objects per partial: {partial_counts:?}");
})
.await;
}
#[tokio::test]
#[serial]
async fn cold_compacted_replicas_finish_within_their_independent_object_budgets() {
let temp = tempfile::tempdir().expect("fixture directory");
let first = temp.path().join("a");
let other = temp.path().join("b");
let mut cache = seed_flat_bucket(&first, 40).await;
tokio::fs::create_dir_all(&other).await.expect("fresh replica");
for index in (0..40).rev() {
let name = format!("object-{index:04}");
let metadata = tokio::fs::read(first.join("bucket").join(&name).join("xl.meta"))
.await
.expect("original metadata");
write_test_object_metadata_bytes(&other, "bucket", &name, &metadata).await;
}
let budget = ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
};
super::super::TEST_FORCE_COMPACTION_WIDTH
.scope(16, async {
for round in 0..(2 * 40_usize.div_ceil(6) + 1) {
let root = if round % 2 == 0 { &first } else { &other };
let peer = if round % 2 == 0 { &other } else { &first };
let step = scan_one_replica_round(root, Some(peer), cache.clone(), budget).await;
if !step.finished {
assert!(
step.outcome
.durable_scan_progress()
.advanced_from(&cache.durable_scan_progress()),
"cross-disk compacted work should advance durable scheduling, round {round}"
);
}
cache = step.outcome;
if step.finished {
assert!(round > 1);
assert!(cache.info.snapshot_complete);
let total = cache.checked_flatten("bucket").expect("complete switched source");
assert_eq!((total.objects, total.size), (40, 40));
return;
}
}
panic!("two cold physical replicas did not converge within two independent object work budgets");
})
.await;
}
#[tokio::test]
#[serial]
async fn cold_compacted_identical_names_do_not_mix_replica_version_sizes() {
use super::checkpoint_fixture::write_checkpoint_object;
let temp = tempfile::tempdir().expect("fixture directory");
let a = temp.path().join("a");
let b = temp.path().join("b");
let start = seed_flat_bucket(&a, 40).await;
seed_flat_bucket(&b, 40).await;
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(100)), 11), (Some(Uuid::from_u128(101)), 23)]).await;
let config = ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
};
super::super::TEST_FORCE_COMPACTION_WIDTH
.scope(16, async {
let first = scan_one_replica_round(&a, Some(&b), start, config).await;
assert!(!first.finished);
let mut cache = first.outcome;
for _ in 0..10 {
let round = scan_one_replica_round(&b, Some(&a), cache, config).await;
cache = round.outcome;
if round.finished {
break;
}
}
let complete = cache;
assert!(complete.info.snapshot_complete);
let counted = complete.checked_flatten("bucket").expect("complete B accounting");
// 39 one-byte objects and both retained versions of the remaining key.
assert_eq!((counted.objects, counted.size, counted.versions), (40, 73, 41));
})
.await;
}
#[tokio::test]
#[serial]
async fn legacy_unbound_partial_restarts_on_the_actual_physical_replica() {
use super::checkpoint_fixture::write_checkpoint_object;
let temp = tempfile::tempdir().expect("fixture");
let a = temp.path().join("a");
let b = temp.path().join("b");
let cache = seed_flat_bucket(&a, 40).await;
seed_flat_bucket(&b, 40).await;
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(200)), 12), (Some(Uuid::from_u128(201)), 31)]).await;
let config = ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
};
let first = scan_one_restart_round(&a, cache, config).await;
let mut legacy = first.outcome;
assert!(legacy.validated_scan_frontier().is_some());
legacy.info.scan_directory_source = None;
legacy.info.scan_directory_work.clear();
legacy.info.scan_source_trees.clear();
let upgraded = run_until_finished(&b, legacy, config, 10).await;
let total = upgraded.checked_flatten("bucket").expect("B complete");
assert_eq!((total.objects, total.size, total.versions), (40, 82, 41));
}
#[tokio::test]
#[serial]
async fn cold_linked_prefixes_resume_their_own_replica_coverage() {
use super::checkpoint_fixture::write_checkpoint_object;
let temp = tempfile::tempdir().expect("fixture directory");
let a = temp.path().join("a");
let b = temp.path().join("b");
let mut cache = seed_flat_bucket(&a, 40).await;
seed_flat_bucket(&b, 40).await;
write_checkpoint_object(&b, "object-0000", &[(Some(Uuid::from_u128(100)), 11), (Some(Uuid::from_u128(101)), 23)]).await;
let config = ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
};
// Six turns per disk leave one final B quantum. Each cold process must
// recover that disk's linked tree as well as its complete child coverage.
for round in 0..(2 * 40_usize.div_ceil(6) + 1) {
let (root, peer) = if round < 12 && round % 2 == 0 { (&a, &b) } else { (&b, &a) };
let previous_source = cache.info.scan_directory_source.clone();
let before = cache.durable_scan_progress();
let result = scan_one_replica_round(root, Some(peer), cache, config).await;
if !result.finished {
assert!(
result.outcome.durable_scan_progress().advanced_from(&before),
"linked source must earn another turn, round {round}"
);
}
cache = result.outcome;
if round == 1 {
let previous_source = previous_source.expect("A's physical source");
let endpoints = [
previous_source.as_str(),
cache.info.scan_directory_source.as_deref().expect("B's physical source"),
]
.map(|source| source.split_once('|').expect("disk generation and endpoint").1.to_owned())
.into_iter()
.collect();
let mut only_switched = cache.clone();
only_switched.activate_directory_source(previous_source, &endpoints);
assert!(only_switched.validated_scan_frontier().is_some(), "A's linked coverage is retained");
assert!(
!only_switched
.durable_scan_progress()
.advanced_from(&cache.durable_scan_progress()),
"moving a linked source between active and dormant earns no new turn"
);
}
if result.finished {
assert_eq!(root, &b);
assert!(cache.info.snapshot_complete);
let root = cache.find("bucket").expect("complete linked bucket");
assert!(!root.compacted);
assert_eq!(root.children.len(), 40);
let counted = cache.checked_flatten("bucket").expect("full B view");
assert_eq!((counted.objects, counted.size, counted.versions), (40, 73, 41));
return;
}
}
panic!("two independent cold linked prefix scans did not converge");
}
#[tokio::test]
#[serial]
async fn cold_compacted_changed_replica_does_not_import_another_disks_totals() {
let temp = tempfile::tempdir().expect("fixture directory");
let first = temp.path().join("a");
let other = temp.path().join("b");
let baseline = seed_flat_bucket(&first, 40).await;
seed_flat_bucket(&other, 40).await;
// Object zero was already counted on A but never existed in B's view.
// Importing A's subtotal based only on a matching child boundary would
// produce 40 objects on B even though its authoritative listing has 39.
tokio::fs::remove_dir_all(other.join("bucket/object-0000"))
.await
.expect("different replica membership");
let config = ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
};
super::super::TEST_FORCE_COMPACTION_WIDTH
.scope(16, async {
let first = scan_one_restart_round(&first, baseline, config).await;
assert!(!first.finished);
let complete = run_until_finished(&other, first.outcome, config, 14).await;
let root = complete.checked_flatten("bucket").expect("finished replica");
assert_eq!((root.objects, root.versions, root.size), (39, 39, 39));
})
.await;
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[tokio::test(start_paused = true)]
#[serial]
async fn cold_wide_listing_rotates_physical_endpoints_before_eof() {
let temp = tempfile::tempdir().expect("fixture directory");
let first = temp.path().join("a");
let other = temp.path().join("b");
let mut cache = seed_flat_bucket(&first, 49).await;
seed_flat_bucket(&other, 49).await;
let config = ScannerCycleBudgetConfig {
max_duration: Some(8 * PHYSICAL_READ_COST),
..Default::default()
};
for round in 0..22 {
let disk = if round % 2 == 0 { &first } else { &other };
let peer = if round % 2 == 0 { &other } else { &first };
let physical = disk.join("bucket").canonicalize().expect("physical directory");
let before = cache.durable_scan_progress();
let step = SLOW_DIRECTORY
.scope((physical, Cell::new(0)), scan_one_replica_round(disk, Some(peer), cache, config))
.await;
if !step.finished {
assert!(
step.outcome.durable_scan_progress().advanced_from(&before),
"each independent native resume must advance durable scheduling, round {round}"
);
}
cache = step.outcome;
if step.finished {
assert!(round > 2);
assert!(cache.info.snapshot_complete);
let total = cache.checked_flatten("bucket").expect("switched cold listing");
assert_eq!((total.objects, total.size), (49, 49));
return;
}
}
panic!("rotating before EOF repeatedly overwrote an endpoint's physical progress");
}
#[cfg(any(target_os = "linux", target_os = "macos"))]
#[tokio::test(start_paused = true)]
#[serial]
async fn cold_wide_listing_reaches_eof_after_every_executor_restart() {
let temp = tempfile::tempdir().expect("fixture directory");
let root = temp.path().join("disk");
let mut cache = seed_flat_bucket(&root, 49).await;
let budget = ScannerCycleBudgetConfig {
max_duration: Some(8 * PHYSICAL_READ_COST),
..Default::default()
};
SLOW_DIRECTORY
.scope((root.join("bucket").canonicalize().expect("physical path"), Cell::new(0)), async {
for round in 0..16 {
let step = scan_one_restart_round(&root, cache.clone(), budget).await;
cache = step.outcome;
if step.finished {
assert!(round > 1);
assert!(cache.info.snapshot_complete);
let total = cache.checked_flatten("bucket").expect("complete bucket");
assert_eq!((total.objects, total.size), (49, 49));
return;
}
assert!(!cache.info.snapshot_complete);
}
let reads = SLOW_DIRECTORY.with(|(_, reads)| reads.get());
panic!("cold physical listing did not reach EOF after 16 rounds and {reads} directory reads");
})
.await;
}
#[tokio::test]
#[serial]
async fn wide_compacted_directory_reuses_only_finished_children() {
const WIDTH: usize = 40;
let temp = tempfile::tempdir().expect("fixture directory");
let disk_root = temp.path().join("disk");
let mut cache = seed_flat_bucket(&disk_root, WIDTH).await;
let disk = open_fixture_disk(&disk_root).await;
let parent = CancellationToken::new();
super::super::TEST_FORCE_COMPACTION_WIDTH
.scope(16, async {
for round in 0..16 {
let budget = ScannerCycleBudget::new_with_progress_tracking(
&parent,
ScannerCycleBudgetConfig {
max_objects: Some(6),
..Default::default()
},
);
let result = scan_data_folder(
budget.token(),
budget,
vec![disk.clone()],
disk.clone(),
cache.clone(),
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await;
match result {
Ok(completed) => {
assert!(round > 1);
assert!(completed.info.snapshot_complete);
let root = completed.find("bucket").expect("compacted bucket");
assert!(root.compacted);
assert_eq!((root.objects, root.size), (WIDTH, WIDTH));
return;
}
Err(ScannerError::PartialCache(partial)) => {
cache = DataUsageCache::unmarshal(&partial.marshal_msg().expect("save compacted checkpoint"))
.expect("reload partial");
}
Err(error) => panic!("unexpected compacted scan failure: {error}"),
}
}
panic!("fixed object budget failed to finish compacted children");
})
.await;
}
#[tokio::test]
#[serial]
async fn wide_directory_replacement_invalidates_parked_entries() {
const WIDTH: usize = 48;
let temp = tempfile::tempdir().expect("fixture directory");
let disk_root = temp.path().join("disk");
let initial = seed_flat_bucket(&disk_root, WIDTH).await;
let disk = open_fixture_disk(&disk_root).await;
let parent = CancellationToken::new();
let config = ScannerCycleBudgetConfig {
max_entries: Some(3),
..Default::default()
};
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
let Err(ScannerError::PartialCache(partial)) = scan_data_folder(
budget.token(),
budget,
vec![disk.clone()],
disk.clone(),
initial,
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await
else {
panic!("the original listing must stop before EOF")
};
assert!(
super::super::parked_bucket_disks("bucket")
.iter()
.any(|scope| scope.ptr_eq(&Arc::downgrade(&disk)))
);
let mut cache = *partial;
// Rename pins the old inode. Reusing its open stream or its name pages
// would visit the vanished objects and omit this replacement's metadata.
tokio::fs::rename(disk_root.join("bucket"), temp.path().join("previous"))
.await
.expect("move old namespace");
let metadata = metadata_for_object_version("bucket", "successor", Some(Uuid::from_u128(100)));
write_test_object_metadata_bytes(&disk_root, "bucket", "successor", &metadata).await;
for _ in 0..6 {
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, config);
match scan_data_folder(
budget.token(),
budget,
vec![disk.clone()],
disk.clone(),
cache.clone(),
None,
HealScanMode::Normal,
SCANNER_SLEEPER.clone(),
)
.await
{
Ok(complete) => {
assert!(complete.info.snapshot_complete);
let total = complete.checked_flatten("bucket").expect("current root");
assert_eq!((total.objects, total.size), (1, 1));
assert!(complete.find("bucket/successor").is_some());
assert!(
!super::super::parked_bucket_disks("bucket")
.iter()
.any(|scope| scope.ptr_eq(&Arc::downgrade(&disk)))
);
return;
}
Err(ScannerError::PartialCache(partial)) => cache = *partial,
Err(error) => panic!("replacement scan failed: {error}"),
}
}
panic!("replacement directory never finished");
}
#[tokio::test]
#[serial]
async fn wide_directory_deleted_bucket_releases_a_slot_under_pressure() {
use super::super::directory_cursor::{DirectoryScope, PARKED_DIRECTORIES_PER_DISK, ScannerDirectory, is_parked};
let temp = tempfile::tempdir().expect("fixture directory");
let disk_root = temp.path().join("disk");
let cache = seed_flat_bucket(&disk_root, 0).await;
let disk = open_fixture_disk(&disk_root).await;
let scope = DirectoryScope::new(&disk, &cache, FolderScanMode::Namespace, None).expect("physical source and owner");
let parent = CancellationToken::new();
let budget = ScannerCycleBudget::new_with_progress_tracking(&parent, ScannerCycleBudgetConfig::default());
budget.cancel_for_runtime();
let retired = disk_root.join("retired");
for index in 0..PARKED_DIRECTORIES_PER_DISK {
let path = retired.join(index.to_string());
tokio::fs::create_dir_all(&path).await.expect("create parked directory");
let path = path.to_str().expect("fixture path");
let directory = ScannerDirectory::open(path, "bucket", Some(scope.clone()), budget.clone(), None)
.await
.expect("open source");
drop(directory);
assert!(is_parked(path));
}
let next = disk_root.join("replacement");
tokio::fs::create_dir(&next).await.expect("current source");
let next = next.to_str().expect("fixture path");
let directory = ScannerDirectory::open(next, "bucket", Some(scope.clone()), budget.clone(), None)
.await
.expect("open next source");
drop(directory);
assert!(!is_parked(next), "active owners retain their bounded slots");
tokio::fs::remove_dir_all(retired).await.expect("retire directories");
let directory = ScannerDirectory::open(next, "bucket", Some(scope), budget, None)
.await
.expect("admit after retirement");
drop(directory);
assert!(is_parked(next), "an obsolete source must not indefinitely exclude new work");
super::super::directory_cursor::forget_bucket(&disk, &cache.info);
}
+7
View File
@@ -1843,6 +1843,7 @@ pub(crate) struct ScannerCycleResult {
usage_watch_proofs: Vec<ScannerUsageWatchProof>,
usage_watch_retirements: Vec<ScannerUsageWatchProof>,
pub(crate) status: ScannerCycleStatus,
pub(crate) partial_stage: Option<&'static str>,
publication_epoch: Option<u64>,
activity_digest: Option<[u8; 32]>,
activity_baseline: Option<Arc<crate::scanner::ScannerActivitySnapshot>>,
@@ -1869,6 +1870,7 @@ impl ScannerCycleResult {
usage_watch_proofs: Vec::new(),
usage_watch_retirements: Vec::new(),
status,
partial_stage: None,
publication_epoch: None,
activity_digest: None,
activity_baseline: None,
@@ -1890,6 +1892,11 @@ impl ScannerCycleResult {
}
}
fn with_partial_stage(mut self, stage: Option<&'static str>) -> Self {
self.partial_stage = stage;
self
}
pub(crate) fn with_publication_epoch(mut self, publication_epoch: Option<u64>) -> Self {
self.publication_expectation = None;
self.publication_epoch = publication_epoch;
+3 -2
View File
@@ -792,6 +792,7 @@ pub(super) fn completed_data_usage_info(
cancelled: bool,
) -> Option<(DataUsageInfo, SystemTime)> {
completed_usage_candidate(results, scope, tier_registry_names, bucket_plan_complete, cancelled)
.ok()
.map(|candidate| (candidate.data, candidate.last_update))
}
@@ -810,7 +811,7 @@ pub(super) fn completed_usage_candidate(
tier_registry_names: &[String],
bucket_plan_complete: bool,
cancelled: bool,
) -> Option<ValidatedUsageCandidate> {
) -> std::result::Result<ValidatedUsageCandidate, CandidateOutcome> {
let candidate: std::result::Result<ValidatedUsageCandidate, CandidateOutcome> = (|| {
if !bucket_plan_complete {
return Err(CandidateOutcome::InventoryIncomplete);
@@ -934,7 +935,7 @@ pub(super) fn completed_usage_candidate(
Err(outcome) => *outcome,
},
);
candidate.ok()
candidate
}
fn tier_accounting_proof_is_publishable(
+54 -25
View File
@@ -15,6 +15,9 @@ use super::*;
/// ScannerIOCache implementation for SetDisks: bucket ordering, worker fan-out, merge, and publish.
use crate::data_usage_define::clone_cache_value;
mod walker_slots;
use walker_slots::AvailableWalkers;
pub(super) async fn collect_completed_bucket_results(
ctx: CancellationToken,
mut cache: DataUsageCache,
@@ -989,12 +992,12 @@ impl ScannerIOCache for SetDisks {
if turn_bucket_count == 0 {
bucket_work_complete.cancel();
}
for NamespaceScannerWalker {
disk,
mode: worker_mode,
local_admission,
} in walkers
{
let walker_count = walkers.len();
let available_walkers = Arc::new(AvailableWalkers::new(walkers, |disk| {
crate::storage_api::owner::EcstoreDiskAPI::endpoint(disk.as_ref()).to_string()
}));
for _ in 0..walker_count {
let available_walkers = available_walkers.clone();
let service_cohort_clone = service_cohort.clone();
let usage_watch_cohort_clone = usage_watch_cohort.clone();
let usage_watch_store_clone = usage_watch_store.clone();
@@ -1026,18 +1029,18 @@ impl ScannerIOCache for SetDisks {
let worker_retired_buckets = retired_buckets.clone();
#[cfg(feature = "e2e-test-hooks")]
let worker_incarnations = current_bucket_incarnations.clone();
let remote_capabilities = match worker_mode {
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
NamespaceScannerWorkerMode::Coordinator => None,
};
#[cfg(test)]
let worker_index = futs.len();
futs.push(AbortOnDropHandle::new(tokio::spawn(async move {
#[cfg(test)]
super::tests::observe_coordinator_bucket_worker_started(worker_index).await;
let remote_session_id = uuid::Uuid::new_v4();
let mut remote_session_sequence = 0_u64;
// Returning at the top of the loop keeps the retiring disk unavailable
// on every break, cancellation, or panic, including after requeue.
let mut previous_walker = None;
loop {
if let Some(slot) = previous_walker.take() {
available_walkers.release(slot);
}
// Do not prefetch a FIFO member into an independently
// scheduled permit waiter: that can reorder admissions.
let permit_wait_start = Instant::now();
@@ -1050,18 +1053,39 @@ impl ScannerIOCache for SetDisks {
break;
}
let permit_wait_elapsed = permit_wait_start.elapsed();
let bucket = tokio::select! {
let (bucket, slot) = tokio::select! {
_ = bucket_work_complete_clone.cancelled() => break,
_ = ctx_clone.cancelled() => break,
bucket = async { bucket_rx_mutex_clone.lock().await.recv().await } => {
let Some(bucket) = bucket else {
break;
};
bucket
selected = async {
let mut receiver = bucket_rx_mutex_clone.lock().await;
let bucket = receiver.recv().await?;
// Select while still owning FIFO admission, so the oldest bucket
// gets its preferred disk before later waiters can borrow it.
let parked = crate::scanner_folder::parked_bucket_disks(&bucket.name);
let slot = available_walkers.take_for_bucket(&bucket.name, |disk| {
parked.iter().any(|owner| owner.ptr_eq(&Arc::downgrade(disk)))
});
Some((bucket, slot))
} => match selected {
Some(pair) => pair,
None => break,
}
};
let mut work_guard =
BucketWorkGuard::new(remaining_bucket_work_clone.clone(), bucket_work_complete_clone.clone());
let Some(slot) = slot else {
// Each live queue consumer owns at most one walker. On retirement a
// consumer and its last walker leave together.
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
break;
};
let slot = previous_walker.insert(slot);
let disk = slot.walker.disk.clone();
let local_admission = &slot.walker.local_admission;
let remote_capabilities = match slot.walker.mode {
NamespaceScannerWorkerMode::RemoteV4(capabilities) => Some(capabilities),
NamespaceScannerWorkerMode::Coordinator => None,
};
// Prefix hints are process-local. Never hand one to a
// remote or legacy-coordinator disk path.
let prefix_scan_scope = disk.is_local().then(|| scope_clone.prefix_scope_for(&bucket.name)).flatten();
@@ -1181,8 +1205,8 @@ impl ScannerIOCache for SetDisks {
let bucket_checkpoint_plan_digest = bucket_plans.coverage;
if let Some(capabilities) = remote_capabilities {
let request_sequence = remote_session_sequence;
let Some(next_sequence) = remote_session_sequence.checked_add(1) else {
let request_sequence = slot.remote_session_sequence;
let Some(next_sequence) = request_sequence.checked_add(1) else {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
target: "rustfs::scanner::io",
@@ -1195,7 +1219,7 @@ impl ScannerIOCache for SetDisks {
);
break;
};
remote_session_sequence = next_sequence;
slot.remote_session_sequence = next_sequence;
let remote_scan = crate::remote_scanner::scan_remote_bucket(
&disk,
if capabilities.supports_durable_progress || capabilities.supports_complete_observation || capabilities.supports_usage_watch {
@@ -1209,7 +1233,7 @@ impl ScannerIOCache for SetDisks {
next_cycle: want_cycle,
leader_epoch,
server_epoch: capabilities.server_epoch,
session_id: remote_session_id,
session_id: slot.remote_session_id,
session_sequence: request_sequence,
plans: bucket_plans,
supports_durable_progress: capabilities.supports_durable_progress,
@@ -1402,7 +1426,7 @@ impl ScannerIOCache for SetDisks {
continue;
}
let _local_admission = if let Some(share) = &local_admission {
let _local_admission = if let Some(share) = local_admission {
match share.acquire(&disk) {
Ok(admission) => Some(admission),
Err(e) => {
@@ -1485,8 +1509,13 @@ impl ScannerIOCache for SetDisks {
};
let mut cache = DataUsageCache::default();
let mut revisions = match cache.load_with_revisions(store_clone_clone.clone(), &cache_name).await {
Ok(revisions) => revisions,
let mut revisions = match Box::pin(cache.load_for_scan(store_clone_clone.clone(), &cache_name, &scan_ctx, &bucket_budget)).await {
Ok(Some(revisions)) => revisions,
Ok(None) => {
work_guard.mark_service_interrupted();
record_partial_dirty_bucket(&partial_dirty_buckets_clone, &bucket.name).await;
continue;
}
Err(e) => {
record_failed_dirty_bucket(&failed_dirty_buckets_clone, &bucket.name).await;
error!(
@@ -0,0 +1,128 @@
// Copyright 2026 RustFS Team
// Licensed under the Apache License, Version 2.0.
use super::NamespaceScannerWalker;
use sha2::{Digest as _, Sha256};
use std::sync::Mutex;
use uuid::Uuid;
/// All state belonging to one physical walker travels with it, even when a
/// different queue consumer makes the next request on that endpoint.
pub(super) struct WalkerSlot<T> {
pub(super) walker: NamespaceScannerWalker<T>,
pub(super) remote_session_id: Uuid,
pub(super) remote_session_sequence: u64,
endpoint: String,
}
pub(super) struct AvailableWalkers<T>(Mutex<Vec<WalkerSlot<T>>>);
impl<T> AvailableWalkers<T> {
pub(super) fn new(walkers: Vec<NamespaceScannerWalker<T>>, endpoint: impl Fn(&T) -> String) -> Self {
Self(Mutex::new(
walkers
.into_iter()
.map(|walker| WalkerSlot {
endpoint: endpoint(&walker.disk),
walker,
remote_session_id: Uuid::new_v4(),
remote_session_sequence: 0,
})
.collect(),
))
}
/// Rank available physical endpoints independently of random inventory
/// order, waiter wakeups or the number of walker slots on a local disk.
pub(super) fn take_for_bucket(&self, bucket: &str, parked: impl Fn(&T) -> bool) -> Option<WalkerSlot<T>> {
let mut idle = self.0.lock().unwrap_or_else(|err| err.into_inner());
let mut prefix = Sha256::new();
prefix.update(bucket.as_bytes());
prefix.update([0]);
let index = idle
.iter()
.enumerate()
.map(|(index, slot)| {
let rank = prefix.clone().chain_update(slot.endpoint.as_bytes()).finalize();
(parked(&slot.walker.disk), rank, slot.endpoint.as_str(), index)
})
.max()
.map(|(_, _, _, index)| index)?;
Some(idle.swap_remove(index))
}
pub(super) fn release(&self, slot: WalkerSlot<T>) {
self.0.lock().unwrap_or_else(|err| err.into_inner()).push(slot);
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::scanner_io::NamespaceScannerWorkerMode;
fn pool(endpoints: &[&'static str]) -> AvailableWalkers<&'static str> {
let walkers = endpoints
.iter()
.copied()
.map(|disk| NamespaceScannerWalker {
disk,
mode: NamespaceScannerWorkerMode::Coordinator,
local_admission: None,
})
.collect();
AvailableWalkers::new(walkers, |disk| (*disk).to_owned())
}
#[test]
fn bucket_resumes_on_same_free_endpoint_despite_reordering_or_borrower_changes() {
let first = pool(&["disk-c", "disk-a", "disk-b"]);
let reordered = pool(&["disk-b", "disk-c", "disk-a"]);
for bucket in ["wide", "documents", "snapshots", "videos"] {
let mut preferred = first.take_for_bucket(bucket, |_| false).expect("preferred walker");
let another = reordered
.take_for_bucket(bucket, |_| false)
.expect("reordered preferred walker");
assert_eq!(preferred.endpoint, another.endpoint);
let next = first
.take_for_bucket(bucket, |_| false)
.expect("fallback when primary is busy");
assert_ne!(preferred.endpoint, next.endpoint);
let alternative = next.endpoint.clone();
let id = preferred.remote_session_id;
preferred.remote_session_sequence += 1;
let sequence = preferred.remote_session_sequence;
first.release(preferred);
let resumed = first.take_for_bucket(bucket, |_| false).expect("released primary");
assert_eq!((resumed.remote_session_id, resumed.remote_session_sequence), (id, sequence));
first.release(resumed);
first.release(next);
let parked = first
.take_for_bucket(bucket, |disk| *disk == alternative)
.expect("parked directory");
assert_eq!(parked.endpoint, alternative, "follow a parked stream on a previous fallback disk");
first.release(parked);
reordered.release(another);
}
}
#[test]
fn retired_slot_leaves_same_endpoint_alternative_and_does_not_leak() {
let slots = pool(&["disk-a", "disk-a"]);
let retiring = slots.take_for_bucket("wide", |_| false).expect("first walker");
let survivor = slots.take_for_bucket("wide", |_| false).expect("second local walker");
assert!(slots.take_for_bucket("wide", |_| false).is_none());
assert_ne!(retiring.remote_session_id, survivor.remote_session_id);
let id = survivor.remote_session_id;
drop(retiring);
slots.release(survivor);
assert_eq!(
slots
.take_for_bucket("wide", |_| false)
.expect("surviving walker")
.remote_session_id,
id
);
assert!(slots.take_for_bucket("wide", |_| false).is_none());
}
}
+99 -7
View File
@@ -41,6 +41,48 @@ struct CollectorSlotSummary<'a> {
source_difference_sample: Option<CollectorSlotSample<'a>>,
}
impl CollectorSlotSummary<'_> {
/// Select one observed blocker. A source emitted in a different slot is valid when the set of sources matches.
fn partial_stage(
&self,
bucket_plan_complete: bool,
bucket_scan_status: ScannerBucketScanStatus,
dirty_usage_status: DirtyUsageSnapshotStatus,
scan_scope_matches: bool,
candidate_rejection: Option<rustfs_scanner_metrics::usage_candidate::ScannerUsageCandidateOutcome>,
) -> &'static str {
if !bucket_plan_complete {
"inventory_incomplete"
} else if self.unobserved > 0 {
"source_result_missing"
} else if bucket_scan_status == ScannerBucketScanStatus::Failed {
"bucket_worker_failed"
} else if bucket_scan_status == ScannerBucketScanStatus::NamespaceNotFound {
"bucket_namespace_not_found"
} else if bucket_scan_status == ScannerBucketScanStatus::Partial {
"bucket_walk_incomplete"
} else if dirty_usage_status == DirtyUsageSnapshotStatus::Unverified {
"dirty_generation_unverified"
} else if self.missing_source > 0 {
"missing_source"
} else if self.cycle_mismatch > 0 {
"source_cycle_mismatch"
} else if self.epoch_mismatch > 0 {
"source_epoch_mismatch"
} else if self.missing_plan > 0 || self.plan_mismatch > 0 {
"source_plan_mismatch"
} else if !scan_scope_matches {
"source_membership"
} else if self.incomplete > 0 {
"source_snapshot_incomplete"
} else if self.missing_last_update > 0 {
"missing_last_update"
} else {
candidate_rejection.map_or("root_coverage_incomplete", |outcome| outcome.as_label())
}
}
}
fn summarize_collector_slots<'a>(
slots: impl IntoIterator<Item = (usize, DataUsageCacheSource, bool, &'a DataUsageCache)>,
cycle: u64,
@@ -797,7 +839,9 @@ where
if all_buckets.is_empty() {
reset_set_scan_gauges();
if !bucket_plan_complete {
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
.with_partial_stage(Some("inventory_incomplete"))
.with_publication_epoch(publication_epoch));
}
let publication_blocked = store.scanner_data_usage_publication_blocked().await;
let activity_after = probe_scanner_cycle_activity(
@@ -834,7 +878,9 @@ where
tier_registry_generation: Some(tier_registry_generation),
},
) else {
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
.with_partial_stage(Some("empty_namespace_guard"))
.with_publication_epoch(publication_epoch));
};
let (empty_usage, publication_expectation) = candidate.prepare(status, &pending_buckets);
let usage_update_emitted = if should_publish_observational_snapshot(status) {
@@ -843,7 +889,11 @@ where
publish_usage_snapshot(&updates, status, &pending_buckets, empty_usage).await?
};
if !usage_update_emitted {
return Ok(ScannerCycleResult::new(status, None).with_publication_epoch(publication_epoch));
let partial_stage =
(dirty_usage_status == DirtyUsageSnapshotStatus::Unverified).then_some("dirty_generation_unverified");
return Ok(ScannerCycleResult::new(status, None)
.with_partial_stage(partial_stage)
.with_publication_epoch(publication_epoch));
}
if status == ScannerCycleStatus::Complete {
complete_tier_registry_cycle(want_cycle, leader_epoch);
@@ -886,7 +936,9 @@ where
"Scanner set state update detected missing disk sets"
);
reset_set_scan_gauges();
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None).with_publication_epoch(publication_epoch));
return Ok(ScannerCycleResult::new(ScannerCycleStatus::Incomplete, None)
.with_partial_stage(Some("source_inventory_empty"))
.with_publication_epoch(publication_epoch));
}
let dispatch_limit = scanner_max_concurrent_set_scans(total_results);
@@ -1261,7 +1313,7 @@ where
remote_certification_attempted,
);
let all_bucket_names = all_buckets.iter().map(|bucket| bucket.name.clone()).collect::<Vec<_>>();
let mut completed_usage = completed_usage_candidate(
let completed_candidate = completed_usage_candidate(
&results,
&ScannerSnapshotScope {
sources: &expected_sources,
@@ -1278,6 +1330,8 @@ where
bucket_plan_complete,
owner_ctx.is_cancelled(),
);
let candidate_rejection = completed_candidate.as_ref().err().copied();
let mut completed_usage = completed_candidate.ok();
if let (Some(candidate), Some(baseline)) = (
completed_usage.as_mut(),
persisted_usage_baseline
@@ -1308,7 +1362,11 @@ where
dirty_usage_status,
activity_status,
);
if cycle_status == ScannerCycleStatus::Incomplete && result.is_ok() && !budget_elapsed && !owner_ctx.is_cancelled() {
let partial_stage = if cycle_status == ScannerCycleStatus::Incomplete
&& result.is_ok()
&& !budget_elapsed
&& !owner_ctx.is_cancelled()
{
let slots =
set_disks
.iter()
@@ -1319,6 +1377,35 @@ where
(index, DataUsageCacheSource::new(set.pool_index, set.set_index), *observed, snapshot)
});
let summary = summarize_collector_slots(slots, want_cycle, leader_epoch, scan_plan_digest);
let mut stage = summary.partial_stage(
bucket_plan_complete,
bucket_scan_status,
dirty_usage_status,
scan_scope_matches,
candidate_rejection,
);
// A fully traversed mixed-plan observation needs another certification sweep. Only
// attribute the stop to that fence when every uncarried bucket has fresh source evidence.
if stage == "bucket_walk_incomplete"
&& let Some((snapshot, _)) = &observational_usage
&& partial_buckets.difference(&carried_partial_buckets).all(|bucket| {
snapshot
.usage_snapshot_bucket_observations
.get(bucket)
.is_some_and(|observation| {
observation
.sources
.iter()
.all(|source| source.scanner_epoch == leader_epoch && source.scanner_cycle == want_cycle)
&& observation
.sources
.iter()
.any(|source| source.started_plan != source.requested_plan)
})
})
{
stage = "mixed_plan_recertification_pending";
}
let sample = summary.sample;
let source_difference = summary.source_difference_sample;
let sample_snapshot = sample.filter(|sample| sample.observed).map(|sample| sample.snapshot);
@@ -1328,6 +1415,7 @@ where
component = LOG_COMPONENT_SCANNER,
subsystem = LOG_SUBSYSTEM_IO,
state = "incomplete_cycle_collector_summary",
partial_stage = stage,
cycle = want_cycle,
leader_epoch,
configured_sets = set_disks.len(),
@@ -1373,7 +1461,10 @@ where
sample_snapshot_plan_matches = sample_snapshot.and_then(|snapshot| snapshot.info.scan_plan_digest.map(|plan| plan == scan_plan_digest)),
"Scanner recorded collector observations for an incomplete cycle"
);
}
Some(stage)
} else {
None
};
// Certification is per bucket: the pending set is read after the
// guardrail verdict so a bucket written during the walk is reported in
// the same publication that certifies the others. A bucket whose walk
@@ -1566,6 +1657,7 @@ where
complete_tier_registry_cycle(want_cycle, leader_epoch);
}
Ok(ScannerCycleResult::new(cycle_status, dirty_usage_clear)
.with_partial_stage(partial_stage)
.with_usage_watch_proofs(usage_watch_proofs.into_values().collect())
.with_usage_watch_retirements(usage_watch_retirements)
.with_publication_epoch(publication_epoch)
@@ -127,6 +127,8 @@ async fn collector_reports_an_unobserved_slot_after_a_stale_set_snapshot() {
let events = collector_events(&events);
assert_eq!(events.len(), 1, "a non-budget incomplete cycle needs one collector observation");
let fields = events[0];
assert_eq!(fields["partial_stage"], "source_result_missing");
assert_eq!(result.partial_stage, Some("source_result_missing"));
assert_eq!(fields["cycle"], 7);
assert_eq!(fields["leader_epoch"], LEADER_EPOCH);
assert_eq!(fields["expected_sources"], 1);
@@ -171,6 +173,7 @@ async fn collector_reports_received_incomplete_cache_only_for_non_budget_partial
let events = collector_events(&events);
assert_eq!(events.len(), 1, "a non-budget incomplete cycle needs one collector observation");
let fields = events[0];
assert_eq!(fields["partial_stage"], second.partial_stage.expect("non-budget incomplete stage"));
assert_eq!(fields["observed_slots"], 1);
assert_eq!(fields["unobserved_slots"], 0);
assert_eq!(fields["sample_observed"], true);
@@ -187,6 +190,7 @@ async fn collector_reports_received_incomplete_cache_only_for_non_budget_partial
let ((third, snapshot, _), events) = capture_cycle(run_cycle(&store, 3, None, false)).await;
assert_eq!(third.status, ScannerCycleStatus::Complete);
assert_eq!(third.partial_stage, None);
assert!(snapshot.is_some_and(|snapshot| snapshot.usage_snapshot_complete));
assert!(
collector_events(&events).is_empty(),
@@ -82,7 +82,9 @@ fn completed_outcome(
},
};
assert_one_outcome(Kind::Completed, outcome, || {
completed_usage_candidate(caches, &scope, &[], plan_complete, owner_cancelled)
let candidate = completed_usage_candidate(caches, &scope, &[], plan_complete, owner_cancelled);
assert_eq!(candidate.as_ref().err().copied().unwrap_or(Outcome::Accepted), outcome);
candidate.ok()
});
}
+1 -1
View File
@@ -15,7 +15,7 @@ An unchanged interval requires matching owner, process session, watch token, buc
Watch renewal keeps the original starting revision. A caller may extend its conservative local monotonic deadline after a successful renewal, but it must not replace the original proof with the renewed revision. A persisted checkpoint stores identity and revision, not a usable `Instant`; after restart, the old producer token must be checked again against the complete current owner set.
One keeper belongs to each leader-local cohort, with weak store and cohort references. It revisits retained watches every sixty seconds, with four concurrent queries and a five-second total limit per keepalive query. At the 256-watch limit one pass is bounded by 380 seconds, below the ten-minute producer lease. Retained checkpoints and finished sources awaiting root publication and ACK keep their original proof alive. A subsequent cycle may restart a cancelled keeper for the same instance, but cannot reset the original token, revision, or expired deadline.
One keeper belongs to each service cohort, with weak store and cohort references. After a cycle deadline the scanner supervisor retains its cohort only when the worker has stopped, cycle state has been persisted, and the epoch has been fenced. A successful successor claim on the same storage instance can resume the cohort only from the exact fenced revision and immediately following epoch. An intervening claim, lock contention, or recovery drops the continuation. The cohort preserves its bounded watch renewals and scheduling order across the retry interval. The new leader needs its own fences for publication and dirty acknowledgements. The keeper revisits retained watches every sixty seconds, with four concurrent queries and a five-second total limit per keepalive query. At the 256-watch limit one pass is bounded by 380 seconds, below the ten-minute producer lease. Retained checkpoints and finished sources awaiting root publication and ACK keep their original proof alive. A subsequent cycle may restart a cancelled keeper for the same instance, but cannot reset the original token, revision, or expired deadline.
## Consumer and persistence order
+14 -3
View File
@@ -82,7 +82,13 @@ The startup usage preflight is read-only and has a 30-second deadline. If it can
Cold-cache startup and startup without active replication avoid an extra bucket listing. Maintenance discovery keeps looking for replication after finding lifecycle rules in an earlier bucket. Once replication is found, both scheduling decisions are settled and remaining buckets need no startup inspection. Reapplying an unchanged resolved configuration does not reconfigure per-disk budgets or emit another configuration wakeup.
The `cycle_max_duration`, `cycle_max_objects`, and `cycle_max_directories` controls bound a cycle's work; they do not guarantee that every budget can complete a sweep in bounded chunks. A partial scan preserves useful checkpoints and coverage, but reopening ancestors and revalidating directory entries costs work again. A budget below the minimum replay or traversal cost can repeatedly stop at the same directory. Cross-bucket fairness also needs to be distinguished from progress within one bucket. Use pacing controls to limit background I/O, and choose cycle budgets large enough for the namespace to advance.
The `cycle_max_duration`, `cycle_max_objects` and `cycle_max_directories` settings also apply to resumed walks. For a stable wide directory, a checkpoint records which child names have been discovered, which children have finished and the usage accumulated from those children. The next executor can restore that work after a cycle deadline or process restart. Objects from a child that was interrupted before completion are accounted for when that child finishes.
On Linux and macOS, the scanner can reopen a partially enumerated physical directory and verify its last saved position against the directory generation and actual entries. Each physical replica has its own position. A replacement or changed directory requires enumeration again. When an executor moves to another disk, it advances that disk's own checkpoint. Returning to the first disk resumes its saved subtotal. This keeps sizes and version counts tied to the replica whose object metadata was read. Other platforms can reuse a live directory handle and already completed work on an identifiable disk. An interrupted native directory read may need to start again after a process restart.
After an upgrade, an older partial traversal without a recorded physical disk starts a fresh walk. This preserves the existing published usage snapshot when the partial checkpoint is discarded. To verify a handoff, enable debug logging for `rustfs_scanner::scanner_folder::directory_cursor` and look for `state="directory_checkpoint_restored"`. The event identifies the bucket, directory, physical disk, leader epoch and cycle. `restore_mode="native_position"` records how many entries had already been read. `restore_mode="sealed_membership"` records reuse of a fully enumerated directory and includes `completed_through` when the checkpoint contains a compacted subtotal.
The bucket usage API can return a carried value or a newer observation while a walk is incomplete. Check `usage_value_state` and `usage_certification` to identify the last certified value. If writes overlap a resumed walk, reaching the end can produce an observation that still needs a fresh sweep before certification. Progress requires enough budget for individual I/O operations and a successful checkpoint save.
Budget cancellation alone does not publish complete usage or acknowledge dirty buckets. A partial cycle must retain that uncertainty; a growing directories-scanned counter is not proof of new coverage or published totals. Inspect checkpoint/frontier changes and publication status across attempts before changing budgets. A budget cancellation does not persist a disabled scheduler flag and does not justify deleting scanner state.
@@ -230,7 +236,7 @@ Only `xl.meta` files enter object metadata accounting. Other payload files consu
UUID candidates under a directory without `xl.meta` are classified during their normal directory traversal. Each directory open and entry is charged once; traversal yields every 64 entries and continues under the budget until a regular `part.N` is found or EOF is reached. A full batch is not evidence of absence. Candidate metadata accounting is deferred until classification finishes. A data directory retains the existing missing-parent-metadata and non-destructive heal path; a namespace continues from the same listing, without a second probe/open/entry pass. Data directories do not consume namespace-directory quota or a preset folder delay; a candidate proven to be a namespace consumes that quota and delay once.
Candidate entries use the normal raw-enumeration progress and checkpoint path. Cancellation or an iteration error leaves classification incomplete, never an EOF or successful coverage result. A child already absent or replaced by a non-directory at open is handled as vanished, without stale namespace totals or a missing-metadata heal request. A replacement retains the existing coverage gap; mid-iteration errors never become EOF. A clean namespace covered by a restored frontier reuses its prior totals only after current classification reaches EOF; metadata is not read or charged to object quota again. A later data candidate prevents completed namespace siblings from certifying the unresolved parent. Existing raw-page indexes are membership receipts, not portable filesystem seek offsets: a later cycle or process restart reopens and revalidates the directory, with actual reads charged again. This does not guarantee that a directory larger than the configured cycle budget can finish within that budget.
Candidate entries use the normal raw-enumeration progress and checkpoint path. Cancellation or an iteration error leaves classification incomplete, never an EOF or successful coverage result. A child already absent or replaced by a non-directory at open is handled as vanished, without stale namespace totals or a missing-metadata heal request. A replacement retains the existing coverage gap; mid-iteration errors never become EOF. A clean namespace covered by a restored frontier reuses its prior totals only after current classification reaches EOF; metadata is not read or charged to object quota again. A later data candidate prevents completed namespace siblings from certifying the unresolved parent. The disk worker parks the physical enumeration between turns and checks directory identity and modification state before reuse. A cold process reopens the directory and pays for its physical reads again. Raw-page indexes identify visited names; the scanner still requires EOF, complete child accounting and a valid publication fence before certifying totals.
- Media is detected once per disk path from the backing block device's `rotational` flag on Linux, through partitions, `dm-*`, and software RAID; any spindle in the stack counts as rotational. Undetectable media, including non-Linux hosts and overlay filesystems without a bind-mounted volume, uses the rotational defaults because a spindle is the case that needs protection.
- `auto` resolves to `100` operations/s and `16 MiB/s` on rotational or unknown media and to `4000` operations/s and `256 MiB/s` on flash. The rotational values leave roughly half of a 7200 rpm spindle's random reads to foreground traffic; the flash values are a few percent of a SATA SSD. The `fastest` speed preset resolves `auto` to no cap; explicit numbers always apply.
@@ -313,6 +319,7 @@ metrics.maintenance_control.primary_control
metrics.source_work
metrics.replication_repair
metrics.scan_checkpoint
metrics.last_cycle_partial_stage
metrics.cycle_timeout_total
metrics.cycle_last_progress_age
metrics.leader_lease_without_progress
@@ -335,6 +342,8 @@ catch_up_estimate.discovered_expiry_items
catch_up_estimate.discovered_transition_items
```
For a partial cycle, `metrics.last_cycle_partial_stage` records the blocking check for that cycle. For example, `entry_budget` counts raw directory enumeration, `source_result_missing` means an expected set supplied no result, and `activity_baseline_unverified` means the activity proof could not certify publication. A `mixed_plan_recertification_pending` stage means every outstanding bucket completed traversal and supplied a fresh cross-plan observation that still requires certification. Candidate rejections retain the existing candidate outcome labels such as `invalid_root` and `tier_proof`. The field clears on the next non-partial cycle outcome. Inspect `metrics.usage_publication` for the outcome of writing the root snapshot. When a cycle ends with incomplete coverage or an unverifiable observation, the terminal error event includes the cycle, leader epoch and blocking stage. Budget stops and saved checkpoints remain visible in scanner status and warning logs. A supervisor deadline that interrupts the worker before a terminal result increments `metrics.cycle_timeout_total`.
### Deep scan cadence state
`bitrot` reports how the scanner decided the scan mode of the running or most recent cycle:
@@ -361,6 +370,8 @@ A resumed walk retains its previous durable coverage while initial enumeration h
Shared compacted accumulators defer periodic emission until the owning subtree completes; replay inside that compacted subtree remains possible after an interruption. When bucket checkpoint preparation returns `Reset` for existing resume state, it logs `checkpoint_reset` with the validation reason. The `scan_checkpoint_used` counter includes validated coverage adoption by the forward sweep.
Fenced per-bucket checkpoints keep small payloads inline. Large checkpoints stage immutable, checksum-addressed pages and then conditionally replace a small manifest. The primary and backup use separate page objects. Interruption before the manifest write leaves the prior checkpoint readable. Cold recovery verifies the referenced pages and can carry verified restore progress across local time slices. A reader that predates the page format can rebuild a partial checkpoint and can read compact complete totals when the compatibility projection fits the manifest.
A confirmed periodic save does not prove restart resumability. Adoption still validates bucket/set identity and, for non-empty caches, connected cache scope. Skipping completed subtrees additionally requires a validated coverage frontier; raw enumeration state alone does not prove completed subtree coverage.
### Cycle recovery on followers
@@ -932,7 +943,7 @@ For a mostly idle single-node, single-disk deployment with sustained CPU usage w
6. If `primary_pressure` is `throttle_pause` and pause ratios are low, raise `scanner.delay` first.
7. If individual sleeps are too short, raise `scanner.max_wait`.
8. If each scan cycle finishes but starts too often, raise `scanner.cycle`.
9. To bound one cycle, set `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`; confirm that checkpoints advance under that budget. These controls do not guarantee resumable bounded chunks for every directory shape; see [Startup and cycle budgets](#startup-and-cycle-budgets).
9. To bound one cycle, set `scanner.cycle_max_duration`, `scanner.cycle_max_objects`, or `scanner.cycle_max_directories`; confirm that checkpoints advance under that budget. For the conditions that permit a wide directory to advance across turns and process restarts, see [Startup and cycle budgets](#startup-and-cycle-budgets).
10. Recheck `pacing_pressure`, `maintenance_control`, source work, and lifecycle transition status after one or more scanner cycles.
Do not rely only on a longer cycle interval if lifecycle, replication, heal, or bitrot work must keep moving; use source work and transition status to confirm that background maintenance still progresses.
+1 -1
View File
@@ -64,7 +64,7 @@ A coverage receipt binds the completed traversal frontier to its scope, starting
The Normal-to-Deep regression holds the mutation plan fixed, changes metadata in a previously visited prefix, and verifies that the real Deep disk-scan entry point reads that prefix again. It also checks that a complete Normal cache cannot satisfy the Deep `Current` path. The fixture disables heal side effects; it proves traversal re-entry, not actual bitrot detection or repair. Additional tests cover map round-trips, stale identities, existing-but-uncovered cursors, coverage gaps, and per-instance metadata ownership.
This fixture bounds object processing after directory enumeration. It does not prove fixed-budget enumeration of arbitrarily wide directories or real process-restart convergence. Those gates require a storage-owned resumable enumeration capability, including its initial construction cost; a readdir offset, an in-memory iterator or a last-name filter is not that capability.
The independent `wide_directory_runtime_budget_reaches_eof` regression charges virtual time for every physical directory read, including replay and EOF, and checks exact object and byte totals across capped rounds. The `paged_checkpoint` tests exercise immutable page reuse, interrupted root commits, stale writers and cold restoration. The process-level regression in [the E2E crate](../../crates/e2e_test/src/scanner_wide_directory_test.rs) seeds objects through S3, restarts the server after a partial save, and checks the certified admin usage against a paginated S3 listing using the same scanner budget on both boots.
For every saved partial cache, the fixture also passes its progress through the production authenticated remote terminal-frame writer and stream consumer. A remote partial result must remain partial even when its progress reports visited objects. This covers the return-frame contract; it does not execute the remote RPC server, distributed locks, EC quorum persistence, mixed-version peers, process crashes, or fsync durability. The file backend models revision preconditions and persistence errors, not a concurrent object store.
-1
View File
@@ -420,7 +420,6 @@ object = { workspace = true, optional = true, features = ["read", "std", "elf"]
symbolic-demangle = { version = "=13.9.0", default-features = false, features = ["rust"], optional = true }
[dev-dependencies]
h2.workspace = true
rustfs-targets = { workspace = true, features = ["test-support"] }
uuid = { workspace = true, features = ["v4", "v5", "fast-rng", "macro-diagnostics"] }
rustfs-iam = { workspace = true, features = ["test-util"] }
+7 -17
View File
@@ -359,22 +359,18 @@ mod tests {
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, AtomicU64};
use time::OffsetDateTime;
/// An in-memory test store with the small authoritative revision surface
/// needed by derived-session authorization. All other persistence methods
/// remain unreachable in these gate tests.
#[derive(Clone, Default)]
struct EmptyStore {
regular_users: Arc<std::sync::RwLock<HashMap<String, (UserIdentity, String)>>>,
}
/// A `Store` whose methods are never invoked by these tests: the gate only
/// reaches the persistence layer for non-owner principals, and the
/// `IamCache` we build starts with an empty in-memory snapshot, so
/// `get_user` resolves to `None` (unknown user) before any `api` call.
#[derive(Clone)]
struct EmptyStore;
#[async_trait::async_trait]
impl Store for EmptyStore {
fn has_watcher(&self) -> bool {
false
}
async fn regular_user_revision(&self, name: &str) -> rustfs_iam::error::Result<Option<(UserIdentity, String)>> {
Ok(self.regular_users.read().expect("test store lock").get(name).cloned())
}
async fn save_iam_config<Item: Serialize + Send>(
&self,
_item: Item,
@@ -500,7 +496,7 @@ mod tests {
let (send_chan, _rx) = tokio::sync::mpsc::channel::<i64>(1);
IamCache {
cache: Cache::default(),
api: EmptyStore::default(),
api: EmptyStore,
state: Arc::new(AtomicU8::new(0)),
loading: Arc::new(AtomicBool::new(false)),
roles: HashMap::new(),
@@ -872,12 +868,6 @@ mod tests {
store
.cache
.add_or_update_user(&user.access_key, &UserIdentity::new(user.clone()), now);
store
.api
.regular_users
.write()
.expect("test store lock")
.insert(user.access_key.clone(), (UserIdentity::new(user.clone()), "self-service".into()));
store
.cache
.add_or_update_user_policy(&user.access_key, &MappedPolicy::new(""), now);
@@ -205,12 +205,6 @@ fn listen_notification_streams_close_after_identity_policy_or_bucket_revocation(
)
.await
.expect("listener");
let parent_credentials = rustfs_credentials::Credentials {
access_key: key.clone(),
secret_key: "listen-user-secret".into(),
status: "on".into(),
..Default::default()
};
let policy = serde_json::json!({"Version":"2012-10-17","Statement":[{"Effect":"Allow",
"Action":["s3:ListenNotification","s3:ListenBucketNotification"],"Resource":"arn:aws:s3:::*"}]});
iam.set_policy(
@@ -233,7 +227,7 @@ fn listen_notification_streams_close_after_identity_policy_or_bucket_revocation(
let mut cred = rustfs_policy::auth::get_new_credentials_with_metadata(&claims, "listen-root-secret")
.expect("signed temporary credentials");
cred.parent_user = key.clone();
iam.set_temp_user_from_parent(&cred, &parent_credentials)
iam.set_temp_user(&cred.access_key, &cred, None)
.await
.expect("temporary listener");
Some(cred)
+14 -39
View File
@@ -355,16 +355,9 @@ impl Operation for AddUser {
args.policy = None;
updated_at
} else {
if validate_admin_request(
req.headers(),
&cred,
owner,
false,
vec![Action::AdminAction(AdminAction::EnableUserAdminAction)],
req.policy_source_addr(),
)
.await
.is_ok()
if authorize_admin_envelope(req.envelope(), vec![Action::AdminAction(AdminAction::EnableUserAdminAction)])
.await
.is_ok()
{
iam_store.create_user(ak, &args).await
} else {
@@ -2088,46 +2081,28 @@ mod tests {
assert_shared_gate_wiring(block, handler, &[action], binds_credentials);
}
// These handlers need context the shared primary gate does not return, so they retain the
// explicit `check_key_valid` + `validate_admin_request` preamble. AddUser separately probes
// EnableUser after CreateUser authorization to decide whether a requested status may apply.
for (handler, marker, secondary_action) in [
("AddUser", "should_check_deny_only(ak, &cred)", Some("EnableUserAdminAction")),
("GetUserInfo", "should_check_deny_only(ak, &cred)", None),
("ImportIam", "if !owner {", None),
// AddUser and GetUserInfo derive a per-request `deny_only` from `should_check_deny_only`, and
// ImportIam consumes the `owner` flag while restoring service accounts. The shared gate pins
// `deny_only = false` and hands back only the credentials, so these three keep the explicit
// `check_key_valid` + `validate_admin_request` preamble.
for (handler, marker) in [
("AddUser", "should_check_deny_only(ak, &cred)"),
("GetUserInfo", "should_check_deny_only(ak, &cred)"),
("ImportIam", "if !owner {"),
] {
let block = source_block(production, &format!("impl Operation for {handler}"));
assert!(
block.contains(marker),
"{handler} must keep the context that rules out the shared primary gate ({marker})"
"{handler} must keep the context that rules out the shared gate ({marker})"
);
assert!(
!block.contains("authorize_admin_request("),
"{handler} must not be routed through the shared primary gate"
!block.contains("authorize_admin_envelope("),
"{handler} must not be routed through the shared gate"
);
assert!(
block.contains("validate_admin_request("),
"{handler} must keep its explicit validate_admin_request call"
);
if let Some(action) = secondary_action {
assert_eq!(
block.matches("authorize_admin_envelope(").count(),
1,
"{handler} must perform exactly one secondary capability check"
);
assert!(block.contains(&format!("AdminAction::{action}")), "{handler} must probe {action}");
let primary = block.find("validate_admin_request(").expect("explicit primary gate");
let secondary = block.find("authorize_admin_envelope(").expect("secondary capability gate");
assert!(
primary < secondary,
"{handler} must complete primary authorization before the secondary capability check"
);
} else {
assert!(
!block.contains("authorize_admin_envelope("),
"{handler} must not use a secondary shared gate"
);
}
}
}
}
+3
View File
@@ -3159,11 +3159,14 @@ mod tests {
..Default::default()
})
.expect("valid form policy")
.with_metadata(&[("purpose".to_owned(), "form".to_owned())])
.expect("valid form metadata")
.install_gateway(&mut req);
let projected = &req.extensions.get::<PostObjectPolicyHeaders>().expect("projected headers").0;
assert!(!projected.contains_key("if-match"));
assert_eq!(projected.get("if-none-match"), Some(&HeaderValue::from_static("*")));
assert_eq!(projected.get("x-amz-server-side-encryption"), Some(&HeaderValue::from_static("form")));
assert_eq!(projected.get("x-amz-meta-purpose"), Some(&HeaderValue::from_static("form")));
assert_eq!(
projected.get("x-amz-server-side-encryption-customer-algorithm"),
Some(&HeaderValue::from_static("AES256"))
-375
View File
@@ -1,375 +0,0 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Transport regressions for SDK clients that withhold DATA until 100 Continue.
use std::{convert::Infallible, future::Future, sync::Arc, time::Duration};
use bytes::Bytes;
use http::{Request, Response, StatusCode};
use http_body_util::{BodyExt, Empty};
use hyper::body::{Body, Incoming};
use hyper_util::{
rt::{TokioExecutor, TokioIo},
server::conn::auto::Builder,
service::TowerToHyperService,
};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
sync::Notify,
task::JoinHandle,
};
use tower::{Service, service_fn};
async fn bounded<F: Future>(future: F) -> F::Output {
tokio::time::timeout(Duration::from_secs(3), future)
.await
.expect("transport must progress while the client waits for Continue")
}
struct H2Client {
sender: h2::client::SendRequest<Bytes>,
tasks: [JoinHandle<()>; 2],
}
impl Drop for H2Client {
fn drop(&mut self) {
for task in &self.tasks {
task.abort();
}
}
}
async fn connect<S>(service: S) -> H2Client
where
S: Service<Request<Incoming>, Response = Response<Empty<Bytes>>, Error = Infallible> + Clone + Send + 'static,
S::Future: Send,
{
let (client, server) = tokio::io::duplex(64 * 1024);
let server_task = tokio::spawn(async move {
let _ = Builder::new(TokioExecutor::new())
.serve_connection(TokioIo::new(server), TowerToHyperService::new(service))
.await;
});
let (sender, connection) = h2::client::handshake(client).await.expect("H2 handshake");
let client_task = tokio::spawn(async move {
let _ = connection.await;
});
H2Client {
sender,
tasks: [server_task, client_task],
}
}
fn request(expect: Option<&str>, end_stream: bool) -> Request<()> {
let mut request = Request::put("https://localhost/bucket/key?partNumber=1&uploadId=test");
if let Some(expect) = expect {
request = request.header("expect", expect);
}
request
.header("content-length", if end_stream { "0" } else { "5" })
.body(())
.expect("request")
}
async fn consume(request: Request<Incoming>) -> Result<Response<Empty<Bytes>>, Infallible> {
assert_eq!(
request.into_body().collect().await.expect("request body").to_bytes(),
Bytes::from_static(b"hello")
);
Ok(Response::new(Empty::new()))
}
async fn continued(response: &mut h2::client::ResponseFuture) {
let info = std::future::poll_fn(|cx| response.poll_informational(cx))
.await
.expect("100 Continue before DATA")
.expect("valid informational response");
assert_eq!(info.status(), StatusCode::CONTINUE);
}
async fn final_response(mut response: h2::client::ResponseFuture, status: StatusCode) {
assert!(
std::future::poll_fn(|cx| response.poll_informational(cx)).await.is_none(),
"unexpected extra informational response"
);
assert_eq!(response.await.expect("final response").status(), status);
}
#[tokio::test]
async fn h2_expect_continue_unblocks_sequential_upload_parts_once_each() {
bounded(async {
let mut client = connect(service_fn(consume)).await;
for expect in ["100-continue", "100-CONTINUE"] {
let (mut response, mut body) = client
.sender
.send_request(request(Some(expect), false), false)
.expect("request headers");
continued(&mut response).await;
body.send_data(Bytes::from_static(b"hello"), true)
.expect("DATA after Continue");
final_response(response, StatusCode::OK).await;
}
})
.await;
}
#[tokio::test]
async fn h2_expect_continue_wakes_when_body_is_read_in_spawned_task() {
bounded(async {
let mut client = connect(service_fn(|request: Request<Incoming>| async move {
tokio::spawn(async move { consume(request).await })
.await
.expect("body reader task")
}))
.await;
let (mut response, mut body) = client
.sender
.send_request(request(Some("100-continue"), false), false)
.expect("request headers");
continued(&mut response).await;
body.send_data(Bytes::from_static(b"hello"), true)
.expect("DATA after Continue");
final_response(response, StatusCode::OK).await;
})
.await;
}
#[tokio::test]
async fn h2_expect_continue_preserves_size_hint_trailers_and_eof() {
bounded(async {
let mut client = connect(service_fn(|request: Request<Incoming>| async move {
let mut body = request.into_body();
assert_eq!(body.size_hint().exact(), Some(5));
assert!(!body.is_end_stream());
let data = body
.frame()
.await
.expect("DATA frame")
.expect("valid DATA")
.into_data()
.expect("first frame is DATA");
assert_eq!(data, Bytes::from_static(b"hello"));
assert_eq!(body.size_hint().exact(), Some(0));
let remaining = (&mut body).collect().await.expect("remaining frames");
assert_eq!(remaining.trailers().expect("request trailers")["x-test-trailer"], "preserved");
assert!(remaining.to_bytes().is_empty());
assert!(body.is_end_stream());
assert!(body.frame().await.is_none());
Ok::<_, Infallible>(Response::new(Empty::new()))
}))
.await;
// Both paths use the same H2 body representation after Continue has fired.
for expect in [Some("100-continue"), None] {
let (mut response, mut body) = client
.sender
.send_request(request(expect, false), false)
.expect("request headers");
if expect.is_some() {
continued(&mut response).await;
}
body.send_data(Bytes::from_static(b"hello"), false)
.expect("DATA without END_STREAM");
let mut trailers = http::HeaderMap::new();
trailers.insert("x-test-trailer", http::HeaderValue::from_static("preserved"));
body.send_trailers(trailers).expect("trailers with END_STREAM");
final_response(response, StatusCode::OK).await;
}
})
.await;
}
#[tokio::test]
async fn h2_without_expect_does_not_send_continue() {
bounded(async {
let mut client = connect(service_fn(consume)).await;
let (response, mut body) = client.sender.send_request(request(None, false), false).expect("request");
body.send_data(Bytes::from_static(b"hello"), true).expect("DATA");
final_response(response, StatusCode::OK).await;
})
.await;
}
#[tokio::test]
async fn h2_unknown_expectation_does_not_send_continue() {
bounded(async {
let mut client = connect(service_fn(consume)).await;
let (response, mut body) = client
.sender
.send_request(request(Some("unsupported-expectation"), false), false)
.expect("request headers");
body.send_data(Bytes::from_static(b"hello"), true).expect("eager DATA");
final_response(response, StatusCode::OK).await;
})
.await;
}
#[tokio::test]
async fn h2_expect_continue_waits_until_service_polls_body() {
bounded(async {
let entered = Arc::new(Notify::new());
let release = Arc::new(Notify::new());
let service_entered = Arc::clone(&entered);
let service_release = Arc::clone(&release);
let mut client = connect(service_fn(move |request: Request<Incoming>| {
let entered = Arc::clone(&service_entered);
let release = Arc::clone(&service_release);
async move {
entered.notify_one();
release.notified().await;
consume(request).await
}
}))
.await;
let (mut response, mut body) = client
.sender
.send_request(request(Some("100-continue"), false), false)
.expect("request headers");
entered.notified().await;
std::future::poll_fn(|cx| {
assert!(
response.poll_informational(cx).is_pending(),
"Continue must wait for the application's body poll"
);
std::task::Poll::Ready(())
})
.await;
release.notify_one();
continued(&mut response).await;
body.send_data(Bytes::from_static(b"hello"), true)
.expect("DATA after Continue");
final_response(response, StatusCode::OK).await;
})
.await;
}
#[tokio::test]
async fn h2_early_rejection_does_not_invite_upload() {
bounded(async {
let mut client = connect(service_fn(|request: Request<Incoming>| async move {
drop(request);
Ok::<_, Infallible>(
Response::builder()
.status(StatusCode::FORBIDDEN)
.body(Empty::new())
.expect("response"),
)
}))
.await;
let (response, _body) = client
.sender
.send_request(request(Some("100-continue"), false), false)
.expect("request headers");
final_response(response, StatusCode::FORBIDDEN).await;
})
.await;
}
#[tokio::test]
async fn h2_end_stream_does_not_send_continue() {
bounded(async {
let mut client = connect(service_fn(|request: Request<Incoming>| async move {
assert!(request.into_body().collect().await.expect("empty body").to_bytes().is_empty());
Ok::<_, Infallible>(Response::new(Empty::new()))
}))
.await;
let (response, _body) = client
.sender
.send_request(request(Some("100-continue"), true), true)
.expect("request headers");
final_response(response, StatusCode::OK).await;
})
.await;
}
#[tokio::test]
async fn h2_client_that_sends_body_without_waiting_completes() {
bounded(async {
let ready = Arc::new(Notify::new());
let gate = Arc::clone(&ready);
let mut client = connect(service_fn(move |request: Request<Incoming>| {
let gate = Arc::clone(&gate);
async move {
gate.notified().await;
consume(request).await
}
}))
.await;
let (mut response, mut body) = client
.sender
.send_request(request(Some("100-continue"), false), false)
.expect("request headers");
body.send_data(Bytes::from_static(b"hello"), true).expect("eager DATA");
ready.notify_one();
// Depending on transport scheduling, the first body poll can precede DATA arrival.
// In either case, an eager client must complete and receive at most one Continue.
if let Some(info) = std::future::poll_fn(|cx| response.poll_informational(cx)).await {
assert_eq!(info.expect("informational response").status(), StatusCode::CONTINUE);
}
final_response(response, StatusCode::OK).await;
})
.await;
}
#[tokio::test]
async fn h2_cancelled_waiting_upload_does_not_block_another_stream() {
bounded(async {
let mut client = connect(service_fn(|request: Request<Incoming>| async move {
if let Ok(body) = request.into_body().collect().await {
assert_eq!(body.to_bytes(), Bytes::from_static(b"hello"));
}
Ok::<_, Infallible>(Response::new(Empty::new()))
}))
.await;
let (mut cancelled, mut abandoned_body) = client
.sender
.send_request(request(Some("100-continue"), false), false)
.expect("first stream");
let (mut response, mut body) = client
.sender
.send_request(request(Some("100-continue"), false), false)
.expect("second stream");
continued(&mut cancelled).await;
abandoned_body.send_reset(h2::Reason::CANCEL);
drop(cancelled);
continued(&mut response).await;
body.send_data(Bytes::from_static(b"hello"), true)
.expect("second stream DATA");
final_response(response, StatusCode::OK).await;
})
.await;
}
#[tokio::test]
async fn http1_expect_continue_control() {
bounded(async {
let (mut client, server) = tokio::io::duplex(64 * 1024);
let server_task = tokio::spawn(async move {
let _ = Builder::new(TokioExecutor::new())
.serve_connection(TokioIo::new(server), TowerToHyperService::new(service_fn(consume)))
.await;
});
client.write_all(b"PUT /bucket/key HTTP/1.1\r\nHost: localhost\r\nContent-Length: 5\r\nExpect: 100-continue\r\nConnection: close\r\n\r\n").await.expect("headers");
let mut informational = Vec::new();
while !informational.ends_with(b"\r\n\r\n") {
informational.push(client.read_u8().await.expect("Continue bytes"));
}
assert_eq!(informational, b"HTTP/1.1 100 Continue\r\n\r\n");
client.write_all(b"hello").await.expect("body after Continue");
let mut response = Vec::new();
client.read_to_end(&mut response).await.expect("final response");
assert!(response.starts_with(b"HTTP/1.1 200 OK\r\n"));
server_task.await.expect("server task");
}).await;
}
-346
View File
@@ -1,346 +0,0 @@
# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
#
# When uploading crates to the registry Cargo will automatically
# "normalize" Cargo.toml files for maximal compatibility
# with all versions of Cargo and also rewrite `path` dependencies
# to registry (e.g., crates.io) dependencies.
#
# If you are reading this file be aware that the original Cargo.toml
# will likely look very different (and much more reasonable).
# See Cargo.toml.orig for the original contents.
[package]
edition = "2021"
rust-version = "1.63"
name = "hyper"
version = "1.12.0"
authors = ["Sean McArthur <sean@seanmonstar.com>"]
build = false
include = [
"Cargo.toml",
"LICENSE",
"src/**/*",
]
autolib = false
autobins = false
autoexamples = false
autotests = false
autobenches = false
description = "A protective and efficient HTTP library for all."
homepage = "https://hyper.rs"
documentation = "https://docs.rs/hyper"
readme = "README.md"
keywords = [
"http",
"hyper",
"hyperium",
]
categories = [
"network-programming",
"web-programming::http-client",
"web-programming::http-server",
]
license = "MIT"
repository = "https://github.com/hyperium/hyper"
[package.metadata.docs.rs]
features = [
"ffi",
"full",
"tracing",
]
rustdoc-args = [
"--cfg",
"hyper_unstable_ffi",
"--cfg",
"hyper_unstable_tracing",
]
[package.metadata.playground]
features = ["full"]
[package.metadata.capi.header]
generation = false
subdirectory = false
[[package.metadata.capi.install.include.asset]]
from = "capi/include/hyper.h"
[features]
capi = []
client = [
"dep:want",
"dep:pin-project-lite",
"dep:smallvec",
]
default = []
ffi = [
"dep:http-body-util",
"dep:futures-util",
]
full = [
"client",
"http1",
"http2",
"server",
]
http1 = [
"dep:atomic-waker",
"dep:futures-core",
"dep:httparse",
"dep:itoa",
]
http2 = [
"dep:atomic-waker",
"dep:futures-channel",
"dep:futures-core",
"dep:h2",
]
nightly = []
server = [
"dep:httpdate",
"dep:pin-project-lite",
"dep:smallvec",
]
tracing = ["dep:tracing"]
[lib]
name = "hyper"
path = "src/lib.rs"
[dependencies.atomic-waker]
version = "1.1.2"
optional = true
[dependencies.bytes]
version = "1.2"
[dependencies.futures-channel]
version = "0.3"
optional = true
[dependencies.futures-core]
version = "0.3.31"
optional = true
[dependencies.futures-util]
version = "0.3"
features = ["alloc"]
optional = true
default-features = false
[dependencies.h2]
version = "0.4.14"
optional = true
[dependencies.http]
version = "1"
[dependencies.http-body]
version = "1"
[dependencies.http-body-util]
version = "0.1"
optional = true
[dependencies.httparse]
version = "1.9"
optional = true
[dependencies.httpdate]
version = "1.0"
optional = true
[dependencies.itoa]
version = "1"
optional = true
[dependencies.pin-project-lite]
version = "0.2.4"
optional = true
[dependencies.smallvec]
version = "1.12"
features = [
"const_generics",
"const_new",
]
optional = true
[dependencies.tokio]
version = "1"
features = ["sync"]
[dependencies.tracing]
version = "0.1"
features = ["std"]
optional = true
default-features = false
[dependencies.want]
version = "0.3.2"
optional = true
[dev-dependencies.form_urlencoded]
version = "1"
[dev-dependencies.futures-channel]
version = "0.3"
features = ["sink"]
[dev-dependencies.futures-util]
version = "0.3"
features = [
"alloc",
"sink",
]
default-features = false
[dev-dependencies.http-body-util]
version = "0.1"
[dev-dependencies.pin-project-lite]
version = "0.2.4"
[dev-dependencies.pretty_env_logger]
version = "0.5"
[dev-dependencies.serde]
version = "1.0"
features = ["derive"]
[dev-dependencies.serde_json]
version = "1.0"
[dev-dependencies.spmc]
version = "0.3"
[dev-dependencies.tokio]
version = "1"
features = [
"fs",
"macros",
"net",
"io-std",
"io-util",
"rt",
"rt-multi-thread",
"sync",
"time",
"test-util",
]
[dev-dependencies.tokio-test]
version = "0.4"
[dev-dependencies.tokio-util]
version = "0.7.10"
[dev-dependencies.tracing]
version = "0.1"
[dev-dependencies.tracing-subscriber]
version = "0.3"
[lints.clippy]
absolute_paths = "allow"
arbitrary_source_item_ordering = "allow"
arithmetic_side_effects = "allow"
as_conversions = "allow"
blanket_clippy_restriction_lints = "allow"
cast_precision_loss = "allow"
checked_conversions = "allow"
clone_on_ref_ptr = "allow"
cognitive_complexity = "allow"
decimal_literal_representation = "allow"
default_numeric_fallback = "allow"
definition_in_module_root = "allow"
else_if_without_else = "allow"
error_impl_error = "allow"
expect_used = "allow"
field_scoped_visibility_modifiers = "allow"
float_arithmetic = "allow"
if_not_else = "allow"
if_then_some_else_none = "allow"
impl_trait_in_params = "allow"
implicit_return = "allow"
indexing_slicing = "allow"
inline_modules = "allow"
inline_trait_bounds = "allow"
items_after_statements = "allow"
let_underscore_must_use = "allow"
let_underscore_untyped = "allow"
let_unit_value = "allow"
match_wild_err_arm = "allow"
min_ident_chars = "allow"
missing_assert_message = "allow"
missing_docs_in_private_items = "allow"
missing_inline_in_public_items = "allow"
missing_trait_methods = "allow"
mod_module_files = "allow"
module_inception = "allow"
module_name_repetitions = "allow"
multiple_inherent_impl = "allow"
multiple_unsafe_ops_per_block = "allow"
must_use_candidate = "allow"
needless_bool = "allow"
needless_pass_by_value = "allow"
new_without_default = "allow"
panic_in_result_fn = "allow"
pattern_type_mismatch = "allow"
pub_use = "allow"
pub_with_shorthand = "allow"
question_mark_used = "allow"
redundant_closure_for_method_calls = "allow"
redundant_else = "allow"
renamed_function_params = "allow"
rest_pattern_accessible_field = "allow"
same_name_method = "allow"
self_named_module_files = "deny"
semicolon_inside_block = "allow"
semicolon_outside_block = "allow"
shadow_reuse = "allow"
shadow_same = "allow"
shadow_unrelated = "allow"
single_call_fn = "allow"
std_instead_of_alloc = "allow"
std_instead_of_core = "allow"
struct_excessive_bools = "allow"
struct_field_names = "allow"
too_many_lines = "allow"
trivially_copy_pass_by_ref = "allow"
type_complexity = "allow"
unnecessary_rest_pattern = "allow"
unnecessary_safety_comment = "allow"
unnested_or_patterns = "allow"
unreachable = "allow"
unseparated_literal_suffix = "allow"
unused_async = "allow"
unused_trait_names = "allow"
unwrap_in_result = "allow"
used_underscore_items = "allow"
wildcard_enum_match_arm = "allow"
[lints.clippy.pedantic]
level = "warn"
priority = -1
[lints.clippy.restriction]
level = "warn"
priority = -2
[lints.rust.unexpected_cfgs]
level = "warn"
priority = 0
check-cfg = [
"cfg(hyper_unstable_tracing)",
"cfg(hyper_unstable_ffi)",
]
[profile.bench]
codegen-units = 1
incremental = false
[profile.release]
codegen-units = 1
incremental = false
-19
View File
@@ -1,19 +0,0 @@
Copyright (c) 2014-2026 Sean McArthur
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE.
-44
View File
@@ -1,44 +0,0 @@
# [hyper](https://hyper.rs)
[![crates.io](https://img.shields.io/crates/v/hyper.svg)](https://crates.io/crates/hyper)
[![Released API docs](https://docs.rs/hyper/badge.svg)](https://docs.rs/hyper)
[![MIT licensed](https://img.shields.io/badge/license-MIT-blue.svg)](./LICENSE)
[![CI](https://github.com/hyperium/hyper/workflows/CI/badge.svg)](https://github.com/hyperium/hyper/actions?query=workflow%3ACI)
[![Discord chat][discord-badge]][discord-url]
A protective and efficient HTTP library for all.
- HTTP/1 and HTTP/2
- Asynchronous design
- Leading in performance
- Tested and **correct**
- Extensive production use
- Client and Server APIs
**Get started** by looking over the [guides](https://hyper.rs/guides/1/).
## "Low-level"
hyper is a relatively low-level library, meant to be a building block for
libraries and applications.
If you are looking for a convenient HTTP client, then you may wish to consider
[reqwest](https://github.com/seanmonstar/reqwest).
If you are not sure what HTTP server to choose, then you may want to consider
[axum](https://github.com/tokio-rs/axum) or
[warp](https://github.com/seanmonstar/warp), the latter taking a more functional
approach. Both are built on top of this library.
## Contributing
To get involved, take a look at [CONTRIBUTING](CONTRIBUTING.md).
If you prefer chatting, there is an active community in the [Discord server][discord-url].
## License
hyper is provided under the MIT license. See [LICENSE](LICENSE).
[discord-badge]: https://img.shields.io/discord/500028886025895936.svg?logo=discord
[discord-url]: https://discord.gg/kkwpueZ
-11
View File
@@ -1,11 +0,0 @@
# Hyper HTTP/2 Expect support
This directory contains the published Hyper 1.12.0 source with the four-file runtime backport of [hyperium/hyper#4217](https://github.com/hyperium/hyper/pull/4217), commit `6d363f774910ac069a830783c23e409c59371fb1`. The released source corresponds to upstream commit `7f67699ce32d717081deb7018b4da7d679277ac0` and crates.io SHA-256 `2c3e324da4c95177d6291d4c8730197c0d1822f8a9766814a4a44fa5ab797c9c`.
The patch sends one HTTP/2 100 Continue response when the service first waits for request body data. A final response without reading the body takes precedence. HTTP/1 behavior is preserved. Non-Expect traffic uses the existing path without a new allocation. No S3 handler or storage behavior is changed.
The upstream PR is not yet released. Its branch is based on Hyper 1.11.1, so pinning that branch would discard newer HTTP/1 framing fixes. The workspace Cargo patch retains 1.12.0 and applies uniformly to transitive Hyper users; this also affects internode and outbound clients, although the new behavior is server-side only.
Regression coverage lives in `rustfs/tests/http_expect_continue.rs` and runs as a normal RustFS integration test. Remove the Cargo patch and this directory when a released Hyper includes equivalent support; keep the regression tests. See [RustFS #9460](https://github.com/rustfs/rustfs/issues/9460).
The upstream MIT license is retained in `LICENSE`. Apart from this provenance document, upstream metadata and source files are preserved to keep the backport directly auditable.
-316
View File
@@ -1,316 +0,0 @@
use std::fmt;
use std::sync::{Arc, Mutex};
use std::task::{Context, Poll};
use atomic_waker::AtomicWaker;
use bytes::Bytes;
use http::HeaderMap;
use crate::common::lock::LockResultExt;
pub(crate) fn channel(wanter: bool) -> (Sender, Receiver) {
let shared = Arc::new(Shared {
state: Mutex::new(State {
item: None,
pending_error: None,
trailers: None,
sender_open: true,
receiver_open: true,
want: !wanter,
}),
sender_waker: AtomicWaker::new(),
receiver_waker: AtomicWaker::new(),
});
(
Sender {
shared: Arc::clone(&shared),
trailers_sent: false,
},
Receiver {
shared,
terminated: false,
},
)
}
#[must_use = "Sender does nothing unless sent on"]
pub(crate) struct Sender {
shared: Arc<Shared>,
trailers_sent: bool,
}
pub(crate) struct Receiver {
shared: Arc<Shared>,
terminated: bool,
}
struct Shared {
state: Mutex<State>,
sender_waker: AtomicWaker,
receiver_waker: AtomicWaker,
}
struct State {
item: Option<Result<Bytes, crate::Error>>,
// An error must not displace data which was already accepted. The old
// mpsc channel achieved this by sending the error from a cloned sender.
pending_error: Option<crate::Error>,
trailers: Option<HeaderMap>,
sender_open: bool,
receiver_open: bool,
want: bool,
}
impl Sender {
pub(crate) fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<crate::Result<()>> {
self.shared.sender_waker.register(cx.waker());
let state = self.shared.state.lock().panic_if_poisoned();
if !state.receiver_open {
Poll::Ready(Err(crate::Error::new_closed()))
} else if state.want && state.item.is_none() && state.pending_error.is_none() {
Poll::Ready(Ok(()))
} else {
Poll::Pending
}
}
#[cfg(test)]
pub(crate) async fn ready(&mut self) -> crate::Result<()> {
futures_util::future::poll_fn(|cx| self.poll_ready(cx)).await
}
#[cfg(test)]
#[allow(dead_code)]
pub(crate) async fn send_data(&mut self, chunk: Bytes) -> crate::Result<()> {
self.ready().await?;
self.try_send_data(chunk)
.map_err(|_| crate::Error::new_closed())
}
pub(crate) fn try_send_data(&mut self, chunk: Bytes) -> Result<(), Bytes> {
let mut state = self.shared.state.lock().panic_if_poisoned();
if !state.receiver_open
|| !state.want
|| state.item.is_some()
|| state.pending_error.is_some()
{
return Err(chunk);
}
state.item = Some(Ok(chunk));
drop(state);
self.shared.receiver_waker.wake();
Ok(())
}
pub(crate) fn try_send_trailers(
&mut self,
trailers: HeaderMap,
) -> Result<(), Option<HeaderMap>> {
if self.trailers_sent {
return Err(None);
}
self.trailers_sent = true;
let mut state = self.shared.state.lock().panic_if_poisoned();
if !state.receiver_open {
return Err(Some(trailers));
}
state.trailers = Some(trailers);
drop(state);
self.shared.receiver_waker.wake();
Ok(())
}
#[allow(dead_code)]
#[allow(clippy::unused_async_trait_impl)]
#[allow(clippy::map_err_ignore)]
pub(crate) async fn send_trailers(&mut self, trailers: HeaderMap) -> crate::Result<()> {
self.try_send_trailers(trailers)
.map_err(|_| crate::Error::new_closed())
}
pub(crate) fn send_error(&mut self, err: crate::Error) {
let mut state = self.shared.state.lock().panic_if_poisoned();
if !state.receiver_open {
return;
}
if state.item.is_none() {
state.item = Some(Err(err));
} else if state.pending_error.is_none() {
state.pending_error = Some(err);
}
drop(state);
self.shared.receiver_waker.wake();
}
#[cfg(test)]
pub(crate) fn abort(mut self) {
self.send_error(crate::Error::new_body_write_aborted());
}
fn is_closed(&self) -> bool {
!self.shared.state.lock().panic_if_poisoned().receiver_open
}
}
impl Receiver {
pub(crate) fn poll_next(
&mut self,
cx: &mut Context<'_>,
) -> Poll<Option<Result<bytes::Bytes, crate::Error>>> {
if self.terminated {
return Poll::Ready(None);
}
self.shared.receiver_waker.register(cx.waker());
let mut state = self.shared.state.lock().panic_if_poisoned();
let wake_sender = if !state.want {
state.want = true;
true
} else {
false
};
if let Some(item) = state.item.take() {
drop(state);
self.shared.sender_waker.wake();
return Poll::Ready(Some(item));
}
if let Some(err) = state.pending_error.take() {
drop(state);
if wake_sender {
self.shared.sender_waker.wake();
}
return Poll::Ready(Some(Err(err)));
}
let sender_open = state.sender_open;
drop(state);
if wake_sender {
self.shared.sender_waker.wake();
}
if sender_open {
Poll::Pending
} else {
self.terminated = true;
Poll::Ready(None)
}
}
pub(crate) fn take_trailers(&mut self) -> Option<HeaderMap> {
debug_assert!(self.terminated, "data channel still open before trailers");
let mut state = self.shared.state.lock().panic_if_poisoned();
state.trailers.take()
}
}
impl Drop for Sender {
fn drop(&mut self) {
let mut state = self.shared.state.lock().panic_if_poisoned();
state.sender_open = false;
let receiver_open = state.receiver_open;
drop(state);
if receiver_open {
self.shared.receiver_waker.wake();
}
}
}
impl Drop for Receiver {
fn drop(&mut self) {
let mut state = self.shared.state.lock().panic_if_poisoned();
state.receiver_open = false;
let sender_open = state.sender_open;
drop(state);
if sender_open {
self.shared.sender_waker.wake();
}
}
}
impl fmt::Debug for Sender {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
if self.is_closed() {
f.debug_tuple("Sender").field(&"Closed").finish()
} else {
f.debug_tuple("Sender").field(&"Open").finish()
}
}
}
#[cfg(test)]
mod tests {
use std::{
sync::atomic::{AtomicUsize, Ordering},
task::{Wake, Waker},
};
use super::*;
async fn recv(rx: &mut Receiver) -> Option<Result<Bytes, crate::Error>> {
futures_util::future::poll_fn(|cx| rx.poll_next(cx)).await
}
#[tokio::test]
async fn error_queued_behind_accepted_data() {
let (mut tx, mut rx) = channel(false);
tx.try_send_data(Bytes::from_static(b"data")).unwrap();
tx.send_error(crate::Error::new_incomplete());
drop(tx);
assert_eq!(recv(&mut rx).await.unwrap().unwrap(), "data");
assert!(recv(&mut rx).await.unwrap().is_err());
assert!(recv(&mut rx).await.is_none());
}
#[tokio::test]
async fn trailers_follow_data_close() {
let (mut tx, mut rx) = channel(false);
let mut trailers = HeaderMap::new();
trailers.insert("x-trailer", "value".parse().unwrap());
tx.try_send_trailers(trailers).unwrap();
drop(tx);
assert!(recv(&mut rx).await.is_none());
assert_eq!(rx.take_trailers().unwrap()["x-trailer"], "value");
}
#[test]
fn drop_wakes_only_an_open_peer() {
struct Count(AtomicUsize);
impl Wake for Count {
fn wake(self: Arc<Self>) {
self.0.fetch_add(1, Ordering::Relaxed);
}
}
let count = Arc::new(Count(AtomicUsize::new(0)));
let waker = Waker::from(count.clone());
let mut cx = Context::from_waker(&waker);
let wakes = || count.0.load(Ordering::Relaxed);
// A receiver dropped after the sender does not wake the sender's stale waker.
let (mut tx, rx) = channel(true);
assert!(tx.poll_ready(&mut cx).is_pending());
drop(tx);
drop(rx);
assert_eq!(wakes(), 0);
// Nor does a sender dropped after the receiver.
let (tx, mut rx) = channel(false);
assert!(rx.poll_next(&mut cx).is_pending());
drop(rx);
drop(tx);
assert_eq!(wakes(), 0);
// An open peer is still woken to observe the close.
let (mut tx, rx) = channel(true);
assert!(tx.poll_ready(&mut cx).is_pending());
drop(rx);
assert_eq!(wakes(), 1);
let (tx, mut rx) = channel(false);
assert!(rx.poll_next(&mut cx).is_pending());
drop(tx);
assert_eq!(wakes(), 2);
}
}
-560
View File
@@ -1,560 +0,0 @@
use std::fmt;
use std::pin::Pin;
use std::task::{Context, Poll};
use bytes::Bytes;
#[cfg(all(feature = "http2", feature = "server"))]
use futures_channel::oneshot;
#[cfg(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
))]
use futures_core::ready;
use http_body::{Body, Frame, SizeHint};
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
use super::chan;
#[cfg(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
))]
use super::DecodedLength;
#[cfg(all(feature = "http2", any(feature = "client", feature = "server")))]
use crate::proto::h2::ping;
/// A stream of `Bytes`, used when receiving bodies from the network.
///
/// Note that Users should not instantiate this struct directly. When working with the hyper client,
/// `Incoming` is returned to you in responses. Similarly, when operating with the hyper server,
/// it is provided within requests.
///
/// # Examples
///
/// ```rust,ignore
/// async fn echo(
/// req: Request<hyper::body::Incoming>,
/// ) -> Result<Response<BoxBody<Bytes, hyper::Error>>, hyper::Error> {
/// //Here, you can process `Incoming`
/// }
/// ```
#[must_use = "streams do nothing unless polled"]
pub struct Incoming {
kind: Kind,
}
enum Kind {
Empty,
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
Chan {
content_length: DecodedLength,
rx: chan::Receiver,
},
#[cfg(all(feature = "http2", any(feature = "client", feature = "server")))]
H2 {
content_length: DecodedLength,
data_done: bool,
ping: ping::Recorder,
recv: h2::RecvStream,
},
#[cfg(all(feature = "http2", feature = "server"))]
ExpectContinue(Box<ExpectContinue>),
#[cfg(feature = "ffi")]
Ffi(crate::ffi::UserBody),
}
#[cfg(all(feature = "http2", feature = "server"))]
struct ExpectContinue {
body: Incoming,
tx: oneshot::Sender<()>,
}
/// A sender half created through [`Body::channel()`].
///
/// Useful when wanting to stream chunks from another thread.
///
/// ## Body Closing
///
/// Note that the request body will always be closed normally when the sender is dropped (meaning
/// that the empty terminating chunk will be sent to the remote). If you desire to close the
/// connection with an incomplete response (e.g. in the case of an error during asynchronous
/// processing), call the [`Sender::abort()`] method to abort the body in an abnormal fashion.
///
/// [`Body::channel()`]: struct.Body.html#method.channel
/// [`Sender::abort()`]: struct.Sender.html#method.abort
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
pub(crate) use super::chan::Sender;
impl Incoming {
/// Create a `Body` stream with an associated sender half.
///
/// Useful when wanting to stream chunks from another thread.
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[inline]
#[cfg(test)]
pub(crate) fn channel() -> (Sender, Incoming) {
Self::new_channel(DecodedLength::CHUNKED, /*wanter =*/ false)
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
pub(crate) fn new_channel(content_length: DecodedLength, wanter: bool) -> (Sender, Incoming) {
let (tx, rx) = chan::channel(wanter);
let rx = Incoming::new(Kind::Chan { content_length, rx });
(tx, rx)
}
fn new(kind: Kind) -> Incoming {
Incoming { kind }
}
#[allow(dead_code)]
pub(crate) fn empty() -> Incoming {
Incoming::new(Kind::Empty)
}
#[cfg(feature = "ffi")]
pub(crate) fn ffi() -> Incoming {
Incoming::new(Kind::Ffi(crate::ffi::UserBody::new()))
}
#[cfg(all(feature = "http2", any(feature = "client", feature = "server")))]
pub(crate) fn h2(
recv: h2::RecvStream,
mut content_length: DecodedLength,
ping: ping::Recorder,
) -> Self {
// If the stream is already EOS, then the "unknown length" is clearly
// actually ZERO.
if !content_length.is_exact() && recv.is_end_stream() {
content_length = DecodedLength::ZERO;
}
Incoming::new(Kind::H2 {
data_done: false,
ping,
content_length,
recv,
})
}
#[cfg(all(feature = "http2", feature = "server"))]
pub(crate) fn with_expect_continue(self, tx: oneshot::Sender<()>) -> Self {
Incoming::new(Kind::ExpectContinue(Box::new(ExpectContinue {
body: self,
tx,
})))
}
#[cfg(feature = "ffi")]
pub(crate) fn as_ffi_mut(&mut self) -> &mut crate::ffi::UserBody {
if !matches!(self.kind, Kind::Ffi(_)) {
self.kind = Kind::Ffi(crate::ffi::UserBody::new());
}
match &mut self.kind {
Kind::Ffi(body) => body,
_ => unreachable!(),
}
}
}
impl Body for Incoming {
type Data = Bytes;
type Error = crate::Error;
fn poll_frame(
#[cfg_attr(
not(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
)),
allow(unused_mut)
)]
mut self: Pin<&mut Self>,
#[cfg_attr(
not(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
)),
allow(unused_variables)
)]
cx: &mut Context<'_>,
) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
match &mut self.kind {
Kind::Empty => Poll::Ready(None),
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
Kind::Chan {
content_length: len,
rx,
} => {
if let Some(chunk) = ready!(rx.poll_next(cx)?) {
len.sub_if(chunk.len() as u64);
return Poll::Ready(Some(Ok(Frame::data(chunk))));
}
Poll::Ready(rx.take_trailers().map(Frame::trailers).map(Ok))
}
#[cfg(all(feature = "http2", any(feature = "client", feature = "server")))]
Kind::H2 {
data_done,
ping,
recv: h2,
content_length: len,
} => {
if !*data_done {
match ready!(h2.poll_data(cx)) {
Some(Ok(bytes)) => {
let _ = h2.flow_control().release_capacity(bytes.len());
len.sub_if(bytes.len() as u64);
ping.record_data(bytes.len());
return Poll::Ready(Some(Ok(Frame::data(bytes))));
}
Some(Err(e)) => {
if let Some(h2::Reason::NO_ERROR) = e.reason() {
// As mentioned in RFC 7540 Section 8.1, a RST_STREAM with NO_ERROR
// indicates an early response, and should cause the body reading
// to stop, but not fail it:
return Poll::Ready(None);
} else {
return Poll::Ready(Some(Err(crate::Error::new_body(e))));
}
}
None => {
*data_done = true;
// fall through to trailers
}
}
}
// after data, check trailers
match ready!(h2.poll_trailers(cx)) {
Ok(t) => {
ping.record_non_data();
Poll::Ready(Ok(t.map(Frame::trailers)).transpose())
}
Err(e) => {
if let Some(h2::Reason::NO_ERROR) = e.reason() {
// Same as above, a RST_STREAM with NO_ERROR indicates an early
// response, and should cause reading the trailers to stop, but
// not fail it:
Poll::Ready(None)
} else {
Poll::Ready(Some(Err(crate::Error::new_h2(e))))
}
}
}
}
#[cfg(all(feature = "http2", feature = "server"))]
Kind::ExpectContinue(_) => {
let expect = match std::mem::replace(&mut self.kind, Kind::Empty) {
Kind::ExpectContinue(expect) => expect,
_ => unreachable!(),
};
let ExpectContinue { body, tx } = *expect;
*self = body;
let res = self.as_mut().poll_frame(cx);
// Only ask the client to continue if the body hasn't already arrived.
if res.is_pending() {
let _ = tx.send(());
}
res
}
#[cfg(feature = "ffi")]
Kind::Ffi(body) => body.poll_data(cx),
}
}
fn is_end_stream(&self) -> bool {
match &self.kind {
Kind::Empty => true,
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
Kind::Chan { content_length, .. } => *content_length == DecodedLength::ZERO,
#[cfg(all(feature = "http2", any(feature = "client", feature = "server")))]
Kind::H2 { recv: h2, .. } => h2.is_end_stream(),
#[cfg(all(feature = "http2", feature = "server"))]
Kind::ExpectContinue(expect) => expect.body.is_end_stream(),
#[cfg(feature = "ffi")]
Kind::Ffi(..) => false,
}
}
fn size_hint(&self) -> SizeHint {
#[cfg(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
))]
fn opt_len(decoded_length: DecodedLength) -> SizeHint {
if let Some(content_length) = decoded_length.into_opt() {
SizeHint::with_exact(content_length)
} else {
SizeHint::default()
}
}
match &self.kind {
Kind::Empty => SizeHint::with_exact(0),
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
Kind::Chan { content_length, .. } => opt_len(*content_length),
#[cfg(all(feature = "http2", any(feature = "client", feature = "server")))]
Kind::H2 { content_length, .. } => opt_len(*content_length),
#[cfg(all(feature = "http2", feature = "server"))]
Kind::ExpectContinue(expect) => expect.body.size_hint(),
#[cfg(feature = "ffi")]
Kind::Ffi(..) => SizeHint::default(),
}
}
}
impl fmt::Debug for Incoming {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
#[cfg(any(
all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
),
feature = "ffi"
))]
#[derive(Debug)]
struct Streaming;
#[derive(Debug)]
struct Empty;
let mut builder = f.debug_tuple("Body");
match self.kind {
Kind::Empty => builder.field(&Empty),
#[cfg(any(
all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
),
feature = "ffi"
))]
_ => builder.field(&Streaming),
};
builder.finish()
}
}
#[cfg(test)]
mod tests {
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
use std::mem;
#[cfg(all(feature = "nightly", not(miri)))]
use std::pin::Pin;
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
use std::task::Poll;
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
use super::{Body, Incoming, SizeHint};
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
use super::{DecodedLength, Sender};
#[cfg(all(feature = "nightly", not(miri)))]
use bytes::Bytes;
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
use http_body_util::BodyExt;
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[test]
fn test_size_of() {
// These are mostly to help catch *accidentally* increasing
// the size by too much.
let body_size = mem::size_of::<Incoming>();
let body_expected_size = mem::size_of::<u64>() * 5;
assert!(
body_size <= body_expected_size,
"Body size = {} <= {}",
body_size,
body_expected_size,
);
assert_eq!(
mem::size_of::<Sender>(),
mem::size_of::<usize>() * 2,
"Sender"
);
assert_eq!(
mem::size_of::<Sender>(),
mem::size_of::<Option<Sender>>(),
"Option<Sender>"
);
}
#[cfg(all(feature = "nightly", not(miri)))]
#[bench]
fn bench_channel_create_and_drop(b: &mut test::Bencher) {
b.iter(|| {
let _ = test::black_box(Incoming::new_channel(
DecodedLength::CHUNKED,
/* wanter = */ false,
));
});
}
#[cfg(all(feature = "nightly", not(miri)))]
#[bench]
fn bench_channel_data_handoff(b: &mut test::Bencher) {
let (mut tx, mut body) =
Incoming::new_channel(DecodedLength::CHUNKED, /* wanter = */ false);
let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
b.iter(|| {
assert!(tx.poll_ready(&mut cx).is_ready());
tx.try_send_data(Bytes::from_static(b"hello world"))
.unwrap();
let frame = match Pin::new(&mut body).poll_frame(&mut cx) {
Poll::Ready(Some(Ok(frame))) => frame,
unexpected => panic!("unexpected body poll: {unexpected:?}"),
};
test::black_box(frame);
});
}
#[cfg(all(feature = "nightly", not(miri)))]
#[bench]
fn bench_channel_want_transition(b: &mut test::Bencher) {
let mut cx = std::task::Context::from_waker(std::task::Waker::noop());
b.iter(|| {
let (mut tx, mut body) =
Incoming::new_channel(DecodedLength::CHUNKED, /* wanter = */ true);
assert!(tx.poll_ready(&mut cx).is_pending());
assert!(Pin::new(&mut body).poll_frame(&mut cx).is_pending());
assert!(tx.poll_ready(&mut cx).is_ready());
let _ = test::black_box((tx, body));
});
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[test]
fn size_hint() {
fn eq(body: Incoming, b: SizeHint, note: &str) {
let a = body.size_hint();
assert_eq!(a.lower(), b.lower(), "lower for {:?}", note);
assert_eq!(a.upper(), b.upper(), "upper for {:?}", note);
}
eq(Incoming::empty(), SizeHint::with_exact(0), "empty");
eq(Incoming::channel().1, SizeHint::new(), "channel");
eq(
Incoming::new_channel(DecodedLength::new(4), /*wanter =*/ false).1,
SizeHint::with_exact(4),
"channel with length",
);
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[cfg(not(miri))]
#[tokio::test]
async fn channel_abort() {
let (tx, mut rx) = Incoming::channel();
tx.abort();
let err = rx.frame().await.unwrap().unwrap_err();
assert!(err.is_body_write_aborted(), "{:?}", err);
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[cfg(all(not(miri), feature = "http1"))]
#[tokio::test]
async fn channel_abort_when_buffer_is_full() {
let (mut tx, mut rx) = Incoming::channel();
tx.try_send_data("chunk 1".into()).expect("send 1");
// buffer is full, but can still send abort
tx.abort();
let chunk1 = rx
.frame()
.await
.expect("item 1")
.expect("chunk 1")
.into_data()
.unwrap();
assert_eq!(chunk1, "chunk 1");
let err = rx.frame().await.unwrap().unwrap_err();
assert!(err.is_body_write_aborted(), "{:?}", err);
}
#[cfg(feature = "http1")]
#[test]
fn channel_buffers_one() {
let (mut tx, _rx) = Incoming::channel();
tx.try_send_data("chunk 1".into()).expect("send 1");
// buffer is now full
let chunk2 = tx.try_send_data("chunk 2".into()).expect_err("send 2");
assert_eq!(chunk2, "chunk 2");
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[cfg(not(miri))]
#[tokio::test]
async fn channel_empty() {
let (_, mut rx) = Incoming::channel();
assert!(rx.frame().await.is_none());
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[test]
fn channel_ready() {
let (mut tx, _rx) = Incoming::new_channel(DecodedLength::CHUNKED, /*wanter = */ false);
let mut tx_ready = tokio_test::task::spawn(tx.ready());
assert!(tx_ready.poll().is_ready(), "tx is ready immediately");
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[test]
fn channel_wanter() {
let (mut tx, mut rx) =
Incoming::new_channel(DecodedLength::CHUNKED, /*wanter = */ true);
let mut tx_ready = tokio_test::task::spawn(tx.ready());
let mut rx_data = tokio_test::task::spawn(rx.frame());
assert!(
tx_ready.poll().is_pending(),
"tx isn't ready before rx has been polled"
);
assert!(rx_data.poll().is_pending(), "poll rx.data");
assert!(tx_ready.is_woken(), "rx poll wakes tx");
assert!(
tx_ready.poll().is_ready(),
"tx is ready after rx has been polled"
);
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
#[test]
fn channel_notices_closure() {
let (mut tx, rx) = Incoming::new_channel(DecodedLength::CHUNKED, /*wanter = */ true);
let mut tx_ready = tokio_test::task::spawn(tx.ready());
assert!(
tx_ready.poll().is_pending(),
"tx isn't ready before rx has been polled"
);
drop(rx);
assert!(tx_ready.is_woken(), "dropping rx wakes tx");
match &tx_ready.poll() {
Poll::Ready(Err(e)) if e.is_closed() => (),
unexpected => panic!("tx poll ready unexpected: {:?}", unexpected),
}
}
}
-129
View File
@@ -1,129 +0,0 @@
use std::fmt;
#[derive(Clone, Copy, PartialEq, Eq)]
pub(crate) struct DecodedLength(u64);
#[cfg(any(feature = "http1", feature = "http2"))]
impl From<Option<u64>> for DecodedLength {
fn from(len: Option<u64>) -> Self {
len.and_then(|len| {
// If the length is u64::MAX, oh well, just reported chunked.
Self::checked_new(len).ok()
})
.unwrap_or(DecodedLength::CHUNKED)
}
}
#[cfg(any(feature = "http1", feature = "http2", test))]
const MAX_LEN: u64 = u64::MAX - 2;
impl DecodedLength {
pub(crate) const CLOSE_DELIMITED: DecodedLength = DecodedLength(u64::MAX);
pub(crate) const CHUNKED: DecodedLength = DecodedLength(u64::MAX - 1);
pub(crate) const ZERO: DecodedLength = DecodedLength(0);
#[cfg(test)]
pub(crate) fn new(len: u64) -> Self {
debug_assert!(len <= MAX_LEN);
DecodedLength(len)
}
/// Takes the length as a content-length without other checks.
///
/// Should only be called if previously confirmed this isn't
/// `CLOSE_DELIMITED` or `CHUNKED`.
#[inline]
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(crate) fn danger_len(self) -> u64 {
debug_assert!(self.0 < Self::CHUNKED.0);
self.0
}
/// Converts to an Option<u64> representing a Known or Unknown length.
#[cfg(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
))]
pub(crate) fn into_opt(self) -> Option<u64> {
match self {
DecodedLength::CHUNKED | DecodedLength::CLOSE_DELIMITED => None,
DecodedLength(known) => Some(known),
}
}
/// Checks the `u64` is within the maximum allowed for content-length.
#[cfg(any(feature = "http1", feature = "http2"))]
pub(crate) fn checked_new(len: u64) -> Result<Self, crate::error::Parse> {
if len <= MAX_LEN {
Ok(DecodedLength(len))
} else {
warn!("content-length bigger than maximum: {} > {}", len, MAX_LEN);
Err(crate::error::Parse::TooLarge)
}
}
#[cfg(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
))]
pub(crate) fn sub_if(&mut self, amt: u64) {
match self {
&mut DecodedLength::CHUNKED | &mut DecodedLength::CLOSE_DELIMITED => (),
DecodedLength(known) => {
*known -= amt;
}
}
}
/// Returns whether this represents an exact length.
///
/// This includes 0, which of course is an exact known length.
///
/// It would return false if "chunked" or otherwise size-unknown.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
pub(crate) fn is_exact(&self) -> bool {
self.0 <= MAX_LEN
}
}
impl fmt::Debug for DecodedLength {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
DecodedLength::CLOSE_DELIMITED => f.write_str("CLOSE_DELIMITED"),
DecodedLength::CHUNKED => f.write_str("CHUNKED"),
DecodedLength(n) => f.debug_tuple("DecodedLength").field(&n).finish(),
}
}
}
impl fmt::Display for DecodedLength {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match *self {
DecodedLength::CLOSE_DELIMITED => f.write_str("close-delimited"),
DecodedLength::CHUNKED => f.write_str("chunked encoding"),
DecodedLength::ZERO => f.write_str("empty"),
DecodedLength(n) => write!(f, "content-length ({n} bytes)"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn sub_if_known() {
let mut len = DecodedLength::new(30);
len.sub_if(20);
assert_eq!(len.0, 10);
}
#[test]
fn sub_if_chunked() {
let mut len = DecodedLength::CHUNKED;
len.sub_if(20);
assert_eq!(len, DecodedLength::CHUNKED);
}
}
-97
View File
@@ -1,97 +0,0 @@
//! Streaming bodies for Requests and Responses.
//!
//! For both [Clients](crate::client) and [Servers](crate::server), requests and
//! responses use streaming bodies, instead of complete buffering. This
//! allows applications to not use memory they don't need, and allows exerting
//! back-pressure on connections by only reading when asked.
//!
//! There are two pieces to this in hyper:
//!
//! - **The [`Body`] trait** describes all possible bodies.
//! hyper allows any body type that implements `Body`, allowing
//! applications to have fine-grained control over their streaming.
//! - **The [`Incoming`] concrete type**, which is an implementation
//! of `Body`, and returned by hyper as a "receive stream" (so, for server
//! requests and client responses).
//!
//! There are additional implementations available in [`http-body-util`][],
//! such as a `Full` or `Empty` body.
//!
//! ## Reading a body
//!
//! The [`BodyExt`][] extension trait provides an asynchronous way to read the
//! frames of a body. A frame can contain either data or trailers:
//!
//! ```
//! use http_body_util::BodyExt as _;
//! use hyper::body::Incoming;
//!
//! async fn read_body(mut body: Incoming) -> Result<(), hyper::Error> {
//! while let Some(frame) = body.frame().await {
//! let frame = frame?;
//!
//! if let Some(data) = frame.data_ref() {
//! println!("received {} bytes", data.len());
//! }
//!
//! if let Some(trailers) = frame.trailers_ref() {
//! println!("received trailers: {trailers:?}");
//! }
//! }
//!
//! Ok(())
//! }
//! ```
//!
//! A body only advances when it is polled. Processing each frame before
//! polling for the next one preserves back-pressure on the connection.
//!
//! If a body is known to be small, it can be collected into memory instead:
//!
//! ```
//! use http_body_util::BodyExt as _;
//! use hyper::body::{Bytes, Incoming};
//!
//! /// Consider using `Limited` if the body is untrusted.
//! async fn read_entire_body(body: Incoming) -> Result<Bytes, hyper::Error> {
//! Ok(body.collect().await?.to_bytes())
//! }
//! ```
//!
//! Collecting buffers the whole body, so it should be avoided for large or
//! untrusted bodies unless their size is limited.
//!
//! [`http-body-util`]: https://docs.rs/http-body-util
//! [`BodyExt`]: https://docs.rs/http-body-util/latest/http_body_util/trait.BodyExt.html
pub use bytes::{Buf, Bytes};
pub use http_body::Body;
pub use http_body::Frame;
pub use http_body::SizeHint;
pub use self::incoming::Incoming;
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(crate) use self::incoming::Sender;
#[cfg(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
))]
pub(crate) use self::length::DecodedLength;
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
mod chan;
mod incoming;
#[cfg(all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server")
))]
mod length;
fn _assert_send_sync() {
fn _assert_send<T: Send>() {}
fn _assert_sync<T: Sync>() {}
_assert_send::<Incoming>();
_assert_sync::<Incoming>();
}
-44
View File
@@ -1,44 +0,0 @@
macro_rules! cfg_feature {
(
#![$meta:meta]
$($item:item)*
) => {
$(
#[cfg($meta)]
#[cfg_attr(docsrs, doc(cfg($meta)))]
$item
)*
}
}
macro_rules! cfg_proto {
($($item:item)*) => {
cfg_feature! {
#![all(
any(feature = "http1", feature = "http2"),
any(feature = "client", feature = "server"),
)]
$($item)*
}
}
}
cfg_proto! {
macro_rules! cfg_client {
($($item:item)*) => {
cfg_feature! {
#![feature = "client"]
$($item)*
}
}
}
macro_rules! cfg_server {
($($item:item)*) => {
cfg_feature! {
#![feature = "server"]
$($item)*
}
}
}
}
-687
View File
@@ -1,687 +0,0 @@
//! HTTP/1 client connections.
use std::error::Error as StdError;
use std::fmt;
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use crate::rt::{Read, Write};
use bytes::Bytes;
use futures_core::ready;
use http::{Request, Response};
use httparse::ParserConfig;
use super::super::dispatch::{self, TrySendError};
use crate::body::{Body, Incoming as IncomingBody};
use crate::proto;
type Dispatcher<T, B> =
proto::dispatch::Dispatcher<proto::dispatch::Client<B>, B, T, proto::h1::ClientTransaction>;
/// The sender side of an established connection.
pub struct SendRequest<B> {
dispatch: dispatch::Sender<Request<B>, Response<IncomingBody>>,
}
/// Deconstructed parts of a `Connection`.
///
/// This allows taking apart a `Connection` at a later time, in order to
/// reclaim the IO object, and additional related pieces.
#[derive(Debug)]
#[non_exhaustive]
pub struct Parts<T> {
/// The original IO object used in the handshake.
pub io: T,
/// A buffer of bytes that have been read but not processed as HTTP.
///
/// For instance, if the `Connection` is used for an HTTP upgrade request,
/// it is possible the server sent back the first bytes of the new protocol
/// along with the response upgrade.
///
/// You will want to check for any existing bytes if you plan to continue
/// communicating on the IO object.
pub read_buf: Bytes,
}
/// A future that processes all HTTP state for the IO object.
///
/// In most cases, this should just be spawned into an executor, so that it
/// can process incoming and outgoing messages, notice hangups, and the like.
///
/// Instances of this type are typically created via the [`handshake`] function.
///
/// # Drop behavior
///
/// Dropping the `Connection` will close the underlying IO resource.
/// Any in-flight requests that have not received a response will be
/// interrupted. If graceful shutdown is desired, poll the connection
/// until it completes instead of dropping.
#[must_use = "futures do nothing unless polled"]
pub struct Connection<T, B>
where
T: Read + Write,
B: Body + 'static,
{
inner: Dispatcher<T, B>,
}
impl<T, B> Connection<T, B>
where
T: Read + Write + Unpin,
B: Body + 'static,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
/// Return the inner IO object, and additional information.
///
/// Only works for HTTP/1 connections. HTTP/2 connections will panic.
pub fn into_parts(self) -> Parts<T> {
let (io, read_buf, _) = self.inner.into_inner();
Parts { io, read_buf }
}
/// Poll the connection for completion, but without calling `shutdown`
/// on the underlying IO.
///
/// This is useful to allow running a connection while doing an HTTP
/// upgrade. Once the upgrade is completed, the connection would be "done",
/// but it is not desired to actually shutdown the IO object. Instead you
/// would take it back using `into_parts`.
///
/// Use [`poll_fn`](https://docs.rs/futures/0.1.25/futures/future/fn.poll_fn.html)
/// and [`try_ready!`](https://docs.rs/futures/0.1.25/futures/macro.try_ready.html)
/// to work with this function; or use the `without_shutdown` wrapper.
pub fn poll_without_shutdown(&mut self, cx: &mut Context<'_>) -> Poll<crate::Result<()>> {
self.inner.poll_without_shutdown(cx)
}
/// Prevent shutdown of the underlying IO object at the end of service the request,
/// instead run `into_parts`. This is a convenience wrapper over `poll_without_shutdown`.
///
/// # Errors
///
/// Returns an error if the connection encounters an error while being polled to completion.
#[allow(clippy::missing_panics_doc)]
pub async fn without_shutdown(self) -> crate::Result<Parts<T>> {
let mut conn = Some(self);
crate::common::future::poll_fn(move |cx| -> Poll<crate::Result<Parts<T>>> {
ready!(conn
.as_mut()
.expect("client connection polled after completion")
.poll_without_shutdown(cx))?;
Poll::Ready(Ok(conn
.take()
.expect("client connection missing before completion")
.into_parts()))
})
.await
}
}
/// A builder to configure an HTTP connection.
///
/// After setting options, the builder is used to create a handshake future.
///
/// **Note**: The default values of options are *not considered stable*. They
/// are subject to change at any time.
#[derive(Clone, Debug)]
pub struct Builder {
h09_responses: bool,
h1_parser_config: ParserConfig,
h1_writev: Option<bool>,
h1_title_case_headers: bool,
h1_preserve_header_case: bool,
h1_max_headers: Option<usize>,
h1_max_header_size: Option<usize>,
#[cfg(feature = "ffi")]
h1_preserve_header_order: bool,
h1_read_buf_exact_size: Option<usize>,
h1_max_buf_size: Option<usize>,
}
/// Returns a handshake future over some IO.
///
/// This is a shortcut for `Builder::new().handshake(io)`.
/// See [`client::conn`](crate::client::conn) for more.
///
/// # Errors
///
/// Returns an error if the HTTP/1 connection handshake fails.
pub async fn handshake<T, B>(io: T) -> crate::Result<(SendRequest<B>, Connection<T, B>)>
where
T: Read + Write + Unpin,
B: Body + 'static,
B::Data: Send,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
Builder::new().handshake(io).await
}
// ===== impl SendRequest
impl<B> SendRequest<B> {
/// Polls to determine whether this sender can be used yet for a request.
///
/// If the associated connection is closed, this returns an Error.
pub fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<crate::Result<()>> {
self.dispatch.poll_ready(cx)
}
/// Waits until the dispatcher is ready.
///
/// # Errors
///
/// If the associated connection is closed, this returns an Error.
pub async fn ready(&mut self) -> crate::Result<()> {
crate::common::future::poll_fn(|cx| self.poll_ready(cx)).await
}
/// Checks if the connection is currently ready to send a request.
///
/// # Note
///
/// This is mostly a hint. Due to inherent latency of networks, it is
/// possible that even after checking this is ready, sending a request
/// may still fail because the connection was closed in the meantime.
pub fn is_ready(&self) -> bool {
self.dispatch.is_ready()
}
/// Checks if the connection side has been closed.
pub fn is_closed(&self) -> bool {
self.dispatch.is_closed()
}
}
impl<B> SendRequest<B>
where
B: Body + 'static,
{
/// Sends a `Request` on the associated connection.
///
/// Returns a future that if successful, yields the `Response`.
///
/// `req` must have a `Host` header.
///
/// # Uri
///
/// The `Uri` of the request is serialized as-is.
///
/// - Usually you want origin-form (`/path?query`).
/// - For sending to an HTTP proxy, you want to send in absolute-form
/// (`https://hyper.rs/guides`).
///
/// This is however not enforced or validated and it is up to the user
/// of this method to ensure the `Uri` is correct for their intended purpose.
///
/// # Cancel safety
///
/// Dropping the returned future is the supported way to cancel an
/// in-flight HTTP/1 request. Because HTTP/1 has no in-protocol way to
/// abort a single request without affecting the shared connection,
/// hyper closes the underlying connection when a request future is
/// dropped before completion. Any subsequent calls on the same
/// [`SendRequest`] will return a `canceled` error.
///
/// # Errors
///
/// Returns an error if the connection is not ready or if an error occurs while
/// processing the request.
pub fn send_request(
&mut self,
req: Request<B>,
) -> impl Future<Output = crate::Result<Response<IncomingBody>>> {
let sent = self.dispatch.send(req);
async move {
match sent {
Ok(rx) => match rx.await {
Ok(Ok(resp)) => Ok(resp),
Ok(Err(err)) => Err(err),
// this is definite bug if it happens, but it shouldn't happen!
Err(_canceled) => unreachable!("dispatch dropped without returning error"),
},
Err(_req) => {
debug!("connection was not ready");
Err(crate::Error::new_canceled().with("connection was not ready"))
}
}
}
}
/// Sends a `Request` on the associated connection.
///
/// Returns a future that if successful, yields the `Response`.
///
/// # Errors
///
/// If there was an error before trying to serialize the request to the
/// connection, the message will be returned as part of this error.
#[allow(clippy::result_large_err)]
pub fn try_send_request(
&mut self,
req: Request<B>,
) -> impl Future<Output = Result<Response<IncomingBody>, TrySendError<Request<B>>>> {
let sent = self.dispatch.try_send(req);
async move {
match sent {
Ok(rx) => match rx.await {
Ok(Ok(res)) => Ok(res),
Ok(Err(err)) => Err(err),
// this is definite bug if it happens, but it shouldn't happen!
Err(_) => unreachable!("dispatch dropped without returning error"),
},
Err(req) => {
debug!("connection was not ready");
let error = crate::Error::new_canceled().with("connection was not ready");
Err(TrySendError {
error,
message: Some(req),
})
}
}
}
}
}
impl<B> fmt::Debug for SendRequest<B> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SendRequest").finish()
}
}
// ===== impl Connection
impl<T, B> Connection<T, B>
where
T: Read + Write + Unpin + Send,
B: Body + 'static,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
/// Enable this connection to support higher-level HTTP upgrades.
///
/// See [the `upgrade` module](crate::upgrade) for more.
pub fn with_upgrades(self) -> upgrades::UpgradeableConnection<T, B> {
upgrades::UpgradeableConnection { inner: Some(self) }
}
}
impl<T, B> fmt::Debug for Connection<T, B>
where
T: Read + Write + fmt::Debug,
B: Body + 'static,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Connection").finish()
}
}
impl<T, B> Future for Connection<T, B>
where
T: Read + Write + Unpin,
B: Body + 'static,
B::Data: Send,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
type Output = crate::Result<()>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match ready!(Pin::new(&mut self.inner).poll(cx))? {
proto::Dispatched::Shutdown => Poll::Ready(Ok(())),
proto::Dispatched::Upgrade(pending) => {
// With no `Send` bound on `I`, we can't try to do
// upgrades here. In case a user was trying to use
// `upgrade` with this API, send a special
// error letting them know about that.
pending.manual();
Poll::Ready(Ok(()))
}
}
}
}
// ===== impl Builder
impl Builder {
/// Creates a new connection builder.
#[inline]
pub fn new() -> Builder {
Builder {
h09_responses: false,
h1_writev: None,
h1_read_buf_exact_size: None,
h1_parser_config: ParserConfig::default(),
h1_title_case_headers: false,
h1_preserve_header_case: false,
h1_max_headers: None,
h1_max_header_size: None,
#[cfg(feature = "ffi")]
h1_preserve_header_order: false,
h1_max_buf_size: None,
}
}
/// Set whether HTTP/0.9 responses should be tolerated.
///
/// Default is false.
pub fn http09_responses(&mut self, enabled: bool) -> &mut Builder {
self.h09_responses = enabled;
self
}
/// Set whether HTTP/1 connections will accept spaces between header names
/// and the colon that follow them in responses.
///
/// You probably don't need this, here is what [RFC 7230 Section 3.2.4.] has
/// to say about it:
///
/// > No whitespace is allowed between the header field-name and colon. In
/// > the past, differences in the handling of such whitespace have led to
/// > security vulnerabilities in request routing and response handling. A
/// > server MUST reject any received request message that contains
/// > whitespace between a header field-name and colon with a response code
/// > of 400 (Bad Request). A proxy MUST remove any such whitespace from a
/// > response message before forwarding the message downstream.
///
/// Default is false.
///
/// [RFC 7230 Section 3.2.4.]: https://tools.ietf.org/html/rfc7230#section-3.2.4
pub fn allow_spaces_after_header_name_in_responses(&mut self, enabled: bool) -> &mut Builder {
self.h1_parser_config
.allow_spaces_after_header_name_in_responses(enabled);
self
}
/// Set whether HTTP/1 connections will accept obsolete line folding for
/// header values.
///
/// Newline codepoints (`\r` and `\n`) will be transformed to spaces when
/// parsing.
///
/// You probably don't need this, here is what [RFC 7230 Section 3.2.4.] has
/// to say about it:
///
/// > A server that receives an obs-fold in a request message that is not
/// > within a message/http container MUST either reject the message by
/// > sending a 400 (Bad Request), preferably with a representation
/// > explaining that obsolete line folding is unacceptable, or replace
/// > each received obs-fold with one or more SP octets prior to
/// > interpreting the field value or forwarding the message downstream.
///
/// > A proxy or gateway that receives an obs-fold in a response message
/// > that is not within a message/http container MUST either discard the
/// > message and replace it with a 502 (Bad Gateway) response, preferably
/// > with a representation explaining that unacceptable line folding was
/// > received, or replace each received obs-fold with one or more SP
/// > octets prior to interpreting the field value or forwarding the
/// > message downstream.
///
/// > A user agent that receives an obs-fold in a response message that is
/// > not within a message/http container MUST replace each received
/// > obs-fold with one or more SP octets prior to interpreting the field
/// > value.
///
/// Default is false.
///
/// [RFC 7230 Section 3.2.4.]: https://tools.ietf.org/html/rfc7230#section-3.2.4
pub fn allow_obsolete_multiline_headers_in_responses(&mut self, enabled: bool) -> &mut Builder {
self.h1_parser_config
.allow_obsolete_multiline_headers_in_responses(enabled);
self
}
/// Set whether HTTP/1 connections will silently ignored malformed header lines.
///
/// If this is enabled and a header line does not start with a valid header
/// name, or does not include a colon at all, the line will be silently ignored
/// and no error will be reported.
///
/// Default is false.
pub fn ignore_invalid_headers_in_responses(&mut self, enabled: bool) -> &mut Builder {
self.h1_parser_config
.ignore_invalid_headers_in_responses(enabled);
self
}
/// Set whether HTTP/1 connections should try to use vectored writes,
/// or always flatten into a single buffer.
///
/// Note that setting this to false may mean more copies of body data,
/// but may also improve performance when an IO transport doesn't
/// support vectored writes well, such as most TLS implementations.
///
/// Setting this to true will force hyper to use queued strategy,
/// which may eliminate unnecessary cloning on some TLS backends.
///
/// Default is `auto`. In this mode hyper will try to guess which
/// mode to use.
pub fn writev(&mut self, enabled: bool) -> &mut Builder {
self.h1_writev = Some(enabled);
self
}
/// Set whether HTTP/1 connections will write header names as title case at
/// the socket level.
///
/// Default is false.
pub fn title_case_headers(&mut self, enabled: bool) -> &mut Builder {
self.h1_title_case_headers = enabled;
self
}
/// Set whether to support preserving original header cases.
///
/// Currently, this will record the original cases received, and store them
/// in a private extension on the `Response`. It will also look for and use
/// such an extension in any provided `Request`.
///
/// Since the relevant extension is still private, there is no way to
/// interact with the original cases. The only effect this can have now is
/// to forward the cases in a proxy-like fashion.
///
/// Default is false.
pub fn preserve_header_case(&mut self, enabled: bool) -> &mut Builder {
self.h1_preserve_header_case = enabled;
self
}
/// Set the maximum number of headers.
///
/// When a response is received, the parser will reserve a buffer to store headers for optimal
/// performance.
///
/// If client receives more headers than the buffer size, the error "message header too large"
/// is returned.
///
/// Note that headers is allocated on the stack by default, which has higher performance. After
/// setting this value, headers will be allocated in heap memory, that is, heap memory
/// allocation will occur for each response, and there will be a performance drop of about 5%.
///
/// Default is 100.
pub fn max_headers(&mut self, val: usize) -> &mut Self {
self.h1_max_headers = Some(val);
self
}
/// Set the maximum size of response headers (including the status line) in bytes.
///
/// If the server sends headers exceeding this limit, the error "message head is too large"
/// is returned.
///
/// If not configured, then the [`max_buf_size`](Builder::max_buf_size) will naturally be reached and applied.
///
/// This value is also used as the maximum size limit for chunked trailers.
///
/// Default is `None`.
pub fn max_header_size(&mut self, val: usize) -> &mut Self {
self.h1_max_header_size = Some(val);
self
}
/// Set whether to support preserving original header order.
///
/// Currently, this will record the order in which headers are received, and store this
/// ordering in a private extension on the `Response`. It will also look for and use
/// such an extension in any provided `Request`.
///
/// Default is false.
#[cfg(feature = "ffi")]
pub fn preserve_header_order(&mut self, enabled: bool) -> &mut Builder {
self.h1_preserve_header_order = enabled;
self
}
/// Sets the exact size of the read buffer to *always* use.
///
/// Note that setting this option unsets the `max_buf_size` option.
///
/// Default is an adaptive read buffer.
pub fn read_buf_exact_size(&mut self, sz: Option<usize>) -> &mut Builder {
self.h1_read_buf_exact_size = sz;
self.h1_max_buf_size = None;
self
}
/// Set the maximum buffer size for the connection.
///
/// Default is ~400kb.
///
/// Note that setting this option unsets the `read_exact_buf_size` option.
///
/// # Panics
///
/// The minimum value allowed is 8192. This method panics if the passed `max` is less than the minimum.
pub fn max_buf_size(&mut self, max: usize) -> &mut Self {
assert!(
max >= proto::h1::MINIMUM_MAX_BUFFER_SIZE,
"the max_buf_size cannot be smaller than the minimum that h1 specifies."
);
self.h1_max_buf_size = Some(max);
self.h1_read_buf_exact_size = None;
self
}
/// Constructs a connection with the configured options and IO.
/// See [`client::conn`](crate::client::conn) for more.
///
/// Note, if [`Connection`] is not `await`-ed, [`SendRequest`] will
/// do nothing.
///
/// # Errors
///
/// Returns an error if the HTTP/1connection handshake fails.
pub fn handshake<T, B>(
&self,
io: T,
) -> impl Future<Output = crate::Result<(SendRequest<B>, Connection<T, B>)>>
where
T: Read + Write + Unpin,
B: Body + 'static,
B::Data: Send,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
let opts = self.clone();
async move {
trace!("client handshake HTTP/1");
let (tx, rx) = dispatch::channel();
let mut conn = proto::Conn::new(io);
conn.set_h1_parser_config(opts.h1_parser_config);
if let Some(writev) = opts.h1_writev {
if writev {
conn.set_write_strategy_queue();
} else {
conn.set_write_strategy_flatten();
}
}
if opts.h1_title_case_headers {
conn.set_title_case_headers();
}
if opts.h1_preserve_header_case {
conn.set_preserve_header_case();
}
if let Some(max_headers) = opts.h1_max_headers {
conn.set_http1_max_headers(max_headers);
}
if let Some(max_header_size) = opts.h1_max_header_size {
conn.set_http1_max_header_size(max_header_size);
}
#[cfg(feature = "ffi")]
if opts.h1_preserve_header_order {
conn.set_preserve_header_order();
}
if opts.h09_responses {
conn.set_h09_responses();
}
if let Some(sz) = opts.h1_read_buf_exact_size {
conn.set_read_buf_exact_size(sz);
}
if let Some(max) = opts.h1_max_buf_size {
conn.set_max_buf_size(max);
}
let cd = proto::h1::dispatch::Client::new(rx);
let proto = proto::h1::Dispatcher::new(cd, conn);
Ok((SendRequest { dispatch: tx }, Connection { inner: proto }))
}
}
}
mod upgrades {
use super::{Connection, Context, Future, Parts, Pin, Poll, Read, StdError, Write};
use crate::body::Body;
use crate::proto::Dispatched;
use crate::upgrade::Upgraded;
use futures_core::ready;
// A future binding a connection with a Service with Upgrade support.
//
// This type is unnameable outside the crate.
#[must_use = "futures do nothing unless polled"]
#[allow(missing_debug_implementations)]
pub struct UpgradeableConnection<T, B>
where
T: Read + Write + Unpin + Send + 'static,
B: Body + 'static,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
pub(super) inner: Option<Connection<T, B>>,
}
impl<I, B> Future for UpgradeableConnection<I, B>
where
I: Read + Write + Unpin + Send + 'static,
B: Body + 'static,
B::Data: Send,
B::Error: Into<Box<dyn StdError + Send + Sync>>,
{
type Output = crate::Result<()>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match ready!(Pin::new(
&mut self
.inner
.as_mut()
.expect("upgradeable client connection polled after upgrade")
.inner,
)
.poll(cx))
{
Ok(Dispatched::Shutdown) => Poll::Ready(Ok(())),
Ok(Dispatched::Upgrade(pending)) => {
let Parts { io, read_buf } = self
.inner
.take()
.expect("upgradeable client connection missing after upgrade")
.into_parts();
pending.fulfill(Upgraded::new(io, read_buf));
Poll::Ready(Ok(()))
}
Err(e) => Poll::Ready(Err(e)),
}
}
}
}
-773
View File
@@ -1,773 +0,0 @@
//! HTTP/2 client connections.
use std::error::Error;
use std::fmt;
use std::future::Future;
use std::marker::PhantomData;
use std::pin::Pin;
use std::sync::Arc;
use std::task::{Context, Poll};
use std::time::Duration;
use crate::rt::{Read, Write};
use futures_core::ready;
use http::{Request, Response};
use super::super::dispatch::{self, TrySendError};
use crate::body::{Body, Incoming as IncomingBody};
use crate::common::time::Time;
use crate::proto;
use crate::rt::bounds::Http2ClientConnExec;
use crate::rt::Timer;
/// The sender side of an established connection.
pub struct SendRequest<B> {
dispatch: dispatch::UnboundedSender<Request<B>, Response<IncomingBody>>,
}
impl<B> Clone for SendRequest<B> {
fn clone(&self) -> SendRequest<B> {
SendRequest {
dispatch: self.dispatch.clone(),
}
}
}
/// A future that processes all HTTP state for the IO object.
///
/// In most cases, this should just be spawned into an executor, so that it
/// can process incoming and outgoing messages, notice hangups, and the like.
///
/// Instances of this type are typically created via the [`handshake`] function.
///
/// # Drop behavior
///
/// Dropping the `Connection` will close the underlying IO resource.
/// Any in-flight requests that have not received a response will be
/// interrupted. If graceful shutdown is desired, poll the connection
/// until it completes instead of dropping.
#[must_use = "futures do nothing unless polled"]
pub struct Connection<T, B, E>
where
T: Read + Write + Unpin,
B: Body + 'static,
E: Http2ClientConnExec<B, T> + Unpin,
B::Error: Into<Box<dyn Error + Send + Sync>>,
{
inner: (PhantomData<T>, proto::h2::ClientTask<B, E, T>),
}
/// A builder to configure an HTTP connection.
///
/// After setting options, the builder is used to create a handshake future.
///
/// **Note**: The default values of options are *not considered stable*. They
/// are subject to change at any time.
#[derive(Clone, Debug)]
pub struct Builder<Ex> {
pub(super) exec: Ex,
pub(super) timer: Time,
h2_builder: proto::h2::client::Config,
}
/// Returns a handshake future over some IO.
///
/// This is a shortcut for `Builder::new(exec).handshake(io)`.
/// See [`client::conn`](crate::client::conn) for more.
///
/// # Errors
///
/// Returns an error if the HTTP/2 connection handshake fails.
pub async fn handshake<E, T, B>(
exec: E,
io: T,
) -> crate::Result<(SendRequest<B>, Connection<T, B, E>)>
where
T: Read + Write + Unpin,
B: Body + 'static,
B::Data: Send,
B::Error: Into<Box<dyn Error + Send + Sync>>,
E: Http2ClientConnExec<B, T> + Unpin + Clone,
{
Builder::new(exec).handshake(io).await
}
// ===== impl SendRequest
impl<B> SendRequest<B> {
/// Polls to determine whether this sender can be used yet for a request.
///
/// If the associated connection is closed, this returns an Error.
pub fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<crate::Result<()>> {
if self.is_closed() {
Poll::Ready(Err(crate::Error::new_closed()))
} else {
Poll::Ready(Ok(()))
}
}
/// Waits until the dispatcher is ready.
///
/// # Errors
///
/// If the associated connection is closed, this returns an Error.
pub async fn ready(&mut self) -> crate::Result<()> {
crate::common::future::poll_fn(|cx| self.poll_ready(cx)).await
}
/// Checks if the connection is currently ready to send a request.
///
/// # Note
///
/// This is mostly a hint. Due to inherent latency of networks, it is
/// possible that even after checking this is ready, sending a request
/// may still fail because the connection was closed in the meantime.
pub fn is_ready(&self) -> bool {
self.dispatch.is_ready()
}
/// Checks if the connection side has been closed.
pub fn is_closed(&self) -> bool {
self.dispatch.is_closed()
}
}
impl<B> SendRequest<B>
where
B: Body + 'static,
{
/// Sends a `Request` on the associated connection.
///
/// Returns a future that if successful, yields the `Response`.
///
/// `req` must have a `Host` header.
///
/// Absolute-form `Uri`s are not required. If received, they will be serialized
/// as-is.
///
/// # Cancel safety
///
/// Dropping the returned future is the supported way to cancel an
/// in-flight HTTP/2 request. The stream is reset with `RST_STREAM`
/// (`CANCEL` error code); the shared connection remains usable for
/// other in-flight and future requests. The peer is notified
/// immediately rather than continuing to send a response body that
/// would be discarded.
///
/// # Errors
///
/// Returns an error if the connection is not ready or if an error occurs while
/// processing the request.
pub fn send_request(
&mut self,
req: Request<B>,
) -> impl Future<Output = crate::Result<Response<IncomingBody>>> {
let sent = self.dispatch.send(req);
async move {
match sent {
Ok(rx) => match rx.await {
Ok(Ok(resp)) => Ok(resp),
Ok(Err(err)) => Err(err),
// this is definite bug if it happens, but it shouldn't happen!
Err(_canceled) => unreachable!("dispatch dropped without returning error"),
},
Err(_req) => {
debug!("connection was not ready");
Err(crate::Error::new_canceled().with("connection was not ready"))
}
}
}
}
/// Sends a `Request` on the associated connection.
///
/// Returns a future that if successful, yields the `Response`.
///
/// # Errors
///
/// If there was an error before trying to serialize the request to the
/// connection, the message will be returned as part of this error.
#[allow(clippy::result_large_err)]
pub fn try_send_request(
&mut self,
req: Request<B>,
) -> impl Future<Output = Result<Response<IncomingBody>, TrySendError<Request<B>>>> {
let sent = self.dispatch.try_send(req);
async move {
match sent {
Ok(rx) => match rx.await {
Ok(Ok(res)) => Ok(res),
Ok(Err(err)) => Err(err),
// this is definite bug if it happens, but it shouldn't happen!
Err(_) => unreachable!("dispatch dropped without returning error"),
},
Err(req) => {
debug!("connection was not ready");
let error = crate::Error::new_canceled().with("connection was not ready");
Err(TrySendError {
error,
message: Some(req),
})
}
}
}
}
}
impl<B> fmt::Debug for SendRequest<B> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("SendRequest").finish()
}
}
// ===== impl Connection
impl<T, B, E> Connection<T, B, E>
where
T: Read + Write + Unpin + 'static,
B: Body + Unpin + 'static,
B::Data: Send,
B::Error: Into<Box<dyn Error + Send + Sync>>,
E: Http2ClientConnExec<B, T> + Unpin,
{
/// Returns whether the [extended CONNECT protocol][1] is enabled or not.
///
/// This setting is configured by the server peer by sending the
/// [`SETTINGS_ENABLE_CONNECT_PROTOCOL` parameter][2] in a `SETTINGS` frame.
/// This method returns the currently acknowledged value received from the
/// remote.
///
/// [1]: https://datatracker.ietf.org/doc/html/rfc8441#section-4
/// [2]: https://datatracker.ietf.org/doc/html/rfc8441#section-3
pub fn is_extended_connect_protocol_enabled(&self) -> bool {
self.inner.1.is_extended_connect_protocol_enabled()
}
/// Returns the current maximum send stream count.
///
/// This setting is configured in a [`SETTINGS_MAX_CONCURRENT_STREAMS` parameter][1] in a `SETTINGS` frame,
/// and may change throughout the connection lifetime.
///
/// [1]: https://datatracker.ietf.org/doc/html/rfc7540#section-5.1.2
pub fn current_max_send_streams(&self) -> usize {
self.inner.1.current_max_send_streams()
}
/// Returns the current maximum receive stream count.
///
/// This setting is configured in a [`SETTINGS_MAX_CONCURRENT_STREAMS` parameter][1] in a `SETTINGS` frame,
/// and may change throughout the connection lifetime.
///
/// [1]: https://datatracker.ietf.org/doc/html/rfc7540#section-5.1.2
pub fn current_max_recv_streams(&self) -> usize {
self.inner.1.current_max_recv_streams()
}
}
impl<T, B, E> fmt::Debug for Connection<T, B, E>
where
T: Read + Write + fmt::Debug + 'static + Unpin,
B: Body + 'static,
E: Http2ClientConnExec<B, T> + Unpin,
B::Error: Into<Box<dyn Error + Send + Sync>>,
{
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Connection").finish()
}
}
impl<T, B, E> Future for Connection<T, B, E>
where
T: Read + Write + Unpin + 'static,
B: Body + 'static + Unpin,
B::Data: Send,
E: Unpin,
B::Error: Into<Box<dyn Error + Send + Sync>>,
E: Http2ClientConnExec<B, T> + Unpin,
{
type Output = crate::Result<()>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match ready!(Pin::new(&mut self.inner.1).poll(cx))? {
proto::Dispatched::Shutdown => Poll::Ready(Ok(())),
#[cfg(feature = "http1")]
proto::Dispatched::Upgrade(_pending) => unreachable!("http2 cannot upgrade"),
}
}
}
// ===== impl Builder
impl<Ex> Builder<Ex>
where
Ex: Clone,
{
/// Creates a new connection builder.
#[inline]
pub fn new(exec: Ex) -> Builder<Ex> {
Builder {
exec,
timer: Time::Empty,
h2_builder: proto::h2::client::Config::default(),
}
}
/// Provide a timer to execute background HTTP2 tasks.
pub fn timer<M>(&mut self, timer: M) -> &mut Builder<Ex>
where
M: Timer + Send + Sync + 'static,
{
self.timer = Time::Timer(Arc::new(timer));
self
}
/// Sets the [`SETTINGS_INITIAL_WINDOW_SIZE`][spec] option for HTTP2
/// stream-level flow control.
///
/// Passing `None` will do nothing.
///
/// If not set, hyper will use a default.
///
/// [spec]: https://httpwg.org/specs/rfc9113.html#SETTINGS_INITIAL_WINDOW_SIZE
pub fn initial_stream_window_size(&mut self, sz: impl Into<Option<u32>>) -> &mut Self {
if let Some(sz) = sz.into() {
self.h2_builder.adaptive_window = false;
self.h2_builder.initial_stream_window_size = sz;
}
self
}
/// Sets the max connection-level flow control for HTTP2.
///
/// Passing `None` will do nothing.
///
/// If not set, hyper will use a default.
pub fn initial_connection_window_size(&mut self, sz: impl Into<Option<u32>>) -> &mut Self {
if let Some(sz) = sz.into() {
self.h2_builder.adaptive_window = false;
self.h2_builder.initial_conn_window_size = sz;
}
self
}
/// Sets the initial maximum of locally initiated (send) streams.
///
/// This value will be overwritten by the value included in the initial
/// SETTINGS frame received from the peer as part of a [connection preface].
///
/// Passing `None` will do nothing.
///
/// If not set, hyper will use a default.
///
/// [connection preface]: https://httpwg.org/specs/rfc9113.html#preface
pub fn initial_max_send_streams(&mut self, initial: impl Into<Option<usize>>) -> &mut Self {
if let Some(initial) = initial.into() {
self.h2_builder.initial_max_send_streams = initial;
}
self
}
/// Sets whether to use an adaptive flow control.
///
/// Enabling this will override the limits set in
/// `initial_stream_window_size` and
/// `initial_connection_window_size`.
pub fn adaptive_window(&mut self, enabled: bool) -> &mut Self {
use proto::h2::SPEC_WINDOW_SIZE;
self.h2_builder.adaptive_window = enabled;
if enabled {
self.h2_builder.initial_conn_window_size = SPEC_WINDOW_SIZE;
self.h2_builder.initial_stream_window_size = SPEC_WINDOW_SIZE;
}
self
}
/// Sets the maximum frame size to use for HTTP2.
///
/// Default is currently 16KB, but can change.
pub fn max_frame_size(&mut self, sz: impl Into<Option<u32>>) -> &mut Self {
self.h2_builder.max_frame_size = sz.into();
self
}
/// Sets the max size of received header frames.
///
/// Default is currently 16KB, but can change.
pub fn max_header_list_size(&mut self, max: u32) -> &mut Self {
self.h2_builder.max_header_list_size = max;
self
}
/// Sets the header table size.
///
/// This setting informs the peer of the maximum size of the header compression
/// table used to encode header blocks, in octets. The encoder may select any value
/// equal to or less than the header table size specified by the sender.
///
/// The default value of crate `h2` is 4,096.
pub fn header_table_size(&mut self, size: impl Into<Option<u32>>) -> &mut Self {
self.h2_builder.header_table_size = size.into();
self
}
/// Sets the maximum number of concurrent streams.
///
/// The maximum concurrent streams setting only controls the maximum number
/// of streams that can be initiated by the remote peer. In other words,
/// when this setting is set to 100, this does not limit the number of
/// concurrent streams that can be created by the caller.
///
/// It is recommended that this value be no smaller than 100, so as to not
/// unnecessarily limit parallelism. However, any value is legal, including
/// 0. If `max` is set to 0, then the remote will not be permitted to
/// initiate streams.
///
/// Note that streams in the reserved state, i.e., push promises that have
/// been reserved but the stream has not started, do not count against this
/// setting.
///
/// Also note that if the remote *does* exceed the value set here, it is not
/// a protocol level error. Instead, the `h2` library will immediately reset
/// the stream.
///
/// See [Section 5.1.2] in the HTTP/2 spec for more details.
///
/// [Section 5.1.2]: https://httpwg.org/specs/rfc7540.html#rfc.section.5.1.2
pub fn max_concurrent_streams(&mut self, max: impl Into<Option<u32>>) -> &mut Self {
self.h2_builder.max_concurrent_streams = max.into();
self
}
/// Sets an interval for HTTP2 Ping frames should be sent to keep a
/// connection alive.
///
/// Pass `None` to disable HTTP2 keep-alive.
///
/// Default is currently disabled.
pub fn keep_alive_interval(&mut self, interval: impl Into<Option<Duration>>) -> &mut Self {
self.h2_builder.keep_alive_interval = interval.into();
self
}
/// Sets a timeout for receiving an acknowledgement of the keep-alive ping.
///
/// If the ping is not acknowledged within the timeout, the connection will
/// be closed. Does nothing if `keep_alive_interval` is disabled.
///
/// Default is 20 seconds.
pub fn keep_alive_timeout(&mut self, timeout: Duration) -> &mut Self {
self.h2_builder.keep_alive_timeout = timeout;
self
}
/// Sets whether HTTP2 keep-alive should apply while the connection is idle.
///
/// If disabled, keep-alive pings are only sent while there are open
/// request/responses streams. If enabled, pings are also sent when no
/// streams are active. Does nothing if `keep_alive_interval` is
/// disabled.
///
/// Default is `false`.
pub fn keep_alive_while_idle(&mut self, enabled: bool) -> &mut Self {
self.h2_builder.keep_alive_while_idle = enabled;
self
}
/// Sets the maximum number of HTTP2 concurrent locally reset streams.
///
/// See the documentation of [`h2::client::Builder::max_concurrent_reset_streams`] for more
/// details.
///
/// The default value is determined by the `h2` crate.
///
/// [`h2::client::Builder::max_concurrent_reset_streams`]: https://docs.rs/h2/client/struct.Builder.html#method.max_concurrent_reset_streams
pub fn max_concurrent_reset_streams(&mut self, max: usize) -> &mut Self {
self.h2_builder.max_concurrent_reset_streams = Some(max);
self
}
/// Set the maximum write buffer size for each HTTP/2 stream.
///
/// Default is currently 1MB, but may change.
///
/// # Panics
///
/// The value must be no larger than `u32::MAX`.
pub fn max_send_buf_size(&mut self, max: usize) -> &mut Self {
assert!(max <= u32::MAX as usize);
self.h2_builder.max_send_buffer_size = max;
self
}
/// Configures the maximum number of pending reset streams allowed before a GOAWAY will be sent.
///
/// This will default to the default value set by the [`h2` crate](https://crates.io/crates/h2).
/// As of v0.4.0, it is 20.
///
/// See <https://github.com/hyperium/hyper/issues/2877> for more information.
pub fn max_pending_accept_reset_streams(&mut self, max: impl Into<Option<usize>>) -> &mut Self {
self.h2_builder.max_pending_accept_reset_streams = max.into();
self
}
/// Configures the maximum number of local resets due to protocol errors made by the remote end.
///
/// See the documentation of [`h2::client::Builder::max_local_error_reset_streams`] for more
/// details.
///
/// The default value is 1024.
pub fn max_local_error_reset_streams(&mut self, max: impl Into<Option<usize>>) -> &mut Self {
self.h2_builder.max_local_error_reset_streams = max.into();
self
}
/// Sets the duration to remember locally reset streams.
///
/// When a stream is explicitly reset by either the client or the server,
/// the HTTP/2 specification requires that any further frames received for
/// that stream must be ignored for "some time".
///
/// In order to satisfy the specification, internal state must be maintained
/// to implement the behavior. This state grows linearly with the number of
/// streams that are locally reset.
///
/// The `reset_stream_duration` setting configures the max amount of time
/// this state will be maintained in memory. Once the duration elapses, the
/// stream state is purged from memory.
///
/// Once the stream has been fully purged from memory, any additional frames
/// received for that stream will result in a connection level protocol
/// error, forcing the connection to terminate.
///
/// The default value is determined by the `h2` crate, and is currently
/// 1 second.
///
/// See the documentation of [`h2::client::Builder::reset_stream_duration`] for more
/// details.
///
/// [`h2::client::Builder::reset_stream_duration`]: https://docs.rs/h2/client/struct.Builder.html#method.reset_stream_duration
pub fn reset_stream_duration(&mut self, dur: Duration) -> &mut Self {
self.h2_builder.reset_stream_duration = Some(dur);
self
}
/// Constructs a connection with the configured options and IO.
/// See [`client::conn`](crate::client::conn) for more.
///
/// Note, if [`Connection`] is not `await`-ed, [`SendRequest`] will
/// do nothing.
///
/// # Errors
///
/// Returns an error if the HTTP/2 connection handshake fails.
pub fn handshake<T, B>(
&self,
io: T,
) -> impl Future<Output = crate::Result<(SendRequest<B>, Connection<T, B, Ex>)>>
where
T: Read + Write + Unpin,
B: Body + 'static,
B::Data: Send,
B::Error: Into<Box<dyn Error + Send + Sync>>,
Ex: Http2ClientConnExec<B, T> + Unpin,
{
let opts = self.clone();
async move {
trace!("client handshake HTTP/2");
let (tx, rx) = dispatch::channel();
let h2 = proto::h2::client::handshake(io, rx, &opts.h2_builder, opts.exec, opts.timer)
.await?;
Ok((
SendRequest {
dispatch: tx.unbound(),
},
Connection {
inner: (PhantomData, h2),
},
))
}
}
}
#[cfg(test)]
mod tests {
#[tokio::test]
#[ignore] // only compilation is checked
async fn send_sync_executor_of_non_send_futures() {
#[derive(Clone)]
struct LocalTokioExecutor;
impl<F> crate::rt::Executor<F> for LocalTokioExecutor
where
F: std::future::Future + 'static, // not requiring `Send`
{
fn execute(&self, fut: F) {
// This will spawn into the currently running `LocalSet`.
tokio::task::spawn_local(fut);
}
}
#[allow(unused)]
async fn run(io: impl crate::rt::Read + crate::rt::Write + Unpin + 'static) {
let (_sender, conn) = crate::client::conn::http2::handshake::<
_,
_,
http_body_util::Empty<bytes::Bytes>,
>(LocalTokioExecutor, io)
.await
.unwrap();
tokio::task::spawn_local(async move {
conn.await.unwrap();
});
}
}
#[tokio::test]
#[ignore] // only compilation is checked
async fn not_send_not_sync_executor_of_not_send_futures() {
#[derive(Clone)]
struct LocalTokioExecutor {
_x: std::marker::PhantomData<std::rc::Rc<()>>,
}
impl<F> crate::rt::Executor<F> for LocalTokioExecutor
where
F: std::future::Future + 'static, // not requiring `Send`
{
fn execute(&self, fut: F) {
// This will spawn into the currently running `LocalSet`.
tokio::task::spawn_local(fut);
}
}
#[allow(unused)]
async fn run(io: impl crate::rt::Read + crate::rt::Write + Unpin + 'static) {
let (_sender, conn) =
crate::client::conn::http2::handshake::<_, _, http_body_util::Empty<bytes::Bytes>>(
LocalTokioExecutor {
_x: Default::default(),
},
io,
)
.await
.unwrap();
tokio::task::spawn_local(async move {
conn.await.unwrap();
});
}
}
#[tokio::test]
#[ignore] // only compilation is checked
async fn send_not_sync_executor_of_not_send_futures() {
#[derive(Clone)]
struct LocalTokioExecutor {
_x: std::marker::PhantomData<std::cell::Cell<()>>,
}
impl<F> crate::rt::Executor<F> for LocalTokioExecutor
where
F: std::future::Future + 'static, // not requiring `Send`
{
fn execute(&self, fut: F) {
// This will spawn into the currently running `LocalSet`.
tokio::task::spawn_local(fut);
}
}
#[allow(unused)]
async fn run(io: impl crate::rt::Read + crate::rt::Write + Unpin + 'static) {
let (_sender, conn) =
crate::client::conn::http2::handshake::<_, _, http_body_util::Empty<bytes::Bytes>>(
LocalTokioExecutor {
_x: Default::default(),
},
io,
)
.await
.unwrap();
tokio::task::spawn_local(async move {
conn.await.unwrap();
});
}
}
#[tokio::test]
#[ignore] // only compilation is checked
async fn send_sync_executor_of_send_futures() {
#[derive(Clone)]
struct TokioExecutor;
impl<F> crate::rt::Executor<F> for TokioExecutor
where
F: std::future::Future + 'static + Send,
F::Output: Send + 'static,
{
fn execute(&self, fut: F) {
tokio::task::spawn(fut);
}
}
#[allow(unused)]
async fn run(io: impl crate::rt::Read + crate::rt::Write + Send + Unpin + 'static) {
let (_sender, conn) = crate::client::conn::http2::handshake::<
_,
_,
http_body_util::Empty<bytes::Bytes>,
>(TokioExecutor, io)
.await
.unwrap();
tokio::task::spawn(async move {
conn.await.unwrap();
});
}
}
#[tokio::test]
#[ignore] // only compilation is checked
async fn send_not_sync_executor_of_send_futures() {
#[derive(Clone)]
struct TokioExecutor {
// !Sync
_x: std::marker::PhantomData<std::cell::Cell<()>>,
}
impl<F> crate::rt::Executor<F> for TokioExecutor
where
F: std::future::Future + 'static + Send,
F::Output: Send + 'static,
{
fn execute(&self, fut: F) {
tokio::task::spawn(fut);
}
}
#[allow(unused)]
async fn run(io: impl crate::rt::Read + crate::rt::Write + Send + Unpin + 'static) {
let (_sender, conn) =
crate::client::conn::http2::handshake::<_, _, http_body_util::Empty<bytes::Bytes>>(
TokioExecutor {
_x: Default::default(),
},
io,
)
.await
.unwrap();
tokio::task::spawn_local(async move {
// can't use spawn here because when executor is !Send
conn.await.unwrap();
});
}
}
}
-22
View File
@@ -1,22 +0,0 @@
//! Lower-level client connection API.
//!
//! The types in this module are to provide a lower-level API based around a
//! single connection. Connecting to a host, pooling connections, and the like
//! are not handled at this level. This module provides the building blocks to
//! customize those things externally.
//!
//! If you are looking for a convenient HTTP client, then you may wish to
//! consider [reqwest](https://github.com/seanmonstar/reqwest) for a high level
//! client or [`hyper-util`'s client](https://docs.rs/hyper-util/latest/hyper_util/client/index.html)
//! if you want to keep it more low level / basic.
//!
//! ## Example
//!
//! See the [client guide](https://hyper.rs/guides/1/client/basic/).
#[cfg(feature = "http1")]
pub mod http1;
#[cfg(feature = "http2")]
pub mod http2;
pub use super::dispatch::TrySendError;
-539
View File
@@ -1,539 +0,0 @@
use std::task::{Context, Poll};
#[cfg(feature = "http2")]
use std::{future::Future, pin::Pin};
#[cfg(feature = "http2")]
use http::{Request, Response};
#[cfg(feature = "http2")]
use http_body::Body;
#[cfg(feature = "http2")]
use pin_project_lite::pin_project;
use tokio::sync::{mpsc, oneshot};
#[cfg(feature = "http2")]
use crate::{body::Incoming, proto::h2::client::ResponseFutMap};
pub(crate) type RetryPromise<T, U> = oneshot::Receiver<Result<U, TrySendError<T>>>;
pub(crate) type Promise<T> = oneshot::Receiver<Result<T, crate::Error>>;
/// An error when calling `try_send_request`.
///
/// There is a possibility of an error occurring on a connection in-between the
/// time that a request is queued and when it is actually written to the IO
/// transport. If that happens, it is safe to return the request back to the
/// caller, as it was never fully sent.
#[derive(Debug)]
pub struct TrySendError<T> {
pub(crate) error: crate::Error,
pub(crate) message: Option<T>,
}
pub(crate) fn channel<T, U>() -> (Sender<T, U>, Receiver<T, U>) {
let (tx, rx) = mpsc::unbounded_channel();
let (giver, taker) = want::new();
let tx = Sender {
#[cfg(feature = "http1")]
buffered_once: false,
giver,
inner: tx,
};
let rx = Receiver { inner: rx, taker };
(tx, rx)
}
/// A bounded sender of requests and callbacks for when responses are ready.
///
/// While the inner sender is unbounded, the Giver is used to determine
/// if the Receiver is ready for another request.
pub(crate) struct Sender<T, U> {
/// One message is always allowed, even if the Receiver hasn't asked
/// for it yet. This boolean keeps track of whether we've sent one
/// without notice.
#[cfg(feature = "http1")]
buffered_once: bool,
/// The Giver helps watch that the Receiver side has been polled
/// when the queue is empty. This helps us know when a request and
/// response have been fully processed, and a connection is ready
/// for more.
giver: want::Giver,
/// Actually bounded by the Giver, plus `buffered_once`.
inner: mpsc::UnboundedSender<Envelope<T, U>>,
}
/// An unbounded version.
///
/// Cannot poll the Giver, but can still use it to determine if the Receiver
/// has been dropped. However, this version can be cloned.
#[cfg(feature = "http2")]
pub(crate) struct UnboundedSender<T, U> {
/// Only used for `is_closed`, since `mpsc::UnboundedSender` cannot be checked.
giver: want::SharedGiver,
inner: mpsc::UnboundedSender<Envelope<T, U>>,
}
impl<T, U> Sender<T, U> {
#[cfg(feature = "http1")]
pub(crate) fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<crate::Result<()>> {
self.giver
.poll_want(cx)
.map_err(|_| crate::Error::new_closed())
}
#[cfg(feature = "http1")]
pub(crate) fn is_ready(&self) -> bool {
self.giver.is_wanting()
}
#[cfg(feature = "http1")]
pub(crate) fn is_closed(&self) -> bool {
self.giver.is_canceled()
}
#[cfg(feature = "http1")]
fn can_send(&mut self) -> bool {
if self.giver.give() || !self.buffered_once {
// If the receiver is ready *now*, then of course we can send.
//
// If the receiver isn't ready yet, but we don't have anything
// in the channel yet, then allow one message.
self.buffered_once = true;
true
} else {
false
}
}
#[cfg(feature = "http1")]
pub(crate) fn try_send(&mut self, val: T) -> Result<RetryPromise<T, U>, T> {
if !self.can_send() {
return Err(val);
}
let (tx, rx) = oneshot::channel();
self.inner
.send(Envelope(Some((val, Callback::Retry(Some(tx))))))
.map(move |()| rx)
.map_err(|mut e| (e.0).0.take().expect("envelope not dropped").0)
}
#[cfg(feature = "http1")]
pub(crate) fn send(&mut self, val: T) -> Result<Promise<U>, T> {
if !self.can_send() {
return Err(val);
}
let (tx, rx) = oneshot::channel();
self.inner
.send(Envelope(Some((val, Callback::NoRetry(Some(tx))))))
.map(move |()| rx)
.map_err(|mut e| (e.0).0.take().expect("envelope not dropped").0)
}
#[cfg(feature = "http2")]
pub(crate) fn unbound(self) -> UnboundedSender<T, U> {
UnboundedSender {
giver: self.giver.shared(),
inner: self.inner,
}
}
}
#[cfg(feature = "http2")]
impl<T, U> UnboundedSender<T, U> {
pub(crate) fn is_ready(&self) -> bool {
!self.giver.is_canceled()
}
pub(crate) fn is_closed(&self) -> bool {
self.giver.is_canceled()
}
pub(crate) fn try_send(&mut self, val: T) -> Result<RetryPromise<T, U>, T> {
let (tx, rx) = oneshot::channel();
self.inner
.send(Envelope(Some((val, Callback::Retry(Some(tx))))))
.map(move |()| rx)
.map_err(|mut e| (e.0).0.take().expect("envelope not dropped").0)
}
pub(crate) fn send(&mut self, val: T) -> Result<Promise<U>, T> {
let (tx, rx) = oneshot::channel();
self.inner
.send(Envelope(Some((val, Callback::NoRetry(Some(tx))))))
.map(move |()| rx)
.map_err(|mut e| (e.0).0.take().expect("envelope not dropped").0)
}
}
#[cfg(feature = "http2")]
impl<T, U> Clone for UnboundedSender<T, U> {
fn clone(&self) -> Self {
UnboundedSender {
giver: self.giver.clone(),
inner: self.inner.clone(),
}
}
}
pub(crate) struct Receiver<T, U> {
inner: mpsc::UnboundedReceiver<Envelope<T, U>>,
taker: want::Taker,
}
impl<T, U> Receiver<T, U> {
pub(crate) fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<(T, Callback<T, U>)>> {
match self.inner.poll_recv(cx) {
Poll::Ready(item) => {
// A want signaled after finding the queue empty can land after
// the Sender's `give()` for the message taken here, as can one
// signaled on a coop-budget Pending with a message queued.
// Withdraw it, or the Sender reports this connection as ready
// while it is still serving this message.
self.taker.unwant();
Poll::Ready(item.map(|mut env| env.0.take().expect("envelope not dropped")))
}
Poll::Pending => {
self.taker.want();
Poll::Pending
}
}
}
#[cfg(feature = "http1")]
pub(crate) fn close(&mut self) {
self.taker.cancel();
self.inner.close();
}
#[cfg(feature = "http1")]
pub(crate) fn try_recv(&mut self) -> Option<(T, Callback<T, U>)> {
match crate::common::task::now_or_never(self.inner.recv()) {
Some(Some(mut env)) => env.0.take(),
_ => None,
}
}
}
impl<T, U> Drop for Receiver<T, U> {
fn drop(&mut self) {
// Notify the giver about the closure first, before dropping
// the mpsc::Receiver.
self.taker.cancel();
}
}
struct Envelope<T, U>(Option<(T, Callback<T, U>)>);
impl<T, U> Drop for Envelope<T, U> {
fn drop(&mut self) {
if let Some((val, cb)) = self.0.take() {
cb.send(Err(TrySendError {
error: crate::Error::new_canceled().with("connection closed"),
message: Some(val),
}));
}
}
}
pub(crate) enum Callback<T, U> {
#[allow(unused)]
Retry(Option<oneshot::Sender<Result<U, TrySendError<T>>>>),
NoRetry(Option<oneshot::Sender<Result<U, crate::Error>>>),
}
impl<T, U> Drop for Callback<T, U> {
fn drop(&mut self) {
match self {
Callback::Retry(tx) => {
if let Some(tx) = tx.take() {
let _ = tx.send(Err(TrySendError {
error: dispatch_gone(),
message: None,
}));
}
}
Callback::NoRetry(tx) => {
if let Some(tx) = tx.take() {
let _ = tx.send(Err(dispatch_gone()));
}
}
}
}
}
#[cold]
fn dispatch_gone() -> crate::Error {
// FIXME(nox): What errors do we want here?
crate::Error::new_user_dispatch_gone().with(if std::thread::panicking() {
"user code panicked"
} else {
"runtime dropped the dispatch task"
})
}
impl<T, U> Callback<T, U> {
#[cfg(feature = "http2")]
pub(crate) fn is_canceled(&self) -> bool {
match self {
Callback::Retry(Some(tx)) => tx.is_closed(),
Callback::NoRetry(Some(tx)) => tx.is_closed(),
_ => unreachable!(),
}
}
pub(crate) fn poll_canceled(&mut self, cx: &mut Context<'_>) -> Poll<()> {
match self {
Callback::Retry(Some(tx)) => tx.poll_closed(cx),
Callback::NoRetry(Some(tx)) => tx.poll_closed(cx),
_ => unreachable!(),
}
}
pub(crate) fn send(mut self, val: Result<U, TrySendError<T>>) {
match &mut self {
Callback::Retry(tx) => {
let _ = tx
.take()
.expect("callback sender not dropped before send")
.send(val);
}
Callback::NoRetry(tx) => {
let _ = tx
.take()
.expect("callback sender not dropped before send")
.send(val.map_err(|e| e.error));
}
}
}
}
impl<T> TrySendError<T> {
/// Take the message from this error.
///
/// The message will not always have been recovered. If an error occurs
/// after the message has been serialized onto the connection, it will not
/// be available here.
pub fn take_message(&mut self) -> Option<T> {
self.message.take()
}
/// Returns a reference to the recovered message.
///
/// The message will not always have been recovered. If an error occurs
/// after the message has been serialized onto the connection, it will not
/// be available here.
pub fn message(&self) -> Option<&T> {
self.message.as_ref()
}
/// Consumes this to return the inner error.
pub fn into_error(self) -> crate::Error {
self.error
}
/// Returns a reference to the inner error.
pub fn error(&self) -> &crate::Error {
&self.error
}
}
#[cfg(feature = "http2")]
pin_project! {
pub struct SendWhen<B, E>
where
B: Body,
B: 'static,
{
#[pin]
pub(crate) when: ResponseFutMap<B, E>,
#[pin]
pub(crate) call_back: Option<Callback<Request<B>, Response<Incoming>>>,
}
}
#[cfg(feature = "http2")]
impl<B, E> Future for SendWhen<B, E>
where
B: Body + 'static,
E: crate::rt::bounds::Http2UpgradedExec<B::Data>,
{
type Output = ();
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
let mut this = self.project();
let mut call_back = this.call_back.take().expect("polled after complete");
match Pin::new(&mut this.when).poll(cx) {
Poll::Ready(Ok(res)) => {
call_back.send(Ok(res));
Poll::Ready(())
}
Poll::Pending => {
// check if the callback is canceled
match call_back.poll_canceled(cx) {
Poll::Ready(v) => v,
Poll::Pending => {
// Move call_back back to struct before return
this.call_back.set(Some(call_back));
return Poll::Pending;
}
}
trace!("send_when canceled");
// Tell pipe_task to reset the h2 stream so that
// RST_STREAM is sent and flow-control capacity freed.
this.when.as_mut().cancel();
Poll::Ready(())
}
Poll::Ready(Err((error, message))) => {
call_back.send(Err(TrySendError { error, message }));
Poll::Ready(())
}
}
}
}
#[cfg(test)]
mod tests {
#[cfg(feature = "nightly")]
extern crate test;
use std::future::Future;
use std::pin::Pin;
use std::task::{Context, Poll};
use super::{channel, Callback, Receiver};
#[derive(Debug)]
struct Custom(#[allow(dead_code)] i32);
impl<T, U> Future for Receiver<T, U> {
type Output = Option<(T, Callback<T, U>)>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
self.poll_recv(cx)
}
}
/// Helper to check if the future is ready after polling once.
struct PollOnce<'fut, F>(&'fut mut F);
impl<F, T> Future for PollOnce<'_, F>
where
F: Future<Output = T> + Unpin,
{
type Output = Option<()>;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match Pin::new(&mut self.0).poll(cx) {
Poll::Ready(_) => Poll::Ready(Some(())),
Poll::Pending => Poll::Ready(None),
}
}
}
#[cfg(not(miri))]
#[tokio::test]
async fn drop_receiver_sends_cancel_errors() {
let _ = pretty_env_logger::try_init();
let (mut tx, mut rx) = channel::<Custom, ()>();
// must poll once for try_send to succeed
assert!(PollOnce(&mut rx).await.is_none(), "rx empty");
let promise = tx.try_send(Custom(43)).unwrap();
drop(rx);
let fulfilled = promise.await;
let err = fulfilled
.expect("fulfilled")
.expect_err("promise should error");
match (err.error.is_canceled(), err.message) {
(true, Some(_)) => (),
e => panic!("expected Error::Cancel(_), found {:?}", e),
}
}
#[cfg(not(miri))]
#[tokio::test]
async fn sender_checks_for_want_on_send() {
let (mut tx, mut rx) = channel::<Custom, ()>();
// one is allowed to buffer, second is rejected
tx.try_send(Custom(1)).expect("1 buffered");
tx.try_send(Custom(2)).expect_err("2 not ready");
assert!(PollOnce(&mut rx).await.is_some(), "rx once");
// Even though 1 has been popped, only 1 could be buffered for the
// lifetime of the channel.
tx.try_send(Custom(2)).expect_err("2 still not ready");
assert!(PollOnce(&mut rx).await.is_none(), "rx empty");
tx.try_send(Custom(2)).expect("2 ready");
}
#[cfg(feature = "http2")]
#[test]
fn unbounded_sender_doesnt_bound_on_want() {
let (tx, rx) = channel::<Custom, ()>();
let mut tx = tx.unbound();
tx.try_send(Custom(1)).unwrap();
tx.try_send(Custom(2)).unwrap();
tx.try_send(Custom(3)).unwrap();
drop(rx);
tx.try_send(Custom(4)).unwrap_err();
}
#[cfg(feature = "nightly")]
#[bench]
fn giver_queue_throughput(b: &mut test::Bencher) {
use crate::{body::Incoming, Request, Response};
let rt = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let (mut tx, mut rx) = channel::<Request<Incoming>, Response<Incoming>>();
b.iter(move || {
tx.send(Request::new(Incoming::empty())).unwrap();
rt.block_on(async {
loop {
let poll_once = PollOnce(&mut rx);
let opt = poll_once.await;
if opt.is_none() {
break;
}
}
});
})
}
#[cfg(feature = "nightly")]
#[bench]
fn giver_queue_not_ready(b: &mut test::Bencher) {
let rt = tokio::runtime::Builder::new_current_thread()
.build()
.unwrap();
let (_tx, mut rx) = channel::<i32, ()>();
b.iter(move || {
rt.block_on(async {
let poll_once = PollOnce(&mut rx);
assert!(poll_once.await.is_none());
});
})
}
#[cfg(feature = "nightly")]
#[bench]
fn giver_queue_cancel(b: &mut test::Bencher) {
let (_tx, mut rx) = channel::<i32, ()>();
b.iter(move || {
rx.taker.cancel();
})
}
}
-22
View File
@@ -1,22 +0,0 @@
//! HTTP Client.
//!
//! hyper provides HTTP over a single connection. See the [`conn`] module.
//!
//! ## Examples
//!
//! * [`client`] - A simple CLI http client that requests the url passed in parameters and outputs the response content and details to the stdout, reading content chunk-by-chunk.
//!
//! * [`client_json`] - A simple program that GETs some json, reads the body asynchronously, parses it with serde and outputs the result.
//!
//! [`client`]: https://github.com/hyperium/hyper/blob/master/examples/client.rs
//! [`client_json`]: https://github.com/hyperium/hyper/blob/master/examples/client_json.rs
#[cfg(test)]
mod tests;
cfg_feature! {
#![any(feature = "http1", feature = "http2")]
pub mod conn;
pub(super) mod dispatch;
}
-261
View File
@@ -1,261 +0,0 @@
/*
// FIXME: re-implement tests with `async/await`
#[test]
fn retryable_request() {
let _ = pretty_env_logger::try_init();
let mut rt = Runtime::new().expect("new rt");
let mut connector = MockConnector::new();
let sock1 = connector.mock("http://mock.local");
let sock2 = connector.mock("http://mock.local");
let client = Client::builder()
.build::<_, crate::Body>(connector);
client.pool.no_timer();
{
let req = Request::builder()
.uri("http://mock.local/a")
.body(Default::default())
.unwrap();
let res1 = client.request(req);
let srv1 = poll_fn(|| {
try_ready!(sock1.read(&mut [0u8; 512]));
try_ready!(sock1.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv1 poll_fn error: {}", e));
rt.block_on(res1.join(srv1)).expect("res1");
}
drop(sock1);
let req = Request::builder()
.uri("http://mock.local/b")
.body(Default::default())
.unwrap();
let res2 = client.request(req)
.map(|res| {
assert_eq!(res.status().as_u16(), 222);
});
let srv2 = poll_fn(|| {
try_ready!(sock2.read(&mut [0u8; 512]));
try_ready!(sock2.write(b"HTTP/1.1 222 OK\r\nContent-Length: 0\r\n\r\n"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv2 poll_fn error: {}", e));
rt.block_on(res2.join(srv2)).expect("res2");
}
#[test]
fn conn_reset_after_write() {
let _ = pretty_env_logger::try_init();
let mut rt = Runtime::new().expect("new rt");
let mut connector = MockConnector::new();
let sock1 = connector.mock("http://mock.local");
let client = Client::builder()
.build::<_, crate::Body>(connector);
client.pool.no_timer();
{
let req = Request::builder()
.uri("http://mock.local/a")
.body(Default::default())
.unwrap();
let res1 = client.request(req);
let srv1 = poll_fn(|| {
try_ready!(sock1.read(&mut [0u8; 512]));
try_ready!(sock1.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv1 poll_fn error: {}", e));
rt.block_on(res1.join(srv1)).expect("res1");
}
let req = Request::builder()
.uri("http://mock.local/a")
.body(Default::default())
.unwrap();
let res2 = client.request(req);
let mut sock1 = Some(sock1);
let srv2 = poll_fn(|| {
// We purposefully keep the socket open until the client
// has written the second request, and THEN disconnect.
//
// Not because we expect servers to be jerks, but to trigger
// state where we write on an assumedly good connection, and
// only reset the close AFTER we wrote bytes.
try_ready!(sock1.as_mut().unwrap().read(&mut [0u8; 512]));
sock1.take();
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv2 poll_fn error: {}", e));
let err = rt.block_on(res2.join(srv2)).expect_err("res2");
assert!(err.is_incomplete_message(), "{:?}", err);
}
#[test]
fn checkout_win_allows_connect_future_to_be_pooled() {
let _ = pretty_env_logger::try_init();
let mut rt = Runtime::new().expect("new rt");
let mut connector = MockConnector::new();
let (tx, rx) = oneshot::channel::<()>();
let sock1 = connector.mock("http://mock.local");
let sock2 = connector.mock_fut("http://mock.local", rx);
let client = Client::builder()
.build::<_, crate::Body>(connector);
client.pool.no_timer();
let uri = "http://mock.local/a".parse::<crate::Uri>().expect("uri parse");
// First request just sets us up to have a connection able to be put
// back in the pool. *However*, it doesn't insert immediately. The
// body has 1 pending byte, and we will only drain in request 2, once
// the connect future has been started.
let mut body = {
let res1 = client.get(uri.clone())
.map(|res| res.into_body().concat2());
let srv1 = poll_fn(|| {
try_ready!(sock1.read(&mut [0u8; 512]));
// Chunked is used so as to force 2 body reads.
try_ready!(sock1.write(b"\
HTTP/1.1 200 OK\r\n\
transfer-encoding: chunked\r\n\
\r\n\
1\r\nx\r\n\
0\r\n\r\n\
"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv1 poll_fn error: {}", e));
rt.block_on(res1.join(srv1)).expect("res1").0
};
// The second request triggers the only mocked connect future, but then
// the drained body allows the first socket to go back to the pool,
// "winning" the checkout race.
{
let res2 = client.get(uri.clone());
let drain = poll_fn(move || {
body.poll()
});
let srv2 = poll_fn(|| {
try_ready!(sock1.read(&mut [0u8; 512]));
try_ready!(sock1.write(b"HTTP/1.1 200 OK\r\nConnection: close\r\n\r\nx"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv2 poll_fn error: {}", e));
rt.block_on(res2.join(drain).join(srv2)).expect("res2");
}
// "Release" the mocked connect future, and let the runtime spin once so
// it's all setup...
{
let mut tx = Some(tx);
let client = &client;
let key = client.pool.h1_key("http://mock.local");
let mut tick_cnt = 0;
let fut = poll_fn(move || {
tx.take();
if client.pool.idle_count(&key) == 0 {
tick_cnt += 1;
assert!(tick_cnt < 10, "ticked too many times waiting for idle");
trace!("no idle yet; tick count: {}", tick_cnt);
::futures::task::current().notify();
Ok(Async::NotReady)
} else {
Ok::<_, ()>(Async::Ready(()))
}
});
rt.block_on(fut).unwrap();
}
// Third request just tests out that the "loser" connection was pooled. If
// it isn't, this will panic since the MockConnector doesn't have any more
// mocks to give out.
{
let res3 = client.get(uri);
let srv3 = poll_fn(|| {
try_ready!(sock2.read(&mut [0u8; 512]));
try_ready!(sock2.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv3 poll_fn error: {}", e));
rt.block_on(res3.join(srv3)).expect("res3");
}
}
#[cfg(feature = "nightly")]
#[bench]
fn bench_http1_get_0b(b: &mut test::Bencher) {
let _ = pretty_env_logger::try_init();
let mut rt = Runtime::new().expect("new rt");
let mut connector = MockConnector::new();
let client = Client::builder()
.build::<_, crate::Body>(connector.clone());
client.pool.no_timer();
let uri = Uri::from_static("http://mock.local/a");
b.iter(move || {
let sock1 = connector.mock("http://mock.local");
let res1 = client
.get(uri.clone())
.and_then(|res| {
res.into_body().for_each(|_| Ok(()))
});
let srv1 = poll_fn(|| {
try_ready!(sock1.read(&mut [0u8; 512]));
try_ready!(sock1.write(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\n\r\n"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv1 poll_fn error: {}", e));
rt.block_on(res1.join(srv1)).expect("res1");
});
}
#[cfg(feature = "nightly")]
#[bench]
fn bench_http1_get_10b(b: &mut test::Bencher) {
let _ = pretty_env_logger::try_init();
let mut rt = Runtime::new().expect("new rt");
let mut connector = MockConnector::new();
let client = Client::builder()
.build::<_, crate::Body>(connector.clone());
client.pool.no_timer();
let uri = Uri::from_static("http://mock.local/a");
b.iter(move || {
let sock1 = connector.mock("http://mock.local");
let res1 = client
.get(uri.clone())
.and_then(|res| {
res.into_body().for_each(|_| Ok(()))
});
let srv1 = poll_fn(|| {
try_ready!(sock1.read(&mut [0u8; 512]));
try_ready!(sock1.write(b"HTTP/1.1 200 OK\r\nContent-Length: 10\r\n\r\n0123456789"));
Ok(Async::Ready(()))
}).map_err(|e: std::io::Error| panic!("srv1 poll_fn error: {}", e));
rt.block_on(res1.join(srv1)).expect("res1");
});
}
*/
-150
View File
@@ -1,150 +0,0 @@
use std::collections::VecDeque;
use std::io::IoSlice;
use bytes::{Buf, BufMut, Bytes, BytesMut};
pub(crate) struct BufList<T> {
bufs: VecDeque<T>,
}
impl<T: Buf> BufList<T> {
pub(crate) fn new() -> BufList<T> {
BufList {
bufs: VecDeque::new(),
}
}
#[inline]
pub(crate) fn push(&mut self, buf: T) {
debug_assert!(buf.has_remaining());
self.bufs.push_back(buf);
}
#[inline]
pub(crate) fn bufs_cnt(&self) -> usize {
self.bufs.len()
}
}
impl<T: Buf> Buf for BufList<T> {
#[inline]
fn remaining(&self) -> usize {
self.bufs.iter().map(|buf| buf.remaining()).sum()
}
#[inline]
fn chunk(&self) -> &[u8] {
self.bufs.front().map(Buf::chunk).unwrap_or_default()
}
#[inline]
fn advance(&mut self, mut cnt: usize) {
while cnt > 0 {
{
let front = &mut self.bufs[0];
let rem = front.remaining();
if rem > cnt {
front.advance(cnt);
return;
} else {
front.advance(rem);
cnt -= rem;
}
}
self.bufs.pop_front();
}
}
#[inline]
fn chunks_vectored<'data>(&'data self, dst: &mut [IoSlice<'data>]) -> usize {
if dst.is_empty() {
return 0;
}
let mut vecs = 0;
for buf in &self.bufs {
vecs += buf.chunks_vectored(&mut dst[vecs..]);
if vecs == dst.len() {
break;
}
}
vecs
}
#[inline]
fn copy_to_bytes(&mut self, len: usize) -> Bytes {
// Our inner buffer may have an optimized version of copy_to_bytes, and if the whole
// request can be fulfilled by the front buffer, we can take advantage.
match self.bufs.front_mut() {
Some(front) if front.remaining() == len => {
let b = front.copy_to_bytes(len);
self.bufs.pop_front();
b
}
Some(front) if front.remaining() > len => front.copy_to_bytes(len),
_ => {
assert!(len <= self.remaining(), "`len` greater than remaining");
let mut bm = BytesMut::with_capacity(len);
bm.put(self.take(len));
bm.freeze()
}
}
}
}
#[cfg(test)]
mod tests {
use std::ptr;
use super::*;
fn hello_world_buf() -> BufList<Bytes> {
BufList {
bufs: vec![Bytes::from("Hello"), Bytes::from(" "), Bytes::from("World")].into(),
}
}
#[test]
fn to_bytes_shorter() {
let mut bufs = hello_world_buf();
let old_ptr = bufs.chunk().as_ptr();
let start = bufs.copy_to_bytes(4);
assert_eq!(start, "Hell");
assert!(ptr::eq(old_ptr, start.as_ptr()));
assert_eq!(bufs.chunk(), b"o");
assert!(ptr::eq(old_ptr.wrapping_add(4), bufs.chunk().as_ptr()));
assert_eq!(bufs.remaining(), 7);
}
#[test]
fn to_bytes_eq() {
let mut bufs = hello_world_buf();
let old_ptr = bufs.chunk().as_ptr();
let start = bufs.copy_to_bytes(5);
assert_eq!(start, "Hello");
assert!(ptr::eq(old_ptr, start.as_ptr()));
assert_eq!(bufs.chunk(), b" ");
assert_eq!(bufs.remaining(), 6);
}
#[test]
fn to_bytes_longer() {
let mut bufs = hello_world_buf();
let start = bufs.copy_to_bytes(7);
assert_eq!(start, "Hello W");
assert_eq!(bufs.remaining(), 4);
}
#[test]
fn one_long_buf_to_bytes() {
let mut buf = BufList::new();
buf.push(b"Hello World" as &[_]);
assert_eq!(buf.copy_to_bytes(5), "Hello");
assert_eq!(buf.chunk(), b" World");
}
#[test]
#[should_panic(expected = "`len` greater than remaining")]
fn buf_to_bytes_too_many() {
hello_world_buf().copy_to_bytes(42);
}
}
-143
View File
@@ -1,143 +0,0 @@
use std::cell::RefCell;
use std::fmt::{self, Write};
use std::str;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
#[cfg(feature = "http2")]
use http::header::HeaderValue;
use httpdate::HttpDate;
// "Sun, 06 Nov 1994 08:49:37 GMT".len()
pub(crate) const DATE_VALUE_LENGTH: usize = 29;
#[cfg(feature = "http1")]
pub(crate) fn extend(dst: &mut Vec<u8>) {
CACHED.with(|cache| {
dst.extend_from_slice(cache.borrow().buffer());
});
}
#[cfg(feature = "http1")]
pub(crate) fn update() {
CACHED.with(|cache| {
cache.borrow_mut().check();
});
}
#[cfg(feature = "http2")]
pub(crate) fn update_and_header_value() -> HeaderValue {
CACHED.with(|cache| {
let mut cache = cache.borrow_mut();
cache.check();
cache.header_value.clone()
})
}
struct CachedDate {
bytes: [u8; DATE_VALUE_LENGTH],
pos: usize,
#[cfg(feature = "http2")]
header_value: HeaderValue,
next_update: SystemTime,
}
thread_local!(static CACHED: RefCell<CachedDate> = RefCell::new(CachedDate::new()));
impl CachedDate {
fn new() -> Self {
let mut cache = CachedDate {
bytes: [0; DATE_VALUE_LENGTH],
pos: 0,
#[cfg(feature = "http2")]
header_value: HeaderValue::from_static(""),
next_update: SystemTime::now(),
};
cache.update(cache.next_update);
cache
}
fn buffer(&self) -> &[u8] {
&self.bytes[..]
}
fn check(&mut self) {
let now = SystemTime::now();
if now > self.next_update {
self.update(now);
}
}
fn update(&mut self, now: SystemTime) {
let nanos = now
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.subsec_nanos();
self.render(now);
self.next_update = now + Duration::new(1, 0) - Duration::from_nanos(u64::from(nanos));
}
fn render(&mut self, now: SystemTime) {
self.pos = 0;
let _ = write!(self, "{}", HttpDate::from(now));
debug_assert_eq!(self.pos, DATE_VALUE_LENGTH);
self.render_http2();
}
#[cfg(feature = "http2")]
fn render_http2(&mut self) {
self.header_value = HeaderValue::from_bytes(self.buffer())
.expect("Date format should be valid HeaderValue");
}
#[cfg(not(feature = "http2"))]
fn render_http2(&mut self) {}
}
impl fmt::Write for CachedDate {
fn write_str(&mut self, s: &str) -> fmt::Result {
let len = s.len();
self.bytes[self.pos..self.pos + len].copy_from_slice(s.as_bytes());
self.pos += len;
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[cfg(feature = "nightly")]
use test::Bencher;
#[test]
fn test_date_len() {
assert_eq!(DATE_VALUE_LENGTH, "Sun, 06 Nov 1994 08:49:37 GMT".len());
}
#[cfg(feature = "nightly")]
#[bench]
fn bench_date_check(b: &mut Bencher) {
let mut date = CachedDate::new();
// cache the first update
date.check();
b.iter(|| {
date.check();
});
}
#[cfg(feature = "nightly")]
#[bench]
fn bench_date_render(b: &mut Bencher) {
let mut date = CachedDate::new();
let now = SystemTime::now();
date.render(now);
b.bytes = date.buffer().len() as u64;
b.iter(|| {
date.render(now);
test::black_box(&date);
});
}
}
-46
View File
@@ -1,46 +0,0 @@
use pin_project_lite::pin_project;
use std::{
future::Future,
pin::Pin,
task::{Context, Poll},
};
pin_project! {
/// One of two possible futures that have the same output type.
#[project = EitherProj]
pub(crate) enum Either<F1, F2> {
Left {
#[pin]
fut: F1
},
Right {
#[pin]
fut: F2,
},
}
}
impl<F1, F2> Either<F1, F2> {
pub(crate) fn left(fut: F1) -> Self {
Either::Left { fut }
}
pub(crate) fn right(fut: F2) -> Self {
Either::Right { fut }
}
}
impl<F1, F2> Future for Either<F1, F2>
where
F1: Future,
F2: Future<Output = F1::Output>,
{
type Output = F1::Output;
fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
match self.project() {
EitherProj::Left { fut } => fut.poll(cx),
EitherProj::Right { fut } => fut.poll(cx),
}
}
}
-30
View File
@@ -1,30 +0,0 @@
use std::{
future::Future,
pin::Pin,
task::{Context, Poll},
};
// TODO: replace with `std::future::poll_fn` once MSRV >= 1.64
pub(crate) fn poll_fn<T, F>(f: F) -> PollFn<F>
where
F: FnMut(&mut Context<'_>) -> Poll<T>,
{
PollFn { f }
}
pub(crate) struct PollFn<F> {
f: F,
}
impl<F> Unpin for PollFn<F> {}
impl<T, F> Future for PollFn<F>
where
F: FnMut(&mut Context<'_>) -> Poll<T>,
{
type Output = T;
fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Self::Output> {
(self.as_mut().f)(cx)
}
}
-163
View File
@@ -1,163 +0,0 @@
use std::pin::Pin;
use std::task::{Context, Poll};
/// This adapts from `hyper` IO traits to the ones in Tokio.
///
/// This is currently used by `h2`, and by hyper internal unit tests.
#[derive(Debug)]
pub(crate) struct Compat<T>(pub(crate) T);
impl<T> Compat<T> {
pub(crate) fn new(io: T) -> Self {
Compat(io)
}
fn p(self: Pin<&mut Self>) -> Pin<&mut T> {
// SAFETY: The simplest of projections. This is just
// a wrapper, we don't do anything that would undo the projection.
unsafe { self.map_unchecked_mut(|me| &mut me.0) }
}
}
impl<T> tokio::io::AsyncRead for Compat<T>
where
T: crate::rt::Read,
{
/// `poll_read` fn implementation for `Compat<T>`.
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
tbuf: &mut tokio::io::ReadBuf<'_>,
) -> Poll<Result<(), std::io::Error>> {
let init = tbuf.initialized().len();
let filled = tbuf.filled().len();
// SAFETY:
// 1. `tbuf.inner_mut()` returns a raw pointer/mutable slice which we wrap into
// a `crate::rt::ReadBuf` that is layout-compatible with the source `tokio::io::ReadBuf`.
// 2. We explicitly restore the `init` and `filled` states from the original buffer
// to maintain the invariant that the new `ReadBuf` tracks the same progress.
// 3. The underlying memory remains valid and uniquely accessible via `tbuf` for
// the duration of this poll operation.
let (new_init, new_filled) = unsafe {
let mut buf = crate::rt::ReadBuf::uninit(tbuf.inner_mut());
buf.set_init(init);
buf.set_filled(filled);
match crate::rt::Read::poll_read(self.p(), cx, buf.unfilled()) {
Poll::Ready(Ok(())) => (buf.init_len(), buf.len()),
other => return other,
}
};
let n_init = new_init - init;
// SAFETY:
// 1. `tbuf.assume_init(n_init)` is safe because `crate::rt::Read::poll_read`
// guarantees that the bytes written into the buffer were initialized.
// 2. `tbuf.set_filled(new_filled)` is safe because `new_filled` is derived
// directly from the buffer state after the successful read operation.
unsafe {
tbuf.assume_init(n_init);
tbuf.set_filled(new_filled);
}
Poll::Ready(Ok(()))
}
}
impl<T> tokio::io::AsyncWrite for Compat<T>
where
T: crate::rt::Write,
{
fn poll_write(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<Result<usize, std::io::Error>> {
crate::rt::Write::poll_write(self.p(), cx, buf)
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), std::io::Error>> {
crate::rt::Write::poll_flush(self.p(), cx)
}
fn poll_shutdown(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Result<(), std::io::Error>> {
crate::rt::Write::poll_shutdown(self.p(), cx)
}
fn is_write_vectored(&self) -> bool {
crate::rt::Write::is_write_vectored(&self.0)
}
fn poll_write_vectored(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
bufs: &[std::io::IoSlice<'_>],
) -> Poll<Result<usize, std::io::Error>> {
crate::rt::Write::poll_write_vectored(self.p(), cx, bufs)
}
}
#[cfg(test)]
impl<T> crate::rt::Read for Compat<T>
where
T: tokio::io::AsyncRead,
{
fn poll_read(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
mut buf: crate::rt::ReadBufCursor<'_>,
) -> Poll<Result<(), std::io::Error>> {
let n = unsafe {
let mut tbuf = tokio::io::ReadBuf::uninit(buf.as_mut());
match tokio::io::AsyncRead::poll_read(self.p(), cx, &mut tbuf) {
Poll::Ready(Ok(())) => tbuf.filled().len(),
other => return other,
}
};
unsafe {
buf.advance(n);
}
Poll::Ready(Ok(()))
}
}
#[cfg(test)]
impl<T> crate::rt::Write for Compat<T>
where
T: tokio::io::AsyncWrite,
{
fn poll_write(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<Result<usize, std::io::Error>> {
tokio::io::AsyncWrite::poll_write(self.p(), cx, buf)
}
fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Result<(), std::io::Error>> {
tokio::io::AsyncWrite::poll_flush(self.p(), cx)
}
fn poll_shutdown(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
) -> Poll<Result<(), std::io::Error>> {
tokio::io::AsyncWrite::poll_shutdown(self.p(), cx)
}
fn is_write_vectored(&self) -> bool {
tokio::io::AsyncWrite::is_write_vectored(&self.0)
}
fn poll_write_vectored(
self: Pin<&mut Self>,
cx: &mut Context<'_>,
bufs: &[std::io::IoSlice<'_>],
) -> Poll<Result<usize, std::io::Error>> {
tokio::io::AsyncWrite::poll_write_vectored(self.p(), cx, bufs)
}
}
-7
View File
@@ -1,7 +0,0 @@
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
mod compat;
mod rewind;
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
pub(crate) use self::compat::Compat;
pub(crate) use self::rewind::Rewind;
-166
View File
@@ -1,166 +0,0 @@
use std::pin::Pin;
use std::task::{Context, Poll};
use std::{cmp, io};
use bytes::{Buf, Bytes};
use crate::rt::{Read, ReadBufCursor, Write};
/// Combine a buffer with an IO, rewinding reads to use the buffer.
#[derive(Debug)]
pub(crate) struct Rewind<T> {
pre: Option<Bytes>,
inner: T,
}
impl<T> Rewind<T> {
#[cfg(all(
test,
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
pub(crate) fn new(io: T) -> Self {
Rewind {
pre: None,
inner: io,
}
}
pub(crate) fn new_buffered(io: T, buf: Bytes) -> Self {
Rewind {
pre: Some(buf),
inner: io,
}
}
#[cfg(all(
test,
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
pub(crate) fn rewind(&mut self, bs: Bytes) {
debug_assert!(self.pre.is_none());
self.pre = Some(bs);
}
pub(crate) fn into_inner(self) -> (T, Bytes) {
(self.inner, self.pre.unwrap_or_default())
}
}
impl<T> Read for Rewind<T>
where
T: Read + Unpin,
{
fn poll_read(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
mut buf: ReadBufCursor<'_>,
) -> Poll<io::Result<()>> {
if let Some(mut prefix) = self.pre.take() {
// If there are no remaining bytes, let the bytes get dropped.
if !prefix.is_empty() {
let copy_len = cmp::min(prefix.len(), buf.remaining());
// TODO: There should be a way to do following two lines cleaner...
buf.put_slice(&prefix[..copy_len]);
prefix.advance(copy_len);
// Put back what's left
if !prefix.is_empty() {
self.pre = Some(prefix);
}
return Poll::Ready(Ok(()));
}
}
Pin::new(&mut self.inner).poll_read(cx, buf)
}
}
impl<T> Write for Rewind<T>
where
T: Write + Unpin,
{
fn poll_write(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
buf: &[u8],
) -> Poll<io::Result<usize>> {
Pin::new(&mut self.inner).poll_write(cx, buf)
}
fn poll_write_vectored(
mut self: Pin<&mut Self>,
cx: &mut Context<'_>,
bufs: &[io::IoSlice<'_>],
) -> Poll<io::Result<usize>> {
Pin::new(&mut self.inner).poll_write_vectored(cx, bufs)
}
fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Pin::new(&mut self.inner).poll_flush(cx)
}
fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<io::Result<()>> {
Pin::new(&mut self.inner).poll_shutdown(cx)
}
fn is_write_vectored(&self) -> bool {
self.inner.is_write_vectored()
}
}
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2"),
))]
#[cfg(test)]
mod tests {
use super::super::Compat;
use super::Rewind;
use bytes::Bytes;
use tokio::io::AsyncReadExt;
#[cfg(not(miri))]
#[tokio::test]
async fn partial_rewind() {
let underlying = [104, 101, 108, 108, 111];
let mock = tokio_test::io::Builder::new().read(&underlying).build();
let mut stream = Compat::new(Rewind::new(Compat::new(mock)));
// Read off some bytes, ensure we filled o1
let mut buf = [0; 2];
stream.read_exact(&mut buf).await.expect("read1");
// Rewind the stream so that it is as if we never read in the first place.
stream.0.rewind(Bytes::copy_from_slice(&buf[..]));
let mut buf = [0; 5];
stream.read_exact(&mut buf).await.expect("read1");
// At this point we should have read everything that was in the MockStream
assert_eq!(&buf, &underlying);
}
#[cfg(not(miri))]
#[tokio::test]
async fn full_rewind() {
let underlying = [104, 101, 108, 108, 111];
let mock = tokio_test::io::Builder::new().read(&underlying).build();
let mut stream = Compat::new(Rewind::new(Compat::new(mock)));
let mut buf = [0; 5];
stream.read_exact(&mut buf).await.expect("read1");
// Rewind the stream so that it is as if we never read in the first place.
stream.0.rewind(Bytes::copy_from_slice(&buf[..]));
let mut buf = [0; 5];
stream.read_exact(&mut buf).await.expect("read1");
assert_eq!(&buf, &underlying);
}
}
-16
View File
@@ -1,16 +0,0 @@
use std::sync::LockResult;
pub(crate) trait LockResultExt<T> {
fn panic_if_poisoned(self) -> T;
}
#[allow(clippy::panic)]
impl<T> LockResultExt<T> for LockResult<T> {
#[track_caller]
fn panic_if_poisoned(self) -> T {
match self {
Ok(inner) => inner,
Err(err) => panic!("lock poisoned by panic: {err}"),
}
}
}
-20
View File
@@ -1,20 +0,0 @@
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(crate) mod buf;
#[cfg(all(feature = "server", any(feature = "http1", feature = "http2")))]
pub(crate) mod date;
#[cfg(all(feature = "client", feature = "http2"))]
pub(crate) mod either;
#[cfg(any(
all(feature = "client", any(feature = "http1", feature = "http2")),
all(feature = "server", feature = "http1"),
))]
pub(crate) mod future;
pub(crate) mod io;
pub(crate) mod lock;
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(crate) mod task;
#[cfg(any(
all(feature = "server", feature = "http1"),
all(any(feature = "client", feature = "server"), feature = "http2"),
))]
pub(crate) mod time;
-45
View File
@@ -1,45 +0,0 @@
use std::task::{Context, Poll};
#[cfg(feature = "client")]
use std::task::{RawWaker, RawWakerVTable, Waker};
/// A function to help "yield" a future, such that it is re-scheduled immediately.
///
/// Useful for spin counts, so a future doesn't hog too much time.
pub(crate) fn yield_now(cx: &mut Context<'_>) -> Poll<std::convert::Infallible> {
cx.waker().wake_by_ref();
Poll::Pending
}
// TODO: replace with `std::task::Waker::noop()` once MSRV >= 1.85
#[cfg(feature = "client")]
fn noop_waker() -> Waker {
const NOOP_RAW_WAKER: RawWaker = RawWaker::new(std::ptr::null(), &NOOP_VTABLE);
const NOOP_VTABLE: RawWakerVTable = RawWakerVTable::new(
// `clone` returns the same noop waker again
|_: *const ()| NOOP_RAW_WAKER,
// `wake`, `wake_by_ref`, and `drop` do nothing
|_: *const ()| {},
|_: *const ()| {},
|_: *const ()| {},
);
// SAFETY: all functions in the vtable are safe to call, and Waker's safety does not require
// them to actually do anything.
unsafe { Waker::from_raw(NOOP_RAW_WAKER) }
}
/// Poll the future once and return `Some` if it is ready, else `None`.
///
/// If the future wasn't ready, it future likely can't be driven to completion any more: the polling
/// uses a no-op waker, so knowledge of what the pending future was waiting for is lost.
#[cfg(feature = "client")]
pub(crate) fn now_or_never<F: std::future::Future>(fut: F) -> Option<F::Output> {
let waker = noop_waker();
let mut cx = Context::from_waker(&waker);
// TODO: replace with std::pin::pin! once MSRV >= 1.68
tokio::pin!(fut);
match fut.poll(&mut cx) {
Poll::Ready(res) => Some(res),
Poll::Pending => None,
}
}
-90
View File
@@ -1,90 +0,0 @@
#[cfg(any(
all(any(feature = "client", feature = "server"), feature = "http2"),
all(feature = "server", feature = "http1"),
))]
use std::time::Duration;
use std::{fmt, sync::Arc};
use std::{pin::Pin, time::Instant};
use crate::rt::Sleep;
use crate::rt::Timer;
/// A user-provided timer to time background tasks.
#[derive(Clone)]
pub(crate) enum Time {
Timer(Arc<dyn Timer + Send + Sync>),
Empty,
}
#[cfg(all(feature = "server", feature = "http1"))]
#[derive(Clone, Copy, Debug)]
pub(crate) enum Dur {
Default(Option<Duration>),
Configured(Option<Duration>),
}
impl fmt::Debug for Time {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Time").finish()
}
}
impl Time {
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
pub(crate) fn sleep(&self, duration: Duration) -> Pin<Box<dyn Sleep>> {
match &self {
#[allow(clippy::panic)]
Time::Empty => {
panic!("You must supply a timer.")
}
Time::Timer(t) => t.sleep(duration),
}
}
#[cfg(all(feature = "server", feature = "http1"))]
pub(crate) fn sleep_until(&self, deadline: Instant) -> Pin<Box<dyn Sleep>> {
match &self {
#[allow(clippy::panic)]
Time::Empty => {
panic!("You must supply a timer.")
}
Time::Timer(t) => t.sleep_until(deadline),
}
}
pub(crate) fn now(&self) -> Instant {
match &self {
Time::Empty => Instant::now(),
Time::Timer(t) => t.now(),
}
}
pub(crate) fn reset(&self, sleep: &mut Pin<Box<dyn Sleep>>, new_deadline: Instant) {
match &self {
#[allow(clippy::panic)]
Time::Empty => {
panic!("You must supply a timer.")
}
Time::Timer(t) => t.reset(sleep, new_deadline),
}
}
#[cfg(all(feature = "server", feature = "http1"))]
pub(crate) fn check(&self, dur: Dur, name: &'static str) -> Option<Duration> {
match dur {
Dur::Default(Some(dur)) => match self {
Time::Empty => {
warn!("timeout `{}` has default, but no timer set", name,);
None
}
Time::Timer(..) => Some(dur),
},
Dur::Configured(Some(dur)) => match self {
#[allow(clippy::panic)]
Time::Empty => panic!("timeout `{name}` set, but no timer set"),
Time::Timer(..) => Some(dur),
},
Dur::Default(None) | Dur::Configured(None) => None,
}
}
}
-740
View File
@@ -1,740 +0,0 @@
//! Error and Result module.
use std::error::Error as StdError;
use std::fmt;
/// Result type often returned from methods that can have hyper `Error`s.
pub type Result<T> = std::result::Result<T, Error>;
type Cause = Box<dyn StdError + Send + Sync>;
/// Represents errors that can occur handling HTTP streams.
///
/// # Formatting
///
/// The `Display` implementation of this type will only print the details of
/// this level of error, even though it may have been caused by another error
/// and contain that error in its source. To print all the relevant
/// information, including the source chain, using something like
/// `std::error::Report`, or equivalent 3rd party types.
///
/// The contents of the formatted error message of this specific `Error` type
/// is unspecified. **You must not depend on it.** The wording and details may
/// change in any version, with the goal of improving error messages.
///
/// # Source
///
/// A `hyper::Error` may be caused by another error. To aid in debugging,
/// those are exposed in `Error::source()` as erased types. While it is
/// possible to check the exact type of the sources, they **can not be depended
/// on**. They may come from private internal dependencies, and are subject to
/// change at any moment.
pub struct Error {
inner: Box<ErrorImpl>,
}
struct ErrorImpl {
kind: Kind,
cause: Option<Cause>,
}
#[derive(Debug)]
pub(super) enum Kind {
Parse(Parse),
User(User),
/// A message reached EOF, but is not complete.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
IncompleteMessage,
/// A connection received a message (or bytes) when not waiting for one.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
UnexpectedMessage,
/// A pending item was dropped before ever being processed.
Canceled,
/// Indicates a channel (client or body sender) is closed.
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
all(feature = "http2", feature = "client")
))]
ChannelClosed,
/// An `io::Error` that occurred while trying to read or write to a network stream.
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
Io,
/// User took too long to send headers.
#[cfg(all(feature = "http1", feature = "server"))]
HeaderTimeout,
/// Error while reading a body from connection.
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
Body,
/// Error while writing a body to connection.
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
BodyWrite,
/// Error calling `AsyncWrite::shutdown()`.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Shutdown,
/// A general error from h2.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
Http2,
}
#[derive(Debug)]
pub(super) enum Parse {
Method,
#[cfg(feature = "http1")]
Version,
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
VersionH2,
Uri,
#[cfg(all(feature = "http1", feature = "server"))]
UriTooLong,
#[cfg(feature = "http1")]
Header(Header),
#[cfg(any(feature = "http1", feature = "http2"))]
#[cfg_attr(feature = "http2", allow(unused))]
TooLarge,
Status,
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Internal,
}
#[derive(Debug)]
#[cfg(feature = "http1")]
pub(super) enum Header {
Token,
#[cfg(any(feature = "client", feature = "server"))]
ContentLengthInvalid,
#[cfg(feature = "server")]
TransferEncodingInvalid,
#[cfg(any(feature = "client", feature = "server"))]
TransferEncodingUnexpected,
}
#[derive(Debug)]
pub(super) enum User {
/// Error calling the user's `Body::poll_data()`.
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
Body,
/// The user aborted writing of the outgoing body.
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
feature = "ffi"
))]
BodyWriteAborted,
/// User tried to send a connect request with a nonzero body.
#[cfg(all(feature = "client", feature = "http2"))]
InvalidConnectWithBody,
/// Error from future of user's Service.
#[cfg(any(
all(any(feature = "client", feature = "server"), feature = "http1"),
all(feature = "server", feature = "http2")
))]
Service,
/// User tried to send a certain header in an unexpected context.
///
/// For example, sending both `content-length` and `transfer-encoding`.
#[cfg(any(feature = "http1", feature = "http2"))]
#[cfg(feature = "server")]
UnexpectedHeader,
/// User tried to respond with a 1xx (not 101) response code.
#[cfg(feature = "http1")]
#[cfg(feature = "server")]
UnsupportedStatusCode,
/// User tried polling for an upgrade that doesn't exist.
NoUpgrade,
/// User polled for an upgrade, but low-level API is not using upgrades.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
ManualUpgrade,
/// The dispatch task is gone.
#[cfg(all(feature = "client", any(feature = "http1", feature = "http2")))]
DispatchGone,
/// User aborted in an FFI callback.
#[cfg(feature = "ffi")]
AbortedByCallback,
}
// Sentinel type to indicate the error was caused by a timeout.
#[derive(Debug)]
pub(super) struct TimedOut;
impl Error {
/// Returns true if this was an HTTP parse error.
///
/// This can be caused by a malformed HTTP message, an invalid header,
/// an invalid URI, an invalid HTTP version, or a message head that is
/// too large. Use the more specific `is_parse_*` methods to determine
/// the exact cause.
pub fn is_parse(&self) -> bool {
matches!(self.inner.kind, Kind::Parse(_))
}
/// Returns true if this was an HTTP parse error caused by a message that was too large.
///
/// This is triggered when the message head (request line plus headers for
/// HTTP/1, or header frame for HTTP/2) exceeds the configured
/// [`max_buf_size`](crate::server::conn::http1::Builder::max_buf_size).
/// It also covers the case where the URI alone exceeds the internal
/// maximum URI length.
#[cfg(all(feature = "http1", feature = "server"))]
pub fn is_parse_too_large(&self) -> bool {
matches!(
self.inner.kind,
Kind::Parse(Parse::TooLarge) | Kind::Parse(Parse::UriTooLong)
)
}
/// Returns true if this was an HTTP parse error caused by an invalid response status code or
/// reason phrase.
pub fn is_parse_status(&self) -> bool {
matches!(self.inner.kind, Kind::Parse(Parse::Status))
}
/// Returns true if this was an HTTP parse error caused by HTTP2 preface sent over an HTTP1
/// connection.
///
/// This can happen when a client sends an HTTP/2 connection preface to a
/// server that is only expecting HTTP/1.x requests.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub fn is_parse_version_h2(&self) -> bool {
matches!(self.inner.kind, Kind::Parse(Parse::VersionH2))
}
/// Returns true if this error was caused by user code.
///
/// For example, this can be returned when the user's `Service` returns
/// an error, the user's `Body` stream yields an error, or the user
/// sends an unexpected header combination (such as both
/// `content-length` and `transfer-encoding`).
pub fn is_user(&self) -> bool {
matches!(self.inner.kind, Kind::User(_))
}
/// Returns true if this was about a `Request` that was canceled.
///
/// This typically happens when a pending request is dropped before
/// it can be dispatched to the connection, for example because the
/// connection was not ready.
pub fn is_canceled(&self) -> bool {
matches!(self.inner.kind, Kind::Canceled)
}
/// Returns true if a sender's channel is closed.
///
/// This can occur when the other side of a client or body channel
/// has been dropped, indicating that the receiver is no longer
/// interested in the data.
pub fn is_closed(&self) -> bool {
#[cfg(not(any(
all(feature = "http1", any(feature = "client", feature = "server")),
all(feature = "http2", feature = "client")
)))]
return false;
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
all(feature = "http2", feature = "client")
))]
matches!(self.inner.kind, Kind::ChannelClosed)
}
/// Returns true if the connection closed before a message could complete.
///
/// This means that the supplied IO connection reported EOF (closed) while
/// hyper's HTTP state indicates more of the message (either request or
/// response) needed to be transmitted.
///
/// Some cases this could happen (not exhaustive):
///
/// - A request is written on a connection, and the next `read` reports
/// EOF (perhaps a server just closed an "idle" connection).
/// - A message body is only partially receive before the connection
/// reports EOF.
/// - A client writes a request to your server, and then closes the write
/// half while waiting for your response. If you need to support this,
/// consider enabling [`half_close`].
///
/// [`half_close`]: crate::server::conn::http1::Builder::half_close()
pub fn is_incomplete_message(&self) -> bool {
#[cfg(not(all(any(feature = "client", feature = "server"), feature = "http1")))]
return false;
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
matches!(self.inner.kind, Kind::IncompleteMessage)
}
/// Returns true if the body write was aborted.
///
/// This occurs when the user's code explicitly aborts writing of the
/// outgoing body before it completes, for example by dropping the
/// body sender.
pub fn is_body_write_aborted(&self) -> bool {
#[cfg(not(any(
all(feature = "http1", any(feature = "client", feature = "server")),
feature = "ffi"
)))]
return false;
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
feature = "ffi"
))]
matches!(self.inner.kind, Kind::User(User::BodyWriteAborted))
}
/// Returns true if the error was caused while calling `AsyncWrite::shutdown()`.
///
/// This can happen when the connection is being gracefully shut down
/// and the underlying IO reports an error during the shutdown sequence.
pub fn is_shutdown(&self) -> bool {
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
if matches!(self.inner.kind, Kind::Shutdown) {
return true;
}
false
}
/// Returns true if the error was caused by a timeout.
///
/// For HTTP/1 servers, this includes the header read timeout (see
/// [`header_read_timeout`](crate::server::conn::http1::Builder::header_read_timeout)).
/// It also covers any timeout set via a user-provided timer.
pub fn is_timeout(&self) -> bool {
#[cfg(all(feature = "http1", feature = "server"))]
if matches!(self.inner.kind, Kind::HeaderTimeout) {
return true;
}
self.find_source::<TimedOut>().is_some()
}
pub(super) fn new(kind: Kind) -> Error {
Error {
inner: Box::new(ErrorImpl { kind, cause: None }),
}
}
pub(super) fn with<C: Into<Cause>>(mut self, cause: C) -> Error {
self.inner.cause = Some(cause.into());
self
}
#[cfg(any(all(feature = "http1", feature = "server"), feature = "ffi"))]
pub(super) fn kind(&self) -> &Kind {
&self.inner.kind
}
pub(crate) fn find_source<E: StdError + 'static>(&self) -> Option<&E> {
let mut cause = self.source();
while let Some(err) = cause {
if let Some(typed) = err.downcast_ref() {
return Some(typed);
}
cause = err.source();
}
// else
None
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
pub(super) fn h2_reason(&self) -> h2::Reason {
// Find an h2::Reason somewhere in the cause stack, if it exists,
// otherwise assume an INTERNAL_ERROR.
self.find_source::<h2::Error>()
.and_then(|h2_err| h2_err.reason())
.unwrap_or(h2::Reason::INTERNAL_ERROR)
}
pub(super) fn new_canceled() -> Error {
Error::new(Kind::Canceled)
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(super) fn new_incomplete() -> Error {
Error::new(Kind::IncompleteMessage)
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(super) fn new_too_large() -> Error {
Error::new(Kind::Parse(Parse::TooLarge))
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(super) fn new_version_h2() -> Error {
Error::new(Kind::Parse(Parse::VersionH2))
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(super) fn new_unexpected_message() -> Error {
Error::new(Kind::UnexpectedMessage)
}
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
pub(super) fn new_io(cause: std::io::Error) -> Error {
Error::new(Kind::Io).with(cause)
}
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
all(feature = "http2", feature = "client")
))]
pub(super) fn new_closed() -> Error {
Error::new(Kind::ChannelClosed)
}
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
pub(super) fn new_body<E: Into<Cause>>(cause: E) -> Error {
Error::new(Kind::Body).with(cause)
}
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
pub(super) fn new_body_write<E: Into<Cause>>(cause: E) -> Error {
Error::new(Kind::BodyWrite).with(cause)
}
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
feature = "ffi"
))]
pub(super) fn new_body_write_aborted() -> Error {
Error::new(Kind::User(User::BodyWriteAborted))
}
fn new_user(user: User) -> Error {
Error::new(Kind::User(user))
}
#[cfg(any(feature = "http1", feature = "http2"))]
#[cfg(feature = "server")]
pub(super) fn new_user_header() -> Error {
Error::new_user(User::UnexpectedHeader)
}
#[cfg(all(feature = "http1", feature = "server"))]
pub(super) fn new_header_timeout() -> Error {
Error::new(Kind::HeaderTimeout)
}
#[cfg(feature = "http1")]
#[cfg(feature = "server")]
pub(super) fn new_user_unsupported_status_code() -> Error {
Error::new_user(User::UnsupportedStatusCode)
}
pub(super) fn new_user_no_upgrade() -> Error {
Error::new_user(User::NoUpgrade)
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(super) fn new_user_manual_upgrade() -> Error {
Error::new_user(User::ManualUpgrade)
}
#[cfg(any(
all(any(feature = "client", feature = "server"), feature = "http1"),
all(feature = "server", feature = "http2")
))]
pub(super) fn new_user_service<E: Into<Cause>>(cause: E) -> Error {
Error::new_user(User::Service).with(cause)
}
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
pub(super) fn new_user_body<E: Into<Cause>>(cause: E) -> Error {
Error::new_user(User::Body).with(cause)
}
#[cfg(all(feature = "client", feature = "http2"))]
pub(super) fn new_user_invalid_connect() -> Error {
Error::new_user(User::InvalidConnectWithBody)
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
pub(super) fn new_shutdown(cause: std::io::Error) -> Error {
Error::new(Kind::Shutdown).with(cause)
}
#[cfg(feature = "ffi")]
pub(super) fn new_user_aborted_by_callback() -> Error {
Error::new_user(User::AbortedByCallback)
}
#[cfg(all(feature = "client", any(feature = "http1", feature = "http2")))]
pub(super) fn new_user_dispatch_gone() -> Error {
Error::new(Kind::User(User::DispatchGone))
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
pub(super) fn new_h2(cause: ::h2::Error) -> Error {
if cause.is_io() {
Error::new_io(cause.into_io().expect("h2::Error::is_io"))
} else {
Error::new(Kind::Http2).with(cause)
}
}
fn description(&self) -> &str {
match self.inner.kind {
Kind::Parse(Parse::Method) => "invalid HTTP method parsed",
#[cfg(feature = "http1")]
Kind::Parse(Parse::Version) => "invalid HTTP version parsed",
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Kind::Parse(Parse::VersionH2) => "invalid HTTP version parsed (found HTTP2 preface)",
Kind::Parse(Parse::Uri) => "invalid URI",
#[cfg(all(feature = "http1", feature = "server"))]
Kind::Parse(Parse::UriTooLong) => "URI too long",
#[cfg(feature = "http1")]
Kind::Parse(Parse::Header(Header::Token)) => "invalid HTTP header parsed",
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Kind::Parse(Parse::Header(Header::ContentLengthInvalid)) => {
"invalid content-length parsed"
}
#[cfg(all(feature = "http1", feature = "server"))]
Kind::Parse(Parse::Header(Header::TransferEncodingInvalid)) => {
"invalid transfer-encoding parsed"
}
#[cfg(all(feature = "http1", any(feature = "client", feature = "server")))]
Kind::Parse(Parse::Header(Header::TransferEncodingUnexpected)) => {
"unexpected transfer-encoding parsed"
}
#[cfg(any(feature = "http1", feature = "http2"))]
Kind::Parse(Parse::TooLarge) => "message head is too large",
Kind::Parse(Parse::Status) => "invalid HTTP status-code parsed",
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Kind::Parse(Parse::Internal) => {
"internal error inside Hyper and/or its dependencies, please report"
}
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Kind::IncompleteMessage => "connection closed before message completed",
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Kind::UnexpectedMessage => "received unexpected message from connection",
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
all(feature = "http2", feature = "client")
))]
Kind::ChannelClosed => "channel closed",
Kind::Canceled => "operation was canceled",
#[cfg(all(feature = "http1", feature = "server"))]
Kind::HeaderTimeout => "read header from client timeout",
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
Kind::Body => "error reading a body from connection",
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
Kind::BodyWrite => "error writing a body to connection",
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Kind::Shutdown => "error shutting down connection",
#[cfg(all(any(feature = "client", feature = "server"), feature = "http2"))]
Kind::Http2 => "http2 error",
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
Kind::Io => "connection error",
#[cfg(all(
any(feature = "client", feature = "server"),
any(feature = "http1", feature = "http2")
))]
Kind::User(User::Body) => "error from user's Body stream",
#[cfg(any(
all(feature = "http1", any(feature = "client", feature = "server")),
feature = "ffi"
))]
Kind::User(User::BodyWriteAborted) => "user body write aborted",
#[cfg(all(feature = "client", feature = "http2"))]
Kind::User(User::InvalidConnectWithBody) => {
"user sent CONNECT request with non-zero body"
}
#[cfg(any(
all(any(feature = "client", feature = "server"), feature = "http1"),
all(feature = "server", feature = "http2")
))]
Kind::User(User::Service) => "error from user's Service",
#[cfg(any(feature = "http1", feature = "http2"))]
#[cfg(feature = "server")]
Kind::User(User::UnexpectedHeader) => "user sent unexpected header",
#[cfg(feature = "http1")]
#[cfg(feature = "server")]
Kind::User(User::UnsupportedStatusCode) => {
"response has 1xx status code, not supported by server"
}
Kind::User(User::NoUpgrade) => "no upgrade available",
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
Kind::User(User::ManualUpgrade) => "upgrade expected but low level API in use",
#[cfg(all(feature = "client", any(feature = "http1", feature = "http2")))]
Kind::User(User::DispatchGone) => "dispatch task is gone",
#[cfg(feature = "ffi")]
Kind::User(User::AbortedByCallback) => "operation aborted by an application callback",
}
}
}
impl fmt::Debug for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
let mut f = f.debug_tuple("hyper::Error");
f.field(&self.inner.kind);
if let Some(cause) = &self.inner.cause {
f.field(cause);
}
f.finish()
}
}
impl fmt::Display for Error {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str(self.description())
}
}
impl StdError for Error {
fn source(&self) -> Option<&(dyn StdError + 'static)> {
self.inner
.cause
.as_ref()
.map(|cause| &**cause as &(dyn StdError + 'static))
}
}
#[doc(hidden)]
impl From<Parse> for Error {
fn from(err: Parse) -> Error {
Error::new(Kind::Parse(err))
}
}
#[cfg(feature = "http1")]
impl Parse {
#[cfg(any(feature = "client", feature = "server"))]
pub(crate) fn content_length_invalid() -> Self {
Parse::Header(Header::ContentLengthInvalid)
}
#[cfg(feature = "server")]
pub(crate) fn transfer_encoding_invalid() -> Self {
Parse::Header(Header::TransferEncodingInvalid)
}
#[cfg(any(feature = "client", feature = "server"))]
pub(crate) fn transfer_encoding_unexpected() -> Self {
Parse::Header(Header::TransferEncodingUnexpected)
}
}
#[cfg(feature = "http1")]
impl From<httparse::Error> for Parse {
fn from(err: httparse::Error) -> Parse {
match err {
httparse::Error::HeaderName
| httparse::Error::HeaderValue
| httparse::Error::NewLine
| httparse::Error::Token => Parse::Header(Header::Token),
httparse::Error::Status => Parse::Status,
httparse::Error::TooManyHeaders => Parse::TooLarge,
httparse::Error::Version => Parse::Version,
}
}
}
impl From<http::method::InvalidMethod> for Parse {
fn from(_: http::method::InvalidMethod) -> Parse {
Parse::Method
}
}
impl From<http::status::InvalidStatusCode> for Parse {
fn from(_: http::status::InvalidStatusCode) -> Parse {
Parse::Status
}
}
impl From<http::uri::InvalidUri> for Parse {
fn from(_: http::uri::InvalidUri) -> Parse {
Parse::Uri
}
}
impl From<http::uri::InvalidUriParts> for Parse {
fn from(_: http::uri::InvalidUriParts) -> Parse {
Parse::Uri
}
}
// ===== impl TimedOut ====
impl fmt::Display for TimedOut {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.write_str("operation timed out")
}
}
impl StdError for TimedOut {}
#[cfg(test)]
mod tests {
use super::*;
use std::mem;
fn assert_send_sync<T: Send + Sync + 'static>() {}
#[test]
fn error_satisfies_send_sync() {
assert_send_sync::<Error>()
}
#[test]
fn error_size_of() {
assert_eq!(mem::size_of::<Error>(), mem::size_of::<usize>());
}
#[cfg(feature = "http2")]
#[test]
fn h2_reason_unknown() {
let closed = Error::new_closed();
assert_eq!(closed.h2_reason(), h2::Reason::INTERNAL_ERROR);
}
#[cfg(feature = "http2")]
#[test]
fn h2_reason_one_level() {
let body_err = Error::new_user_body(h2::Error::from(h2::Reason::ENHANCE_YOUR_CALM));
assert_eq!(body_err.h2_reason(), h2::Reason::ENHANCE_YOUR_CALM);
}
#[cfg(feature = "http2")]
#[test]
fn h2_reason_nested() {
let recvd = Error::new_h2(h2::Error::from(h2::Reason::HTTP_1_1_REQUIRED));
// Suppose a user were proxying the received error
let svc_err = Error::new_user_service(recvd);
assert_eq!(svc_err.h2_reason(), h2::Reason::HTTP_1_1_REQUIRED);
}
}
-226
View File
@@ -1,226 +0,0 @@
use bytes::Bytes;
/// A reason phrase in an HTTP/1 response.
///
/// # Clients
///
/// For clients, a `ReasonPhrase` will be present in the extensions of the `http::Response` returned
/// for a request if the reason phrase is different from the canonical reason phrase for the
/// response's status code. For example, if a server returns `HTTP/1.1 200 Awesome`, the
/// `ReasonPhrase` will be present and contain `Awesome`, but if a server returns `HTTP/1.1 200 OK`,
/// the response will not contain a `ReasonPhrase`.
///
/// ```no_run
/// # #[cfg(all(feature = "tcp", feature = "client", feature = "http1"))]
/// # async fn fake_fetch() -> hyper::Result<()> {
/// use hyper::{Client, Uri};
/// use hyper::ext::ReasonPhrase;
///
/// let res = Client::new().get(Uri::from_static("http://example.com/non_canonical_reason")).await?;
///
/// // Print out the non-canonical reason phrase, if it has one...
/// if let Some(reason) = res.extensions().get::<ReasonPhrase>() {
/// println!("non-canonical reason: {}", std::str::from_utf8(reason.as_bytes()).unwrap());
/// }
/// # Ok(())
/// # }
/// ```
///
/// # Servers
///
/// When a `ReasonPhrase` is present in the extensions of the `http::Response` written by a server,
/// its contents will be written in place of the canonical reason phrase when responding via HTTP/1.
#[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub struct ReasonPhrase(Bytes);
impl ReasonPhrase {
/// Gets the reason phrase as bytes.
pub fn as_bytes(&self) -> &[u8] {
&self.0
}
/// Converts a static byte slice to a reason phrase.
///
/// # Panics
///
/// This method will panic if 'reason' contains an invalid byte. In a const context,
/// this is a compile-time error instead.
pub const fn from_static(reason: &'static [u8]) -> Self {
assert!(
find_invalid_byte(reason).is_none(),
"invalid byte in static reason phrase"
);
Self(Bytes::from_static(reason))
}
// Not public on purpose.
/// Converts a `Bytes` directly into a `ReasonPhrase` without validating.
///
/// Use with care; invalid bytes in a reason phrase can cause serious security problems if
/// emitted in a response.
#[cfg(feature = "client")]
pub(crate) fn from_bytes_unchecked(reason: Bytes) -> Self {
Self(reason)
}
}
impl TryFrom<&[u8]> for ReasonPhrase {
type Error = InvalidReasonPhrase;
fn try_from(reason: &[u8]) -> Result<Self, Self::Error> {
if let Some(bad_byte) = find_invalid_byte(reason) {
Err(InvalidReasonPhrase { bad_byte })
} else {
Ok(Self(Bytes::copy_from_slice(reason)))
}
}
}
impl TryFrom<Vec<u8>> for ReasonPhrase {
type Error = InvalidReasonPhrase;
fn try_from(reason: Vec<u8>) -> Result<Self, Self::Error> {
if let Some(bad_byte) = find_invalid_byte(&reason) {
Err(InvalidReasonPhrase { bad_byte })
} else {
Ok(Self(Bytes::from(reason)))
}
}
}
impl TryFrom<String> for ReasonPhrase {
type Error = InvalidReasonPhrase;
fn try_from(reason: String) -> Result<Self, Self::Error> {
if let Some(bad_byte) = find_invalid_byte(reason.as_bytes()) {
Err(InvalidReasonPhrase { bad_byte })
} else {
Ok(Self(Bytes::from(reason)))
}
}
}
impl TryFrom<Bytes> for ReasonPhrase {
type Error = InvalidReasonPhrase;
fn try_from(reason: Bytes) -> Result<Self, Self::Error> {
if let Some(bad_byte) = find_invalid_byte(&reason) {
Err(InvalidReasonPhrase { bad_byte })
} else {
Ok(Self(reason))
}
}
}
impl From<ReasonPhrase> for Bytes {
fn from(reason: ReasonPhrase) -> Self {
reason.0
}
}
impl AsRef<[u8]> for ReasonPhrase {
fn as_ref(&self) -> &[u8] {
&self.0
}
}
/// Error indicating an invalid byte when constructing a `ReasonPhrase`.
///
/// See [the spec][spec] for details on allowed bytes.
///
/// [spec]: https://httpwg.org/http-core/draft-ietf-httpbis-messaging-latest.html#rfc.section.4.p.7
#[derive(Debug)]
pub struct InvalidReasonPhrase {
bad_byte: u8,
}
impl std::fmt::Display for InvalidReasonPhrase {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(f, "Invalid byte in reason phrase: {}", self.bad_byte)
}
}
impl std::error::Error for InvalidReasonPhrase {}
const fn is_valid_byte(b: u8) -> bool {
// See https://www.rfc-editor.org/rfc/rfc5234.html#appendix-B.1
const fn is_vchar(b: u8) -> bool {
0x21 <= b && b <= 0x7E
}
// See https://httpwg.org/http-core/draft-ietf-httpbis-semantics-latest.html#fields.values
//
// The 0xFF comparison is technically redundant, but it matches the text of the spec more
// clearly and will be optimized away.
#[allow(unused_comparisons, clippy::absurd_extreme_comparisons)]
const fn is_obs_text(b: u8) -> bool {
0x80 <= b && b <= 0xFF
}
// See https://httpwg.org/http-core/draft-ietf-httpbis-messaging-latest.html#rfc.section.4.p.7
b == b'\t' || b == b' ' || is_vchar(b) || is_obs_text(b)
}
const fn find_invalid_byte(bytes: &[u8]) -> Option<u8> {
let mut i = 0;
while i < bytes.len() {
let b = bytes[i];
if !is_valid_byte(b) {
return Some(b);
}
i += 1;
}
None
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn basic_valid() {
const PHRASE: &[u8] = b"OK";
assert_eq!(ReasonPhrase::from_static(PHRASE).as_bytes(), PHRASE);
assert_eq!(ReasonPhrase::try_from(PHRASE).unwrap().as_bytes(), PHRASE);
}
#[test]
fn empty_valid() {
const PHRASE: &[u8] = b"";
assert_eq!(ReasonPhrase::from_static(PHRASE).as_bytes(), PHRASE);
assert_eq!(ReasonPhrase::try_from(PHRASE).unwrap().as_bytes(), PHRASE);
}
#[test]
fn obs_text_valid() {
const PHRASE: &[u8] = b"hyp\xe9r";
assert_eq!(ReasonPhrase::from_static(PHRASE).as_bytes(), PHRASE);
assert_eq!(ReasonPhrase::try_from(PHRASE).unwrap().as_bytes(), PHRASE);
}
const NEWLINE_PHRASE: &[u8] = b"hyp\ner";
#[test]
#[should_panic]
fn newline_invalid_panic() {
ReasonPhrase::from_static(NEWLINE_PHRASE);
}
#[test]
fn newline_invalid_err() {
assert!(ReasonPhrase::try_from(NEWLINE_PHRASE).is_err());
}
const CR_PHRASE: &[u8] = b"hyp\rer";
#[test]
#[should_panic]
fn cr_invalid_panic() {
ReasonPhrase::from_static(CR_PHRASE);
}
#[test]
fn cr_invalid_err() {
assert!(ReasonPhrase::try_from(CR_PHRASE).is_err());
}
}
-86
View File
@@ -1,86 +0,0 @@
use std::sync::Arc;
#[derive(Clone)]
pub(crate) struct OnInformational(Arc<dyn OnInformationalCallback + Send + Sync>);
/// Add a callback for 1xx informational responses.
///
/// # Example
///
/// ```
/// # let some_body = ();
/// let mut req = hyper::Request::new(some_body);
///
/// hyper::ext::on_informational(&mut req, |res| {
/// println!("informational: {:?}", res.status());
/// });
///
/// // send request on a client connection...
/// ```
pub fn on_informational<B, F>(req: &mut http::Request<B>, callback: F)
where
F: Fn(Response<'_>) + Send + Sync + 'static,
{
on_informational_raw(req, OnInformationalClosure(callback));
}
pub(crate) fn on_informational_raw<B, C>(req: &mut http::Request<B>, callback: C)
where
C: OnInformationalCallback + Send + Sync + 'static,
{
req.extensions_mut()
.insert(OnInformational(Arc::new(callback)));
}
// Sealed, not actually nameable bounds
pub(crate) trait OnInformationalCallback {
fn on_informational(&self, res: http::Response<()>);
}
impl OnInformational {
pub(crate) fn call(&self, res: http::Response<()>) {
self.0.on_informational(res);
}
}
struct OnInformationalClosure<F>(F);
impl<F> OnInformationalCallback for OnInformationalClosure<F>
where
F: Fn(Response<'_>) + Send + Sync + 'static,
{
fn on_informational(&self, res: http::Response<()>) {
let res = Response(&res);
(self.0)(res);
}
}
// A facade over http::Response.
//
// It purposefully hides being able to move the response out of the closure,
// while also not being able to expect it to be a reference `&Response`.
// (Otherwise, a closure can be written as `|res: &_|`, and then be broken if
// we make the closure take ownership.)
//
// With the type not being nameable, we could change from being a facade to
// being either a real reference, or moving the http::Response into the closure,
// in a backwards-compatible change in the future.
#[derive(Debug)]
pub struct Response<'resp>(&'resp http::Response<()>);
impl Response<'_> {
#[inline]
pub fn status(&self) -> http::StatusCode {
self.0.status()
}
#[inline]
pub fn version(&self) -> http::Version {
self.0.version()
}
#[inline]
pub fn headers(&self) -> &http::HeaderMap {
self.0.headers()
}
}
-263
View File
@@ -1,263 +0,0 @@
//! Extensions for HTTP messages in Hyper.
//!
//! This module provides types and utilities that extend the capabilities of HTTP requests and responses
//! in Hyper. Extensions are additional pieces of information or features that can be attached to HTTP
//! messages via the [`http::Extensions`] map, which is
//! accessible through methods like [`http::Request::extensions`] and [`http::Response::extensions`].
//!
//! # What are extensions?
//!
//! Extensions allow Hyper to associate extra metadata or behaviors with HTTP messages, beyond the standard
//! headers and body. These can be used by advanced users and library authors to access protocol-specific
//! features, track original header casing, handle informational responses, and more.
//!
//! # How to access extensions
//!
//! Extensions are stored in the `Extensions` map of a request or response. You can access them using:
//!
//! ```rust
//! # let response = http::Response::new(());
//! if let Some(ext) = response.extensions().get::<hyper::ext::ReasonPhrase>() {
//! // use the extension
//! }
//! ```
//!
//! # Extension Groups
//!
//! The extensions in this module can be grouped as follows:
//!
//! - **HTTP/1 Reason Phrase**: [`ReasonPhrase`] — Access non-canonical reason phrases in HTTP/1 responses.
//! - **Informational Responses**: [`on_informational`] — Register callbacks for 1xx HTTP/1 responses on the client.
//! - **Header Case Tracking**: Internal types for tracking the original casing and order of headers as received.
//! - **HTTP/2 Protocol Extensions**: [`Protocol`] — Access the `:protocol` pseudo-header for Extended CONNECT in HTTP/2.
//!
//! Some extensions are only available for specific protocols (HTTP/1 or HTTP/2) or use cases (client, server, FFI).
//!
//! See the documentation on each item for details about its usage and requirements.
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
use bytes::Bytes;
#[cfg(any(
all(any(feature = "client", feature = "server"), feature = "http1"),
feature = "ffi"
))]
use http::header::HeaderName;
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
use http::header::{HeaderMap, IntoHeaderName, ValueIter};
#[cfg(feature = "ffi")]
use std::collections::HashMap;
#[cfg(feature = "http2")]
use std::fmt;
#[cfg(any(feature = "http1", feature = "ffi"))]
mod h1_reason_phrase;
#[cfg(any(feature = "http1", feature = "ffi"))]
pub use h1_reason_phrase::ReasonPhrase;
#[cfg(all(feature = "http1", feature = "client"))]
mod informational;
#[cfg(all(feature = "http1", feature = "client"))]
pub use informational::on_informational;
#[cfg(all(feature = "http1", feature = "client"))]
pub(crate) use informational::OnInformational;
#[cfg(all(feature = "http1", feature = "client", feature = "ffi"))]
pub(crate) use informational::{on_informational_raw, OnInformationalCallback};
#[cfg(feature = "http2")]
/// Extension type representing the `:protocol` pseudo-header in HTTP/2.
///
/// The `Protocol` extension allows access to the value of the `:protocol` pseudo-header
/// used by the [Extended CONNECT Protocol](https://datatracker.ietf.org/doc/html/rfc8441#section-4).
/// This extension is only sent on HTTP/2 CONNECT requests, most commonly with the value `websocket`.
///
/// # Example
///
/// ```rust
/// use hyper::ext::Protocol;
/// use http::{Request, Method, Version};
///
/// let mut req = Request::new(());
/// *req.method_mut() = Method::CONNECT;
/// *req.version_mut() = Version::HTTP_2;
/// req.extensions_mut().insert(Protocol::from_static("websocket"));
/// // Now the request will include the `:protocol` pseudo-header with value "websocket"
/// ```
#[derive(Clone, Eq, PartialEq)]
pub struct Protocol {
inner: h2::ext::Protocol,
}
#[cfg(feature = "http2")]
impl Protocol {
/// Converts a static string to a protocol name.
pub const fn from_static(value: &'static str) -> Self {
Self {
inner: h2::ext::Protocol::from_static(value),
}
}
/// Returns a str representation of the header.
pub fn as_str(&self) -> &str {
self.inner.as_str()
}
#[cfg(feature = "server")]
pub(crate) fn from_inner(inner: h2::ext::Protocol) -> Self {
Self { inner }
}
#[cfg(all(feature = "client", feature = "http2"))]
pub(crate) fn into_inner(self) -> h2::ext::Protocol {
self.inner
}
}
#[cfg(feature = "http2")]
impl<'proto> From<&'proto str> for Protocol {
fn from(value: &'proto str) -> Self {
Self {
inner: h2::ext::Protocol::from(value),
}
}
}
#[cfg(feature = "http2")]
impl AsRef<[u8]> for Protocol {
fn as_ref(&self) -> &[u8] {
self.inner.as_ref()
}
}
#[cfg(feature = "http2")]
impl fmt::Debug for Protocol {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
self.inner.fmt(f)
}
}
/// A map from header names to their original casing as received in an HTTP message.
///
/// If an HTTP/1 response `res` is parsed on a connection whose option
/// [`preserve_header_case`] was set to true and the response included
/// the following headers:
///
/// ```ignore
/// x-Bread: Baguette
/// X-BREAD: Pain
/// x-bread: Ficelle
/// ```
///
/// Then `res.extensions().get::<HeaderCaseMap>()` will return a map with:
///
/// ```ignore
/// HeaderCaseMap({
/// "x-bread": ["x-Bread", "X-BREAD", "x-bread"],
/// })
/// ```
///
/// [`preserve_header_case`]: /client/struct.Client.html#method.preserve_header_case
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
#[derive(Clone, Debug)]
pub(crate) struct HeaderCaseMap(HeaderMap<Bytes>);
#[cfg(all(any(feature = "client", feature = "server"), feature = "http1"))]
impl HeaderCaseMap {
/// Returns a view of all spellings associated with that header name,
/// in the order they were found.
#[cfg(feature = "client")]
pub(crate) fn get_all<'hdr>(
&'hdr self,
name: &HeaderName,
) -> impl Iterator<Item = impl AsRef<[u8]> + 'hdr> + 'hdr {
self.get_all_internal(name)
}
/// Returns a view of all spellings associated with that header name,
/// in the order they were found.
#[cfg(any(feature = "client", feature = "server"))]
pub(crate) fn get_all_internal(&self, name: &HeaderName) -> ValueIter<'_, Bytes> {
self.0.get_all(name).into_iter()
}
#[cfg(any(feature = "client", feature = "server"))]
pub(crate) fn default() -> Self {
Self(HeaderMap::default())
}
#[cfg(any(test, feature = "ffi"))]
pub(crate) fn insert(&mut self, name: HeaderName, orig: Bytes) {
self.0.insert(name, orig);
}
#[cfg(any(feature = "client", feature = "server"))]
pub(crate) fn append<N>(&mut self, name: N, orig: Bytes)
where
N: IntoHeaderName,
{
self.0.append(name, orig);
}
}
#[cfg(feature = "ffi")]
#[derive(Clone, Debug)]
/// Hashmap<Headername, numheaders with that name>
pub(crate) struct OriginalHeaderOrder {
/// Stores how many entries a Headername maps to. This is used
/// for accounting.
num_entries: HashMap<HeaderName, usize>,
/// Stores the ordering of the headers. ex: `vec[i] = (headerName, idx)`,
/// The vector is ordered such that the ith element
/// represents the ith header that came in off the line.
/// The `HeaderName` and `idx` are then used elsewhere to index into
/// the multi map that stores the header values.
entry_order: Vec<(HeaderName, usize)>,
}
#[cfg(all(feature = "http1", feature = "ffi"))]
impl OriginalHeaderOrder {
pub(crate) fn default() -> Self {
OriginalHeaderOrder {
num_entries: HashMap::new(),
entry_order: Vec::new(),
}
}
pub(crate) fn insert(&mut self, name: HeaderName) {
if !self.num_entries.contains_key(&name) {
let idx = 0;
self.num_entries.insert(name.clone(), 1);
self.entry_order.push((name, idx));
}
// Replacing an already existing element does not
// change ordering, so we only care if its the first
// header name encountered
}
pub(crate) fn append<N>(&mut self, name: N)
where
N: IntoHeaderName + Into<HeaderName> + Clone,
{
let name: HeaderName = name.into();
let idx = match self.num_entries.entry(name.clone()) {
std::collections::hash_map::Entry::Occupied(mut entry) => {
let idx = *entry.get();
*entry.get_mut() += 1;
idx
}
std::collections::hash_map::Entry::Vacant(entry) => {
entry.insert(1);
0
}
};
self.entry_order.push((name, idx));
}
// No doc test is run here because `RUSTFLAGS='--cfg hyper_unstable_ffi'`
// is needed to compile. Once ffi is stabilized `no_run` should be removed
// here.
/// This returns an iterator that provides header names and indexes
/// in the original order received.
pub(crate) fn get_in_order(&self) -> impl Iterator<Item = &(HeaderName, usize)> {
self.entry_order.iter()
}
}
-302
View File
@@ -1,302 +0,0 @@
use std::ffi::{c_int, c_void};
use std::mem::ManuallyDrop;
use std::ptr;
use std::task::{Context, Poll};
use http_body_util::BodyExt as _;
use super::task::{hyper_context, hyper_task, hyper_task_return_type, AsTaskType};
use super::{UserDataPointer, HYPER_ITER_CONTINUE};
use crate::body::{Bytes, Frame, Incoming as IncomingBody};
use crate::ffi::size_t;
/// A streaming HTTP body.
///
/// This is used both for sending requests (with `hyper_request_set_body`) and
/// for receiving responses (with `hyper_response_body`).
///
/// For outgoing request bodies, call `hyper_body_set_data_func` to provide the
/// data.
///
/// For incoming response bodies, call `hyper_body_data` to get a task that will
/// yield a chunk of data each time it is polled. That task must be then be
/// added to the executor with `hyper_executor_push`.
///
/// Methods:
///
/// - hyper_body_new: Create a new “empty” body.
/// - hyper_body_set_userdata: Set userdata on this body, which will be passed to callback functions.
/// - hyper_body_set_data_func: Set the data callback for this body.
/// - hyper_body_data: Creates a task that will poll a response body for the next buffer of data.
/// - hyper_body_foreach: Creates a task to execute the callback with each body chunk received.
/// - hyper_body_free: Free a body.
pub struct hyper_body(pub(super) IncomingBody);
/// A buffer of bytes that is sent or received on a `hyper_body`.
///
/// Obtain one of these in the callback of `hyper_body_foreach` or by receiving
/// a task of type `HYPER_TASK_BUF` from `hyper_executor_poll` (after calling
/// `hyper_body_data` and pushing the resulting task).
///
/// Methods:
///
/// - hyper_buf_bytes: Get a pointer to the bytes in this buffer.
/// - hyper_buf_copy: Create a new hyper_buf * by copying the provided bytes.
/// - hyper_buf_free: Free this buffer.
/// - hyper_buf_len: Get the length of the bytes this buffer contains.
pub struct hyper_buf(pub(crate) Bytes);
pub(crate) struct UserBody {
data_func: hyper_body_data_callback,
userdata: *mut c_void,
}
// ===== Body =====
type hyper_body_foreach_callback = extern "C" fn(*mut c_void, *const hyper_buf) -> c_int;
type hyper_body_data_callback =
extern "C" fn(*mut c_void, *mut hyper_context<'_>, *mut *mut hyper_buf) -> c_int;
ffi_fn! {
/// Creates a new "empty" body.
///
/// If not configured, this body acts as an empty payload.
///
/// To avoid a memory leak, the body must eventually be consumed by
/// `hyper_body_free`, `hyper_body_foreach`, or `hyper_request_set_body`.
fn hyper_body_new() -> *mut hyper_body {
Box::into_raw(Box::new(hyper_body(IncomingBody::ffi())))
} ?= ptr::null_mut()
}
ffi_fn! {
/// Free a body.
///
/// This should only be used if the request isn't consumed by
/// `hyper_body_foreach` or `hyper_request_set_body`.
fn hyper_body_free(body: *mut hyper_body) {
drop(non_null!(Box::from_raw(body) ?= ()));
}
}
ffi_fn! {
/// Creates a task that will poll a response body for the next buffer of data.
///
/// The task may have different types depending on the outcome:
///
/// - `HYPER_TASK_BUF`: Success, and more data was received.
/// - `HYPER_TASK_ERROR`: An error retrieving the data.
/// - `HYPER_TASK_EMPTY`: The body has finished streaming data.
///
/// When the application receives the task from `hyper_executor_poll`,
/// if the task type is `HYPER_TASK_BUF`, it should cast the task to
/// `hyper_buf *` and consume all the bytes in the buffer. Then
/// the application should call `hyper_body_data` again for the same
/// `hyper_body *`, to create a task for the next buffer of data.
/// Repeat until the polled task type is `HYPER_TASK_ERROR` or
/// `HYPER_TASK_EMPTY`.
///
/// To avoid a memory leak, the task must eventually be consumed by
/// `hyper_task_free`, or taken ownership of by `hyper_executor_push`
/// without subsequently being given back by `hyper_executor_poll`.
///
/// This does not consume the `hyper_body *`, so it may be used again.
/// However, the `hyper_body *` MUST NOT be used or freed until the
/// related task is returned from `hyper_executor_poll`.
///
/// For a more convenient method, see also `hyper_body_foreach`.
fn hyper_body_data(body: *mut hyper_body) -> *mut hyper_task {
// This doesn't take ownership of the Body, so don't allow destructor
let mut body = ManuallyDrop::new(non_null!(Box::from_raw(body) ?= ptr::null_mut()));
Box::into_raw(hyper_task::boxed(async move {
loop {
match body.0.frame().await {
Some(Ok(frame)) => {
if let Ok(data) = frame.into_data() {
return Ok(Some(hyper_buf(data)));
} else {
continue;
}
},
Some(Err(e)) => return Err(e),
None => return Ok(None),
}
}
}))
} ?= ptr::null_mut()
}
ffi_fn! {
/// Creates a task to execute the callback with each body chunk received.
///
/// To avoid a memory leak, the task must eventually be consumed by
/// `hyper_task_free`, or taken ownership of by `hyper_executor_push`
/// without subsequently being given back by `hyper_executor_poll`.
///
/// The `hyper_buf` pointer is only a borrowed reference. It cannot live outside
/// the execution of the callback. You must make a copy of the bytes to retain them.
///
/// The callback should return `HYPER_ITER_CONTINUE` to continue iterating
/// chunks as they are received, or `HYPER_ITER_BREAK` to cancel. Each
/// invocation of the callback must consume all the bytes it is provided.
/// There is no mechanism to signal to Hyper that only a subset of bytes were
/// consumed.
///
/// This will consume the `hyper_body *`, you shouldn't use it anymore or free it.
fn hyper_body_foreach(body: *mut hyper_body, func: hyper_body_foreach_callback, userdata: *mut c_void) -> *mut hyper_task {
let mut body = non_null!(Box::from_raw(body) ?= ptr::null_mut());
let userdata = UserDataPointer(userdata);
Box::into_raw(hyper_task::boxed(async move {
let _ = &userdata;
while let Some(item) = body.0.frame().await {
let frame = item?;
if let Ok(chunk) = frame.into_data() {
if HYPER_ITER_CONTINUE != func(userdata.0, &hyper_buf(chunk)) {
return Err(crate::Error::new_user_aborted_by_callback());
}
}
}
Ok(())
}))
} ?= ptr::null_mut()
}
ffi_fn! {
/// Set userdata on this body, which will be passed to callback functions.
fn hyper_body_set_userdata(body: *mut hyper_body, userdata: *mut c_void) {
let b = non_null!(&mut *body ?= ());
b.0.as_ffi_mut().userdata = userdata;
}
}
ffi_fn! {
/// Set the outgoing data callback for this body.
///
/// The callback is called each time hyper needs to send more data for the
/// body. It is passed the value from `hyper_body_set_userdata`.
///
/// If there is data available, the `hyper_buf **` argument should be set
/// to a `hyper_buf *` containing the data, and `HYPER_POLL_READY` should
/// be returned.
///
/// Returning `HYPER_POLL_READY` while the `hyper_buf **` argument points
/// to `NULL` will indicate the body has completed all data.
///
/// If there is more data to send, but it isn't yet available, a
/// `hyper_waker` should be saved from the `hyper_context *` argument, and
/// `HYPER_POLL_PENDING` should be returned. You must wake the saved waker
/// to signal the task when data is available.
///
/// If some error has occurred, you can return `HYPER_POLL_ERROR` to abort
/// the body.
fn hyper_body_set_data_func(body: *mut hyper_body, func: hyper_body_data_callback) {
let b = non_null!{ &mut *body ?= () };
b.0.as_ffi_mut().data_func = func;
}
}
// ===== impl UserBody =====
impl UserBody {
pub(crate) fn new() -> UserBody {
UserBody {
data_func: data_noop,
userdata: std::ptr::null_mut(),
}
}
pub(crate) fn poll_data(
&mut self,
cx: &mut Context<'_>,
) -> Poll<Option<crate::Result<Frame<Bytes>>>> {
let mut out = std::ptr::null_mut();
match (self.data_func)(self.userdata, hyper_context::wrap(cx), &mut out) {
super::task::HYPER_POLL_READY => {
if out.is_null() {
Poll::Ready(None)
} else {
let buf = unsafe { Box::from_raw(out) };
Poll::Ready(Some(Ok(Frame::data(buf.0))))
}
}
super::task::HYPER_POLL_PENDING => Poll::Pending,
super::task::HYPER_POLL_ERROR => {
Poll::Ready(Some(Err(crate::Error::new_body_write_aborted())))
}
unexpected => Poll::Ready(Some(Err(crate::Error::new_body_write(format!(
"unexpected hyper_body_data_func return code {}",
unexpected
))))),
}
}
}
/// cbindgen:ignore
extern "C" fn data_noop(
_userdata: *mut c_void,
_: *mut hyper_context<'_>,
_: *mut *mut hyper_buf,
) -> c_int {
super::task::HYPER_POLL_READY
}
unsafe impl Send for UserBody {}
unsafe impl Sync for UserBody {}
// ===== Bytes =====
ffi_fn! {
/// Create a new `hyper_buf *` by copying the provided bytes.
///
/// This makes an owned copy of the bytes, so the `buf` argument can be
/// freed (with `hyper_buf_free`) or changed afterwards.
///
/// To avoid a memory leak, the copy must eventually be consumed by
/// `hyper_buf_free`.
///
/// This returns `NULL` if allocating a new buffer fails.
fn hyper_buf_copy(buf: *const u8, len: size_t) -> *mut hyper_buf {
let slice = unsafe {
std::slice::from_raw_parts(buf, len)
};
Box::into_raw(Box::new(hyper_buf(Bytes::copy_from_slice(slice))))
} ?= ptr::null_mut()
}
ffi_fn! {
/// Get a pointer to the bytes in this buffer.
///
/// This should be used in conjunction with `hyper_buf_len` to get the length
/// of the bytes data.
///
/// This pointer is borrowed data, and not valid once the `hyper_buf` is
/// consumed/freed.
fn hyper_buf_bytes(buf: *const hyper_buf) -> *const u8 {
unsafe { (*buf).0.as_ptr() }
} ?= ptr::null()
}
ffi_fn! {
/// Get the length of the bytes this buffer contains.
fn hyper_buf_len(buf: *const hyper_buf) -> size_t {
unsafe { (*buf).0.len() }
}
}
ffi_fn! {
/// Free this buffer.
///
/// This should be used for any buffer once it is no longer needed.
fn hyper_buf_free(buf: *mut hyper_buf) {
drop(unsafe { Box::from_raw(buf) });
}
}
unsafe impl AsTaskType for hyper_buf {
fn as_task_type(&self) -> hyper_task_return_type {
hyper_task_return_type::HYPER_TASK_BUF
}
}
-274
View File
@@ -1,274 +0,0 @@
use std::ffi::c_int;
use std::ptr;
use std::sync::Arc;
use crate::client::conn;
use crate::rt::Executor as _;
use super::error::hyper_code;
use super::http_types::{hyper_request, hyper_response};
use super::io::hyper_io;
use super::task::{hyper_executor, hyper_task, hyper_task_return_type, AsTaskType, WeakExec};
/// An options builder to configure an HTTP client connection.
///
/// Methods:
///
/// - hyper_clientconn_options_new: Creates a new set of HTTP clientconn options to be used in a handshake.
/// - hyper_clientconn_options_exec: Set the client background task executor.
/// - hyper_clientconn_options_http2: Set whether to use HTTP2.
/// - hyper_clientconn_options_set_preserve_header_case: Set whether header case is preserved.
/// - hyper_clientconn_options_set_preserve_header_order: Set whether header order is preserved.
/// - hyper_clientconn_options_http1_allow_multiline_headers: Set whether HTTP/1 connections accept obsolete line folding for header values.
/// - hyper_clientconn_options_free: Free a set of HTTP clientconn options.
pub struct hyper_clientconn_options {
http1_allow_obsolete_multiline_headers_in_responses: bool,
http1_preserve_header_case: bool,
http1_preserve_header_order: bool,
http2: bool,
/// Use a `Weak` to prevent cycles.
exec: WeakExec,
}
/// An HTTP client connection handle.
///
/// These are used to send one or more requests on a single connection.
///
/// It's possible to send multiple requests on a single connection, such
/// as when HTTP/1 keep-alive or HTTP/2 is used.
///
/// To create a `hyper_clientconn`:
///
/// 1. Create a `hyper_io` with `hyper_io_new`.
/// 2. Create a `hyper_clientconn_options` with `hyper_clientconn_options_new`.
/// 3. Call `hyper_clientconn_handshake` with the `hyper_io` and `hyper_clientconn_options`.
/// This creates a `hyper_task`.
/// 5. Call `hyper_task_set_userdata` to assign an application-specific pointer to the task.
/// This allows keeping track of multiple connections that may be handshaking
/// simultaneously.
/// 4. Add the `hyper_task` to an executor with `hyper_executor_push`.
/// 5. Poll that executor until it yields a task of type `HYPER_TASK_CLIENTCONN`.
/// 6. Extract the `hyper_clientconn` from the task with `hyper_task_value`.
/// This will require a cast from `void *` to `hyper_clientconn *`.
///
/// This process results in a `hyper_clientconn` that permanently owns the
/// `hyper_io`. Because the `hyper_io` in turn owns a TCP or TLS connection, that means
/// the `hyper_clientconn` owns the connection for both the clientconn's lifetime
/// and the connection's lifetime.
///
/// In other words, each connection (`hyper_io`) must have exactly one `hyper_clientconn`
/// associated with it. That's because `hyper_clientconn_handshake` sends the
/// [HTTP/2 Connection Preface] (for HTTP/2 connections). Since that preface can't
/// be sent twice, handshake can't be called twice.
///
/// [HTTP/2 Connection Preface]: https://datatracker.ietf.org/doc/html/rfc9113#name-http-2-connection-preface
///
/// Methods:
///
/// - hyper_clientconn_handshake: Creates an HTTP client handshake task.
/// - hyper_clientconn_send: Creates a task to send a request on the client connection.
/// - hyper_clientconn_free: Free a hyper_clientconn *.
pub struct hyper_clientconn {
tx: Tx,
}
enum Tx {
#[cfg(feature = "http1")]
Http1(conn::http1::SendRequest<crate::body::Incoming>),
#[cfg(feature = "http2")]
Http2(conn::http2::SendRequest<crate::body::Incoming>),
}
// ===== impl hyper_clientconn =====
ffi_fn! {
/// Creates an HTTP client handshake task.
///
/// Both the `io` and the `options` are consumed in this function call.
/// They should not be used or freed afterwards.
///
/// The returned task must be polled with an executor until the handshake
/// completes, at which point the value can be taken.
///
/// To avoid a memory leak, the task must eventually be consumed by
/// `hyper_task_free`, or taken ownership of by `hyper_executor_push`
/// without subsequently being given back by `hyper_executor_poll`.
fn hyper_clientconn_handshake(io: *mut hyper_io, options: *mut hyper_clientconn_options) -> *mut hyper_task {
let options = non_null! { Box::from_raw(options) ?= ptr::null_mut() };
let io = non_null! { Box::from_raw(io) ?= ptr::null_mut() };
Box::into_raw(hyper_task::boxed(async move {
#[cfg(feature = "http2")]
{
if options.http2 {
return conn::http2::Builder::new(options.exec.clone())
.handshake::<_, crate::body::Incoming>(io)
.await
.map(|(tx, conn)| {
options.exec.execute(Box::pin(async move {
let _ = conn.await;
}));
hyper_clientconn { tx: Tx::Http2(tx) }
});
}
}
conn::http1::Builder::new()
.allow_obsolete_multiline_headers_in_responses(options.http1_allow_obsolete_multiline_headers_in_responses)
.preserve_header_case(options.http1_preserve_header_case)
.preserve_header_order(options.http1_preserve_header_order)
.handshake::<_, crate::body::Incoming>(io)
.await
.map(|(tx, conn)| {
options.exec.execute(Box::pin(async move {
let _ = conn.await;
}));
hyper_clientconn { tx: Tx::Http1(tx) }
})
}))
} ?= std::ptr::null_mut()
}
ffi_fn! {
/// Creates a task to send a request on the client connection.
///
/// This consumes the request. You should not use or free the request
/// afterwards.
///
/// Returns a task that needs to be polled until it is ready. When ready, the
/// task yields a `hyper_response *`.
///
/// To avoid a memory leak, the task must eventually be consumed by
/// `hyper_task_free`, or taken ownership of by `hyper_executor_push`
/// without subsequently being given back by `hyper_executor_poll`.
fn hyper_clientconn_send(conn: *mut hyper_clientconn, req: *mut hyper_request) -> *mut hyper_task {
let mut req = non_null! { Box::from_raw(req) ?= ptr::null_mut() };
// Update request with original-case map of headers
req.finalize_request();
let fut = match &mut non_null! { &mut *conn ?= ptr::null_mut() }.tx {
Tx::Http1(tx) => futures_util::future::Either::Left(tx.send_request(req.0)),
Tx::Http2(tx) => futures_util::future::Either::Right(tx.send_request(req.0)),
};
let fut = async move {
fut.await.map(hyper_response::wrap)
};
Box::into_raw(hyper_task::boxed(fut))
} ?= std::ptr::null_mut()
}
ffi_fn! {
/// Free a `hyper_clientconn *`.
///
/// This should be used for any connection once it is no longer needed.
fn hyper_clientconn_free(conn: *mut hyper_clientconn) {
drop(non_null! { Box::from_raw(conn) ?= () });
}
}
unsafe impl AsTaskType for hyper_clientconn {
fn as_task_type(&self) -> hyper_task_return_type {
hyper_task_return_type::HYPER_TASK_CLIENTCONN
}
}
// ===== impl hyper_clientconn_options =====
ffi_fn! {
/// Creates a new set of HTTP clientconn options to be used in a handshake.
///
/// To avoid a memory leak, the options must eventually be consumed by
/// `hyper_clientconn_options_free` or `hyper_clientconn_handshake`.
fn hyper_clientconn_options_new() -> *mut hyper_clientconn_options {
Box::into_raw(Box::new(hyper_clientconn_options {
http1_allow_obsolete_multiline_headers_in_responses: false,
http1_preserve_header_case: false,
http1_preserve_header_order: false,
http2: false,
exec: WeakExec::new(),
}))
} ?= std::ptr::null_mut()
}
ffi_fn! {
/// Set whether header case is preserved.
///
/// Pass `0` to allow lowercase normalization (default), `1` to retain original case.
fn hyper_clientconn_options_set_preserve_header_case(opts: *mut hyper_clientconn_options, enabled: c_int) {
let opts = non_null! { &mut *opts ?= () };
opts.http1_preserve_header_case = enabled != 0;
}
}
ffi_fn! {
/// Set whether header order is preserved.
///
/// Pass `0` to allow reordering (default), `1` to retain original ordering.
fn hyper_clientconn_options_set_preserve_header_order(opts: *mut hyper_clientconn_options, enabled: c_int) {
let opts = non_null! { &mut *opts ?= () };
opts.http1_preserve_header_order = enabled != 0;
}
}
ffi_fn! {
/// Free a set of HTTP clientconn options.
///
/// This should only be used if the options aren't consumed by
/// `hyper_clientconn_handshake`.
fn hyper_clientconn_options_free(opts: *mut hyper_clientconn_options) {
drop(non_null! { Box::from_raw(opts) ?= () });
}
}
ffi_fn! {
/// Set the client background task executor.
///
/// This does not consume the `options` or the `exec`.
fn hyper_clientconn_options_exec(opts: *mut hyper_clientconn_options, exec: *const hyper_executor) {
let opts = non_null! { &mut *opts ?= () };
let exec = non_null! { Arc::from_raw(exec) ?= () };
let weak_exec = hyper_executor::downgrade(&exec);
std::mem::forget(exec);
opts.exec = weak_exec;
}
}
ffi_fn! {
/// Set whether to use HTTP2.
///
/// Pass `0` to disable, `1` to enable.
fn hyper_clientconn_options_http2(opts: *mut hyper_clientconn_options, enabled: c_int) -> hyper_code {
#[cfg(feature = "http2")]
{
let opts = non_null! { &mut *opts ?= hyper_code::HYPERE_INVALID_ARG };
opts.http2 = enabled != 0;
hyper_code::HYPERE_OK
}
#[cfg(not(feature = "http2"))]
{
drop(opts);
drop(enabled);
hyper_code::HYPERE_FEATURE_NOT_ENABLED
}
}
}
ffi_fn! {
/// Set whether HTTP/1 connections accept obsolete line folding for header values.
///
/// Newline codepoints (\r and \n) will be transformed to spaces when parsing.
///
/// Pass `0` to disable, `1` to enable.
///
fn hyper_clientconn_options_http1_allow_multiline_headers(opts: *mut hyper_clientconn_options, enabled: c_int) -> hyper_code {
let opts = non_null! { &mut *opts ?= hyper_code::HYPERE_INVALID_ARG };
opts.http1_allow_obsolete_multiline_headers_in_responses = enabled != 0;
hyper_code::HYPERE_OK
}
}
-96
View File
@@ -1,96 +0,0 @@
use crate::ffi::size_t;
/// A more detailed error object returned by some hyper functions.
///
/// Compare with `hyper_code`, which is a simpler error returned from
/// some hyper functions.
///
/// Methods:
///
/// - hyper_error_code: Get an equivalent hyper_code from this error.
/// - hyper_error_print: Print the details of this error to a buffer.
/// - hyper_error_free: Frees a hyper_error.
pub struct hyper_error(crate::Error);
/// A return code for many of hyper's methods.
#[repr(C)]
pub enum hyper_code {
/// All is well.
HYPERE_OK,
/// General error, details in the `hyper_error *`.
HYPERE_ERROR,
/// A function argument was invalid.
HYPERE_INVALID_ARG,
/// The IO transport returned an EOF when one wasn't expected.
///
/// This typically means an HTTP request or response was expected, but the
/// connection closed cleanly without sending (all of) it.
HYPERE_UNEXPECTED_EOF,
/// Aborted by a user supplied callback.
HYPERE_ABORTED_BY_CALLBACK,
/// An optional hyper feature was not enabled.
#[cfg_attr(feature = "http2", allow(unused))]
HYPERE_FEATURE_NOT_ENABLED,
/// The peer sent an HTTP message that could not be parsed.
HYPERE_INVALID_PEER_MESSAGE,
}
// ===== impl hyper_error =====
impl hyper_error {
fn code(&self) -> hyper_code {
use crate::error::Kind as ErrorKind;
use crate::error::User;
match self.0.kind() {
ErrorKind::Parse(_) => hyper_code::HYPERE_INVALID_PEER_MESSAGE,
ErrorKind::IncompleteMessage => hyper_code::HYPERE_UNEXPECTED_EOF,
ErrorKind::User(User::AbortedByCallback) => hyper_code::HYPERE_ABORTED_BY_CALLBACK,
// TODO: add more variants
_ => hyper_code::HYPERE_ERROR,
}
}
fn print_to(&self, dst: &mut [u8]) -> usize {
use std::io::Write;
let mut dst = std::io::Cursor::new(dst);
// A write! error doesn't matter. As much as possible will have been
// written, and the Cursor position will know how far that is (even
// if that is zero).
let _ = write!(dst, "{}", &self.0);
dst.position() as usize
}
}
ffi_fn! {
/// Frees a `hyper_error`.
///
/// This should be used for any error once it is no longer needed.
fn hyper_error_free(err: *mut hyper_error) {
drop(non_null!(Box::from_raw(err) ?= ()));
}
}
ffi_fn! {
/// Get an equivalent `hyper_code` from this error.
fn hyper_error_code(err: *const hyper_error) -> hyper_code {
non_null!(&*err ?= hyper_code::HYPERE_INVALID_ARG).code()
}
}
ffi_fn! {
/// Print the details of this error to a buffer.
///
/// The `dst_len` value must be the maximum length that the buffer can
/// store.
///
/// The return value is number of bytes that were written to `dst`.
fn hyper_error_print(err: *const hyper_error, dst: *mut u8, dst_len: size_t) -> size_t {
let dst = unsafe {
std::slice::from_raw_parts_mut(dst, dst_len)
};
non_null!(&*err ?= 0).print_to(dst)
}
}
-707
View File
@@ -1,707 +0,0 @@
use std::ffi::{c_int, c_void};
use bytes::Bytes;
use super::body::hyper_body;
use super::error::hyper_code;
use super::task::{hyper_task_return_type, AsTaskType};
use super::{UserDataPointer, HYPER_ITER_CONTINUE};
use crate::body::Incoming as IncomingBody;
use crate::ext::{HeaderCaseMap, OriginalHeaderOrder, ReasonPhrase};
use crate::ffi::size_t;
use crate::header::{HeaderName, HeaderValue};
use crate::{HeaderMap, Method, Request, Response, Uri};
/// An HTTP request.
///
/// Once you've finished constructing a request, you can send it with
/// `hyper_clientconn_send`.
///
/// Methods:
///
/// - hyper_request_new: Construct a new HTTP request.
/// - hyper_request_headers: Gets a mutable reference to the HTTP headers of this request
/// - hyper_request_set_body: Set the body of the request.
/// - hyper_request_set_method: Set the HTTP Method of the request.
/// - hyper_request_set_uri: Set the URI of the request.
/// - hyper_request_set_uri_parts: Set the URI of the request with separate scheme, authority, and path/query strings.
/// - hyper_request_set_version: Set the preferred HTTP version of the request.
/// - hyper_request_on_informational: Set an informational (1xx) response callback.
/// - hyper_request_free: Free an HTTP request.
pub struct hyper_request(pub(super) Request<IncomingBody>);
/// An HTTP response.
///
/// Obtain one of these by making a request with `hyper_clientconn_send`, then
/// polling the executor unntil you get a `hyper_task` of type
/// `HYPER_TASK_RESPONSE`. To figure out which request this response
/// corresponds to, check the userdata of the task, which you should
/// previously have set to an application-specific identifier for the
/// request.
///
/// Methods:
///
/// - hyper_response_status: Get the HTTP-Status code of this response.
/// - hyper_response_version: Get the HTTP version used by this response.
/// - hyper_response_reason_phrase: Get a pointer to the reason-phrase of this response.
/// - hyper_response_reason_phrase_len: Get the length of the reason-phrase of this response.
/// - hyper_response_headers: Gets a reference to the HTTP headers of this response.
/// - hyper_response_body: Take ownership of the body of this response.
/// - hyper_response_free: Free an HTTP response.
pub struct hyper_response(pub(super) Response<IncomingBody>);
/// An HTTP header map.
///
/// These can be part of a request or response.
///
/// Obtain a pointer to read or modify these from `hyper_request_headers`
/// or `hyper_response_headers`.
///
/// Methods:
///
/// - hyper_headers_add: Adds the provided value to the list of the provided name.
/// - hyper_headers_foreach: Iterates the headers passing each name and value pair to the callback.
/// - hyper_headers_set: Sets the header with the provided name to the provided value.
#[derive(Clone)]
pub struct hyper_headers {
pub(super) headers: HeaderMap,
orig_casing: HeaderCaseMap,
orig_order: OriginalHeaderOrder,
}
#[derive(Clone)]
struct OnInformational {
func: hyper_request_on_informational_callback,
data: UserDataPointer,
}
type hyper_request_on_informational_callback = extern "C" fn(*mut c_void, *mut hyper_response);
// ===== impl hyper_request =====
ffi_fn! {
/// Construct a new HTTP request.
///
/// The default request has an empty body. To send a body, call `hyper_request_set_body`.
///
///
/// To avoid a memory leak, the request must eventually be consumed by
/// `hyper_request_free` or `hyper_clientconn_send`.
fn hyper_request_new() -> *mut hyper_request {
Box::into_raw(Box::new(hyper_request(Request::new(IncomingBody::empty()))))
} ?= std::ptr::null_mut()
}
ffi_fn! {
/// Free an HTTP request.
///
/// This should only be used if the request isn't consumed by
/// `hyper_clientconn_send`.
fn hyper_request_free(req: *mut hyper_request) {
drop(non_null!(Box::from_raw(req) ?= ()));
}
}
ffi_fn! {
/// Set the HTTP Method of the request.
fn hyper_request_set_method(req: *mut hyper_request, method: *const u8, method_len: size_t) -> hyper_code {
let bytes = unsafe {
std::slice::from_raw_parts(method, method_len as usize)
};
let req = non_null!(&mut *req ?= hyper_code::HYPERE_INVALID_ARG);
match Method::from_bytes(bytes) {
Ok(m) => {
*req.0.method_mut() = m;
hyper_code::HYPERE_OK
},
Err(_) => {
hyper_code::HYPERE_INVALID_ARG
}
}
}
}
ffi_fn! {
/// Set the URI of the request.
///
/// The request's URI is best described as the `request-target` from the RFCs. So in HTTP/1,
/// whatever is set will get sent as-is in the first line (GET $uri HTTP/1.1). It
/// supports the 4 defined variants, origin-form, absolute-form, authority-form, and
/// asterisk-form.
///
/// The underlying type was built to efficiently support HTTP/2 where the request-target is
/// split over :scheme, :authority, and :path. As such, each part can be set explicitly, or the
/// type can parse a single contiguous string and if a scheme is found, that slot is "set". If
/// the string just starts with a path, only the path portion is set. All pseudo headers that
/// have been parsed/set are sent when the connection type is HTTP/2.
///
/// To set each slot explicitly, use `hyper_request_set_uri_parts`.
fn hyper_request_set_uri(req: *mut hyper_request, uri: *const u8, uri_len: size_t) -> hyper_code {
let bytes = unsafe {
std::slice::from_raw_parts(uri, uri_len as usize)
};
let req = non_null!(&mut *req ?= hyper_code::HYPERE_INVALID_ARG);
match Uri::from_maybe_shared(bytes) {
Ok(u) => {
*req.0.uri_mut() = u;
hyper_code::HYPERE_OK
},
Err(_) => {
hyper_code::HYPERE_INVALID_ARG
}
}
}
}
ffi_fn! {
/// Set the URI of the request with separate scheme, authority, and
/// path/query strings.
///
/// Each of `scheme`, `authority`, and `path_and_query` should either be
/// null, to skip providing a component, or point to a UTF-8 encoded
/// string. If any string pointer argument is non-null, its corresponding
/// `len` parameter must be set to the string's length.
fn hyper_request_set_uri_parts(
req: *mut hyper_request,
scheme: *const u8,
scheme_len: size_t,
authority: *const u8,
authority_len: size_t,
path_and_query: *const u8,
path_and_query_len: size_t
) -> hyper_code {
let mut builder = Uri::builder();
if !scheme.is_null() {
let scheme_bytes = unsafe {
std::slice::from_raw_parts(scheme, scheme_len as usize)
};
builder = builder.scheme(scheme_bytes);
}
if !authority.is_null() {
let authority_bytes = unsafe {
std::slice::from_raw_parts(authority, authority_len as usize)
};
builder = builder.authority(authority_bytes);
}
if !path_and_query.is_null() {
let path_and_query_bytes = unsafe {
std::slice::from_raw_parts(path_and_query, path_and_query_len as usize)
};
builder = builder.path_and_query(path_and_query_bytes);
}
let req = non_null!(&mut *req ?= hyper_code::HYPERE_INVALID_ARG);
match builder.build() {
Ok(u) => {
*req.0.uri_mut() = u;
hyper_code::HYPERE_OK
},
Err(_) => {
hyper_code::HYPERE_INVALID_ARG
}
}
}
}
ffi_fn! {
/// Set the preferred HTTP version of the request.
///
/// The version value should be one of the `HYPER_HTTP_VERSION_` constants.
///
/// Note that this won't change the major HTTP version of the connection,
/// since that is determined at the handshake step.
fn hyper_request_set_version(req: *mut hyper_request, version: c_int) -> hyper_code {
use http::Version;
let req = non_null!(&mut *req ?= hyper_code::HYPERE_INVALID_ARG);
*req.0.version_mut() = match version {
super::HYPER_HTTP_VERSION_NONE => Version::HTTP_11,
super::HYPER_HTTP_VERSION_1_0 => Version::HTTP_10,
super::HYPER_HTTP_VERSION_1_1 => Version::HTTP_11,
super::HYPER_HTTP_VERSION_2 => Version::HTTP_2,
_ => {
// We don't know this version
return hyper_code::HYPERE_INVALID_ARG;
}
};
hyper_code::HYPERE_OK
}
}
ffi_fn! {
/// Gets a mutable reference to the HTTP headers of this request
///
/// This is not an owned reference, so it should not be accessed after the
/// `hyper_request` has been consumed.
fn hyper_request_headers(req: *mut hyper_request) -> *mut hyper_headers {
let req = non_null!(&mut *req ?= std::ptr::null_mut());
hyper_headers::get_or_default(req.0.extensions_mut())
} ?= std::ptr::null_mut()
}
ffi_fn! {
/// Set the body of the request.
///
/// You can get a `hyper_body` by calling `hyper_body_new`.
///
/// This takes ownership of the `hyper_body *`, you must not use it or
/// free it after setting it on the request.
fn hyper_request_set_body(req: *mut hyper_request, body: *mut hyper_body) -> hyper_code {
let body = non_null!(Box::from_raw(body) ?= hyper_code::HYPERE_INVALID_ARG);
let req = non_null!(&mut *req ?= hyper_code::HYPERE_INVALID_ARG);
*req.0.body_mut() = body.0;
hyper_code::HYPERE_OK
}
}
ffi_fn! {
/// Set an informational (1xx) response callback.
///
/// The callback is called each time hyper receives an informational (1xx)
/// response for this request.
///
/// The third argument is an opaque user data pointer, which is passed to
/// the callback each time.
///
/// The callback is passed the `void *` data pointer, and a
/// `hyper_response *` which can be inspected as any other response. The
/// body of the response will always be empty.
///
/// NOTE: The `hyper_response *` is just borrowed data, and will not
/// be valid after the callback finishes. You must copy any data you wish
/// to persist.
fn hyper_request_on_informational(req: *mut hyper_request, callback: hyper_request_on_informational_callback, data: *mut c_void) -> hyper_code {
#[cfg(feature = "client")]
{
let ext = OnInformational {
func: callback,
data: UserDataPointer(data),
};
let req = non_null!(&mut *req ?= hyper_code::HYPERE_INVALID_ARG);
crate::ext::on_informational_raw(&mut req.0, ext);
hyper_code::HYPERE_OK
}
#[cfg(not(feature = "client"))]
{
drop((req, callback, data));
hyper_code::HYPERE_FEATURE_NOT_ENABLED
}
}
}
impl hyper_request {
pub(super) fn finalize_request(&mut self) {
if let Some(headers) = self.0.extensions_mut().remove::<hyper_headers>() {
*self.0.headers_mut() = headers.headers;
self.0.extensions_mut().insert(headers.orig_casing);
self.0.extensions_mut().insert(headers.orig_order);
}
}
}
// ===== impl hyper_response =====
ffi_fn! {
/// Free an HTTP response.
///
/// This should be used for any response once it is no longer needed.
fn hyper_response_free(resp: *mut hyper_response) {
drop(non_null!(Box::from_raw(resp) ?= ()));
}
}
ffi_fn! {
/// Get the HTTP-Status code of this response.
///
/// It will always be within the range of 100-599.
fn hyper_response_status(resp: *const hyper_response) -> u16 {
non_null!(&*resp ?= 0).0.status().as_u16()
}
}
ffi_fn! {
/// Get a pointer to the reason-phrase of this response.
///
/// This buffer is not null-terminated.
///
/// This buffer is owned by the response, and should not be used after
/// the response has been freed.
///
/// Use `hyper_response_reason_phrase_len()` to get the length of this
/// buffer.
fn hyper_response_reason_phrase(resp: *const hyper_response) -> *const u8 {
non_null!(&*resp ?= std::ptr::null()).reason_phrase().as_ptr()
} ?= std::ptr::null()
}
ffi_fn! {
/// Get the length of the reason-phrase of this response.
///
/// Use `hyper_response_reason_phrase()` to get the buffer pointer.
fn hyper_response_reason_phrase_len(resp: *const hyper_response) -> size_t {
non_null!(&*resp ?= 0).reason_phrase().len()
}
}
ffi_fn! {
/// Get the HTTP version used by this response.
///
/// The returned value could be:
///
/// - `HYPER_HTTP_VERSION_1_0`
/// - `HYPER_HTTP_VERSION_1_1`
/// - `HYPER_HTTP_VERSION_2`
/// - `HYPER_HTTP_VERSION_NONE` if newer (or older).
fn hyper_response_version(resp: *const hyper_response) -> c_int {
use http::Version;
match non_null!(&*resp ?= 0).0.version() {
Version::HTTP_10 => super::HYPER_HTTP_VERSION_1_0,
Version::HTTP_11 => super::HYPER_HTTP_VERSION_1_1,
Version::HTTP_2 => super::HYPER_HTTP_VERSION_2,
_ => super::HYPER_HTTP_VERSION_NONE,
}
}
}
ffi_fn! {
/// Gets a reference to the HTTP headers of this response.
///
/// This is not an owned reference, so it should not be accessed after the
/// `hyper_response` has been freed.
fn hyper_response_headers(resp: *mut hyper_response) -> *mut hyper_headers {
let resp = non_null!(&mut *resp ?= std::ptr::null_mut());
hyper_headers::get_or_default(resp.0.extensions_mut())
} ?= std::ptr::null_mut()
}
ffi_fn! {
/// Take ownership of the body of this response.
///
/// It is safe to free the response even after taking ownership of its body.
///
/// To avoid a memory leak, the body must eventually be consumed by
/// `hyper_body_free`, `hyper_body_foreach`, or `hyper_request_set_body`.
fn hyper_response_body(resp: *mut hyper_response) -> *mut hyper_body {
let body = std::mem::replace(non_null!(&mut *resp ?= std::ptr::null_mut()).0.body_mut(), IncomingBody::empty());
Box::into_raw(Box::new(hyper_body(body)))
} ?= std::ptr::null_mut()
}
impl hyper_response {
pub(super) fn wrap(mut resp: Response<IncomingBody>) -> hyper_response {
let headers = std::mem::take(resp.headers_mut());
let orig_casing = resp
.extensions_mut()
.remove::<HeaderCaseMap>()
.unwrap_or_else(HeaderCaseMap::default);
let orig_order = resp
.extensions_mut()
.remove::<OriginalHeaderOrder>()
.unwrap_or_else(OriginalHeaderOrder::default);
resp.extensions_mut().insert(hyper_headers {
headers,
orig_casing,
orig_order,
});
hyper_response(resp)
}
fn reason_phrase(&self) -> &[u8] {
if let Some(reason) = self.0.extensions().get::<ReasonPhrase>() {
return reason.as_bytes();
}
if let Some(reason) = self.0.status().canonical_reason() {
return reason.as_bytes();
}
&[]
}
}
unsafe impl AsTaskType for hyper_response {
fn as_task_type(&self) -> hyper_task_return_type {
hyper_task_return_type::HYPER_TASK_RESPONSE
}
}
// ===== impl Headers =====
type hyper_headers_foreach_callback =
extern "C" fn(*mut c_void, *const u8, size_t, *const u8, size_t) -> c_int;
impl hyper_headers {
pub(super) fn get_or_default(ext: &mut http::Extensions) -> &mut hyper_headers {
if let None = ext.get_mut::<hyper_headers>() {
ext.insert(hyper_headers::default());
}
ext.get_mut::<hyper_headers>()
.expect("hyper headers inserted into extensions")
}
}
ffi_fn! {
/// Iterates the headers passing each name and value pair to the callback.
///
/// The `userdata` pointer is also passed to the callback.
///
/// The callback should return `HYPER_ITER_CONTINUE` to keep iterating, or
/// `HYPER_ITER_BREAK` to stop.
fn hyper_headers_foreach(headers: *const hyper_headers, func: hyper_headers_foreach_callback, userdata: *mut c_void) {
let headers = non_null!(&*headers ?= ());
// For each header name/value pair, there may be a value in the casemap
// that corresponds to the HeaderValue. So, we iterator all the keys,
// and for each one, try to pair the originally cased name with the value.
//
// TODO: consider adding http::HeaderMap::entries() iterator
let mut ordered_iter = headers.orig_order.get_in_order().peekable();
if ordered_iter.peek().is_some() {
for (name, idx) in ordered_iter {
let (name_ptr, name_len) = if let Some(orig_name) = headers.orig_casing.get_all(name).nth(*idx) {
(orig_name.as_ref().as_ptr(), orig_name.as_ref().len())
} else {
(
name.as_str().as_bytes().as_ptr(),
name.as_str().as_bytes().len(),
)
};
let val_ptr;
let val_len;
if let Some(value) = headers.headers.get_all(name).iter().nth(*idx) {
val_ptr = value.as_bytes().as_ptr();
val_len = value.as_bytes().len();
} else {
// Stop iterating, something has gone wrong.
return;
}
if HYPER_ITER_CONTINUE != func(userdata, name_ptr, name_len, val_ptr, val_len) {
return;
}
}
} else {
for name in headers.headers.keys() {
let mut names = headers.orig_casing.get_all(name);
for value in headers.headers.get_all(name) {
let (name_ptr, name_len) = if let Some(orig_name) = names.next() {
(orig_name.as_ref().as_ptr(), orig_name.as_ref().len())
} else {
(
name.as_str().as_bytes().as_ptr(),
name.as_str().as_bytes().len(),
)
};
let val_ptr = value.as_bytes().as_ptr();
let val_len = value.as_bytes().len();
if HYPER_ITER_CONTINUE != func(userdata, name_ptr, name_len, val_ptr, val_len) {
return;
}
}
}
}
}
}
ffi_fn! {
/// Sets the header with the provided name to the provided value.
///
/// This overwrites any previous value set for the header.
fn hyper_headers_set(headers: *mut hyper_headers, name: *const u8, name_len: size_t, value: *const u8, value_len: size_t) -> hyper_code {
let headers = non_null!(&mut *headers ?= hyper_code::HYPERE_INVALID_ARG);
match unsafe { raw_name_value(name, name_len, value, value_len) } {
Ok((name, value, orig_name)) => {
headers.headers.insert(&name, value);
headers.orig_casing.insert(name.clone(), orig_name.clone());
headers.orig_order.insert(name);
hyper_code::HYPERE_OK
}
Err(code) => code,
}
}
}
ffi_fn! {
/// Adds the provided value to the list of the provided name.
///
/// If there were already existing values for the name, this will append the
/// new value to the internal list.
fn hyper_headers_add(headers: *mut hyper_headers, name: *const u8, name_len: size_t, value: *const u8, value_len: size_t) -> hyper_code {
let headers = non_null!(&mut *headers ?= hyper_code::HYPERE_INVALID_ARG);
match unsafe { raw_name_value(name, name_len, value, value_len) } {
Ok((name, value, orig_name)) => {
headers.headers.append(&name, value);
headers.orig_casing.append(&name, orig_name.clone());
headers.orig_order.append(name);
hyper_code::HYPERE_OK
}
Err(code) => code,
}
}
}
impl Default for hyper_headers {
fn default() -> Self {
Self {
headers: Default::default(),
orig_casing: HeaderCaseMap::default(),
orig_order: OriginalHeaderOrder::default(),
}
}
}
unsafe fn raw_name_value(
name: *const u8,
name_len: size_t,
value: *const u8,
value_len: size_t,
) -> Result<(HeaderName, HeaderValue, Bytes), hyper_code> {
let name = std::slice::from_raw_parts(name, name_len);
let orig_name = Bytes::copy_from_slice(name);
let name = match HeaderName::from_bytes(name) {
Ok(name) => name,
Err(_) => return Err(hyper_code::HYPERE_INVALID_ARG),
};
let value = std::slice::from_raw_parts(value, value_len);
let value = match HeaderValue::from_bytes(value) {
Ok(val) => val,
Err(_) => return Err(hyper_code::HYPERE_INVALID_ARG),
};
Ok((name, value, orig_name))
}
// ===== impl OnInformational =====
#[cfg(feature = "client")]
impl crate::ext::OnInformationalCallback for OnInformational {
fn on_informational(&self, res: http::Response<()>) {
let res = res.map(|()| IncomingBody::empty());
let mut res = hyper_response::wrap(res);
(self.func)(self.data.0, &mut res);
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_headers_foreach_cases_preserved() {
let mut headers = hyper_headers::default();
let name1 = b"Set-CookiE";
let value1 = b"a=b";
hyper_headers_add(
&mut headers,
name1.as_ptr(),
name1.len(),
value1.as_ptr(),
value1.len(),
);
let name2 = b"SET-COOKIE";
let value2 = b"c=d";
hyper_headers_add(
&mut headers,
name2.as_ptr(),
name2.len(),
value2.as_ptr(),
value2.len(),
);
let mut vec = Vec::<u8>::new();
hyper_headers_foreach(&headers, concat, &mut vec as *mut _ as *mut c_void);
assert_eq!(vec, b"Set-CookiE: a=b\r\nSET-COOKIE: c=d\r\n");
extern "C" fn concat(
vec: *mut c_void,
name: *const u8,
name_len: usize,
value: *const u8,
value_len: usize,
) -> c_int {
unsafe {
let vec = &mut *(vec as *mut Vec<u8>);
let name = std::slice::from_raw_parts(name, name_len);
let value = std::slice::from_raw_parts(value, value_len);
vec.extend(name);
vec.extend(b": ");
vec.extend(value);
vec.extend(b"\r\n");
}
HYPER_ITER_CONTINUE
}
}
#[cfg(all(feature = "http1", feature = "ffi"))]
#[test]
fn test_headers_foreach_order_preserved() {
let mut headers = hyper_headers::default();
let name1 = b"Set-CookiE";
let value1 = b"a=b";
hyper_headers_add(
&mut headers,
name1.as_ptr(),
name1.len(),
value1.as_ptr(),
value1.len(),
);
let name2 = b"Content-Encoding";
let value2 = b"gzip";
hyper_headers_add(
&mut headers,
name2.as_ptr(),
name2.len(),
value2.as_ptr(),
value2.len(),
);
let name3 = b"SET-COOKIE";
let value3 = b"c=d";
hyper_headers_add(
&mut headers,
name3.as_ptr(),
name3.len(),
value3.as_ptr(),
value3.len(),
);
let mut vec = Vec::<u8>::new();
hyper_headers_foreach(&headers, concat, &mut vec as *mut _ as *mut c_void);
println!("{}", std::str::from_utf8(&vec).unwrap());
assert_eq!(
vec,
b"Set-CookiE: a=b\r\nContent-Encoding: gzip\r\nSET-COOKIE: c=d\r\n"
);
extern "C" fn concat(
vec: *mut c_void,
name: *const u8,
name_len: usize,
value: *const u8,
value_len: usize,
) -> c_int {
unsafe {
let vec = &mut *(vec as *mut Vec<u8>);
let name = std::slice::from_raw_parts(name, name_len);
let value = std::slice::from_raw_parts(value, value_len);
vec.extend(name);
vec.extend(b": ");
vec.extend(value);
vec.extend(b"\r\n");
}
HYPER_ITER_CONTINUE
}
}
}

Some files were not shown because too many files have changed in this diff Show More