Compare commits

..

2 Commits

Author SHA1 Message Date
overtrue da3fd83aa7 style: cargo fmt 2026-08-22 10:11:51 +08:00
overtrue a2e7036cb1 refactor(data-usage): ReplicationStats -> ReplicationTargetUsage
Rename the data-usage crate's ReplicationStats to ReplicationTargetUsage.
Serde field names are byte-identical (only the Rust type name changed;
field identifiers that rmp encodes are untouched). An rmp round-trip test
guards against future drift.

Scanner test imports updated to match.
2026-08-22 10:11:51 +08:00
8 changed files with 74 additions and 103 deletions
-7
View File
@@ -57,13 +57,6 @@ pub const DEFAULT_MAX_IO_EVENTS_PER_TICK: usize = 1024;
pub const DEFAULT_EVENT_INTERVAL: u32 = 61; pub const DEFAULT_EVENT_INTERVAL: u32 = 61;
pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random pub const DEFAULT_RNG_SEED: Option<u64> = None; // None means random
/// Dedicated blocking thread pool for fsync/fdatasync operations.
/// When > 1, fsync operations are isolated from the main blocking pool to
/// prevent device-bound fsync from starving read operations (pread/stat/open).
/// Default 0 means auto (no isolation, use main runtime).
pub const ENV_FSYNC_BLOCKING_THREADS: &str = "RUSTFS_RUNTIME_FSYNC_BLOCKING_THREADS";
pub const DEFAULT_FSYNC_BLOCKING_THREADS: usize = 0;
// Dial9 Tokio Telemetry Default values // Dial9 Tokio Telemetry Default values
pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default pub const DEFAULT_RUNTIME_DIAL9_ENABLED: bool = false; // Disabled by default
pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry"; pub const DEFAULT_RUNTIME_DIAL9_OUTPUT_DIR: &str = "/var/log/rustfs/telemetry";
+64 -18
View File
@@ -585,9 +585,12 @@ impl VersionsHistogram {
} }
} }
/// Replication statistics for a single target /// Replication statistics for a single target.
#[derive(Debug, Default, Clone, Serialize, Deserialize)] ///
pub struct ReplicationStats { /// Renamed from `ReplicationStats`; serde field names are preserved
/// byte-identically to maintain wire compatibility with existing snapshots.
#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)]
pub struct ReplicationTargetUsage {
pub pending_size: u64, pub pending_size: u64,
pub replicated_size: u64, pub replicated_size: u64,
pub failed_size: u64, pub failed_size: u64,
@@ -600,7 +603,7 @@ pub struct ReplicationStats {
pub replicated_count: u64, pub replicated_count: u64,
} }
impl ReplicationStats { impl ReplicationTargetUsage {
pub fn is_empty(&self) -> bool { pub fn is_empty(&self) -> bool {
let Self { let Self {
pending_size, pending_size,
@@ -636,7 +639,7 @@ impl ReplicationStats {
/// Replication statistics for all targets /// Replication statistics for all targets
#[derive(Debug, Default, Clone, Serialize, Deserialize)] #[derive(Debug, Default, Clone, Serialize, Deserialize)]
pub struct ReplicationAllStats { pub struct ReplicationAllStats {
pub targets: HashMap<String, ReplicationStats>, pub targets: HashMap<String, ReplicationTargetUsage>,
pub replica_size: u64, pub replica_size: u64,
pub replica_count: u64, pub replica_count: u64,
} }
@@ -649,7 +652,7 @@ impl ReplicationAllStats {
targets, targets,
} = self; } = self;
*replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationStats::is_empty) *replica_size == 0 && *replica_count == 0 && targets.values().all(ReplicationTargetUsage::is_empty)
} }
#[deprecated(note = "use is_empty instead")] #[deprecated(note = "use is_empty instead")]
@@ -2466,7 +2469,7 @@ mod tests {
#[test] #[test]
fn replication_stats_empty_checks_every_field() { fn replication_stats_empty_checks_every_field() {
type SetField = fn(&mut ReplicationStats); type SetField = fn(&mut ReplicationTargetUsage);
let cases: [(&str, SetField); 10] = [ let cases: [(&str, SetField); 10] = [
("pending_size", |stats| stats.pending_size = 1), ("pending_size", |stats| stats.pending_size = 1),
@@ -2481,9 +2484,9 @@ mod tests {
("replicated_count", |stats| stats.replicated_count = 1), ("replicated_count", |stats| stats.replicated_count = 1),
]; ];
assert!(ReplicationStats::default().is_empty()); assert!(ReplicationTargetUsage::default().is_empty());
for (field, set_nonzero) in cases { for (field, set_nonzero) in cases {
let mut stats = ReplicationStats::default(); let mut stats = ReplicationTargetUsage::default();
set_nonzero(&mut stats); set_nonzero(&mut stats);
assert!(!stats.is_empty(), "{field} must make replication stats non-empty"); assert!(!stats.is_empty(), "{field} must make replication stats non-empty");
} }
@@ -2514,17 +2517,17 @@ mod tests {
} }
let empty_targets = ReplicationAllStats { let empty_targets = ReplicationAllStats {
targets: HashMap::from([("arn:test:empty".to_string(), ReplicationStats::default())]), targets: HashMap::from([("arn:test:empty".to_string(), ReplicationTargetUsage::default())]),
..Default::default() ..Default::default()
}; };
assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty"); assert!(empty_targets.is_empty(), "all-empty targets must keep aggregate stats empty");
let stats = ReplicationAllStats { let stats = ReplicationAllStats {
targets: HashMap::from([ targets: HashMap::from([
("arn:test:empty".to_string(), ReplicationStats::default()), ("arn:test:empty".to_string(), ReplicationTargetUsage::default()),
( (
"arn:test:non-empty".to_string(), "arn:test:non-empty".to_string(),
ReplicationStats { ReplicationTargetUsage {
pending_count: 1, pending_count: 1,
..Default::default() ..Default::default()
}, },
@@ -2565,7 +2568,7 @@ mod tests {
replication_stats: Some(ReplicationAllStats { replication_stats: Some(ReplicationAllStats {
targets: HashMap::from([( targets: HashMap::from([(
"arn:test:pending".to_string(), "arn:test:pending".to_string(),
ReplicationStats { ReplicationTargetUsage {
pending_count: 1, pending_count: 1,
..Default::default() ..Default::default()
}, },
@@ -2714,7 +2717,7 @@ mod tests {
targets: HashMap::from([ targets: HashMap::from([
( (
"arn:self-only".to_string(), "arn:self-only".to_string(),
ReplicationStats { ReplicationTargetUsage {
pending_size: 7, pending_size: 7,
pending_count: 1, pending_count: 1,
..Default::default() ..Default::default()
@@ -2722,7 +2725,7 @@ mod tests {
), ),
( (
"arn:shared".to_string(), "arn:shared".to_string(),
ReplicationStats { ReplicationTargetUsage {
failed_size: 3, failed_size: 3,
failed_count: 1, failed_count: 1,
missed_threshold_size: 2, missed_threshold_size: 2,
@@ -2741,7 +2744,7 @@ mod tests {
targets: HashMap::from([ targets: HashMap::from([
( (
"arn:shared".to_string(), "arn:shared".to_string(),
ReplicationStats { ReplicationTargetUsage {
failed_size: 5, failed_size: 5,
failed_count: 2, failed_count: 2,
after_threshold_size: 4, after_threshold_size: 4,
@@ -2751,7 +2754,7 @@ mod tests {
), ),
( (
"arn:other-only".to_string(), "arn:other-only".to_string(),
ReplicationStats { ReplicationTargetUsage {
replicated_size: 11, replicated_size: 11,
replicated_count: 3, replicated_count: 3,
..Default::default() ..Default::default()
@@ -2993,7 +2996,9 @@ mod tests {
fn replication_target_deserialization_preserves_large_historical_maps() { fn replication_target_deserialization_preserves_large_historical_maps() {
let mut stats = ReplicationAllStats::default(); let mut stats = ReplicationAllStats::default();
for index in 0..=1024 { for index in 0..=1024 {
stats.targets.insert(format!("target-{index}"), ReplicationStats::default()); stats
.targets
.insert(format!("target-{index}"), ReplicationTargetUsage::default());
} }
let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode"); let encoded = rmp_serde::to_vec_named(&stats).expect("large replication target fixture should encode");
let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded) let decoded = rmp_serde::from_slice::<ReplicationAllStats>(&encoded)
@@ -3002,6 +3007,47 @@ mod tests {
assert_eq!(decoded.targets.len(), stats.targets.len()); assert_eq!(decoded.targets.len(), stats.targets.len());
} }
/// Round-trip test: encoding a [`ReplicationTargetUsage`] and decoding it back
/// must produce the exact same value. This guards against accidental serde
/// field-name drift during the `ReplicationStats` -> `ReplicationTargetUsage`
/// rename. Wire-level field names are the serialized Rust field identifiers,
/// which must remain byte-identical.
#[test]
fn replication_target_usage_rmp_round_trip() {
let original = ReplicationTargetUsage {
pending_size: 100,
replicated_size: 2_000,
failed_size: 50,
failed_count: 3,
pending_count: 7,
missed_threshold_size: 11,
after_threshold_size: 22,
missed_threshold_count: 1,
after_threshold_count: 2,
replicated_count: 99,
};
let buf = rmp_serde::to_vec_named(&original).expect("encode ReplicationTargetUsage to msgpack");
let decoded: ReplicationTargetUsage = rmp_serde::from_slice(&buf).expect("decode ReplicationTargetUsage from msgpack");
assert_eq!(original, decoded, "round-trip through rmp must preserve every field");
// Also verify that encoding as an unnamed sequence and then decoding
// with named fields produces the correct mapping (this catches reordering).
let named_buf = rmp_serde::to_vec_named(&original).expect("re-encode for field-name pinning");
// Spot-check that known field names appear in the named encoding.
let named_str = String::from_utf8_lossy(&named_buf);
assert!(named_str.contains("pending_size"), "field 'pending_size' must survive the rename");
assert!(named_str.contains("replicated_size"), "field 'replicated_size' must survive the rename");
assert!(
named_str.contains("missed_threshold_size"),
"field 'missed_threshold_size' must survive the rename"
);
assert!(
named_str.contains("after_threshold_count"),
"field 'after_threshold_count' must survive the rename"
);
}
#[test] #[test]
fn checked_merge_rejects_noncanonical_histograms_without_mutation() { fn checked_merge_rejects_noncanonical_histograms_without_mutation() {
let mut entry = DataUsageEntry { let mut entry = DataUsageEntry {
+4 -42
View File
@@ -315,7 +315,7 @@ pub async fn fsync_dir(dir: impl AsRef<Path>) -> io::Result<()> {
#[cfg(unix)] #[cfg(unix)]
{ {
let dir = dir.as_ref().to_path_buf(); let dir = dir.as_ref().to_path_buf();
fsync_spawn_blocking(move || fsync_dir_std(dir)).await? tokio::task::spawn_blocking(move || fsync_dir_std(dir)).await?
} }
#[cfg(not(unix))] #[cfg(not(unix))]
@@ -683,7 +683,7 @@ async fn fsync_open_dst_dir_group(group: &DstDirFsyncGroup) -> io::Result<()> {
#[cfg(test)] #[cfg(test)]
let dir = group.dir.clone(); let dir = group.dir.clone();
let dir_file = group.dir_file.clone(); let dir_file = group.dir_file.clone();
fsync_spawn_blocking(move || { tokio::task::spawn_blocking(move || {
#[cfg(test)] #[cfg(test)]
{ {
if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) { if let Some(kind) = fsync_dir_recorder::take_grouped_failure(&dir) {
@@ -1080,44 +1080,6 @@ const TEST_GLOBAL_FILE_SYNCS: usize = 64;
static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit())); static FILE_SYNC_PERMITS: LazyLock<Semaphore> = LazyLock::new(|| Semaphore::new(global_file_sync_limit()));
static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new())); static DISK_FILE_SYNC_LIMITERS: LazyLock<Mutex<HashMap<PathBuf, Weak<Semaphore>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
/// Dedicated tokio runtime for fsync/fdatasync blocking operations. When
/// configured with >1 threads, isolates device-bound fsync from the main
/// blocking pool so reads (pread/stat/open) are not starved. `None` means
/// fall back to the main runtime (zero behavior change).
static FSYNC_RUNTIME: LazyLock<Option<tokio::runtime::Runtime>> = LazyLock::new(|| {
let threads =
rustfs_utils::get_env_usize(rustfs_config::ENV_FSYNC_BLOCKING_THREADS, rustfs_config::DEFAULT_FSYNC_BLOCKING_THREADS);
if threads <= 1 {
return None;
}
let mut builder = tokio::runtime::Builder::new_multi_thread();
builder
.worker_threads(num_cpus::get().min(8))
.max_blocking_threads(threads)
.thread_name("rustfs-fsync")
.thread_stack_size(512 * 1024)
.enable_all();
match builder.build() {
Ok(rt) => {
tracing::info!(threads, "fsync dedicated blocking pool enabled");
Some(rt)
}
Err(err) => {
tracing::warn!(%err, "failed to build fsync runtime, falling back to main pool");
None
}
}
});
/// Spawn a blocking task on the fsync-dedicated runtime if configured,
/// otherwise fall back to the main tokio blocking pool.
fn fsync_spawn_blocking<T: Send + 'static>(f: impl FnOnce() -> T + Send + 'static) -> tokio::task::JoinHandle<T> {
match FSYNC_RUNTIME.as_ref() {
Some(rt) => rt.spawn_blocking(f),
None => tokio::task::spawn_blocking(f),
}
}
static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> = static DISK_VOLUME_MUTATION_LOCKS: LazyLock<Mutex<HashMap<PathBuf, Weak<RwLock<()>>>>> =
LazyLock::new(|| Mutex::new(HashMap::new())); LazyLock::new(|| Mutex::new(HashMap::new()));
type NamespaceMutationLock = AsyncMutex<()>; type NamespaceMutationLock = AsyncMutex<()>;
@@ -1255,7 +1217,7 @@ where
F: FnOnce() -> io::Result<T> + Send + 'static, F: FnOnce() -> io::Result<T> + Send + 'static,
{ {
let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?; let (disk_permit, global_permit) = acquire_file_sync_permits(disk_permits).await?;
let result = fsync_spawn_blocking(move || { let result = tokio::task::spawn_blocking(move || {
let _disk_permit = disk_permit; let _disk_permit = disk_permit;
work() work()
}) })
@@ -2184,7 +2146,7 @@ async fn run_blocking_namespace_file_sync_operation_with_global<T: Send + 'stati
wait_started, wait_started,
); );
let disk_permit = admission.disk_permit.clone(); let disk_permit = admission.disk_permit.clone();
let result = fsync_spawn_blocking(move || { let result = tokio::task::spawn_blocking(move || {
let _lease = lease; let _lease = lease;
let _disk_permit = disk_permit; let _disk_permit = disk_permit;
operation() operation()
-4
View File
@@ -2106,9 +2106,6 @@ pub enum ChannelClass {
Bulk, Bulk,
} }
// Keep multiplexed unary RPCs below h2's per-connection small-frame budget.
const INTERNODE_RPC_CONCURRENCY_LIMIT: usize = 64;
/// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout). /// Whether control/bulk channel isolation is enabled (env-gated, default off for safe rollout).
fn channel_isolation_enabled() -> bool { fn channel_isolation_enabled() -> bool {
rustfs_utils::get_env_bool( rustfs_utils::get_env_bool(
@@ -2191,7 +2188,6 @@ async fn build_channel(dial_addr: &str, cache_key: &str) -> Result<Channel, Box<
let mut connector = Endpoint::from_shared(dial_addr.to_string())? let mut connector = Endpoint::from_shared(dial_addr.to_string())?
// Fast connection timeout for dead peer detection // Fast connection timeout for dead peer detection
.connect_timeout(connect_timeout) .connect_timeout(connect_timeout)
.concurrency_limit(INTERNODE_RPC_CONCURRENCY_LIMIT)
// TCP-level keepalive - OS will probe connection // TCP-level keepalive - OS will probe connection
.tcp_keepalive(Some(tcp_keepalive)) .tcp_keepalive(Some(tcp_keepalive))
// Disable Nagle so latency-sensitive control-plane RPCs (locks/health) are not batched // Disable Nagle so latency-sensitive control-plane RPCs (locks/health) are not batched
@@ -16,7 +16,7 @@ use super::persistence::DataUsageCacheLoadAttempt;
use super::*; use super::*;
use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO}; use crate::storage_api::scanner_io::{HTTPRangeSpec, ObjectIO};
use crate::{ScannerGetObjectReader, ScannerPutObjReader}; use crate::{ScannerGetObjectReader, ScannerPutObjReader};
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats}; use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
use serde_json::Value; use serde_json::Value;
use std::io::Cursor; use std::io::Cursor;
use std::pin::Pin; use std::pin::Pin;
@@ -1636,7 +1636,7 @@ fn size_recursive_prunes_empty_and_preserves_threshold_replication_stats() {
replication_stats: Some(ReplicationAllStats { replication_stats: Some(ReplicationAllStats {
targets: HashMap::from([( targets: HashMap::from([(
"arn:test:threshold".to_string(), "arn:test:threshold".to_string(),
ReplicationStats { ReplicationTargetUsage {
after_threshold_count: 1, after_threshold_count: 1,
..Default::default() ..Default::default()
}, },
@@ -13,7 +13,7 @@
// limitations under the License. // limitations under the License.
use super::*; use super::*;
use rustfs_data_usage::{ReplicationAllStats, ReplicationStats}; use rustfs_data_usage::{ReplicationAllStats, ReplicationTargetUsage};
const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]); const TEST_PLAN_DIGEST: DataUsageScanPlanDigest = DataUsageScanPlanDigest([7; 32]);
@@ -271,7 +271,7 @@ fn completed_data_usage_info_flattens_nested_bucket_entries() {
replication_stats: Some(ReplicationAllStats { replication_stats: Some(ReplicationAllStats {
targets: HashMap::from([( targets: HashMap::from([(
"arn:target".to_string(), "arn:target".to_string(),
ReplicationStats { ReplicationTargetUsage {
replicated_size: 2048, replicated_size: 2048,
replicated_count: 2, replicated_count: 2,
..Default::default() ..Default::default()
+1 -26
View File
@@ -206,13 +206,6 @@ def check_runner_selection(root: Path) -> list[str]:
return errors return errors
def check_s3_tests_runner(root: Path) -> list[str]:
runner = (root / "scripts/s3-tests/run.sh").read_text()
if "--showlocals" in runner:
return ["scripts/s3-tests/run.sh: pytest failure diagnostics must not dump local values"]
return []
def profile_selection(root: Path, profile: str) -> str: def profile_selection(root: Path, profile: str) -> str:
if not re.fullmatch(r"e2e-[a-z0-9-]+", profile): if not re.fullmatch(r"e2e-[a-z0-9-]+", profile):
raise ValueError(f"invalid e2e profile name: {profile}") raise ValueError(f"invalid e2e profile name: {profile}")
@@ -279,7 +272,6 @@ def validate(root: Path) -> list[str]:
errors.extend(check_e2e_modules(root)) errors.extend(check_e2e_modules(root))
errors.extend(check_fuzz_targets(root)) errors.extend(check_fuzz_targets(root))
errors.extend(check_runner_selection(root)) errors.extend(check_runner_selection(root))
errors.extend(check_s3_tests_runner(root))
errors.extend(check_profile_definitions(root)) errors.extend(check_profile_definitions(root))
return errors return errors
@@ -349,23 +341,6 @@ class SelfTests(unittest.TestCase):
) )
self.assertEqual(len(check_fuzz_targets(root)), 1) self.assertEqual(len(check_fuzz_targets(root)), 1)
def test_s3_runner_rejects_unbounded_failure_locals(self) -> None:
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
runner = root / "scripts/s3-tests/run.sh"
runner.parent.mkdir(parents=True)
runner.write_text("tox -- -vv -ra --tb=long\n")
self.assertEqual(check_s3_tests_runner(root), [])
runner.write_text("tox -- -vv -ra --showlocals --tb=long\n")
self.assertEqual(len(check_s3_tests_runner(root)), 1)
with (
mock.patch(__name__ + ".check_e2e_modules", return_value=[]),
mock.patch(__name__ + ".check_fuzz_targets", return_value=[]),
mock.patch(__name__ + ".check_runner_selection", return_value=[]),
mock.patch(__name__ + ".check_profile_definitions", return_value=[]),
):
self.assertEqual(len(validate(root)), 1)
def test_profile_listing_enforces_selection(self) -> None: def test_profile_listing_enforces_selection(self) -> None:
with tempfile.TemporaryDirectory() as tmp: with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp) root = Path(tmp)
@@ -436,7 +411,7 @@ def main() -> int:
for error in errors: for error in errors:
print(f"ERROR: {error}", file=sys.stderr) print(f"ERROR: {error}", file=sys.stderr)
return 1 return 1
print("OK: e2e modules, runner selection, fuzz matrices, profiles, and bounded diagnostics are wired") print("OK: e2e modules, runner selection, fuzz matrices, and profile guards are wired")
return 0 return 0
+1 -2
View File
@@ -1028,11 +1028,10 @@ else
fi fi
# Run tests from s3tests/functional # Run tests from s3tests/functional
# Failure locals can contain multi-MiB request bodies; keep tracebacks without expanding local values.
set +e set +e
S3TEST_CONF="${CONF_OUTPUT_PATH}" \ S3TEST_CONF="${CONF_OUTPUT_PATH}" \
tox -- \ tox -- \
-vv -ra --tb=long \ -vv -ra --showlocals --tb=long \
--maxfail="${MAXFAIL}" \ --maxfail="${MAXFAIL}" \
--timeout="${TEST_TIMEOUT}" \ --timeout="${TEST_TIMEOUT}" \
--junitxml="${ARTIFACTS_DIR}/junit.xml" \ --junitxml="${ARTIFACTS_DIR}/junit.xml" \