mirror of
https://github.com/rustfs/rustfs.git
synced 2026-11-17 21:44:21 +00:00
Compare commits
17 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 739699426b | |||
| 0d2a0ea345 | |||
| c1f97d27f6 | |||
| 1066ab4d1d | |||
| b0e824437b | |||
| 913a5df515 | |||
| f483cfedc8 | |||
| f2177505da | |||
| a758b58bf2 | |||
| 9075131b23 | |||
| 050be16dc7 | |||
| 8efa1a96c0 | |||
| 444dbd9956 | |||
| 33e0fe8c6f | |||
| 205227b441 | |||
| 3e5d37742d | |||
| 8e6265907a |
@@ -31,3 +31,14 @@
|
||||
# the flag, so the two can no longer drift apart unnoticed.
|
||||
#
|
||||
# For CPU profiling, add `-C force-frame-pointers=yes` to that RUSTFLAGS value.
|
||||
|
||||
# Exact internode HTTP/2 PING ACK timeout accounting is also deliberately
|
||||
# default-off. Hyper exposes the exact timeout only through its unstable tracing
|
||||
# event, so diagnostic builds must opt in to both the feature and cfg:
|
||||
#
|
||||
# RUSTFLAGS="--cfg hyper_unstable_tracing" \
|
||||
# cargo build -p rustfs --features diagnostic-internode-http2-ping
|
||||
#
|
||||
# `crates/protos/build.rs` and `crates/obs/build.rs` fail fast if the feature and
|
||||
# cfg drift apart. Do not move this cfg into global rustflags: doing so would
|
||||
# recompile every dependency under an unstable, non-semver build contract.
|
||||
|
||||
@@ -39,6 +39,8 @@ script-tests: ## Run shell script tests
|
||||
./scripts/test_entrypoint_directories.sh
|
||||
./scripts/test_internode_grpc_ab_bench.sh
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_grpc_window.py
|
||||
$(RUSTFS_PYTHON_BIN) ./scripts/test_internode_msgpack_peer_canary.py
|
||||
./scripts/test_internode_msgpack_peer_canary_runner.sh
|
||||
./scripts/test_object_batch_bench_enhanced.sh
|
||||
./scripts/test_package_service_scripts.sh
|
||||
./scripts/test_hotpath_warp_ab_gate.sh
|
||||
|
||||
Generated
+52
-26
@@ -255,7 +255,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -266,7 +266,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1618,15 +1618,6 @@ dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bincode"
|
||||
version = "1.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "bindgen"
|
||||
version = "0.71.1"
|
||||
@@ -2024,7 +2015,7 @@ dependencies = [
|
||||
"maybe-owned",
|
||||
"rustix",
|
||||
"rustix-linux-procfs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
"winx",
|
||||
]
|
||||
|
||||
@@ -4142,7 +4133,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4456,7 +4447,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e35d6e78f4cebb90e51d543db8fe41528851d6deeef6ef21ebd4bfdc1e784776"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5596,6 +5587,7 @@ dependencies = [
|
||||
"pin-project-lite",
|
||||
"smallvec",
|
||||
"tokio",
|
||||
"tracing",
|
||||
"want",
|
||||
]
|
||||
|
||||
@@ -7017,11 +7009,10 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nixl-sys"
|
||||
version = "1.5.2"
|
||||
source = "git+https://github.com/ai-dynamo/nixl?tag=v1.5.2#e72c52c78478d2e1ca9e9c57b872114e113ad767"
|
||||
version = "1.6.0"
|
||||
source = "git+https://github.com/houseme/nixl?rev=cc636c83f13d692e85b67e55aed092413d16fb41#cc636c83f13d692e85b67e55aed092413d16fb41"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"bincode",
|
||||
"bindgen 0.71.1",
|
||||
"cc",
|
||||
"libc",
|
||||
@@ -7030,6 +7021,7 @@ dependencies = [
|
||||
"serde",
|
||||
"thiserror 2.0.21",
|
||||
"tracing",
|
||||
"wincode",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7081,7 +7073,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -7974,6 +7966,12 @@ dependencies = [
|
||||
"phc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pastey"
|
||||
version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2ee67f1008b1ba2321834326597b8e186293b049a023cdef258527550b9935b4"
|
||||
|
||||
[[package]]
|
||||
name = "path-absolutize"
|
||||
version = "4.0.1"
|
||||
@@ -9028,7 +9026,7 @@ dependencies = [
|
||||
"once_cell",
|
||||
"socket2",
|
||||
"tracing",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11030,6 +11028,8 @@ version = "1.0.5"
|
||||
dependencies = [
|
||||
"flatbuffers",
|
||||
"hotpath",
|
||||
"hyper",
|
||||
"hyper-util",
|
||||
"prost 0.14.6",
|
||||
"rcgen",
|
||||
"rmp-serde",
|
||||
@@ -11048,6 +11048,7 @@ dependencies = [
|
||||
"tonic-prost",
|
||||
"tonic-prost-build",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
@@ -11640,7 +11641,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11722,7 +11723,7 @@ dependencies = [
|
||||
"security-framework",
|
||||
"security-framework-sys",
|
||||
"webpki-root-certs",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12539,7 +12540,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d1e2c7f27f8d4cb10542a02c49005dbd6e93095799d6f3be745fae9f8fedd4"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12690,7 +12691,7 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"libc",
|
||||
"psm",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12998,10 +12999,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.3",
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14116,7 +14117,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -14125,6 +14126,31 @@ version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "wincode"
|
||||
version = "0.6.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "063b5affe63fb4e584a33101d8808f01f2348b4f533db6223bf27263a0fdb9e9"
|
||||
dependencies = [
|
||||
"pastey",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"thiserror 2.0.21",
|
||||
"wincode-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wincode-derive"
|
||||
version = "0.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "11d582d9fc9d813264407a9e16bfa16846ccf15ef032f9bbcd700741bdc00255"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn 2.0.119",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.62.2"
|
||||
|
||||
@@ -313,6 +313,7 @@ libsystemd = "0.7.2"
|
||||
local-ip-address = "0.6.13"
|
||||
log = "0.4"
|
||||
memmap2 = "0.9.12"
|
||||
nixl-sys = { git = "https://github.com/houseme/nixl", rev = "cc636c83f13d692e85b67e55aed092413d16fb41" }
|
||||
lz4 = "1.28.1"
|
||||
lz4_flex = { version = "=0.14.0", default-features = false, features = ["std", "frame"] }
|
||||
matchit = "0.9.2"
|
||||
|
||||
@@ -23,7 +23,80 @@ pub static GLOBAL_LOCAL_NODE_NAME: LazyLock<RwLock<String>> = LazyLock::new(|| R
|
||||
pub static GLOBAL_RUSTFS_HOST: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
pub static GLOBAL_RUSTFS_PORT: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("9000".to_string()));
|
||||
pub static GLOBAL_RUSTFS_ADDR: LazyLock<RwLock<String>> = LazyLock::new(|| RwLock::new("".to_string()));
|
||||
static NEXT_GRPC_CHANNEL_ID: AtomicU64 = AtomicU64::new(1);
|
||||
static NEXT_GRPC_CONNECTION_GENERATION: AtomicU64 = AtomicU64::new(1);
|
||||
|
||||
#[derive(Debug)]
|
||||
struct GrpcConnectionStateInner {
|
||||
channel_id: u64,
|
||||
generation: AtomicU64,
|
||||
}
|
||||
|
||||
/// Lock-free identity and physical-connection generation shared by every clone of one gRPC channel.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct GrpcConnectionState(Arc<GrpcConnectionStateInner>);
|
||||
|
||||
/// A point-in-time identity for the physical connection currently used by a gRPC channel.
|
||||
#[derive(Clone, Copy, Debug, Eq, Hash, PartialEq)]
|
||||
pub struct GrpcConnectionSnapshot {
|
||||
pub channel_id: u64,
|
||||
pub generation: u64,
|
||||
}
|
||||
|
||||
impl GrpcConnectionState {
|
||||
pub fn new() -> Self {
|
||||
Self(Arc::new(GrpcConnectionStateInner {
|
||||
channel_id: next_nonzero_id(&NEXT_GRPC_CHANNEL_ID),
|
||||
generation: AtomicU64::new(0),
|
||||
}))
|
||||
}
|
||||
|
||||
/// Publish a successfully opened physical transport before tonic makes it request-ready.
|
||||
pub fn record_connected(&self) -> GrpcConnectionSnapshot {
|
||||
let generation = next_nonzero_id(&NEXT_GRPC_CONNECTION_GENERATION);
|
||||
self.0.generation.store(generation, Ordering::Release);
|
||||
GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
}
|
||||
}
|
||||
|
||||
/// Return `None` until the first physical transport has connected successfully.
|
||||
pub fn snapshot(&self) -> Option<GrpcConnectionSnapshot> {
|
||||
let generation = self.0.generation.load(Ordering::Acquire);
|
||||
(generation != 0).then_some(GrpcConnectionSnapshot {
|
||||
channel_id: self.0.channel_id,
|
||||
generation,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for GrpcConnectionState {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
fn next_nonzero_id(counter: &AtomicU64) -> u64 {
|
||||
loop {
|
||||
let id = counter.fetch_add(1, Ordering::Relaxed);
|
||||
if id != 0 {
|
||||
return id;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub static GLOBAL_CONN_MAP: LazyLock<RwLock<HashMap<String, Channel>>> = LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TrackedGrpcConnection {
|
||||
channel: Channel,
|
||||
connection_state: GrpcConnectionState,
|
||||
}
|
||||
|
||||
static GLOBAL_TRACKED_CONN_MAP: LazyLock<RwLock<HashMap<String, TrackedGrpcConnection>>> =
|
||||
LazyLock::new(|| RwLock::new(HashMap::new()));
|
||||
static GLOBAL_CONN_CACHE_WRITER: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
pub static GLOBAL_ROOT_CERT: LazyLock<RwLock<Option<Vec<u8>>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_MTLS_IDENTITY: LazyLock<RwLock<Option<MtlsIdentityPem>>> = LazyLock::new(|| RwLock::new(None));
|
||||
pub static GLOBAL_OUTBOUND_TLS_GENERATION: LazyLock<AtomicU64> = LazyLock::new(|| AtomicU64::new(0));
|
||||
@@ -231,6 +304,16 @@ pub async fn evict_connection(addr: &str) {
|
||||
|
||||
/// Evict a stale/dead connection from the global connection cache with an explicit log level.
|
||||
pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) {
|
||||
let _ = remove_connection_with_log_level(addr, log_level).await;
|
||||
}
|
||||
|
||||
/// Remove a cached connection and report whether it existed. Transport owners use this to
|
||||
/// attach a rebuild reason only to real cache transitions.
|
||||
pub async fn remove_connection_with_log_level(addr: &str, log_level: ConnectionEvictionLogLevel) -> bool {
|
||||
// Writes are rare and serialized so the optimized tracked cache and public compatibility map
|
||||
// cannot publish different channel clones for one key.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(addr);
|
||||
let removed = GLOBAL_CONN_MAP.write().await.remove(addr);
|
||||
if removed.is_some() {
|
||||
match log_level {
|
||||
@@ -254,6 +337,7 @@ pub async fn evict_connection_with_log_level(addr: &str, log_level: ConnectionEv
|
||||
}
|
||||
}
|
||||
}
|
||||
removed.is_some()
|
||||
}
|
||||
|
||||
/// Get a cached gRPC connection for the given address.
|
||||
@@ -261,8 +345,37 @@ pub async fn cached_connection(addr: &str) -> Option<Channel> {
|
||||
GLOBAL_CONN_MAP.read().await.get(addr).cloned()
|
||||
}
|
||||
|
||||
/// Get a cached channel together with its shared physical-connection state, when tracked.
|
||||
pub async fn cached_connection_with_state(addr: &str) -> Option<(Channel, Option<GrpcConnectionState>)> {
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
// A miss may race a low-frequency publish/evict. Serialize only this fallback, recheck the
|
||||
// tracked map, then consult legacy entries injected through the compatibility API.
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
if let Some(cached) = GLOBAL_TRACKED_CONN_MAP.read().await.get(addr).cloned() {
|
||||
return Some((cached.channel, Some(cached.connection_state)));
|
||||
}
|
||||
cached_connection(addr).await.map(|channel| (channel, None))
|
||||
}
|
||||
|
||||
/// Cache a gRPC connection for the given address.
|
||||
pub async fn cache_connection(addr: String, channel: Channel) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.remove(&addr);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
/// Cache a gRPC channel and its physical-connection state as one coherent entry.
|
||||
pub async fn cache_connection_with_state(addr: String, channel: Channel, connection_state: GrpcConnectionState) {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.insert(
|
||||
addr.clone(),
|
||||
TrackedGrpcConnection {
|
||||
channel: channel.clone(),
|
||||
connection_state,
|
||||
},
|
||||
);
|
||||
GLOBAL_CONN_MAP.write().await.insert(addr, channel);
|
||||
}
|
||||
|
||||
@@ -279,6 +392,8 @@ pub async fn has_cached_connection(addr: &str) -> bool {
|
||||
|
||||
/// Clear all cached connections. Useful for full cluster reset/recovery.
|
||||
pub async fn clear_all_connections() {
|
||||
let _writer = GLOBAL_CONN_CACHE_WRITER.lock().await;
|
||||
GLOBAL_TRACKED_CONN_MAP.write().await.clear();
|
||||
let mut map = GLOBAL_CONN_MAP.write().await;
|
||||
let count = map.len();
|
||||
map.clear();
|
||||
@@ -363,4 +478,46 @@ mod tls_snapshot_tests {
|
||||
assert!(!debug.contains("PRIVATE-KEY-MARKER"));
|
||||
assert!(debug.contains("[REDACTED]"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn conditional_connection_removal_preserves_legacy_evict_api() {
|
||||
let key = "http://conditional-eviction-test.invalid";
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(!remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
let _: () = evict_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cached_connection_keeps_channel_and_transport_state_coherent() {
|
||||
let key = "http://tracked-connection-test.invalid";
|
||||
let channel = tonic::transport::Endpoint::from_static("http://127.0.0.1:1").connect_lazy();
|
||||
let state = GrpcConnectionState::new();
|
||||
assert!(state.snapshot().is_none());
|
||||
let first = state.record_connected();
|
||||
cache_connection_with_state(key.to_owned(), channel, state.clone()).await;
|
||||
|
||||
let (_, cached_state) = cached_connection_with_state(key).await.expect("tracked cache entry");
|
||||
let cached_state = cached_state.expect("tracked cache state");
|
||||
assert_eq!(cached_state.snapshot(), Some(first));
|
||||
let second = state.record_connected();
|
||||
assert_eq!(cached_state.snapshot(), Some(second));
|
||||
assert_eq!(first.channel_id, second.channel_id);
|
||||
assert_ne!(first.generation, second.generation);
|
||||
|
||||
cache_connection(
|
||||
key.to_owned(),
|
||||
tonic::transport::Endpoint::from_static("http://127.0.0.1:2").connect_lazy(),
|
||||
)
|
||||
.await;
|
||||
let (_, replacement_state) = cached_connection_with_state(key).await.expect("legacy replacement entry");
|
||||
assert!(replacement_state.is_none(), "legacy replacement must remove stale tracked state");
|
||||
|
||||
assert!(remove_connection_with_log_level(key, ConnectionEvictionLogLevel::Debug).await);
|
||||
assert!(cached_connection_with_state(key).await.is_none());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,9 +121,17 @@ pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY: bool = false;
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: bool = false;
|
||||
|
||||
/// Permit current per-peer capability proof to replace fleet-wide confirmation.
|
||||
///
|
||||
/// This remains ineffective unless [`ENV_INTERNODE_RPC_MSGPACK_ONLY`] is also enabled. Unknown,
|
||||
/// unsupported, revoked, or stale connection generations keep dual-writing JSON.
|
||||
pub const ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: &str = "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY";
|
||||
pub const DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: bool = false;
|
||||
|
||||
// Compile-time invariants: dual-write by default so the base build is byte-for-byte legacy behavior.
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED);
|
||||
const _: () = assert!(!DEFAULT_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY);
|
||||
|
||||
/// Require target-bound v2 signatures on every internode gRPC request, rejecting the legacy
|
||||
/// constant-target fallback instead of accepting it (<https://github.com/rustfs/backlog/issues/1327>).
|
||||
@@ -294,6 +302,14 @@ pub const DEFAULT_INTERNODE_BULK_CONCURRENCY_LIMIT: usize = 64;
|
||||
pub const ENV_INTERNODE_BULK_BUFFER_SIZE: &str = "RUSTFS_INTERNODE_BULK_BUFFER_SIZE";
|
||||
pub const DEFAULT_INTERNODE_BULK_BUFFER_SIZE: usize = 1024;
|
||||
|
||||
/// Per-peer limit held through complete bulk-unary response decoding and validation. Zero keeps
|
||||
/// the compatibility behavior with no additional admission gate.
|
||||
pub const ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: &str = "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT: usize = 0;
|
||||
pub const ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: &str = "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS";
|
||||
pub const DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS: u64 = 1_000;
|
||||
const _: () = assert!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT == 0);
|
||||
|
||||
/// Profile selector for conservative internode HTTP data-plane client tuning.
|
||||
pub const ENV_INTERNODE_HTTP_TUNING_PROFILE: &str = "RUSTFS_INTERNODE_HTTP_TUNING_PROFILE";
|
||||
pub const DEFAULT_INTERNODE_HTTP_TUNING_PROFILE: &str = "legacy";
|
||||
@@ -501,8 +517,12 @@ mod tests {
|
||||
fn internode_channel_isolation_defaults_and_env_names() {
|
||||
// The isolation-off default is asserted at compile time next to the constant definition.
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_CHANNELS, 2);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, 0);
|
||||
assert_eq!(DEFAULT_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, 1_000);
|
||||
assert_eq!(ENV_INTERNODE_CHANNEL_ISOLATION, "RUSTFS_INTERNODE_CHANNEL_ISOLATION");
|
||||
assert_eq!(ENV_INTERNODE_BULK_CHANNELS, "RUSTFS_INTERNODE_BULK_CHANNELS");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT, "RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT");
|
||||
assert_eq!(ENV_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS, "RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -513,6 +533,7 @@ mod tests {
|
||||
ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED,
|
||||
"RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED"
|
||||
);
|
||||
assert_eq!(ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY, "RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -708,13 +708,14 @@ pub mod rpc {
|
||||
sign_ns_scanner_capability_with_checkpoint_coverage, sign_ns_scanner_capability_with_durable_progress,
|
||||
sign_ns_scanner_capability_with_tier_registry_generation, sign_ns_scanner_capability_with_usage_capabilities,
|
||||
sign_ns_scanner_usage_watch_capability, sign_put_file_capability, sign_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability,
|
||||
verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_response_proof, verify_tonic_rpc_signature, verify_tonic_rpc_signature_with_bootstrap,
|
||||
tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_boot_epoch_response_headers_with_msgpack_capability,
|
||||
tonic_msgpack_manifest_capability_requested, tonic_rpc_auth_failure_reason, verify_ns_scanner_capability,
|
||||
verify_ns_scanner_capability_with_checkpoint_coverage, verify_ns_scanner_capability_with_durable_progress,
|
||||
verify_ns_scanner_capability_with_tier_registry_generation, verify_ns_scanner_capability_with_usage_capabilities,
|
||||
verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer, verify_put_file_capability, verify_rpc_signature,
|
||||
verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||
verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_response_proof, verify_tonic_rpc_signature,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -78,6 +78,9 @@ pub const RPC_BOOT_EPOCH_CHALLENGE_HEADER: &str = "x-rustfs-rpc-boot-epoch-chall
|
||||
pub const RPC_BOOT_EPOCH_PROOF_HEADER: &str = "x-rustfs-rpc-boot-epoch-proof";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_HEADER: &str = "x-rustfs-rpc-replay-cache-capability";
|
||||
pub(crate) const RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-replay-cache-capability-proof";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-request";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-capability";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER: &str = "x-rustfs-rpc-msgpack-manifest-capability-proof";
|
||||
const RPC_REPLAY_SCOPE_VERSION_V3: &str = "3";
|
||||
const RPC_RESPONSE_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-response-proof-v1\0";
|
||||
|
||||
@@ -124,6 +127,8 @@ const RPC_REPLAY_SCOPE_DOMAIN: &[u8] = b"rustfs-rpc-replay-scope-v3\0";
|
||||
const RPC_BOOT_EPOCH_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-boot-epoch-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-replay-cache-capability-proof-v1\0";
|
||||
const RPC_REPLAY_CACHE_CAPABILITY_V1: &str = "dynamic-replay-cache-v1";
|
||||
const RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_DOMAIN: &[u8] = b"rustfs-rpc-msgpack-manifest-capability-proof-v1\0";
|
||||
pub(crate) const RPC_MSGPACK_MANIFEST_CAPABILITY_V1: &str = "msgpack-compat-manifest-v1";
|
||||
const HTTP_PUT_FILE_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-auth-v1\0";
|
||||
const HTTP_PUT_FILE_CAPABILITY_AUTH_DOMAIN: &[u8] = b"rustfs-http-put-file-capability-v1\0";
|
||||
const UNSIGNED_PAYLOAD: &str = "UNSIGNED-PAYLOAD";
|
||||
@@ -1238,6 +1243,50 @@ fn verify_replay_cache_capability_proof(
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC replay cache capability proof"))
|
||||
}
|
||||
|
||||
fn update_msgpack_manifest_capability_proof(mac: &mut HmacSha256, audience: &str, challenge: Uuid, boot_epoch: Uuid) {
|
||||
mac.update(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_DOMAIN);
|
||||
for part in [
|
||||
audience.as_bytes(),
|
||||
b"|",
|
||||
challenge.as_bytes(),
|
||||
b"|",
|
||||
boot_epoch.as_bytes(),
|
||||
b"|",
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_V1.as_bytes(),
|
||||
] {
|
||||
mac.update(part);
|
||||
}
|
||||
}
|
||||
|
||||
fn generate_msgpack_manifest_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
) -> std::io::Result<String> {
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_msgpack_manifest_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
Ok(base64_simd::STANDARD.encode_to_string(mac.finalize().into_bytes()))
|
||||
}
|
||||
|
||||
fn verify_msgpack_manifest_capability_proof(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
proof: &str,
|
||||
) -> std::io::Result<()> {
|
||||
let proof = base64_simd::STANDARD
|
||||
.decode_to_vec(proof)
|
||||
.map_err(|_| std::io::Error::other("Invalid RPC msgpack manifest capability proof"))?;
|
||||
let mut mac =
|
||||
<HmacSha256 as KeyInit>::new_from_slice(secret.as_bytes()).map_err(|_| std::io::Error::other("Invalid RPC HMAC key"))?;
|
||||
update_msgpack_manifest_capability_proof(&mut mac, audience, challenge, boot_epoch);
|
||||
mac.verify_slice(&proof)
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::PermissionDenied, "Invalid RPC msgpack manifest capability proof"))
|
||||
}
|
||||
|
||||
fn non_nil_uuid(value: &str, name: &str) -> std::io::Result<Uuid> {
|
||||
let value = Uuid::parse_str(value).map_err(|_| std::io::Error::other(format!("Invalid {name}")))?;
|
||||
(!value.is_nil())
|
||||
@@ -1337,8 +1386,24 @@ pub fn tonic_boot_epoch_challenge(headers: &HeaderMap) -> std::io::Result<Option
|
||||
.transpose()
|
||||
}
|
||||
|
||||
/// Whether the request opts into the optional msgpack manifest capability proof.
|
||||
pub fn tonic_msgpack_manifest_capability_requested(headers: &HeaderMap) -> bool {
|
||||
headers
|
||||
.get(RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER)
|
||||
.is_some_and(|value| value == RPC_MSGPACK_MANIFEST_CAPABILITY_V1)
|
||||
}
|
||||
|
||||
/// Build the authenticated response headers for a client boot-epoch challenge.
|
||||
pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std::io::Result<HeaderMap> {
|
||||
tonic_boot_epoch_response_headers_with_msgpack_capability(audience, challenge, false)
|
||||
}
|
||||
|
||||
/// Build authenticated boot-epoch headers and, when requested, the msgpack manifest capability.
|
||||
pub fn tonic_boot_epoch_response_headers_with_msgpack_capability(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
include_msgpack_manifest: bool,
|
||||
) -> std::io::Result<HeaderMap> {
|
||||
let boot_epoch = tonic_rpc_boot_epoch();
|
||||
let secret = get_shared_secret()?;
|
||||
let proof = generate_boot_epoch_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
@@ -1354,6 +1419,17 @@ pub fn tonic_boot_epoch_response_headers(audience: &str, challenge: Uuid) -> std
|
||||
RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER,
|
||||
header_value(&capability_proof, RPC_REPLAY_CACHE_CAPABILITY_PROOF_HEADER)?,
|
||||
);
|
||||
if include_msgpack_manifest {
|
||||
let msgpack_capability_proof = generate_msgpack_manifest_capability_proof(&secret, audience, challenge, boot_epoch)?;
|
||||
headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER,
|
||||
HeaderValue::from_static(RPC_MSGPACK_MANIFEST_CAPABILITY_V1),
|
||||
);
|
||||
headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER,
|
||||
header_value(&msgpack_capability_proof, RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER)?,
|
||||
);
|
||||
}
|
||||
Ok(headers)
|
||||
}
|
||||
|
||||
@@ -1387,6 +1463,12 @@ pub(crate) struct AuthenticatedPeerReplayCapabilities {
|
||||
pub(crate) dynamic_replay_cache: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) struct AuthenticatedPeerMsgpackCapabilities {
|
||||
pub(crate) boot_epoch: Uuid,
|
||||
pub(crate) msgpack_compat_manifest: bool,
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_peer_replay_capabilities_response(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
@@ -1418,6 +1500,57 @@ pub(crate) fn verify_tonic_peer_replay_capabilities_response(
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn verify_tonic_peer_msgpack_capabilities_response(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
let secret = get_shared_secret()?;
|
||||
let boot_epoch = verify_tonic_boot_epoch_response_with_secret(&secret, audience, challenge, headers)?;
|
||||
verify_tonic_peer_msgpack_capability_for_epoch_with_secret(&secret, audience, challenge, boot_epoch, headers)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_peer_msgpack_capability_for_authenticated_epoch(
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
verify_tonic_peer_msgpack_capability_for_epoch_with_secret(&get_shared_secret()?, audience, challenge, boot_epoch, headers)
|
||||
}
|
||||
|
||||
fn verify_tonic_peer_msgpack_capability_for_epoch_with_secret(
|
||||
secret: &str,
|
||||
audience: &str,
|
||||
challenge: Uuid,
|
||||
boot_epoch: Uuid,
|
||||
headers: &HeaderMap,
|
||||
) -> std::io::Result<AuthenticatedPeerMsgpackCapabilities> {
|
||||
let capability = headers.get(RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER);
|
||||
let proof = headers.get(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER);
|
||||
if capability.is_none() && proof.is_none() {
|
||||
return Ok(AuthenticatedPeerMsgpackCapabilities {
|
||||
boot_epoch,
|
||||
msgpack_compat_manifest: false,
|
||||
});
|
||||
}
|
||||
let capability = capability
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC msgpack manifest capability"))?;
|
||||
if capability != RPC_MSGPACK_MANIFEST_CAPABILITY_V1 {
|
||||
return Err(std::io::Error::other("Unsupported RPC msgpack manifest capability"));
|
||||
}
|
||||
let proof = proof
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.ok_or_else(|| std::io::Error::other("Missing RPC msgpack manifest capability proof"))?;
|
||||
verify_msgpack_manifest_capability_proof(secret, audience, challenge, boot_epoch, proof)?;
|
||||
Ok(AuthenticatedPeerMsgpackCapabilities {
|
||||
boot_epoch,
|
||||
msgpack_compat_manifest: true,
|
||||
})
|
||||
}
|
||||
|
||||
fn valid_content_sha256(value: &str) -> bool {
|
||||
value == UNSIGNED_PAYLOAD
|
||||
|| (value.len() == 64
|
||||
@@ -3147,6 +3280,55 @@ mod tests {
|
||||
assert!(verify_tonic_peer_replay_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_manifest_capability_is_requested_explicitly_and_proved_for_one_epoch() {
|
||||
ensure_test_rpc_secret();
|
||||
let challenge = Uuid::new_v4();
|
||||
let mut request_headers = HeaderMap::new();
|
||||
assert!(!tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
request_headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER,
|
||||
HeaderValue::from_static(RPC_MSGPACK_MANIFEST_CAPABILITY_V1),
|
||||
);
|
||||
assert!(tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
request_headers.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_REQUEST_HEADER,
|
||||
HeaderValue::from_static("msgpack-compat-manifest-v2"),
|
||||
);
|
||||
assert!(!tonic_msgpack_manifest_capability_requested(&request_headers));
|
||||
|
||||
let legacy_headers = tonic_boot_epoch_response_headers("node-a:9000", challenge).expect("legacy capability headers");
|
||||
let legacy = verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &legacy_headers)
|
||||
.expect("authenticated response without msgpack capability");
|
||||
assert_eq!(legacy.boot_epoch, tonic_rpc_boot_epoch());
|
||||
assert!(!legacy.msgpack_compat_manifest);
|
||||
|
||||
let headers = tonic_boot_epoch_response_headers_with_msgpack_capability("node-a:9000", challenge, true)
|
||||
.expect("msgpack capability headers");
|
||||
let capability = verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &headers)
|
||||
.expect("matching msgpack capability proof");
|
||||
assert_eq!(capability.boot_epoch, tonic_rpc_boot_epoch());
|
||||
assert!(capability.msgpack_compat_manifest);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-b:9000", challenge, &headers).is_err());
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", Uuid::new_v4(), &headers).is_err());
|
||||
|
||||
let mut changed_capability = headers.clone();
|
||||
changed_capability.insert(
|
||||
RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER,
|
||||
HeaderValue::from_static("msgpack-compat-manifest-v2"),
|
||||
);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &changed_capability).is_err());
|
||||
let mut forged_proof = headers;
|
||||
forged_proof.insert(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER, HeaderValue::from_static("forged"));
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &forged_proof).is_err());
|
||||
let mut missing_proof = forged_proof.clone();
|
||||
missing_proof.remove(RPC_MSGPACK_MANIFEST_CAPABILITY_PROOF_HEADER);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &missing_proof).is_err());
|
||||
let mut missing_value = forged_proof;
|
||||
missing_value.remove(RPC_MSGPACK_MANIFEST_CAPABILITY_HEADER);
|
||||
assert!(verify_tonic_peer_msgpack_capabilities_response("node-a:9000", challenge, &missing_value).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn tonic_rpc_auth_failure_reason_maps_security_relevant_errors() {
|
||||
for (message, reason) in [
|
||||
|
||||
@@ -43,8 +43,9 @@ pub use http_auth::{
|
||||
set_tonic_rolling_mutation_body_digest, sign_ns_scanner_capability, sign_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_ns_scanner_capability_with_durable_progress, sign_ns_scanner_capability_with_tier_registry_generation,
|
||||
sign_ns_scanner_capability_with_usage_capabilities, sign_ns_scanner_usage_watch_capability, sign_put_file_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason,
|
||||
verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_boot_epoch_response_headers_with_msgpack_capability, tonic_msgpack_manifest_capability_requested,
|
||||
tonic_rpc_auth_failure_reason, verify_ns_scanner_capability, verify_ns_scanner_capability_with_checkpoint_coverage,
|
||||
verify_ns_scanner_capability_with_durable_progress, verify_ns_scanner_capability_with_tier_registry_generation,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_ns_scanner_usage_watch_capability, verify_put_file_auth_trailer,
|
||||
verify_put_file_capability, verify_rpc_signature, verify_tonic_boot_epoch_response, verify_tonic_canonical_body_digest,
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -29,7 +29,7 @@ use rustfs_protos::proto_gen::node_service::{
|
||||
LockAcquireStatus, LockReleaseStatus, PingRequest,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
ConnectionEvictionLogLevel, evict_failed_connection_with_log_level, models::PingBodyBuilder,
|
||||
ChannelRebuildReason, ConnectionEvictionLogLevel, evict_failed_connection_with_reason, models::PingBodyBuilder,
|
||||
proto_gen::node_service::node_service_client::NodeServiceClient,
|
||||
};
|
||||
use std::collections::HashMap;
|
||||
@@ -498,7 +498,7 @@ impl RemoteClient {
|
||||
))
|
||||
}
|
||||
|
||||
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str) {
|
||||
async fn evict_connection(&self, op: &'static str, reason: &str, resource_summary: &str, trigger: EvictionTrigger) {
|
||||
let log_level = if Self::is_scanner_leader_lock(resource_summary) {
|
||||
debug!(
|
||||
addr = %self.addr,
|
||||
@@ -518,7 +518,11 @@ impl RemoteClient {
|
||||
);
|
||||
ConnectionEvictionLogLevel::Warn
|
||||
};
|
||||
evict_failed_connection_with_log_level(&self.addr, log_level).await;
|
||||
let rebuild_reason = match trigger {
|
||||
EvictionTrigger::Timeout => ChannelRebuildReason::LockTimeout,
|
||||
EvictionTrigger::Transport => ChannelRebuildReason::LockTransport,
|
||||
};
|
||||
evict_failed_connection_with_reason(&self.addr, rebuild_reason, log_level).await;
|
||||
}
|
||||
|
||||
fn summarize_resources(requests: &[LockRequest]) -> String {
|
||||
@@ -772,7 +776,7 @@ impl RemoteClient {
|
||||
});
|
||||
if verdict == EvictionVerdict::Evict {
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_channel_eviction(&self.addr, trigger.as_str());
|
||||
self.evict_connection(op, reason, resource_summary).await;
|
||||
self.evict_connection(op, reason, resource_summary, trigger).await;
|
||||
return;
|
||||
}
|
||||
rustfs_io_metrics::lock_metrics::record_remote_lock_channel_eviction_suppressed(&self.addr, verdict.as_str());
|
||||
|
||||
@@ -30,6 +30,7 @@ harness = false
|
||||
|
||||
[features]
|
||||
default = []
|
||||
diagnostic-internode-http2-ping = []
|
||||
diagnostic-read-version-caller = []
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
|
||||
@@ -433,6 +433,13 @@ const INTERNODE_OPERATION_DURATION_MS: &str = "rustfs_system_network_internode_o
|
||||
const INTERNODE_OPERATION_SERVER_DURATION_MS: &str = "rustfs_system_network_internode_operation_server_duration_ms";
|
||||
const INTERNODE_RPC_ACTIVE: &str = "rustfs_system_network_internode_rpc_active";
|
||||
const INTERNODE_RPC_COMPLETIONS_TOTAL: &str = "rustfs_system_network_internode_rpc_completions_total";
|
||||
const INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL: &str = "rustfs_system_network_internode_channel_rebuild_attempts_total";
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
const INTERNODE_HTTP2_PING_TIMEOUTS_TOTAL: &str = "rustfs_system_network_internode_http2_ping_timeouts_total";
|
||||
const INTERNODE_BULK_BODY_WAIT_DURATION_MS: &str = "rustfs_system_network_internode_bulk_body_wait_duration_ms";
|
||||
const INTERNODE_BULK_BODY_WAITERS: &str = "rustfs_system_network_internode_bulk_body_waiters";
|
||||
const INTERNODE_BULK_BODY_ACTIVE: &str = "rustfs_system_network_internode_bulk_body_active";
|
||||
const INTERNODE_BULK_BODY_ADMISSIONS_TOTAL: &str = "rustfs_system_network_internode_bulk_body_admissions_total";
|
||||
const INTERNODE_OPERATION_STAGE_DURATION_MS: &str = "rustfs_system_network_internode_operation_stage_duration_ms";
|
||||
const INTERNODE_OPERATION_CLASSIFIED_ERRORS_TOTAL: &str = "rustfs_system_network_internode_operation_classified_errors_total";
|
||||
const INTERNODE_OPERATION_RETRIES_TOTAL: &str = "rustfs_system_network_internode_operation_retries_total";
|
||||
@@ -447,8 +454,26 @@ const INTERNODE_OPERATION_LARGE_PAYLOADS_TOTAL: &str = "rustfs_system_network_in
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_total";
|
||||
const INTERNODE_MSGPACK_JSON_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_fallback_total";
|
||||
const INTERNODE_MSGPACK_JSON_DECODE_ERROR_TOTAL: &str = "rustfs_system_network_internode_msgpack_json_decode_error_total";
|
||||
const INTERNODE_MSGPACK_SEND_DECISIONS_TOTAL: &str = "rustfs_system_network_internode_msgpack_send_decisions_total";
|
||||
const INTERNODE_SIGNATURE_V1_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_signature_v1_fallback_total";
|
||||
const INTERNODE_BODY_DIGEST_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_body_digest_fallback_total";
|
||||
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
tokio::task_local! {
|
||||
static INTERNODE_HTTP2_CLIENT_TASK: ();
|
||||
}
|
||||
|
||||
/// Scope a tonic/Hyper background task as an internode HTTP/2 client task.
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub async fn scope_internode_http2_client_task<T>(future: impl std::future::Future<Output = T>) -> T {
|
||||
INTERNODE_HTTP2_CLIENT_TASK.scope((), future).await
|
||||
}
|
||||
|
||||
/// Whether the current task belongs to an internode tonic/Hyper client.
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub fn is_internode_http2_client_task() -> bool {
|
||||
INTERNODE_HTTP2_CLIENT_TASK.try_with(|()| ()).is_ok()
|
||||
}
|
||||
const INTERNODE_REPLAY_SCOPE_FALLBACK_TOTAL: &str = "rustfs_system_network_internode_replay_scope_fallback_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_TOTAL: &str = "rustfs_system_network_internode_replay_cache_overflow_total";
|
||||
const INTERNODE_REPLAY_CACHE_OVERFLOW_BY_OPERATION_TOTAL: &str =
|
||||
@@ -468,6 +493,8 @@ pub struct InternodeOperationMetricDescriptor {
|
||||
const SERVER_OPERATION_BACKEND_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_SIDE_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, "side"];
|
||||
const SERVER_OPERATION_BACKEND_OUTCOME_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, "outcome"];
|
||||
const SERVER_CLASS_REASON_OUTCOME_LABELS: &[&str] = &[SERVER_LABEL, "class", REASON_LABEL, "outcome"];
|
||||
const SERVER_OUTCOME_LABELS: &[&str] = &[SERVER_LABEL, "outcome"];
|
||||
const SERVER_OPERATION_BACKEND_CLASSIFICATION_LABELS: &[&str] =
|
||||
&[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, CLASSIFICATION_LABEL];
|
||||
const SERVER_OPERATION_BACKEND_HTTP_VERSION_LABELS: &[&str] = &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL, HTTP_VERSION_LABEL];
|
||||
@@ -580,6 +607,26 @@ pub const INTERNODE_OPERATION_METRICS: &[InternodeOperationMetricDescriptor] = &
|
||||
name: INTERNODE_RPC_COMPLETIONS_TOTAL,
|
||||
labels: SERVER_OPERATION_BACKEND_OUTCOME_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL,
|
||||
labels: SERVER_CLASS_REASON_OUTCOME_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_WAIT_DURATION_MS,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_WAITERS,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_ACTIVE,
|
||||
labels: SERVER_LABELS,
|
||||
},
|
||||
InternodeOperationMetricDescriptor {
|
||||
name: INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
labels: SERVER_OUTCOME_LABELS,
|
||||
},
|
||||
];
|
||||
|
||||
static STABLE_SERVER_LABEL: OnceLock<String> = OnceLock::new();
|
||||
@@ -752,6 +799,169 @@ pub async fn measure_client_rpc<T, E>(
|
||||
result
|
||||
}
|
||||
|
||||
/// Record one attempt to publish a replacement after a bounded cached-channel eviction.
|
||||
pub fn record_channel_rebuild_attempt(class: &'static str, reason: &'static str, connected: bool) {
|
||||
if !crate::metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
let class = match class {
|
||||
"control" | "bulk" | "recovery" => class,
|
||||
_ => "unknown",
|
||||
};
|
||||
let reason = match reason {
|
||||
"rpc_failure"
|
||||
| "operation_timeout"
|
||||
| "transport_failure"
|
||||
| "lock_timeout"
|
||||
| "lock_transport"
|
||||
| "peer_probe"
|
||||
| "tls_generation_changed"
|
||||
| "manual" => reason,
|
||||
_ => "unknown",
|
||||
};
|
||||
metrics::counter!(
|
||||
INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
"class" => class,
|
||||
REASON_LABEL => reason,
|
||||
"outcome" => if connected { "connected" } else { "failed_or_cancelled" }
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record one Hyper-confirmed HTTP/2 keepalive PING ACK timeout.
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub fn record_http2_ping_timeout() {
|
||||
if !crate::metrics_enabled() {
|
||||
return;
|
||||
}
|
||||
metrics::counter!(
|
||||
INTERNODE_HTTP2_PING_TIMEOUTS_TOTAL,
|
||||
SERVER_LABEL => current_server_label()
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct BulkBodyAdmissionMetricHandles {
|
||||
wait_duration: metrics::Histogram,
|
||||
waiters: metrics::Gauge,
|
||||
active: metrics::Gauge,
|
||||
admitted: metrics::Counter,
|
||||
cancelled: metrics::Counter,
|
||||
timed_out: metrics::Counter,
|
||||
}
|
||||
|
||||
impl BulkBodyAdmissionMetricHandles {
|
||||
fn new() -> Self {
|
||||
let server = current_server_label();
|
||||
Self {
|
||||
wait_duration: metrics::histogram!(INTERNODE_BULK_BODY_WAIT_DURATION_MS, SERVER_LABEL => server),
|
||||
waiters: metrics::gauge!(INTERNODE_BULK_BODY_WAITERS, SERVER_LABEL => server),
|
||||
active: metrics::gauge!(INTERNODE_BULK_BODY_ACTIVE, SERVER_LABEL => server),
|
||||
admitted: metrics::counter!(
|
||||
INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
"outcome" => "admitted"
|
||||
),
|
||||
cancelled: metrics::counter!(
|
||||
INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
"outcome" => "cancelled"
|
||||
),
|
||||
timed_out: metrics::counter!(
|
||||
INTERNODE_BULK_BODY_ADMISSIONS_TOTAL,
|
||||
SERVER_LABEL => server,
|
||||
"outcome" => "timeout"
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
static BULK_BODY_ADMISSION_METRICS: LazyLock<BulkBodyAdmissionMetricHandles> = LazyLock::new(BulkBodyAdmissionMetricHandles::new);
|
||||
|
||||
#[derive(Clone, Copy, Eq, PartialEq)]
|
||||
enum BulkBodyAdmissionState {
|
||||
Waiting,
|
||||
Active,
|
||||
Finished,
|
||||
}
|
||||
|
||||
/// Tracks one wait for complete-unary admission and, after admission, keeps the active gauge
|
||||
/// elevated until the decoded and validated response is released.
|
||||
pub struct InternodeBulkBodyAdmissionGuard {
|
||||
started_at: Option<Instant>,
|
||||
handles: Option<BulkBodyAdmissionMetricHandles>,
|
||||
state: BulkBodyAdmissionState,
|
||||
}
|
||||
|
||||
impl InternodeBulkBodyAdmissionGuard {
|
||||
pub fn waiting() -> Self {
|
||||
#[cfg(not(test))]
|
||||
let handles =
|
||||
(crate::metrics_enabled() && STABLE_SERVER_LABEL.get().is_some()).then(|| BULK_BODY_ADMISSION_METRICS.clone());
|
||||
#[cfg(test)]
|
||||
let handles = crate::metrics_enabled().then(BulkBodyAdmissionMetricHandles::new);
|
||||
if let Some(handles) = &handles {
|
||||
handles.waiters.increment(1.0);
|
||||
}
|
||||
Self {
|
||||
started_at: handles.as_ref().map(|_| Instant::now()),
|
||||
handles,
|
||||
state: BulkBodyAdmissionState::Waiting,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn admitted(&mut self) {
|
||||
if self.state == BulkBodyAdmissionState::Active {
|
||||
return;
|
||||
}
|
||||
if let Some(handles) = &self.handles {
|
||||
handles.waiters.decrement(1.0);
|
||||
handles.active.increment(1.0);
|
||||
handles.admitted.increment(1);
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
handles.wait_duration.record(started_at.elapsed().as_secs_f64() * 1000.0);
|
||||
}
|
||||
}
|
||||
self.state = BulkBodyAdmissionState::Active;
|
||||
}
|
||||
|
||||
pub fn timed_out(&mut self) {
|
||||
if self.state != BulkBodyAdmissionState::Waiting {
|
||||
return;
|
||||
}
|
||||
if let Some(handles) = &self.handles {
|
||||
handles.waiters.decrement(1.0);
|
||||
handles.timed_out.increment(1);
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
handles.wait_duration.record(started_at.elapsed().as_secs_f64() * 1000.0);
|
||||
}
|
||||
}
|
||||
self.state = BulkBodyAdmissionState::Finished;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for InternodeBulkBodyAdmissionGuard {
|
||||
fn drop(&mut self) {
|
||||
let Some(handles) = &self.handles else {
|
||||
return;
|
||||
};
|
||||
match self.state {
|
||||
BulkBodyAdmissionState::Waiting => {
|
||||
handles.waiters.decrement(1.0);
|
||||
handles.cancelled.increment(1);
|
||||
if let Some(started_at) = self.started_at.take() {
|
||||
handles.wait_duration.record(started_at.elapsed().as_secs_f64() * 1000.0);
|
||||
}
|
||||
}
|
||||
BulkBodyAdmissionState::Active => handles.active.decrement(1.0),
|
||||
BulkBodyAdmissionState::Finished => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(test))]
|
||||
struct GrpcReadVersionMetricHandles {
|
||||
sent_bytes: metrics::Counter,
|
||||
@@ -1395,6 +1605,25 @@ impl InternodeMetrics {
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
/// Record one eligible request's fixed-cardinality JSON/msgpack send decision.
|
||||
pub fn record_msgpack_send_decision(&self, decision: &'static str, reason: &'static str) {
|
||||
let decision = match decision {
|
||||
"msgpack_only" | "dual_write" => decision,
|
||||
_ => "dual_write",
|
||||
};
|
||||
let reason = match reason {
|
||||
"manual_fleet" | "peer_capable" | "peer_unknown" | "peer_revoked" | "not_requested" => reason,
|
||||
_ => "peer_unknown",
|
||||
};
|
||||
counter!(
|
||||
INTERNODE_MSGPACK_SEND_DECISIONS_TOTAL,
|
||||
SERVER_LABEL => current_server_label(),
|
||||
"decision" => decision,
|
||||
REASON_LABEL => reason
|
||||
)
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
#[doc(hidden)]
|
||||
pub fn msgpack_json_decode_error_total_for_test(&self) -> u64 {
|
||||
self.msgpack_json_decode_error_total.load(Ordering::Relaxed)
|
||||
@@ -1849,6 +2078,80 @@ mod tests {
|
||||
assert_eq!(active_sides, HashSet::from(["client".to_owned(), "server".to_owned()]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_lifecycle_metrics_use_bounded_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let previously_enabled = crate::metrics_enabled();
|
||||
crate::set_metrics_enabled(true);
|
||||
with_local_recorder(&recorder, || {
|
||||
record_channel_rebuild_attempt("bulk", "tls_generation_changed", true);
|
||||
record_channel_rebuild_attempt("control", "transport_failure", false);
|
||||
record_channel_rebuild_attempt("untrusted-class", "untrusted-reason", false);
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
record_http2_ping_timeout();
|
||||
});
|
||||
crate::set_metrics_enabled(previously_enabled);
|
||||
let entries = snapshotter.snapshot().into_vec();
|
||||
let rebuild = entries
|
||||
.iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL)
|
||||
.count();
|
||||
assert_eq!(rebuild, 3);
|
||||
assert!(entries.iter().any(|(key, _, _, _)| {
|
||||
key.key().name() == INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL
|
||||
&& key.key().labels().filter(|label| label.value() == "unknown").count() == 2
|
||||
}));
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
assert!(entries.iter().any(|(key, _, _, value)| {
|
||||
key.key().name() == INTERNODE_HTTP2_PING_TIMEOUTS_TOTAL && matches!(value, DebugValue::Counter(1))
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bulk_body_admission_metrics_balance_wait_cancel_and_active() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let previously_enabled = crate::metrics_enabled();
|
||||
crate::set_metrics_enabled(true);
|
||||
with_local_recorder(&recorder, || {
|
||||
drop(InternodeBulkBodyAdmissionGuard::waiting());
|
||||
let mut active = InternodeBulkBodyAdmissionGuard::waiting();
|
||||
active.admitted();
|
||||
drop(active);
|
||||
let mut timed_out = InternodeBulkBodyAdmissionGuard::waiting();
|
||||
timed_out.timed_out();
|
||||
drop(timed_out);
|
||||
});
|
||||
crate::set_metrics_enabled(previously_enabled);
|
||||
let entries = snapshotter.snapshot().into_vec();
|
||||
for name in [INTERNODE_BULK_BODY_WAITERS, INTERNODE_BULK_BODY_ACTIVE] {
|
||||
let value = entries
|
||||
.iter()
|
||||
.find(|(key, _, _, _)| key.key().name() == name)
|
||||
.map(|(_, _, _, value)| value)
|
||||
.expect("admission gauge");
|
||||
assert!(matches!(value, DebugValue::Gauge(value) if value.into_inner() == 0.0));
|
||||
}
|
||||
let outcomes = entries
|
||||
.iter()
|
||||
.filter(|(key, _, _, _)| key.key().name() == INTERNODE_BULK_BODY_ADMISSIONS_TOTAL)
|
||||
.map(|(key, _, _, value)| {
|
||||
assert!(matches!(value, DebugValue::Counter(1)));
|
||||
key.key()
|
||||
.labels()
|
||||
.find(|label| label.key() == "outcome")
|
||||
.expect("outcome")
|
||||
.value()
|
||||
.to_owned()
|
||||
})
|
||||
.collect::<HashSet<_>>();
|
||||
assert_eq!(
|
||||
outcomes,
|
||||
HashSet::from(["admitted".to_owned(), "cancelled".to_owned(), "timeout".to_owned()])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn snapshot_reports_recorded_values() {
|
||||
let metrics = global_internode_metrics();
|
||||
@@ -2019,11 +2322,21 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn operation_metric_descriptors_include_backend_and_operation_labels() {
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 25);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS.len(), 30);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[23].name, INTERNODE_RPC_ACTIVE);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[23].labels, SERVER_OPERATION_BACKEND_SIDE_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[24].name, INTERNODE_RPC_COMPLETIONS_TOTAL);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[24].labels, SERVER_OPERATION_BACKEND_OUTCOME_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[25].name, INTERNODE_CHANNEL_REBUILD_ATTEMPTS_TOTAL);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[25].labels, SERVER_CLASS_REASON_OUTCOME_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[26].name, INTERNODE_BULK_BODY_WAIT_DURATION_MS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[26].labels, SERVER_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[27].name, INTERNODE_BULK_BODY_WAITERS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[27].labels, SERVER_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[28].name, INTERNODE_BULK_BODY_ACTIVE);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[28].labels, SERVER_LABELS);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[29].name, INTERNODE_BULK_BODY_ADMISSIONS_TOTAL);
|
||||
assert_eq!(INTERNODE_OPERATION_METRICS[29].labels, SERVER_OUTCOME_LABELS);
|
||||
for metric in &INTERNODE_OPERATION_METRICS[..6] {
|
||||
assert_eq!(metric.labels, &[SERVER_LABEL, OPERATION_LABEL, BACKEND_LABEL]);
|
||||
}
|
||||
@@ -2407,6 +2720,52 @@ mod tests {
|
||||
assert!(observed.contains(&(INTERNODE_MSGPACK_DIRECTION_RESPONSE.to_string(), "RawFileInfo".to_string(), 1)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_send_decision_counter_keeps_fixed_decision_and_reason_labels() {
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let metrics = InternodeMetrics::default();
|
||||
metrics::with_local_recorder(&recorder, || {
|
||||
for (decision, reason) in [
|
||||
("msgpack_only", "manual_fleet"),
|
||||
("msgpack_only", "peer_capable"),
|
||||
("dual_write", "peer_unknown"),
|
||||
("dual_write", "peer_revoked"),
|
||||
("dual_write", "not_requested"),
|
||||
] {
|
||||
metrics.record_msgpack_send_decision(decision, reason);
|
||||
}
|
||||
metrics.record_msgpack_send_decision("untrusted", "untrusted");
|
||||
});
|
||||
|
||||
let observed: HashMap<(String, String), u64> = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.filter(|(composite, _, _, _)| composite.key().name() == INTERNODE_MSGPACK_SEND_DECISIONS_TOTAL)
|
||||
.map(|(composite, _, _, value)| {
|
||||
let labels: HashMap<String, String> = composite
|
||||
.key()
|
||||
.labels()
|
||||
.map(|label| (label.key().to_string(), label.value().to_string()))
|
||||
.collect();
|
||||
let count = match value {
|
||||
DebugValue::Counter(count) => count,
|
||||
other => panic!("msgpack send decision must be a counter, got {other:?}"),
|
||||
};
|
||||
(
|
||||
(
|
||||
labels.get("decision").cloned().unwrap_or_default(),
|
||||
labels.get(REASON_LABEL).cloned().unwrap_or_default(),
|
||||
),
|
||||
count,
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(observed.len(), 5);
|
||||
assert_eq!(observed.get(&("dual_write".to_string(), "peer_unknown".to_string())), Some(&2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn msgpack_json_decode_counter_tracks_codec_direction_and_message() {
|
||||
let metrics = InternodeMetrics::default();
|
||||
|
||||
@@ -22,7 +22,7 @@ workspace = true
|
||||
base64-simd = { workspace = true, optional = true }
|
||||
bytes = { workspace = true, optional = true }
|
||||
memmap2 = { workspace = true, optional = true }
|
||||
nixl-sys = { git = "https://github.com/ai-dynamo/nixl", tag = "v1.5.2", optional = true }
|
||||
nixl-sys = { workspace = true, optional = true }
|
||||
parking_lot = { workspace = true, optional = true }
|
||||
thiserror = { workspace = true, optional = true }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread", "sync", "time"], optional = true }
|
||||
|
||||
@@ -29,6 +29,9 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
|
||||
default = []
|
||||
# Default-off JSON isolation for sampled local-batch diagnostic root events.
|
||||
diagnostic-local-batch = []
|
||||
# Exact HTTP/2 PING ACK timeout observation. The root feature also enables
|
||||
# Hyper's unstable tracing event in rustfs-protos.
|
||||
diagnostic-internode-http2-ping = ["rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
|
||||
+12
-1
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! Enforces that the `dial9` feature is paired with `--cfg tokio_unstable`.
|
||||
//! Enforces the cfg contracts of default-off unstable instrumentation features.
|
||||
//!
|
||||
//! `dial9` hooks Tokio's unstable runtime instrumentation. The
|
||||
//! flag lives in `RUSTFLAGS`, which a caller can silently clobber by exporting
|
||||
@@ -25,6 +25,8 @@ fn main() {
|
||||
|
||||
let dial9_enabled = std::env::var_os("CARGO_FEATURE_DIAL9").is_some();
|
||||
let tokio_unstable = std::env::var_os("CARGO_CFG_TOKIO_UNSTABLE").is_some();
|
||||
let ping_observability_enabled = std::env::var_os("CARGO_FEATURE_DIAGNOSTIC_INTERNODE_HTTP2_PING").is_some();
|
||||
let hyper_unstable_tracing = std::env::var_os("CARGO_CFG_HYPER_UNSTABLE_TRACING").is_some();
|
||||
|
||||
if dial9_enabled && !tokio_unstable {
|
||||
panic!(
|
||||
@@ -39,4 +41,13 @@ fn main() {
|
||||
present in the value you export.\n"
|
||||
);
|
||||
}
|
||||
|
||||
if ping_observability_enabled && !hyper_unstable_tracing {
|
||||
panic!(
|
||||
"\n\
|
||||
rustfs-obs: the `diagnostic-internode-http2-ping` feature requires\n\
|
||||
`--cfg hyper_unstable_tracing`. Build with:\n\
|
||||
\n RUSTFLAGS=\"--cfg hyper_unstable_tracing\" cargo build -p rustfs --features diagnostic-internode-http2-ping\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -24,6 +24,9 @@ use tracing_subscriber::{
|
||||
filter::{FilterFn, LevelFilter, filter_fn},
|
||||
};
|
||||
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
use super::http2_ping::HYPER_HTTP2_CLIENT_TARGET;
|
||||
|
||||
pub const HTTP_SERVER_LOG_TARGET: &str = "rustfs::server::http";
|
||||
|
||||
/// Pyroscope emits raw reqwest errors from its background session manager.
|
||||
@@ -57,7 +60,9 @@ pub(super) fn s3s_log_filter() -> FilterFn {
|
||||
/// `RUST_LOG` environment variable takes precedence over `logger_level`.
|
||||
/// For non-verbose levels (`info`, `warn`, `error`), noisy dependency targets
|
||||
/// are automatically suppressed based on the effective log configuration.
|
||||
/// Transport internals are disabled, while `s3s` remains visible at WARN.
|
||||
/// Transport internals are disabled, while `s3s` remains visible at WARN. The
|
||||
/// diagnostic HTTP/2 PING feature adds back only Hyper's client target at DEBUG
|
||||
/// so its exact keepalive timeout event reaches the metrics layer.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `logger_level` - The desired log level string (e.g., `"info"`, `"debug"`).
|
||||
@@ -294,6 +299,14 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
{
|
||||
let directive = format!("{HYPER_HTTP2_CLIENT_TARGET}=debug")
|
||||
.parse()
|
||||
.expect("static Hyper HTTP/2 PING tracing directive must parse");
|
||||
filter = filter.add_directive(directive);
|
||||
}
|
||||
|
||||
filter
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use tracing::{Event, Subscriber, field::Visit};
|
||||
use tracing_subscriber::{Layer, layer::Context};
|
||||
|
||||
pub(super) const HYPER_HTTP2_CLIENT_TARGET: &str = "hyper::proto::h2::client";
|
||||
const HYPER_KEEPALIVE_TIMEOUT_MESSAGE: &str = "connection keep-alive timed out";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Default)]
|
||||
pub(crate) struct Http2PingTimeoutLayer;
|
||||
|
||||
pub(crate) const fn http2_ping_timeout_layer() -> Option<Http2PingTimeoutLayer> {
|
||||
if cfg!(feature = "diagnostic-internode-http2-ping") {
|
||||
Some(Http2PingTimeoutLayer)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct TimeoutMessageVisitor {
|
||||
matched: bool,
|
||||
}
|
||||
|
||||
impl Visit for TimeoutMessageVisitor {
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||
if field.name() == "message" && format!("{value:?}") == HYPER_KEEPALIVE_TIMEOUT_MESSAGE {
|
||||
self.matched = true;
|
||||
}
|
||||
}
|
||||
|
||||
fn record_str(&mut self, field: &tracing::field::Field, value: &str) {
|
||||
if field.name() == "message" && value == HYPER_KEEPALIVE_TIMEOUT_MESSAGE {
|
||||
self.matched = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for Http2PingTimeoutLayer
|
||||
where
|
||||
S: Subscriber,
|
||||
{
|
||||
fn on_event(&self, event: &Event<'_>, _context: Context<'_, S>) {
|
||||
if event.metadata().target() != HYPER_HTTP2_CLIENT_TARGET
|
||||
|| !rustfs_io_metrics::internode_metrics::is_internode_http2_client_task()
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
let mut visitor = TimeoutMessageVisitor::default();
|
||||
event.record(&mut visitor);
|
||||
if visitor.matched {
|
||||
rustfs_io_metrics::internode_metrics::record_http2_ping_timeout();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(all(test, feature = "diagnostic-internode-http2-ping"))]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use metrics_util::debugging::{DebugValue, DebuggingRecorder};
|
||||
use tracing_subscriber::{Registry, layer::SubscriberExt};
|
||||
|
||||
static METRICS_TEST_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(());
|
||||
|
||||
#[tokio::test]
|
||||
async fn exact_scoped_hyper_timeout_event_records_once_without_close_inference() {
|
||||
let _test_guard = METRICS_TEST_LOCK.lock().await;
|
||||
let recorder = DebuggingRecorder::new();
|
||||
let snapshotter = recorder.snapshotter();
|
||||
let _recorder_guard = metrics::set_default_local_recorder(&recorder);
|
||||
let previously_enabled = rustfs_io_metrics::metrics_enabled();
|
||||
rustfs_io_metrics::set_metrics_enabled(true);
|
||||
|
||||
let subscriber = Registry::default()
|
||||
.with(crate::telemetry::filter::build_env_filter("info", Some("info")))
|
||||
.with(http2_ping_timeout_layer());
|
||||
let dispatch = tracing::Dispatch::new(subscriber);
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
tracing::debug!(target: HYPER_HTTP2_CLIENT_TARGET, "connection keep-alive timed out");
|
||||
});
|
||||
rustfs_io_metrics::internode_metrics::scope_internode_http2_client_task(async {
|
||||
tracing::dispatcher::with_default(&dispatch, || {
|
||||
tracing::debug!(target: HYPER_HTTP2_CLIENT_TARGET, "connection keep-alive timed out");
|
||||
tracing::debug!(target: HYPER_HTTP2_CLIENT_TARGET, "connection error");
|
||||
tracing::debug!(target: "rustfs::test", "connection keep-alive timed out");
|
||||
});
|
||||
})
|
||||
.await;
|
||||
rustfs_io_metrics::set_metrics_enabled(previously_enabled);
|
||||
|
||||
let timeouts = snapshotter
|
||||
.snapshot()
|
||||
.into_vec()
|
||||
.into_iter()
|
||||
.find(|(key, _, _, _)| key.key().name() == "rustfs_system_network_internode_http2_ping_timeouts_total")
|
||||
.map(|(_, _, _, value)| value)
|
||||
.expect("exact HTTP/2 PING timeout counter");
|
||||
assert!(matches!(timeouts, DebugValue::Counter(1)));
|
||||
}
|
||||
}
|
||||
@@ -421,6 +421,7 @@ fn init_stdout_only(_config: &OtelConfig, logger_level: &str, is_production: boo
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
@@ -534,6 +535,7 @@ fn init_file_logging_internal(
|
||||
|
||||
tracing_subscriber::registry()
|
||||
.with(env_filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
|
||||
@@ -43,6 +43,8 @@
|
||||
pub mod dial9;
|
||||
mod filter;
|
||||
mod guard;
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
mod http2_ping;
|
||||
mod local;
|
||||
mod otel;
|
||||
mod recorder;
|
||||
@@ -53,12 +55,19 @@ use crate::TelemetryError;
|
||||
use crate::config::OtelConfig;
|
||||
pub use filter::HTTP_SERVER_LOG_TARGET;
|
||||
pub use guard::OtelGuard;
|
||||
#[cfg(feature = "diagnostic-internode-http2-ping")]
|
||||
pub(super) use http2_ping::http2_ping_timeout_layer;
|
||||
pub(crate) use recorder::process_global_meter;
|
||||
pub use recorder::{Recorder, retire_metric_series};
|
||||
use rustfs_config::observability::ENV_OBS_LOG_DIRECTORY;
|
||||
use rustfs_config::{DEFAULT_LOG_LEVEL, ENVIRONMENT, observability::DEFAULT_OBS_ENVIRONMENT_PRODUCTION};
|
||||
use rustfs_utils::get_env_opt_str;
|
||||
|
||||
#[cfg(not(feature = "diagnostic-internode-http2-ping"))]
|
||||
pub(super) const fn http2_ping_timeout_layer() -> Option<tracing_subscriber::layer::Identity> {
|
||||
None
|
||||
}
|
||||
|
||||
/// Initialize the telemetry subsystem according to the provided configuration.
|
||||
///
|
||||
/// Evaluates three routing rules in priority order and delegates to the
|
||||
|
||||
@@ -107,6 +107,7 @@ const OBJECT_LOCK_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
const INTERNODE_DURATION_HISTOGRAM_METRICS: &[&str] = &[
|
||||
"rustfs_system_network_internode_operation_duration_ms",
|
||||
"rustfs_system_network_internode_operation_server_duration_ms",
|
||||
"rustfs_system_network_internode_bulk_body_wait_duration_ms",
|
||||
];
|
||||
// Internode instruments record milliseconds, including sub-millisecond control RPCs.
|
||||
const INTERNODE_DURATION_HISTOGRAM_BUCKETS: &[f64] = &[
|
||||
@@ -311,6 +312,7 @@ pub(super) fn init_observability_http(
|
||||
let filter = build_env_filter(logger_level, None);
|
||||
tracing_subscriber::registry()
|
||||
.with(filter)
|
||||
.with(super::http2_ping_timeout_layer())
|
||||
.with(pyroscope_log_filter())
|
||||
.with(s3s_log_filter())
|
||||
.with(ErrorLayer::default())
|
||||
@@ -1181,6 +1183,7 @@ mod tests {
|
||||
(OBJECT_LOCK_DURATION_HISTOGRAM_METRICS[1], [0.002, 12.9, 31.0]),
|
||||
(INTERNODE_DURATION_HISTOGRAM_METRICS[0], [0.04, 3.2, 31000.0]),
|
||||
(INTERNODE_DURATION_HISTOGRAM_METRICS[1], [0.02, 1.2, 1300.0]),
|
||||
(INTERNODE_DURATION_HISTOGRAM_METRICS[2], [0.01, 4.2, 1200.0]),
|
||||
] {
|
||||
let histogram = recorder.register_histogram(&metrics::Key::from_name(name), &metadata);
|
||||
for seconds in observations {
|
||||
@@ -1200,6 +1203,7 @@ mod tests {
|
||||
"rustfs_object_lock_diag_hold_duration_seconds" => [(0.002, 1), (15.0, 1), (60.0, 1)],
|
||||
"rustfs_system_network_internode_operation_duration_ms" => [(0.05, 1), (4.0, 1), (60000.0, 1)],
|
||||
"rustfs_system_network_internode_operation_server_duration_ms" => [(0.025, 1), (1.5, 1), (2500.0, 1)],
|
||||
"rustfs_system_network_internode_bulk_body_wait_duration_ms" => [(0.01, 1), (5.0, 1), (2500.0, 1)],
|
||||
_ => continue,
|
||||
};
|
||||
let AggregatedMetrics::F64(MetricData::Histogram(histogram)) = metric.data() else {
|
||||
@@ -1215,7 +1219,7 @@ mod tests {
|
||||
assert_eq!(occupied, expected, "seconds remain distinct at millisecond and long-hold scales");
|
||||
checked += 1;
|
||||
}
|
||||
assert_eq!(checked, 4, "lock and internode duration histograms were exported");
|
||||
assert_eq!(checked, 5, "lock and internode duration histograms were exported");
|
||||
provider.shutdown().expect("stop lock histogram test provider");
|
||||
}
|
||||
|
||||
|
||||
@@ -34,6 +34,9 @@ path = "src/main.rs"
|
||||
|
||||
[features]
|
||||
default = []
|
||||
# Exact HTTP/2 keepalive timeout observation through Hyper's unstable tracing
|
||||
# event. Requires `RUSTFLAGS="--cfg hyper_unstable_tracing"`.
|
||||
diagnostic-internode-http2-ping = ["dep:hyper", "rustfs-io-metrics/diagnostic-internode-http2-ping"]
|
||||
hotpath = [
|
||||
"hotpath/hotpath",
|
||||
"hotpath/tokio",
|
||||
@@ -75,6 +78,8 @@ prost = { workspace = true }
|
||||
tonic = { workspace = true, features = ["transport", "tls-native-roots", "tls-aws-lc", "gzip", "deflate"] }
|
||||
tonic-prost = { workspace = true }
|
||||
tonic-prost-build = { workspace = true }
|
||||
hyper = { workspace = true, optional = true, features = ["tracing"] }
|
||||
hyper-util = { workspace = true, features = ["client-legacy", "tokio"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread"] }
|
||||
tracing = { workspace = true }
|
||||
serde = { workspace = true }
|
||||
@@ -90,3 +95,4 @@ serde_json = { workspace = true }
|
||||
temp-env = { workspace = true, features = ["async_closure"] }
|
||||
tempfile = { workspace = true }
|
||||
tokio = { workspace = true, features = ["macros"] }
|
||||
tracing-subscriber = { workspace = true }
|
||||
|
||||
@@ -0,0 +1,31 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
fn main() {
|
||||
println!("cargo::rerun-if-changed=build.rs");
|
||||
println!("cargo::rerun-if-env-changed=RUSTFLAGS");
|
||||
|
||||
let ping_observability_enabled = std::env::var_os("CARGO_FEATURE_DIAGNOSTIC_INTERNODE_HTTP2_PING").is_some();
|
||||
let hyper_unstable_tracing = std::env::var_os("CARGO_CFG_HYPER_UNSTABLE_TRACING").is_some();
|
||||
|
||||
if ping_observability_enabled && !hyper_unstable_tracing {
|
||||
panic!(
|
||||
"\n\
|
||||
rustfs-protos: the `diagnostic-internode-http2-ping` feature requires\n\
|
||||
`--cfg hyper_unstable_tracing` so Hyper emits the exact keepalive\n\
|
||||
timeout event. Build with:\n\
|
||||
\n RUSTFLAGS=\"--cfg hyper_unstable_tracing\" cargo build -p rustfs --features diagnostic-internode-http2-ping\n"
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -73,7 +73,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "batch_read_version_req",
|
||||
bin_field: "batch_read_version_req_bin",
|
||||
},
|
||||
json_encoder: "let batch_read_version_req = compat_json(&req)?;",
|
||||
json_encoder: "let batch_read_version_req = compat_json(batch_rpc_req.as_ref(), &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -118,7 +118,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "read_multiple_req",
|
||||
bin_field: "read_multiple_req_bin",
|
||||
},
|
||||
json_encoder: "let read_multiple_req = compat_json(&req)?;",
|
||||
json_encoder: "let read_multiple_req = compat_json(&req, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -127,7 +127,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));",
|
||||
json_encoder: "let opts_str = compat_json(opts, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -136,7 +136,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
json_encoder: "let file_info = compat_json(&fi, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -145,7 +145,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
json_encoder: "let file_info = compat_json(&fi, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -154,7 +154,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let opts_str = compat_json(&opts)?;",
|
||||
json_encoder: "let opts_str = compat_json(opts, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
@@ -163,7 +163,7 @@ pub const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
json_encoder: "let file_info = compat_json(&fi, &policy)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
];
|
||||
|
||||
+827
-45
File diff suppressed because it is too large
Load Diff
@@ -34,6 +34,10 @@ pub(crate) fn record_grpc_dial_result(duration: Duration, success: bool) {
|
||||
global_internode_metrics().record_dial_result(duration, success);
|
||||
}
|
||||
|
||||
pub(crate) fn record_channel_rebuild_attempt(class: &'static str, reason: &'static str, connected: bool) {
|
||||
rustfs_io_metrics::internode_metrics::record_channel_rebuild_attempt(class, reason, connected);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn grpc_dial_errors_total() -> u64 {
|
||||
global_internode_metrics().snapshot().dial_errors_total
|
||||
|
||||
@@ -43,6 +43,9 @@ allow-git = [
|
||||
# Completion ownership patch fixed to rustfs/uring#20.
|
||||
# owner: overtrue review: 2026-10
|
||||
"https://github.com/rustfs/uring.git",
|
||||
# NIXL bindings fork pinned for the optional RustFS RDMA transport.
|
||||
# owner: houseme review: 2027-02
|
||||
"https://github.com/houseme/nixl",
|
||||
]
|
||||
|
||||
[bans]
|
||||
|
||||
@@ -60,6 +60,7 @@ Run **before** at the baseline column and **after** at the enabled column, every
|
||||
| P1 isolation | `RUSTFS_INTERNODE_CHANNEL_ISOLATION` | `false` (default) | `true` |
|
||||
| P1 bulk pool | `RUSTFS_INTERNODE_BULK_CHANNELS` | unset | `2` |
|
||||
| P2 msgpack-only | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` + `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` | both `false` (default) | per the convergence runbook |
|
||||
| P2 per-peer msgpack | `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY` + `RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY` | both `false` (default) | both `true`, fleet confirmation `false`, stable direct/affine peer connections only |
|
||||
| P3 prewarm | `RUSTFS_INTERNODE_PREWARM` | `false` (default) | `true` |
|
||||
| P3 offline bypass | `RUSTFS_INTERNODE_OFFLINE_BYPASS` | `false` (default) | `true` |
|
||||
| P3 reprobe / threshold | `RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS` / `RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD` | defaults | defaults (`DEFAULT_INTERNODE_OFFLINE_REPROBE_SECS`, `DEFAULT_INTERNODE_OFFLINE_FAILURE_THRESHOLD`) |
|
||||
@@ -81,6 +82,11 @@ Explicit channel dial metrics start after endpoint settings validate and cover T
|
||||
| `rustfs_system_network_internode_operation_payload_bytes` | payload size distribution (P0/P1 sizing) |
|
||||
| `rustfs_system_network_internode_operation_large_payloads_total` | large unary RPCs sharing a channel (P1 target) |
|
||||
| `rustfs_system_network_internode_dial_avg_time_nanos`, `rustfs_system_network_internode_dial_errors_total` | connect cost and failures (P3) |
|
||||
| `rustfs_system_network_internode_channel_rebuild_attempts_total{class,reason,outcome}` | explicit RustFS cache rebuild attempts after a bounded eviction reason |
|
||||
| `rustfs_system_network_internode_http2_ping_timeouts_total` | exact Hyper keepalive PING ACK timeouts in diagnostic builds; generic closes and RPC errors do not contribute |
|
||||
| `rustfs_system_network_internode_bulk_body_{waiters,active}` | work waiting for or holding complete bulk-unary admission |
|
||||
| `rustfs_system_network_internode_bulk_body_wait_duration_ms` | time spent before a complete bulk-unary attempt enters tonic |
|
||||
| `rustfs_system_network_internode_bulk_body_admissions_total{outcome}` | admitted attempts and waits cancelled by an outer deadline |
|
||||
| `rustfs_system_network_internode_msgpack_json_decode_total{direction,message,codec}`, `..._msgpack_json_fallback_total`, `..._msgpack_json_decode_error_total` | P2 gate inputs |
|
||||
| `rustfs_cluster_servers_offline_total` | offline detection correctness (P3 bypass) |
|
||||
| lock p99 (lock metrics, needs `RUSTFS_OBJECT_LOCK_DIAG_ENABLE=true`) | P1 head-of-line-blocking win |
|
||||
@@ -99,11 +105,28 @@ P1 phase files clear bulk-specific overrides so an isolation experiment inherits
|
||||
| `RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE` | Inherit `RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE` | Bulk response receive credit per connection, bytes |
|
||||
| `RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT` | `64` | Active tonic transport response futures per bulk channel; minimum 1 |
|
||||
| `RUSTFS_INTERNODE_BULK_BUFFER_SIZE` | `1024` | Pending tonic service buffer capacity per bulk channel; minimum 1 |
|
||||
| `RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT` | `0` | Complete bulk-unary attempts per peer, held through response decoding and validation; `0` disables the additional gate |
|
||||
| `RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS` | `1000` | Local wait budget before returning `DiskOngoingReq`; minimum 1 ms |
|
||||
|
||||
Zero receive-window values select the linked library default. In the locked Hyper 1.12 client this is 2 MiB per stream and 5 MiB per connection, while RustFS explicitly defaults to 1 MiB and 2 MiB. Record which policy the experiment actually selects. Values above the HTTP/2 31-bit limit fail channel acquisition with an invalid-input error. Change connection receive credit first, then stream credit only when measured bandwidth-delay product and resource budgets justify it. For example, compare the default 2 MiB connection window with `RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE=8388608`, holding other settings fixed. Request-upload flow control is configured by the receiving server. Keep server-wide window changes in a separate experiment.
|
||||
|
||||
Bulk selection advances independently for each peer. Cursor bookkeeping caches up to 512 peer addresses. Each entry keeps four immutable slot keys so common bulk pools borrow their cache keys on hits. Larger pools construct the remaining slot keys on demand with the same cache identity. Replacing an entry restarts its selection sequence while established channels keep their existing cache keys. HTTPS cache hits share the TLS-generation read lock, with generation validation retained before handoff. Connection misses still coalesce under the existing per-key dial lock.
|
||||
|
||||
The complete-body gate is per peer and applies to every RPC routed through the bulk client, including recovery inventories. Acquire it before the existing remote-operation deadline so queued work does not occupy tonic's transport queue or mark a healthy peer faulty. Its own bounded wait returns `DiskOngoingReq`. The permit remains owned by the caller through protobuf body decoding and application response validation. It does not reduce memory already owned by a queued WriteAll caller. Start with an explicit value such as `32`, check waiters, active work, p99 and memory, and keep the value and wait timeout identical across all nodes in one cell.
|
||||
|
||||
The body limit is read when remote-disk handles are built and therefore requires a process restart. The per-peer registry retains weak entries and uses one conservative shared overflow gate only if more than 512 peer identities remain live. Values above Tokio's semaphore capacity fail startup instead of being truncated.
|
||||
|
||||
Explicit rebuild reasons are `rpc_failure`, `operation_timeout`, `transport_failure`, `lock_timeout`, `lock_transport`, `peer_probe`, `tls_generation_changed`, and `manual`; unexpected inputs collapse to `unknown`. Outcomes are `connected` and `failed_or_cancelled`; `connected` means a replacement passed TLS-generation validation and was published or observed in the cache. A transport connection rejected during publication counts as `failed_or_cancelled` and retries under the same pending reason. Concurrent cache hits atomically consume the successful reason once. One eviction can schedule control and bulk rebuilds independently, and every failed attempt remains visible until a replacement is published. These metrics do not claim that tonic's private reconnect loop was observed.
|
||||
|
||||
The locked tonic/Hyper public API does not expose HTTP/2 PING ACK timeout outcomes. A default-off diagnostic build can enable Hyper's exact internal timeout event and the RustFS counter without adding a second timer or changing reconnect behavior:
|
||||
|
||||
```bash
|
||||
RUSTFLAGS="--cfg hyper_unstable_tracing" \
|
||||
cargo build -p rustfs --features diagnostic-internode-http2-ping
|
||||
```
|
||||
|
||||
The build scripts reject the feature when the required cfg is missing. The subscriber enables only Hyper's HTTP/2 client debug target and counts the exact `connection keep-alive timed out` event. Other low-frequency debug events from that target can also reach configured diagnostic log/export sinks, but generic connection errors, TCP closure, GOAWAY and RPC timeouts are not classified as PING failures. The feature is diagnostic-only because Hyper documents its tracing event contract as unstable; ordinary builds do not enable the dependency feature or pay its tracing cost. Tonic-internal reconnects without a RustFS cache eviction remain separate from explicit rebuild attempts. A stable production-default metric still requires an upstream typed observer.
|
||||
|
||||
Custom-CA channels use tonic's endpoint URI handling for certificate names and SNI, including IPv6 literals. The regression suite exercises real control and bulk TLS connections to IPv6 loopback and verifies that a trusted certificate for a different host is rejected.
|
||||
|
||||
When the outbound snapshot has no custom CA, channels explicitly load the platform trust roots. A configured mTLS client identity applies to both platform-root and custom-CA channels. An explicit custom CA selects its own trust store. TLS configuration loads and parses certificates on the blocking pool. The acquisition deadline covers waiting for that configuration. A timed-out blocking task can finish later and drops its endpoint, while the acquisition future owns connection establishment and channel publication. The native-root regression uses a process-local `SSL_CERT_FILE` fixture and exercises both channel classes, required client authentication, and rejection of a server outside the explicit private trust store.
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Internode msgpack/JSON convergence runbook
|
||||
|
||||
**Use this when:** operating or changing the staged retirement of the JSON compatibility fields on internode gRPC metadata RPCs, deciding whether a fleet may enable `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY`, or adding a new `*_bin` proto field.
|
||||
**Source of truth:** `crates/config/src/constants/internode.rs` (`ENV_INTERNODE_RPC_MSGPACK_ONLY`, `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` and their compile-time default-off asserts), `crates/protos/src/node.proto` (`*_bin` fields), `decode_msgpack_or_json` in `rustfs/src/storage/rpc/node_service/disk.rs` (server side) and `crates/ecstore/src/cluster/rpc/remote_disk.rs` (client side), `crates/io-metrics/src/internode_metrics.rs` (counters).
|
||||
**Source of truth:** `crates/config/src/constants/internode.rs` (`ENV_INTERNODE_RPC_MSGPACK_ONLY`, `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED`, `ENV_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY` and their compile-time default-off asserts), `crates/protos/src/node.proto` (`*_bin` fields), `decode_msgpack_or_json` in `rustfs/src/storage/rpc/node_service/disk.rs` (server side) and `crates/ecstore/src/cluster/rpc/remote_disk.rs` (client side), `crates/io-metrics/src/internode_metrics.rs` (counters).
|
||||
|
||||
This is a cross-version change and proceeds by observation-gated stages, never in one step.
|
||||
|
||||
@@ -16,6 +16,7 @@ Internode RPCs dual-encode each metadata value as a msgpack binary field (`*_bin
|
||||
| `rustfs_system_network_internode_msgpack_json_decode_total` | `direction`, `message`, `codec` | a msgpack or JSON decode succeeded |
|
||||
| `rustfs_system_network_internode_msgpack_json_fallback_total` | `direction`, `message` | a decode fell back to JSON because `_bin` was empty |
|
||||
| `rustfs_system_network_internode_msgpack_json_decode_error_total` | `direction`, `message`, `codec` | either codec failed to decode (`codec` = the failed codec) |
|
||||
| `rustfs_system_network_internode_msgpack_send_decisions_total` | `decision`, `reason` | an eligible RPC selected `msgpack_only` or `dual_write`; reasons are fixed to `manual_fleet`, `peer_capable`, `peer_unknown`, `peer_revoked`, or `not_requested` |
|
||||
|
||||
`direction="request"` is a server decoding a peer's request (`node_service/disk.rs`); `direction="response"` is a client decoding a peer's response (`cluster/rpc/remote_disk.rs`), including the list-level `ReadMultiple` / `BatchReadVersion` fallbacks. `message` is the value name (`FileInfo`, `RawFileInfo`, `ReadMultipleResp`, ...).
|
||||
|
||||
@@ -92,15 +93,55 @@ Run the fleet with both flags at their default (`false`) for at least one full o
|
||||
|
||||
## Stage 1 — Stop writing JSON (env-gated)
|
||||
|
||||
Flag semantics are defined on `ENV_INTERNODE_RPC_MSGPACK_ONLY` and `ENV_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED` in `crates/config/src/constants/internode.rs`; both default off and only their conjunction stops the JSON write for convergence-ready fields. Decoders keep the JSON fallback in every state.
|
||||
Flag semantics are defined in `crates/config/src/constants/internode.rs`; all three flags default off. `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` requests msgpack-only, and JSON stops only when either fleet confirmation is true or the peer-capability gate is true and the exact current peer boot epoch plus physical connection generation has an authenticated capability proof. Unknown, unsupported, revoked, stale, or reconnecting peers continue dual-write. Decoders keep the JSON fallback in every state.
|
||||
|
||||
1. Rehearse `request-only`: `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true`, `..._FLEET_CONFIRMED=false`. Behaviour is unchanged (still dual-write) and safe with old peers; any fallback or decode-error increment still blocks.
|
||||
2. Canary: set both flags `true` on one node, restart that node only, and watch the fallback and decode-error counters for a soak period with real internode traffic.
|
||||
3. Fleet: if the counters stay zero, enable both flags fleet-wide with a rolling restart.
|
||||
4. Rollback: set either flag to `false` (or unset it) and restart; no wire format changed, so rollback is immediate.
|
||||
|
||||
An alternative per-peer canary keeps `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED=false`, sets `RUSTFS_INTERNODE_RPC_MSGPACK_ONLY=true` and `RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY=true`, and confirms `reason="peer_capable"` only for proved peers while every `peer_unknown` or `peer_revoked` decision remains `dual_write`. Enable this mode only when one client HTTP/2 connection has stable affinity to one RustFS peer process; an L7 proxy that routes streams from one connection to different backend versions invalidates the connection-generation proof. A connection-generation change may rebuild one request as dual-write only when the retained local guard proves rejection before the first payload frame; remote status text, a dispatched request, or an ordinary transport error never authorizes that compatibility retry. Disable either the base request flag or peer-capability flag to roll back immediately.
|
||||
|
||||
The benchmark driver pins these operator states as dry-run phases (`before`, `request-only`, `canary`, `after`, `rollback`); see [internode-grpc-benchmark-runbook.md](internode-grpc-benchmark-runbook.md).
|
||||
|
||||
### Per-peer mixed-version evidence gate
|
||||
|
||||
The per-peer gate is not qualified by a successful command transcript alone. Record three ordered four-node measurement phases: `mixed-old-peer` (three candidate nodes and one previous-version node), `all-new` (four candidate nodes), and `rollback-old-peer` (replace one candidate with the previous-version image and verify readback). Every candidate node uses `MSGPACK_ONLY=true`, `MSGPACK_PEER_CAPABILITY=true`, and `MSGPACK_ONLY_FLEET_CONFIRMED=false`; previous-version nodes use all three false. Each phase must run real PUT/GET/metadata traffic and retain the exact candidate/baseline revisions, topology, timestamps, workload result, environment, and raw Prometheus HTTP API JSON.
|
||||
|
||||
Capture these PromQL vectors over each non-overlapping phase window and store the complete HTTP API response, not extracted numbers:
|
||||
|
||||
```promql
|
||||
sum by (decision, reason) (increase(rustfs_system_network_internode_msgpack_send_decisions_total[$WINDOW]))
|
||||
sum(increase(rustfs_system_network_internode_msgpack_json_fallback_total[$WINDOW])) or vector(0)
|
||||
sum(increase(rustfs_system_network_internode_msgpack_json_decode_error_total[$WINDOW])) or vector(0)
|
||||
(sum(increase(rustfs_system_network_internode_operation_errors_total{backend="grpc",operation!="grpc_read_version"}[$WINDOW])) or vector(0)) + (sum(increase(rustfs_system_network_internode_operation_classified_errors_total{backend="grpc",operation="grpc_read_version",classification!="remote_response"}[$WINDOW])) or vector(0))
|
||||
```
|
||||
|
||||
Build a schema-1 manifest with `scope="internode-msgpack-peer-canary"`, `source_revision`, `baseline_revision`, `stable_connection_affinity=true`, `max_metric_lag_secs`, and the three phase records. Each phase records integer `start_unix`/`end_unix`, exact topology counts plus a `node1..node4` revision map, the new/old node environment maps, the exact four PromQL strings with a window equal to the phase duration, a relative workload JSON path, and relative paths for `decisions`, `fallback`, `decode_errors`, and `rpc_errors`. The workload artifact records `status`, a non-zero integer `requests`, and `rollback_readback` (true only for `rollback-old-peer`). Validate without rewriting raw evidence:
|
||||
|
||||
```bash
|
||||
python3 scripts/check_internode_msgpack_peer_canary.py \
|
||||
--manifest /path/to/evidence/manifest.json \
|
||||
--out /path/to/evidence/verification.json
|
||||
```
|
||||
|
||||
The verifier rejects reused/escaping/overwritten artifacts, duplicate or non-exact decision labels, labeled scalar queries, mismatched PromQL windows, wrong per-node revisions, missing peer-capable traffic, missing mixed/rollback dual-write traffic, manual-fleet or not-requested leakage, failed workload/readback, non-zero fallback/decode/gRPC transport-or-protocol errors, stale timestamps, invalid topology, and candidate/baseline identity reuse. The gRPC error query checks every operation except `grpc_read_version` through the total error counter. Read-version uses its classified counter and excludes only `remote_response`, because a successful gRPC exchange can carry an expected application-level disk response; `grpc_status` and `response_decode` still block the gate. TCP/HTTP operation outcomes remain outside this msgpack gRPC compatibility gate and the workload's zero-error contract remains independently enforced. The workload artifact binds its raw benchmark `summary.csv` by relative path and SHA-256; rollback also records equal source/download content hashes. Verifier output includes SHA-256 receipts for every raw query, workload, and summary artifact. Passing this gate is compatibility evidence only; production soak and ABBA performance drift gates remain separate.
|
||||
|
||||
Use the orchestration runner to create the isolated Compose project, execute the three phases, preserve a known object across node2 rollback, capture the four raw Prometheus responses, build the manifest, and invoke the verifier. Without `--execute` it prints a redacted plan and makes no external changes. Execution refuses an existing Compose project/volume set and removes its isolated containers and volumes on exit unless `--keep-up` is explicit.
|
||||
|
||||
```bash
|
||||
scripts/run_internode_msgpack_peer_canary.sh \
|
||||
--candidate-image rustfs/rustfs:candidate \
|
||||
--baseline-image rustfs/rustfs:previous \
|
||||
--candidate-revision <40-hex-candidate-commit> \
|
||||
--baseline-revision <40-hex-previous-commit> \
|
||||
--prometheus-query-url http://127.0.0.1:9090/api/v1/query \
|
||||
--out-dir /path/to/evidence \
|
||||
--execute
|
||||
```
|
||||
|
||||
The runner requires Docker Compose, AWS CLI, curl, Warp through `run_object_batch_bench.sh`, Python, the four-node local cluster compose file, prebuilt candidate/baseline images, and an already reachable Prometheus/OTLP pipeline. It never builds, pulls, or starts observability images implicitly. Execution rejects reused Compose resources or a non-empty evidence directory and performs a Prometheus vector preflight before starting the cluster. The workload summary must contain only successful zero-error rows with parseable request rates; the rollback phase must read byte-identical object data written before the version replacement.
|
||||
|
||||
## Stage 2 — Remove the proto JSON fields
|
||||
|
||||
Only after Stage 1 has been stable fleet-wide for a full window with the counters still zero.
|
||||
@@ -113,7 +154,7 @@ Only after Stage 1 has been stable fleet-wide for a full window with the counter
|
||||
| Stage | Wire format broken? | Rollback |
|
||||
| --- | --- | --- |
|
||||
| 0 Observe | no | n/a (metrics only) |
|
||||
| 1 msgpack-only send | no | unset either flag and restart |
|
||||
| 1 msgpack-only send | no | unset the base request flag, or unset the selected fleet/peer gate, and restart |
|
||||
| 2 remove fields | yes | redeploy the prior release; field numbers stay `reserved` |
|
||||
|
||||
## Related
|
||||
|
||||
@@ -101,6 +101,12 @@ dial9 = ["rustfs-obs/dial9"]
|
||||
diagnostic-allocator-live = []
|
||||
# Default-off sampled local batch wall/CPU and request correlation diagnostics.
|
||||
diagnostic-local-batch = ["rustfs-ecstore/diagnostic-local-batch", "rustfs-obs/diagnostic-local-batch"]
|
||||
# Exact Hyper HTTP/2 PING ACK timeout accounting. Diagnostic-only because the
|
||||
# upstream tracing event is unstable and requires an explicit build cfg.
|
||||
diagnostic-internode-http2-ping = [
|
||||
"rustfs-obs/diagnostic-internode-http2-ping",
|
||||
"rustfs-protos/diagnostic-internode-http2-ping",
|
||||
]
|
||||
# Diagnostic-only S3-operation attribution for internode ReadVersion and list-walker metrics.
|
||||
diagnostic-read-version-caller = ["rustfs-ecstore/diagnostic-read-version-caller", "rustfs-io-metrics/diagnostic-read-version-caller", "rustfs-scanner/diagnostic-read-version-caller"]
|
||||
# Diagnostic-only fixed-prefix attribution for actual Scanner metadata visits.
|
||||
|
||||
@@ -2255,9 +2255,16 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[test]
|
||||
#[serial_test::serial]
|
||||
async fn completion_absent_binding_refusal_order_baseline() {
|
||||
fn completion_absent_binding_refusal_order_baseline() {
|
||||
crate::app::gating_test_env::run_large_stack_test(
|
||||
"completion-absent-binding-refusal-order",
|
||||
completion_absent_binding_refusal_order_baseline_inner,
|
||||
);
|
||||
}
|
||||
|
||||
async fn completion_absent_binding_refusal_order_baseline_inner() {
|
||||
use crate::app::storage_api::test::bucket::quota::BucketQuota;
|
||||
use crate::app::storage_api::test::contract::bucket::{BucketOperations as _, MakeBucketOptions};
|
||||
|
||||
|
||||
@@ -52,7 +52,7 @@ use crate::storage_api::server::http::tonic_service::make_server;
|
||||
use crate::storage_api::server::http::tonic_service::make_server_for_slot;
|
||||
use crate::storage_api::server::http::{
|
||||
ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, verify_tonic_rpc_signature_with_bootstrap,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use futures::stream::FuturesUnordered;
|
||||
@@ -451,13 +451,21 @@ where
|
||||
method: req.method().clone(),
|
||||
};
|
||||
req.extensions_mut().insert(target);
|
||||
let msgpack_manifest_capability_requested = storage::tonic_msgpack_manifest_capability_requested(req.headers());
|
||||
let response_headers = tonic_boot_epoch_challenge(req.headers())
|
||||
.ok()
|
||||
.flatten()
|
||||
.and_then(|challenge| {
|
||||
storage::try_current_local_node_name()
|
||||
.and_then(|node| normalize_tonic_rpc_audience(&node).ok())
|
||||
.and_then(|audience| tonic_boot_epoch_response_headers(&audience, challenge).ok())
|
||||
.and_then(|audience| {
|
||||
storage::tonic_boot_epoch_response_headers_with_msgpack_capability(
|
||||
&audience,
|
||||
challenge,
|
||||
msgpack_manifest_capability_requested,
|
||||
)
|
||||
.ok()
|
||||
})
|
||||
});
|
||||
let future = self.inner.call(req);
|
||||
Box::pin(async move {
|
||||
@@ -6355,6 +6363,62 @@ mod tests {
|
||||
assert_eq!(captured.method, Method::POST);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rpc_request_path_service_advertises_msgpack_capability_only_when_requested() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("rpc-msgpack-capability-test-secret".to_string());
|
||||
let previous_node_name = rustfs_common::get_global_local_node_name().await;
|
||||
let audience = "127.0.0.1:9000";
|
||||
rustfs_common::set_global_local_node_name(audience).await;
|
||||
let challenge = uuid::Uuid::new_v4();
|
||||
|
||||
let response_for = |request_capability: bool| async move {
|
||||
let mut request = HttpRequest::builder()
|
||||
.method(Method::POST)
|
||||
.uri("/node_service.NodeService/Ping")
|
||||
.body(())
|
||||
.expect("capability request");
|
||||
request.headers_mut().insert(
|
||||
"x-rustfs-rpc-boot-epoch-challenge",
|
||||
challenge.to_string().parse().expect("challenge header"),
|
||||
);
|
||||
if request_capability {
|
||||
request.headers_mut().insert(
|
||||
"x-rustfs-rpc-msgpack-manifest-request",
|
||||
HeaderValue::from_static("msgpack-compat-manifest-v1"),
|
||||
);
|
||||
}
|
||||
RpcRequestPathService::new(RpcPathObserver)
|
||||
.call(request)
|
||||
.await
|
||||
.expect("capability response")
|
||||
};
|
||||
|
||||
let legacy = response_for(false).await;
|
||||
assert!(!legacy.headers().contains_key("x-rustfs-rpc-msgpack-manifest-capability"));
|
||||
assert!(
|
||||
!legacy
|
||||
.headers()
|
||||
.contains_key("x-rustfs-rpc-msgpack-manifest-capability-proof")
|
||||
);
|
||||
|
||||
let capable = response_for(true).await;
|
||||
assert_eq!(
|
||||
capable
|
||||
.headers()
|
||||
.get("x-rustfs-rpc-msgpack-manifest-capability")
|
||||
.and_then(|value| value.to_str().ok()),
|
||||
Some("msgpack-compat-manifest-v1")
|
||||
);
|
||||
assert!(
|
||||
capable
|
||||
.headers()
|
||||
.contains_key("x-rustfs-rpc-msgpack-manifest-capability-proof")
|
||||
);
|
||||
|
||||
rustfs_common::set_global_local_node_name(&previous_node_name).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial]
|
||||
async fn rpc_completion_waits_for_the_grpc_stream_trailer() {
|
||||
|
||||
@@ -681,17 +681,17 @@ pub(crate) mod ecstore_rpc {
|
||||
SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC, TONIC_RPC_PREFIX, check_and_record_signed_rpc_nonce,
|
||||
decode_heal_bucket_rpc_options, normalize_tonic_rpc_audience, rpc_auth_is_resource_exhausted, rpc_background_admission,
|
||||
sign_ns_scanner_capability_with_usage_capabilities, sign_ns_scanner_usage_watch_capability, sign_put_file_capability,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers,
|
||||
tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer, verify_rpc_signature, verify_tonic_canonical_body_digest,
|
||||
verify_tonic_mutation_body_digest, verify_tonic_mutation_body_digest_reject_unsigned,
|
||||
verify_tonic_rpc_signature_with_bootstrap,
|
||||
sign_tonic_rpc_response_proof, tonic_boot_epoch_challenge, tonic_boot_epoch_response_headers_with_msgpack_capability,
|
||||
tonic_msgpack_manifest_capability_requested, tonic_rpc_auth_failure_reason, verify_put_file_auth_trailer,
|
||||
verify_rpc_signature, verify_tonic_canonical_body_digest, verify_tonic_mutation_body_digest,
|
||||
verify_tonic_mutation_body_digest_reject_unsigned, verify_tonic_rpc_signature_with_bootstrap,
|
||||
};
|
||||
#[cfg(test)]
|
||||
pub(crate) use rustfs_ecstore::api::rpc::{
|
||||
RpcReplayCacheAdmissionError, ScannerScopedDirtyUsageAckEntry, build_put_file_auth_trailer, gen_signature_headers,
|
||||
gen_tonic_replay_scope_headers, gen_tonic_signature_headers, set_tonic_canonical_body_digest,
|
||||
verify_ns_scanner_capability_with_usage_capabilities, verify_put_file_capability, verify_tonic_boot_epoch_response,
|
||||
verify_tonic_rpc_response_proof,
|
||||
tonic_boot_epoch_response_headers, verify_ns_scanner_capability_with_usage_capabilities, verify_put_file_capability,
|
||||
verify_tonic_boot_epoch_response, verify_tonic_rpc_response_proof,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -2539,10 +2539,23 @@ pub(crate) fn tonic_boot_epoch_challenge(headers: &http::HeaderMap) -> std::io::
|
||||
ecstore_rpc::tonic_boot_epoch_challenge(headers)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(crate) fn tonic_boot_epoch_response_headers(audience: &str, challenge: uuid::Uuid) -> std::io::Result<http::HeaderMap> {
|
||||
ecstore_rpc::tonic_boot_epoch_response_headers(audience, challenge)
|
||||
}
|
||||
|
||||
pub(crate) fn tonic_boot_epoch_response_headers_with_msgpack_capability(
|
||||
audience: &str,
|
||||
challenge: uuid::Uuid,
|
||||
include_msgpack_manifest: bool,
|
||||
) -> std::io::Result<http::HeaderMap> {
|
||||
ecstore_rpc::tonic_boot_epoch_response_headers_with_msgpack_capability(audience, challenge, include_msgpack_manifest)
|
||||
}
|
||||
|
||||
pub(crate) fn tonic_msgpack_manifest_capability_requested(headers: &http::HeaderMap) -> bool {
|
||||
ecstore_rpc::tonic_msgpack_manifest_capability_requested(headers)
|
||||
}
|
||||
|
||||
pub(crate) fn verify_tonic_canonical_body_digest<T>(request: &tonic::Request<T>, canonical_body: &[u8]) -> std::io::Result<()> {
|
||||
ecstore_rpc::verify_tonic_canonical_body_digest(request, canonical_body)
|
||||
}
|
||||
|
||||
@@ -193,8 +193,8 @@ pub(crate) mod server {
|
||||
pub(crate) use crate::storage::storage_api::{
|
||||
RemoteBackgroundIo, ServerContextSlot, TONIC_RPC_PREFIX, normalize_tonic_rpc_audience,
|
||||
rpc_auth_is_resource_exhausted, rpc_background_admission, tonic_boot_epoch_challenge,
|
||||
tonic_boot_epoch_response_headers, tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap,
|
||||
with_remote_background_io,
|
||||
tonic_boot_epoch_response_headers_with_msgpack_capability, tonic_msgpack_manifest_capability_requested,
|
||||
tonic_rpc_auth_failure_reason, verify_tonic_rpc_signature_with_bootstrap, with_remote_background_io,
|
||||
};
|
||||
|
||||
pub(crate) fn try_current_local_node_name() -> Option<String> {
|
||||
@@ -206,7 +206,7 @@ pub(crate) mod server {
|
||||
Endpoint, EndpointServerPools, Endpoints, PeerRestClient, PoolEndpoints, ScannerScopedDirtyUsageAckEntry,
|
||||
find_local_disk_by_ref, gen_signature_headers, gen_tonic_replay_scope_headers, gen_tonic_signature_headers,
|
||||
get_global_lock_client, get_global_lock_clients, init_local_disks, init_lock_clients,
|
||||
verify_tonic_boot_epoch_response,
|
||||
tonic_boot_epoch_response_headers, verify_tonic_boot_epoch_response,
|
||||
};
|
||||
|
||||
pub(crate) mod ecfs {
|
||||
|
||||
@@ -0,0 +1,328 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Validate measured mixed-version evidence for the internode msgpack peer gate."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
from typing import Any
|
||||
|
||||
|
||||
PHASES = ("mixed-old-peer", "all-new", "rollback-old-peer")
|
||||
TOPOLOGIES = {
|
||||
"mixed-old-peer": (3, 1),
|
||||
"all-new": (4, 0),
|
||||
"rollback-old-peer": (3, 1),
|
||||
}
|
||||
OLD_NODES = {
|
||||
"mixed-old-peer": {"node4"},
|
||||
"all-new": set(),
|
||||
"rollback-old-peer": {"node2"},
|
||||
}
|
||||
ARTIFACTS = ("decisions", "fallback", "decode_errors", "rpc_errors")
|
||||
DECISIONS = {"msgpack_only", "dual_write"}
|
||||
REASONS = {"manual_fleet", "peer_capable", "peer_unknown", "peer_revoked", "not_requested"}
|
||||
REVISION = re.compile(r"^[0-9a-f]{40}$")
|
||||
SHA256 = re.compile(r"^[0-9a-f]{64}$")
|
||||
MAX_DOCUMENT_BYTES = 1024 * 1024
|
||||
|
||||
|
||||
class EvidenceError(ValueError):
|
||||
pass
|
||||
|
||||
|
||||
def require(condition: bool, message: str) -> None:
|
||||
if not condition:
|
||||
raise EvidenceError(message)
|
||||
|
||||
|
||||
def unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
|
||||
result: dict[str, Any] = {}
|
||||
for key, value in pairs:
|
||||
require(key not in result, f"duplicate JSON key: {key}")
|
||||
result[key] = value
|
||||
return result
|
||||
|
||||
|
||||
def load_json(path: Path) -> tuple[dict[str, Any], str]:
|
||||
require(path.is_file(), f"missing evidence artifact: {path}")
|
||||
size = path.stat().st_size
|
||||
require(0 < size <= MAX_DOCUMENT_BYTES, f"invalid evidence artifact size: {path}")
|
||||
raw = path.read_bytes()
|
||||
document = json.loads(raw, object_pairs_hook=unique_object)
|
||||
require(isinstance(document, dict), f"evidence artifact must be an object: {path}")
|
||||
return document, hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def load_raw(path: Path) -> str:
|
||||
require(path.is_file(), f"missing evidence artifact: {path}")
|
||||
size = path.stat().st_size
|
||||
require(0 < size <= MAX_DOCUMENT_BYTES, f"invalid evidence artifact size: {path}")
|
||||
return hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
|
||||
|
||||
def bounded_number(value: Any, name: str) -> float:
|
||||
require(type(value) in (int, float), f"{name} must be numeric")
|
||||
result = float(value)
|
||||
require(math.isfinite(result) and result >= 0, f"{name} must be finite and non-negative")
|
||||
return result
|
||||
|
||||
|
||||
def uint(value: Any, name: str) -> int:
|
||||
require(type(value) is int and value >= 0, f"{name} must be a non-negative integer")
|
||||
return value
|
||||
|
||||
|
||||
def timestamped_value(row: dict[str, Any], start: float, end: float, lag: float, name: str) -> float:
|
||||
value = row.get("value")
|
||||
require(isinstance(value, list) and len(value) == 2, f"{name} must contain one Prometheus value")
|
||||
timestamp = bounded_number(value[0], f"{name} timestamp")
|
||||
require(end <= timestamp <= end + lag, f"{name} timestamp is outside its phase completion window")
|
||||
try:
|
||||
number = float(value[1])
|
||||
except (TypeError, ValueError) as error:
|
||||
raise EvidenceError(f"{name} value is not numeric") from error
|
||||
require(math.isfinite(number) and number >= 0, f"{name} value must be finite and non-negative")
|
||||
return number
|
||||
|
||||
|
||||
def prometheus_rows(document: dict[str, Any], name: str) -> list[dict[str, Any]]:
|
||||
require(document.get("status") == "success", f"{name} Prometheus query failed")
|
||||
data = document.get("data")
|
||||
require(isinstance(data, dict) and data.get("resultType") == "vector", f"{name} must be a Prometheus vector")
|
||||
rows = data.get("result")
|
||||
require(isinstance(rows, list) and len(rows) <= 64, f"{name} has invalid vector cardinality")
|
||||
require(all(isinstance(row, dict) for row in rows), f"{name} contains a non-object result")
|
||||
return rows
|
||||
|
||||
|
||||
def decision_values(document: dict[str, Any], start: float, end: float, lag: float) -> dict[tuple[str, str], float]:
|
||||
result: dict[tuple[str, str], float] = {}
|
||||
for index, row in enumerate(prometheus_rows(document, "decisions")):
|
||||
labels = row.get("metric")
|
||||
require(isinstance(labels, dict) and set(labels) == {"decision", "reason"},
|
||||
"decision evidence must contain only decision/reason labels")
|
||||
decision = labels.get("decision")
|
||||
reason = labels.get("reason")
|
||||
require(decision in DECISIONS and reason in REASONS, "decision evidence contains an unbounded label")
|
||||
key = (decision, reason)
|
||||
require(key not in result, f"duplicate decision series: {key}")
|
||||
result[key] = timestamped_value(row, start, end, lag, f"decision[{index}]")
|
||||
return result
|
||||
|
||||
|
||||
def scalar_sum(document: dict[str, Any], start: float, end: float, lag: float, name: str) -> float:
|
||||
total = 0.0
|
||||
rows = prometheus_rows(document, name)
|
||||
require(len(rows) == 1, f"{name} scalar query must return exactly one row")
|
||||
for index, row in enumerate(rows):
|
||||
labels = row.get("metric")
|
||||
require(labels == {}, f"{name} scalar query must not retain labels")
|
||||
total += timestamped_value(row, start, end, lag, f"{name}[{index}]")
|
||||
return total
|
||||
|
||||
|
||||
def artifact_path(root: Path, relative: Any, used: set[Path], name: str) -> Path:
|
||||
require(isinstance(relative, str) and relative and not Path(relative).is_absolute(), f"invalid {name} artifact path")
|
||||
path = (root / relative).resolve()
|
||||
try:
|
||||
path.relative_to(root)
|
||||
except ValueError as error:
|
||||
raise EvidenceError(f"{name} artifact escapes the evidence directory") from error
|
||||
require(path not in used, f"artifact reused across phases: {relative}")
|
||||
used.add(path)
|
||||
return path
|
||||
|
||||
|
||||
def bool_value(value: Any, name: str) -> bool:
|
||||
require(type(value) is bool, f"{name} must be boolean")
|
||||
return value
|
||||
|
||||
|
||||
def exact_env(document: Any, name: str, old: bool) -> None:
|
||||
require(isinstance(document, dict), f"{name} environment must be an object")
|
||||
expected = {
|
||||
"msgpack_only": not old,
|
||||
"peer_capability": not old,
|
||||
"fleet_confirmed": False,
|
||||
}
|
||||
require(document == expected, f"{name} environment does not match the peer-gate canary contract")
|
||||
|
||||
|
||||
def expected_queries(window: int) -> dict[str, str]:
|
||||
return {
|
||||
"decisions": (
|
||||
"sum by (decision, reason) "
|
||||
f"(increase(rustfs_system_network_internode_msgpack_send_decisions_total[{window}s]))"
|
||||
),
|
||||
"fallback": f"sum(increase(rustfs_system_network_internode_msgpack_json_fallback_total[{window}s])) or vector(0)",
|
||||
"decode_errors": (
|
||||
f"sum(increase(rustfs_system_network_internode_msgpack_json_decode_error_total[{window}s])) or vector(0)"
|
||||
),
|
||||
"rpc_errors": (
|
||||
f'(sum(increase(rustfs_system_network_internode_operation_errors_total{{backend="grpc",'
|
||||
f'operation!="grpc_read_version"}}[{window}s])) or vector(0)) + '
|
||||
"(sum(increase(rustfs_system_network_internode_operation_classified_errors_total"
|
||||
f'{{backend="grpc",operation="grpc_read_version",classification!="remote_response"}}[{window}s])) '
|
||||
"or vector(0))"
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def check(manifest: dict[str, Any], root: Path) -> dict[str, Any]:
|
||||
root = root.resolve()
|
||||
require(type(manifest.get("schema")) is int and manifest.get("schema") == 1, "unsupported canary evidence schema")
|
||||
require(manifest.get("scope") == "internode-msgpack-peer-canary", "unexpected evidence scope")
|
||||
source = manifest.get("source_revision")
|
||||
baseline = manifest.get("baseline_revision")
|
||||
require(isinstance(source, str) and REVISION.fullmatch(source) is not None, "invalid source revision")
|
||||
require(isinstance(baseline, str) and REVISION.fullmatch(baseline) is not None, "invalid baseline revision")
|
||||
require(source != baseline, "candidate and baseline revisions must differ")
|
||||
require(bool_value(manifest.get("stable_connection_affinity"), "stable_connection_affinity"),
|
||||
"stable connection-to-peer affinity is required")
|
||||
lag = bounded_number(manifest.get("max_metric_lag_secs"), "max_metric_lag_secs")
|
||||
require(0 <= lag <= 300, "max_metric_lag_secs exceeds the bounded limit")
|
||||
phases = manifest.get("phases")
|
||||
require(isinstance(phases, list) and len(phases) == len(PHASES), "exactly three canary phases are required")
|
||||
|
||||
used: set[Path] = set()
|
||||
receipts: list[dict[str, Any]] = []
|
||||
summaries: list[dict[str, Any]] = []
|
||||
violations: list[str] = []
|
||||
previous_end = 0.0
|
||||
for expected_name, phase in zip(PHASES, phases):
|
||||
require(isinstance(phase, dict) and phase.get("name") == expected_name, "canary phases are missing or out of order")
|
||||
start = uint(phase.get("start_unix"), f"{expected_name} start")
|
||||
end = uint(phase.get("end_unix"), f"{expected_name} end")
|
||||
require(previous_end < start < end and end - start <= 3600, f"invalid {expected_name} measurement window")
|
||||
previous_end = end
|
||||
topology = phase.get("topology")
|
||||
require(isinstance(topology, dict), f"{expected_name} topology must be an object")
|
||||
expected_new, expected_old = TOPOLOGIES[expected_name]
|
||||
nodes = topology.get("nodes")
|
||||
require(isinstance(nodes, dict) and set(nodes) == {"node1", "node2", "node3", "node4"},
|
||||
f"{expected_name} node inventory is incomplete")
|
||||
expected_nodes = {
|
||||
node: baseline if node in OLD_NODES[expected_name] else source
|
||||
for node in ("node1", "node2", "node3", "node4")
|
||||
}
|
||||
require(nodes == expected_nodes, f"{expected_name} node revisions do not match the mixed-version matrix")
|
||||
new_nodes = uint(topology.get("new_nodes"), f"{expected_name} new_nodes")
|
||||
old_nodes = uint(topology.get("old_nodes"), f"{expected_name} old_nodes")
|
||||
require(new_nodes == expected_new and old_nodes == expected_old,
|
||||
f"{expected_name} topology counts do not match the four-node matrix")
|
||||
environments = phase.get("environment")
|
||||
require(isinstance(environments, dict), f"{expected_name} environment is missing")
|
||||
exact_env(environments.get("new_nodes"), f"{expected_name} new_nodes", old=False)
|
||||
exact_env(environments.get("old_nodes"), f"{expected_name} old_nodes", old=True)
|
||||
queries = phase.get("queries")
|
||||
require(queries == expected_queries(end - start), f"{expected_name} PromQL does not match its exact window")
|
||||
workload_path = artifact_path(root, phase.get("workload"), used, f"{expected_name}.workload")
|
||||
workload, workload_digest = load_json(workload_path)
|
||||
summary_path = artifact_path(root, workload.get("summary"), used, f"{expected_name}.workload-summary")
|
||||
summary_digest = load_raw(summary_path)
|
||||
require(workload.get("summary_sha256") == summary_digest, f"{expected_name} workload summary hash mismatch")
|
||||
workload_success = workload.get("status") == "success"
|
||||
requests = uint(workload.get("requests"), f"{expected_name} workload requests")
|
||||
require(requests > 0, f"{expected_name} workload has no requests")
|
||||
if not workload_success:
|
||||
violations.append(f"{expected_name}: workload failed")
|
||||
rollback_readback = bool_value(workload.get("rollback_readback"), f"{expected_name} rollback_readback")
|
||||
if expected_name != "rollback-old-peer":
|
||||
require(not rollback_readback, f"{expected_name} cannot claim rollback readback")
|
||||
if expected_name == "rollback-old-peer" and not rollback_readback:
|
||||
violations.append(f"{expected_name}: rollback readback failed")
|
||||
source_digest = workload.get("readback_source_sha256")
|
||||
downloaded_digest = workload.get("readback_download_sha256")
|
||||
if expected_name == "rollback-old-peer":
|
||||
require(isinstance(source_digest, str) and SHA256.fullmatch(source_digest) is not None,
|
||||
"rollback source digest is invalid")
|
||||
require(source_digest == downloaded_digest, "rollback readback digest mismatch")
|
||||
else:
|
||||
require(source_digest is None and downloaded_digest is None,
|
||||
f"{expected_name} cannot claim rollback content digests")
|
||||
receipts.append({"phase": expected_name, "kind": "workload", "path": phase["workload"],
|
||||
"sha256": workload_digest})
|
||||
receipts.append({"phase": expected_name, "kind": "workload-summary", "path": workload["summary"],
|
||||
"sha256": summary_digest})
|
||||
|
||||
artifacts = phase.get("artifacts")
|
||||
require(isinstance(artifacts, dict) and set(artifacts) == set(ARTIFACTS), f"{expected_name} artifact set is incomplete")
|
||||
documents: dict[str, dict[str, Any]] = {}
|
||||
for artifact in ARTIFACTS:
|
||||
path = artifact_path(root, artifacts[artifact], used, f"{expected_name}.{artifact}")
|
||||
document, digest = load_json(path)
|
||||
documents[artifact] = document
|
||||
receipts.append({"phase": expected_name, "kind": artifact, "path": artifacts[artifact], "sha256": digest})
|
||||
|
||||
decisions = decision_values(documents["decisions"], start, end, lag)
|
||||
peer_capable = decisions.get(("msgpack_only", "peer_capable"), 0.0)
|
||||
compatibility = decisions.get(("dual_write", "peer_unknown"), 0.0) + decisions.get(
|
||||
("dual_write", "peer_revoked"), 0.0
|
||||
)
|
||||
if peer_capable <= 0:
|
||||
violations.append(f"{expected_name}: no peer_capable msgpack-only decision")
|
||||
if expected_old > 0 and compatibility <= 0:
|
||||
violations.append(f"{expected_name}: no old/unknown peer dual-write decision")
|
||||
if decisions.get(("msgpack_only", "manual_fleet"), 0.0) > 0:
|
||||
violations.append(f"{expected_name}: manual fleet path was unexpectedly active")
|
||||
if decisions.get(("dual_write", "not_requested"), 0.0) > 0:
|
||||
violations.append(f"{expected_name}: peer gate was not active on an eligible new-node request")
|
||||
|
||||
errors = {
|
||||
"fallback": scalar_sum(documents["fallback"], start, end, lag, "fallback"),
|
||||
"decode_errors": scalar_sum(documents["decode_errors"], start, end, lag, "decode_errors"),
|
||||
"rpc_errors": scalar_sum(documents["rpc_errors"], start, end, lag, "rpc_errors"),
|
||||
}
|
||||
for name, value in errors.items():
|
||||
if value != 0:
|
||||
violations.append(f"{expected_name}: nonzero {name} ({value})")
|
||||
summaries.append({
|
||||
"name": expected_name,
|
||||
"start_unix": start,
|
||||
"end_unix": end,
|
||||
"topology": topology,
|
||||
"workload_requests": requests,
|
||||
"peer_capable_decisions": peer_capable,
|
||||
"compatibility_decisions": compatibility,
|
||||
**errors,
|
||||
})
|
||||
|
||||
return {
|
||||
"observed_pass": not violations,
|
||||
"scope": manifest["scope"],
|
||||
"source_revision": source,
|
||||
"baseline_revision": baseline,
|
||||
"phases": summaries,
|
||||
"violations": violations,
|
||||
"artifacts": receipts,
|
||||
"limits": "Measured four-node compatibility evidence only; production soak and performance drift gates remain required.",
|
||||
}
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--manifest", type=Path, required=True)
|
||||
parser.add_argument("--out", type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
root = args.manifest.resolve().parent
|
||||
manifest, digest = load_json(args.manifest.resolve())
|
||||
result = check(manifest, root)
|
||||
inputs = {args.manifest.resolve()} | {(root / item["path"]).resolve() for item in result["artifacts"]}
|
||||
require(args.out.resolve() not in inputs, "output would overwrite raw canary evidence")
|
||||
result["manifest_sha256"] = digest
|
||||
args.out.write_text(json.dumps(result, indent=2, sort_keys=True, allow_nan=False) + "\n", encoding="utf-8")
|
||||
return 0 if result["observed_pass"] else 1
|
||||
except (EvidenceError, OSError, TypeError, KeyError, AttributeError, UnicodeError, json.JSONDecodeError) as error:
|
||||
print(f"Invalid internode msgpack peer canary evidence: {error}", file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -166,7 +166,7 @@ ENV_LINES="$(stage_env "${STAGE}" "${PHASE}")"
|
||||
# sourcing before/after in one shell cannot silently carry baseline overrides into a candidate.
|
||||
case "${STAGE}" in
|
||||
p0) RESET_VARS=(RUSTFS_INTERNODE_RPC_TCP_NODELAY RUSTFS_INTERNODE_RPC_HTTP2_STREAM_WINDOW_SIZE RUSTFS_INTERNODE_RPC_HTTP2_CONN_WINDOW_SIZE RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE) ;;
|
||||
p1) RESET_VARS=(RUSTFS_INTERNODE_CHANNEL_ISOLATION RUSTFS_INTERNODE_BULK_CHANNELS RUSTFS_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT RUSTFS_INTERNODE_BULK_BUFFER_SIZE) ;;
|
||||
p1) RESET_VARS=(RUSTFS_INTERNODE_CHANNEL_ISOLATION RUSTFS_INTERNODE_BULK_CHANNELS RUSTFS_INTERNODE_BULK_HTTP2_STREAM_WINDOW_SIZE RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT RUSTFS_INTERNODE_BULK_BUFFER_SIZE RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS) ;;
|
||||
p2) RESET_VARS=(RUSTFS_INTERNODE_RPC_MSGPACK_ONLY RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED) ;;
|
||||
p3) RESET_VARS=(RUSTFS_INTERNODE_PREWARM RUSTFS_INTERNODE_OFFLINE_BYPASS RUSTFS_INTERNODE_OFFLINE_REPROBE_SECS RUSTFS_INTERNODE_OFFLINE_FAILURE_THRESHOLD) ;;
|
||||
esac
|
||||
|
||||
Executable
+496
@@ -0,0 +1,496 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
CLUSTER_COMPOSE="${CLUSTER_COMPOSE:-${PROJECT_ROOT}/.docker/compose/docker-compose.cluster.local-build.yml}"
|
||||
PROJECT_NAME="${PROJECT_NAME:-rustfs-msgpack-peer-canary}"
|
||||
OUT_DIR="${OUT_DIR:-${PROJECT_ROOT}/target/validation/internode-msgpack-peer-$(date +%Y%m%d-%H%M%S)}"
|
||||
PROMETHEUS_QUERY_URL="${PROMETHEUS_QUERY_URL:-http://127.0.0.1:9090/api/v1/query}"
|
||||
WORKLOAD_RUNNER="${WORKLOAD_RUNNER:-${PROJECT_ROOT}/scripts/run_object_batch_bench.sh}"
|
||||
PYTHON_BIN="${RUSTFS_PYTHON_BIN:-python3}"
|
||||
CANDIDATE_IMAGE=""
|
||||
BASELINE_IMAGE=""
|
||||
CANDIDATE_REVISION=""
|
||||
BASELINE_REVISION=""
|
||||
EXECUTE=false
|
||||
KEEP_UP=false
|
||||
WAIT_TIMEOUT_SECS="${WAIT_TIMEOUT_SECS:-180}"
|
||||
METRICS_SETTLE_SECS="${METRICS_SETTLE_SECS:-15}"
|
||||
WORKLOAD_DURATION="${WORKLOAD_DURATION:-20s}"
|
||||
WORKLOAD_CONCURRENCY="${WORKLOAD_CONCURRENCY:-8}"
|
||||
WORKLOAD_SIZES="${WORKLOAD_SIZES:-4KiB,1MiB}"
|
||||
RUSTFS_ACCESS_KEY="${RUSTFS_ACCESS_KEY:-rustfsadmin-local}"
|
||||
RUSTFS_SECRET_KEY="${RUSTFS_SECRET_KEY:-rustfssecret-local}"
|
||||
S3_ENDPOINT="${S3_ENDPOINT:-http://127.0.0.1:9000}"
|
||||
CANARY_BUCKET="${CANARY_BUCKET:-rustfs-msgpack-peer-canary}"
|
||||
LAST_PHASE_END=0
|
||||
ACTIVE_OVERRIDE=""
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/run_internode_msgpack_peer_canary.sh \
|
||||
--candidate-image IMAGE --baseline-image IMAGE \
|
||||
--candidate-revision 40HEX --baseline-revision 40HEX [--execute]
|
||||
|
||||
Options:
|
||||
--candidate-image IMAGE Candidate RustFS image.
|
||||
--baseline-image IMAGE Previous-version image lacking the peer capability.
|
||||
--candidate-revision 40HEX Exact source revision used by the candidate image.
|
||||
--baseline-revision 40HEX Exact source revision used by the baseline image.
|
||||
--cluster-compose PATH Four-node compose file.
|
||||
--project-name NAME Isolated Docker Compose project name.
|
||||
--prometheus-query-url URL Prometheus /api/v1/query endpoint.
|
||||
--workload-runner PATH Object batch workload runner.
|
||||
--out-dir PATH Evidence directory.
|
||||
--execute Run Docker, workload and Prometheus capture. Without this, print the plan.
|
||||
--keep-up Leave the Compose project running after execution.
|
||||
-h, --help Show this help.
|
||||
|
||||
Environment tuning:
|
||||
RUSTFS_ACCESS_KEY RUSTFS_SECRET_KEY S3_ENDPOINT CANARY_BUCKET
|
||||
WAIT_TIMEOUT_SECS METRICS_SETTLE_SECS WORKLOAD_DURATION
|
||||
WORKLOAD_CONCURRENCY WORKLOAD_SIZES RUSTFS_PYTHON_BIN
|
||||
|
||||
Phases:
|
||||
mixed-old-peer node1/2/3 candidate, node4 baseline
|
||||
all-new node1/2/3/4 candidate
|
||||
rollback-old-peer node1/3/4 candidate, node2 baseline, known-object readback required
|
||||
USAGE
|
||||
}
|
||||
|
||||
die() {
|
||||
printf 'ERROR: %s\n' "$*" >&2
|
||||
exit 2
|
||||
}
|
||||
|
||||
log() {
|
||||
printf '[msgpack-peer-canary] %s\n' "$*"
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
command -v "$1" >/dev/null 2>&1 || die "command not found: $1"
|
||||
}
|
||||
|
||||
validate_token() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
[[ "$value" =~ ^[A-Za-z0-9][A-Za-z0-9._/:@-]*$ ]] || die "invalid ${name}: ${value}"
|
||||
}
|
||||
|
||||
validate_revision() {
|
||||
local name="$1"
|
||||
local value="$2"
|
||||
[[ "$value" =~ ^[0-9a-f]{40}$ ]] || die "${name} must be a lowercase 40-hex revision"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--candidate-image) CANDIDATE_IMAGE="${2:-}"; shift 2 ;;
|
||||
--baseline-image) BASELINE_IMAGE="${2:-}"; shift 2 ;;
|
||||
--candidate-revision) CANDIDATE_REVISION="${2:-}"; shift 2 ;;
|
||||
--baseline-revision) BASELINE_REVISION="${2:-}"; shift 2 ;;
|
||||
--cluster-compose) CLUSTER_COMPOSE="${2:-}"; shift 2 ;;
|
||||
--project-name) PROJECT_NAME="${2:-}"; shift 2 ;;
|
||||
--prometheus-query-url) PROMETHEUS_QUERY_URL="${2:-}"; shift 2 ;;
|
||||
--workload-runner) WORKLOAD_RUNNER="${2:-}"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="${2:-}"; shift 2 ;;
|
||||
--execute) EXECUTE=true; shift ;;
|
||||
--keep-up) KEEP_UP=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*) die "unknown argument: $1" ;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
validate() {
|
||||
[[ -n "$CANDIDATE_IMAGE" && -n "$BASELINE_IMAGE" ]] || die "candidate and baseline images are required"
|
||||
[[ "$CANDIDATE_IMAGE" != "$BASELINE_IMAGE" ]] || die "candidate and baseline images must differ"
|
||||
validate_token candidate-image "$CANDIDATE_IMAGE"
|
||||
validate_token baseline-image "$BASELINE_IMAGE"
|
||||
validate_token project-name "$PROJECT_NAME"
|
||||
[[ "$CANARY_BUCKET" =~ ^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$ ]] || die "invalid S3 canary bucket: $CANARY_BUCKET"
|
||||
validate_revision candidate-revision "$CANDIDATE_REVISION"
|
||||
validate_revision baseline-revision "$BASELINE_REVISION"
|
||||
[[ "$CANDIDATE_REVISION" != "$BASELINE_REVISION" ]] || die "candidate and baseline revisions must differ"
|
||||
[[ -f "$CLUSTER_COMPOSE" ]] || die "cluster compose file not found: $CLUSTER_COMPOSE"
|
||||
[[ -x "$WORKLOAD_RUNNER" ]] || die "workload runner is not executable: $WORKLOAD_RUNNER"
|
||||
[[ "$WAIT_TIMEOUT_SECS" =~ ^[0-9]+$ && "$WAIT_TIMEOUT_SECS" -gt 0 ]] || die "invalid WAIT_TIMEOUT_SECS"
|
||||
[[ "$METRICS_SETTLE_SECS" =~ ^[0-9]+$ ]] || die "invalid METRICS_SETTLE_SECS"
|
||||
[[ "$WORKLOAD_CONCURRENCY" =~ ^[0-9]+$ && "$WORKLOAD_CONCURRENCY" -gt 0 ]] || die "invalid WORKLOAD_CONCURRENCY"
|
||||
[[ "$WORKLOAD_DURATION" =~ ^[1-9][0-9]*(s|m)$ ]] || die "invalid WORKLOAD_DURATION"
|
||||
}
|
||||
|
||||
phase_is_old() {
|
||||
local phase="$1"
|
||||
local node="$2"
|
||||
[[ "$phase" == "mixed-old-peer" && "$node" == "node4" ]] ||
|
||||
[[ "$phase" == "rollback-old-peer" && "$node" == "node2" ]]
|
||||
}
|
||||
|
||||
phase_image() {
|
||||
if phase_is_old "$1" "$2"; then
|
||||
printf '%s\n' "$BASELINE_IMAGE"
|
||||
else
|
||||
printf '%s\n' "$CANDIDATE_IMAGE"
|
||||
fi
|
||||
}
|
||||
|
||||
write_override() {
|
||||
local phase="$1"
|
||||
local path="$2"
|
||||
local node image requested peer
|
||||
{
|
||||
printf 'services:\n'
|
||||
for node in node1 node2 node3 node4; do
|
||||
image="$(phase_image "$phase" "$node")"
|
||||
if phase_is_old "$phase" "$node"; then
|
||||
requested=false
|
||||
peer=false
|
||||
else
|
||||
requested=true
|
||||
peer=true
|
||||
fi
|
||||
printf ' %s:\n' "$node"
|
||||
printf ' image: "%s"\n' "$image"
|
||||
printf ' environment:\n'
|
||||
printf ' RUSTFS_INTERNODE_RPC_MSGPACK_ONLY: "%s"\n' "$requested"
|
||||
printf ' RUSTFS_INTERNODE_RPC_MSGPACK_PEER_CAPABILITY: "%s"\n' "$peer"
|
||||
printf ' RUSTFS_INTERNODE_RPC_MSGPACK_ONLY_FLEET_CONFIRMED: "false"\n'
|
||||
done
|
||||
} >"$path"
|
||||
}
|
||||
|
||||
compose() {
|
||||
docker compose --project-name "$PROJECT_NAME" -f "$CLUSTER_COMPOSE" -f "$ACTIVE_OVERRIDE" "$@"
|
||||
}
|
||||
|
||||
cleanup() {
|
||||
if [[ "$EXECUTE" == "true" && "$KEEP_UP" != "true" && -n "$ACTIVE_OVERRIDE" ]]; then
|
||||
compose down --volumes --remove-orphans >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
preflight_isolated_project() {
|
||||
local containers volumes
|
||||
containers="$(docker ps -aq --filter "label=com.docker.compose.project=${PROJECT_NAME}")"
|
||||
volumes="$(docker volume ls -q --filter "label=com.docker.compose.project=${PROJECT_NAME}")"
|
||||
[[ -z "$containers" ]] || die "Compose project already has containers: ${PROJECT_NAME}"
|
||||
[[ -z "$volumes" ]] || die "Compose project already has volumes: ${PROJECT_NAME}"
|
||||
}
|
||||
|
||||
preflight_prometheus() {
|
||||
local output="$OUT_DIR/prometheus-preflight.json"
|
||||
prometheus_query "vector(0)" "$(date +%s)" "$output"
|
||||
"$PYTHON_BIN" - "$output" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
document = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8"))
|
||||
if document.get("status") != "success" or document.get("data", {}).get("resultType") != "vector":
|
||||
raise SystemExit("Prometheus preflight did not return a successful vector")
|
||||
PY
|
||||
}
|
||||
|
||||
node_port() {
|
||||
case "$1" in
|
||||
node1) printf '9000\n' ;;
|
||||
node2) printf '9001\n' ;;
|
||||
node3) printf '9002\n' ;;
|
||||
node4) printf '9003\n' ;;
|
||||
*) die "unknown node: $1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
wait_node() {
|
||||
local node="$1"
|
||||
local port start now
|
||||
port="$(node_port "$node")"
|
||||
start="$(date +%s)"
|
||||
while ! curl -fsS --connect-timeout 2 --max-time 3 "http://127.0.0.1:${port}/health" >/dev/null 2>&1; do
|
||||
now="$(date +%s)"
|
||||
(( now - start < WAIT_TIMEOUT_SECS )) || die "timed out waiting for ${node}"
|
||||
sleep 2
|
||||
done
|
||||
}
|
||||
|
||||
wait_cluster() {
|
||||
local node
|
||||
for node in node1 node2 node3 node4; do
|
||||
wait_node "$node"
|
||||
done
|
||||
}
|
||||
|
||||
aws_s3() {
|
||||
AWS_ACCESS_KEY_ID="$RUSTFS_ACCESS_KEY" AWS_SECRET_ACCESS_KEY="$RUSTFS_SECRET_KEY" \
|
||||
AWS_EC2_METADATA_DISABLED=true aws --endpoint-url "$S3_ENDPOINT" --region us-east-1 "$@"
|
||||
}
|
||||
|
||||
prepare_readback_object() {
|
||||
local payload="$OUT_DIR/readback-source.bin"
|
||||
printf 'rustfs-msgpack-peer-canary\nsource=%s\nbaseline=%s\n' "$CANDIDATE_REVISION" "$BASELINE_REVISION" >"$payload"
|
||||
if ! aws_s3 s3api head-bucket --bucket "$CANARY_BUCKET" >/dev/null 2>&1; then
|
||||
aws_s3 s3api create-bucket --bucket "$CANARY_BUCKET" >/dev/null
|
||||
fi
|
||||
aws_s3 s3api put-object --bucket "$CANARY_BUCKET" --key rollback/readback.bin --body "$payload" >/dev/null
|
||||
}
|
||||
|
||||
verify_readback_object() {
|
||||
local downloaded="$OUT_DIR/rollback-readback.bin"
|
||||
aws_s3 s3api get-object --bucket "$CANARY_BUCKET" --key rollback/readback.bin "$downloaded" >/dev/null
|
||||
cmp -s "$OUT_DIR/readback-source.bin" "$downloaded"
|
||||
}
|
||||
|
||||
duration_seconds() {
|
||||
"$PYTHON_BIN" - "$WORKLOAD_DURATION" <<'PY'
|
||||
import re
|
||||
import sys
|
||||
value = sys.argv[1]
|
||||
match = re.fullmatch(r"([1-9][0-9]*)(s|m)", value)
|
||||
if not match:
|
||||
raise SystemExit("duration must be positive integer seconds or minutes")
|
||||
amount = int(match.group(1))
|
||||
print(amount if match.group(2) == "s" else amount * 60)
|
||||
PY
|
||||
}
|
||||
|
||||
write_workload_evidence() {
|
||||
local summary="$1"
|
||||
local rollback="$2"
|
||||
local output="$3"
|
||||
local seconds
|
||||
seconds="$(duration_seconds)"
|
||||
"$PYTHON_BIN" - "$OUT_DIR" "$summary" "$seconds" "$rollback" "$output" <<'PY'
|
||||
import csv
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import sys
|
||||
|
||||
root, summary = Path(sys.argv[1]).resolve(), Path(sys.argv[2]).resolve()
|
||||
seconds, rollback, output = int(sys.argv[3]), sys.argv[4] == "true", Path(sys.argv[5])
|
||||
rows = list(csv.DictReader(summary.open(newline="", encoding="utf-8")))
|
||||
if not rows or any(row.get("status") != "ok" or row.get("error_count") != "0" for row in rows):
|
||||
raise SystemExit("workload summary contains failed or error rows")
|
||||
requests = 0
|
||||
for row in rows:
|
||||
match = re.match(r"([0-9]+(?:\.[0-9]+)?)", row.get("requests_per_sec", ""))
|
||||
if not match:
|
||||
raise SystemExit("workload summary lacks requests_per_sec")
|
||||
requests += max(1, int(float(match.group(1)) * seconds))
|
||||
summary_bytes = summary.read_bytes()
|
||||
source_digest = downloaded_digest = None
|
||||
if rollback:
|
||||
source_digest = hashlib.sha256((root / "readback-source.bin").read_bytes()).hexdigest()
|
||||
downloaded_digest = hashlib.sha256((root / "rollback-readback.bin").read_bytes()).hexdigest()
|
||||
output.write_text(json.dumps({
|
||||
"status": "success",
|
||||
"requests": requests,
|
||||
"rollback_readback": rollback,
|
||||
"summary": summary.relative_to(root).as_posix(),
|
||||
"summary_sha256": hashlib.sha256(summary_bytes).hexdigest(),
|
||||
"readback_source_sha256": source_digest,
|
||||
"readback_download_sha256": downloaded_digest,
|
||||
}, sort_keys=True) + "\n", encoding="utf-8")
|
||||
PY
|
||||
}
|
||||
|
||||
run_workload() {
|
||||
local phase="$1"
|
||||
local phase_dir="$2"
|
||||
local bench_dir="$phase_dir/workload-bench"
|
||||
if ! "$WORKLOAD_RUNNER" \
|
||||
--tool warp \
|
||||
--endpoint "http://127.0.0.1:9000,http://127.0.0.1:9001,http://127.0.0.1:9002,http://127.0.0.1:9003" \
|
||||
--access-key "$RUSTFS_ACCESS_KEY" \
|
||||
--secret-key "$RUSTFS_SECRET_KEY" \
|
||||
--bucket-prefix "msgpack-peer-${phase}" \
|
||||
--auto-new-bucket \
|
||||
--warp-mode mixed \
|
||||
--concurrency "$WORKLOAD_CONCURRENCY" \
|
||||
--duration "$WORKLOAD_DURATION" \
|
||||
--sizes "$WORKLOAD_SIZES" \
|
||||
--out-dir "$bench_dir" >"$phase_dir/workload-runner.log" 2>&1; then
|
||||
cat "$phase_dir/workload-runner.log" >&2
|
||||
die "workload runner failed for ${phase}"
|
||||
fi
|
||||
[[ -f "$bench_dir/summary.csv" ]] || die "workload did not produce summary.csv for ${phase}"
|
||||
printf '%s\n' "$bench_dir/summary.csv"
|
||||
}
|
||||
|
||||
prometheus_query() {
|
||||
local query="$1"
|
||||
local at="$2"
|
||||
local output="$3"
|
||||
local tmp="${output}.tmp"
|
||||
curl -fsS -G "$PROMETHEUS_QUERY_URL" \
|
||||
--data-urlencode "query=${query}" \
|
||||
--data-urlencode "time=${at}" >"$tmp"
|
||||
mv "$tmp" "$output"
|
||||
}
|
||||
|
||||
wait_next_phase_second() {
|
||||
local now
|
||||
while true; do
|
||||
now="$(date +%s)"
|
||||
if (( now > LAST_PHASE_END )); then
|
||||
printf '%s\n' "$now"
|
||||
return
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
}
|
||||
|
||||
capture_metrics() {
|
||||
local phase_dir="$1"
|
||||
local start="$2"
|
||||
local end="$3"
|
||||
local window=$((end - start))
|
||||
local decisions fallback decode_errors rpc_errors
|
||||
decisions="sum by (decision, reason) (increase(rustfs_system_network_internode_msgpack_send_decisions_total[${window}s]))"
|
||||
fallback="sum(increase(rustfs_system_network_internode_msgpack_json_fallback_total[${window}s])) or vector(0)"
|
||||
decode_errors="sum(increase(rustfs_system_network_internode_msgpack_json_decode_error_total[${window}s])) or vector(0)"
|
||||
rpc_errors="(sum(increase(rustfs_system_network_internode_operation_errors_total{backend=\"grpc\",operation!=\"grpc_read_version\"}[${window}s])) or vector(0)) + (sum(increase(rustfs_system_network_internode_operation_classified_errors_total{backend=\"grpc\",operation=\"grpc_read_version\",classification!=\"remote_response\"}[${window}s])) or vector(0))"
|
||||
prometheus_query "$decisions" "$end" "$phase_dir/decisions.json"
|
||||
prometheus_query "$fallback" "$end" "$phase_dir/fallback.json"
|
||||
prometheus_query "$decode_errors" "$end" "$phase_dir/decode_errors.json"
|
||||
prometheus_query "$rpc_errors" "$end" "$phase_dir/rpc_errors.json"
|
||||
"$PYTHON_BIN" - "$phase_dir/queries.json" "$decisions" "$fallback" "$decode_errors" "$rpc_errors" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
Path(sys.argv[1]).write_text(json.dumps({"decisions": sys.argv[2], "fallback": sys.argv[3],
|
||||
"decode_errors": sys.argv[4], "rpc_errors": sys.argv[5]},
|
||||
sort_keys=True) + "\n", encoding="utf-8")
|
||||
PY
|
||||
}
|
||||
|
||||
phase_nodes() {
|
||||
case "$1" in
|
||||
mixed-old-peer) printf 'node1 node2 node3 node4\n' ;;
|
||||
all-new) printf 'node4\n' ;;
|
||||
rollback-old-peer) printf 'node2\n' ;;
|
||||
*) die "unknown phase: $1" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
run_phase() {
|
||||
local phase="$1"
|
||||
local phase_dir="$OUT_DIR/$phase"
|
||||
local override="$phase_dir/compose.override.yml"
|
||||
local summary start end rollback=false
|
||||
local -a nodes
|
||||
mkdir -p "$phase_dir"
|
||||
write_override "$phase" "$override"
|
||||
ACTIVE_OVERRIDE="$override"
|
||||
read -r -a nodes <<<"$(phase_nodes "$phase")"
|
||||
log "applying ${phase}: ${nodes[*]}"
|
||||
compose up -d --no-build --force-recreate "${nodes[@]}"
|
||||
wait_cluster
|
||||
if [[ "$phase" == "mixed-old-peer" ]]; then
|
||||
prepare_readback_object
|
||||
fi
|
||||
start="$(wait_next_phase_second)"
|
||||
summary="$(run_workload "$phase" "$phase_dir")"
|
||||
if [[ "$phase" == "rollback-old-peer" ]]; then
|
||||
verify_readback_object || die "rollback readback mismatch"
|
||||
rollback=true
|
||||
fi
|
||||
sleep "$METRICS_SETTLE_SECS"
|
||||
end="$(date +%s)"
|
||||
while (( end <= start )); do sleep 1; end="$(date +%s)"; done
|
||||
LAST_PHASE_END="$end"
|
||||
write_workload_evidence "$summary" "$rollback" "$phase_dir/workload.json"
|
||||
capture_metrics "$phase_dir" "$start" "$end"
|
||||
printf '%s\t%s\t%s\n' "$phase" "$start" "$end" >>"$OUT_DIR/phases.tsv"
|
||||
}
|
||||
|
||||
write_manifest() {
|
||||
PYTHONDONTWRITEBYTECODE=1 PYTHONPATH="$PROJECT_ROOT/scripts" "$PYTHON_BIN" - \
|
||||
"$OUT_DIR" "$CANDIDATE_REVISION" "$BASELINE_REVISION" <<'PY'
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
from check_internode_msgpack_peer_canary import expected_queries, OLD_NODES
|
||||
|
||||
root, candidate, baseline = Path(sys.argv[1]), sys.argv[2], sys.argv[3]
|
||||
phases = []
|
||||
for line in (root / "phases.tsv").read_text(encoding="utf-8").splitlines():
|
||||
name, start, end = line.split("\t")
|
||||
start, end = int(start), int(end)
|
||||
old = OLD_NODES[name]
|
||||
nodes = {node: (baseline if node in old else candidate) for node in ("node1", "node2", "node3", "node4")}
|
||||
phases.append({
|
||||
"name": name,
|
||||
"start_unix": start,
|
||||
"end_unix": end,
|
||||
"topology": {"new_nodes": 4 - len(old), "old_nodes": len(old), "nodes": nodes},
|
||||
"environment": {
|
||||
"new_nodes": {"msgpack_only": True, "peer_capability": True, "fleet_confirmed": False},
|
||||
"old_nodes": {"msgpack_only": False, "peer_capability": False, "fleet_confirmed": False},
|
||||
},
|
||||
"queries": expected_queries(end - start),
|
||||
"workload": f"{name}/workload.json",
|
||||
"artifacts": {kind: f"{name}/{kind}.json" for kind in
|
||||
("decisions", "fallback", "decode_errors", "rpc_errors")},
|
||||
})
|
||||
manifest = {
|
||||
"schema": 1,
|
||||
"scope": "internode-msgpack-peer-canary",
|
||||
"source_revision": candidate,
|
||||
"baseline_revision": baseline,
|
||||
"stable_connection_affinity": True,
|
||||
"max_metric_lag_secs": 120,
|
||||
"phases": phases,
|
||||
}
|
||||
(root / "manifest.json").write_text(json.dumps(manifest, indent=2, sort_keys=True) + "\n", encoding="utf-8")
|
||||
PY
|
||||
}
|
||||
|
||||
print_plan() {
|
||||
printf 'candidate_image=%s\n' "$CANDIDATE_IMAGE"
|
||||
printf 'baseline_image=%s\n' "$BASELINE_IMAGE"
|
||||
printf 'candidate_revision=%s\n' "$CANDIDATE_REVISION"
|
||||
printf 'baseline_revision=%s\n' "$BASELINE_REVISION"
|
||||
printf 'project_name=%s\n' "$PROJECT_NAME"
|
||||
printf 'out_dir=%s\n' "$OUT_DIR"
|
||||
printf 'phases=mixed-old-peer all-new rollback-old-peer\n'
|
||||
printf 'verifier=scripts/check_internode_msgpack_peer_canary.py\n'
|
||||
printf 'execute=false\n'
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
validate
|
||||
if [[ "$EXECUTE" != "true" ]]; then
|
||||
print_plan
|
||||
return 0
|
||||
fi
|
||||
require_cmd docker
|
||||
require_cmd curl
|
||||
require_cmd aws
|
||||
require_cmd cmp
|
||||
require_cmd "$PYTHON_BIN"
|
||||
docker info >/dev/null
|
||||
docker image inspect "$CANDIDATE_IMAGE" "$BASELINE_IMAGE" >/dev/null
|
||||
preflight_isolated_project
|
||||
if [[ -d "$OUT_DIR" && -n "$(find "$OUT_DIR" -mindepth 1 -maxdepth 1 -print -quit)" ]]; then
|
||||
die "evidence directory is not empty: $OUT_DIR"
|
||||
fi
|
||||
mkdir -p "$OUT_DIR"
|
||||
preflight_prometheus
|
||||
: >"$OUT_DIR/phases.tsv"
|
||||
trap cleanup EXIT
|
||||
run_phase mixed-old-peer
|
||||
run_phase all-new
|
||||
run_phase rollback-old-peer
|
||||
write_manifest
|
||||
"$PYTHON_BIN" "$PROJECT_ROOT/scripts/check_internode_msgpack_peer_canary.py" \
|
||||
--manifest "$OUT_DIR/manifest.json" \
|
||||
--out "$OUT_DIR/verification.json"
|
||||
log "verified evidence: $OUT_DIR/verification.json"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
@@ -104,10 +104,14 @@ echo "internode grpc A/B bench env tests passed"
|
||||
[[ "$RUSTFS_INTERNODE_RPC_MAX_MESSAGE_SIZE" == 4194304 ]]
|
||||
export RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE=33554432
|
||||
export RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT=1
|
||||
export RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT=1
|
||||
export RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS=25
|
||||
# shellcheck source=/dev/null
|
||||
source "$P1_AFTER_ENV"
|
||||
[[ "$RUSTFS_INTERNODE_CHANNEL_ISOLATION" == true ]]
|
||||
[[ -z "${RUSTFS_INTERNODE_BULK_HTTP2_CONN_WINDOW_SIZE+x}" && -z "${RUSTFS_INTERNODE_BULK_CONCURRENCY_LIMIT+x}" ]]
|
||||
[[ -z "${RUSTFS_INTERNODE_BULK_BODY_CONCURRENCY_LIMIT+x}" ]]
|
||||
[[ -z "${RUSTFS_INTERNODE_BULK_BODY_WAIT_TIMEOUT_MS+x}" ]]
|
||||
# shellcheck source=/dev/null
|
||||
source "$P1_BEFORE_ENV"
|
||||
[[ "$RUSTFS_INTERNODE_CHANNEL_ISOLATION" == false && -z "${RUSTFS_INTERNODE_BULK_CHANNELS+x}" ]]
|
||||
|
||||
@@ -0,0 +1,286 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import importlib.util
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
|
||||
SCRIPT = Path(__file__).with_name("check_internode_msgpack_peer_canary.py")
|
||||
SPEC = importlib.util.spec_from_file_location("msgpack_peer_canary", SCRIPT)
|
||||
assert SPEC and SPEC.loader
|
||||
canary = importlib.util.module_from_spec(SPEC)
|
||||
SPEC.loader.exec_module(canary)
|
||||
|
||||
|
||||
def prometheus(rows):
|
||||
return {"status": "success", "data": {"resultType": "vector", "result": rows}}
|
||||
|
||||
|
||||
def row(timestamp, value, **labels):
|
||||
return {"metric": labels, "value": [timestamp, str(value)]}
|
||||
|
||||
|
||||
class MsgpackPeerCanaryTests(unittest.TestCase):
|
||||
def test_rpc_error_query_scopes_application_response_exclusion_to_read_version(self):
|
||||
query = canary.expected_queries(30)["rpc_errors"]
|
||||
self.assertIn('internode_operation_errors_total{backend="grpc",operation!="grpc_read_version"}', query)
|
||||
self.assertIn("internode_operation_classified_errors_total", query)
|
||||
self.assertIn(
|
||||
'backend="grpc",operation="grpc_read_version",classification!="remote_response"',
|
||||
query,
|
||||
)
|
||||
self.assertNotIn("http_status_other", query)
|
||||
|
||||
def fixture(self):
|
||||
temp = tempfile.TemporaryDirectory()
|
||||
root = Path(temp.name)
|
||||
phases = []
|
||||
layouts = [
|
||||
("mixed-old-peer", 100, 110, 3, 1, [row(110, 4, decision="msgpack_only", reason="peer_capable"),
|
||||
row(110, 2, decision="dual_write", reason="peer_unknown")]),
|
||||
("all-new", 120, 130, 4, 0, [row(130, 8, decision="msgpack_only", reason="peer_capable")]),
|
||||
("rollback-old-peer", 140, 150, 3, 1, [row(150, 5, decision="msgpack_only", reason="peer_capable"),
|
||||
row(150, 3, decision="dual_write", reason="peer_revoked")]),
|
||||
]
|
||||
for name, start, end, new_nodes, old_nodes, decisions in layouts:
|
||||
nodes = {
|
||||
node: ("b" * 40 if node in canary.OLD_NODES[name] else "a" * 40)
|
||||
for node in ("node1", "node2", "node3", "node4")
|
||||
}
|
||||
artifacts = {}
|
||||
for kind, document in [
|
||||
("decisions", prometheus(decisions)),
|
||||
("fallback", prometheus([row(end, 0)])),
|
||||
("decode_errors", prometheus([row(end, 0)])),
|
||||
("rpc_errors", prometheus([row(end, 0)])),
|
||||
]:
|
||||
relative = f"{name}/{kind}.json"
|
||||
path = root / relative
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(document), encoding="utf-8")
|
||||
artifacts[kind] = relative
|
||||
workload_relative = f"{name}/workload.json"
|
||||
summary_relative = f"{name}/workload-bench/summary.csv"
|
||||
summary_path = root / summary_relative
|
||||
summary_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
summary_bytes = b"size,status,requests_per_sec,error_count\n4KiB,ok,10 req/s,0\n"
|
||||
summary_path.write_bytes(summary_bytes)
|
||||
readback_digest = "c" * 64 if name == "rollback-old-peer" else None
|
||||
(root / workload_relative).write_text(
|
||||
json.dumps({
|
||||
"status": "success",
|
||||
"requests": 100,
|
||||
"rollback_readback": name == "rollback-old-peer",
|
||||
"summary": summary_relative,
|
||||
"summary_sha256": hashlib.sha256(summary_bytes).hexdigest(),
|
||||
"readback_source_sha256": readback_digest,
|
||||
"readback_download_sha256": readback_digest,
|
||||
}),
|
||||
encoding="utf-8",
|
||||
)
|
||||
phases.append({
|
||||
"name": name,
|
||||
"start_unix": start,
|
||||
"end_unix": end,
|
||||
"topology": {"new_nodes": new_nodes, "old_nodes": old_nodes, "nodes": nodes},
|
||||
"environment": {
|
||||
"new_nodes": {"msgpack_only": True, "peer_capability": True, "fleet_confirmed": False},
|
||||
"old_nodes": {"msgpack_only": False, "peer_capability": False, "fleet_confirmed": False},
|
||||
},
|
||||
"queries": canary.expected_queries(end - start),
|
||||
"workload": workload_relative,
|
||||
"artifacts": artifacts,
|
||||
})
|
||||
manifest = {
|
||||
"schema": 1,
|
||||
"scope": "internode-msgpack-peer-canary",
|
||||
"source_revision": "a" * 40,
|
||||
"baseline_revision": "b" * 40,
|
||||
"stable_connection_affinity": True,
|
||||
"max_metric_lag_secs": 30,
|
||||
"phases": phases,
|
||||
}
|
||||
return temp, root, manifest
|
||||
|
||||
def test_valid_four_node_matrix_passes_with_receipts(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
result = canary.check(manifest, root)
|
||||
self.assertTrue(result["observed_pass"])
|
||||
self.assertEqual(result["violations"], [])
|
||||
self.assertEqual(len(result["artifacts"]), 18)
|
||||
self.assertTrue(all(len(item["sha256"]) == 64 for item in result["artifacts"]))
|
||||
|
||||
def test_nonzero_decode_error_is_valid_evidence_but_fails_gate(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
phase = manifest["phases"][1]
|
||||
path = root / phase["artifacts"]["decode_errors"]
|
||||
path.write_text(json.dumps(prometheus([row(phase["end_unix"], 1)])), encoding="utf-8")
|
||||
result = canary.check(manifest, root)
|
||||
self.assertFalse(result["observed_pass"])
|
||||
self.assertIn("all-new: nonzero decode_errors (1.0)", result["violations"])
|
||||
|
||||
def test_old_peer_phase_requires_a_compatibility_decision(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
phase = manifest["phases"][0]
|
||||
path = root / phase["artifacts"]["decisions"]
|
||||
path.write_text(
|
||||
json.dumps(prometheus([row(phase["end_unix"], 4, decision="msgpack_only", reason="peer_capable")])),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = canary.check(manifest, root)
|
||||
self.assertFalse(result["observed_pass"])
|
||||
self.assertIn("mixed-old-peer: no old/unknown peer dual-write decision", result["violations"])
|
||||
|
||||
def test_manual_fleet_and_not_requested_series_fail_peer_gate_canary(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
phase = manifest["phases"][1]
|
||||
path = root / phase["artifacts"]["decisions"]
|
||||
path.write_text(
|
||||
json.dumps(prometheus([
|
||||
row(phase["end_unix"], 4, decision="msgpack_only", reason="peer_capable"),
|
||||
row(phase["end_unix"], 1, decision="msgpack_only", reason="manual_fleet"),
|
||||
row(phase["end_unix"], 1, decision="dual_write", reason="not_requested"),
|
||||
])),
|
||||
encoding="utf-8",
|
||||
)
|
||||
result = canary.check(manifest, root)
|
||||
self.assertFalse(result["observed_pass"])
|
||||
self.assertIn("all-new: manual fleet path was unexpectedly active", result["violations"])
|
||||
self.assertIn("all-new: peer gate was not active on an eligible new-node request", result["violations"])
|
||||
|
||||
def test_invalid_affinity_path_and_revision_are_rejected(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
manifest["schema"] = True
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "schema"):
|
||||
canary.check(manifest, root)
|
||||
manifest["schema"] = 1
|
||||
manifest["stable_connection_affinity"] = False
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "affinity"):
|
||||
canary.check(manifest, root)
|
||||
manifest["stable_connection_affinity"] = True
|
||||
manifest["baseline_revision"] = manifest["source_revision"]
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "must differ"):
|
||||
canary.check(manifest, root)
|
||||
manifest["baseline_revision"] = "b" * 40
|
||||
manifest["phases"][0]["topology"]["nodes"]["node4"] = "a" * 40
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "node revisions"):
|
||||
canary.check(manifest, root)
|
||||
manifest["phases"][0]["topology"]["nodes"]["node4"] = "b" * 40
|
||||
manifest["phases"][0]["queries"]["decisions"] = "sum(unrelated_metric)"
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "PromQL"):
|
||||
canary.check(manifest, root)
|
||||
manifest["phases"][0]["queries"] = canary.expected_queries(10)
|
||||
manifest["phases"][0]["artifacts"]["decisions"] = "../escape.json"
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "escapes"):
|
||||
canary.check(manifest, root)
|
||||
|
||||
def test_duplicate_series_and_unbounded_labels_are_rejected(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
phase = manifest["phases"][2]
|
||||
path = root / phase["artifacts"]["decisions"]
|
||||
duplicate = row(phase["end_unix"], 1, decision="msgpack_only", reason="peer_capable")
|
||||
path.write_text(json.dumps(prometheus([duplicate, duplicate])), encoding="utf-8")
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "duplicate decision series"):
|
||||
canary.check(manifest, root)
|
||||
path.write_text(
|
||||
json.dumps(prometheus([row(phase["end_unix"], 1, decision="msgpack_only", reason="peer-address")])),
|
||||
encoding="utf-8",
|
||||
)
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "unbounded label"):
|
||||
canary.check(manifest, root)
|
||||
|
||||
def test_empty_error_vector_cannot_masquerade_as_zero(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
phase = manifest["phases"][1]
|
||||
path = root / phase["artifacts"]["rpc_errors"]
|
||||
path.write_text(json.dumps(prometheus([])), encoding="utf-8")
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "exactly one row"):
|
||||
canary.check(manifest, root)
|
||||
|
||||
def test_workload_summary_and_rollback_hashes_are_bound(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
mixed_workload_path = root / manifest["phases"][0]["workload"]
|
||||
mixed_workload = json.loads(mixed_workload_path.read_text(encoding="utf-8"))
|
||||
(root / mixed_workload["summary"]).write_text("changed\n", encoding="utf-8")
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "summary hash mismatch"):
|
||||
canary.check(manifest, root)
|
||||
|
||||
temp.cleanup()
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
rollback_path = root / manifest["phases"][2]["workload"]
|
||||
rollback = json.loads(rollback_path.read_text(encoding="utf-8"))
|
||||
rollback["readback_download_sha256"] = "d" * 64
|
||||
rollback_path.write_text(json.dumps(rollback), encoding="utf-8")
|
||||
with self.assertRaisesRegex(canary.EvidenceError, "readback digest mismatch"):
|
||||
canary.check(manifest, root)
|
||||
|
||||
def test_cli_exit_codes_distinguish_pass_gate_failure_and_invalid_input(self):
|
||||
temp, root, manifest = self.fixture()
|
||||
self.addCleanup(temp.cleanup)
|
||||
manifest_path = root / "manifest.json"
|
||||
out = root / "verification.json"
|
||||
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
result = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--manifest", str(manifest_path), "--out", str(out)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
verified = json.loads(out.read_text(encoding="utf-8"))
|
||||
self.assertTrue(verified["observed_pass"])
|
||||
self.assertEqual(len(verified["manifest_sha256"]), 64)
|
||||
|
||||
raw_artifact = root / manifest["phases"][0]["workload"]
|
||||
overwrite = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--manifest", str(manifest_path), "--out", str(raw_artifact)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(overwrite.returncode, 2)
|
||||
self.assertIn("overwrite raw canary evidence", overwrite.stderr)
|
||||
|
||||
rollback_workload = root / manifest["phases"][2]["workload"]
|
||||
rollback_payload = json.loads(rollback_workload.read_text(encoding="utf-8"))
|
||||
rollback_payload["rollback_readback"] = False
|
||||
rollback_workload.write_text(json.dumps(rollback_payload), encoding="utf-8")
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
failed = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--manifest", str(manifest_path), "--out", str(out)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(failed.returncode, 1, failed.stderr)
|
||||
self.assertFalse(json.loads(out.read_text(encoding="utf-8"))["observed_pass"])
|
||||
|
||||
manifest["scope"] = "command-transcript"
|
||||
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
||||
invalid = subprocess.run(
|
||||
[sys.executable, str(SCRIPT), "--manifest", str(manifest_path), "--out", str(out)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
self.assertEqual(invalid.returncode, 2)
|
||||
self.assertIn("unexpected evidence scope", invalid.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
+160
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
RUNNER="$PROJECT_ROOT/scripts/run_internode_msgpack_peer_canary.sh"
|
||||
TMP_DIR="$(mktemp -d)"
|
||||
trap 'rm -rf "$TMP_DIR"' EXIT
|
||||
FAKE_BIN="$TMP_DIR/bin"
|
||||
mkdir -p "$FAKE_BIN"
|
||||
|
||||
cat >"$FAKE_BIN/docker" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
printf '%s\n' "$*" >>"$FAKE_DOCKER_LOG"
|
||||
exit 0
|
||||
SH
|
||||
|
||||
cat >"$FAKE_BIN/curl" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
query=""
|
||||
at="0"
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
http://127.0.0.1:*/health) printf '{}\n'; exit 0 ;;
|
||||
query=*) query="${arg#query=}" ;;
|
||||
time=*) at="${arg#time=}" ;;
|
||||
esac
|
||||
done
|
||||
if [[ "$query" == *msgpack_send_decisions_total* ]]; then
|
||||
index=0
|
||||
[[ -f "$FAKE_DECISION_INDEX" ]] && index="$(cat "$FAKE_DECISION_INDEX")"
|
||||
case "$index" in
|
||||
0) rows='[{"metric":{"decision":"msgpack_only","reason":"peer_capable"},"value":['"$at"',"4"]},{"metric":{"decision":"dual_write","reason":"peer_unknown"},"value":['"$at"',"2"]}]' ;;
|
||||
1) rows='[{"metric":{"decision":"msgpack_only","reason":"peer_capable"},"value":['"$at"',"8"]}]' ;;
|
||||
*) rows='[{"metric":{"decision":"msgpack_only","reason":"peer_capable"},"value":['"$at"',"5"]},{"metric":{"decision":"dual_write","reason":"peer_revoked"},"value":['"$at"',"3"]}]' ;;
|
||||
esac
|
||||
printf '%s\n' "$((index + 1))" >"$FAKE_DECISION_INDEX"
|
||||
printf '{"status":"success","data":{"resultType":"vector","result":%s}}\n' "$rows"
|
||||
exit 0
|
||||
fi
|
||||
value=0
|
||||
if [[ "${FAKE_NONZERO_ERRORS:-false}" == "true" && "$query" == *decode_error_total* ]]; then
|
||||
value=1
|
||||
fi
|
||||
printf '{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[%s,"%s"]}]}}\n' "$at" "$value"
|
||||
SH
|
||||
|
||||
cat >"$FAKE_BIN/aws" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
operation=""
|
||||
body=""
|
||||
destination=""
|
||||
for ((i = 1; i <= $#; i++)); do
|
||||
arg="${!i}"
|
||||
case "$arg" in
|
||||
head-bucket|create-bucket|put-object|get-object) operation="$arg" ;;
|
||||
--body) next=$((i + 1)); body="${!next}" ;;
|
||||
esac
|
||||
done
|
||||
case "$operation" in
|
||||
head-bucket|create-bucket) exit 0 ;;
|
||||
put-object) cp "$body" "$FAKE_AWS_STATE" ;;
|
||||
get-object) destination="${!#}"; cp "$FAKE_AWS_STATE" "$destination" ;;
|
||||
*) echo "unexpected fake aws invocation: $*" >&2; exit 1 ;;
|
||||
esac
|
||||
SH
|
||||
|
||||
cat >"$TMP_DIR/workload" <<'SH'
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
out=""
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--out-dir) out="$2"; shift 2 ;;
|
||||
*) shift ;;
|
||||
esac
|
||||
done
|
||||
[[ -n "$out" ]]
|
||||
mkdir -p "$out"
|
||||
cat >"$out/summary.csv" <<'CSV'
|
||||
size,tool,concurrency,status,throughput,requests_per_sec,avg_latency,error_count,log_file
|
||||
4KiB,warp,2,ok,1 MiB/s,10 req/s,1 ms,0,fake.log
|
||||
CSV
|
||||
SH
|
||||
|
||||
chmod +x "$FAKE_BIN/docker" "$FAKE_BIN/curl" "$FAKE_BIN/aws" "$TMP_DIR/workload"
|
||||
|
||||
CANDIDATE_REVISION="aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
|
||||
BASELINE_REVISION="bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"
|
||||
|
||||
plan="$TMP_DIR/plan.txt"
|
||||
"$RUNNER" \
|
||||
--candidate-image rustfs/candidate:test \
|
||||
--baseline-image rustfs/baseline:test \
|
||||
--candidate-revision "$CANDIDATE_REVISION" \
|
||||
--baseline-revision "$BASELINE_REVISION" >"$plan"
|
||||
rg -q '^phases=mixed-old-peer all-new rollback-old-peer$' "$plan"
|
||||
rg -q '^execute=false$' "$plan"
|
||||
if rg -q 'rustfssecret-local' "$plan"; then
|
||||
echo "plan leaked the default secret" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
run_once() {
|
||||
local name="$1"
|
||||
local nonzero="$2"
|
||||
local out="$TMP_DIR/$name"
|
||||
: >"$TMP_DIR/docker.log"
|
||||
rm -f "$TMP_DIR/decision.index" "$TMP_DIR/aws.object"
|
||||
set +e
|
||||
PATH="$FAKE_BIN:$PATH" \
|
||||
FAKE_DOCKER_LOG="$TMP_DIR/docker.log" \
|
||||
FAKE_DECISION_INDEX="$TMP_DIR/decision.index" \
|
||||
FAKE_AWS_STATE="$TMP_DIR/aws.object" \
|
||||
FAKE_NONZERO_ERRORS="$nonzero" \
|
||||
METRICS_SETTLE_SECS=0 \
|
||||
WORKLOAD_DURATION=1s \
|
||||
WORKLOAD_CONCURRENCY=2 \
|
||||
WORKLOAD_SIZES=4KiB \
|
||||
"$RUNNER" \
|
||||
--candidate-image rustfs/candidate:test \
|
||||
--baseline-image rustfs/baseline:test \
|
||||
--candidate-revision "$CANDIDATE_REVISION" \
|
||||
--baseline-revision "$BASELINE_REVISION" \
|
||||
--project-name "msgpack-${name}" \
|
||||
--workload-runner "$TMP_DIR/workload" \
|
||||
--out-dir "$out" \
|
||||
--execute
|
||||
status=$?
|
||||
set -e
|
||||
printf '%s\n' "$status" >"$TMP_DIR/$name.status"
|
||||
}
|
||||
|
||||
run_once pass false
|
||||
[[ "$(cat "$TMP_DIR/pass.status")" == "0" ]]
|
||||
python3 - "$TMP_DIR/pass/verification.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
data = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert data["observed_pass"] is True
|
||||
assert len(data["artifacts"]) == 18
|
||||
PY
|
||||
rg -q 'compose .*up -d --no-build --force-recreate node4' "$TMP_DIR/docker.log"
|
||||
rg -q 'compose .*up -d --no-build --force-recreate node2' "$TMP_DIR/docker.log"
|
||||
rg -q 'compose .*down --volumes --remove-orphans' "$TMP_DIR/docker.log"
|
||||
|
||||
run_once fail true
|
||||
[[ "$(cat "$TMP_DIR/fail.status")" == "1" ]]
|
||||
python3 - "$TMP_DIR/fail/verification.json" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
data = json.load(open(sys.argv[1], encoding="utf-8"))
|
||||
assert data["observed_pass"] is False
|
||||
assert any("decode_errors" in item for item in data["violations"])
|
||||
PY
|
||||
|
||||
bash -n "$RUNNER"
|
||||
echo "internode msgpack peer canary runner tests passed"
|
||||
Reference in New Issue
Block a user