mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-27 15:37:02 +00:00
refactor: move ecstore owner layout modules (#3932)
This commit is contained in:
Zhengchao An
@@ -0,0 +1,404 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::data_usage::{DATA_USAGE_CACHE_NAME, DATA_USAGE_ROOT, load_data_usage_from_backend};
|
||||
use crate::error::{Error, Result};
|
||||
use crate::rpc::{TonicInterceptor, gen_tonic_signature_interceptor, node_service_time_out_client};
|
||||
use crate::{disk::endpoint::Endpoint, runtime_sources};
|
||||
|
||||
use crate::data_usage::load_data_usage_cache;
|
||||
use crate::storage_api_contracts::admin::StorageAdminApi;
|
||||
use rustfs_common::heal_channel::DriveState;
|
||||
use rustfs_madmin::{
|
||||
BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
models::{PingBody, PingBodyBuilder},
|
||||
proto_gen::node_service::{PingRequest, PingResponse},
|
||||
};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
time::Duration,
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::time::timeout;
|
||||
use tonic::Request;
|
||||
use tracing::warn;
|
||||
|
||||
use shadow_rs::shadow;
|
||||
|
||||
shadow!(build);
|
||||
|
||||
const SERVER_PING_TIMEOUT: Duration = Duration::from_secs(1);
|
||||
|
||||
// pub const ITEM_OFFLINE: &str = "offline";
|
||||
// pub const ITEM_INITIALIZING: &str = "initializing";
|
||||
// pub const ITEM_ONLINE: &str = "online";
|
||||
|
||||
// #[derive(Debug, Default, Serialize, Deserialize)]
|
||||
// pub struct MemStats {
|
||||
// alloc: u64,
|
||||
// total_alloc: u64,
|
||||
// mallocs: u64,
|
||||
// frees: u64,
|
||||
// heap_alloc: u64,
|
||||
// }
|
||||
|
||||
// #[derive(Debug, Default, Serialize, Deserialize)]
|
||||
// pub struct ServerProperties {
|
||||
// pub state: String,
|
||||
// pub endpoint: String,
|
||||
// pub scheme: String,
|
||||
// pub uptime: u64,
|
||||
// pub version: String,
|
||||
// pub commit_id: String,
|
||||
// pub network: HashMap<String, String>,
|
||||
// pub disks: Vec<madmin::Disk>,
|
||||
// pub pool_number: i32,
|
||||
// pub pool_numbers: Vec<i32>,
|
||||
// pub mem_stats: MemStats,
|
||||
// pub max_procs: u64,
|
||||
// pub num_cpu: u64,
|
||||
// pub runtime_version: String,
|
||||
// pub rustfs_env_vars: HashMap<String, String>,
|
||||
// }
|
||||
|
||||
async fn is_server_resolvable(endpoint: &Endpoint) -> Result<()> {
|
||||
let addr = format!(
|
||||
"{}://{}:{}",
|
||||
endpoint.url.scheme(),
|
||||
endpoint.url.host_str().unwrap(),
|
||||
endpoint.url.port().unwrap()
|
||||
);
|
||||
|
||||
let ping_task = async {
|
||||
let mut fbb = flatbuffers::FlatBufferBuilder::new();
|
||||
let payload = fbb.create_vector(b"hello world");
|
||||
|
||||
let mut builder = PingBodyBuilder::new(&mut fbb);
|
||||
builder.add_payload(payload);
|
||||
let root = builder.finish();
|
||||
fbb.finish(root, None);
|
||||
|
||||
let finished_data = fbb.finished_data();
|
||||
|
||||
let decoded_payload = flatbuffers::root::<PingBody>(finished_data);
|
||||
assert!(decoded_payload.is_ok());
|
||||
|
||||
let mut client = node_service_time_out_client(&addr, TonicInterceptor::Signature(gen_tonic_signature_interceptor()))
|
||||
.await
|
||||
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
|
||||
|
||||
let request = Request::new(PingRequest {
|
||||
version: 1,
|
||||
body: bytes::Bytes::copy_from_slice(finished_data),
|
||||
});
|
||||
|
||||
let response: PingResponse = client.ping(request).await?.into_inner();
|
||||
|
||||
let ping_response_body = flatbuffers::root::<PingBody>(&response.body);
|
||||
if let Err(e) = ping_response_body {
|
||||
eprintln!("{e}");
|
||||
} else {
|
||||
println!("ping_resp:body(flatbuffer): {ping_response_body:?}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
};
|
||||
|
||||
timeout(SERVER_PING_TIMEOUT, ping_task)
|
||||
.await
|
||||
.map_err(|_| Error::other("server ping timeout"))?
|
||||
}
|
||||
|
||||
pub async fn get_local_server_property() -> ServerProperties {
|
||||
let addr = runtime_sources::local_node_name().await;
|
||||
let mut pool_numbers = HashSet::new();
|
||||
let mut network = HashMap::new();
|
||||
|
||||
let endpoints = match runtime_sources::endpoint_pools() {
|
||||
Some(eps) => eps,
|
||||
None => return ServerProperties::default(),
|
||||
};
|
||||
for ep in endpoints.as_ref().iter() {
|
||||
for endpoint in ep.endpoints.as_ref().iter() {
|
||||
let node_name = match endpoint.url.host_str() {
|
||||
Some(s) => s.to_string(),
|
||||
None => addr.clone(),
|
||||
};
|
||||
if endpoint.is_local {
|
||||
pool_numbers.insert(endpoint.pool_idx + 1);
|
||||
network.insert(node_name, ITEM_ONLINE.to_string());
|
||||
continue;
|
||||
}
|
||||
if let std::collections::hash_map::Entry::Vacant(e) = network.entry(node_name) {
|
||||
if is_server_resolvable(endpoint).await.is_err() {
|
||||
e.insert(ITEM_OFFLINE.to_string());
|
||||
} else {
|
||||
e.insert(ITEM_ONLINE.to_string());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// todo: mem collect
|
||||
// let mem_stats =
|
||||
|
||||
let mut props = ServerProperties {
|
||||
endpoint: addr,
|
||||
uptime: runtime_sources::boot_uptime_secs(),
|
||||
network,
|
||||
version: get_commit_id(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
for pool_num in pool_numbers.iter() {
|
||||
props.pool_numbers.push(*pool_num);
|
||||
}
|
||||
props.pool_numbers.sort();
|
||||
props.pool_number = if props.pool_numbers.len() == 1 {
|
||||
props.pool_numbers[0]
|
||||
} else {
|
||||
i32::MAX
|
||||
};
|
||||
|
||||
// let mut sensitive = HashSet::new();
|
||||
// sensitive.insert(rustfs_config::ENV_RUSTFS_ACCESS_KEY.to_string());
|
||||
// sensitive.insert(rustfs_config::ENV_RUSTFS_SECRET_KEY.to_string());
|
||||
if let Some(store) = runtime_sources::object_store_handle() {
|
||||
let storage_info = StorageAdminApi::local_storage_info(store.as_ref()).await;
|
||||
props.state = ITEM_ONLINE.to_string();
|
||||
props.disks = storage_info.disks;
|
||||
} else {
|
||||
props.state = ITEM_INITIALIZING.to_string();
|
||||
};
|
||||
|
||||
props
|
||||
}
|
||||
|
||||
pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
let nowt: OffsetDateTime = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_server_info start {:?}", nowt);
|
||||
|
||||
let local = get_local_server_property().await;
|
||||
|
||||
let after1 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_local_server_property end {:?}", after1 - nowt);
|
||||
|
||||
let mut servers = {
|
||||
if let Some(sys) = runtime_sources::notification_sys() {
|
||||
sys.server_info().await
|
||||
} else {
|
||||
vec![]
|
||||
}
|
||||
};
|
||||
|
||||
let after2 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("server_info end {:?}", after2 - after1);
|
||||
servers.push(local);
|
||||
|
||||
let mut buckets = rustfs_madmin::Buckets::default();
|
||||
let mut objects = rustfs_madmin::Objects::default();
|
||||
let mut versions = rustfs_madmin::Versions::default();
|
||||
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
|
||||
let mut usage = rustfs_madmin::Usage::default();
|
||||
let mut mode = ITEM_INITIALIZING;
|
||||
let mut backend = rustfs_madmin::ErasureBackend::default();
|
||||
let mut pools: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
|
||||
|
||||
if let Some(store) = runtime_sources::object_store_handle() {
|
||||
mode = ITEM_ONLINE;
|
||||
match load_data_usage_from_backend(store.clone()).await {
|
||||
Ok(res) => {
|
||||
buckets.count = res.buckets_count;
|
||||
objects.count = res.objects_total_count;
|
||||
versions.count = res.versions_total_count;
|
||||
delete_markers.count = res.delete_markers_total_count;
|
||||
usage.size = res.objects_total_size;
|
||||
}
|
||||
Err(err) => {
|
||||
buckets.error = Some(err.to_string());
|
||||
objects.error = Some(err.to_string());
|
||||
versions.error = Some(err.to_string());
|
||||
delete_markers.error = Some(err.to_string());
|
||||
usage.error = Some(err.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let after3 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("load_data_usage_from_backend end {:?}", after3 - after2);
|
||||
|
||||
let backend_info = StorageAdminApi::backend_info(store.as_ref()).await;
|
||||
|
||||
let after4 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("backend_info end {:?}", after4 - after3);
|
||||
|
||||
let mut all_disks: Vec<Disk> = Vec::new();
|
||||
for server in servers.iter() {
|
||||
all_disks.extend(server.disks.clone());
|
||||
}
|
||||
let (online_disks, offline_disks) = get_online_offline_disks_stats(&all_disks);
|
||||
|
||||
let after5 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_online_offline_disks_stats end {:?}", after5 - after4);
|
||||
backend = rustfs_madmin::ErasureBackend {
|
||||
backend_type: rustfs_madmin::BackendType::ErasureType,
|
||||
online_disks: online_disks.sum(),
|
||||
offline_disks: offline_disks.sum(),
|
||||
standard_sc_parity: backend_info.standard_sc_parity,
|
||||
rr_sc_parity: backend_info.rr_sc_parity,
|
||||
total_sets: backend_info.total_sets,
|
||||
drives_per_set: backend_info.drives_per_set,
|
||||
};
|
||||
if get_pools {
|
||||
pools = get_pools_info(&all_disks).await.unwrap_or_default();
|
||||
let after6 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_pools_info end {:?}", after6 - after5);
|
||||
}
|
||||
}
|
||||
|
||||
let services = rustfs_madmin::Services::default();
|
||||
|
||||
InfoMessage {
|
||||
mode: Some(mode.to_string()),
|
||||
domain: None,
|
||||
region: None,
|
||||
sqs_arn: None,
|
||||
deployment_id: runtime_sources::deployment_id(),
|
||||
buckets: Some(buckets),
|
||||
objects: Some(objects),
|
||||
versions: Some(versions),
|
||||
delete_markers: Some(delete_markers),
|
||||
usage: Some(usage),
|
||||
backend: Some(backend),
|
||||
services: Some(services),
|
||||
servers: Some(servers),
|
||||
pools: Some(pools),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_online_offline_disks_stats(disks_info: &[Disk]) -> (BackendDisks, BackendDisks) {
|
||||
let mut online_disks: HashMap<String, usize> = HashMap::new();
|
||||
let mut offline_disks: HashMap<String, usize> = HashMap::new();
|
||||
|
||||
for disk in disks_info {
|
||||
let ep = &disk.endpoint;
|
||||
offline_disks.entry(ep.clone()).or_insert(0);
|
||||
online_disks.entry(ep.clone()).or_insert(0);
|
||||
}
|
||||
|
||||
for disk in disks_info {
|
||||
let ep = &disk.endpoint;
|
||||
let state = &disk.state;
|
||||
if *state != DriveState::Ok.to_string() && *state != DriveState::Unformatted.to_string() {
|
||||
*offline_disks.get_mut(ep).unwrap() += 1;
|
||||
continue;
|
||||
}
|
||||
*online_disks.get_mut(ep).unwrap() += 1;
|
||||
}
|
||||
|
||||
let mut root_disk_count = 0;
|
||||
for di in disks_info {
|
||||
if di.root_disk {
|
||||
root_disk_count += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if disks_info.len() == (root_disk_count + offline_disks.values().sum::<usize>()) {
|
||||
return (BackendDisks(online_disks), BackendDisks(offline_disks));
|
||||
}
|
||||
|
||||
for disk in disks_info {
|
||||
let ep = &disk.endpoint;
|
||||
if disk.root_disk {
|
||||
*offline_disks.get_mut(ep).unwrap() += 1;
|
||||
*online_disks.get_mut(ep).unwrap() -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
(BackendDisks(online_disks), BackendDisks(offline_disks))
|
||||
}
|
||||
|
||||
async fn get_pools_info(all_disks: &[Disk]) -> Result<HashMap<i32, HashMap<i32, ErasureSetInfo>>> {
|
||||
let Some(store) = runtime_sources::object_store_handle() else {
|
||||
return Err(Error::other("ServerNotInitialized"));
|
||||
};
|
||||
|
||||
let mut pools_info: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
|
||||
for d in all_disks {
|
||||
let pool_info = pools_info.entry(d.pool_index).or_default();
|
||||
let erasure_set = pool_info.entry(d.set_index).or_default();
|
||||
|
||||
if erasure_set.id == 0 {
|
||||
erasure_set.id = d.set_index;
|
||||
if let Ok(cache) = load_data_usage_cache(
|
||||
&store.pools[d.pool_index as usize].disk_set[d.set_index as usize].clone(),
|
||||
DATA_USAGE_CACHE_NAME,
|
||||
)
|
||||
.await
|
||||
{
|
||||
let data_usage_info = cache.dui(DATA_USAGE_ROOT, &Vec::<String>::new());
|
||||
erasure_set.objects_count = data_usage_info.objects_total_count;
|
||||
erasure_set.versions_count = data_usage_info.versions_total_count;
|
||||
erasure_set.delete_markers_count = data_usage_info.delete_markers_total_count;
|
||||
erasure_set.usage = data_usage_info.objects_total_size;
|
||||
};
|
||||
}
|
||||
|
||||
erasure_set.raw_capacity += d.total_space;
|
||||
erasure_set.raw_usage += d.used_space;
|
||||
if d.healing {
|
||||
erasure_set.heal_disks = 1;
|
||||
}
|
||||
}
|
||||
Ok(pools_info)
|
||||
}
|
||||
|
||||
#[allow(clippy::const_is_empty)]
|
||||
pub fn get_commit_id() -> String {
|
||||
let ver = if !build::TAG.is_empty() {
|
||||
build::TAG.to_string()
|
||||
} else if !build::SHORT_COMMIT.is_empty() {
|
||||
build::SHORT_COMMIT.to_string()
|
||||
} else {
|
||||
build::PKG_VERSION.to_string()
|
||||
};
|
||||
|
||||
format!("{}@{}", build::COMMIT_DATE_3339, ver)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use serial_test::serial;
|
||||
|
||||
use crate::runtime_sources;
|
||||
|
||||
use super::get_server_info;
|
||||
|
||||
#[serial]
|
||||
#[tokio::test]
|
||||
async fn server_info_includes_global_deployment_id() {
|
||||
let expected_deployment_id = runtime_sources::deployment_id();
|
||||
let info = get_server_info(false).await;
|
||||
|
||||
assert_eq!(info.deployment_id, expected_deployment_id);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::disk::error::DiskError;
|
||||
use crate::error::StorageError;
|
||||
use std::io;
|
||||
|
||||
pub(crate) const GET_OBJECT_PATH_CODEC_STREAMING: &str = "codec_streaming";
|
||||
pub(crate) const GET_OBJECT_PATH_EMPTY: &str = "empty";
|
||||
pub(crate) const GET_OBJECT_PATH_LEGACY_DUPLEX: &str = "legacy_duplex";
|
||||
pub(crate) const GET_OBJECT_PATH_REMOTE_TRANSITION: &str = "remote_transition";
|
||||
|
||||
pub(crate) const GET_STAGE_DECODE: &str = "decode";
|
||||
pub(crate) const GET_STAGE_EMIT: &str = "emit";
|
||||
pub(crate) const GET_STAGE_FILL: &str = "fill";
|
||||
pub(crate) const GET_STAGE_FIRST_BYTE: &str = "first_byte";
|
||||
pub(crate) const GET_STAGE_FIRST_METADATA_RESPONSE: &str = "first_metadata_response";
|
||||
pub(crate) const GET_STAGE_FIRST_VALID_METADATA_RESPONSE: &str = "first_valid_metadata_response";
|
||||
pub(crate) const GET_STAGE_FIRST_SHARD_READ: &str = "first_shard_read";
|
||||
pub(crate) const GET_STAGE_FULL_BODY: &str = "full_body";
|
||||
pub(crate) const GET_STAGE_METADATA: &str = "metadata";
|
||||
pub(crate) const GET_STAGE_METADATA_FANOUT: &str = "metadata_fanout";
|
||||
pub(crate) const GET_STAGE_OUTPUT_LOCK_WAIT: &str = "output_lock_wait";
|
||||
pub(crate) const GET_STAGE_OUTPUT_POLL: &str = "output_poll";
|
||||
pub(crate) const GET_STAGE_QUORUM_REACHED: &str = "quorum_reached";
|
||||
pub(crate) const GET_STAGE_RANGE: &str = "range";
|
||||
pub(crate) const GET_STAGE_READER_SETUP: &str = "reader_setup";
|
||||
pub(crate) const GET_STAGE_RECONSTRUCT: &str = "reconstruct";
|
||||
pub(crate) const GET_STAGE_RESPONSE_HANDOFF: &str = "response_handoff";
|
||||
pub(crate) const GET_STAGE_SLOWEST_METADATA_RESPONSE: &str = "slowest_metadata_response";
|
||||
pub(crate) const GET_STAGE_STRIPE_READ: &str = "stripe_read";
|
||||
pub(crate) const GET_STAGE_STRIPE_READ_FIRST_SHARD: &str = "stripe_read_first_shard";
|
||||
pub(crate) const GET_STAGE_STRIPE_READ_QUORUM: &str = "stripe_read_quorum";
|
||||
pub(crate) const GET_STAGE_BITROT_VERIFY: &str = "bitrot_verify";
|
||||
|
||||
pub(crate) const GET_READER_BUFFER_OUTPUT: &str = "output";
|
||||
pub(crate) const GET_READER_BUFFER_PREFETCH: &str = "prefetch";
|
||||
pub(crate) const GET_READER_PREFETCH_DIRECT: &str = "direct";
|
||||
pub(crate) const GET_READER_PREFETCH_EOF: &str = "eof";
|
||||
pub(crate) const GET_READER_PREFETCH_ERROR_DEFERRED: &str = "error_deferred";
|
||||
pub(crate) const GET_READER_PREFETCH_ERROR_IMMEDIATE: &str = "error_immediate";
|
||||
pub(crate) const GET_READER_PREFETCH_STORED: &str = "stored";
|
||||
pub(crate) const GET_READER_POLL_PENDING: &str = "pending";
|
||||
pub(crate) const GET_READER_POLL_READY_DATA: &str = "ready_data";
|
||||
pub(crate) const GET_READER_POLL_READY_EMPTY: &str = "ready_empty";
|
||||
pub(crate) const GET_READER_POLL_READY_ERROR: &str = "ready_error";
|
||||
pub(crate) const GET_SHARD_READ_OUTCOME_ERROR: &str = "error";
|
||||
pub(crate) const GET_SHARD_READ_OUTCOME_MISSING: &str = "missing";
|
||||
pub(crate) const GET_SHARD_READ_OUTCOME_SUCCESS: &str = "success";
|
||||
pub(crate) const GET_SHARD_READ_COST_LOCAL: &str = "local";
|
||||
pub(crate) const GET_SHARD_READ_COST_REMOTE: &str = "remote";
|
||||
pub(crate) const GET_SHARD_READ_COST_SAME_NODE: &str = "same_node";
|
||||
pub(crate) const GET_SHARD_READ_COST_UNKNOWN: &str = "unknown";
|
||||
pub(crate) const GET_SHARD_READ_ERROR_MISSING: &str = "missing";
|
||||
pub(crate) const GET_SHARD_READ_ERROR_NONE: &str = "none";
|
||||
pub(crate) const GET_SHARD_ROLE_DATA: &str = "data";
|
||||
pub(crate) const GET_SHARD_ROLE_PARITY: &str = "parity";
|
||||
|
||||
pub(crate) const GET_METADATA_RESPONSE_CORRUPT: &str = "corrupt";
|
||||
pub(crate) const GET_METADATA_RESPONSE_DISK_NOT_FOUND: &str = "disk_not_found";
|
||||
pub(crate) const GET_METADATA_RESPONSE_ERROR: &str = "error";
|
||||
pub(crate) const GET_METADATA_RESPONSE_IGNORED: &str = "ignored";
|
||||
pub(crate) const GET_METADATA_RESPONSE_NOT_FOUND: &str = "not_found";
|
||||
pub(crate) const GET_METADATA_RESPONSE_TIMEOUT: &str = "timeout";
|
||||
pub(crate) const GET_METADATA_RESPONSE_VALID: &str = "valid";
|
||||
pub(crate) const GET_METADATA_RESPONSE_VERSION_NOT_FOUND: &str = "version_not_found";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA: &str = "conflicting_metadata";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER: &str = "delete_marker";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_ERROR: &str = "error";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM: &str = "insufficient_quorum";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_NOT_FOUND: &str = "not_found";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST: &str = "unsafe_request";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM: &str = "valid_quorum";
|
||||
pub(crate) const GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND: &str = "version_not_found";
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
pub(crate) enum GetObjectFailureReason {
|
||||
BitrotMismatch,
|
||||
DecodeError,
|
||||
DownstreamClosed,
|
||||
Io,
|
||||
RangeOrLengthInvalid,
|
||||
ReadQuorum,
|
||||
ShortRead,
|
||||
Timeout,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl GetObjectFailureReason {
|
||||
pub(crate) const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::BitrotMismatch => "bitrot_mismatch",
|
||||
Self::DecodeError => "decode_error",
|
||||
Self::DownstreamClosed => "downstream_closed",
|
||||
Self::Io => "io",
|
||||
Self::RangeOrLengthInvalid => "range_or_length_invalid",
|
||||
Self::ReadQuorum => "read_quorum",
|
||||
Self::ShortRead => "short_read",
|
||||
Self::Timeout => "timeout",
|
||||
Self::Unknown => "unknown",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn classify_storage_error(err: &StorageError) -> GetObjectFailureReason {
|
||||
match err {
|
||||
StorageError::ErasureReadQuorum | StorageError::InsufficientReadQuorum(_, _) => GetObjectFailureReason::ReadQuorum,
|
||||
StorageError::FileCorrupt => GetObjectFailureReason::BitrotMismatch,
|
||||
StorageError::InvalidRangeSpec(_) => GetObjectFailureReason::RangeOrLengthInvalid,
|
||||
StorageError::Io(io_err) => classify_io_error(io_err),
|
||||
_ => GetObjectFailureReason::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn classify_disk_error(err: &DiskError) -> GetObjectFailureReason {
|
||||
match err {
|
||||
DiskError::ErasureReadQuorum => GetObjectFailureReason::ReadQuorum,
|
||||
DiskError::FileCorrupt | DiskError::PartMissingOrCorrupt => GetObjectFailureReason::BitrotMismatch,
|
||||
DiskError::LessData => GetObjectFailureReason::ShortRead,
|
||||
DiskError::Timeout => GetObjectFailureReason::Timeout,
|
||||
DiskError::Io(io_err) => classify_io_error(io_err),
|
||||
_ => GetObjectFailureReason::Unknown,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn classify_io_error(err: &io::Error) -> GetObjectFailureReason {
|
||||
match err.kind() {
|
||||
io::ErrorKind::BrokenPipe | io::ErrorKind::ConnectionReset => GetObjectFailureReason::DownstreamClosed,
|
||||
io::ErrorKind::TimedOut => GetObjectFailureReason::Timeout,
|
||||
io::ErrorKind::UnexpectedEof => GetObjectFailureReason::ShortRead,
|
||||
io::ErrorKind::InvalidInput | io::ErrorKind::InvalidData => GetObjectFailureReason::RangeOrLengthInvalid,
|
||||
_ => GetObjectFailureReason::Io,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn record_get_object_pipeline_failure(stage: &'static str, reason: GetObjectFailureReason) {
|
||||
rustfs_io_metrics::record_get_object_pipeline_failure(stage, reason.as_str());
|
||||
}
|
||||
|
||||
pub(crate) fn record_get_object_pipeline_failure_for_path(
|
||||
path: &'static str,
|
||||
stage: &'static str,
|
||||
reason: GetObjectFailureReason,
|
||||
) {
|
||||
rustfs_io_metrics::record_get_object_pipeline_failure_for_path(path, stage, reason.as_str());
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn classifies_storage_errors_for_get_pipeline() {
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::ErasureReadQuorum),
|
||||
GetObjectFailureReason::ReadQuorum
|
||||
);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::InsufficientReadQuorum("bucket".to_string(), "object".to_string(),)),
|
||||
GetObjectFailureReason::ReadQuorum
|
||||
);
|
||||
assert_eq!(classify_storage_error(&StorageError::FileCorrupt), GetObjectFailureReason::BitrotMismatch);
|
||||
assert_eq!(
|
||||
classify_storage_error(&StorageError::InvalidRangeSpec("bad range".to_string())),
|
||||
GetObjectFailureReason::RangeOrLengthInvalid
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_disk_errors_for_get_pipeline() {
|
||||
assert_eq!(classify_disk_error(&DiskError::ErasureReadQuorum), GetObjectFailureReason::ReadQuorum);
|
||||
assert_eq!(classify_disk_error(&DiskError::FileCorrupt), GetObjectFailureReason::BitrotMismatch);
|
||||
assert_eq!(
|
||||
classify_disk_error(&DiskError::PartMissingOrCorrupt),
|
||||
GetObjectFailureReason::BitrotMismatch
|
||||
);
|
||||
assert_eq!(classify_disk_error(&DiskError::LessData), GetObjectFailureReason::ShortRead);
|
||||
assert_eq!(classify_disk_error(&DiskError::Timeout), GetObjectFailureReason::Timeout);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn classifies_io_errors_for_get_pipeline() {
|
||||
let cases = [
|
||||
(io::ErrorKind::BrokenPipe, GetObjectFailureReason::DownstreamClosed),
|
||||
(io::ErrorKind::ConnectionReset, GetObjectFailureReason::DownstreamClosed),
|
||||
(io::ErrorKind::TimedOut, GetObjectFailureReason::Timeout),
|
||||
(io::ErrorKind::UnexpectedEof, GetObjectFailureReason::ShortRead),
|
||||
(io::ErrorKind::InvalidInput, GetObjectFailureReason::RangeOrLengthInvalid),
|
||||
(io::ErrorKind::InvalidData, GetObjectFailureReason::RangeOrLengthInvalid),
|
||||
(io::ErrorKind::Other, GetObjectFailureReason::Io),
|
||||
];
|
||||
|
||||
for (kind, expected) in cases {
|
||||
let err = io::Error::from(kind);
|
||||
assert_eq!(classify_io_error(&err), expected, "kind={kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn keeps_metric_labels_stable() {
|
||||
assert_eq!(GetObjectFailureReason::ReadQuorum.as_str(), "read_quorum");
|
||||
assert_eq!(GetObjectFailureReason::ShortRead.as_str(), "short_read");
|
||||
assert_eq!(GetObjectFailureReason::DownstreamClosed.as_str(), "downstream_closed");
|
||||
assert_eq!(GetObjectFailureReason::BitrotMismatch.as_str(), "bitrot_mismatch");
|
||||
assert_eq!(GetObjectFailureReason::DecodeError.as_str(), "decode_error");
|
||||
assert_eq!(GET_READER_BUFFER_OUTPUT, "output");
|
||||
assert_eq!(GET_READER_BUFFER_PREFETCH, "prefetch");
|
||||
assert_eq!(GET_READER_PREFETCH_DIRECT, "direct");
|
||||
assert_eq!(GET_READER_PREFETCH_STORED, "stored");
|
||||
assert_eq!(GET_READER_PREFETCH_EOF, "eof");
|
||||
assert_eq!(GET_READER_PREFETCH_ERROR_DEFERRED, "error_deferred");
|
||||
assert_eq!(GET_READER_PREFETCH_ERROR_IMMEDIATE, "error_immediate");
|
||||
assert_eq!(GET_READER_POLL_PENDING, "pending");
|
||||
assert_eq!(GET_READER_POLL_READY_DATA, "ready_data");
|
||||
assert_eq!(GET_READER_POLL_READY_EMPTY, "ready_empty");
|
||||
assert_eq!(GET_READER_POLL_READY_ERROR, "ready_error");
|
||||
assert_eq!(GET_STAGE_DECODE, "decode");
|
||||
assert_eq!(GET_STAGE_EMIT, "emit");
|
||||
assert_eq!(GET_STAGE_FILL, "fill");
|
||||
assert_eq!(GET_STAGE_FIRST_BYTE, "first_byte");
|
||||
assert_eq!(GET_STAGE_FIRST_METADATA_RESPONSE, "first_metadata_response");
|
||||
assert_eq!(GET_STAGE_FIRST_VALID_METADATA_RESPONSE, "first_valid_metadata_response");
|
||||
assert_eq!(GET_STAGE_FIRST_SHARD_READ, "first_shard_read");
|
||||
assert_eq!(GET_STAGE_FULL_BODY, "full_body");
|
||||
assert_eq!(GET_STAGE_METADATA, "metadata");
|
||||
assert_eq!(GET_STAGE_METADATA_FANOUT, "metadata_fanout");
|
||||
assert_eq!(GET_STAGE_OUTPUT_LOCK_WAIT, "output_lock_wait");
|
||||
assert_eq!(GET_STAGE_OUTPUT_POLL, "output_poll");
|
||||
assert_eq!(GET_STAGE_QUORUM_REACHED, "quorum_reached");
|
||||
assert_eq!(GET_STAGE_RANGE, "range");
|
||||
assert_eq!(GET_STAGE_READER_SETUP, "reader_setup");
|
||||
assert_eq!(GET_STAGE_RECONSTRUCT, "reconstruct");
|
||||
assert_eq!(GET_STAGE_RESPONSE_HANDOFF, "response_handoff");
|
||||
assert_eq!(GET_STAGE_SLOWEST_METADATA_RESPONSE, "slowest_metadata_response");
|
||||
assert_eq!(GET_STAGE_STRIPE_READ, "stripe_read");
|
||||
assert_eq!(GET_STAGE_STRIPE_READ_FIRST_SHARD, "stripe_read_first_shard");
|
||||
assert_eq!(GET_STAGE_STRIPE_READ_QUORUM, "stripe_read_quorum");
|
||||
assert_eq!(GET_STAGE_BITROT_VERIFY, "bitrot_verify");
|
||||
assert_eq!(GET_SHARD_READ_OUTCOME_ERROR, "error");
|
||||
assert_eq!(GET_SHARD_READ_OUTCOME_MISSING, "missing");
|
||||
assert_eq!(GET_SHARD_READ_OUTCOME_SUCCESS, "success");
|
||||
assert_eq!(GET_SHARD_READ_COST_LOCAL, "local");
|
||||
assert_eq!(GET_SHARD_READ_COST_REMOTE, "remote");
|
||||
assert_eq!(GET_SHARD_READ_COST_SAME_NODE, "same_node");
|
||||
assert_eq!(GET_SHARD_READ_COST_UNKNOWN, "unknown");
|
||||
assert_eq!(GET_SHARD_READ_ERROR_MISSING, "missing");
|
||||
assert_eq!(GET_SHARD_READ_ERROR_NONE, "none");
|
||||
assert_eq!(GET_SHARD_ROLE_DATA, "data");
|
||||
assert_eq!(GET_SHARD_ROLE_PARITY, "parity");
|
||||
assert_eq!(GET_METADATA_RESPONSE_CORRUPT, "corrupt");
|
||||
assert_eq!(GET_METADATA_RESPONSE_DISK_NOT_FOUND, "disk_not_found");
|
||||
assert_eq!(GET_METADATA_RESPONSE_ERROR, "error");
|
||||
assert_eq!(GET_METADATA_RESPONSE_IGNORED, "ignored");
|
||||
assert_eq!(GET_METADATA_RESPONSE_NOT_FOUND, "not_found");
|
||||
assert_eq!(GET_METADATA_RESPONSE_TIMEOUT, "timeout");
|
||||
assert_eq!(GET_METADATA_RESPONSE_VALID, "valid");
|
||||
assert_eq!(GET_METADATA_RESPONSE_VERSION_NOT_FOUND, "version_not_found");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_CONFLICTING_METADATA, "conflicting_metadata");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_DELETE_MARKER, "delete_marker");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_ERROR, "error");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_INSUFFICIENT_QUORUM, "insufficient_quorum");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_NOT_FOUND, "not_found");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_UNSAFE_REQUEST, "unsafe_request");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_VALID_QUORUM, "valid_quorum");
|
||||
assert_eq!(GET_METADATA_EARLY_STOP_REASON_VERSION_NOT_FOUND, "version_not_found");
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user