gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-07-26 08:18:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

feat(site-replication): make repair operations durable (#5217)

Browse Source 
* feat(site-replication): make repair operations durable

* fix(site-replication): serialize durable repair execution
This commit is contained in:
cxymds
2026-07-25 10:35:37 +08:00
committed by GitHub
parent dfb0a20048
commit 59361ed786
4 changed files with 1369 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/site_replication.rs
+1288 -26
View File
File diff suppressed because it is too large Load Diff
rustfs/src/admin/handlers/system.rs
+68 -3
View File
@@ -62,6 +62,8 @@ pub(crate) const RUNTIME_CAPABILITIES_ROUTE_SUFFIX: &str = "/v4/runtime/capabili
const SITE_REPLICATION_INFO_ROUTE: &str = "/rustfs/admin/v3/site-replication/info";
const SITE_REPLICATION_EDIT_ROUTE: &str = "/rustfs/admin/v3/site-replication/edit";
const SITE_REPLICATION_RESYNC_ROUTE: &str = "/rustfs/admin/v3/site-replication/resync/op";
const SITE_REPLICATION_REPAIR_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair";
const SITE_REPLICATION_REPAIR_STATUS_ROUTE: &str = "/rustfs/admin/v3/site-replication/repair/status";
macro_rules! log_system_request_rejected {
($operation:expr, $reason:expr) => {
@@ -631,6 +633,8 @@ pub struct RuntimeCapabilitiesSummary {
pub site_replication_edit: CapabilityStatus,
#[serde(default)]
pub site_replication_resync: CapabilityStatus,
#[serde(default)]
pub site_replication_repair: CapabilityStatus,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
@@ -639,6 +643,7 @@ pub struct RuntimeCapabilitiesResponse {
pub diagnostic_probes: DiagnosticProbeCapabilities,
pub inspect_archive: super::inspect_archive::InspectArchiveCapability,
pub storage_classes: StorageClassCapabilities,
pub site_replication_repair: SiteReplicationRepairCapabilities,
pub cluster_snapshot_path: String,
pub cluster_snapshot_summary: Option<CapabilityStatus>,
pub observability: crate::admin::storage_api::cluster::ObservabilitySnapshot,
@@ -647,6 +652,42 @@ pub struct RuntimeCapabilitiesResponse {
pub topology_status: CapabilityStatus,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct SiteReplicationRepairCapabilities {
pub contract_version: u32,
pub status: CapabilityStatus,
pub modes: [&'static str; 2],
pub execute_route: &'static str,
pub status_route: &'static str,
pub preflight_token_contract: &'static str,
pub operation_id_format: &'static str,
pub max_retained_successful_operations: usize,
pub disabled_by_default: bool,
}
impl SiteReplicationRepairCapabilities {
fn current() -> Self {
let execute = admin_route_capability(HttpMethod::Put, SITE_REPLICATION_REPAIR_ROUTE);
let status = admin_route_capability(HttpMethod::Get, SITE_REPLICATION_REPAIR_STATUS_ROUTE);
let supported = execute.state == CapabilityState::Supported && status.state == CapabilityState::Supported;
Self {
contract_version: 1,
status: if supported {
CapabilityStatus::supported().with_reason("durable dry-run, execute, and status contracts are registered")
} else {
CapabilityStatus::unsupported().with_reason("one or more durable repair routes are unavailable")
},
modes: ["dry-run", "execute"],
execute_route: SITE_REPLICATION_REPAIR_ROUTE,
status_route: SITE_REPLICATION_REPAIR_STATUS_ROUTE,
preflight_token_contract: "hmac-sha256-v1",
operation_id_format: "uuid",
max_retained_successful_operations: 32,
disabled_by_default: false,
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize)]
pub struct DiagnosticProbeCapabilities {
pub contract_version: u32,
@@ -789,6 +830,7 @@ pub(crate) async fn build_runtime_capabilities_response()
diagnostic_probes: DiagnosticProbeCapabilities::current(),
inspect_archive: super::inspect_archive::InspectArchiveCapability::current(local_drive_count),
storage_classes: StorageClassCapabilities::current(),
site_replication_repair: SiteReplicationRepairCapabilities::current(),
cluster_snapshot_path: usecase.cluster_snapshot_route().to_string(),
cluster_snapshot_summary: cluster_snapshot_discovery.summary,
observability,
@@ -865,6 +907,7 @@ fn build_runtime_capabilities_summary(
site_replication_info: admin_route_capability(HttpMethod::Get, SITE_REPLICATION_INFO_ROUTE),
site_replication_edit: admin_route_capability(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE),
site_replication_resync: admin_route_capability(HttpMethod::Put, SITE_REPLICATION_RESYNC_ROUTE),
site_replication_repair: SiteReplicationRepairCapabilities::current().status,
}
}
@@ -997,9 +1040,10 @@ impl Operation for DataUsageInfoHandler {
mod tests {
use super::{
OBSERVABILITY_SUMMARY_RESOLVED, RuntimeCapabilitiesHandler, SITE_REPLICATION_EDIT_ROUTE, SITE_REPLICATION_INFO_ROUTE,
SITE_REPLICATION_RESYNC_ROUTE, ServerInfoResponse, TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED,
admin_route_capability_from_inventory, build_runtime_capabilities_response, build_runtime_capabilities_summary,
data_usage_info_gate_actions, runtime_capabilities_gate_actions, system_admin_discovery,
SITE_REPLICATION_REPAIR_ROUTE, SITE_REPLICATION_REPAIR_STATUS_ROUTE, SITE_REPLICATION_RESYNC_ROUTE, ServerInfoResponse,
TOPOLOGY_SNAPSHOT_NOT_AVAILABLE, TOPOLOGY_SUMMARY_RESOLVED, admin_route_capability_from_inventory,
build_runtime_capabilities_response, build_runtime_capabilities_summary, data_usage_info_gate_actions,
runtime_capabilities_gate_actions, system_admin_discovery,
};
use crate::admin::router::Operation;
use crate::admin::runtime_sources::DefaultAdminUsecase;
@@ -1051,6 +1095,16 @@ mod tests {
assert_eq!(response.summary.site_replication_info.state, CapabilityState::Supported);
assert_eq!(response.summary.site_replication_edit.state, CapabilityState::Supported);
assert_eq!(response.summary.site_replication_resync.state, CapabilityState::Supported);
assert_eq!(response.summary.site_replication_repair.state, CapabilityState::Supported);
assert_eq!(response.site_replication_repair.contract_version, 1);
assert_eq!(response.site_replication_repair.status.state, CapabilityState::Supported);
assert_eq!(response.site_replication_repair.modes, ["dry-run", "execute"]);
assert_eq!(response.site_replication_repair.execute_route, SITE_REPLICATION_REPAIR_ROUTE);
assert_eq!(response.site_replication_repair.status_route, SITE_REPLICATION_REPAIR_STATUS_ROUTE);
assert_eq!(response.site_replication_repair.preflight_token_contract, "hmac-sha256-v1");
assert_eq!(response.site_replication_repair.operation_id_format, "uuid");
assert_eq!(response.site_replication_repair.max_retained_successful_operations, 32);
assert!(!response.site_replication_repair.disabled_by_default);
assert_eq!(response.diagnostic_probes.contract_version, 1);
assert_eq!(response.diagnostic_probes.health_info.status.state, CapabilityState::Supported);
assert_eq!(response.diagnostic_probes.observed_drive_metrics.mode, "observed_storage_metrics");
@@ -1087,6 +1141,14 @@ mod tests {
assert_eq!(value["summary"]["site_replication_info"]["state"], "supported");
assert_eq!(value["summary"]["site_replication_edit"]["state"], "supported");
assert_eq!(value["summary"]["site_replication_resync"]["state"], "supported");
assert_eq!(value["summary"]["site_replication_repair"]["state"], "supported");
assert_eq!(value["site_replication_repair"]["contract_version"], 1);
assert_eq!(value["site_replication_repair"]["status"]["state"], "supported");
assert_eq!(value["site_replication_repair"]["modes"], json!(["dry-run", "execute"]));
assert_eq!(value["site_replication_repair"]["execute_route"], SITE_REPLICATION_REPAIR_ROUTE);
assert_eq!(value["site_replication_repair"]["status_route"], SITE_REPLICATION_REPAIR_STATUS_ROUTE);
assert_eq!(value["site_replication_repair"]["max_retained_successful_operations"], 32);
assert_eq!(value["site_replication_repair"]["disabled_by_default"], false);
assert_eq!(value["diagnostic_probes"]["client_devnull"]["status"]["state"], "supported");
assert_eq!(
value["diagnostic_probes"]["client_devnull"]["max_bytes"],
@@ -1112,6 +1174,8 @@ mod tests {
(HttpMethod::Get, SITE_REPLICATION_INFO_ROUTE),
(HttpMethod::Put, SITE_REPLICATION_EDIT_ROUTE),
(HttpMethod::Put, SITE_REPLICATION_RESYNC_ROUTE),
(HttpMethod::Put, SITE_REPLICATION_REPAIR_ROUTE),
(HttpMethod::Get, SITE_REPLICATION_REPAIR_STATUS_ROUTE),
] {
assert_eq!(
admin_route_capability_from_inventory(
@@ -1170,6 +1234,7 @@ mod tests {
assert_eq!(summary.site_replication_info.state, CapabilityState::Unknown);
assert_eq!(summary.site_replication_edit.state, CapabilityState::Unknown);
assert_eq!(summary.site_replication_resync.state, CapabilityState::Unknown);
assert_eq!(summary.site_replication_repair.state, CapabilityState::Unknown);
}
#[tokio::test]
rustfs/src/admin/route_policy.rs
+11
View File
@@ -637,6 +637,12 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
SITE_REPLICATION_OPERATION,
RouteRiskLevel::High,
),
admin(
HttpMethod::Get,
"/rustfs/admin/v3/site-replication/repair/status",
SITE_REPLICATION_OPERATION,
RouteRiskLevel::Sensitive,
),
admin(HttpMethod::Get, "/rustfs/admin/debug/pprof/profile", PROFILING, RouteRiskLevel::High),
admin(HttpMethod::Get, "/rustfs/admin/debug/pprof/status", PROFILING, RouteRiskLevel::High),
admin(HttpMethod::Get, "/rustfs/admin/debug/tls/status", PROFILING, RouteRiskLevel::High),
@@ -1854,6 +1860,11 @@ mod tests {
#[test]
fn route_policy_requires_operation_for_site_replication_repair() {
assert_action(HttpMethod::Put, "/rustfs/admin/v3/site-replication/repair", SITE_REPLICATION_OPERATION);
assert_action(
HttpMethod::Get,
"/rustfs/admin/v3/site-replication/repair/status",
SITE_REPLICATION_OPERATION,
);
}
fn route_policy_inventory_keys() -> BTreeSet<String> {
rustfs/src/admin/route_registration_test.rs
+2
View File
@@ -280,6 +280,7 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
admin_route(Method::PUT, "/v3/site-replication/resync/op"),
admin_route(Method::PUT, "/v3/site-replication/state/edit"),
admin_route(Method::PUT, "/v3/site-replication/repair"),
admin_route(Method::GET, "/v3/site-replication/repair/status"),
admin_route(Method::GET, "/debug/pprof/profile"),
admin_route(Method::GET, "/debug/pprof/status"),
admin_route(Method::POST, "/v3/profiling/start"),
@@ -1206,6 +1207,7 @@ fn test_register_routes_cover_representative_admin_paths() {
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/resync/op"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/state/edit"));
assert_route(&router, Method::PUT, &admin_path("/v3/site-replication/repair"));
assert_route(&router, Method::GET, &admin_path("/v3/site-replication/repair/status"));
assert_route(&router, Method::GET, &admin_path("/debug/pprof/profile"));
assert_route(&router, Method::GET, &admin_path("/debug/tls/status"));