mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-09 22:59:59 +00:00
fix(tiering): bind recovery to transaction metadata (#5409)
* fix(tiering): bind recovery to transaction metadata * fix(tier): add operator transition reconciliation (#5410) * fix(tier): add operator transition reconciliation * fix(tiering): require live fleet capability proof (#5423)
This commit is contained in:
cxymds
@@ -20,8 +20,10 @@ use crate::admin::storage_api::error::StorageError;
|
||||
use crate::admin::storage_api::lifecycle::{
|
||||
ManualTransitionCancelCheck, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionProgressSink,
|
||||
ManualTransitionQueueSnapshot, ManualTransitionRunOptions, ManualTransitionRunReport, ManualTransitionScopeAdmission,
|
||||
ManualTransitionScopeAdmissionClaim, claim_manual_transition_scope_admission,
|
||||
delete_manual_transition_scope_admission_if_current, enqueue_transition_for_existing_objects_scoped,
|
||||
ManualTransitionScopeAdmissionClaim, TransitionOperatorDeleteResult, TransitionOperatorError,
|
||||
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
||||
delete_transition_candidate_for_operator, enqueue_transition_for_existing_objects_scoped,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
load_manual_transition_job_record, load_manual_transition_job_record_with_etag, load_manual_transition_scope_admission,
|
||||
manual_transition_job_lease_expired, manual_transition_queue_snapshot, manual_transition_scope_admission_lease_expired,
|
||||
persist_manual_transition_job_progress, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
|
||||
@@ -33,6 +35,7 @@ use crate::server::{ADMIN_PREFIX, RemoteAddr};
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use hyper::{Method, StatusCode};
|
||||
use matchit::Params;
|
||||
use rustfs_config::MAX_ADMIN_REQUEST_BODY_SIZE;
|
||||
use rustfs_policy::policy::action::{Action, AdminAction};
|
||||
use rustfs_utils::{
|
||||
MaskedAccessKey,
|
||||
@@ -56,6 +59,7 @@ const MAX_MANUAL_TRANSITION_DURATION_SECONDS: u64 = 3600;
|
||||
const LOG_COMPONENT_ADMIN: &str = "admin";
|
||||
const LOG_SUBSYSTEM_ILM_TRANSITION: &str = "ilm_transition";
|
||||
const EVENT_ADMIN_ILM_TRANSITION_STATE: &str = "admin_ilm_transition_state";
|
||||
const EVENT_ADMIN_ILM_TRANSITION_RECONCILE: &str = "admin_ilm_transition_reconcile";
|
||||
|
||||
static ACTIVE_MANUAL_TRANSITION_SCOPES: OnceLock<Mutex<Vec<ManualTransitionRunScope>>> = OnceLock::new();
|
||||
#[cfg(feature = "e2e-test-hooks")]
|
||||
@@ -218,6 +222,16 @@ pub fn register_ilm_transition_route(r: &mut S3Router<AdminOperation>) -> std::i
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/jobs/{{job_id}}").as_str(),
|
||||
AdminOperation(&ManualTransitionJobCancelHandler {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::GET,
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
|
||||
AdminOperation(&TransitionReconcileInspectHandler {}),
|
||||
)?;
|
||||
r.insert(
|
||||
Method::POST,
|
||||
format!("{ADMIN_PREFIX}/v3/ilm/transition/reconcile/{{transaction_id}}").as_str(),
|
||||
AdminOperation(&TransitionReconcileApplyHandler {}),
|
||||
)?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -389,6 +403,10 @@ fn log_manual_transition_completed(
|
||||
}
|
||||
|
||||
async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<String> {
|
||||
authorize_transition_admin_request(req, AdminAction::SetTierAction).await
|
||||
}
|
||||
|
||||
async fn authorize_transition_admin_request(req: &S3Request<Body>, action: AdminAction) -> S3Result<String> {
|
||||
let Some(input_cred) = req.credentials.as_ref() else {
|
||||
return Err(s3_error!(InvalidRequest, "authentication required"));
|
||||
};
|
||||
@@ -401,19 +419,122 @@ async fn authorize_manual_transition_request(req: &S3Request<Body>) -> S3Result<
|
||||
.get::<Option<RemoteAddr>>()
|
||||
.and_then(|opt| opt.map(|addr| addr.0));
|
||||
|
||||
validate_admin_request(
|
||||
&req.headers,
|
||||
&cred,
|
||||
owner,
|
||||
false,
|
||||
vec![Action::AdminAction(AdminAction::SetTierAction)],
|
||||
remote_addr,
|
||||
)
|
||||
.await?;
|
||||
validate_admin_request(&req.headers, &cred, owner, false, vec![Action::AdminAction(action)], remote_addr).await?;
|
||||
|
||||
Ok(actor)
|
||||
}
|
||||
|
||||
fn transition_transaction_id_from_params(params: &Params<'_, '_>) -> S3Result<Uuid> {
|
||||
Uuid::parse_str(params.get("transaction_id").unwrap_or(""))
|
||||
.map_err(|_| s3_error!(InvalidArgument, "invalid transition transaction id"))
|
||||
}
|
||||
|
||||
fn map_transition_operator_error(err: TransitionOperatorError) -> S3Error {
|
||||
match err {
|
||||
TransitionOperatorError::NotFound => s3_error!(NoSuchKey, "transition transaction not found"),
|
||||
TransitionOperatorError::NotExpired => {
|
||||
s3_error!(OperationAborted, "transition transaction is still inside its active ownership window")
|
||||
}
|
||||
TransitionOperatorError::InvalidState(_) => {
|
||||
s3_error!(OperationAborted, "transition transaction is not eligible for operator reconciliation")
|
||||
}
|
||||
TransitionOperatorError::RemoteVersionRequired => {
|
||||
s3_error!(InvalidArgument, "an exact non-empty remote version is required")
|
||||
}
|
||||
TransitionOperatorError::CandidateNotMissing(_) => {
|
||||
s3_error!(OperationAborted, "remote candidate is not proven missing")
|
||||
}
|
||||
TransitionOperatorError::CandidateVersionMismatch { .. } => {
|
||||
s3_error!(OperationAborted, "remote candidate version does not match requested exact version")
|
||||
}
|
||||
TransitionOperatorError::Store(_) | TransitionOperatorError::Remote(_) => {
|
||||
s3_error!(InternalError, "transition reconciliation failed")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum TransitionReconcileAction {
|
||||
DeleteCandidate,
|
||||
FinalizeMissing,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct TransitionReconcileRequest {
|
||||
action: TransitionReconcileAction,
|
||||
confirm: bool,
|
||||
#[serde(default)]
|
||||
remote_version_id: Option<String>,
|
||||
}
|
||||
|
||||
enum ValidatedTransitionReconcileAction<'a> {
|
||||
DeleteCandidate(&'a str),
|
||||
FinalizeMissing,
|
||||
}
|
||||
|
||||
fn validate_transition_reconcile_request(
|
||||
request: &TransitionReconcileRequest,
|
||||
) -> S3Result<ValidatedTransitionReconcileAction<'_>> {
|
||||
if !request.confirm {
|
||||
return Err(s3_error!(
|
||||
InvalidRequest,
|
||||
"transition reconciliation requires confirm=true; use GET to inspect without changes"
|
||||
));
|
||||
}
|
||||
match request.action {
|
||||
TransitionReconcileAction::DeleteCandidate => request
|
||||
.remote_version_id
|
||||
.as_deref()
|
||||
.filter(|version_id| !version_id.is_empty())
|
||||
.map(ValidatedTransitionReconcileAction::DeleteCandidate)
|
||||
.ok_or_else(|| s3_error!(InvalidArgument, "delete_candidate requires remote_version_id")),
|
||||
TransitionReconcileAction::FinalizeMissing if request.remote_version_id.is_none() => {
|
||||
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
|
||||
}
|
||||
TransitionReconcileAction::FinalizeMissing => {
|
||||
Err(s3_error!(InvalidArgument, "finalize_missing must not include remote_version_id"))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TransitionCandidateDeleteResponse {
|
||||
outcome: &'static str,
|
||||
result: TransitionOperatorDeleteResult,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct TransitionFinalizeMissingResponse {
|
||||
outcome: &'static str,
|
||||
journal_retained: bool,
|
||||
transaction_id: Uuid,
|
||||
}
|
||||
|
||||
fn log_transition_reconcile_applied(
|
||||
transaction_id: Uuid,
|
||||
action: &str,
|
||||
outcome: &str,
|
||||
request_id: &str,
|
||||
actor: &str,
|
||||
remote_addr: &str,
|
||||
) {
|
||||
info!(
|
||||
event = EVENT_ADMIN_ILM_TRANSITION_RECONCILE,
|
||||
component = LOG_COMPONENT_ADMIN,
|
||||
subsystem = LOG_SUBSYSTEM_ILM_TRANSITION,
|
||||
operation = "transition_operator_reconcile",
|
||||
transaction_id = %transaction_id,
|
||||
action,
|
||||
outcome,
|
||||
request_id = %request_id,
|
||||
actor = %actor,
|
||||
remote_addr = %remote_addr,
|
||||
"admin transition reconciliation applied"
|
||||
);
|
||||
}
|
||||
|
||||
fn response_state(report: &ManualTransitionRunReport) -> &'static str {
|
||||
if report.was_truncated() || report.has_partial_enqueue() || report.tier_failure > 0 || report.transition_failed > 0 {
|
||||
"partial"
|
||||
@@ -902,6 +1023,81 @@ impl Operation for ManualTransitionJobCancelHandler {
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransitionReconcileInspectHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for TransitionReconcileInspectHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
authorize_transition_admin_request(&req, AdminAction::ListTierAction).await?;
|
||||
let transaction_id = transition_transaction_id_from_params(¶ms)?;
|
||||
let Some(store) = object_store_from_extensions(&req.extensions) else {
|
||||
return Err(s3_error!(InternalError, "object store is not initialized"));
|
||||
};
|
||||
let status = inspect_transition_transaction_for_operator(store, transaction_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
json_response(&status, StatusCode::OK)
|
||||
}
|
||||
}
|
||||
|
||||
pub struct TransitionReconcileApplyHandler {}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Operation for TransitionReconcileApplyHandler {
|
||||
async fn call(&self, req: S3Request<Body>, params: Params<'_, '_>) -> S3Result<S3Response<(StatusCode, Body)>> {
|
||||
let request_id = admin_request_id(&req.headers).unwrap_or_default().to_string();
|
||||
let remote_addr = admin_remote_addr(&req).unwrap_or_default();
|
||||
let actor = authorize_transition_admin_request(&req, AdminAction::SetTierAction).await?;
|
||||
let transaction_id = transition_transaction_id_from_params(¶ms)?;
|
||||
let Some(store) = object_store_from_extensions(&req.extensions) else {
|
||||
return Err(s3_error!(InternalError, "object store is not initialized"));
|
||||
};
|
||||
let mut input = req.input;
|
||||
let body = input
|
||||
.store_all_limited(MAX_ADMIN_REQUEST_BODY_SIZE)
|
||||
.await
|
||||
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation body is too large or unreadable"))?;
|
||||
let request: TransitionReconcileRequest = serde_json::from_slice(&body)
|
||||
.map_err(|_| s3_error!(InvalidRequest, "transition reconciliation request must be valid JSON"))?;
|
||||
|
||||
match validate_transition_reconcile_request(&request)? {
|
||||
ValidatedTransitionReconcileAction::DeleteCandidate(remote_version_id) => {
|
||||
let result = delete_transition_candidate_for_operator(store, transaction_id, remote_version_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
let outcome = if result.journal_observed_after_delete {
|
||||
"exact_delete_completed_journal_observed"
|
||||
} else {
|
||||
"exact_delete_completed_journal_already_finalized"
|
||||
};
|
||||
log_transition_reconcile_applied(transaction_id, "delete_candidate", outcome, &request_id, &actor, &remote_addr);
|
||||
json_response(&TransitionCandidateDeleteResponse { outcome, result }, StatusCode::OK)
|
||||
}
|
||||
ValidatedTransitionReconcileAction::FinalizeMissing => {
|
||||
finalize_missing_transition_transaction_for_operator(store, transaction_id)
|
||||
.await
|
||||
.map_err(map_transition_operator_error)?;
|
||||
log_transition_reconcile_applied(
|
||||
transaction_id,
|
||||
"finalize_missing",
|
||||
"journal_deleted_after_missing_probe",
|
||||
&request_id,
|
||||
&actor,
|
||||
&remote_addr,
|
||||
);
|
||||
json_response(
|
||||
&TransitionFinalizeMissingResponse {
|
||||
outcome: "journal_finalized",
|
||||
journal_retained: false,
|
||||
transaction_id,
|
||||
},
|
||||
StatusCode::OK,
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -931,6 +1127,65 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_request_is_explicit_and_fail_closed() {
|
||||
let unconfirmed: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":false,"remote_version_id":"v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&unconfirmed).is_err());
|
||||
|
||||
let missing_version: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true}"#).expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&missing_version).is_err());
|
||||
|
||||
let unsafe_finalize: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true,"remote_version_id":"v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(validate_transition_reconcile_request(&unsafe_finalize).is_err());
|
||||
|
||||
let delete: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"delete_candidate","confirm":true,"remote_version_id":"opaque-v1"}"#)
|
||||
.expect("request should decode");
|
||||
assert!(matches!(
|
||||
validate_transition_reconcile_request(&delete),
|
||||
Ok(ValidatedTransitionReconcileAction::DeleteCandidate("opaque-v1"))
|
||||
));
|
||||
|
||||
let finalize: TransitionReconcileRequest =
|
||||
serde_json::from_slice(br#"{"action":"finalize_missing","confirm":true}"#).expect("request should decode");
|
||||
assert!(matches!(
|
||||
validate_transition_reconcile_request(&finalize),
|
||||
Ok(ValidatedTransitionReconcileAction::FinalizeMissing)
|
||||
));
|
||||
|
||||
assert!(
|
||||
serde_json::from_slice::<TransitionReconcileRequest>(
|
||||
br#"{"action":"finalize_missing","confirm":true,"unexpected":true}"#
|
||||
)
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transition_reconcile_routes_use_read_and_write_tier_actions() {
|
||||
let src = include_str!("ilm_transition.rs");
|
||||
let inspect = src
|
||||
.split("impl Operation for TransitionReconcileInspectHandler")
|
||||
.nth(1)
|
||||
.and_then(|block| block.split("impl Operation for TransitionReconcileApplyHandler").next())
|
||||
.expect("inspect handler block");
|
||||
assert!(inspect.contains("AdminAction::ListTierAction"));
|
||||
assert!(!inspect.contains("AdminAction::SetTierAction"));
|
||||
|
||||
let apply = src
|
||||
.split("impl Operation for TransitionReconcileApplyHandler")
|
||||
.nth(1)
|
||||
.and_then(|block| block.split("#[cfg(test)]").next())
|
||||
.expect("apply handler block");
|
||||
assert!(apply.contains("AdminAction::SetTierAction"));
|
||||
assert!(!apply.contains("AdminAction::ListTierAction"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_query_defaults_to_bounded_run() {
|
||||
let (bucket, options, run_mode) =
|
||||
|
||||
@@ -432,6 +432,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
SET_TIER,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
LIST_TIER,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Post,
|
||||
"/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
SET_TIER,
|
||||
RouteRiskLevel::High,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v3/audit/target/list",
|
||||
@@ -1825,13 +1837,17 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn route_policy_requires_set_tier_for_manual_transition_routes() {
|
||||
fn route_policy_uses_tier_actions_for_transition_routes() {
|
||||
assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SET_TIER);
|
||||
assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
|
||||
assert_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SET_TIER);
|
||||
assert_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
|
||||
assert_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
|
||||
assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/run", SERVER_INFO);
|
||||
assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
|
||||
assert_not_action(HttpMethod::Delete, "/rustfs/admin/v3/ilm/transition/jobs/{job_id}", SERVER_INFO);
|
||||
assert_not_action(HttpMethod::Get, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", SET_TIER);
|
||||
assert_not_action(HttpMethod::Post, "/rustfs/admin/v3/ilm/transition/reconcile/{transaction_id}", LIST_TIER);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -202,6 +202,16 @@ fn expected_admin_route_matrix() -> Vec<RouteMatrixEntry> {
|
||||
"/v3/ilm/transition/jobs/{job_id}",
|
||||
"/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111",
|
||||
),
|
||||
admin_route_sample(
|
||||
Method::GET,
|
||||
"/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
"/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
|
||||
),
|
||||
admin_route_sample(
|
||||
Method::POST,
|
||||
"/v3/ilm/transition/reconcile/{transaction_id}",
|
||||
"/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111",
|
||||
),
|
||||
admin_route_sample(
|
||||
Method::DELETE,
|
||||
"/v3/ilm/transition/jobs/{job_id}",
|
||||
@@ -856,6 +866,16 @@ fn test_register_routes_cover_representative_admin_paths() {
|
||||
Method::DELETE,
|
||||
&admin_path("/v3/ilm/transition/jobs/11111111-1111-4111-8111-111111111111"),
|
||||
);
|
||||
assert_route(
|
||||
&router,
|
||||
Method::GET,
|
||||
&admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
|
||||
);
|
||||
assert_route(
|
||||
&router,
|
||||
Method::POST,
|
||||
&admin_path("/v3/ilm/transition/reconcile/11111111-1111-4111-8111-111111111111"),
|
||||
);
|
||||
|
||||
assert_route(&router, Method::GET, &table_catalog_path("/config"));
|
||||
assert_route(&router, Method::PUT, &table_catalog_path("/buckets/analytics"));
|
||||
|
||||
@@ -210,6 +210,10 @@ pub(crate) mod lifecycle {
|
||||
pub(crate) type ManualTransitionRunOptions =
|
||||
super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunOptions;
|
||||
pub(crate) type ManualTransitionRunReport = super::ecstore_bucket::lifecycle::bucket_lifecycle_ops::ManualTransitionRunReport;
|
||||
pub(crate) use super::ecstore_bucket::lifecycle::transition_transaction::{
|
||||
TransitionOperatorDeleteResult, TransitionOperatorError, delete_transition_candidate_for_operator,
|
||||
finalize_missing_transition_transaction_for_operator, inspect_transition_transaction_for_operator,
|
||||
};
|
||||
|
||||
pub(crate) async fn enqueue_transition_for_existing_objects_scoped(
|
||||
api: std::sync::Arc<super::ECStore>,
|
||||
|
||||
@@ -52,7 +52,7 @@ use std::{
|
||||
collections::HashMap,
|
||||
io::Cursor,
|
||||
pin::Pin,
|
||||
sync::{Arc, OnceLock},
|
||||
sync::{Arc, LazyLock, OnceLock},
|
||||
};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::spawn;
|
||||
@@ -128,6 +128,7 @@ fn remove_heal_control_replay(
|
||||
}
|
||||
|
||||
static HEAL_CONTROL_REPLAY_CACHE: OnceLock<tokio::sync::Mutex<HashMap<String, Arc<HealControlReplayEntry>>>> = OnceLock::new();
|
||||
static NODE_CAPABILITY_SERVER_EPOCH: LazyLock<Uuid> = LazyLock::new(Uuid::new_v4);
|
||||
|
||||
fn admit_heal_control_replay(
|
||||
replay_cache: &mut HashMap<String, Arc<HealControlReplayEntry>>,
|
||||
@@ -465,6 +466,19 @@ impl HealControlRpcService {
|
||||
pub(crate) async fn initialize_heal_topology_fingerprint(
|
||||
cache: Arc<tokio::sync::OnceCell<String>>,
|
||||
endpoint_pools: EndpointServerPools,
|
||||
) -> Result<(), String> {
|
||||
initialize_heal_topology_fingerprint_with_probe(
|
||||
cache,
|
||||
endpoint_pools,
|
||||
crate::storage::storage_api::start_remote_version_state_fleet_probe,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn initialize_heal_topology_fingerprint_with_probe(
|
||||
cache: Arc<tokio::sync::OnceCell<String>>,
|
||||
endpoint_pools: EndpointServerPools,
|
||||
start_probe: impl FnOnce(String),
|
||||
) -> Result<(), String> {
|
||||
if cache.get().is_some() {
|
||||
return Ok(());
|
||||
@@ -472,7 +486,8 @@ pub(crate) async fn initialize_heal_topology_fingerprint(
|
||||
let fingerprint = tokio::task::spawn_blocking(move || heal::heal_topology_fingerprint(&endpoint_pools))
|
||||
.await
|
||||
.map_err(|_| "heal control topology calculation task failed".to_string())??;
|
||||
let _ = cache.set(fingerprint);
|
||||
let _ = cache.set(fingerprint.clone());
|
||||
start_probe(fingerprint);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -806,6 +821,35 @@ impl heal_control_service_server::HealControlService for HealControlRpcService {
|
||||
response_proof: Bytes::new(),
|
||||
}));
|
||||
}
|
||||
if rustfs_protos::is_remote_version_state_capability_probe(&request.get_ref().command) {
|
||||
let topology_member = self
|
||||
.endpoint_pools()
|
||||
.await
|
||||
.ok_or_else(|| Status::failed_precondition("heal control topology is not initialized"))?
|
||||
.peers()
|
||||
.1;
|
||||
if topology_member.is_empty() {
|
||||
return Err(Status::failed_precondition("local topology member identity is unavailable"));
|
||||
}
|
||||
let result =
|
||||
rustfs_protos::encode_remote_version_state_capability(&topology_member, NODE_CAPABILITY_SERVER_EPOCH.as_bytes())
|
||||
.map_err(|_| Status::internal("remote version state capability length cannot be represented"))?;
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
request.get_ref().version,
|
||||
&request.get_ref().topology_fingerprint,
|
||||
&request.get_ref().command,
|
||||
&result,
|
||||
)
|
||||
.map_err(|_| Status::internal("heal control response length cannot be represented"))?;
|
||||
let response_proof = sign_tonic_rpc_response_proof(&canonical_response)
|
||||
.map_err(|_| Status::internal("heal control response proof is unavailable"))?;
|
||||
return Ok(Response::new(HealControlResponse {
|
||||
success: true,
|
||||
result: result.into(),
|
||||
error_info: None,
|
||||
response_proof: response_proof.into(),
|
||||
}));
|
||||
}
|
||||
let endpoints = self
|
||||
.endpoint_pools()
|
||||
.await
|
||||
@@ -2090,10 +2134,10 @@ mod tests {
|
||||
PEER_RESTDRY_RUN, PEER_RESTSIGNAL, PEER_RESTSUB_SYS, SCANNER_ACTIVITY_LEGACY_PROTOCOL_VERSION,
|
||||
SCANNER_ACTIVITY_PREVIOUS_PROTOCOL_VERSION, SERVICE_SIGNAL_REFRESH_CONFIG, SERVICE_SIGNAL_RELOAD_DYNAMIC,
|
||||
STORAGE_CLASS_SUB_SYS, admit_heal_control_replay, background_rebalance_start_error_message,
|
||||
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint, legacy_scanner_activity_response,
|
||||
make_heal_control_server, make_heal_control_server_with_cache, make_server, make_server_for_context,
|
||||
make_tier_mutation_control_server_for_context, previous_scanner_activity_response, remove_heal_control_replay,
|
||||
scanner_activity_response, stop_rebalance_response,
|
||||
execute_heal_control_envelope_with_manager, initialize_heal_topology_fingerprint,
|
||||
initialize_heal_topology_fingerprint_with_probe, legacy_scanner_activity_response, make_heal_control_server,
|
||||
make_heal_control_server_with_cache, make_server, make_server_for_context, make_tier_mutation_control_server_for_context,
|
||||
previous_scanner_activity_response, remove_heal_control_replay, scanner_activity_response, stop_rebalance_response,
|
||||
};
|
||||
use crate::storage::rpc::node_service::heal::heal_topology_fingerprint;
|
||||
use crate::storage::storage_api::rpc_consumer::node_service::{HealBucketInfo, HealEndpoint};
|
||||
@@ -2147,6 +2191,7 @@ mod tests {
|
||||
use tokio::time::Duration;
|
||||
use tokio_stream::wrappers::TcpListenerStream;
|
||||
use tonic::{Request, Response, Status};
|
||||
use uuid::Uuid;
|
||||
|
||||
const DISK_MUTATION_RPC_METHODS: [&str; 18] = [
|
||||
"renamedata",
|
||||
@@ -3132,6 +3177,60 @@ mod tests {
|
||||
assert_eq!(non_coordinator.code(), tonic::Code::FailedPrecondition);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_version_state_probe_authenticates_topology_challenge_and_process_epoch() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("remote-version-state-node-service-test-secret".to_string());
|
||||
let endpoints = heal_control_test_endpoints_with_coordinator("node-d", true);
|
||||
let fingerprint = heal_topology_fingerprint(&endpoints).expect("test topology should hash");
|
||||
let (service, source) = super::make_heal_control_server_for_source();
|
||||
*source.write().await = Some(endpoints);
|
||||
let probe_command = rustfs_protos::remote_version_state_capability_probe(&[7; 16]);
|
||||
let mut request = Request::new(HealControlRequest {
|
||||
version: rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
topology_fingerprint: fingerprint.clone(),
|
||||
command: Bytes::from(probe_command.clone()),
|
||||
});
|
||||
let body = rustfs_protos::canonical_heal_control_request_body(
|
||||
request.get_ref().version,
|
||||
&request.get_ref().topology_fingerprint,
|
||||
&request.get_ref().command,
|
||||
)
|
||||
.expect("probe should encode");
|
||||
set_tonic_canonical_body_digest(&mut request, &body).expect("digest metadata should encode");
|
||||
mark_v2_authenticated(&mut request);
|
||||
let response = service
|
||||
.heal_control(request)
|
||||
.await
|
||||
.expect("matching topology should be acknowledged")
|
||||
.into_inner();
|
||||
|
||||
let (topology_member, process_epoch) =
|
||||
rustfs_protos::decode_remote_version_state_capability(&response.result).expect("capability response should decode");
|
||||
assert_eq!(topology_member, "node-a:9000");
|
||||
let server_epoch = Uuid::from_slice(process_epoch).expect("server epoch should be a UUID");
|
||||
assert!(!server_epoch.is_nil());
|
||||
let canonical_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
&fingerprint,
|
||||
&probe_command,
|
||||
&response.result,
|
||||
)
|
||||
.expect("response should encode");
|
||||
crate::storage::storage_api::verify_tonic_rpc_response_proof(&canonical_response, &response.response_proof)
|
||||
.expect("outer proof should bind the response to the request");
|
||||
|
||||
let different_probe = rustfs_protos::remote_version_state_capability_probe(&[8; 16]);
|
||||
let different_response = rustfs_protos::canonical_heal_control_response_body(
|
||||
rustfs_protos::HEAL_CONTROL_PROTOCOL_VERSION,
|
||||
&fingerprint,
|
||||
&different_probe,
|
||||
&response.result,
|
||||
)
|
||||
.expect("different response should encode");
|
||||
crate::storage::storage_api::verify_tonic_rpc_response_proof(&different_response, &response.response_proof)
|
||||
.expect_err("proof from one challenge must not be reusable");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn heal_control_coordinator_rejects_expired_and_non_admin_starts() {
|
||||
let _ = rustfs_credentials::set_global_rpc_secret("heal-control-node-service-test-secret".to_string());
|
||||
@@ -3194,10 +3293,15 @@ mod tests {
|
||||
let topology = heal_control_test_endpoints("node-d");
|
||||
let expected = heal_topology_fingerprint(&topology).expect("test topology should hash");
|
||||
let cache = Arc::new(tokio::sync::OnceCell::new());
|
||||
initialize_heal_topology_fingerprint(Arc::clone(&cache), topology)
|
||||
.await
|
||||
.expect("valid topology should initialize");
|
||||
let started_probe = Arc::new(std::sync::Mutex::new(None));
|
||||
let started_probe_capture = Arc::clone(&started_probe);
|
||||
initialize_heal_topology_fingerprint_with_probe(Arc::clone(&cache), topology, move |fingerprint| {
|
||||
*started_probe_capture.lock().expect("probe capture should not poison") = Some(fingerprint);
|
||||
})
|
||||
.await
|
||||
.expect("valid topology should initialize");
|
||||
assert_eq!(cache.get(), Some(&expected));
|
||||
assert_eq!(started_probe.lock().expect("probe capture should not poison").as_ref(), Some(&expected));
|
||||
|
||||
let mut invalid = heal_control_test_endpoints("node-d");
|
||||
invalid.as_mut()[0].endpoints.as_mut()[0].pool_idx = -1;
|
||||
|
||||
@@ -470,7 +470,7 @@ pub(crate) mod ecstore_metrics {
|
||||
#[allow(unused_imports)]
|
||||
pub(crate) mod ecstore_notification {
|
||||
pub(crate) use rustfs_ecstore::api::notification::{
|
||||
NotificationSys, get_global_notification_sys, new_global_notification_sys,
|
||||
NotificationSys, get_global_notification_sys, new_global_notification_sys, start_remote_version_state_fleet_probe,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -961,6 +961,10 @@ pub(crate) async fn new_global_notification_sys(endpoint_pools: EndpointServerPo
|
||||
ecstore_notification::new_global_notification_sys(endpoint_pools).await
|
||||
}
|
||||
|
||||
pub(crate) fn start_remote_version_state_fleet_probe(topology_fingerprint: String) {
|
||||
ecstore_notification::start_remote_version_state_fleet_probe(topology_fingerprint);
|
||||
}
|
||||
|
||||
pub(crate) async fn read_config(api: Arc<ECStore>, file: &str) -> Result<Vec<u8>> {
|
||||
ecstore_config::com::read_config(api, file).await
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user