mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 12:09:12 +00:00
Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 950cce6f57 |
@@ -293,7 +293,7 @@ pub mod cache {
|
|||||||
pub mod capacity {
|
pub mod capacity {
|
||||||
pub use crate::core::pools::{
|
pub use crate::core::pools::{
|
||||||
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||||
path2_bucket_object, path2_bucket_object_with_base_path,
|
is_pool_activation_fleet_proof_error, path2_bucket_object, path2_bucket_object_with_base_path,
|
||||||
};
|
};
|
||||||
pub use crate::store::utils::is_reserved_or_invalid_bucket;
|
pub use crate::store::utils::is_reserved_or_invalid_bucket;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -3385,7 +3385,7 @@ pub(crate) async fn acquire_pool_activation_fleet_proof(
|
|||||||
.ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
|
.ok_or_else(|| Error::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED))
|
||||||
}
|
}
|
||||||
|
|
||||||
pub(crate) fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
pub fn is_pool_activation_fleet_proof_error(err: &Error) -> bool {
|
||||||
// Save-stage helpers add context by formatting the original error, so the
|
// Save-stage helpers add context by formatting the original error, so the
|
||||||
// marker may be nested in the display string. Restrict matching to the
|
// marker may be nested in the display string. Restrict matching to the
|
||||||
// `Error::other` I/O shape used by this activation path.
|
// `Error::other` I/O shape used by this activation path.
|
||||||
|
|||||||
@@ -452,14 +452,6 @@ async fn scoped_scan_same_cycle_maintenance_rewalks_after_root_delivery_failure(
|
|||||||
.put_object(bucket, "initial", &mut reader, &ScannerObjectOptions::default())
|
.put_object(bucket, "initial", &mut reader, &ScannerObjectOptions::default())
|
||||||
.await
|
.await
|
||||||
.expect("initial object should persist");
|
.expect("initial object should persist");
|
||||||
let lock = store.pools[0].disk_set[0]
|
|
||||||
.new_ns_lock(bucket, "initial")
|
|
||||||
.await
|
|
||||||
.expect("fixture namespace lock should be created");
|
|
||||||
let _settled = lock
|
|
||||||
.get_write_lock(Duration::from_secs(30))
|
|
||||||
.await
|
|
||||||
.expect("fixture rename tail should finish before the usage scan");
|
|
||||||
}
|
}
|
||||||
let ctx = CancellationToken::new();
|
let ctx = CancellationToken::new();
|
||||||
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
let budget = ScannerCycleBudget::new(&ctx, ScannerCycleBudgetConfig::default());
|
||||||
@@ -509,15 +501,6 @@ async fn scoped_scan_same_cycle_maintenance_rewalks_after_root_delivery_failure(
|
|||||||
.put_object("cold-bucket", "new", &mut reader, &ScannerObjectOptions::default())
|
.put_object("cold-bucket", "new", &mut reader, &ScannerObjectOptions::default())
|
||||||
.await
|
.await
|
||||||
.expect("new cold object should persist");
|
.expect("new cold object should persist");
|
||||||
let lock = store.pools[0].disk_set[0]
|
|
||||||
.new_ns_lock("cold-bucket", "new")
|
|
||||||
.await
|
|
||||||
.expect("fixture namespace lock should be created");
|
|
||||||
let _settled = lock
|
|
||||||
.get_write_lock(Duration::from_secs(30))
|
|
||||||
.await
|
|
||||||
.expect("fixture rename tail should finish before the usage scan");
|
|
||||||
drop(_settled);
|
|
||||||
record_dirty_usage_bucket("hot-bucket");
|
record_dirty_usage_bucket("hot-bucket");
|
||||||
if scan_mode == HealScanMode::Normal && !requires_full_scan {
|
if scan_mode == HealScanMode::Normal && !requires_full_scan {
|
||||||
record_dirty_usage_bucket("cold-bucket");
|
record_dirty_usage_bucket("cold-bucket");
|
||||||
|
|||||||
@@ -63,6 +63,7 @@ const EVENT_ADMIN_REQUEST_STATE: &str = "admin_request_state";
|
|||||||
const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
|
const EVENT_ADMIN_REQUEST_REJECTED: &str = "admin_request_rejected";
|
||||||
const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
|
const EVENT_ADMIN_REQUEST_FAILED: &str = "admin_request_failed";
|
||||||
const EVENT_ADMIN_RESPONSE_EMITTED: &str = "admin_response_emitted";
|
const EVENT_ADMIN_RESPONSE_EMITTED: &str = "admin_response_emitted";
|
||||||
|
const POOL_ACTIVATION_FLEET_PROOF_REQUIRED: &str = "pool activation requires a live fleet capability proof";
|
||||||
|
|
||||||
fn admin_request_id(headers: &HeaderMap) -> Option<&str> {
|
fn admin_request_id(headers: &HeaderMap) -> Option<&str> {
|
||||||
headers
|
headers
|
||||||
@@ -321,6 +322,17 @@ fn contextualize_admin_pool_api_error(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn decommission_start_api_error(err: crate::storage_api::error::StorageError) -> ApiError {
|
||||||
|
if crate::storage_api::capacity::is_pool_activation_fleet_proof_error(&err) {
|
||||||
|
return ApiError {
|
||||||
|
code: S3ErrorCode::InternalError,
|
||||||
|
message: POOL_ACTIVATION_FLEET_PROOF_REQUIRED.to_string(),
|
||||||
|
source: Some(Box::new(err)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
ApiError::from(err)
|
||||||
|
}
|
||||||
|
|
||||||
fn decommission_admin_not_initialized_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error {
|
fn decommission_admin_not_initialized_error_with_audit(operation: &str, audit: PoolAuditContext<'_>) -> S3Error {
|
||||||
error!(
|
error!(
|
||||||
event = EVENT_ADMIN_REQUEST_FAILED,
|
event = EVENT_ADMIN_REQUEST_FAILED,
|
||||||
@@ -790,7 +802,24 @@ impl Operation for StartDecommission {
|
|||||||
store
|
store
|
||||||
.decommission(ctx.clone(), pools_indices.clone())
|
.decommission(ctx.clone(), pools_indices.clone())
|
||||||
.await
|
.await
|
||||||
.map_err(ApiError::from)
|
.map_err(|err| {
|
||||||
|
error!(
|
||||||
|
event = EVENT_ADMIN_REQUEST_FAILED,
|
||||||
|
component = LOG_COMPONENT_ADMIN_API,
|
||||||
|
subsystem = LOG_SUBSYSTEM_POOL_ADMIN,
|
||||||
|
operation = "start_decommission",
|
||||||
|
action = "start_decommission",
|
||||||
|
result = "failed",
|
||||||
|
reason = "storage_decommission_failed",
|
||||||
|
request_id = %request_id,
|
||||||
|
actor = %actor,
|
||||||
|
remote_addr = %remote_addr,
|
||||||
|
pool_indices = ?pools_indices,
|
||||||
|
error = %err,
|
||||||
|
"admin request failed"
|
||||||
|
);
|
||||||
|
decommission_start_api_error(err)
|
||||||
|
})
|
||||||
.map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
|
.map_err(|err| contextualize_admin_pool_api_error(err, "start decommission", &pool_context))?;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1018,9 +1047,10 @@ impl Operation for ClearDecommission {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod pools_handler_tests {
|
mod pools_handler_tests {
|
||||||
use super::{
|
use super::{
|
||||||
AdminPoolStatus, Body, CancelDecommission, ClearDecommission, HeaderMap, ListPools, Method, Operation, Params,
|
AdminPoolStatus, Body, CancelDecommission, ClearDecommission, HeaderMap, ListPools, Method, Operation,
|
||||||
PoolAuditContext, S3ErrorCode, S3Request, StartDecommission, StatusDecommission, StatusPool, Uri,
|
POOL_ACTIVATION_FLEET_PROOF_REQUIRED, Params, PoolAuditContext, S3ErrorCode, S3Request, StartDecommission,
|
||||||
contextualize_admin_pool_api_error, decommission_admin_not_initialized_error_with_audit, decommission_peer_target,
|
StatusDecommission, StatusPool, Uri, contextualize_admin_pool_api_error,
|
||||||
|
decommission_admin_not_initialized_error_with_audit, decommission_peer_target, decommission_start_api_error,
|
||||||
has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query,
|
has_duplicate_indices, parse_mutation_pool_query, parse_pool_idx_by_id, parse_status_pool_query,
|
||||||
pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
|
pool_admin_missing_credentials_error, pool_admin_missing_credentials_error_with_request,
|
||||||
pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit,
|
pool_admin_pool_index_error_with_audit, pool_admin_pool_not_found_error_with_audit,
|
||||||
@@ -1209,6 +1239,21 @@ mod pools_handler_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn test_decommission_start_api_error_preserves_fleet_proof_retry_marker() {
|
||||||
|
let err = crate::storage_api::error::StorageError::other(POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
|
||||||
|
|
||||||
|
let err = decommission_start_api_error(err);
|
||||||
|
|
||||||
|
assert_eq!(err.code, s3s::S3ErrorCode::InternalError);
|
||||||
|
assert_eq!(err.message, POOL_ACTIVATION_FLEET_PROOF_REQUIRED);
|
||||||
|
assert!(err.source.is_some());
|
||||||
|
|
||||||
|
let unrelated = decommission_start_api_error(crate::storage_api::error::StorageError::other("disk read failed"));
|
||||||
|
assert_eq!(unrelated.code, s3s::S3ErrorCode::InternalError);
|
||||||
|
assert_eq!(unrelated.message, "We encountered an internal error, please try again.");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn test_contextualize_admin_pool_api_error_preserves_source() {
|
fn test_contextualize_admin_pool_api_error_preserves_source() {
|
||||||
let err = contextualize_admin_pool_api_error(
|
let err = contextualize_admin_pool_api_error(
|
||||||
|
|||||||
@@ -417,7 +417,7 @@ pub(crate) mod ecstore_bucket {
|
|||||||
pub(crate) mod ecstore_capacity {
|
pub(crate) mod ecstore_capacity {
|
||||||
pub(crate) use rustfs_ecstore::api::capacity::{
|
pub(crate) use rustfs_ecstore::api::capacity::{
|
||||||
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
DecommissionUnresolvedEntry, PoolDecommissionInfo, PoolStatus, get_total_usable_capacity, get_total_usable_capacity_free,
|
||||||
is_reserved_or_invalid_bucket,
|
is_pool_activation_fleet_proof_error, is_reserved_or_invalid_bucket,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -18,6 +18,8 @@
|
|||||||
use rustfs_storage_api as storage_contracts;
|
use rustfs_storage_api as storage_contracts;
|
||||||
|
|
||||||
pub(crate) mod capacity {
|
pub(crate) mod capacity {
|
||||||
|
pub(crate) use crate::storage::storage_api::ecstore_capacity::is_pool_activation_fleet_proof_error;
|
||||||
|
|
||||||
pub(crate) mod service {
|
pub(crate) mod service {
|
||||||
pub(crate) use crate::storage::storage_api::{all_local_disk, disk_drive_path, disk_endpoint};
|
pub(crate) use crate::storage::storage_api::{all_local_disk, disk_drive_path, disk_endpoint};
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user