mirror of
https://github.com/rustfs/rustfs.git
synced 2026-07-27 16:48:58 +00:00
feat(ecstore): attribute manual transition worker failures (#5324)
* feat(ecstore): attribute manual transition worker failures Add manual transition worker failure reason tracking and persistence recovery compatibility for checksum validation. Co-Authored-By: heihutu <heihutu@gmail.com> * chore(ilm): add manual transition diagnostics scripts Co-Authored-By: heihutu <heihutu@gmail.com> * style(ecstore): format manual transition attribution Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): avoid copying failure reasons via clone Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
Generated
+23
-23
@@ -56,9 +56,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aes"
|
||||
version = "0.9.1"
|
||||
version = "0.9.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138"
|
||||
checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58"
|
||||
dependencies = [
|
||||
"cipher 0.5.2",
|
||||
"cpubits",
|
||||
@@ -73,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes 0.9.1",
|
||||
"aes 0.9.2",
|
||||
"cipher 0.5.2",
|
||||
"ctr",
|
||||
"ghash",
|
||||
@@ -1528,7 +1528,7 @@ version = "0.10.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1547,7 +1547,7 @@ version = "0.3.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1870,7 +1870,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -2328,7 +2328,7 @@ version = "0.5.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rand_core 0.6.4",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -2353,11 +2353,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"typenum",
|
||||
]
|
||||
|
||||
@@ -3554,7 +3554,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3811,7 +3811,7 @@ dependencies = [
|
||||
"crypto-bigint 0.5.5",
|
||||
"digest 0.10.7",
|
||||
"ff 0.13.1",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"group 0.13.0",
|
||||
"hkdf 0.12.4",
|
||||
"pem-rfc7468 0.7.0",
|
||||
@@ -4257,9 +4257,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -4272,7 +4272,7 @@ version = "1.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||
dependencies = [
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"rustversion",
|
||||
"typenum",
|
||||
]
|
||||
@@ -5272,7 +5272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||
dependencies = [
|
||||
"block-padding 0.3.3",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5745,9 +5745,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
||||
|
||||
[[package]]
|
||||
name = "libflate"
|
||||
version = "2.3.0"
|
||||
version = "2.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df"
|
||||
checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c"
|
||||
dependencies = [
|
||||
"adler32",
|
||||
"crc32fast",
|
||||
@@ -7485,7 +7485,7 @@ version = "0.8.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"aes 0.9.2",
|
||||
"aes-gcm",
|
||||
"cbc 0.2.1",
|
||||
"der 0.8.1",
|
||||
@@ -8681,7 +8681,7 @@ version = "0.62.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"aes 0.9.2",
|
||||
"aws-lc-rs",
|
||||
"bitflags 2.13.1",
|
||||
"block-padding 0.4.2",
|
||||
@@ -10503,7 +10503,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
||||
dependencies = [
|
||||
"base16ct 0.2.0",
|
||||
"der 0.7.10",
|
||||
"generic-array 0.14.7",
|
||||
"generic-array 0.14.9",
|
||||
"pkcs8 0.10.2",
|
||||
"subtle",
|
||||
"zeroize",
|
||||
@@ -11123,7 +11123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
|
||||
dependencies = [
|
||||
"aead",
|
||||
"aes 0.9.1",
|
||||
"aes 0.9.2",
|
||||
"aes-gcm",
|
||||
"chacha20",
|
||||
"cipher 0.5.2",
|
||||
@@ -13068,7 +13068,7 @@ version = "8.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||
dependencies = [
|
||||
"aes 0.9.1",
|
||||
"aes 0.9.2",
|
||||
"bzip2",
|
||||
"constant_time_eq",
|
||||
"crc32fast",
|
||||
|
||||
+3
-2
@@ -68,7 +68,7 @@ resolver = "3"
|
||||
edition = "2024"
|
||||
license = "Apache-2.0"
|
||||
repository = "https://github.com/rustfs/rustfs"
|
||||
rust-version = "1.96.0"
|
||||
rust-version = "1.97.1"
|
||||
version = "1.0.0-beta.11"
|
||||
homepage = "https://rustfs.com"
|
||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||
@@ -211,7 +211,7 @@ zeroize = { version = "1.9.0" }
|
||||
# Time and Date
|
||||
chrono = { version = "0.4.45" }
|
||||
humantime = "2.4.0"
|
||||
jiff = { version = "0.2.34" }
|
||||
jiff = { version = "0.2.35" }
|
||||
time = { version = "0.3.54" }
|
||||
|
||||
# Database
|
||||
@@ -243,6 +243,7 @@ crossbeam-channel = "0.5.16"
|
||||
crossbeam-deque = "0.8.7"
|
||||
crossbeam-utils = "0.8.22"
|
||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
||||
#datafusion = { default-features = false, version = "54.1.0" }
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.14"
|
||||
faster-hex = "0.10.0"
|
||||
|
||||
@@ -23,13 +23,13 @@ use crate::bucket::lifecycle::lifecycle::{
|
||||
};
|
||||
use crate::bucket::lifecycle::manual_transition_job::{
|
||||
MANUAL_TRANSITION_JOB_RECORD_PREFIX, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission,
|
||||
ManualTransitionScopeAdmissionClaim, ManualTransitionTaskRecord, ManualTransitionWorkerResult,
|
||||
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
||||
ManualTransitionScopeAdmissionClaim, ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason,
|
||||
ManualTransitionWorkerResult, claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
||||
load_manual_transition_job_record, load_manual_transition_job_record_with_etag,
|
||||
manual_transition_job_id_from_record_object_name, manual_transition_job_lease_expired,
|
||||
manual_transition_worker_result_task_key, persist_manual_transition_job_progress, reconcile_manual_transition_worker_results,
|
||||
record_manual_transition_worker_result, renew_manual_transition_job_lease, save_manual_transition_job_record_if_current,
|
||||
save_manual_transition_task_if_absent,
|
||||
record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason,
|
||||
renew_manual_transition_job_lease, save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent,
|
||||
};
|
||||
use crate::bucket::lifecycle::replication_sink;
|
||||
use crate::bucket::lifecycle::replication_sink::{
|
||||
@@ -49,7 +49,9 @@ use crate::disk::error::DiskError;
|
||||
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
|
||||
use crate::error::Error;
|
||||
use crate::error::StorageError;
|
||||
use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
|
||||
use crate::error::{
|
||||
error_resp_to_object_err, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down,
|
||||
};
|
||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||
use crate::services::tier::{
|
||||
tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata},
|
||||
@@ -94,7 +96,7 @@ use s3s::dto::{
|
||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::any::Any;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||
use std::env;
|
||||
use std::future::Future;
|
||||
use std::pin::Pin;
|
||||
@@ -1658,11 +1660,12 @@ impl TransitionState {
|
||||
.await
|
||||
{
|
||||
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
|
||||
record_manual_transition_worker_result_for_task(
|
||||
record_manual_transition_worker_result_for_task_with_reason(
|
||||
api.clone(),
|
||||
job_id,
|
||||
result_key,
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
manual_transition_worker_failure_reason(&err),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
@@ -1818,6 +1821,78 @@ async fn record_manual_transition_worker_result_for_task(
|
||||
}
|
||||
}
|
||||
|
||||
async fn record_manual_transition_worker_result_for_task_with_reason(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
result_key: &str,
|
||||
result: ManualTransitionWorkerResult,
|
||||
reason: ManualTransitionWorkerFailureReason,
|
||||
) {
|
||||
if let Err(err) = record_manual_transition_worker_result_with_reason(
|
||||
api,
|
||||
job_id,
|
||||
result_key,
|
||||
result,
|
||||
manual_transition_queue_snapshot(),
|
||||
Some(reason),
|
||||
)
|
||||
.await
|
||||
{
|
||||
warn!(
|
||||
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||
component = LOG_COMPONENT_ECSTORE,
|
||||
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||
job_id = %job_id,
|
||||
error = %err,
|
||||
reason = ?reason,
|
||||
state = "manual_transition_worker_result_failed",
|
||||
"Manual transition worker failed to persist job result"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn manual_transition_worker_failure_reason(err: &Error) -> ManualTransitionWorkerFailureReason {
|
||||
if is_err_object_not_found(err) || is_err_version_not_found(err) {
|
||||
return ManualTransitionWorkerFailureReason::NotFound;
|
||||
}
|
||||
if is_err_permission_denied(err) {
|
||||
return ManualTransitionWorkerFailureReason::PermissionDenied;
|
||||
}
|
||||
if is_err_read_quorum(err) {
|
||||
return ManualTransitionWorkerFailureReason::Quorum;
|
||||
}
|
||||
if is_timeout(err) {
|
||||
return ManualTransitionWorkerFailureReason::Timeout;
|
||||
}
|
||||
if is_slow_down(err) {
|
||||
return ManualTransitionWorkerFailureReason::SlowDown;
|
||||
}
|
||||
if is_network_or_host_down(&err.to_string(), false) {
|
||||
return ManualTransitionWorkerFailureReason::Network;
|
||||
}
|
||||
ManualTransitionWorkerFailureReason::Unknown
|
||||
}
|
||||
|
||||
fn is_err_permission_denied(err: &Error) -> bool {
|
||||
match err {
|
||||
Error::VolumeAccessDenied | Error::FileAccessDenied | Error::PrefixAccessDenied(_, _) | Error::DiskAccessDenied => true,
|
||||
Error::Io(io) => io.kind() == std::io::ErrorKind::PermissionDenied,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_timeout(err: &Error) -> bool {
|
||||
match err {
|
||||
Error::Timeout => true,
|
||||
Error::Io(io) => io.kind() == std::io::ErrorKind::TimedOut,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
fn is_slow_down(err: &Error) -> bool {
|
||||
matches!(err, Error::SlowDown)
|
||||
}
|
||||
|
||||
pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
|
||||
//globalILMConfig.getExpirationWorkers()
|
||||
@@ -3233,6 +3308,8 @@ pub struct ManualTransitionRunReport {
|
||||
#[serde(default, skip_serializing_if = "is_zero_u64")]
|
||||
pub transition_failed: u64,
|
||||
pub tier_failure: u64,
|
||||
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||
pub tier_failure_by_reason: BTreeMap<ManualTransitionWorkerFailureReason, u64>,
|
||||
pub truncated_by_limit: bool,
|
||||
pub truncated_by_duration: bool,
|
||||
pub cancelled: bool,
|
||||
@@ -3303,12 +3380,18 @@ impl ManualTransitionRunReport {
|
||||
}
|
||||
|
||||
pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
|
||||
let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
|
||||
for (reason, count) in &scan_report.tier_failure_by_reason {
|
||||
let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
|
||||
tier_failure_by_reason.insert(*reason, current.max(*count));
|
||||
}
|
||||
let transition_completed = self.transition_completed;
|
||||
let transition_failed = self.transition_failed;
|
||||
*self = scan_report.clone();
|
||||
self.transition_completed = transition_completed;
|
||||
self.transition_failed = transition_failed;
|
||||
self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
|
||||
self.tier_failure_by_reason = tier_failure_by_reason;
|
||||
}
|
||||
|
||||
pub fn worker_transition_pending(&self) -> bool {
|
||||
@@ -4658,12 +4741,13 @@ mod tests {
|
||||
lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
|
||||
lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets,
|
||||
manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink,
|
||||
manual_transition_version_marker, mark_delete_opts_skip_decommissioned_on_remote_success,
|
||||
merge_stale_multipart_candidate, persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint,
|
||||
recover_manual_transition_job, recover_manual_transition_jobs, replication_state_for_delete,
|
||||
resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout,
|
||||
resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop,
|
||||
select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
manual_transition_version_marker, manual_transition_worker_failure_reason,
|
||||
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
|
||||
persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job,
|
||||
recover_manual_transition_jobs, replication_state_for_delete, resolve_tier_free_version_recovery_enabled,
|
||||
resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
|
||||
resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location,
|
||||
set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||
should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state,
|
||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||
};
|
||||
@@ -4676,16 +4760,17 @@ mod tests {
|
||||
use crate::bucket::lifecycle::config_boundary;
|
||||
use crate::bucket::lifecycle::manual_transition_job::{
|
||||
ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim,
|
||||
ManualTransitionTaskRecord, ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord,
|
||||
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
||||
legacy_manual_transition_scope_key, load_manual_transition_job_record, load_manual_transition_scope_admission,
|
||||
ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason, ManualTransitionWorkerResult,
|
||||
ManualTransitionWorkerResultRecord, claim_manual_transition_scope_admission,
|
||||
delete_manual_transition_scope_admission_if_current, legacy_manual_transition_scope_key,
|
||||
load_manual_transition_job_record, load_manual_transition_scope_admission,
|
||||
load_manual_transition_scope_admission_with_etag, load_manual_transition_task_record,
|
||||
manual_transition_scope_record_object_name, manual_transition_worker_result_object_name,
|
||||
manual_transition_worker_result_task_key, reconcile_manual_transition_worker_results,
|
||||
record_manual_transition_worker_result, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
|
||||
save_manual_transition_job_record, save_manual_transition_scope_admission_if_absent,
|
||||
save_manual_transition_scope_admission_if_current, save_manual_transition_task_if_absent,
|
||||
save_manual_transition_worker_result_if_absent,
|
||||
record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason,
|
||||
renew_manual_transition_job_lease, request_manual_transition_job_cancel, save_manual_transition_job_record,
|
||||
save_manual_transition_scope_admission_if_absent, save_manual_transition_scope_admission_if_current,
|
||||
save_manual_transition_task_if_absent, save_manual_transition_worker_result_if_absent,
|
||||
};
|
||||
use crate::bucket::lifecycle::replication_sink::{
|
||||
ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, VersionPurgeStatusType,
|
||||
@@ -7539,6 +7624,37 @@ mod tests {
|
||||
assert!(record.completed_at_unix_nanos.is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_worker_failure_reason_classifies_known_failures() {
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::ObjectNotFound("bucket".to_string(), "obj".to_string())),
|
||||
ManualTransitionWorkerFailureReason::NotFound
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::Io(std::io::Error::new(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
"file access denied",
|
||||
))),
|
||||
ManualTransitionWorkerFailureReason::PermissionDenied
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::ErasureReadQuorum),
|
||||
ManualTransitionWorkerFailureReason::Quorum
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::Timeout),
|
||||
ManualTransitionWorkerFailureReason::Timeout
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::SlowDown),
|
||||
ManualTransitionWorkerFailureReason::SlowDown
|
||||
);
|
||||
assert_eq!(
|
||||
manual_transition_worker_failure_reason(&Error::MethodNotAllowed),
|
||||
ManualTransitionWorkerFailureReason::Unknown
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn manual_transition_counts_already_transitioned_object() {
|
||||
let lc = latest_transition_lifecycle();
|
||||
@@ -8471,6 +8587,49 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn manual_transition_worker_result_stores_failure_reason() {
|
||||
let (_paths, ecstore) = setup_test_env().await;
|
||||
let job_id = Uuid::new_v4();
|
||||
let bucket = format!("manual-worker-failure-reason-{}", job_id.simple());
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let mut record = ManualTransitionJobRecord::new(job_id, &bucket, &options, "worker-owner");
|
||||
record.complete(
|
||||
ManualTransitionRunReport {
|
||||
bucket: bucket.clone(),
|
||||
enqueued: 1,
|
||||
..Default::default()
|
||||
},
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
);
|
||||
save_manual_transition_job_record(ecstore.clone(), &record)
|
||||
.await
|
||||
.expect("worker result job record should save");
|
||||
|
||||
let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None);
|
||||
let final_record = record_manual_transition_worker_result_with_reason(
|
||||
ecstore.clone(),
|
||||
job_id,
|
||||
&task_key,
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
Some(ManualTransitionWorkerFailureReason::Network),
|
||||
)
|
||||
.await
|
||||
.expect("worker result with failure reason should persist");
|
||||
|
||||
assert_eq!(
|
||||
final_record
|
||||
.report
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::Network),
|
||||
Some(&1)
|
||||
);
|
||||
assert_eq!(final_record.report.transition_failed, 1);
|
||||
assert_eq!(final_record.report.tier_failure, 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial]
|
||||
async fn manual_transition_worker_result_reconcile_applies_marker_and_releases_admission() {
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||
@@ -194,6 +195,15 @@ impl ManualTransitionJobRecord {
|
||||
}
|
||||
|
||||
pub fn record_worker_result(&mut self, result: ManualTransitionWorkerResult, queue_snapshot: ManualTransitionQueueSnapshot) {
|
||||
self.record_worker_result_with_reason(result, queue_snapshot, None);
|
||||
}
|
||||
|
||||
pub fn record_worker_result_with_reason(
|
||||
&mut self,
|
||||
result: ManualTransitionWorkerResult,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||
) {
|
||||
if self.is_terminal() {
|
||||
return;
|
||||
}
|
||||
@@ -204,6 +214,8 @@ impl ManualTransitionJobRecord {
|
||||
ManualTransitionWorkerResult::TierFailure => {
|
||||
self.report.transition_failed = self.report.transition_failed.saturating_add(1);
|
||||
self.report.tier_failure = self.report.tier_failure.saturating_add(1);
|
||||
let reason = failure_reason.unwrap_or(ManualTransitionWorkerFailureReason::Unknown);
|
||||
*self.report.tier_failure_by_reason.entry(reason).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
self.queue_snapshot = queue_snapshot;
|
||||
@@ -215,6 +227,7 @@ impl ManualTransitionJobRecord {
|
||||
&mut self,
|
||||
completed: u64,
|
||||
failed: u64,
|
||||
failure_reasons: &BTreeMap<ManualTransitionWorkerFailureReason, u64>,
|
||||
task_queued: u64,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> bool {
|
||||
@@ -237,11 +250,17 @@ impl ManualTransitionJobRecord {
|
||||
{
|
||||
return false;
|
||||
}
|
||||
let mut scan_tier_failure_by_reason = self.report.tier_failure_by_reason.clone();
|
||||
for (reason, count) in failure_reasons {
|
||||
let current = scan_tier_failure_by_reason.get(reason).copied().unwrap_or_default();
|
||||
scan_tier_failure_by_reason.insert(*reason, current.max(*count));
|
||||
}
|
||||
let scan_tier_failure = self.report.tier_failure.saturating_sub(self.report.transition_failed);
|
||||
self.report.enqueued = enqueued;
|
||||
self.report.transition_completed = transition_completed;
|
||||
self.report.transition_failed = transition_failed;
|
||||
self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
|
||||
self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
|
||||
self.queue_snapshot = queue_snapshot;
|
||||
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||
self.mark_terminal_if_worker_drained();
|
||||
@@ -401,9 +420,24 @@ impl ManualTransitionJobRecord {
|
||||
return Err(ManualTransitionJobError::Corrupt("content checksum is not a sha256 checksum"));
|
||||
}
|
||||
let mut job = persisted.job;
|
||||
let job_bytes = serde_json::to_vec(&job)?;
|
||||
let mut job_bytes = serde_json::to_vec(&job)?;
|
||||
let actual_checksum = hex_sha256(&job_bytes, ToOwned::to_owned);
|
||||
if persisted.content_sha256 != actual_checksum {
|
||||
let checksum_match = if persisted.content_sha256 == actual_checksum {
|
||||
true
|
||||
} else {
|
||||
let mut job_value = serde_json::to_value(&job)?;
|
||||
if let Some(job_obj) = job_value.as_object_mut() {
|
||||
if let Some(report) = job_obj.get_mut("report").and_then(|value| value.as_object_mut()) {
|
||||
report.remove("tier_failure_by_reason");
|
||||
}
|
||||
} else {
|
||||
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||
}
|
||||
job_bytes = serde_json::to_vec(&job_value)?;
|
||||
let fallback_checksum = hex_sha256(&job_bytes, ToOwned::to_owned);
|
||||
persisted.content_sha256 == fallback_checksum
|
||||
};
|
||||
if !checksum_match {
|
||||
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||
}
|
||||
if job.job_id != expected_job_id {
|
||||
@@ -449,6 +483,18 @@ pub enum ManualTransitionWorkerResult {
|
||||
TierFailure,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum ManualTransitionWorkerFailureReason {
|
||||
Unknown,
|
||||
NotFound,
|
||||
Network,
|
||||
PermissionDenied,
|
||||
Timeout,
|
||||
Quorum,
|
||||
SlowDown,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ManualTransitionTaskRecord {
|
||||
@@ -554,17 +600,22 @@ struct PersistedManualTransitionTaskRecord {
|
||||
record: ManualTransitionTaskRecord,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ManualTransitionWorkerResultStats {
|
||||
pub completed: u64,
|
||||
pub failed: u64,
|
||||
pub tier_failure_by_reason: BTreeMap<ManualTransitionWorkerFailureReason, u64>,
|
||||
}
|
||||
|
||||
impl ManualTransitionWorkerResultStats {
|
||||
fn record(&mut self, result: ManualTransitionWorkerResult) {
|
||||
fn record(&mut self, result: ManualTransitionWorkerResult, failure_reason: Option<ManualTransitionWorkerFailureReason>) {
|
||||
match result {
|
||||
ManualTransitionWorkerResult::Completed => self.completed = self.completed.saturating_add(1),
|
||||
ManualTransitionWorkerResult::TierFailure => self.failed = self.failed.saturating_add(1),
|
||||
ManualTransitionWorkerResult::TierFailure => {
|
||||
self.failed = self.failed.saturating_add(1);
|
||||
let reason = failure_reason.unwrap_or(ManualTransitionWorkerFailureReason::Unknown);
|
||||
*self.tier_failure_by_reason.entry(reason).or_insert(0) += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -576,16 +627,28 @@ pub struct ManualTransitionWorkerResultRecord {
|
||||
pub job_id: Uuid,
|
||||
pub task_key: String,
|
||||
pub result: ManualTransitionWorkerResult,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||
pub completed_at_unix_nanos: i128,
|
||||
}
|
||||
|
||||
impl ManualTransitionWorkerResultRecord {
|
||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||
Self::new_with_reason(job_id, task_key, result, None)
|
||||
}
|
||||
|
||||
pub fn new_with_reason(
|
||||
job_id: Uuid,
|
||||
task_key: impl Into<String>,
|
||||
result: ManualTransitionWorkerResult,
|
||||
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||
) -> Self {
|
||||
Self {
|
||||
schema: MANUAL_TRANSITION_WORKER_RESULT_SCHEMA.to_string(),
|
||||
job_id,
|
||||
task_key: task_key.into(),
|
||||
result,
|
||||
failure_reason,
|
||||
completed_at_unix_nanos: OffsetDateTime::now_utc().unix_timestamp_nanos(),
|
||||
}
|
||||
}
|
||||
@@ -620,9 +683,23 @@ impl ManualTransitionWorkerResultRecord {
|
||||
));
|
||||
}
|
||||
let record = persisted.record;
|
||||
let record_bytes = serde_json::to_vec(&record)?;
|
||||
let mut record_bytes = serde_json::to_vec(&record)?;
|
||||
let actual_checksum = hex_sha256(&record_bytes, ToOwned::to_owned);
|
||||
if persisted.content_sha256 != actual_checksum {
|
||||
let checksum_match = if persisted.content_sha256 == actual_checksum {
|
||||
true
|
||||
} else {
|
||||
if let Some(record_value) = serde_json::to_value(&record)?.as_object_mut() {
|
||||
if record.failure_reason.is_none() {
|
||||
record_value.remove("failure_reason");
|
||||
}
|
||||
record_bytes = serde_json::to_vec(record_value)?;
|
||||
} else {
|
||||
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||
}
|
||||
let fallback_checksum = hex_sha256(&record_bytes, ToOwned::to_owned);
|
||||
persisted.content_sha256 == fallback_checksum
|
||||
};
|
||||
if !checksum_match {
|
||||
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||
}
|
||||
if record.job_id != expected_job_id {
|
||||
@@ -1120,7 +1197,7 @@ async fn scan_manual_transition_worker_result_journal(
|
||||
Ok(result) => result,
|
||||
Err(err) => return Ok(ManualTransitionWorkerResultJournal::Corrupt(err.to_string())),
|
||||
};
|
||||
stats.record(result.result);
|
||||
stats.record(result.result, result.failure_reason);
|
||||
}
|
||||
if !page.is_truncated {
|
||||
return Ok(ManualTransitionWorkerResultJournal::Stats(stats));
|
||||
@@ -1151,7 +1228,13 @@ pub async fn reconcile_manual_transition_worker_results(
|
||||
};
|
||||
for _ in 0..4 {
|
||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||
let changed = record.apply_worker_result_counts(stats.completed, stats.failed, task_stats.queued, queue_snapshot);
|
||||
let changed = record.apply_worker_result_counts(
|
||||
stats.completed,
|
||||
stats.failed,
|
||||
&stats.tier_failure_by_reason,
|
||||
task_stats.queued,
|
||||
queue_snapshot,
|
||||
);
|
||||
if !changed {
|
||||
return Ok(record);
|
||||
}
|
||||
@@ -1446,7 +1529,18 @@ pub async fn record_manual_transition_worker_result(
|
||||
result: ManualTransitionWorkerResult,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let result_record = ManualTransitionWorkerResultRecord::new(job_id, task_key, result);
|
||||
record_manual_transition_worker_result_with_reason(api, job_id, task_key, result, queue_snapshot, None).await
|
||||
}
|
||||
|
||||
pub async fn record_manual_transition_worker_result_with_reason(
|
||||
api: Arc<ECStore>,
|
||||
job_id: Uuid,
|
||||
task_key: &str,
|
||||
result: ManualTransitionWorkerResult,
|
||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||
let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason);
|
||||
if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
|
||||
return load_manual_transition_job_record(api, job_id).await;
|
||||
}
|
||||
@@ -1456,7 +1550,7 @@ pub async fn record_manual_transition_worker_result(
|
||||
if record.is_terminal() {
|
||||
return Ok(record);
|
||||
}
|
||||
record.record_worker_result(result, queue_snapshot);
|
||||
record.record_worker_result_with_reason(result, queue_snapshot, failure_reason);
|
||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||
Ok(()) => {
|
||||
if record.is_terminal() {
|
||||
@@ -1652,6 +1746,52 @@ mod tests {
|
||||
assert_eq!(record.report.tier_failure, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_record_records_worker_failure_reasons() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||
|
||||
record.record_worker_result_with_reason(
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
Some(ManualTransitionWorkerFailureReason::NotFound),
|
||||
);
|
||||
record.record_worker_result_with_reason(
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
Some(ManualTransitionWorkerFailureReason::Network),
|
||||
);
|
||||
record.record_worker_result_with_reason(
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
None,
|
||||
);
|
||||
|
||||
assert_eq!(record.report.transition_failed, 3);
|
||||
assert_eq!(record.report.tier_failure, 3);
|
||||
assert_eq!(
|
||||
record
|
||||
.report
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::NotFound),
|
||||
Some(&1)
|
||||
);
|
||||
assert_eq!(
|
||||
record
|
||||
.report
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::Network),
|
||||
Some(&1)
|
||||
);
|
||||
assert_eq!(
|
||||
record
|
||||
.report
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::Unknown),
|
||||
Some(&1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_cancel_recovery_finishes_after_worker_drain() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
@@ -1709,6 +1849,67 @@ mod tests {
|
||||
assert_eq!(record.report.tier_failure, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_scan_progress_preserves_worker_failure_reasons() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||
record.record_worker_result_with_reason(
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
Some(ManualTransitionWorkerFailureReason::NotFound),
|
||||
);
|
||||
|
||||
record.update_running_progress(
|
||||
ManualTransitionRunReport {
|
||||
bucket: "bucket".to_string(),
|
||||
scanned: 3,
|
||||
enqueued: 1,
|
||||
tier_failure: 2,
|
||||
..Default::default()
|
||||
},
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
record
|
||||
.report
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::NotFound),
|
||||
Some(&1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||
record.record_worker_result_with_reason(
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
ManualTransitionQueueSnapshot::default(),
|
||||
Some(ManualTransitionWorkerFailureReason::NotFound),
|
||||
);
|
||||
|
||||
let mut failure_reasons = BTreeMap::new();
|
||||
failure_reasons.insert(ManualTransitionWorkerFailureReason::PermissionDenied, 2);
|
||||
|
||||
record.apply_worker_result_counts(0, 1, &failure_reasons, 1, ManualTransitionQueueSnapshot::default());
|
||||
|
||||
assert_eq!(
|
||||
record
|
||||
.report
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::NotFound),
|
||||
Some(&1)
|
||||
);
|
||||
assert_eq!(
|
||||
record
|
||||
.report
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||
Some(&2)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_unknown_checkpoint_persists_counters_and_cursor() {
|
||||
let options = ManualTransitionRunOptions {
|
||||
@@ -1955,6 +2156,62 @@ mod tests {
|
||||
assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_worker_result_record_round_trips_without_failure_reason() {
|
||||
let job_id = Uuid::new_v4();
|
||||
let task_key = manual_transition_worker_result_task_key("bucket", "logs/a", None);
|
||||
let record = ManualTransitionWorkerResultRecord::new_with_reason(
|
||||
job_id,
|
||||
&task_key,
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
Some(ManualTransitionWorkerFailureReason::Network),
|
||||
);
|
||||
let encoded = record.encode().expect("worker result record should encode");
|
||||
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded worker result should be json");
|
||||
let record_obj = value["record"].as_object_mut().expect("record object should be present");
|
||||
record_obj.remove("failure_reason");
|
||||
let record_bytes = serde_json::to_vec(&value["record"]).expect("record without reason should encode");
|
||||
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
|
||||
let stripped = serde_json::to_vec(&value).expect("stripped worker result should encode");
|
||||
|
||||
let decoded = ManualTransitionWorkerResultRecord::decode(job_id, &task_key, &stripped)
|
||||
.expect("worker result without reason should decode");
|
||||
|
||||
assert_eq!(decoded.failure_reason, None);
|
||||
assert_eq!(decoded.result, ManualTransitionWorkerResult::TierFailure);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_worker_result_stats_aggregates_reasons() {
|
||||
let mut stats = ManualTransitionWorkerResultStats::default();
|
||||
|
||||
stats.record(ManualTransitionWorkerResult::Completed, None);
|
||||
stats.record(
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
Some(ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||
);
|
||||
stats.record(
|
||||
ManualTransitionWorkerResult::TierFailure,
|
||||
Some(ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||
);
|
||||
stats.record(ManualTransitionWorkerResult::TierFailure, None);
|
||||
|
||||
assert_eq!(stats.completed, 1);
|
||||
assert_eq!(stats.failed, 3);
|
||||
assert_eq!(
|
||||
stats
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||
Some(&2)
|
||||
);
|
||||
assert_eq!(
|
||||
stats
|
||||
.tier_failure_by_reason
|
||||
.get(&ManualTransitionWorkerFailureReason::Unknown),
|
||||
Some(&1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_task_record_rejects_checksum_drift() {
|
||||
let job_id = Uuid::new_v4();
|
||||
@@ -1971,6 +2228,24 @@ mod tests {
|
||||
assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_record_round_trips_legacy_report_without_failure_reason_map() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
let record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||
let encoded = record.encode().expect("job record should encode");
|
||||
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
|
||||
let report = value["job"]["report"].as_object_mut().expect("report should be object");
|
||||
report.remove("tier_failure_by_reason");
|
||||
let record_bytes = serde_json::to_vec(&value["job"]).expect("job without report failure map should encode");
|
||||
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
|
||||
let legacy = serde_json::to_vec(&value).expect("job without report failure map should encode");
|
||||
|
||||
let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job report should decode");
|
||||
|
||||
assert_eq!(decoded.state, ManualTransitionJobState::Running);
|
||||
assert!(decoded.report.tier_failure_by_reason.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn manual_transition_job_record_rejects_unknown_report_fields() {
|
||||
let options = ManualTransitionRunOptions::default();
|
||||
|
||||
Executable
+253
@@ -0,0 +1,253 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
ENDPOINT=""
|
||||
JOB_ID=""
|
||||
ADMIN_TOKEN=""
|
||||
LOG_GLOB="/var/log/rustfs/*.log"
|
||||
METRIC_TYPES="1"
|
||||
METRIC_SAMPLES="1"
|
||||
METRIC_INTERVAL=""
|
||||
BUCKET_FILTER=""
|
||||
SHOW_HELP=false
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/manual_transition_debug.sh --endpoint <admin-api-base> --job-id <job-id> [options]
|
||||
|
||||
Required:
|
||||
--endpoint Admin endpoint base, e.g. http://127.0.0.1:9000
|
||||
--job-id Manual transition job_id (UUID)
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for /admin/v3 endpoints
|
||||
--log-glob Log glob to scan, defaults to /var/log/rustfs/*.log
|
||||
--metric-types Metrics types bitmask, defaults to 1 (SCANNER)
|
||||
--metric-samples Metrics samples, defaults to 1
|
||||
--metric-interval Metrics interval parameter, passed as-is to query string
|
||||
--bucket Optional bucket filter for focused log output
|
||||
-h, --help Show this help
|
||||
|
||||
Output sections:
|
||||
- lifecycle_worker_state structured events (job-scoped)
|
||||
- manual-transition admin job status
|
||||
- lifecycle_tier_operation_failed and admin_ilm_transition_state traces
|
||||
- metrics snapshot (aggregated.scanner.lifecycle_transition)
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: required command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
api_get() {
|
||||
local path="$1"
|
||||
local url="${ENDPOINT%/}${path}"
|
||||
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url"
|
||||
return
|
||||
fi
|
||||
|
||||
curl -sS "$url"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--endpoint)
|
||||
ENDPOINT="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--job-id)
|
||||
JOB_ID="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--admin-token)
|
||||
ADMIN_TOKEN="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--log-glob)
|
||||
LOG_GLOB="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--metric-types)
|
||||
METRIC_TYPES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--metric-samples)
|
||||
METRIC_SAMPLES="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--metric-interval)
|
||||
METRIC_INTERVAL="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
--bucket)
|
||||
BUCKET_FILTER="${2:-}"
|
||||
shift 2
|
||||
;;
|
||||
-h|--help)
|
||||
SHOW_HELP=true
|
||||
shift
|
||||
;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
validate() {
|
||||
if [[ "$SHOW_HELP" == true ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [[ -z "$ENDPOINT" || -z "$JOB_ID" ]]; then
|
||||
echo "ERROR: --endpoint and --job-id are required" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [[ -z "$LOG_GLOB" ]]; then
|
||||
echo "ERROR: --log-glob must not be empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
build_admin_metrics_path() {
|
||||
local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}"
|
||||
if [[ -n "$METRIC_INTERVAL" ]]; then
|
||||
path="${path}&interval=${METRIC_INTERVAL}"
|
||||
fi
|
||||
printf '%s' "$path"
|
||||
}
|
||||
|
||||
run_json_stream() {
|
||||
local query="$1"
|
||||
local filter="$2"
|
||||
local log_lines; log_lines="$(mktemp)"
|
||||
local log_errors; log_errors="$(mktemp)"
|
||||
local log_paths=()
|
||||
local match
|
||||
|
||||
while IFS= read -r match; do
|
||||
[[ -n "$match" ]] && log_paths+=("$match")
|
||||
done < <(compgen -G "$LOG_GLOB" || true)
|
||||
if [[ "${#log_paths[@]}" -eq 0 ]]; then
|
||||
log_paths=("$LOG_GLOB")
|
||||
fi
|
||||
|
||||
if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then
|
||||
:
|
||||
fi
|
||||
|
||||
if [[ -s "$log_errors" ]]; then
|
||||
sed 's/^/DEBUG: /' "$log_errors"
|
||||
fi
|
||||
|
||||
jq -R "$filter" <"$log_lines"
|
||||
|
||||
rm -f "$log_lines" "$log_errors"
|
||||
}
|
||||
|
||||
run_worker_logs() {
|
||||
local base_filter='fromjson? | select(.event == "lifecycle_worker_state")'
|
||||
local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))'
|
||||
local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state'
|
||||
|
||||
if [[ -n "$BUCKET_FILTER" ]]; then
|
||||
manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER)'
|
||||
dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER) | .state'
|
||||
fi
|
||||
|
||||
echo "## lifecycle_worker_state (all)"
|
||||
run_json_stream "lifecycle_worker_state" "$base_filter"
|
||||
|
||||
echo "## lifecycle_worker_state (manual_transition + job filter: ${JOB_ID})"
|
||||
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||
run_json_stream "lifecycle_worker_state" "$manual_filter" || true
|
||||
|
||||
echo "## lifecycle_worker_state state distribution (manual_transition only)"
|
||||
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||
run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true
|
||||
}
|
||||
|
||||
run_admin_job() {
|
||||
echo "## manual transition job status"
|
||||
api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \
|
||||
| jq '{status, mode, job_id, failure_reason, report: .report, queue_snapshot: .queue_snapshot}'
|
||||
}
|
||||
|
||||
run_tier_operation_failures() {
|
||||
echo "## lifecycle_tier_operation_failed events"
|
||||
local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||
if [[ -n "$BUCKET_FILTER" ]]; then
|
||||
filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||
fi
|
||||
|
||||
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||
run_json_stream "lifecycle_tier_operation_failed" "$filter" || true
|
||||
}
|
||||
|
||||
run_admin_events() {
|
||||
echo "## admin_ilm_transition_state (job filter)"
|
||||
local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||
if [[ -n "$BUCKET_FILTER" ]]; then
|
||||
filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||
fi
|
||||
|
||||
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||
run_json_stream "admin_ilm_transition_state" "$filter" || true
|
||||
}
|
||||
|
||||
run_metrics() {
|
||||
echo "## metrics: scanner.lifecycle_transition"
|
||||
local path
|
||||
path="$(build_admin_metrics_path)"
|
||||
api_get "$path" | jq '.aggregated.scanner.lifecycle_transition'
|
||||
|
||||
echo "## metrics: scanner lifecycle cycle"
|
||||
api_get "$path" | jq '.aggregated.scanner | {lifecycle_transition: .lifecycle_transition, current_cycle_ilm_actions: .current_cycle_ilm_actions, current_cycle_lifecycle_transition_actions: .current_cycle_lifecycle_transition_actions}'
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
validate
|
||||
|
||||
require_cmd curl
|
||||
require_cmd jq
|
||||
require_cmd rg
|
||||
|
||||
run_worker_logs
|
||||
run_admin_events
|
||||
run_tier_operation_failures
|
||||
run_admin_job
|
||||
run_metrics
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+590
@@ -0,0 +1,590 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
ENDPOINT=""
|
||||
JOB_ID=""
|
||||
ADMIN_TOKEN=""
|
||||
LOG_GLOB="/var/log/rustfs/*.log"
|
||||
METRIC_TYPES="1"
|
||||
METRIC_SAMPLES="1"
|
||||
METRIC_INTERVAL=""
|
||||
TASK_BUCKET_FILTER=""
|
||||
OUT_DIR=""
|
||||
|
||||
SYS_BUCKET="${RUSTFS_META_BUCKET:-.rustfs.sys}"
|
||||
TASK_PREFIX="ilm/manual-transition/tasks"
|
||||
RESULT_PREFIX="ilm/manual-transition/results"
|
||||
|
||||
MC_BIN="${MC_BIN:-mc}"
|
||||
MC_ALIAS=""
|
||||
MC_ENDPOINT=""
|
||||
MC_ACCESS_KEY=""
|
||||
MC_SECRET_KEY=""
|
||||
MC_ALIAS_CREATED=false
|
||||
|
||||
SHOW_HELP=false
|
||||
DRY_RUN=false
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/manual_transition_journal_audit.sh --endpoint <admin-api> --job-id <job-uuid> [options]
|
||||
|
||||
Required:
|
||||
--endpoint Admin endpoint base, e.g. https://127.0.0.1:9000
|
||||
--job-id Manual transition job_id
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for /admin/v3 endpoints
|
||||
--log-glob Log glob, default /var/log/rustfs/*.log
|
||||
--metric-types /admin/v3/metrics type flags, default 1
|
||||
--metric-samples /admin/v3/metrics sample count, default 1
|
||||
--metric-interval /admin/v3/metrics interval query param
|
||||
--bucket-filter Optional bucket filter for logs
|
||||
--sys-bucket Meta bucket for journal read (default .rustfs.sys)
|
||||
--mc-alias Reuse existing mc alias (skip alias set)
|
||||
--mc-endpoint mc endpoint for .rustfs.sys access
|
||||
--mc-access-key mc access key
|
||||
--mc-secret-key mc secret key
|
||||
--out-dir Artifact output dir
|
||||
--dry-run
|
||||
--help Show usage
|
||||
|
||||
Output:
|
||||
- Journal CSV/JSON samples for manual transition tasks and worker results
|
||||
- Reconcile report (task_count/result_count/missing/mismatch)
|
||||
- /admin/v3 lifecycle_worker_state / admin_ilm_transition_state events
|
||||
- lifecycle_tier_operation_failed logs
|
||||
- aggregated lifecycle transition metrics (admin endpoint sample)
|
||||
USAGE
|
||||
}
|
||||
|
||||
require_bash4() {
|
||||
if (( BASH_VERSINFO[0] < 4 )); then
|
||||
echo "ERROR: manual transition journal audit requires bash >= 4.0" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
arg_value() {
|
||||
local flag="$1"
|
||||
local value="${2:-}"
|
||||
if [[ -z "$value" || "$value" == --* ]]; then
|
||||
echo "ERROR: missing value for $flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
require_nonempty() {
|
||||
local value="$1"
|
||||
local label="$2"
|
||||
if [[ -z "$value" ]]; then
|
||||
echo "ERROR: $label is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
normalize_uuid() {
|
||||
local raw="${1//-/}"
|
||||
printf '%s' "${raw,,}"
|
||||
}
|
||||
|
||||
validate_uuid() {
|
||||
local normalized
|
||||
normalized="$(normalize_uuid "$1")"
|
||||
if ! [[ "$normalized" =~ ^[0-9a-f]{32}$ ]]; then
|
||||
echo "ERROR: invalid uuid (expect UUIDv4): $1" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$normalized"
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--job-id) JOB_ID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--log-glob) LOG_GLOB="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--metric-types) METRIC_TYPES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--metric-samples) METRIC_SAMPLES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--metric-interval) METRIC_INTERVAL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--bucket-filter) TASK_BUCKET_FILTER="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--sys-bucket) SYS_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--mc-alias) MC_ALIAS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--mc-endpoint) MC_ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--mc-access-key) MC_ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--mc-secret-key) MC_SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) SHOW_HELP=true; shift ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
validate_args() {
|
||||
if [[ "$SHOW_HELP" == true ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
|
||||
require_nonempty "$ENDPOINT" "--endpoint"
|
||||
require_nonempty "$JOB_ID" "--job-id"
|
||||
|
||||
JOB_ID="$(validate_uuid "$JOB_ID")"
|
||||
if [[ "$LOG_GLOB" == "" ]]; then
|
||||
echo "ERROR: --log-glob must not be empty" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "$MC_ALIAS" ]]; then
|
||||
if [[ -z "$MC_ENDPOINT" || -z "$MC_ACCESS_KEY" || -z "$MC_SECRET_KEY" ]]; then
|
||||
echo "ERROR: either --mc-alias or (--mc-endpoint + --mc-access-key + --mc-secret-key) is required" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
cleanup_mc_alias() {
|
||||
if [[ "$MC_ALIAS_CREATED" == true ]]; then
|
||||
"${MC_BIN}" alias remove "$MC_ALIAS" >/dev/null 2>&1 || true
|
||||
fi
|
||||
}
|
||||
|
||||
setup_output() {
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-journal-audit/$(date +%Y%m%dT%H%M%S)"
|
||||
fi
|
||||
mkdir -p "$OUT_DIR"
|
||||
SUMMARY_TXT="${OUT_DIR}/audit_summary.txt"
|
||||
TASKS_CSV="${OUT_DIR}/tasks.csv"
|
||||
RESULTS_CSV="${OUT_DIR}/results.csv"
|
||||
RECONCILE_CSV="${OUT_DIR}/journal_reconcile.csv"
|
||||
COMMANDS_TXT="${OUT_DIR}/commands.txt"
|
||||
{
|
||||
echo "task_key,bucket,object,version_id,storage_class,queued_at_unix_nanos,task_object"
|
||||
} > "$TASKS_CSV"
|
||||
{
|
||||
echo "task_key,result,completion_reason,completed_at_unix_nanos,result_object"
|
||||
} > "$RESULTS_CSV"
|
||||
{
|
||||
echo "task_key,bucket,object,version_id,storage_class,task_has_result,result,result_reason,task_job_id_match,result_job_id_match,task_path,result_path"
|
||||
} > "$RECONCILE_CSV"
|
||||
}
|
||||
|
||||
setup_mc_alias() {
|
||||
if [[ "$MC_ALIAS" == "" ]]; then
|
||||
MC_ALIAS="mt-journal-audit-${JOB_ID}"
|
||||
MC_ALIAS_CREATED=true
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "[DRY-RUN] mc alias set ${MC_ALIAS} ${MC_ENDPOINT} ${MC_ACCESS_KEY} ***"
|
||||
else
|
||||
"${MC_BIN}" alias set "${MC_ALIAS}" "${MC_ENDPOINT}" "${MC_ACCESS_KEY}" "${MC_SECRET_KEY}" >/dev/null
|
||||
fi
|
||||
fi
|
||||
if ! [[ "$DRY_RUN" == true ]]; then
|
||||
"${MC_BIN}" ls "${MC_ALIAS}/${SYS_BUCKET}" >/dev/null
|
||||
fi
|
||||
trap cleanup_mc_alias EXIT
|
||||
}
|
||||
|
||||
api_get() {
|
||||
local path="$1"
|
||||
local url="${ENDPOINT%/}${path}"
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url"
|
||||
else
|
||||
curl -sS "$url"
|
||||
fi
|
||||
}
|
||||
|
||||
build_admin_metrics_path() {
|
||||
local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}"
|
||||
if [[ -n "$METRIC_INTERVAL" ]]; then
|
||||
path="${path}&interval=${METRIC_INTERVAL}"
|
||||
fi
|
||||
printf '%s' "$path"
|
||||
}
|
||||
|
||||
run_json_stream() {
|
||||
local query="$1"
|
||||
local filter="$2"
|
||||
local log_lines; log_lines="$(mktemp)"
|
||||
local log_errors; log_errors="$(mktemp)"
|
||||
local log_paths=()
|
||||
local match
|
||||
|
||||
while IFS= read -r match; do
|
||||
[[ -n "$match" ]] && log_paths+=("$match")
|
||||
done < <(compgen -G "$LOG_GLOB" || true)
|
||||
if [[ "${#log_paths[@]}" -eq 0 ]]; then
|
||||
log_paths=("$LOG_GLOB")
|
||||
fi
|
||||
|
||||
if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then
|
||||
:
|
||||
fi
|
||||
|
||||
if [[ -s "$log_errors" ]]; then
|
||||
sed 's/^/DEBUG: /' "$log_errors"
|
||||
fi
|
||||
|
||||
if [[ -s "$log_lines" ]]; then
|
||||
jq -R "$filter" <"$log_lines"
|
||||
else
|
||||
echo "[]"
|
||||
fi
|
||||
|
||||
rm -f "$log_lines" "$log_errors"
|
||||
}
|
||||
|
||||
run_worker_logs() {
|
||||
local base_filter='fromjson? | select(.event == "lifecycle_worker_state")'
|
||||
local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))'
|
||||
local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state'
|
||||
|
||||
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
|
||||
manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER)'
|
||||
dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER) | .state'
|
||||
fi
|
||||
|
||||
echo "## lifecycle_worker_state all"
|
||||
run_json_stream "lifecycle_worker_state" "$base_filter" || true
|
||||
echo
|
||||
echo "## lifecycle_worker_state manual_transition + job filter"
|
||||
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$manual_filter" || true
|
||||
echo
|
||||
echo "## lifecycle_worker_state state distribution"
|
||||
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true
|
||||
}
|
||||
|
||||
run_admin_job() {
|
||||
echo "## manual transition job status"
|
||||
api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \
|
||||
| jq '{status, mode, job_id, report: .report, queue_snapshot: .queue_snapshot, cancel_requested}'
|
||||
}
|
||||
|
||||
run_admin_events() {
|
||||
echo "## admin_ilm_transition_state events"
|
||||
local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
|
||||
filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||
fi
|
||||
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \
|
||||
run_json_stream "admin_ilm_transition_state" "$filter" || true
|
||||
}
|
||||
|
||||
run_tier_operation_failures() {
|
||||
echo "## lifecycle_tier_operation_failed events"
|
||||
local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
|
||||
filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||
fi
|
||||
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \
|
||||
run_json_stream "lifecycle_tier_operation_failed" "$filter" || true
|
||||
}
|
||||
|
||||
run_metrics() {
|
||||
echo "## metrics: scanner.lifecycle_transition"
|
||||
local path
|
||||
path="$(build_admin_metrics_path)"
|
||||
api_get "$path" | jq '.aggregated.scanner.lifecycle_transition'
|
||||
}
|
||||
|
||||
journal_key_prefix() {
|
||||
local prefix="$1"
|
||||
local shard_a="${JOB_ID:0:2}"
|
||||
local shard_b="${JOB_ID:2:2}"
|
||||
printf '%s/%s/%s/%s' "$prefix" "$shard_a" "$shard_b" "$JOB_ID"
|
||||
}
|
||||
|
||||
csv_escape() {
|
||||
local value="${1:-}"
|
||||
value="${value//\"/\"\"}"
|
||||
printf '\"%s\"' "$value"
|
||||
}
|
||||
|
||||
read_json_file() {
|
||||
local object_key="$1"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo ""
|
||||
return 0
|
||||
fi
|
||||
"${MC_BIN}" cat "${MC_ALIAS}/${SYS_BUCKET}/${object_key}"
|
||||
}
|
||||
|
||||
collect_journals() {
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "[DRY-RUN] skip .rustfs.sys object reads."
|
||||
echo "dry_run=true" > "$SUMMARY_TXT"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local task_prefix result_prefix
|
||||
task_prefix="$(journal_key_prefix "$TASK_PREFIX")"
|
||||
result_prefix="$(journal_key_prefix "$RESULT_PREFIX")"
|
||||
|
||||
local task_objs
|
||||
local result_objs
|
||||
local object_name
|
||||
local task_key
|
||||
local raw
|
||||
local record_task_job record_result_job
|
||||
local expected_job
|
||||
expected_job="$JOB_ID"
|
||||
|
||||
local -A task_bucket task_object task_version task_storage task_queued task_job task_path
|
||||
local -A result_status result_reason result_completed result_job result_path
|
||||
local -A task_seen
|
||||
local -A result_seen
|
||||
|
||||
task_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${task_prefix}" | jq -r 'select(.type == "file") | .key')"
|
||||
result_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${result_prefix}" | jq -r 'select(.type == "file") | .key')"
|
||||
|
||||
while IFS= read -r object_name; do
|
||||
[[ -z "$object_name" ]] && continue
|
||||
task_key="${object_name##*/}"
|
||||
task_key="${task_key%.json}"
|
||||
if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
echo "WARN: ignore invalid task object name: ${object_name}" >&2
|
||||
continue
|
||||
fi
|
||||
raw="$(read_json_file "$object_name")"
|
||||
if ! record_task_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then
|
||||
echo "WARN: parse error task object: ${object_name}" >&2
|
||||
continue
|
||||
fi
|
||||
if [[ -z "$record_task_job" ]]; then
|
||||
echo "WARN: task object missing record.job_id: ${object_name}" >&2
|
||||
continue
|
||||
fi
|
||||
task_seen["$task_key"]="1"
|
||||
task_bucket["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.bucket // empty' )"
|
||||
task_object["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.object // empty' )"
|
||||
task_version["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.version_id // empty' )"
|
||||
task_storage["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.storage_class // empty' )"
|
||||
task_queued["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.queued_at_unix_nanos // empty' )"
|
||||
task_job["$task_key"]="$(printf '%s' "$record_task_job")"
|
||||
task_path["$task_key"]="$object_name"
|
||||
printf '%s,%s,%s,%s,%s,%s,%s\n' \
|
||||
"$(csv_escape "$task_key")" \
|
||||
"$(csv_escape "${task_bucket[$task_key]}")" \
|
||||
"$(csv_escape "${task_object[$task_key]}")" \
|
||||
"$(csv_escape "${task_version[$task_key]}")" \
|
||||
"$(csv_escape "${task_storage[$task_key]}")" \
|
||||
"$(csv_escape "${task_queued[$task_key]}")" \
|
||||
"$(csv_escape "$object_name")" >> "$TASKS_CSV"
|
||||
done <<< "$task_objs"
|
||||
|
||||
while IFS= read -r object_name; do
|
||||
[[ -z "$object_name" ]] && continue
|
||||
task_key="${object_name##*/}"
|
||||
task_key="${task_key%.json}"
|
||||
if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then
|
||||
echo "WARN: ignore invalid result object name: ${object_name}" >&2
|
||||
continue
|
||||
fi
|
||||
raw="$(read_json_file "$object_name")"
|
||||
if ! record_result_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then
|
||||
echo "WARN: parse error result object: ${object_name}" >&2
|
||||
continue
|
||||
fi
|
||||
if [[ -z "$record_result_job" ]]; then
|
||||
echo "WARN: result object missing record.job_id: ${object_name}" >&2
|
||||
continue
|
||||
fi
|
||||
result_seen["$task_key"]="1"
|
||||
result_status["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.result // .result // empty' )"
|
||||
result_reason["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.failure_reason // empty' )"
|
||||
result_completed["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.completed_at_unix_nanos // empty' )"
|
||||
result_job["$task_key"]="$(printf '%s' "$record_result_job")"
|
||||
result_path["$task_key"]="$object_name"
|
||||
printf '%s,%s,%s,%s,%s\n' \
|
||||
"$(csv_escape "$task_key")" \
|
||||
"$(csv_escape "${result_status[$task_key]}")" \
|
||||
"$(csv_escape "${result_reason[$task_key]}")" \
|
||||
"$(csv_escape "${result_completed[$task_key]}")" \
|
||||
"$(csv_escape "$object_name")" >> "$RESULTS_CSV"
|
||||
done <<< "$result_objs"
|
||||
|
||||
local task_count=0
|
||||
local result_count=0
|
||||
local missing_count=0
|
||||
local mismatch_count=0
|
||||
local completed_count=0
|
||||
local failed_count=0
|
||||
local task_job_id_match
|
||||
local result_job_id_match
|
||||
local task_record_job
|
||||
local result_record_job
|
||||
local -A failure_by_reason=()
|
||||
|
||||
task_count="${#task_seen[@]}"
|
||||
result_count="${#result_seen[@]}"
|
||||
|
||||
for task_key in "${!task_seen[@]}"; do
|
||||
local result=""
|
||||
local reason=""
|
||||
local task_has_result="false"
|
||||
if [[ -n "${result_seen[$task_key]+x}" ]]; then
|
||||
task_has_result="true"
|
||||
result="${result_status[$task_key]}"
|
||||
reason="${result_reason[$task_key]}"
|
||||
if [[ "${result}" == "Completed" ]]; then
|
||||
completed_count=$((completed_count + 1))
|
||||
elif [[ "${result}" == "TierFailure" ]]; then
|
||||
failed_count=$((failed_count + 1))
|
||||
reason="${reason:-Unknown}"
|
||||
failure_by_reason["$reason"]=$(( ${failure_by_reason["$reason"]:-0} + 1 ))
|
||||
fi
|
||||
else
|
||||
missing_count=$((missing_count + 1))
|
||||
fi
|
||||
|
||||
task_job_id_match="false"
|
||||
result_job_id_match="false"
|
||||
|
||||
task_record_job="${task_job[$task_key]-}"
|
||||
if [[ -n "$task_record_job" && "$(normalize_uuid "$task_record_job")" == "$expected_job" ]]; then
|
||||
task_job_id_match="true"
|
||||
fi
|
||||
result_record_job="${result_job[$task_key]-}"
|
||||
if [[ -n "$result_record_job" && "$(normalize_uuid "$result_record_job")" == "$expected_job" ]]; then
|
||||
result_job_id_match="true"
|
||||
fi
|
||||
if [[ "$task_job_id_match" != "true" ]] || [[ -n "${result_seen[$task_key]+x}" && "$result_job_id_match" != "true" ]]; then
|
||||
mismatch_count=$((mismatch_count + 1))
|
||||
fi
|
||||
|
||||
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
|
||||
"$(csv_escape "$task_key")" \
|
||||
"$(csv_escape "${task_bucket[$task_key]}")" \
|
||||
"$(csv_escape "${task_object[$task_key]}")" \
|
||||
"$(csv_escape "${task_version[$task_key]}")" \
|
||||
"$(csv_escape "${task_storage[$task_key]}")" \
|
||||
"$(csv_escape "$task_has_result")" \
|
||||
"$(csv_escape "${result_status[$task_key]}")" \
|
||||
"$(csv_escape "${result_reason[$task_key]}")" \
|
||||
"$(csv_escape "$task_job_id_match")" \
|
||||
"$(csv_escape "$result_job_id_match")" \
|
||||
"$(csv_escape "${task_path[$task_key]}")" \
|
||||
"$(csv_escape "${result_path[$task_key]-}")" >> "$RECONCILE_CSV"
|
||||
done
|
||||
|
||||
local orphan_count=0
|
||||
for task_key in "${!result_seen[@]}"; do
|
||||
if [[ -z "${task_seen[$task_key]+x}" ]]; then
|
||||
orphan_count=$((orphan_count + 1))
|
||||
printf 'orphan_result_task_key=%s\n' "$task_key" >> "$SUMMARY_TXT"
|
||||
fi
|
||||
done
|
||||
|
||||
{
|
||||
echo "job_id=$JOB_ID"
|
||||
echo "sys_bucket=$SYS_BUCKET"
|
||||
echo "expected_task_prefix=$task_prefix"
|
||||
echo "expected_result_prefix=$result_prefix"
|
||||
echo "task_count=${task_count}"
|
||||
echo "result_count=${result_count}"
|
||||
echo "missing_result_count=${missing_count}"
|
||||
echo "job_id_mismatch_count=${mismatch_count}"
|
||||
echo "orphan_result_count=${orphan_count}"
|
||||
echo "completed_result_count=${completed_count}"
|
||||
echo "failed_result_count=${failed_count}"
|
||||
for reason in "${!failure_by_reason[@]}"; do
|
||||
echo "result_failure_${reason// /_}=${failure_by_reason[$reason]}"
|
||||
done
|
||||
} > "$SUMMARY_TXT"
|
||||
|
||||
{
|
||||
echo
|
||||
echo "## journal summary"
|
||||
echo "- tasks: ${task_count}"
|
||||
echo "- results: ${result_count}"
|
||||
echo "- missing result for task: ${missing_count}"
|
||||
echo "- orphaned result rows: ${orphan_count}"
|
||||
echo "- completed result: ${completed_count}"
|
||||
echo "- failed result: ${failed_count}"
|
||||
} >> "$SUMMARY_TXT"
|
||||
}
|
||||
|
||||
build_command_notes() {
|
||||
{
|
||||
echo "# Run snippets (to use manually)"
|
||||
echo
|
||||
echo "curl -sS ${ENDPOINT}/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES} | jq '.aggregated.scanner.lifecycle_transition'"
|
||||
echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token <token> --sys-bucket ${SYS_BUCKET}"
|
||||
echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token <token> --sys-bucket ${SYS_BUCKET} --out-dir ${OUT_DIR}"
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "# Add --mc-endpoint, --mc-access-key, --mc-secret-key when writing to .rustfs.sys."
|
||||
fi
|
||||
} > "$COMMANDS_TXT"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
if [[ "$SHOW_HELP" == true ]]; then
|
||||
usage
|
||||
exit 0
|
||||
fi
|
||||
require_bash4
|
||||
validate_args
|
||||
require_cmd "$MC_BIN"
|
||||
require_cmd curl
|
||||
require_cmd jq
|
||||
require_cmd rg
|
||||
setup_output
|
||||
setup_mc_alias
|
||||
|
||||
build_command_notes
|
||||
run_worker_logs
|
||||
echo
|
||||
run_admin_events
|
||||
echo
|
||||
run_tier_operation_failures
|
||||
echo
|
||||
run_admin_job
|
||||
echo
|
||||
run_metrics
|
||||
echo
|
||||
collect_journals
|
||||
echo "Artifacts: $OUT_DIR"
|
||||
echo " - $TASKS_CSV"
|
||||
echo " - $RESULTS_CSV"
|
||||
echo " - $RECONCILE_CSV"
|
||||
echo " - $SUMMARY_TXT"
|
||||
echo " - $COMMANDS_TXT"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
+291
@@ -0,0 +1,291 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
ENDPOINT=""
|
||||
ADMIN_TOKEN=""
|
||||
OUT_DIR=""
|
||||
PHASE_MATRIX="request-only:100:0:30,canary:90:10:120,mixed:50:50:240,full-rollout:0:100:360,rollback:100:0:30"
|
||||
CONCURRENCIES="8,16,32"
|
||||
OBJECT_COUNTS="5k,20k,50k"
|
||||
JOB_BUCKET="manual-transition"
|
||||
JOB_PREFIX="journal-mixed-rollout"
|
||||
READ_RATIO="90"
|
||||
RUN_ADMIN_CHECKS=true
|
||||
DRY_RUN=false
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/manual_transition_mixed_rollout_matrix.sh --endpoint <admin-api> [options]
|
||||
|
||||
Required:
|
||||
--endpoint Admin API base, e.g. https://127.0.0.1:9000
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for admin calls
|
||||
--phase-matrix Comma-separated phase spec: name:old_pct:new_pct:duration_min
|
||||
--concurrencies Comma-separated concurrency list
|
||||
--object-counts Comma-separated queue-size list (for plan rows)
|
||||
--job-bucket Manual transition source bucket for sample commands
|
||||
--job-prefix Prefix for generated job notes
|
||||
--read-ratio Target read ratio percentage for mixed workload
|
||||
--no-admin-checks Skip generated admin check commands
|
||||
--out-dir Output dir
|
||||
--dry-run
|
||||
--help
|
||||
|
||||
Examples:
|
||||
scripts/manual_transition_mixed_rollout_matrix.sh \
|
||||
--endpoint https://127.0.0.1:9000 \
|
||||
--admin-token "$TOKEN" \
|
||||
--phase-matrix "request-only:100:0:30,canary:90:10:120,full:0:100:240"
|
||||
USAGE
|
||||
}
|
||||
|
||||
arg_value() {
|
||||
local flag="$1"
|
||||
local value="${2:-}"
|
||||
if [[ -z "$value" || "$value" == --* ]]; then
|
||||
echo "ERROR: missing value for $flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--phase-matrix) PHASE_MATRIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--object-counts) OBJECT_COUNTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--job-bucket) JOB_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--job-prefix) JOB_PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--read-ratio) READ_RATIO="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
trim() {
|
||||
echo "$1" | awk '{$1=$1;print}'
|
||||
}
|
||||
|
||||
parse_phase_row() {
|
||||
local spec="$1"
|
||||
local name old_ratio new_ratio duration_min
|
||||
|
||||
name="$(echo "$spec" | cut -d':' -f1)"
|
||||
old_ratio="$(echo "$spec" | cut -d':' -f2)"
|
||||
new_ratio="$(echo "$spec" | cut -d':' -f3)"
|
||||
duration_min="$(echo "$spec" | cut -d':' -f4)"
|
||||
name="$(trim "$name")"
|
||||
old_ratio="$(trim "$old_ratio")"
|
||||
new_ratio="$(trim "$new_ratio")"
|
||||
duration_min="$(trim "$duration_min")"
|
||||
|
||||
if [[ -z "$name" || -z "$old_ratio" || -z "$new_ratio" || -z "$duration_min" ]]; then
|
||||
echo "ERROR: invalid phase spec: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$old_ratio" =~ ^[0-9]+$ && "$new_ratio" =~ ^[0-9]+$ && "$duration_min" =~ ^[0-9]+$ ]]; then
|
||||
echo "ERROR: phase ratio/duration must be integers: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$name|$old_ratio|$new_ratio|$duration_min"
|
||||
}
|
||||
|
||||
gate_for_phase() {
|
||||
case "$1" in
|
||||
request-only)
|
||||
echo "No mixed-version errors; unknown failures should stay 0 in job report; log ratio must trend down."
|
||||
;;
|
||||
canary)
|
||||
echo "old/new mismatch in worker results = 0; no Unknown job failures."
|
||||
;;
|
||||
mixed)
|
||||
echo "request-only + canary acceptance gates both pass for 2+ windows, then proceed."
|
||||
;;
|
||||
full-rollout|full)
|
||||
echo "full canary-to-full transition completed, no tier_failure in report, worker_result mismatch = 0."
|
||||
;;
|
||||
rollback)
|
||||
echo "all rollout checks reverse cleanly; queued -> completed in job terminal state."
|
||||
;;
|
||||
*)
|
||||
echo "run admin job status + journal reconcile + metric delta sanity check."
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
admin_check_cmd() {
|
||||
local phase="$1"
|
||||
local job_id_ref="$2"
|
||||
local metric_types="$3"
|
||||
local metric_samples="$4"
|
||||
local metric_path
|
||||
local job_url
|
||||
local metric_url
|
||||
local job_filter='.report, .queue_snapshot, .failure_reason'
|
||||
local metric_filter='.aggregated.scanner.lifecycle_transition'
|
||||
|
||||
if [[ "$RUN_ADMIN_CHECKS" != true ]]; then
|
||||
return
|
||||
fi
|
||||
metric_path="/rustfs/admin/v3/metrics?types=${metric_types}&n=${metric_samples}"
|
||||
job_url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id_ref}"
|
||||
metric_url="${ENDPOINT%/}${metric_path}"
|
||||
|
||||
{
|
||||
printf '# check-%s\n' "$phase"
|
||||
printf 'curl -sS'
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\""
|
||||
fi
|
||||
printf ' -X GET "%s" | jq '\''%s'\''\n' "$job_url" "$job_filter"
|
||||
printf 'curl -sS'
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\""
|
||||
fi
|
||||
printf ' -X GET "%s" | jq '\''%s'\''\n' "$metric_url" "$metric_filter"
|
||||
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref"
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
printf " --admin-token \"\${ADMIN_TOKEN}\""
|
||||
fi
|
||||
printf ' --sys-bucket .rustfs.sys\n'
|
||||
}
|
||||
}
|
||||
|
||||
run_rows() {
|
||||
local matrix_csv="${OUT_DIR}/mixed_rollout_matrix.csv"
|
||||
local run_script="${OUT_DIR}/run_phase_commands.sh"
|
||||
local checklist="${OUT_DIR}/mixed_rollout_checklist.md"
|
||||
local command_manifest="${OUT_DIR}/run_phase_notes.txt"
|
||||
local admin_check
|
||||
local admin_check_csv
|
||||
local job_id_ref
|
||||
|
||||
{
|
||||
echo "phase,old_pct,new_pct,duration_min,concurrency,object_count,read_ratio,gate,admin_check"
|
||||
} > "$matrix_csv"
|
||||
|
||||
{
|
||||
echo "#!/usr/bin/env bash"
|
||||
echo "set -euo pipefail"
|
||||
echo ""
|
||||
echo "# generated matrix commands: fill in JOB_ID after each manual transition run"
|
||||
echo ""
|
||||
} > "$run_script"
|
||||
|
||||
{
|
||||
echo "# mixed-version rollout command template"
|
||||
echo "Endpoint: ${ENDPOINT}"
|
||||
echo "Bucket: ${JOB_BUCKET}"
|
||||
echo "Prefix: ${JOB_PREFIX}"
|
||||
echo "Read ratio: ${READ_RATIO}%"
|
||||
echo ""
|
||||
} > "$checklist"
|
||||
|
||||
while IFS=',' read -r phase_spec; do
|
||||
[[ -z "$phase_spec" ]] && continue
|
||||
parsed="$(parse_phase_row "$phase_spec")"
|
||||
IFS='|' read -r phase_name old_ratio new_ratio duration_min <<< "$parsed"
|
||||
|
||||
gate="$(gate_for_phase "$phase_name")"
|
||||
IFS=',' read -r -a counts <<< "$OBJECT_COUNTS"
|
||||
IFS=',' read -r -a concs <<< "$CONCURRENCIES"
|
||||
for conc in "${concs[@]}"; do
|
||||
conc="$(trim "$conc")"
|
||||
[[ -z "$conc" ]] && continue
|
||||
for count in "${counts[@]}"; do
|
||||
count="$(trim "$count")"
|
||||
[[ -z "$count" || -z "$conc" ]] && continue
|
||||
|
||||
job_id_ref="<JOB_ID_${phase_name}_c${conc}_q${count}>"
|
||||
admin_check="$(admin_check_cmd "$phase_name" "$job_id_ref" 1 1 | tr '\n' ';')"
|
||||
admin_check_csv="${admin_check//\"/\"\"}"
|
||||
echo "${phase_name},${old_ratio},${new_ratio},${duration_min},${conc},${count},${READ_RATIO},\"${gate}\",\"${admin_check_csv}\"" >> "$matrix_csv"
|
||||
|
||||
{
|
||||
echo "# phase: ${phase_name} concurrency: ${conc} object_count: ${count}"
|
||||
echo "# old:new = ${old_ratio}:${new_ratio}, duration=${duration_min}min"
|
||||
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref"
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
printf " --admin-token \"\${ADMIN_TOKEN}\""
|
||||
fi
|
||||
printf '\n'
|
||||
echo ""
|
||||
} >> "$run_script"
|
||||
done
|
||||
done
|
||||
done < <(tr ',' '\n' <<< "$PHASE_MATRIX")
|
||||
|
||||
{
|
||||
echo "## Mixed-version rollout matrix artifacts"
|
||||
echo ""
|
||||
echo "- matrix: ${matrix_csv}"
|
||||
echo "- commands: ${run_script}"
|
||||
echo "- checklist: ${checklist}"
|
||||
} > "$command_manifest"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
if [[ -z "$ENDPOINT" ]]; then
|
||||
echo "ERROR: --endpoint is required" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-mixed-rollout/$(date +%Y%m%dT%H%M%S)"
|
||||
fi
|
||||
mkdir -p "$OUT_DIR"
|
||||
require_cmd awk
|
||||
run_rows
|
||||
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "[DRY-RUN] generated files:"
|
||||
fi
|
||||
echo "Generated:"
|
||||
echo " - ${OUT_DIR}/mixed_rollout_matrix.csv"
|
||||
echo " - ${OUT_DIR}/run_phase_commands.sh"
|
||||
echo " - ${OUT_DIR}/mixed_rollout_checklist.md"
|
||||
echo " - ${OUT_DIR}/run_phase_notes.txt"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Executable
+262
@@ -0,0 +1,262 @@
|
||||
#!/usr/bin/env bash
|
||||
# Copyright 2026 RustFS Team
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# software distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||
|
||||
ENDPOINT=""
|
||||
ADMIN_TOKEN=""
|
||||
OUT_DIR=""
|
||||
WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy"
|
||||
SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60"
|
||||
WORKLOAD_SIZES="4KiB,1MiB,16MiB"
|
||||
RUN_ADMIN_CHECKS=true
|
||||
COST_BUDGET="$((24*1024))"
|
||||
DRY_RUN=false
|
||||
|
||||
usage() {
|
||||
cat <<'USAGE'
|
||||
Usage:
|
||||
scripts/manual_transition_soak_matrix.sh --endpoint <admin-api> [options]
|
||||
|
||||
Required:
|
||||
--endpoint Admin API base
|
||||
|
||||
Optional:
|
||||
--admin-token Bearer token for admin calls
|
||||
--window-spec CSV of <name:duration_min:concurrency:ops_per_minute:mix_name>
|
||||
--workload-sizes Comma-separated object size set
|
||||
--soak-ratios Comma-separated mix spec: label:read_pct:write_pct
|
||||
--cost-budget Max budget units for each run, default 24576
|
||||
--out-dir Output dir
|
||||
--no-admin-checks Skip admin check command blocks
|
||||
--dry-run
|
||||
--help
|
||||
USAGE
|
||||
}
|
||||
|
||||
arg_value() {
|
||||
local flag="$1"
|
||||
local value="${2:-}"
|
||||
if [[ -z "$value" || "$value" == --* ]]; then
|
||||
echo "ERROR: missing value for $flag" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s' "$value"
|
||||
}
|
||||
|
||||
require_cmd() {
|
||||
local cmd="$1"
|
||||
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||
echo "ERROR: command not found: $cmd" >&2
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
parse_args() {
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--window-spec) WINDOW_SPEC="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--workload-sizes) WORKLOAD_SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--soak-ratios) SOAK_RATIOS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--cost-budget) COST_BUDGET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||
--no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;;
|
||||
--dry-run) DRY_RUN=true; shift ;;
|
||||
-h|--help) usage; exit 0 ;;
|
||||
*)
|
||||
echo "ERROR: unknown arg: $1" >&2
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
}
|
||||
|
||||
trim() {
|
||||
echo "$1" | awk '{$1=$1;print}'
|
||||
}
|
||||
|
||||
parse_window() {
|
||||
local spec="$1"
|
||||
local name duration_min concurrency ops_per_min mix_name
|
||||
name="$(echo "$spec" | cut -d':' -f1)"
|
||||
duration_min="$(echo "$spec" | cut -d':' -f2)"
|
||||
concurrency="$(echo "$spec" | cut -d':' -f3)"
|
||||
ops_per_min="$(echo "$spec" | cut -d':' -f4)"
|
||||
mix_name="$(echo "$spec" | cut -d':' -f5)"
|
||||
name="$(trim "$name")"
|
||||
duration_min="$(trim "$duration_min")"
|
||||
concurrency="$(trim "$concurrency")"
|
||||
ops_per_min="$(trim "$ops_per_min")"
|
||||
mix_name="$(trim "$mix_name")"
|
||||
if [[ -z "$name" || -z "$duration_min" || -z "$concurrency" || -z "$ops_per_min" || -z "$mix_name" ]]; then
|
||||
echo "ERROR: invalid window spec: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$duration_min" =~ ^[0-9]+$ && "$concurrency" =~ ^[0-9]+$ && "$ops_per_min" =~ ^[0-9]+$ ]]; then
|
||||
echo "ERROR: window duration/concurrency/ops must be integers: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$name|$duration_min|$concurrency|$ops_per_min|$mix_name"
|
||||
}
|
||||
|
||||
parse_ratio() {
|
||||
local spec="$1"
|
||||
local name read_pct write_pct
|
||||
name="$(echo "$spec" | cut -d':' -f1)"
|
||||
read_pct="$(echo "$spec" | cut -d':' -f2)"
|
||||
write_pct="$(echo "$spec" | cut -d':' -f3)"
|
||||
name="$(trim "$name")"
|
||||
read_pct="$(trim "$read_pct")"
|
||||
write_pct="$(trim "$write_pct")"
|
||||
if [[ -z "$name" || -z "$read_pct" || -z "$write_pct" ]]; then
|
||||
echo "ERROR: invalid mix ratio: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
if ! [[ "$read_pct" =~ ^[0-9]+$ && "$write_pct" =~ ^[0-9]+$ ]]; then
|
||||
echo "ERROR: mix percentages must be integers: $spec" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "$name|$read_pct|$write_pct"
|
||||
}
|
||||
|
||||
resolve_mix() {
|
||||
local name="$1"
|
||||
local line
|
||||
line="$(awk -v target="$name" -F: '{
|
||||
if ($1 == target) { print $0; exit }
|
||||
}' < <(tr ',' '\n' <<< "$SOAK_RATIOS"))"
|
||||
if [[ -z "$line" ]]; then
|
||||
return 1
|
||||
fi
|
||||
parse_ratio "$line"
|
||||
}
|
||||
|
||||
run_checks() {
|
||||
local tag="$1"
|
||||
local job_id="$2"
|
||||
local token_hdr=""
|
||||
if [[ -z "$job_id" ]]; then
|
||||
job_id="<JOB_ID_${tag}>"
|
||||
fi
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
token_hdr="-H \"Authorization: Bearer \${ADMIN_TOKEN}\" "
|
||||
fi
|
||||
if [[ "$RUN_ADMIN_CHECKS" != true ]]; then
|
||||
return
|
||||
fi
|
||||
echo "# admin checks for ${tag}"
|
||||
echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}\" | jq '.status, .report, .queue_snapshot, .failure_reason'"
|
||||
echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/metrics?types=1&n=1\" | jq '.aggregated.scanner.lifecycle_transition'"
|
||||
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id"
|
||||
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||
printf " --admin-token \"\${ADMIN_TOKEN}\""
|
||||
fi
|
||||
printf '\n'
|
||||
}
|
||||
|
||||
write_matrix_files() {
|
||||
local matrix_csv="${OUT_DIR}/nightly_soak_matrix.csv"
|
||||
local run_matrix="${OUT_DIR}/run_soak_matrix.sh"
|
||||
local notes="${OUT_DIR}/soak_notes.md"
|
||||
local run_id=1
|
||||
|
||||
{
|
||||
echo "window,window_duration_min,concurrency,ops_per_min,mix_name,read_pct,write_pct,size,expected_ops,expected_run_id,run_check"
|
||||
} > "$matrix_csv"
|
||||
|
||||
echo "#!/usr/bin/env bash" > "$run_matrix"
|
||||
echo "set -euo pipefail" >> "$run_matrix"
|
||||
echo "" >> "$run_matrix"
|
||||
|
||||
{
|
||||
echo "# Nightly/night soak matrix for manual transition stress"
|
||||
echo "- endpoint: ${ENDPOINT}"
|
||||
echo "- cost-budget: ${COST_BUDGET}"
|
||||
echo ""
|
||||
echo "Recommended gate for each row: no unknown failure_reason spikes, job result drift is 0, and queued/task/result counts reconcile."
|
||||
echo ""
|
||||
} > "$notes"
|
||||
|
||||
IFS=',' read -r -a sizes <<< "$WORKLOAD_SIZES"
|
||||
while IFS=',' read -r window_spec; do
|
||||
[[ -z "$window_spec" ]] && continue
|
||||
parsed="$(parse_window "$window_spec")"
|
||||
IFS='|' read -r window_name duration_min concurrency ops_per_min mix_name <<< "$parsed"
|
||||
|
||||
mix="$(resolve_mix "$mix_name" || true)"
|
||||
if [[ -z "$mix" ]]; then
|
||||
echo "ERROR: unknown mix name in window spec: ${window_name} -> ${mix_name}" >&2
|
||||
exit 1
|
||||
fi
|
||||
IFS='|' read -r resolved_mix read_pct write_pct <<< "$mix"
|
||||
|
||||
for size in "${sizes[@]}"; do
|
||||
size="$(trim "$size")"
|
||||
[[ -z "$size" ]] && continue
|
||||
expected_ops=$(( duration_min * ops_per_min ))
|
||||
if (( expected_ops > COST_BUDGET )); then
|
||||
budget_status="over-budget"
|
||||
else
|
||||
budget_status="within-budget"
|
||||
fi
|
||||
echo "${window_name},${duration_min},${concurrency},${ops_per_min},${resolved_mix},${read_pct},${write_pct},${size},${expected_ops},${run_id},${budget_status}" >> "$matrix_csv"
|
||||
{
|
||||
echo "# window=${window_name} size=${size} mix=${resolved_mix} concurrency=${concurrency} duration=${duration_min}m ops_per_min=${ops_per_min}"
|
||||
echo "job_id=\"<JOB_ID_${window_name}_${size}_${concurrency}_${run_id}>\""
|
||||
echo "read_pct=${read_pct}"
|
||||
echo "write_pct=${write_pct}"
|
||||
echo "expected_ops=${expected_ops}"
|
||||
echo "budget_status=${budget_status}"
|
||||
run_checks "${window_name}_${size}_${concurrency}_${run_id}" "<JOB_ID_${window_name}_${size}_${concurrency}_${run_id}>"
|
||||
echo
|
||||
} >> "$run_matrix"
|
||||
run_id=$((run_id + 1))
|
||||
done
|
||||
done < <(tr ',' '\n' <<< "$WINDOW_SPEC")
|
||||
|
||||
{
|
||||
echo "# run target:"
|
||||
echo "bash ${run_matrix}"
|
||||
} >> "$notes"
|
||||
}
|
||||
|
||||
main() {
|
||||
parse_args "$@"
|
||||
if [[ -z "$ENDPOINT" ]]; then
|
||||
echo "ERROR: --endpoint is required" >&2
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
if [[ -z "$OUT_DIR" ]]; then
|
||||
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-nightly-soak/$(date +%Y%m%dT%H%M%S)"
|
||||
fi
|
||||
mkdir -p "$OUT_DIR"
|
||||
require_cmd awk
|
||||
write_matrix_files
|
||||
if [[ "$DRY_RUN" == true ]]; then
|
||||
echo "[DRY-RUN] generated matrix files:"
|
||||
fi
|
||||
echo "Generated:"
|
||||
echo " - ${OUT_DIR}/nightly_soak_matrix.csv"
|
||||
echo " - ${OUT_DIR}/run_soak_matrix.sh"
|
||||
echo " - ${OUT_DIR}/soak_notes.md"
|
||||
}
|
||||
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user