diff --git a/Cargo.lock b/Cargo.lock index 05cdac78b..253b51d35 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -56,9 +56,9 @@ dependencies = [ [[package]] name = "aes" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138" +checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58" dependencies = [ "cipher 0.5.2", "cpubits", @@ -73,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028" dependencies = [ "aead", - "aes 0.9.1", + "aes 0.9.2", "cipher 0.5.2", "ctr", "ghash", @@ -1528,7 +1528,7 @@ version = "0.10.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.9", ] [[package]] @@ -1547,7 +1547,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.9", ] [[package]] @@ -1870,7 +1870,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common 0.1.7", + "crypto-common 0.1.6", "inout 0.1.4", ] @@ -2328,7 +2328,7 @@ version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.9", "rand_core 0.6.4", "subtle", "zeroize", @@ -2353,11 +2353,11 @@ dependencies = [ [[package]] name = "crypto-common" -version = "0.1.7" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.9", "typenum", ] @@ -3554,7 +3554,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", "const-oid 0.9.6", - "crypto-common 0.1.7", + "crypto-common 0.1.6", "subtle", ] @@ -3811,7 +3811,7 @@ dependencies = [ "crypto-bigint 0.5.5", "digest 0.10.7", "ff 0.13.1", - "generic-array 0.14.7", + "generic-array 0.14.9", "group 0.13.0", "hkdf 0.12.4", "pem-rfc7468 0.7.0", @@ -4257,9 +4257,9 @@ dependencies = [ [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -4272,7 +4272,7 @@ version = "1.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b" dependencies = [ - "generic-array 0.14.7", + "generic-array 0.14.9", "rustversion", "typenum", ] @@ -5272,7 +5272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" dependencies = [ "block-padding 0.3.3", - "generic-array 0.14.7", + "generic-array 0.14.9", ] [[package]] @@ -5745,9 +5745,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" [[package]] name = "libflate" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df" +checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c" dependencies = [ "adler32", "crc32fast", @@ -7485,7 +7485,7 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca" dependencies = [ - "aes 0.9.1", + "aes 0.9.2", "aes-gcm", "cbc 0.2.1", "der 0.8.1", @@ -8681,7 +8681,7 @@ version = "0.62.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f" dependencies = [ - "aes 0.9.1", + "aes 0.9.2", "aws-lc-rs", "bitflags 2.13.1", "block-padding 0.4.2", @@ -10503,7 +10503,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" dependencies = [ "base16ct 0.2.0", "der 0.7.10", - "generic-array 0.14.7", + "generic-array 0.14.9", "pkcs8 0.10.2", "subtle", "zeroize", @@ -11123,7 +11123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597" dependencies = [ "aead", - "aes 0.9.1", + "aes 0.9.2", "aes-gcm", "chacha20", "cipher 0.5.2", @@ -13068,7 +13068,7 @@ version = "8.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b" dependencies = [ - "aes 0.9.1", + "aes 0.9.2", "bzip2", "constant_time_eq", "crc32fast", diff --git a/Cargo.toml b/Cargo.toml index 774ec8b01..93306bca1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -68,7 +68,7 @@ resolver = "3" edition = "2024" license = "Apache-2.0" repository = "https://github.com/rustfs/rustfs" -rust-version = "1.96.0" +rust-version = "1.97.1" version = "1.0.0-beta.11" homepage = "https://rustfs.com" description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. " @@ -211,7 +211,7 @@ zeroize = { version = "1.9.0" } # Time and Date chrono = { version = "0.4.45" } humantime = "2.4.0" -jiff = { version = "0.2.34" } +jiff = { version = "0.2.35" } time = { version = "0.3.54" } # Database @@ -243,6 +243,7 @@ crossbeam-channel = "0.5.16" crossbeam-deque = "0.8.7" crossbeam-utils = "0.8.22" datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" } +#datafusion = { default-features = false, version = "54.1.0" } derive_builder = "0.20.2" enumset = "1.1.14" faster-hex = "0.10.0" diff --git a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs index f98b8064c..220052584 100644 --- a/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs +++ b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs @@ -23,13 +23,13 @@ use crate::bucket::lifecycle::lifecycle::{ }; use crate::bucket::lifecycle::manual_transition_job::{ MANUAL_TRANSITION_JOB_RECORD_PREFIX, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission, - ManualTransitionScopeAdmissionClaim, ManualTransitionTaskRecord, ManualTransitionWorkerResult, - claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current, + ManualTransitionScopeAdmissionClaim, ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason, + ManualTransitionWorkerResult, claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current, load_manual_transition_job_record, load_manual_transition_job_record_with_etag, manual_transition_job_id_from_record_object_name, manual_transition_job_lease_expired, manual_transition_worker_result_task_key, persist_manual_transition_job_progress, reconcile_manual_transition_worker_results, - record_manual_transition_worker_result, renew_manual_transition_job_lease, save_manual_transition_job_record_if_current, - save_manual_transition_task_if_absent, + record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason, + renew_manual_transition_job_lease, save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent, }; use crate::bucket::lifecycle::replication_sink; use crate::bucket::lifecycle::replication_sink::{ @@ -49,7 +49,9 @@ use crate::disk::error::DiskError; use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE}; use crate::error::Error; use crate::error::StorageError; -use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down}; +use crate::error::{ + error_resp_to_object_err, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down, +}; use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions}; use crate::services::tier::{ tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata}, @@ -94,7 +96,7 @@ use s3s::dto::{ use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION}; use sha2::{Digest, Sha256}; use std::any::Any; -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use std::env; use std::future::Future; use std::pin::Pin; @@ -1658,11 +1660,12 @@ impl TransitionState { .await { if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) { - record_manual_transition_worker_result_for_task( + record_manual_transition_worker_result_for_task_with_reason( api.clone(), job_id, result_key, ManualTransitionWorkerResult::TierFailure, + manual_transition_worker_failure_reason(&err), ) .await; } @@ -1818,6 +1821,78 @@ async fn record_manual_transition_worker_result_for_task( } } +async fn record_manual_transition_worker_result_for_task_with_reason( + api: Arc, + job_id: Uuid, + result_key: &str, + result: ManualTransitionWorkerResult, + reason: ManualTransitionWorkerFailureReason, +) { + if let Err(err) = record_manual_transition_worker_result_with_reason( + api, + job_id, + result_key, + result, + manual_transition_queue_snapshot(), + Some(reason), + ) + .await + { + warn!( + event = EVENT_LIFECYCLE_WORKER_STATE, + component = LOG_COMPONENT_ECSTORE, + subsystem = LOG_SUBSYSTEM_LIFECYCLE, + job_id = %job_id, + error = %err, + reason = ?reason, + state = "manual_transition_worker_result_failed", + "Manual transition worker failed to persist job result" + ); + } +} + +fn manual_transition_worker_failure_reason(err: &Error) -> ManualTransitionWorkerFailureReason { + if is_err_object_not_found(err) || is_err_version_not_found(err) { + return ManualTransitionWorkerFailureReason::NotFound; + } + if is_err_permission_denied(err) { + return ManualTransitionWorkerFailureReason::PermissionDenied; + } + if is_err_read_quorum(err) { + return ManualTransitionWorkerFailureReason::Quorum; + } + if is_timeout(err) { + return ManualTransitionWorkerFailureReason::Timeout; + } + if is_slow_down(err) { + return ManualTransitionWorkerFailureReason::SlowDown; + } + if is_network_or_host_down(&err.to_string(), false) { + return ManualTransitionWorkerFailureReason::Network; + } + ManualTransitionWorkerFailureReason::Unknown +} + +fn is_err_permission_denied(err: &Error) -> bool { + match err { + Error::VolumeAccessDenied | Error::FileAccessDenied | Error::PrefixAccessDenied(_, _) | Error::DiskAccessDenied => true, + Error::Io(io) => io.kind() == std::io::ErrorKind::PermissionDenied, + _ => false, + } +} + +fn is_timeout(err: &Error) -> bool { + match err { + Error::Timeout => true, + Error::Io(io) => io.kind() == std::io::ErrorKind::TimedOut, + _ => false, + } +} + +fn is_slow_down(err: &Error) -> bool { + matches!(err, Error::SlowDown) +} + pub async fn init_background_expiry(api: Arc) { let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16)); //globalILMConfig.getExpirationWorkers() @@ -3233,6 +3308,8 @@ pub struct ManualTransitionRunReport { #[serde(default, skip_serializing_if = "is_zero_u64")] pub transition_failed: u64, pub tier_failure: u64, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub tier_failure_by_reason: BTreeMap, pub truncated_by_limit: bool, pub truncated_by_duration: bool, pub cancelled: bool, @@ -3303,12 +3380,18 @@ impl ManualTransitionRunReport { } pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) { + let mut tier_failure_by_reason = self.tier_failure_by_reason.clone(); + for (reason, count) in &scan_report.tier_failure_by_reason { + let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default(); + tier_failure_by_reason.insert(*reason, current.max(*count)); + } let transition_completed = self.transition_completed; let transition_failed = self.transition_failed; *self = scan_report.clone(); self.transition_completed = transition_completed; self.transition_failed = transition_failed; self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed); + self.tier_failure_by_reason = tier_failure_by_reason; } pub fn worker_transition_pending(&self) -> bool { @@ -4658,12 +4741,13 @@ mod tests { lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action, lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets, manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink, - manual_transition_version_marker, mark_delete_opts_skip_decommissioned_on_remote_success, - merge_stale_multipart_candidate, persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, - recover_manual_transition_job, recover_manual_transition_jobs, replication_state_for_delete, - resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, - resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, - select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer, + manual_transition_version_marker, manual_transition_worker_failure_reason, + mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate, + persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job, + recover_manual_transition_jobs, replication_state_for_delete, resolve_tier_free_version_recovery_enabled, + resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count, + resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location, + set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer, should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state, transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery, }; @@ -4676,16 +4760,17 @@ mod tests { use crate::bucket::lifecycle::config_boundary; use crate::bucket::lifecycle::manual_transition_job::{ ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim, - ManualTransitionTaskRecord, ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord, - claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current, - legacy_manual_transition_scope_key, load_manual_transition_job_record, load_manual_transition_scope_admission, + ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason, ManualTransitionWorkerResult, + ManualTransitionWorkerResultRecord, claim_manual_transition_scope_admission, + delete_manual_transition_scope_admission_if_current, legacy_manual_transition_scope_key, + load_manual_transition_job_record, load_manual_transition_scope_admission, load_manual_transition_scope_admission_with_etag, load_manual_transition_task_record, manual_transition_scope_record_object_name, manual_transition_worker_result_object_name, manual_transition_worker_result_task_key, reconcile_manual_transition_worker_results, - record_manual_transition_worker_result, renew_manual_transition_job_lease, request_manual_transition_job_cancel, - save_manual_transition_job_record, save_manual_transition_scope_admission_if_absent, - save_manual_transition_scope_admission_if_current, save_manual_transition_task_if_absent, - save_manual_transition_worker_result_if_absent, + record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason, + renew_manual_transition_job_lease, request_manual_transition_job_cancel, save_manual_transition_job_record, + save_manual_transition_scope_admission_if_absent, save_manual_transition_scope_admission_if_current, + save_manual_transition_task_if_absent, save_manual_transition_worker_result_if_absent, }; use crate::bucket::lifecycle::replication_sink::{ ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, VersionPurgeStatusType, @@ -7539,6 +7624,37 @@ mod tests { assert!(record.completed_at_unix_nanos.is_some()); } + #[test] + fn manual_transition_worker_failure_reason_classifies_known_failures() { + assert_eq!( + manual_transition_worker_failure_reason(&Error::ObjectNotFound("bucket".to_string(), "obj".to_string())), + ManualTransitionWorkerFailureReason::NotFound + ); + assert_eq!( + manual_transition_worker_failure_reason(&Error::Io(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "file access denied", + ))), + ManualTransitionWorkerFailureReason::PermissionDenied + ); + assert_eq!( + manual_transition_worker_failure_reason(&Error::ErasureReadQuorum), + ManualTransitionWorkerFailureReason::Quorum + ); + assert_eq!( + manual_transition_worker_failure_reason(&Error::Timeout), + ManualTransitionWorkerFailureReason::Timeout + ); + assert_eq!( + manual_transition_worker_failure_reason(&Error::SlowDown), + ManualTransitionWorkerFailureReason::SlowDown + ); + assert_eq!( + manual_transition_worker_failure_reason(&Error::MethodNotAllowed), + ManualTransitionWorkerFailureReason::Unknown + ); + } + #[tokio::test] async fn manual_transition_counts_already_transitioned_object() { let lc = latest_transition_lifecycle(); @@ -8471,6 +8587,49 @@ mod tests { ); } + #[tokio::test] + #[serial] + async fn manual_transition_worker_result_stores_failure_reason() { + let (_paths, ecstore) = setup_test_env().await; + let job_id = Uuid::new_v4(); + let bucket = format!("manual-worker-failure-reason-{}", job_id.simple()); + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(job_id, &bucket, &options, "worker-owner"); + record.complete( + ManualTransitionRunReport { + bucket: bucket.clone(), + enqueued: 1, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + save_manual_transition_job_record(ecstore.clone(), &record) + .await + .expect("worker result job record should save"); + + let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None); + let final_record = record_manual_transition_worker_result_with_reason( + ecstore.clone(), + job_id, + &task_key, + ManualTransitionWorkerResult::TierFailure, + ManualTransitionQueueSnapshot::default(), + Some(ManualTransitionWorkerFailureReason::Network), + ) + .await + .expect("worker result with failure reason should persist"); + + assert_eq!( + final_record + .report + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::Network), + Some(&1) + ); + assert_eq!(final_record.report.transition_failed, 1); + assert_eq!(final_record.report.tier_failure, 1); + } + #[tokio::test] #[serial] async fn manual_transition_worker_result_reconcile_applies_marker_and_releases_admission() { diff --git a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs index 68ce95cad..7042d2224 100644 --- a/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs +++ b/crates/ecstore/src/bucket/lifecycle/manual_transition_job.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::BTreeMap; use std::sync::Arc; use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum}; @@ -194,6 +195,15 @@ impl ManualTransitionJobRecord { } pub fn record_worker_result(&mut self, result: ManualTransitionWorkerResult, queue_snapshot: ManualTransitionQueueSnapshot) { + self.record_worker_result_with_reason(result, queue_snapshot, None); + } + + pub fn record_worker_result_with_reason( + &mut self, + result: ManualTransitionWorkerResult, + queue_snapshot: ManualTransitionQueueSnapshot, + failure_reason: Option, + ) { if self.is_terminal() { return; } @@ -204,6 +214,8 @@ impl ManualTransitionJobRecord { ManualTransitionWorkerResult::TierFailure => { self.report.transition_failed = self.report.transition_failed.saturating_add(1); self.report.tier_failure = self.report.tier_failure.saturating_add(1); + let reason = failure_reason.unwrap_or(ManualTransitionWorkerFailureReason::Unknown); + *self.report.tier_failure_by_reason.entry(reason).or_insert(0) += 1; } } self.queue_snapshot = queue_snapshot; @@ -215,6 +227,7 @@ impl ManualTransitionJobRecord { &mut self, completed: u64, failed: u64, + failure_reasons: &BTreeMap, task_queued: u64, queue_snapshot: ManualTransitionQueueSnapshot, ) -> bool { @@ -237,11 +250,17 @@ impl ManualTransitionJobRecord { { return false; } + let mut scan_tier_failure_by_reason = self.report.tier_failure_by_reason.clone(); + for (reason, count) in failure_reasons { + let current = scan_tier_failure_by_reason.get(reason).copied().unwrap_or_default(); + scan_tier_failure_by_reason.insert(*reason, current.max(*count)); + } let scan_tier_failure = self.report.tier_failure.saturating_sub(self.report.transition_failed); self.report.enqueued = enqueued; self.report.transition_completed = transition_completed; self.report.transition_failed = transition_failed; self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed); + self.report.tier_failure_by_reason = scan_tier_failure_by_reason; self.queue_snapshot = queue_snapshot; self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos(); self.mark_terminal_if_worker_drained(); @@ -401,9 +420,24 @@ impl ManualTransitionJobRecord { return Err(ManualTransitionJobError::Corrupt("content checksum is not a sha256 checksum")); } let mut job = persisted.job; - let job_bytes = serde_json::to_vec(&job)?; + let mut job_bytes = serde_json::to_vec(&job)?; let actual_checksum = hex_sha256(&job_bytes, ToOwned::to_owned); - if persisted.content_sha256 != actual_checksum { + let checksum_match = if persisted.content_sha256 == actual_checksum { + true + } else { + let mut job_value = serde_json::to_value(&job)?; + if let Some(job_obj) = job_value.as_object_mut() { + if let Some(report) = job_obj.get_mut("report").and_then(|value| value.as_object_mut()) { + report.remove("tier_failure_by_reason"); + } + } else { + return Err(ManualTransitionJobError::ChecksumMismatch); + } + job_bytes = serde_json::to_vec(&job_value)?; + let fallback_checksum = hex_sha256(&job_bytes, ToOwned::to_owned); + persisted.content_sha256 == fallback_checksum + }; + if !checksum_match { return Err(ManualTransitionJobError::ChecksumMismatch); } if job.job_id != expected_job_id { @@ -449,6 +483,18 @@ pub enum ManualTransitionWorkerResult { TierFailure, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ManualTransitionWorkerFailureReason { + Unknown, + NotFound, + Network, + PermissionDenied, + Timeout, + Quorum, + SlowDown, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct ManualTransitionTaskRecord { @@ -554,17 +600,22 @@ struct PersistedManualTransitionTaskRecord { record: ManualTransitionTaskRecord, } -#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ManualTransitionWorkerResultStats { pub completed: u64, pub failed: u64, + pub tier_failure_by_reason: BTreeMap, } impl ManualTransitionWorkerResultStats { - fn record(&mut self, result: ManualTransitionWorkerResult) { + fn record(&mut self, result: ManualTransitionWorkerResult, failure_reason: Option) { match result { ManualTransitionWorkerResult::Completed => self.completed = self.completed.saturating_add(1), - ManualTransitionWorkerResult::TierFailure => self.failed = self.failed.saturating_add(1), + ManualTransitionWorkerResult::TierFailure => { + self.failed = self.failed.saturating_add(1); + let reason = failure_reason.unwrap_or(ManualTransitionWorkerFailureReason::Unknown); + *self.tier_failure_by_reason.entry(reason).or_insert(0) += 1; + } } } } @@ -576,16 +627,28 @@ pub struct ManualTransitionWorkerResultRecord { pub job_id: Uuid, pub task_key: String, pub result: ManualTransitionWorkerResult, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub failure_reason: Option, pub completed_at_unix_nanos: i128, } impl ManualTransitionWorkerResultRecord { pub fn new(job_id: Uuid, task_key: impl Into, result: ManualTransitionWorkerResult) -> Self { + Self::new_with_reason(job_id, task_key, result, None) + } + + pub fn new_with_reason( + job_id: Uuid, + task_key: impl Into, + result: ManualTransitionWorkerResult, + failure_reason: Option, + ) -> Self { Self { schema: MANUAL_TRANSITION_WORKER_RESULT_SCHEMA.to_string(), job_id, task_key: task_key.into(), result, + failure_reason, completed_at_unix_nanos: OffsetDateTime::now_utc().unix_timestamp_nanos(), } } @@ -620,9 +683,23 @@ impl ManualTransitionWorkerResultRecord { )); } let record = persisted.record; - let record_bytes = serde_json::to_vec(&record)?; + let mut record_bytes = serde_json::to_vec(&record)?; let actual_checksum = hex_sha256(&record_bytes, ToOwned::to_owned); - if persisted.content_sha256 != actual_checksum { + let checksum_match = if persisted.content_sha256 == actual_checksum { + true + } else { + if let Some(record_value) = serde_json::to_value(&record)?.as_object_mut() { + if record.failure_reason.is_none() { + record_value.remove("failure_reason"); + } + record_bytes = serde_json::to_vec(record_value)?; + } else { + return Err(ManualTransitionJobError::ChecksumMismatch); + } + let fallback_checksum = hex_sha256(&record_bytes, ToOwned::to_owned); + persisted.content_sha256 == fallback_checksum + }; + if !checksum_match { return Err(ManualTransitionJobError::ChecksumMismatch); } if record.job_id != expected_job_id { @@ -1120,7 +1197,7 @@ async fn scan_manual_transition_worker_result_journal( Ok(result) => result, Err(err) => return Ok(ManualTransitionWorkerResultJournal::Corrupt(err.to_string())), }; - stats.record(result.result); + stats.record(result.result, result.failure_reason); } if !page.is_truncated { return Ok(ManualTransitionWorkerResultJournal::Stats(stats)); @@ -1151,7 +1228,13 @@ pub async fn reconcile_manual_transition_worker_results( }; for _ in 0..4 { let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?; - let changed = record.apply_worker_result_counts(stats.completed, stats.failed, task_stats.queued, queue_snapshot); + let changed = record.apply_worker_result_counts( + stats.completed, + stats.failed, + &stats.tier_failure_by_reason, + task_stats.queued, + queue_snapshot, + ); if !changed { return Ok(record); } @@ -1446,7 +1529,18 @@ pub async fn record_manual_transition_worker_result( result: ManualTransitionWorkerResult, queue_snapshot: ManualTransitionQueueSnapshot, ) -> EcstoreResult { - let result_record = ManualTransitionWorkerResultRecord::new(job_id, task_key, result); + record_manual_transition_worker_result_with_reason(api, job_id, task_key, result, queue_snapshot, None).await +} + +pub async fn record_manual_transition_worker_result_with_reason( + api: Arc, + job_id: Uuid, + task_key: &str, + result: ManualTransitionWorkerResult, + queue_snapshot: ManualTransitionQueueSnapshot, + failure_reason: Option, +) -> EcstoreResult { + let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason); if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? { return load_manual_transition_job_record(api, job_id).await; } @@ -1456,7 +1550,7 @@ pub async fn record_manual_transition_worker_result( if record.is_terminal() { return Ok(record); } - record.record_worker_result(result, queue_snapshot); + record.record_worker_result_with_reason(result, queue_snapshot, failure_reason); match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await { Ok(()) => { if record.is_terminal() { @@ -1652,6 +1746,52 @@ mod tests { assert_eq!(record.report.tier_failure, 1); } + #[test] + fn manual_transition_job_record_records_worker_failure_reasons() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + + record.record_worker_result_with_reason( + ManualTransitionWorkerResult::TierFailure, + ManualTransitionQueueSnapshot::default(), + Some(ManualTransitionWorkerFailureReason::NotFound), + ); + record.record_worker_result_with_reason( + ManualTransitionWorkerResult::TierFailure, + ManualTransitionQueueSnapshot::default(), + Some(ManualTransitionWorkerFailureReason::Network), + ); + record.record_worker_result_with_reason( + ManualTransitionWorkerResult::TierFailure, + ManualTransitionQueueSnapshot::default(), + None, + ); + + assert_eq!(record.report.transition_failed, 3); + assert_eq!(record.report.tier_failure, 3); + assert_eq!( + record + .report + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::NotFound), + Some(&1) + ); + assert_eq!( + record + .report + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::Network), + Some(&1) + ); + assert_eq!( + record + .report + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::Unknown), + Some(&1) + ); + } + #[test] fn manual_transition_job_cancel_recovery_finishes_after_worker_drain() { let options = ManualTransitionRunOptions::default(); @@ -1709,6 +1849,67 @@ mod tests { assert_eq!(record.report.tier_failure, 3); } + #[test] + fn manual_transition_job_scan_progress_preserves_worker_failure_reasons() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + record.record_worker_result_with_reason( + ManualTransitionWorkerResult::TierFailure, + ManualTransitionQueueSnapshot::default(), + Some(ManualTransitionWorkerFailureReason::NotFound), + ); + + record.update_running_progress( + ManualTransitionRunReport { + bucket: "bucket".to_string(), + scanned: 3, + enqueued: 1, + tier_failure: 2, + ..Default::default() + }, + ManualTransitionQueueSnapshot::default(), + ); + + assert_eq!( + record + .report + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::NotFound), + Some(&1) + ); + } + + #[test] + fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() { + let options = ManualTransitionRunOptions::default(); + let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + record.record_worker_result_with_reason( + ManualTransitionWorkerResult::TierFailure, + ManualTransitionQueueSnapshot::default(), + Some(ManualTransitionWorkerFailureReason::NotFound), + ); + + let mut failure_reasons = BTreeMap::new(); + failure_reasons.insert(ManualTransitionWorkerFailureReason::PermissionDenied, 2); + + record.apply_worker_result_counts(0, 1, &failure_reasons, 1, ManualTransitionQueueSnapshot::default()); + + assert_eq!( + record + .report + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::NotFound), + Some(&1) + ); + assert_eq!( + record + .report + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::PermissionDenied), + Some(&2) + ); + } + #[test] fn manual_transition_job_unknown_checkpoint_persists_counters_and_cursor() { let options = ManualTransitionRunOptions { @@ -1955,6 +2156,62 @@ mod tests { assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch)); } + #[test] + fn manual_transition_worker_result_record_round_trips_without_failure_reason() { + let job_id = Uuid::new_v4(); + let task_key = manual_transition_worker_result_task_key("bucket", "logs/a", None); + let record = ManualTransitionWorkerResultRecord::new_with_reason( + job_id, + &task_key, + ManualTransitionWorkerResult::TierFailure, + Some(ManualTransitionWorkerFailureReason::Network), + ); + let encoded = record.encode().expect("worker result record should encode"); + let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded worker result should be json"); + let record_obj = value["record"].as_object_mut().expect("record object should be present"); + record_obj.remove("failure_reason"); + let record_bytes = serde_json::to_vec(&value["record"]).expect("record without reason should encode"); + value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned)); + let stripped = serde_json::to_vec(&value).expect("stripped worker result should encode"); + + let decoded = ManualTransitionWorkerResultRecord::decode(job_id, &task_key, &stripped) + .expect("worker result without reason should decode"); + + assert_eq!(decoded.failure_reason, None); + assert_eq!(decoded.result, ManualTransitionWorkerResult::TierFailure); + } + + #[test] + fn manual_transition_worker_result_stats_aggregates_reasons() { + let mut stats = ManualTransitionWorkerResultStats::default(); + + stats.record(ManualTransitionWorkerResult::Completed, None); + stats.record( + ManualTransitionWorkerResult::TierFailure, + Some(ManualTransitionWorkerFailureReason::PermissionDenied), + ); + stats.record( + ManualTransitionWorkerResult::TierFailure, + Some(ManualTransitionWorkerFailureReason::PermissionDenied), + ); + stats.record(ManualTransitionWorkerResult::TierFailure, None); + + assert_eq!(stats.completed, 1); + assert_eq!(stats.failed, 3); + assert_eq!( + stats + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::PermissionDenied), + Some(&2) + ); + assert_eq!( + stats + .tier_failure_by_reason + .get(&ManualTransitionWorkerFailureReason::Unknown), + Some(&1) + ); + } + #[test] fn manual_transition_task_record_rejects_checksum_drift() { let job_id = Uuid::new_v4(); @@ -1971,6 +2228,24 @@ mod tests { assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch)); } + #[test] + fn manual_transition_job_record_round_trips_legacy_report_without_failure_reason_map() { + let options = ManualTransitionRunOptions::default(); + let record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER); + let encoded = record.encode().expect("job record should encode"); + let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json"); + let report = value["job"]["report"].as_object_mut().expect("report should be object"); + report.remove("tier_failure_by_reason"); + let record_bytes = serde_json::to_vec(&value["job"]).expect("job without report failure map should encode"); + value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned)); + let legacy = serde_json::to_vec(&value).expect("job without report failure map should encode"); + + let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job report should decode"); + + assert_eq!(decoded.state, ManualTransitionJobState::Running); + assert!(decoded.report.tier_failure_by_reason.is_empty()); + } + #[test] fn manual_transition_job_record_rejects_unknown_report_fields() { let options = ManualTransitionRunOptions::default(); diff --git a/scripts/manual_transition_debug.sh b/scripts/manual_transition_debug.sh new file mode 100755 index 000000000..2468d3b48 --- /dev/null +++ b/scripts/manual_transition_debug.sh @@ -0,0 +1,253 @@ +#!/usr/bin/env bash +# +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +ENDPOINT="" +JOB_ID="" +ADMIN_TOKEN="" +LOG_GLOB="/var/log/rustfs/*.log" +METRIC_TYPES="1" +METRIC_SAMPLES="1" +METRIC_INTERVAL="" +BUCKET_FILTER="" +SHOW_HELP=false + +usage() { + cat <<'USAGE' +Usage: + scripts/manual_transition_debug.sh --endpoint --job-id [options] + +Required: + --endpoint Admin endpoint base, e.g. http://127.0.0.1:9000 + --job-id Manual transition job_id (UUID) + +Optional: + --admin-token Bearer token for /admin/v3 endpoints + --log-glob Log glob to scan, defaults to /var/log/rustfs/*.log + --metric-types Metrics types bitmask, defaults to 1 (SCANNER) + --metric-samples Metrics samples, defaults to 1 + --metric-interval Metrics interval parameter, passed as-is to query string + --bucket Optional bucket filter for focused log output + -h, --help Show this help + +Output sections: + - lifecycle_worker_state structured events (job-scoped) + - manual-transition admin job status + - lifecycle_tier_operation_failed and admin_ilm_transition_state traces + - metrics snapshot (aggregated.scanner.lifecycle_transition) +USAGE +} + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: required command not found: $cmd" >&2 + exit 1 + fi +} + +api_get() { + local path="$1" + local url="${ENDPOINT%/}${path}" + + if [[ -n "$ADMIN_TOKEN" ]]; then + curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url" + return + fi + + curl -sS "$url" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --endpoint) + ENDPOINT="${2:-}" + shift 2 + ;; + --job-id) + JOB_ID="${2:-}" + shift 2 + ;; + --admin-token) + ADMIN_TOKEN="${2:-}" + shift 2 + ;; + --log-glob) + LOG_GLOB="${2:-}" + shift 2 + ;; + --metric-types) + METRIC_TYPES="${2:-}" + shift 2 + ;; + --metric-samples) + METRIC_SAMPLES="${2:-}" + shift 2 + ;; + --metric-interval) + METRIC_INTERVAL="${2:-}" + shift 2 + ;; + --bucket) + BUCKET_FILTER="${2:-}" + shift 2 + ;; + -h|--help) + SHOW_HELP=true + shift + ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +validate() { + if [[ "$SHOW_HELP" == true ]]; then + usage + exit 0 + fi + + if [[ -z "$ENDPOINT" || -z "$JOB_ID" ]]; then + echo "ERROR: --endpoint and --job-id are required" >&2 + usage + exit 1 + fi + + if [[ -z "$LOG_GLOB" ]]; then + echo "ERROR: --log-glob must not be empty" >&2 + exit 1 + fi +} + +build_admin_metrics_path() { + local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}" + if [[ -n "$METRIC_INTERVAL" ]]; then + path="${path}&interval=${METRIC_INTERVAL}" + fi + printf '%s' "$path" +} + +run_json_stream() { + local query="$1" + local filter="$2" + local log_lines; log_lines="$(mktemp)" + local log_errors; log_errors="$(mktemp)" + local log_paths=() + local match + + while IFS= read -r match; do + [[ -n "$match" ]] && log_paths+=("$match") + done < <(compgen -G "$LOG_GLOB" || true) + if [[ "${#log_paths[@]}" -eq 0 ]]; then + log_paths=("$LOG_GLOB") + fi + + if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then + : + fi + + if [[ -s "$log_errors" ]]; then + sed 's/^/DEBUG: /' "$log_errors" + fi + + jq -R "$filter" <"$log_lines" + + rm -f "$log_lines" "$log_errors" +} + +run_worker_logs() { + local base_filter='fromjson? | select(.event == "lifecycle_worker_state")' + local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))' + local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state' + + if [[ -n "$BUCKET_FILTER" ]]; then + manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER)' + dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER) | .state' + fi + + echo "## lifecycle_worker_state (all)" + run_json_stream "lifecycle_worker_state" "$base_filter" + + echo "## lifecycle_worker_state (manual_transition + job filter: ${JOB_ID})" + JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \ + run_json_stream "lifecycle_worker_state" "$manual_filter" || true + + echo "## lifecycle_worker_state state distribution (manual_transition only)" + JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \ + run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true +} + +run_admin_job() { + echo "## manual transition job status" + api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \ + | jq '{status, mode, job_id, failure_reason, report: .report, queue_snapshot: .queue_snapshot}' +} + +run_tier_operation_failures() { + echo "## lifecycle_tier_operation_failed events" + local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}' + if [[ -n "$BUCKET_FILTER" ]]; then + filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}' + fi + + JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \ + run_json_stream "lifecycle_tier_operation_failed" "$filter" || true +} + +run_admin_events() { + echo "## admin_ilm_transition_state (job filter)" + local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}' + if [[ -n "$BUCKET_FILTER" ]]; then + filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}' + fi + + JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \ + run_json_stream "admin_ilm_transition_state" "$filter" || true +} + +run_metrics() { + echo "## metrics: scanner.lifecycle_transition" + local path + path="$(build_admin_metrics_path)" + api_get "$path" | jq '.aggregated.scanner.lifecycle_transition' + + echo "## metrics: scanner lifecycle cycle" + api_get "$path" | jq '.aggregated.scanner | {lifecycle_transition: .lifecycle_transition, current_cycle_ilm_actions: .current_cycle_ilm_actions, current_cycle_lifecycle_transition_actions: .current_cycle_lifecycle_transition_actions}' +} + +main() { + parse_args "$@" + validate + + require_cmd curl + require_cmd jq + require_cmd rg + + run_worker_logs + run_admin_events + run_tier_operation_failures + run_admin_job + run_metrics +} + +main "$@" diff --git a/scripts/manual_transition_journal_audit.sh b/scripts/manual_transition_journal_audit.sh new file mode 100755 index 000000000..262bc1b73 --- /dev/null +++ b/scripts/manual_transition_journal_audit.sh @@ -0,0 +1,590 @@ +#!/usr/bin/env bash +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +ENDPOINT="" +JOB_ID="" +ADMIN_TOKEN="" +LOG_GLOB="/var/log/rustfs/*.log" +METRIC_TYPES="1" +METRIC_SAMPLES="1" +METRIC_INTERVAL="" +TASK_BUCKET_FILTER="" +OUT_DIR="" + +SYS_BUCKET="${RUSTFS_META_BUCKET:-.rustfs.sys}" +TASK_PREFIX="ilm/manual-transition/tasks" +RESULT_PREFIX="ilm/manual-transition/results" + +MC_BIN="${MC_BIN:-mc}" +MC_ALIAS="" +MC_ENDPOINT="" +MC_ACCESS_KEY="" +MC_SECRET_KEY="" +MC_ALIAS_CREATED=false + +SHOW_HELP=false +DRY_RUN=false + +usage() { + cat <<'USAGE' +Usage: + scripts/manual_transition_journal_audit.sh --endpoint --job-id [options] + +Required: + --endpoint Admin endpoint base, e.g. https://127.0.0.1:9000 + --job-id Manual transition job_id + +Optional: + --admin-token Bearer token for /admin/v3 endpoints + --log-glob Log glob, default /var/log/rustfs/*.log + --metric-types /admin/v3/metrics type flags, default 1 + --metric-samples /admin/v3/metrics sample count, default 1 + --metric-interval /admin/v3/metrics interval query param + --bucket-filter Optional bucket filter for logs + --sys-bucket Meta bucket for journal read (default .rustfs.sys) + --mc-alias Reuse existing mc alias (skip alias set) + --mc-endpoint mc endpoint for .rustfs.sys access + --mc-access-key mc access key + --mc-secret-key mc secret key + --out-dir Artifact output dir + --dry-run + --help Show usage + +Output: + - Journal CSV/JSON samples for manual transition tasks and worker results + - Reconcile report (task_count/result_count/missing/mismatch) + - /admin/v3 lifecycle_worker_state / admin_ilm_transition_state events + - lifecycle_tier_operation_failed logs + - aggregated lifecycle transition metrics (admin endpoint sample) +USAGE +} + +require_bash4() { + if (( BASH_VERSINFO[0] < 4 )); then + echo "ERROR: manual transition journal audit requires bash >= 4.0" >&2 + exit 1 + fi +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s' "$value" +} + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: command not found: $cmd" >&2 + exit 1 + fi +} + +require_nonempty() { + local value="$1" + local label="$2" + if [[ -z "$value" ]]; then + echo "ERROR: $label is required" >&2 + exit 1 + fi +} + +normalize_uuid() { + local raw="${1//-/}" + printf '%s' "${raw,,}" +} + +validate_uuid() { + local normalized + normalized="$(normalize_uuid "$1")" + if ! [[ "$normalized" =~ ^[0-9a-f]{32}$ ]]; then + echo "ERROR: invalid uuid (expect UUIDv4): $1" >&2 + exit 1 + fi + printf '%s' "$normalized" +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --job-id) JOB_ID="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --log-glob) LOG_GLOB="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --metric-types) METRIC_TYPES="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --metric-samples) METRIC_SAMPLES="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --metric-interval) METRIC_INTERVAL="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --bucket-filter) TASK_BUCKET_FILTER="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --sys-bucket) SYS_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --mc-alias) MC_ALIAS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --mc-endpoint) MC_ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --mc-access-key) MC_ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --mc-secret-key) MC_SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) SHOW_HELP=true; shift ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +validate_args() { + if [[ "$SHOW_HELP" == true ]]; then + usage + exit 0 + fi + + require_nonempty "$ENDPOINT" "--endpoint" + require_nonempty "$JOB_ID" "--job-id" + + JOB_ID="$(validate_uuid "$JOB_ID")" + if [[ "$LOG_GLOB" == "" ]]; then + echo "ERROR: --log-glob must not be empty" >&2 + exit 1 + fi + if [[ -z "$MC_ALIAS" ]]; then + if [[ -z "$MC_ENDPOINT" || -z "$MC_ACCESS_KEY" || -z "$MC_SECRET_KEY" ]]; then + echo "ERROR: either --mc-alias or (--mc-endpoint + --mc-access-key + --mc-secret-key) is required" >&2 + exit 1 + fi + fi +} + +cleanup_mc_alias() { + if [[ "$MC_ALIAS_CREATED" == true ]]; then + "${MC_BIN}" alias remove "$MC_ALIAS" >/dev/null 2>&1 || true + fi +} + +setup_output() { + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/manual-transition-journal-audit/$(date +%Y%m%dT%H%M%S)" + fi + mkdir -p "$OUT_DIR" + SUMMARY_TXT="${OUT_DIR}/audit_summary.txt" + TASKS_CSV="${OUT_DIR}/tasks.csv" + RESULTS_CSV="${OUT_DIR}/results.csv" + RECONCILE_CSV="${OUT_DIR}/journal_reconcile.csv" + COMMANDS_TXT="${OUT_DIR}/commands.txt" + { + echo "task_key,bucket,object,version_id,storage_class,queued_at_unix_nanos,task_object" + } > "$TASKS_CSV" + { + echo "task_key,result,completion_reason,completed_at_unix_nanos,result_object" + } > "$RESULTS_CSV" + { + echo "task_key,bucket,object,version_id,storage_class,task_has_result,result,result_reason,task_job_id_match,result_job_id_match,task_path,result_path" + } > "$RECONCILE_CSV" +} + +setup_mc_alias() { + if [[ "$MC_ALIAS" == "" ]]; then + MC_ALIAS="mt-journal-audit-${JOB_ID}" + MC_ALIAS_CREATED=true + if [[ "$DRY_RUN" == true ]]; then + echo "[DRY-RUN] mc alias set ${MC_ALIAS} ${MC_ENDPOINT} ${MC_ACCESS_KEY} ***" + else + "${MC_BIN}" alias set "${MC_ALIAS}" "${MC_ENDPOINT}" "${MC_ACCESS_KEY}" "${MC_SECRET_KEY}" >/dev/null + fi + fi + if ! [[ "$DRY_RUN" == true ]]; then + "${MC_BIN}" ls "${MC_ALIAS}/${SYS_BUCKET}" >/dev/null + fi + trap cleanup_mc_alias EXIT +} + +api_get() { + local path="$1" + local url="${ENDPOINT%/}${path}" + if [[ -n "$ADMIN_TOKEN" ]]; then + curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url" + else + curl -sS "$url" + fi +} + +build_admin_metrics_path() { + local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}" + if [[ -n "$METRIC_INTERVAL" ]]; then + path="${path}&interval=${METRIC_INTERVAL}" + fi + printf '%s' "$path" +} + +run_json_stream() { + local query="$1" + local filter="$2" + local log_lines; log_lines="$(mktemp)" + local log_errors; log_errors="$(mktemp)" + local log_paths=() + local match + + while IFS= read -r match; do + [[ -n "$match" ]] && log_paths+=("$match") + done < <(compgen -G "$LOG_GLOB" || true) + if [[ "${#log_paths[@]}" -eq 0 ]]; then + log_paths=("$LOG_GLOB") + fi + + if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then + : + fi + + if [[ -s "$log_errors" ]]; then + sed 's/^/DEBUG: /' "$log_errors" + fi + + if [[ -s "$log_lines" ]]; then + jq -R "$filter" <"$log_lines" + else + echo "[]" + fi + + rm -f "$log_lines" "$log_errors" +} + +run_worker_logs() { + local base_filter='fromjson? | select(.event == "lifecycle_worker_state")' + local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))' + local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state' + + if [[ -n "$TASK_BUCKET_FILTER" ]]; then + manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER)' + dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER) | .state' + fi + + echo "## lifecycle_worker_state all" + run_json_stream "lifecycle_worker_state" "$base_filter" || true + echo + echo "## lifecycle_worker_state manual_transition + job filter" + JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$manual_filter" || true + echo + echo "## lifecycle_worker_state state distribution" + JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true +} + +run_admin_job() { + echo "## manual transition job status" + api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \ + | jq '{status, mode, job_id, report: .report, queue_snapshot: .queue_snapshot, cancel_requested}' +} + +run_admin_events() { + echo "## admin_ilm_transition_state events" + local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}' + if [[ -n "$TASK_BUCKET_FILTER" ]]; then + filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}' + fi + JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \ + run_json_stream "admin_ilm_transition_state" "$filter" || true +} + +run_tier_operation_failures() { + echo "## lifecycle_tier_operation_failed events" + local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}' + if [[ -n "$TASK_BUCKET_FILTER" ]]; then + filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}' + fi + JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \ + run_json_stream "lifecycle_tier_operation_failed" "$filter" || true +} + +run_metrics() { + echo "## metrics: scanner.lifecycle_transition" + local path + path="$(build_admin_metrics_path)" + api_get "$path" | jq '.aggregated.scanner.lifecycle_transition' +} + +journal_key_prefix() { + local prefix="$1" + local shard_a="${JOB_ID:0:2}" + local shard_b="${JOB_ID:2:2}" + printf '%s/%s/%s/%s' "$prefix" "$shard_a" "$shard_b" "$JOB_ID" +} + +csv_escape() { + local value="${1:-}" + value="${value//\"/\"\"}" + printf '\"%s\"' "$value" +} + +read_json_file() { + local object_key="$1" + if [[ "$DRY_RUN" == true ]]; then + echo "" + return 0 + fi + "${MC_BIN}" cat "${MC_ALIAS}/${SYS_BUCKET}/${object_key}" +} + +collect_journals() { + if [[ "$DRY_RUN" == true ]]; then + echo "[DRY-RUN] skip .rustfs.sys object reads." + echo "dry_run=true" > "$SUMMARY_TXT" + return 0 + fi + + local task_prefix result_prefix + task_prefix="$(journal_key_prefix "$TASK_PREFIX")" + result_prefix="$(journal_key_prefix "$RESULT_PREFIX")" + + local task_objs + local result_objs + local object_name + local task_key + local raw + local record_task_job record_result_job + local expected_job + expected_job="$JOB_ID" + + local -A task_bucket task_object task_version task_storage task_queued task_job task_path + local -A result_status result_reason result_completed result_job result_path + local -A task_seen + local -A result_seen + + task_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${task_prefix}" | jq -r 'select(.type == "file") | .key')" + result_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${result_prefix}" | jq -r 'select(.type == "file") | .key')" + + while IFS= read -r object_name; do + [[ -z "$object_name" ]] && continue + task_key="${object_name##*/}" + task_key="${task_key%.json}" + if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then + echo "WARN: ignore invalid task object name: ${object_name}" >&2 + continue + fi + raw="$(read_json_file "$object_name")" + if ! record_task_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then + echo "WARN: parse error task object: ${object_name}" >&2 + continue + fi + if [[ -z "$record_task_job" ]]; then + echo "WARN: task object missing record.job_id: ${object_name}" >&2 + continue + fi + task_seen["$task_key"]="1" + task_bucket["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.bucket // empty' )" + task_object["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.object // empty' )" + task_version["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.version_id // empty' )" + task_storage["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.storage_class // empty' )" + task_queued["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.queued_at_unix_nanos // empty' )" + task_job["$task_key"]="$(printf '%s' "$record_task_job")" + task_path["$task_key"]="$object_name" + printf '%s,%s,%s,%s,%s,%s,%s\n' \ + "$(csv_escape "$task_key")" \ + "$(csv_escape "${task_bucket[$task_key]}")" \ + "$(csv_escape "${task_object[$task_key]}")" \ + "$(csv_escape "${task_version[$task_key]}")" \ + "$(csv_escape "${task_storage[$task_key]}")" \ + "$(csv_escape "${task_queued[$task_key]}")" \ + "$(csv_escape "$object_name")" >> "$TASKS_CSV" + done <<< "$task_objs" + + while IFS= read -r object_name; do + [[ -z "$object_name" ]] && continue + task_key="${object_name##*/}" + task_key="${task_key%.json}" + if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then + echo "WARN: ignore invalid result object name: ${object_name}" >&2 + continue + fi + raw="$(read_json_file "$object_name")" + if ! record_result_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then + echo "WARN: parse error result object: ${object_name}" >&2 + continue + fi + if [[ -z "$record_result_job" ]]; then + echo "WARN: result object missing record.job_id: ${object_name}" >&2 + continue + fi + result_seen["$task_key"]="1" + result_status["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.result // .result // empty' )" + result_reason["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.failure_reason // empty' )" + result_completed["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.completed_at_unix_nanos // empty' )" + result_job["$task_key"]="$(printf '%s' "$record_result_job")" + result_path["$task_key"]="$object_name" + printf '%s,%s,%s,%s,%s\n' \ + "$(csv_escape "$task_key")" \ + "$(csv_escape "${result_status[$task_key]}")" \ + "$(csv_escape "${result_reason[$task_key]}")" \ + "$(csv_escape "${result_completed[$task_key]}")" \ + "$(csv_escape "$object_name")" >> "$RESULTS_CSV" + done <<< "$result_objs" + + local task_count=0 + local result_count=0 + local missing_count=0 + local mismatch_count=0 + local completed_count=0 + local failed_count=0 + local task_job_id_match + local result_job_id_match + local task_record_job + local result_record_job + local -A failure_by_reason=() + + task_count="${#task_seen[@]}" + result_count="${#result_seen[@]}" + + for task_key in "${!task_seen[@]}"; do + local result="" + local reason="" + local task_has_result="false" + if [[ -n "${result_seen[$task_key]+x}" ]]; then + task_has_result="true" + result="${result_status[$task_key]}" + reason="${result_reason[$task_key]}" + if [[ "${result}" == "Completed" ]]; then + completed_count=$((completed_count + 1)) + elif [[ "${result}" == "TierFailure" ]]; then + failed_count=$((failed_count + 1)) + reason="${reason:-Unknown}" + failure_by_reason["$reason"]=$(( ${failure_by_reason["$reason"]:-0} + 1 )) + fi + else + missing_count=$((missing_count + 1)) + fi + + task_job_id_match="false" + result_job_id_match="false" + + task_record_job="${task_job[$task_key]-}" + if [[ -n "$task_record_job" && "$(normalize_uuid "$task_record_job")" == "$expected_job" ]]; then + task_job_id_match="true" + fi + result_record_job="${result_job[$task_key]-}" + if [[ -n "$result_record_job" && "$(normalize_uuid "$result_record_job")" == "$expected_job" ]]; then + result_job_id_match="true" + fi + if [[ "$task_job_id_match" != "true" ]] || [[ -n "${result_seen[$task_key]+x}" && "$result_job_id_match" != "true" ]]; then + mismatch_count=$((mismatch_count + 1)) + fi + + printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \ + "$(csv_escape "$task_key")" \ + "$(csv_escape "${task_bucket[$task_key]}")" \ + "$(csv_escape "${task_object[$task_key]}")" \ + "$(csv_escape "${task_version[$task_key]}")" \ + "$(csv_escape "${task_storage[$task_key]}")" \ + "$(csv_escape "$task_has_result")" \ + "$(csv_escape "${result_status[$task_key]}")" \ + "$(csv_escape "${result_reason[$task_key]}")" \ + "$(csv_escape "$task_job_id_match")" \ + "$(csv_escape "$result_job_id_match")" \ + "$(csv_escape "${task_path[$task_key]}")" \ + "$(csv_escape "${result_path[$task_key]-}")" >> "$RECONCILE_CSV" + done + + local orphan_count=0 + for task_key in "${!result_seen[@]}"; do + if [[ -z "${task_seen[$task_key]+x}" ]]; then + orphan_count=$((orphan_count + 1)) + printf 'orphan_result_task_key=%s\n' "$task_key" >> "$SUMMARY_TXT" + fi + done + + { + echo "job_id=$JOB_ID" + echo "sys_bucket=$SYS_BUCKET" + echo "expected_task_prefix=$task_prefix" + echo "expected_result_prefix=$result_prefix" + echo "task_count=${task_count}" + echo "result_count=${result_count}" + echo "missing_result_count=${missing_count}" + echo "job_id_mismatch_count=${mismatch_count}" + echo "orphan_result_count=${orphan_count}" + echo "completed_result_count=${completed_count}" + echo "failed_result_count=${failed_count}" + for reason in "${!failure_by_reason[@]}"; do + echo "result_failure_${reason// /_}=${failure_by_reason[$reason]}" + done + } > "$SUMMARY_TXT" + + { + echo + echo "## journal summary" + echo "- tasks: ${task_count}" + echo "- results: ${result_count}" + echo "- missing result for task: ${missing_count}" + echo "- orphaned result rows: ${orphan_count}" + echo "- completed result: ${completed_count}" + echo "- failed result: ${failed_count}" + } >> "$SUMMARY_TXT" +} + +build_command_notes() { + { + echo "# Run snippets (to use manually)" + echo + echo "curl -sS ${ENDPOINT}/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES} | jq '.aggregated.scanner.lifecycle_transition'" + echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token --sys-bucket ${SYS_BUCKET}" + echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token --sys-bucket ${SYS_BUCKET} --out-dir ${OUT_DIR}" + if [[ "$DRY_RUN" == true ]]; then + echo "# Add --mc-endpoint, --mc-access-key, --mc-secret-key when writing to .rustfs.sys." + fi + } > "$COMMANDS_TXT" +} + +main() { + parse_args "$@" + if [[ "$SHOW_HELP" == true ]]; then + usage + exit 0 + fi + require_bash4 + validate_args + require_cmd "$MC_BIN" + require_cmd curl + require_cmd jq + require_cmd rg + setup_output + setup_mc_alias + + build_command_notes + run_worker_logs + echo + run_admin_events + echo + run_tier_operation_failures + echo + run_admin_job + echo + run_metrics + echo + collect_journals + echo "Artifacts: $OUT_DIR" + echo " - $TASKS_CSV" + echo " - $RESULTS_CSV" + echo " - $RECONCILE_CSV" + echo " - $SUMMARY_TXT" + echo " - $COMMANDS_TXT" +} + +main "$@" diff --git a/scripts/manual_transition_mixed_rollout_matrix.sh b/scripts/manual_transition_mixed_rollout_matrix.sh new file mode 100755 index 000000000..ede17fcf7 --- /dev/null +++ b/scripts/manual_transition_mixed_rollout_matrix.sh @@ -0,0 +1,291 @@ +#!/usr/bin/env bash +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +ENDPOINT="" +ADMIN_TOKEN="" +OUT_DIR="" +PHASE_MATRIX="request-only:100:0:30,canary:90:10:120,mixed:50:50:240,full-rollout:0:100:360,rollback:100:0:30" +CONCURRENCIES="8,16,32" +OBJECT_COUNTS="5k,20k,50k" +JOB_BUCKET="manual-transition" +JOB_PREFIX="journal-mixed-rollout" +READ_RATIO="90" +RUN_ADMIN_CHECKS=true +DRY_RUN=false + +usage() { + cat <<'USAGE' +Usage: + scripts/manual_transition_mixed_rollout_matrix.sh --endpoint [options] + +Required: + --endpoint Admin API base, e.g. https://127.0.0.1:9000 + +Optional: + --admin-token Bearer token for admin calls + --phase-matrix Comma-separated phase spec: name:old_pct:new_pct:duration_min + --concurrencies Comma-separated concurrency list + --object-counts Comma-separated queue-size list (for plan rows) + --job-bucket Manual transition source bucket for sample commands + --job-prefix Prefix for generated job notes + --read-ratio Target read ratio percentage for mixed workload + --no-admin-checks Skip generated admin check commands + --out-dir Output dir + --dry-run + --help + +Examples: + scripts/manual_transition_mixed_rollout_matrix.sh \ + --endpoint https://127.0.0.1:9000 \ + --admin-token "$TOKEN" \ + --phase-matrix "request-only:100:0:30,canary:90:10:120,full:0:100:240" +USAGE +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s' "$value" +} + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: command not found: $cmd" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --phase-matrix) PHASE_MATRIX="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --object-counts) OBJECT_COUNTS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --job-bucket) JOB_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --job-prefix) JOB_PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --read-ratio) READ_RATIO="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +trim() { + echo "$1" | awk '{$1=$1;print}' +} + +parse_phase_row() { + local spec="$1" + local name old_ratio new_ratio duration_min + + name="$(echo "$spec" | cut -d':' -f1)" + old_ratio="$(echo "$spec" | cut -d':' -f2)" + new_ratio="$(echo "$spec" | cut -d':' -f3)" + duration_min="$(echo "$spec" | cut -d':' -f4)" + name="$(trim "$name")" + old_ratio="$(trim "$old_ratio")" + new_ratio="$(trim "$new_ratio")" + duration_min="$(trim "$duration_min")" + + if [[ -z "$name" || -z "$old_ratio" || -z "$new_ratio" || -z "$duration_min" ]]; then + echo "ERROR: invalid phase spec: $spec" >&2 + exit 1 + fi + if ! [[ "$old_ratio" =~ ^[0-9]+$ && "$new_ratio" =~ ^[0-9]+$ && "$duration_min" =~ ^[0-9]+$ ]]; then + echo "ERROR: phase ratio/duration must be integers: $spec" >&2 + exit 1 + fi + echo "$name|$old_ratio|$new_ratio|$duration_min" +} + +gate_for_phase() { + case "$1" in + request-only) + echo "No mixed-version errors; unknown failures should stay 0 in job report; log ratio must trend down." + ;; + canary) + echo "old/new mismatch in worker results = 0; no Unknown job failures." + ;; + mixed) + echo "request-only + canary acceptance gates both pass for 2+ windows, then proceed." + ;; + full-rollout|full) + echo "full canary-to-full transition completed, no tier_failure in report, worker_result mismatch = 0." + ;; + rollback) + echo "all rollout checks reverse cleanly; queued -> completed in job terminal state." + ;; + *) + echo "run admin job status + journal reconcile + metric delta sanity check." + ;; + esac +} + +admin_check_cmd() { + local phase="$1" + local job_id_ref="$2" + local metric_types="$3" + local metric_samples="$4" + local metric_path + local job_url + local metric_url + local job_filter='.report, .queue_snapshot, .failure_reason' + local metric_filter='.aggregated.scanner.lifecycle_transition' + + if [[ "$RUN_ADMIN_CHECKS" != true ]]; then + return + fi + metric_path="/rustfs/admin/v3/metrics?types=${metric_types}&n=${metric_samples}" + job_url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id_ref}" + metric_url="${ENDPOINT%/}${metric_path}" + + { + printf '# check-%s\n' "$phase" + printf 'curl -sS' + if [[ -n "$ADMIN_TOKEN" ]]; then + printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\"" + fi + printf ' -X GET "%s" | jq '\''%s'\''\n' "$job_url" "$job_filter" + printf 'curl -sS' + if [[ -n "$ADMIN_TOKEN" ]]; then + printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\"" + fi + printf ' -X GET "%s" | jq '\''%s'\''\n' "$metric_url" "$metric_filter" + printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref" + if [[ -n "$ADMIN_TOKEN" ]]; then + printf " --admin-token \"\${ADMIN_TOKEN}\"" + fi + printf ' --sys-bucket .rustfs.sys\n' + } +} + +run_rows() { + local matrix_csv="${OUT_DIR}/mixed_rollout_matrix.csv" + local run_script="${OUT_DIR}/run_phase_commands.sh" + local checklist="${OUT_DIR}/mixed_rollout_checklist.md" + local command_manifest="${OUT_DIR}/run_phase_notes.txt" + local admin_check + local admin_check_csv + local job_id_ref + + { + echo "phase,old_pct,new_pct,duration_min,concurrency,object_count,read_ratio,gate,admin_check" + } > "$matrix_csv" + + { + echo "#!/usr/bin/env bash" + echo "set -euo pipefail" + echo "" + echo "# generated matrix commands: fill in JOB_ID after each manual transition run" + echo "" + } > "$run_script" + + { + echo "# mixed-version rollout command template" + echo "Endpoint: ${ENDPOINT}" + echo "Bucket: ${JOB_BUCKET}" + echo "Prefix: ${JOB_PREFIX}" + echo "Read ratio: ${READ_RATIO}%" + echo "" + } > "$checklist" + + while IFS=',' read -r phase_spec; do + [[ -z "$phase_spec" ]] && continue + parsed="$(parse_phase_row "$phase_spec")" + IFS='|' read -r phase_name old_ratio new_ratio duration_min <<< "$parsed" + + gate="$(gate_for_phase "$phase_name")" + IFS=',' read -r -a counts <<< "$OBJECT_COUNTS" + IFS=',' read -r -a concs <<< "$CONCURRENCIES" + for conc in "${concs[@]}"; do + conc="$(trim "$conc")" + [[ -z "$conc" ]] && continue + for count in "${counts[@]}"; do + count="$(trim "$count")" + [[ -z "$count" || -z "$conc" ]] && continue + + job_id_ref="" + admin_check="$(admin_check_cmd "$phase_name" "$job_id_ref" 1 1 | tr '\n' ';')" + admin_check_csv="${admin_check//\"/\"\"}" + echo "${phase_name},${old_ratio},${new_ratio},${duration_min},${conc},${count},${READ_RATIO},\"${gate}\",\"${admin_check_csv}\"" >> "$matrix_csv" + + { + echo "# phase: ${phase_name} concurrency: ${conc} object_count: ${count}" + echo "# old:new = ${old_ratio}:${new_ratio}, duration=${duration_min}min" + printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref" + if [[ -n "$ADMIN_TOKEN" ]]; then + printf " --admin-token \"\${ADMIN_TOKEN}\"" + fi + printf '\n' + echo "" + } >> "$run_script" + done + done + done < <(tr ',' '\n' <<< "$PHASE_MATRIX") + + { + echo "## Mixed-version rollout matrix artifacts" + echo "" + echo "- matrix: ${matrix_csv}" + echo "- commands: ${run_script}" + echo "- checklist: ${checklist}" + } > "$command_manifest" +} + +main() { + parse_args "$@" + if [[ -z "$ENDPOINT" ]]; then + echo "ERROR: --endpoint is required" >&2 + usage + exit 1 + fi + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/manual-transition-mixed-rollout/$(date +%Y%m%dT%H%M%S)" + fi + mkdir -p "$OUT_DIR" + require_cmd awk + run_rows + + if [[ "$DRY_RUN" == true ]]; then + echo "[DRY-RUN] generated files:" + fi + echo "Generated:" + echo " - ${OUT_DIR}/mixed_rollout_matrix.csv" + echo " - ${OUT_DIR}/run_phase_commands.sh" + echo " - ${OUT_DIR}/mixed_rollout_checklist.md" + echo " - ${OUT_DIR}/run_phase_notes.txt" +} + +main "$@" diff --git a/scripts/manual_transition_soak_matrix.sh b/scripts/manual_transition_soak_matrix.sh new file mode 100755 index 000000000..4c2602a50 --- /dev/null +++ b/scripts/manual_transition_soak_matrix.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash +# Copyright 2026 RustFS Team +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# software distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)" + +ENDPOINT="" +ADMIN_TOKEN="" +OUT_DIR="" +WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy" +SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60" +WORKLOAD_SIZES="4KiB,1MiB,16MiB" +RUN_ADMIN_CHECKS=true +COST_BUDGET="$((24*1024))" +DRY_RUN=false + +usage() { + cat <<'USAGE' +Usage: + scripts/manual_transition_soak_matrix.sh --endpoint [options] + +Required: + --endpoint Admin API base + +Optional: + --admin-token Bearer token for admin calls + --window-spec CSV of + --workload-sizes Comma-separated object size set + --soak-ratios Comma-separated mix spec: label:read_pct:write_pct + --cost-budget Max budget units for each run, default 24576 + --out-dir Output dir + --no-admin-checks Skip admin check command blocks + --dry-run + --help +USAGE +} + +arg_value() { + local flag="$1" + local value="${2:-}" + if [[ -z "$value" || "$value" == --* ]]; then + echo "ERROR: missing value for $flag" >&2 + exit 1 + fi + printf '%s' "$value" +} + +require_cmd() { + local cmd="$1" + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "ERROR: command not found: $cmd" >&2 + exit 1 + fi +} + +parse_args() { + while [[ $# -gt 0 ]]; do + case "$1" in + --endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --window-spec) WINDOW_SPEC="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --workload-sizes) WORKLOAD_SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --soak-ratios) SOAK_RATIOS="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --cost-budget) COST_BUDGET="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;; + --no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;; + --dry-run) DRY_RUN=true; shift ;; + -h|--help) usage; exit 0 ;; + *) + echo "ERROR: unknown arg: $1" >&2 + usage + exit 1 + ;; + esac + done +} + +trim() { + echo "$1" | awk '{$1=$1;print}' +} + +parse_window() { + local spec="$1" + local name duration_min concurrency ops_per_min mix_name + name="$(echo "$spec" | cut -d':' -f1)" + duration_min="$(echo "$spec" | cut -d':' -f2)" + concurrency="$(echo "$spec" | cut -d':' -f3)" + ops_per_min="$(echo "$spec" | cut -d':' -f4)" + mix_name="$(echo "$spec" | cut -d':' -f5)" + name="$(trim "$name")" + duration_min="$(trim "$duration_min")" + concurrency="$(trim "$concurrency")" + ops_per_min="$(trim "$ops_per_min")" + mix_name="$(trim "$mix_name")" + if [[ -z "$name" || -z "$duration_min" || -z "$concurrency" || -z "$ops_per_min" || -z "$mix_name" ]]; then + echo "ERROR: invalid window spec: $spec" >&2 + exit 1 + fi + if ! [[ "$duration_min" =~ ^[0-9]+$ && "$concurrency" =~ ^[0-9]+$ && "$ops_per_min" =~ ^[0-9]+$ ]]; then + echo "ERROR: window duration/concurrency/ops must be integers: $spec" >&2 + exit 1 + fi + echo "$name|$duration_min|$concurrency|$ops_per_min|$mix_name" +} + +parse_ratio() { + local spec="$1" + local name read_pct write_pct + name="$(echo "$spec" | cut -d':' -f1)" + read_pct="$(echo "$spec" | cut -d':' -f2)" + write_pct="$(echo "$spec" | cut -d':' -f3)" + name="$(trim "$name")" + read_pct="$(trim "$read_pct")" + write_pct="$(trim "$write_pct")" + if [[ -z "$name" || -z "$read_pct" || -z "$write_pct" ]]; then + echo "ERROR: invalid mix ratio: $spec" >&2 + exit 1 + fi + if ! [[ "$read_pct" =~ ^[0-9]+$ && "$write_pct" =~ ^[0-9]+$ ]]; then + echo "ERROR: mix percentages must be integers: $spec" >&2 + exit 1 + fi + echo "$name|$read_pct|$write_pct" +} + +resolve_mix() { + local name="$1" + local line + line="$(awk -v target="$name" -F: '{ + if ($1 == target) { print $0; exit } + }' < <(tr ',' '\n' <<< "$SOAK_RATIOS"))" + if [[ -z "$line" ]]; then + return 1 + fi + parse_ratio "$line" +} + +run_checks() { + local tag="$1" + local job_id="$2" + local token_hdr="" + if [[ -z "$job_id" ]]; then + job_id="" + fi + if [[ -n "$ADMIN_TOKEN" ]]; then + token_hdr="-H \"Authorization: Bearer \${ADMIN_TOKEN}\" " + fi + if [[ "$RUN_ADMIN_CHECKS" != true ]]; then + return + fi + echo "# admin checks for ${tag}" + echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}\" | jq '.status, .report, .queue_snapshot, .failure_reason'" + echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/metrics?types=1&n=1\" | jq '.aggregated.scanner.lifecycle_transition'" + printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id" + if [[ -n "$ADMIN_TOKEN" ]]; then + printf " --admin-token \"\${ADMIN_TOKEN}\"" + fi + printf '\n' +} + +write_matrix_files() { + local matrix_csv="${OUT_DIR}/nightly_soak_matrix.csv" + local run_matrix="${OUT_DIR}/run_soak_matrix.sh" + local notes="${OUT_DIR}/soak_notes.md" + local run_id=1 + + { + echo "window,window_duration_min,concurrency,ops_per_min,mix_name,read_pct,write_pct,size,expected_ops,expected_run_id,run_check" + } > "$matrix_csv" + + echo "#!/usr/bin/env bash" > "$run_matrix" + echo "set -euo pipefail" >> "$run_matrix" + echo "" >> "$run_matrix" + + { + echo "# Nightly/night soak matrix for manual transition stress" + echo "- endpoint: ${ENDPOINT}" + echo "- cost-budget: ${COST_BUDGET}" + echo "" + echo "Recommended gate for each row: no unknown failure_reason spikes, job result drift is 0, and queued/task/result counts reconcile." + echo "" + } > "$notes" + + IFS=',' read -r -a sizes <<< "$WORKLOAD_SIZES" + while IFS=',' read -r window_spec; do + [[ -z "$window_spec" ]] && continue + parsed="$(parse_window "$window_spec")" + IFS='|' read -r window_name duration_min concurrency ops_per_min mix_name <<< "$parsed" + + mix="$(resolve_mix "$mix_name" || true)" + if [[ -z "$mix" ]]; then + echo "ERROR: unknown mix name in window spec: ${window_name} -> ${mix_name}" >&2 + exit 1 + fi + IFS='|' read -r resolved_mix read_pct write_pct <<< "$mix" + + for size in "${sizes[@]}"; do + size="$(trim "$size")" + [[ -z "$size" ]] && continue + expected_ops=$(( duration_min * ops_per_min )) + if (( expected_ops > COST_BUDGET )); then + budget_status="over-budget" + else + budget_status="within-budget" + fi + echo "${window_name},${duration_min},${concurrency},${ops_per_min},${resolved_mix},${read_pct},${write_pct},${size},${expected_ops},${run_id},${budget_status}" >> "$matrix_csv" + { + echo "# window=${window_name} size=${size} mix=${resolved_mix} concurrency=${concurrency} duration=${duration_min}m ops_per_min=${ops_per_min}" + echo "job_id=\"\"" + echo "read_pct=${read_pct}" + echo "write_pct=${write_pct}" + echo "expected_ops=${expected_ops}" + echo "budget_status=${budget_status}" + run_checks "${window_name}_${size}_${concurrency}_${run_id}" "" + echo + } >> "$run_matrix" + run_id=$((run_id + 1)) + done + done < <(tr ',' '\n' <<< "$WINDOW_SPEC") + + { + echo "# run target:" + echo "bash ${run_matrix}" + } >> "$notes" +} + +main() { + parse_args "$@" + if [[ -z "$ENDPOINT" ]]; then + echo "ERROR: --endpoint is required" >&2 + usage + exit 1 + fi + if [[ -z "$OUT_DIR" ]]; then + OUT_DIR="${PROJECT_ROOT}/target/manual-transition-nightly-soak/$(date +%Y%m%dT%H%M%S)" + fi + mkdir -p "$OUT_DIR" + require_cmd awk + write_matrix_files + if [[ "$DRY_RUN" == true ]]; then + echo "[DRY-RUN] generated matrix files:" + fi + echo "Generated:" + echo " - ${OUT_DIR}/nightly_soak_matrix.csv" + echo " - ${OUT_DIR}/run_soak_matrix.sh" + echo " - ${OUT_DIR}/soak_notes.md" +} + +main "$@"