mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-28 16:07:05 +00:00
feat(ecstore): attribute manual transition worker failures (#5324)
* feat(ecstore): attribute manual transition worker failures Add manual transition worker failure reason tracking and persistence recovery compatibility for checksum validation. Co-Authored-By: heihutu <heihutu@gmail.com> * chore(ilm): add manual transition diagnostics scripts Co-Authored-By: heihutu <heihutu@gmail.com> * style(ecstore): format manual transition attribution Co-Authored-By: heihutu <heihutu@gmail.com> * fix(ecstore): avoid copying failure reasons via clone Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
Generated
+23
-23
@@ -56,9 +56,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "aes"
|
name = "aes"
|
||||||
version = "0.9.1"
|
version = "0.9.2"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "f1fc76eaeac4c9164506c466d4ffdd8ec9d0c5bf57ee97177c4d8eceb3a0e138"
|
checksum = "f8eb277bec05f56a0e0591f155a484cbd0f4f07ff2905051a48c72f004f7ed58"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"cipher 0.5.2",
|
"cipher 0.5.2",
|
||||||
"cpubits",
|
"cpubits",
|
||||||
@@ -73,7 +73,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
|
checksum = "fdf011db2e21ce0d575593d749db5554b47fed37aff429e4dc50bc91ac93a028"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aead",
|
"aead",
|
||||||
"aes 0.9.1",
|
"aes 0.9.2",
|
||||||
"cipher 0.5.2",
|
"cipher 0.5.2",
|
||||||
"ctr",
|
"ctr",
|
||||||
"ghash",
|
"ghash",
|
||||||
@@ -1528,7 +1528,7 @@ version = "0.10.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1547,7 +1547,7 @@ version = "0.3.3"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -1870,7 +1870,7 @@ version = "0.4.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"crypto-common 0.1.7",
|
"crypto-common 0.1.6",
|
||||||
"inout 0.1.4",
|
"inout 0.1.4",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -2328,7 +2328,7 @@ version = "0.5.5"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"rand_core 0.6.4",
|
"rand_core 0.6.4",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
@@ -2353,11 +2353,11 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "crypto-common"
|
name = "crypto-common"
|
||||||
version = "0.1.7"
|
version = "0.1.6"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"typenum",
|
"typenum",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3554,7 +3554,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"block-buffer 0.10.4",
|
"block-buffer 0.10.4",
|
||||||
"const-oid 0.9.6",
|
"const-oid 0.9.6",
|
||||||
"crypto-common 0.1.7",
|
"crypto-common 0.1.6",
|
||||||
"subtle",
|
"subtle",
|
||||||
]
|
]
|
||||||
|
|
||||||
@@ -3811,7 +3811,7 @@ dependencies = [
|
|||||||
"crypto-bigint 0.5.5",
|
"crypto-bigint 0.5.5",
|
||||||
"digest 0.10.7",
|
"digest 0.10.7",
|
||||||
"ff 0.13.1",
|
"ff 0.13.1",
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"group 0.13.0",
|
"group 0.13.0",
|
||||||
"hkdf 0.12.4",
|
"hkdf 0.12.4",
|
||||||
"pem-rfc7468 0.7.0",
|
"pem-rfc7468 0.7.0",
|
||||||
@@ -4257,9 +4257,9 @@ dependencies = [
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "generic-array"
|
name = "generic-array"
|
||||||
version = "0.14.7"
|
version = "0.14.9"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"typenum",
|
"typenum",
|
||||||
"version_check",
|
"version_check",
|
||||||
@@ -4272,7 +4272,7 @@ version = "1.4.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
checksum = "ab4e5aa225bc56696909483320f0ff9b600f1a971b52e07a17d70f3d9b43254b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"rustversion",
|
"rustversion",
|
||||||
"typenum",
|
"typenum",
|
||||||
]
|
]
|
||||||
@@ -5272,7 +5272,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"block-padding 0.3.3",
|
"block-padding 0.3.3",
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
]
|
]
|
||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
@@ -5745,9 +5745,9 @@ checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2"
|
|||||||
|
|
||||||
[[package]]
|
[[package]]
|
||||||
name = "libflate"
|
name = "libflate"
|
||||||
version = "2.3.0"
|
version = "2.3.1"
|
||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "cd96e993e5f3368b0cb8497dae6c860c22af8ff18388c61c6c0b86c58d86b5df"
|
checksum = "a4da9b700e758e57152a1fd1c52cbdc5727c1aa6d8743dc1acda917398f1d76c"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"adler32",
|
"adler32",
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
@@ -7485,7 +7485,7 @@ version = "0.8.1"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
|
checksum = "63d440a804ec8d6fafbb6b84471e013286658d373248927692ab3366686220ca"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes 0.9.1",
|
"aes 0.9.2",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"cbc 0.2.1",
|
"cbc 0.2.1",
|
||||||
"der 0.8.1",
|
"der 0.8.1",
|
||||||
@@ -8681,7 +8681,7 @@ version = "0.62.4"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
|
checksum = "b8b67b5a0d8068c89dcbe9d95df986af7a851d1f3c604525274c37468e60464f"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes 0.9.1",
|
"aes 0.9.2",
|
||||||
"aws-lc-rs",
|
"aws-lc-rs",
|
||||||
"bitflags 2.13.1",
|
"bitflags 2.13.1",
|
||||||
"block-padding 0.4.2",
|
"block-padding 0.4.2",
|
||||||
@@ -10503,7 +10503,7 @@ checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc"
|
|||||||
dependencies = [
|
dependencies = [
|
||||||
"base16ct 0.2.0",
|
"base16ct 0.2.0",
|
||||||
"der 0.7.10",
|
"der 0.7.10",
|
||||||
"generic-array 0.14.7",
|
"generic-array 0.14.9",
|
||||||
"pkcs8 0.10.2",
|
"pkcs8 0.10.2",
|
||||||
"subtle",
|
"subtle",
|
||||||
"zeroize",
|
"zeroize",
|
||||||
@@ -11123,7 +11123,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
|
checksum = "d801accda99469cde6d73da741422610fdf6508a72d9a69d1b55cb241c720597"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aead",
|
"aead",
|
||||||
"aes 0.9.1",
|
"aes 0.9.2",
|
||||||
"aes-gcm",
|
"aes-gcm",
|
||||||
"chacha20",
|
"chacha20",
|
||||||
"cipher 0.5.2",
|
"cipher 0.5.2",
|
||||||
@@ -13068,7 +13068,7 @@ version = "8.6.0"
|
|||||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||||
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"aes 0.9.1",
|
"aes 0.9.2",
|
||||||
"bzip2",
|
"bzip2",
|
||||||
"constant_time_eq",
|
"constant_time_eq",
|
||||||
"crc32fast",
|
"crc32fast",
|
||||||
|
|||||||
+3
-2
@@ -68,7 +68,7 @@ resolver = "3"
|
|||||||
edition = "2024"
|
edition = "2024"
|
||||||
license = "Apache-2.0"
|
license = "Apache-2.0"
|
||||||
repository = "https://github.com/rustfs/rustfs"
|
repository = "https://github.com/rustfs/rustfs"
|
||||||
rust-version = "1.96.0"
|
rust-version = "1.97.1"
|
||||||
version = "1.0.0-beta.11"
|
version = "1.0.0-beta.11"
|
||||||
homepage = "https://rustfs.com"
|
homepage = "https://rustfs.com"
|
||||||
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
description = "RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. "
|
||||||
@@ -211,7 +211,7 @@ zeroize = { version = "1.9.0" }
|
|||||||
# Time and Date
|
# Time and Date
|
||||||
chrono = { version = "0.4.45" }
|
chrono = { version = "0.4.45" }
|
||||||
humantime = "2.4.0"
|
humantime = "2.4.0"
|
||||||
jiff = { version = "0.2.34" }
|
jiff = { version = "0.2.35" }
|
||||||
time = { version = "0.3.54" }
|
time = { version = "0.3.54" }
|
||||||
|
|
||||||
# Database
|
# Database
|
||||||
@@ -243,6 +243,7 @@ crossbeam-channel = "0.5.16"
|
|||||||
crossbeam-deque = "0.8.7"
|
crossbeam-deque = "0.8.7"
|
||||||
crossbeam-utils = "0.8.22"
|
crossbeam-utils = "0.8.22"
|
||||||
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
datafusion = { default-features = false, git = "https://github.com/apache/datafusion.git", rev = "dae03ee062b2abf986de8df12ea82fb1578a2d99" }
|
||||||
|
#datafusion = { default-features = false, version = "54.1.0" }
|
||||||
derive_builder = "0.20.2"
|
derive_builder = "0.20.2"
|
||||||
enumset = "1.1.14"
|
enumset = "1.1.14"
|
||||||
faster-hex = "0.10.0"
|
faster-hex = "0.10.0"
|
||||||
|
|||||||
@@ -23,13 +23,13 @@ use crate::bucket::lifecycle::lifecycle::{
|
|||||||
};
|
};
|
||||||
use crate::bucket::lifecycle::manual_transition_job::{
|
use crate::bucket::lifecycle::manual_transition_job::{
|
||||||
MANUAL_TRANSITION_JOB_RECORD_PREFIX, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission,
|
MANUAL_TRANSITION_JOB_RECORD_PREFIX, ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission,
|
||||||
ManualTransitionScopeAdmissionClaim, ManualTransitionTaskRecord, ManualTransitionWorkerResult,
|
ManualTransitionScopeAdmissionClaim, ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason,
|
||||||
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
ManualTransitionWorkerResult, claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
||||||
load_manual_transition_job_record, load_manual_transition_job_record_with_etag,
|
load_manual_transition_job_record, load_manual_transition_job_record_with_etag,
|
||||||
manual_transition_job_id_from_record_object_name, manual_transition_job_lease_expired,
|
manual_transition_job_id_from_record_object_name, manual_transition_job_lease_expired,
|
||||||
manual_transition_worker_result_task_key, persist_manual_transition_job_progress, reconcile_manual_transition_worker_results,
|
manual_transition_worker_result_task_key, persist_manual_transition_job_progress, reconcile_manual_transition_worker_results,
|
||||||
record_manual_transition_worker_result, renew_manual_transition_job_lease, save_manual_transition_job_record_if_current,
|
record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason,
|
||||||
save_manual_transition_task_if_absent,
|
renew_manual_transition_job_lease, save_manual_transition_job_record_if_current, save_manual_transition_task_if_absent,
|
||||||
};
|
};
|
||||||
use crate::bucket::lifecycle::replication_sink;
|
use crate::bucket::lifecycle::replication_sink;
|
||||||
use crate::bucket::lifecycle::replication_sink::{
|
use crate::bucket::lifecycle::replication_sink::{
|
||||||
@@ -49,7 +49,9 @@ use crate::disk::error::DiskError;
|
|||||||
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
|
use crate::disk::{DeleteOptions, Disk, DiskAPI, RUSTFS_META_BUCKET, RUSTFS_META_MULTIPART_BUCKET, STORAGE_FORMAT_FILE};
|
||||||
use crate::error::Error;
|
use crate::error::Error;
|
||||||
use crate::error::StorageError;
|
use crate::error::StorageError;
|
||||||
use crate::error::{error_resp_to_object_err, is_err_object_not_found, is_err_version_not_found, is_network_or_host_down};
|
use crate::error::{
|
||||||
|
error_resp_to_object_err, is_err_object_not_found, is_err_read_quorum, is_err_version_not_found, is_network_or_host_down,
|
||||||
|
};
|
||||||
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
use crate::object_api::{GetObjectReader, ObjectInfo, ObjectOptions};
|
||||||
use crate::services::tier::{
|
use crate::services::tier::{
|
||||||
tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata},
|
tier::{TierConfigMgr, TierOperationLease, tier_destination_id_from_metadata},
|
||||||
@@ -94,7 +96,7 @@ use s3s::dto::{
|
|||||||
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
|
use s3s::header::{X_AMZ_RESTORE, X_AMZ_SERVER_SIDE_ENCRYPTION};
|
||||||
use sha2::{Digest, Sha256};
|
use sha2::{Digest, Sha256};
|
||||||
use std::any::Any;
|
use std::any::Any;
|
||||||
use std::collections::{HashMap, HashSet};
|
use std::collections::{BTreeMap, HashMap, HashSet};
|
||||||
use std::env;
|
use std::env;
|
||||||
use std::future::Future;
|
use std::future::Future;
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
@@ -1658,11 +1660,12 @@ impl TransitionState {
|
|||||||
.await
|
.await
|
||||||
{
|
{
|
||||||
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
|
if let (Some(job_id), Some(result_key)) = (task.manual_job_id, task.manual_result_key.as_deref()) {
|
||||||
record_manual_transition_worker_result_for_task(
|
record_manual_transition_worker_result_for_task_with_reason(
|
||||||
api.clone(),
|
api.clone(),
|
||||||
job_id,
|
job_id,
|
||||||
result_key,
|
result_key,
|
||||||
ManualTransitionWorkerResult::TierFailure,
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
manual_transition_worker_failure_reason(&err),
|
||||||
)
|
)
|
||||||
.await;
|
.await;
|
||||||
}
|
}
|
||||||
@@ -1818,6 +1821,78 @@ async fn record_manual_transition_worker_result_for_task(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn record_manual_transition_worker_result_for_task_with_reason(
|
||||||
|
api: Arc<ECStore>,
|
||||||
|
job_id: Uuid,
|
||||||
|
result_key: &str,
|
||||||
|
result: ManualTransitionWorkerResult,
|
||||||
|
reason: ManualTransitionWorkerFailureReason,
|
||||||
|
) {
|
||||||
|
if let Err(err) = record_manual_transition_worker_result_with_reason(
|
||||||
|
api,
|
||||||
|
job_id,
|
||||||
|
result_key,
|
||||||
|
result,
|
||||||
|
manual_transition_queue_snapshot(),
|
||||||
|
Some(reason),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
{
|
||||||
|
warn!(
|
||||||
|
event = EVENT_LIFECYCLE_WORKER_STATE,
|
||||||
|
component = LOG_COMPONENT_ECSTORE,
|
||||||
|
subsystem = LOG_SUBSYSTEM_LIFECYCLE,
|
||||||
|
job_id = %job_id,
|
||||||
|
error = %err,
|
||||||
|
reason = ?reason,
|
||||||
|
state = "manual_transition_worker_result_failed",
|
||||||
|
"Manual transition worker failed to persist job result"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn manual_transition_worker_failure_reason(err: &Error) -> ManualTransitionWorkerFailureReason {
|
||||||
|
if is_err_object_not_found(err) || is_err_version_not_found(err) {
|
||||||
|
return ManualTransitionWorkerFailureReason::NotFound;
|
||||||
|
}
|
||||||
|
if is_err_permission_denied(err) {
|
||||||
|
return ManualTransitionWorkerFailureReason::PermissionDenied;
|
||||||
|
}
|
||||||
|
if is_err_read_quorum(err) {
|
||||||
|
return ManualTransitionWorkerFailureReason::Quorum;
|
||||||
|
}
|
||||||
|
if is_timeout(err) {
|
||||||
|
return ManualTransitionWorkerFailureReason::Timeout;
|
||||||
|
}
|
||||||
|
if is_slow_down(err) {
|
||||||
|
return ManualTransitionWorkerFailureReason::SlowDown;
|
||||||
|
}
|
||||||
|
if is_network_or_host_down(&err.to_string(), false) {
|
||||||
|
return ManualTransitionWorkerFailureReason::Network;
|
||||||
|
}
|
||||||
|
ManualTransitionWorkerFailureReason::Unknown
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_err_permission_denied(err: &Error) -> bool {
|
||||||
|
match err {
|
||||||
|
Error::VolumeAccessDenied | Error::FileAccessDenied | Error::PrefixAccessDenied(_, _) | Error::DiskAccessDenied => true,
|
||||||
|
Error::Io(io) => io.kind() == std::io::ErrorKind::PermissionDenied,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_timeout(err: &Error) -> bool {
|
||||||
|
match err {
|
||||||
|
Error::Timeout => true,
|
||||||
|
Error::Io(io) => io.kind() == std::io::ErrorKind::TimedOut,
|
||||||
|
_ => false,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn is_slow_down(err: &Error) -> bool {
|
||||||
|
matches!(err, Error::SlowDown)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn init_background_expiry(api: Arc<ECStore>) {
|
pub async fn init_background_expiry(api: Arc<ECStore>) {
|
||||||
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
|
let mut workers = get_env_usize("RUSTFS_MAX_EXPIRY_WORKERS", std::cmp::min(num_cpus::get(), 16));
|
||||||
//globalILMConfig.getExpirationWorkers()
|
//globalILMConfig.getExpirationWorkers()
|
||||||
@@ -3233,6 +3308,8 @@ pub struct ManualTransitionRunReport {
|
|||||||
#[serde(default, skip_serializing_if = "is_zero_u64")]
|
#[serde(default, skip_serializing_if = "is_zero_u64")]
|
||||||
pub transition_failed: u64,
|
pub transition_failed: u64,
|
||||||
pub tier_failure: u64,
|
pub tier_failure: u64,
|
||||||
|
#[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
|
||||||
|
pub tier_failure_by_reason: BTreeMap<ManualTransitionWorkerFailureReason, u64>,
|
||||||
pub truncated_by_limit: bool,
|
pub truncated_by_limit: bool,
|
||||||
pub truncated_by_duration: bool,
|
pub truncated_by_duration: bool,
|
||||||
pub cancelled: bool,
|
pub cancelled: bool,
|
||||||
@@ -3303,12 +3380,18 @@ impl ManualTransitionRunReport {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
|
pub fn merge_scan_report_preserving_worker(&mut self, scan_report: &ManualTransitionRunReport) {
|
||||||
|
let mut tier_failure_by_reason = self.tier_failure_by_reason.clone();
|
||||||
|
for (reason, count) in &scan_report.tier_failure_by_reason {
|
||||||
|
let current = tier_failure_by_reason.get(reason).copied().unwrap_or_default();
|
||||||
|
tier_failure_by_reason.insert(*reason, current.max(*count));
|
||||||
|
}
|
||||||
let transition_completed = self.transition_completed;
|
let transition_completed = self.transition_completed;
|
||||||
let transition_failed = self.transition_failed;
|
let transition_failed = self.transition_failed;
|
||||||
*self = scan_report.clone();
|
*self = scan_report.clone();
|
||||||
self.transition_completed = transition_completed;
|
self.transition_completed = transition_completed;
|
||||||
self.transition_failed = transition_failed;
|
self.transition_failed = transition_failed;
|
||||||
self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
|
self.tier_failure = scan_report.tier_failure.saturating_add(transition_failed);
|
||||||
|
self.tier_failure_by_reason = tier_failure_by_reason;
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn worker_transition_pending(&self) -> bool {
|
pub fn worker_transition_pending(&self) -> bool {
|
||||||
@@ -4658,12 +4741,13 @@ mod tests {
|
|||||||
lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
|
lifecycle_delete_all_versions_replication_scan, lifecycle_deleted_object, lifecycle_replication_blocks_action,
|
||||||
lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets,
|
lifecycle_rule_has_date_expiration, lifecycle_version_purge_state_from_completed_targets,
|
||||||
manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink,
|
manual_transition_duration_elapsed, manual_transition_has_more_after_limit, manual_transition_recovery_progress_sink,
|
||||||
manual_transition_version_marker, mark_delete_opts_skip_decommissioned_on_remote_success,
|
manual_transition_version_marker, manual_transition_worker_failure_reason,
|
||||||
merge_stale_multipart_candidate, persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint,
|
mark_delete_opts_skip_decommissioned_on_remote_success, merge_stale_multipart_candidate,
|
||||||
recover_manual_transition_job, recover_manual_transition_jobs, replication_state_for_delete,
|
persist_manual_transition_job_progress, persist_manual_transition_page_checkpoint, recover_manual_transition_job,
|
||||||
resolve_tier_free_version_recovery_enabled, resolve_transition_queue_capacity, resolve_transition_queue_send_timeout,
|
recover_manual_transition_jobs, replication_state_for_delete, resolve_tier_free_version_recovery_enabled,
|
||||||
resolve_transition_worker_count, resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop,
|
resolve_transition_queue_capacity, resolve_transition_queue_send_timeout, resolve_transition_worker_count,
|
||||||
select_restore_s3_location, set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
resolve_transition_workers_absolute_max, run_tier_free_version_recovery_loop, select_restore_s3_location,
|
||||||
|
set_lifecycle_observability_observer, set_recovered_free_version_enqueue_observer,
|
||||||
should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state,
|
should_defer_date_expiry_for_recent_config_update, should_reuse_lifecycle_delete_replication_state,
|
||||||
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
transitioned_cleanup_tuple, transitioned_object_delete_opts, wait_for_tier_free_version_recovery,
|
||||||
};
|
};
|
||||||
@@ -4676,16 +4760,17 @@ mod tests {
|
|||||||
use crate::bucket::lifecycle::config_boundary;
|
use crate::bucket::lifecycle::config_boundary;
|
||||||
use crate::bucket::lifecycle::manual_transition_job::{
|
use crate::bucket::lifecycle::manual_transition_job::{
|
||||||
ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim,
|
ManualTransitionJobRecord, ManualTransitionJobState, ManualTransitionScopeAdmission, ManualTransitionScopeAdmissionClaim,
|
||||||
ManualTransitionTaskRecord, ManualTransitionWorkerResult, ManualTransitionWorkerResultRecord,
|
ManualTransitionTaskRecord, ManualTransitionWorkerFailureReason, ManualTransitionWorkerResult,
|
||||||
claim_manual_transition_scope_admission, delete_manual_transition_scope_admission_if_current,
|
ManualTransitionWorkerResultRecord, claim_manual_transition_scope_admission,
|
||||||
legacy_manual_transition_scope_key, load_manual_transition_job_record, load_manual_transition_scope_admission,
|
delete_manual_transition_scope_admission_if_current, legacy_manual_transition_scope_key,
|
||||||
|
load_manual_transition_job_record, load_manual_transition_scope_admission,
|
||||||
load_manual_transition_scope_admission_with_etag, load_manual_transition_task_record,
|
load_manual_transition_scope_admission_with_etag, load_manual_transition_task_record,
|
||||||
manual_transition_scope_record_object_name, manual_transition_worker_result_object_name,
|
manual_transition_scope_record_object_name, manual_transition_worker_result_object_name,
|
||||||
manual_transition_worker_result_task_key, reconcile_manual_transition_worker_results,
|
manual_transition_worker_result_task_key, reconcile_manual_transition_worker_results,
|
||||||
record_manual_transition_worker_result, renew_manual_transition_job_lease, request_manual_transition_job_cancel,
|
record_manual_transition_worker_result, record_manual_transition_worker_result_with_reason,
|
||||||
save_manual_transition_job_record, save_manual_transition_scope_admission_if_absent,
|
renew_manual_transition_job_lease, request_manual_transition_job_cancel, save_manual_transition_job_record,
|
||||||
save_manual_transition_scope_admission_if_current, save_manual_transition_task_if_absent,
|
save_manual_transition_scope_admission_if_absent, save_manual_transition_scope_admission_if_current,
|
||||||
save_manual_transition_worker_result_if_absent,
|
save_manual_transition_task_if_absent, save_manual_transition_worker_result_if_absent,
|
||||||
};
|
};
|
||||||
use crate::bucket::lifecycle::replication_sink::{
|
use crate::bucket::lifecycle::replication_sink::{
|
||||||
ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, VersionPurgeStatusType,
|
ReplicateDecision, ReplicateTargetDecision, ReplicationStatusType, VersionPurgeStatusType,
|
||||||
@@ -7539,6 +7624,37 @@ mod tests {
|
|||||||
assert!(record.completed_at_unix_nanos.is_some());
|
assert!(record.completed_at_unix_nanos.is_some());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manual_transition_worker_failure_reason_classifies_known_failures() {
|
||||||
|
assert_eq!(
|
||||||
|
manual_transition_worker_failure_reason(&Error::ObjectNotFound("bucket".to_string(), "obj".to_string())),
|
||||||
|
ManualTransitionWorkerFailureReason::NotFound
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
manual_transition_worker_failure_reason(&Error::Io(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::PermissionDenied,
|
||||||
|
"file access denied",
|
||||||
|
))),
|
||||||
|
ManualTransitionWorkerFailureReason::PermissionDenied
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
manual_transition_worker_failure_reason(&Error::ErasureReadQuorum),
|
||||||
|
ManualTransitionWorkerFailureReason::Quorum
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
manual_transition_worker_failure_reason(&Error::Timeout),
|
||||||
|
ManualTransitionWorkerFailureReason::Timeout
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
manual_transition_worker_failure_reason(&Error::SlowDown),
|
||||||
|
ManualTransitionWorkerFailureReason::SlowDown
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
manual_transition_worker_failure_reason(&Error::MethodNotAllowed),
|
||||||
|
ManualTransitionWorkerFailureReason::Unknown
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn manual_transition_counts_already_transitioned_object() {
|
async fn manual_transition_counts_already_transitioned_object() {
|
||||||
let lc = latest_transition_lifecycle();
|
let lc = latest_transition_lifecycle();
|
||||||
@@ -8471,6 +8587,49 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
#[serial]
|
||||||
|
async fn manual_transition_worker_result_stores_failure_reason() {
|
||||||
|
let (_paths, ecstore) = setup_test_env().await;
|
||||||
|
let job_id = Uuid::new_v4();
|
||||||
|
let bucket = format!("manual-worker-failure-reason-{}", job_id.simple());
|
||||||
|
let options = ManualTransitionRunOptions::default();
|
||||||
|
let mut record = ManualTransitionJobRecord::new(job_id, &bucket, &options, "worker-owner");
|
||||||
|
record.complete(
|
||||||
|
ManualTransitionRunReport {
|
||||||
|
bucket: bucket.clone(),
|
||||||
|
enqueued: 1,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
);
|
||||||
|
save_manual_transition_job_record(ecstore.clone(), &record)
|
||||||
|
.await
|
||||||
|
.expect("worker result job record should save");
|
||||||
|
|
||||||
|
let task_key = manual_transition_worker_result_task_key(&bucket, "logs/fail", None);
|
||||||
|
let final_record = record_manual_transition_worker_result_with_reason(
|
||||||
|
ecstore.clone(),
|
||||||
|
job_id,
|
||||||
|
&task_key,
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
Some(ManualTransitionWorkerFailureReason::Network),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("worker result with failure reason should persist");
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
final_record
|
||||||
|
.report
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::Network),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
assert_eq!(final_record.report.transition_failed, 1);
|
||||||
|
assert_eq!(final_record.report.tier_failure, 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
#[serial]
|
#[serial]
|
||||||
async fn manual_transition_worker_result_reconcile_applies_marker_and_releases_admission() {
|
async fn manual_transition_worker_result_reconcile_applies_marker_and_releases_admission() {
|
||||||
|
|||||||
@@ -12,6 +12,7 @@
|
|||||||
// See the License for the specific language governing permissions and
|
// See the License for the specific language governing permissions and
|
||||||
// limitations under the License.
|
// limitations under the License.
|
||||||
|
|
||||||
|
use std::collections::BTreeMap;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
use rustfs_utils::crypto::{hex_sha256, is_sha256_checksum};
|
||||||
@@ -194,6 +195,15 @@ impl ManualTransitionJobRecord {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn record_worker_result(&mut self, result: ManualTransitionWorkerResult, queue_snapshot: ManualTransitionQueueSnapshot) {
|
pub fn record_worker_result(&mut self, result: ManualTransitionWorkerResult, queue_snapshot: ManualTransitionQueueSnapshot) {
|
||||||
|
self.record_worker_result_with_reason(result, queue_snapshot, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn record_worker_result_with_reason(
|
||||||
|
&mut self,
|
||||||
|
result: ManualTransitionWorkerResult,
|
||||||
|
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||||
|
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||||
|
) {
|
||||||
if self.is_terminal() {
|
if self.is_terminal() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -204,6 +214,8 @@ impl ManualTransitionJobRecord {
|
|||||||
ManualTransitionWorkerResult::TierFailure => {
|
ManualTransitionWorkerResult::TierFailure => {
|
||||||
self.report.transition_failed = self.report.transition_failed.saturating_add(1);
|
self.report.transition_failed = self.report.transition_failed.saturating_add(1);
|
||||||
self.report.tier_failure = self.report.tier_failure.saturating_add(1);
|
self.report.tier_failure = self.report.tier_failure.saturating_add(1);
|
||||||
|
let reason = failure_reason.unwrap_or(ManualTransitionWorkerFailureReason::Unknown);
|
||||||
|
*self.report.tier_failure_by_reason.entry(reason).or_insert(0) += 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
self.queue_snapshot = queue_snapshot;
|
self.queue_snapshot = queue_snapshot;
|
||||||
@@ -215,6 +227,7 @@ impl ManualTransitionJobRecord {
|
|||||||
&mut self,
|
&mut self,
|
||||||
completed: u64,
|
completed: u64,
|
||||||
failed: u64,
|
failed: u64,
|
||||||
|
failure_reasons: &BTreeMap<ManualTransitionWorkerFailureReason, u64>,
|
||||||
task_queued: u64,
|
task_queued: u64,
|
||||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||||
) -> bool {
|
) -> bool {
|
||||||
@@ -237,11 +250,17 @@ impl ManualTransitionJobRecord {
|
|||||||
{
|
{
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
let mut scan_tier_failure_by_reason = self.report.tier_failure_by_reason.clone();
|
||||||
|
for (reason, count) in failure_reasons {
|
||||||
|
let current = scan_tier_failure_by_reason.get(reason).copied().unwrap_or_default();
|
||||||
|
scan_tier_failure_by_reason.insert(*reason, current.max(*count));
|
||||||
|
}
|
||||||
let scan_tier_failure = self.report.tier_failure.saturating_sub(self.report.transition_failed);
|
let scan_tier_failure = self.report.tier_failure.saturating_sub(self.report.transition_failed);
|
||||||
self.report.enqueued = enqueued;
|
self.report.enqueued = enqueued;
|
||||||
self.report.transition_completed = transition_completed;
|
self.report.transition_completed = transition_completed;
|
||||||
self.report.transition_failed = transition_failed;
|
self.report.transition_failed = transition_failed;
|
||||||
self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
|
self.report.tier_failure = scan_tier_failure.saturating_add(transition_failed);
|
||||||
|
self.report.tier_failure_by_reason = scan_tier_failure_by_reason;
|
||||||
self.queue_snapshot = queue_snapshot;
|
self.queue_snapshot = queue_snapshot;
|
||||||
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
self.updated_at_unix_nanos = OffsetDateTime::now_utc().unix_timestamp_nanos();
|
||||||
self.mark_terminal_if_worker_drained();
|
self.mark_terminal_if_worker_drained();
|
||||||
@@ -401,9 +420,24 @@ impl ManualTransitionJobRecord {
|
|||||||
return Err(ManualTransitionJobError::Corrupt("content checksum is not a sha256 checksum"));
|
return Err(ManualTransitionJobError::Corrupt("content checksum is not a sha256 checksum"));
|
||||||
}
|
}
|
||||||
let mut job = persisted.job;
|
let mut job = persisted.job;
|
||||||
let job_bytes = serde_json::to_vec(&job)?;
|
let mut job_bytes = serde_json::to_vec(&job)?;
|
||||||
let actual_checksum = hex_sha256(&job_bytes, ToOwned::to_owned);
|
let actual_checksum = hex_sha256(&job_bytes, ToOwned::to_owned);
|
||||||
if persisted.content_sha256 != actual_checksum {
|
let checksum_match = if persisted.content_sha256 == actual_checksum {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
let mut job_value = serde_json::to_value(&job)?;
|
||||||
|
if let Some(job_obj) = job_value.as_object_mut() {
|
||||||
|
if let Some(report) = job_obj.get_mut("report").and_then(|value| value.as_object_mut()) {
|
||||||
|
report.remove("tier_failure_by_reason");
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||||
|
}
|
||||||
|
job_bytes = serde_json::to_vec(&job_value)?;
|
||||||
|
let fallback_checksum = hex_sha256(&job_bytes, ToOwned::to_owned);
|
||||||
|
persisted.content_sha256 == fallback_checksum
|
||||||
|
};
|
||||||
|
if !checksum_match {
|
||||||
return Err(ManualTransitionJobError::ChecksumMismatch);
|
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||||
}
|
}
|
||||||
if job.job_id != expected_job_id {
|
if job.job_id != expected_job_id {
|
||||||
@@ -449,6 +483,18 @@ pub enum ManualTransitionWorkerResult {
|
|||||||
TierFailure,
|
TierFailure,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum ManualTransitionWorkerFailureReason {
|
||||||
|
Unknown,
|
||||||
|
NotFound,
|
||||||
|
Network,
|
||||||
|
PermissionDenied,
|
||||||
|
Timeout,
|
||||||
|
Quorum,
|
||||||
|
SlowDown,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct ManualTransitionTaskRecord {
|
pub struct ManualTransitionTaskRecord {
|
||||||
@@ -554,17 +600,22 @@ struct PersistedManualTransitionTaskRecord {
|
|||||||
record: ManualTransitionTaskRecord,
|
record: ManualTransitionTaskRecord,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||||
pub struct ManualTransitionWorkerResultStats {
|
pub struct ManualTransitionWorkerResultStats {
|
||||||
pub completed: u64,
|
pub completed: u64,
|
||||||
pub failed: u64,
|
pub failed: u64,
|
||||||
|
pub tier_failure_by_reason: BTreeMap<ManualTransitionWorkerFailureReason, u64>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ManualTransitionWorkerResultStats {
|
impl ManualTransitionWorkerResultStats {
|
||||||
fn record(&mut self, result: ManualTransitionWorkerResult) {
|
fn record(&mut self, result: ManualTransitionWorkerResult, failure_reason: Option<ManualTransitionWorkerFailureReason>) {
|
||||||
match result {
|
match result {
|
||||||
ManualTransitionWorkerResult::Completed => self.completed = self.completed.saturating_add(1),
|
ManualTransitionWorkerResult::Completed => self.completed = self.completed.saturating_add(1),
|
||||||
ManualTransitionWorkerResult::TierFailure => self.failed = self.failed.saturating_add(1),
|
ManualTransitionWorkerResult::TierFailure => {
|
||||||
|
self.failed = self.failed.saturating_add(1);
|
||||||
|
let reason = failure_reason.unwrap_or(ManualTransitionWorkerFailureReason::Unknown);
|
||||||
|
*self.tier_failure_by_reason.entry(reason).or_insert(0) += 1;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -576,16 +627,28 @@ pub struct ManualTransitionWorkerResultRecord {
|
|||||||
pub job_id: Uuid,
|
pub job_id: Uuid,
|
||||||
pub task_key: String,
|
pub task_key: String,
|
||||||
pub result: ManualTransitionWorkerResult,
|
pub result: ManualTransitionWorkerResult,
|
||||||
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
|
pub failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||||
pub completed_at_unix_nanos: i128,
|
pub completed_at_unix_nanos: i128,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl ManualTransitionWorkerResultRecord {
|
impl ManualTransitionWorkerResultRecord {
|
||||||
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
pub fn new(job_id: Uuid, task_key: impl Into<String>, result: ManualTransitionWorkerResult) -> Self {
|
||||||
|
Self::new_with_reason(job_id, task_key, result, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn new_with_reason(
|
||||||
|
job_id: Uuid,
|
||||||
|
task_key: impl Into<String>,
|
||||||
|
result: ManualTransitionWorkerResult,
|
||||||
|
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||||
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
schema: MANUAL_TRANSITION_WORKER_RESULT_SCHEMA.to_string(),
|
schema: MANUAL_TRANSITION_WORKER_RESULT_SCHEMA.to_string(),
|
||||||
job_id,
|
job_id,
|
||||||
task_key: task_key.into(),
|
task_key: task_key.into(),
|
||||||
result,
|
result,
|
||||||
|
failure_reason,
|
||||||
completed_at_unix_nanos: OffsetDateTime::now_utc().unix_timestamp_nanos(),
|
completed_at_unix_nanos: OffsetDateTime::now_utc().unix_timestamp_nanos(),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -620,9 +683,23 @@ impl ManualTransitionWorkerResultRecord {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
let record = persisted.record;
|
let record = persisted.record;
|
||||||
let record_bytes = serde_json::to_vec(&record)?;
|
let mut record_bytes = serde_json::to_vec(&record)?;
|
||||||
let actual_checksum = hex_sha256(&record_bytes, ToOwned::to_owned);
|
let actual_checksum = hex_sha256(&record_bytes, ToOwned::to_owned);
|
||||||
if persisted.content_sha256 != actual_checksum {
|
let checksum_match = if persisted.content_sha256 == actual_checksum {
|
||||||
|
true
|
||||||
|
} else {
|
||||||
|
if let Some(record_value) = serde_json::to_value(&record)?.as_object_mut() {
|
||||||
|
if record.failure_reason.is_none() {
|
||||||
|
record_value.remove("failure_reason");
|
||||||
|
}
|
||||||
|
record_bytes = serde_json::to_vec(record_value)?;
|
||||||
|
} else {
|
||||||
|
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||||
|
}
|
||||||
|
let fallback_checksum = hex_sha256(&record_bytes, ToOwned::to_owned);
|
||||||
|
persisted.content_sha256 == fallback_checksum
|
||||||
|
};
|
||||||
|
if !checksum_match {
|
||||||
return Err(ManualTransitionJobError::ChecksumMismatch);
|
return Err(ManualTransitionJobError::ChecksumMismatch);
|
||||||
}
|
}
|
||||||
if record.job_id != expected_job_id {
|
if record.job_id != expected_job_id {
|
||||||
@@ -1120,7 +1197,7 @@ async fn scan_manual_transition_worker_result_journal(
|
|||||||
Ok(result) => result,
|
Ok(result) => result,
|
||||||
Err(err) => return Ok(ManualTransitionWorkerResultJournal::Corrupt(err.to_string())),
|
Err(err) => return Ok(ManualTransitionWorkerResultJournal::Corrupt(err.to_string())),
|
||||||
};
|
};
|
||||||
stats.record(result.result);
|
stats.record(result.result, result.failure_reason);
|
||||||
}
|
}
|
||||||
if !page.is_truncated {
|
if !page.is_truncated {
|
||||||
return Ok(ManualTransitionWorkerResultJournal::Stats(stats));
|
return Ok(ManualTransitionWorkerResultJournal::Stats(stats));
|
||||||
@@ -1151,7 +1228,13 @@ pub async fn reconcile_manual_transition_worker_results(
|
|||||||
};
|
};
|
||||||
for _ in 0..4 {
|
for _ in 0..4 {
|
||||||
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
let (mut record, etag) = load_manual_transition_job_record_with_etag(api.clone(), job_id).await?;
|
||||||
let changed = record.apply_worker_result_counts(stats.completed, stats.failed, task_stats.queued, queue_snapshot);
|
let changed = record.apply_worker_result_counts(
|
||||||
|
stats.completed,
|
||||||
|
stats.failed,
|
||||||
|
&stats.tier_failure_by_reason,
|
||||||
|
task_stats.queued,
|
||||||
|
queue_snapshot,
|
||||||
|
);
|
||||||
if !changed {
|
if !changed {
|
||||||
return Ok(record);
|
return Ok(record);
|
||||||
}
|
}
|
||||||
@@ -1446,7 +1529,18 @@ pub async fn record_manual_transition_worker_result(
|
|||||||
result: ManualTransitionWorkerResult,
|
result: ManualTransitionWorkerResult,
|
||||||
queue_snapshot: ManualTransitionQueueSnapshot,
|
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||||
) -> EcstoreResult<ManualTransitionJobRecord> {
|
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||||
let result_record = ManualTransitionWorkerResultRecord::new(job_id, task_key, result);
|
record_manual_transition_worker_result_with_reason(api, job_id, task_key, result, queue_snapshot, None).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn record_manual_transition_worker_result_with_reason(
|
||||||
|
api: Arc<ECStore>,
|
||||||
|
job_id: Uuid,
|
||||||
|
task_key: &str,
|
||||||
|
result: ManualTransitionWorkerResult,
|
||||||
|
queue_snapshot: ManualTransitionQueueSnapshot,
|
||||||
|
failure_reason: Option<ManualTransitionWorkerFailureReason>,
|
||||||
|
) -> EcstoreResult<ManualTransitionJobRecord> {
|
||||||
|
let result_record = ManualTransitionWorkerResultRecord::new_with_reason(job_id, task_key, result, failure_reason);
|
||||||
if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
|
if !save_manual_transition_worker_result_if_absent(api.clone(), &result_record).await? {
|
||||||
return load_manual_transition_job_record(api, job_id).await;
|
return load_manual_transition_job_record(api, job_id).await;
|
||||||
}
|
}
|
||||||
@@ -1456,7 +1550,7 @@ pub async fn record_manual_transition_worker_result(
|
|||||||
if record.is_terminal() {
|
if record.is_terminal() {
|
||||||
return Ok(record);
|
return Ok(record);
|
||||||
}
|
}
|
||||||
record.record_worker_result(result, queue_snapshot);
|
record.record_worker_result_with_reason(result, queue_snapshot, failure_reason);
|
||||||
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
match save_manual_transition_job_record_if_current(api.clone(), &record, &etag).await {
|
||||||
Ok(()) => {
|
Ok(()) => {
|
||||||
if record.is_terminal() {
|
if record.is_terminal() {
|
||||||
@@ -1652,6 +1746,52 @@ mod tests {
|
|||||||
assert_eq!(record.report.tier_failure, 1);
|
assert_eq!(record.report.tier_failure, 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manual_transition_job_record_records_worker_failure_reasons() {
|
||||||
|
let options = ManualTransitionRunOptions::default();
|
||||||
|
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||||
|
|
||||||
|
record.record_worker_result_with_reason(
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
Some(ManualTransitionWorkerFailureReason::NotFound),
|
||||||
|
);
|
||||||
|
record.record_worker_result_with_reason(
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
Some(ManualTransitionWorkerFailureReason::Network),
|
||||||
|
);
|
||||||
|
record.record_worker_result_with_reason(
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
None,
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(record.report.transition_failed, 3);
|
||||||
|
assert_eq!(record.report.tier_failure, 3);
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.report
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::NotFound),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.report
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::Network),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.report
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::Unknown),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn manual_transition_job_cancel_recovery_finishes_after_worker_drain() {
|
fn manual_transition_job_cancel_recovery_finishes_after_worker_drain() {
|
||||||
let options = ManualTransitionRunOptions::default();
|
let options = ManualTransitionRunOptions::default();
|
||||||
@@ -1709,6 +1849,67 @@ mod tests {
|
|||||||
assert_eq!(record.report.tier_failure, 3);
|
assert_eq!(record.report.tier_failure, 3);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manual_transition_job_scan_progress_preserves_worker_failure_reasons() {
|
||||||
|
let options = ManualTransitionRunOptions::default();
|
||||||
|
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||||
|
record.record_worker_result_with_reason(
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
Some(ManualTransitionWorkerFailureReason::NotFound),
|
||||||
|
);
|
||||||
|
|
||||||
|
record.update_running_progress(
|
||||||
|
ManualTransitionRunReport {
|
||||||
|
bucket: "bucket".to_string(),
|
||||||
|
scanned: 3,
|
||||||
|
enqueued: 1,
|
||||||
|
tier_failure: 2,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.report
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::NotFound),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manual_transition_job_apply_worker_result_counts_preserves_existing_failure_reasons() {
|
||||||
|
let options = ManualTransitionRunOptions::default();
|
||||||
|
let mut record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||||
|
record.record_worker_result_with_reason(
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
ManualTransitionQueueSnapshot::default(),
|
||||||
|
Some(ManualTransitionWorkerFailureReason::NotFound),
|
||||||
|
);
|
||||||
|
|
||||||
|
let mut failure_reasons = BTreeMap::new();
|
||||||
|
failure_reasons.insert(ManualTransitionWorkerFailureReason::PermissionDenied, 2);
|
||||||
|
|
||||||
|
record.apply_worker_result_counts(0, 1, &failure_reasons, 1, ManualTransitionQueueSnapshot::default());
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.report
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::NotFound),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
record
|
||||||
|
.report
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||||
|
Some(&2)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn manual_transition_job_unknown_checkpoint_persists_counters_and_cursor() {
|
fn manual_transition_job_unknown_checkpoint_persists_counters_and_cursor() {
|
||||||
let options = ManualTransitionRunOptions {
|
let options = ManualTransitionRunOptions {
|
||||||
@@ -1955,6 +2156,62 @@ mod tests {
|
|||||||
assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch));
|
assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manual_transition_worker_result_record_round_trips_without_failure_reason() {
|
||||||
|
let job_id = Uuid::new_v4();
|
||||||
|
let task_key = manual_transition_worker_result_task_key("bucket", "logs/a", None);
|
||||||
|
let record = ManualTransitionWorkerResultRecord::new_with_reason(
|
||||||
|
job_id,
|
||||||
|
&task_key,
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
Some(ManualTransitionWorkerFailureReason::Network),
|
||||||
|
);
|
||||||
|
let encoded = record.encode().expect("worker result record should encode");
|
||||||
|
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded worker result should be json");
|
||||||
|
let record_obj = value["record"].as_object_mut().expect("record object should be present");
|
||||||
|
record_obj.remove("failure_reason");
|
||||||
|
let record_bytes = serde_json::to_vec(&value["record"]).expect("record without reason should encode");
|
||||||
|
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
|
||||||
|
let stripped = serde_json::to_vec(&value).expect("stripped worker result should encode");
|
||||||
|
|
||||||
|
let decoded = ManualTransitionWorkerResultRecord::decode(job_id, &task_key, &stripped)
|
||||||
|
.expect("worker result without reason should decode");
|
||||||
|
|
||||||
|
assert_eq!(decoded.failure_reason, None);
|
||||||
|
assert_eq!(decoded.result, ManualTransitionWorkerResult::TierFailure);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manual_transition_worker_result_stats_aggregates_reasons() {
|
||||||
|
let mut stats = ManualTransitionWorkerResultStats::default();
|
||||||
|
|
||||||
|
stats.record(ManualTransitionWorkerResult::Completed, None);
|
||||||
|
stats.record(
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
Some(ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||||
|
);
|
||||||
|
stats.record(
|
||||||
|
ManualTransitionWorkerResult::TierFailure,
|
||||||
|
Some(ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||||
|
);
|
||||||
|
stats.record(ManualTransitionWorkerResult::TierFailure, None);
|
||||||
|
|
||||||
|
assert_eq!(stats.completed, 1);
|
||||||
|
assert_eq!(stats.failed, 3);
|
||||||
|
assert_eq!(
|
||||||
|
stats
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::PermissionDenied),
|
||||||
|
Some(&2)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
stats
|
||||||
|
.tier_failure_by_reason
|
||||||
|
.get(&ManualTransitionWorkerFailureReason::Unknown),
|
||||||
|
Some(&1)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn manual_transition_task_record_rejects_checksum_drift() {
|
fn manual_transition_task_record_rejects_checksum_drift() {
|
||||||
let job_id = Uuid::new_v4();
|
let job_id = Uuid::new_v4();
|
||||||
@@ -1971,6 +2228,24 @@ mod tests {
|
|||||||
assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch));
|
assert!(matches!(err, ManualTransitionJobError::ChecksumMismatch));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn manual_transition_job_record_round_trips_legacy_report_without_failure_reason_map() {
|
||||||
|
let options = ManualTransitionRunOptions::default();
|
||||||
|
let record = ManualTransitionJobRecord::new(Uuid::new_v4(), "bucket", &options, TEST_OWNER);
|
||||||
|
let encoded = record.encode().expect("job record should encode");
|
||||||
|
let mut value: serde_json::Value = serde_json::from_slice(&encoded).expect("encoded job should be json");
|
||||||
|
let report = value["job"]["report"].as_object_mut().expect("report should be object");
|
||||||
|
report.remove("tier_failure_by_reason");
|
||||||
|
let record_bytes = serde_json::to_vec(&value["job"]).expect("job without report failure map should encode");
|
||||||
|
value["content_sha256"] = serde_json::Value::String(hex_sha256(&record_bytes, ToOwned::to_owned));
|
||||||
|
let legacy = serde_json::to_vec(&value).expect("job without report failure map should encode");
|
||||||
|
|
||||||
|
let decoded = ManualTransitionJobRecord::decode(record.job_id, &legacy).expect("legacy job report should decode");
|
||||||
|
|
||||||
|
assert_eq!(decoded.state, ManualTransitionJobState::Running);
|
||||||
|
assert!(decoded.report.tier_failure_by_reason.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn manual_transition_job_record_rejects_unknown_report_fields() {
|
fn manual_transition_job_record_rejects_unknown_report_fields() {
|
||||||
let options = ManualTransitionRunOptions::default();
|
let options = ManualTransitionRunOptions::default();
|
||||||
|
|||||||
Executable
+253
@@ -0,0 +1,253 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# Copyright 2026 RustFS Team
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
ENDPOINT=""
|
||||||
|
JOB_ID=""
|
||||||
|
ADMIN_TOKEN=""
|
||||||
|
LOG_GLOB="/var/log/rustfs/*.log"
|
||||||
|
METRIC_TYPES="1"
|
||||||
|
METRIC_SAMPLES="1"
|
||||||
|
METRIC_INTERVAL=""
|
||||||
|
BUCKET_FILTER=""
|
||||||
|
SHOW_HELP=false
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/manual_transition_debug.sh --endpoint <admin-api-base> --job-id <job-id> [options]
|
||||||
|
|
||||||
|
Required:
|
||||||
|
--endpoint Admin endpoint base, e.g. http://127.0.0.1:9000
|
||||||
|
--job-id Manual transition job_id (UUID)
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
--admin-token Bearer token for /admin/v3 endpoints
|
||||||
|
--log-glob Log glob to scan, defaults to /var/log/rustfs/*.log
|
||||||
|
--metric-types Metrics types bitmask, defaults to 1 (SCANNER)
|
||||||
|
--metric-samples Metrics samples, defaults to 1
|
||||||
|
--metric-interval Metrics interval parameter, passed as-is to query string
|
||||||
|
--bucket Optional bucket filter for focused log output
|
||||||
|
-h, --help Show this help
|
||||||
|
|
||||||
|
Output sections:
|
||||||
|
- lifecycle_worker_state structured events (job-scoped)
|
||||||
|
- manual-transition admin job status
|
||||||
|
- lifecycle_tier_operation_failed and admin_ilm_transition_state traces
|
||||||
|
- metrics snapshot (aggregated.scanner.lifecycle_transition)
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
require_cmd() {
|
||||||
|
local cmd="$1"
|
||||||
|
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: required command not found: $cmd" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
api_get() {
|
||||||
|
local path="$1"
|
||||||
|
local url="${ENDPOINT%/}${path}"
|
||||||
|
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url"
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
|
||||||
|
curl -sS "$url"
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--endpoint)
|
||||||
|
ENDPOINT="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--job-id)
|
||||||
|
JOB_ID="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--admin-token)
|
||||||
|
ADMIN_TOKEN="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--log-glob)
|
||||||
|
LOG_GLOB="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--metric-types)
|
||||||
|
METRIC_TYPES="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--metric-samples)
|
||||||
|
METRIC_SAMPLES="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--metric-interval)
|
||||||
|
METRIC_INTERVAL="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
--bucket)
|
||||||
|
BUCKET_FILTER="${2:-}"
|
||||||
|
shift 2
|
||||||
|
;;
|
||||||
|
-h|--help)
|
||||||
|
SHOW_HELP=true
|
||||||
|
shift
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: unknown arg: $1" >&2
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
validate() {
|
||||||
|
if [[ "$SHOW_HELP" == true ]]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$ENDPOINT" || -z "$JOB_ID" ]]; then
|
||||||
|
echo "ERROR: --endpoint and --job-id are required" >&2
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -z "$LOG_GLOB" ]]; then
|
||||||
|
echo "ERROR: --log-glob must not be empty" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
build_admin_metrics_path() {
|
||||||
|
local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}"
|
||||||
|
if [[ -n "$METRIC_INTERVAL" ]]; then
|
||||||
|
path="${path}&interval=${METRIC_INTERVAL}"
|
||||||
|
fi
|
||||||
|
printf '%s' "$path"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_json_stream() {
|
||||||
|
local query="$1"
|
||||||
|
local filter="$2"
|
||||||
|
local log_lines; log_lines="$(mktemp)"
|
||||||
|
local log_errors; log_errors="$(mktemp)"
|
||||||
|
local log_paths=()
|
||||||
|
local match
|
||||||
|
|
||||||
|
while IFS= read -r match; do
|
||||||
|
[[ -n "$match" ]] && log_paths+=("$match")
|
||||||
|
done < <(compgen -G "$LOG_GLOB" || true)
|
||||||
|
if [[ "${#log_paths[@]}" -eq 0 ]]; then
|
||||||
|
log_paths=("$LOG_GLOB")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -s "$log_errors" ]]; then
|
||||||
|
sed 's/^/DEBUG: /' "$log_errors"
|
||||||
|
fi
|
||||||
|
|
||||||
|
jq -R "$filter" <"$log_lines"
|
||||||
|
|
||||||
|
rm -f "$log_lines" "$log_errors"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_worker_logs() {
|
||||||
|
local base_filter='fromjson? | select(.event == "lifecycle_worker_state")'
|
||||||
|
local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))'
|
||||||
|
local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state'
|
||||||
|
|
||||||
|
if [[ -n "$BUCKET_FILTER" ]]; then
|
||||||
|
manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER)'
|
||||||
|
dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.BUCKET_FILTER) | .state'
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "## lifecycle_worker_state (all)"
|
||||||
|
run_json_stream "lifecycle_worker_state" "$base_filter"
|
||||||
|
|
||||||
|
echo "## lifecycle_worker_state (manual_transition + job filter: ${JOB_ID})"
|
||||||
|
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||||
|
run_json_stream "lifecycle_worker_state" "$manual_filter" || true
|
||||||
|
|
||||||
|
echo "## lifecycle_worker_state state distribution (manual_transition only)"
|
||||||
|
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||||
|
run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true
|
||||||
|
}
|
||||||
|
|
||||||
|
run_admin_job() {
|
||||||
|
echo "## manual transition job status"
|
||||||
|
api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \
|
||||||
|
| jq '{status, mode, job_id, failure_reason, report: .report, queue_snapshot: .queue_snapshot}'
|
||||||
|
}
|
||||||
|
|
||||||
|
run_tier_operation_failures() {
|
||||||
|
echo "## lifecycle_tier_operation_failed events"
|
||||||
|
local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||||
|
if [[ -n "$BUCKET_FILTER" ]]; then
|
||||||
|
filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||||
|
fi
|
||||||
|
|
||||||
|
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||||
|
run_json_stream "lifecycle_tier_operation_failed" "$filter" || true
|
||||||
|
}
|
||||||
|
|
||||||
|
run_admin_events() {
|
||||||
|
echo "## admin_ilm_transition_state (job filter)"
|
||||||
|
local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||||
|
if [[ -n "$BUCKET_FILTER" ]]; then
|
||||||
|
filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||||
|
fi
|
||||||
|
|
||||||
|
JOB_ID="$JOB_ID" BUCKET_FILTER="$BUCKET_FILTER" \
|
||||||
|
run_json_stream "admin_ilm_transition_state" "$filter" || true
|
||||||
|
}
|
||||||
|
|
||||||
|
run_metrics() {
|
||||||
|
echo "## metrics: scanner.lifecycle_transition"
|
||||||
|
local path
|
||||||
|
path="$(build_admin_metrics_path)"
|
||||||
|
api_get "$path" | jq '.aggregated.scanner.lifecycle_transition'
|
||||||
|
|
||||||
|
echo "## metrics: scanner lifecycle cycle"
|
||||||
|
api_get "$path" | jq '.aggregated.scanner | {lifecycle_transition: .lifecycle_transition, current_cycle_ilm_actions: .current_cycle_ilm_actions, current_cycle_lifecycle_transition_actions: .current_cycle_lifecycle_transition_actions}'
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
parse_args "$@"
|
||||||
|
validate
|
||||||
|
|
||||||
|
require_cmd curl
|
||||||
|
require_cmd jq
|
||||||
|
require_cmd rg
|
||||||
|
|
||||||
|
run_worker_logs
|
||||||
|
run_admin_events
|
||||||
|
run_tier_operation_failures
|
||||||
|
run_admin_job
|
||||||
|
run_metrics
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+590
@@ -0,0 +1,590 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Copyright 2026 RustFS Team
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
|
|
||||||
|
ENDPOINT=""
|
||||||
|
JOB_ID=""
|
||||||
|
ADMIN_TOKEN=""
|
||||||
|
LOG_GLOB="/var/log/rustfs/*.log"
|
||||||
|
METRIC_TYPES="1"
|
||||||
|
METRIC_SAMPLES="1"
|
||||||
|
METRIC_INTERVAL=""
|
||||||
|
TASK_BUCKET_FILTER=""
|
||||||
|
OUT_DIR=""
|
||||||
|
|
||||||
|
SYS_BUCKET="${RUSTFS_META_BUCKET:-.rustfs.sys}"
|
||||||
|
TASK_PREFIX="ilm/manual-transition/tasks"
|
||||||
|
RESULT_PREFIX="ilm/manual-transition/results"
|
||||||
|
|
||||||
|
MC_BIN="${MC_BIN:-mc}"
|
||||||
|
MC_ALIAS=""
|
||||||
|
MC_ENDPOINT=""
|
||||||
|
MC_ACCESS_KEY=""
|
||||||
|
MC_SECRET_KEY=""
|
||||||
|
MC_ALIAS_CREATED=false
|
||||||
|
|
||||||
|
SHOW_HELP=false
|
||||||
|
DRY_RUN=false
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/manual_transition_journal_audit.sh --endpoint <admin-api> --job-id <job-uuid> [options]
|
||||||
|
|
||||||
|
Required:
|
||||||
|
--endpoint Admin endpoint base, e.g. https://127.0.0.1:9000
|
||||||
|
--job-id Manual transition job_id
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
--admin-token Bearer token for /admin/v3 endpoints
|
||||||
|
--log-glob Log glob, default /var/log/rustfs/*.log
|
||||||
|
--metric-types /admin/v3/metrics type flags, default 1
|
||||||
|
--metric-samples /admin/v3/metrics sample count, default 1
|
||||||
|
--metric-interval /admin/v3/metrics interval query param
|
||||||
|
--bucket-filter Optional bucket filter for logs
|
||||||
|
--sys-bucket Meta bucket for journal read (default .rustfs.sys)
|
||||||
|
--mc-alias Reuse existing mc alias (skip alias set)
|
||||||
|
--mc-endpoint mc endpoint for .rustfs.sys access
|
||||||
|
--mc-access-key mc access key
|
||||||
|
--mc-secret-key mc secret key
|
||||||
|
--out-dir Artifact output dir
|
||||||
|
--dry-run
|
||||||
|
--help Show usage
|
||||||
|
|
||||||
|
Output:
|
||||||
|
- Journal CSV/JSON samples for manual transition tasks and worker results
|
||||||
|
- Reconcile report (task_count/result_count/missing/mismatch)
|
||||||
|
- /admin/v3 lifecycle_worker_state / admin_ilm_transition_state events
|
||||||
|
- lifecycle_tier_operation_failed logs
|
||||||
|
- aggregated lifecycle transition metrics (admin endpoint sample)
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
require_bash4() {
|
||||||
|
if (( BASH_VERSINFO[0] < 4 )); then
|
||||||
|
echo "ERROR: manual transition journal audit requires bash >= 4.0" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
arg_value() {
|
||||||
|
local flag="$1"
|
||||||
|
local value="${2:-}"
|
||||||
|
if [[ -z "$value" || "$value" == --* ]]; then
|
||||||
|
echo "ERROR: missing value for $flag" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
require_cmd() {
|
||||||
|
local cmd="$1"
|
||||||
|
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: command not found: $cmd" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
require_nonempty() {
|
||||||
|
local value="$1"
|
||||||
|
local label="$2"
|
||||||
|
if [[ -z "$value" ]]; then
|
||||||
|
echo "ERROR: $label is required" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
normalize_uuid() {
|
||||||
|
local raw="${1//-/}"
|
||||||
|
printf '%s' "${raw,,}"
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_uuid() {
|
||||||
|
local normalized
|
||||||
|
normalized="$(normalize_uuid "$1")"
|
||||||
|
if ! [[ "$normalized" =~ ^[0-9a-f]{32}$ ]]; then
|
||||||
|
echo "ERROR: invalid uuid (expect UUIDv4): $1" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf '%s' "$normalized"
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--job-id) JOB_ID="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--log-glob) LOG_GLOB="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--metric-types) METRIC_TYPES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--metric-samples) METRIC_SAMPLES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--metric-interval) METRIC_INTERVAL="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--bucket-filter) TASK_BUCKET_FILTER="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--sys-bucket) SYS_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--mc-alias) MC_ALIAS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--mc-endpoint) MC_ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--mc-access-key) MC_ACCESS_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--mc-secret-key) MC_SECRET_KEY="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--dry-run) DRY_RUN=true; shift ;;
|
||||||
|
-h|--help) SHOW_HELP=true; shift ;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: unknown arg: $1" >&2
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
validate_args() {
|
||||||
|
if [[ "$SHOW_HELP" == true ]]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
require_nonempty "$ENDPOINT" "--endpoint"
|
||||||
|
require_nonempty "$JOB_ID" "--job-id"
|
||||||
|
|
||||||
|
JOB_ID="$(validate_uuid "$JOB_ID")"
|
||||||
|
if [[ "$LOG_GLOB" == "" ]]; then
|
||||||
|
echo "ERROR: --log-glob must not be empty" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ -z "$MC_ALIAS" ]]; then
|
||||||
|
if [[ -z "$MC_ENDPOINT" || -z "$MC_ACCESS_KEY" || -z "$MC_SECRET_KEY" ]]; then
|
||||||
|
echo "ERROR: either --mc-alias or (--mc-endpoint + --mc-access-key + --mc-secret-key) is required" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanup_mc_alias() {
|
||||||
|
if [[ "$MC_ALIAS_CREATED" == true ]]; then
|
||||||
|
"${MC_BIN}" alias remove "$MC_ALIAS" >/dev/null 2>&1 || true
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_output() {
|
||||||
|
if [[ -z "$OUT_DIR" ]]; then
|
||||||
|
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-journal-audit/$(date +%Y%m%dT%H%M%S)"
|
||||||
|
fi
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
SUMMARY_TXT="${OUT_DIR}/audit_summary.txt"
|
||||||
|
TASKS_CSV="${OUT_DIR}/tasks.csv"
|
||||||
|
RESULTS_CSV="${OUT_DIR}/results.csv"
|
||||||
|
RECONCILE_CSV="${OUT_DIR}/journal_reconcile.csv"
|
||||||
|
COMMANDS_TXT="${OUT_DIR}/commands.txt"
|
||||||
|
{
|
||||||
|
echo "task_key,bucket,object,version_id,storage_class,queued_at_unix_nanos,task_object"
|
||||||
|
} > "$TASKS_CSV"
|
||||||
|
{
|
||||||
|
echo "task_key,result,completion_reason,completed_at_unix_nanos,result_object"
|
||||||
|
} > "$RESULTS_CSV"
|
||||||
|
{
|
||||||
|
echo "task_key,bucket,object,version_id,storage_class,task_has_result,result,result_reason,task_job_id_match,result_job_id_match,task_path,result_path"
|
||||||
|
} > "$RECONCILE_CSV"
|
||||||
|
}
|
||||||
|
|
||||||
|
setup_mc_alias() {
|
||||||
|
if [[ "$MC_ALIAS" == "" ]]; then
|
||||||
|
MC_ALIAS="mt-journal-audit-${JOB_ID}"
|
||||||
|
MC_ALIAS_CREATED=true
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "[DRY-RUN] mc alias set ${MC_ALIAS} ${MC_ENDPOINT} ${MC_ACCESS_KEY} ***"
|
||||||
|
else
|
||||||
|
"${MC_BIN}" alias set "${MC_ALIAS}" "${MC_ENDPOINT}" "${MC_ACCESS_KEY}" "${MC_SECRET_KEY}" >/dev/null
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
if ! [[ "$DRY_RUN" == true ]]; then
|
||||||
|
"${MC_BIN}" ls "${MC_ALIAS}/${SYS_BUCKET}" >/dev/null
|
||||||
|
fi
|
||||||
|
trap cleanup_mc_alias EXIT
|
||||||
|
}
|
||||||
|
|
||||||
|
api_get() {
|
||||||
|
local path="$1"
|
||||||
|
local url="${ENDPOINT%/}${path}"
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
curl -sS -H "Authorization: Bearer ${ADMIN_TOKEN}" "$url"
|
||||||
|
else
|
||||||
|
curl -sS "$url"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
build_admin_metrics_path() {
|
||||||
|
local path="/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES}"
|
||||||
|
if [[ -n "$METRIC_INTERVAL" ]]; then
|
||||||
|
path="${path}&interval=${METRIC_INTERVAL}"
|
||||||
|
fi
|
||||||
|
printf '%s' "$path"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_json_stream() {
|
||||||
|
local query="$1"
|
||||||
|
local filter="$2"
|
||||||
|
local log_lines; log_lines="$(mktemp)"
|
||||||
|
local log_errors; log_errors="$(mktemp)"
|
||||||
|
local log_paths=()
|
||||||
|
local match
|
||||||
|
|
||||||
|
while IFS= read -r match; do
|
||||||
|
[[ -n "$match" ]] && log_paths+=("$match")
|
||||||
|
done < <(compgen -G "$LOG_GLOB" || true)
|
||||||
|
if [[ "${#log_paths[@]}" -eq 0 ]]; then
|
||||||
|
log_paths=("$LOG_GLOB")
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! rg -n --color=never -- "$query" -- "${log_paths[@]}" >"$log_lines" 2>"$log_errors"; then
|
||||||
|
:
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -s "$log_errors" ]]; then
|
||||||
|
sed 's/^/DEBUG: /' "$log_errors"
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ -s "$log_lines" ]]; then
|
||||||
|
jq -R "$filter" <"$log_lines"
|
||||||
|
else
|
||||||
|
echo "[]"
|
||||||
|
fi
|
||||||
|
|
||||||
|
rm -f "$log_lines" "$log_errors"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_worker_logs() {
|
||||||
|
local base_filter='fromjson? | select(.event == "lifecycle_worker_state")'
|
||||||
|
local manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")))'
|
||||||
|
local dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) ) | .state'
|
||||||
|
|
||||||
|
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
|
||||||
|
manual_filter='fromjson? | select(.event == "lifecycle_worker_state" and .job_id == env.JOB_ID and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER)'
|
||||||
|
dist_filter='fromjson? | select(.event == "lifecycle_worker_state" and (.state | startswith("manual_transition_")) and .bucket == env.TASK_BUCKET_FILTER) | .state'
|
||||||
|
fi
|
||||||
|
|
||||||
|
echo "## lifecycle_worker_state all"
|
||||||
|
run_json_stream "lifecycle_worker_state" "$base_filter" || true
|
||||||
|
echo
|
||||||
|
echo "## lifecycle_worker_state manual_transition + job filter"
|
||||||
|
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$manual_filter" || true
|
||||||
|
echo
|
||||||
|
echo "## lifecycle_worker_state state distribution"
|
||||||
|
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" run_json_stream "lifecycle_worker_state" "$dist_filter" | jq -r . | sort | uniq -c | sort -nr || true
|
||||||
|
}
|
||||||
|
|
||||||
|
run_admin_job() {
|
||||||
|
echo "## manual transition job status"
|
||||||
|
api_get "/rustfs/admin/v3/ilm/transition/jobs/${JOB_ID}" \
|
||||||
|
| jq '{status, mode, job_id, report: .report, queue_snapshot: .queue_snapshot, cancel_requested}'
|
||||||
|
}
|
||||||
|
|
||||||
|
run_admin_events() {
|
||||||
|
echo "## admin_ilm_transition_state events"
|
||||||
|
local filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||||
|
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
|
||||||
|
filter='fromjson? | select(.event == "admin_ilm_transition_state" and .job_id == env.JOB_ID and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, operation: .operation, state: .state, result: .result, failure_reason: .failure_reason, bucket: .bucket, prefix: .prefix}'
|
||||||
|
fi
|
||||||
|
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \
|
||||||
|
run_json_stream "admin_ilm_transition_state" "$filter" || true
|
||||||
|
}
|
||||||
|
|
||||||
|
run_tier_operation_failures() {
|
||||||
|
echo "## lifecycle_tier_operation_failed events"
|
||||||
|
local filter='fromjson? | select(.event == "lifecycle_tier_operation_failed") | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||||
|
if [[ -n "$TASK_BUCKET_FILTER" ]]; then
|
||||||
|
filter='fromjson? | select(.event == "lifecycle_tier_operation_failed" and .bucket == env.TASK_BUCKET_FILTER) | {timestamp, bucket: .bucket, object: .object, version_id: .version_id, tier: .tier, operation: .operation, error: .error}'
|
||||||
|
fi
|
||||||
|
JOB_ID="$JOB_ID" TASK_BUCKET_FILTER="$TASK_BUCKET_FILTER" \
|
||||||
|
run_json_stream "lifecycle_tier_operation_failed" "$filter" || true
|
||||||
|
}
|
||||||
|
|
||||||
|
run_metrics() {
|
||||||
|
echo "## metrics: scanner.lifecycle_transition"
|
||||||
|
local path
|
||||||
|
path="$(build_admin_metrics_path)"
|
||||||
|
api_get "$path" | jq '.aggregated.scanner.lifecycle_transition'
|
||||||
|
}
|
||||||
|
|
||||||
|
journal_key_prefix() {
|
||||||
|
local prefix="$1"
|
||||||
|
local shard_a="${JOB_ID:0:2}"
|
||||||
|
local shard_b="${JOB_ID:2:2}"
|
||||||
|
printf '%s/%s/%s/%s' "$prefix" "$shard_a" "$shard_b" "$JOB_ID"
|
||||||
|
}
|
||||||
|
|
||||||
|
csv_escape() {
|
||||||
|
local value="${1:-}"
|
||||||
|
value="${value//\"/\"\"}"
|
||||||
|
printf '\"%s\"' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
read_json_file() {
|
||||||
|
local object_key="$1"
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo ""
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
"${MC_BIN}" cat "${MC_ALIAS}/${SYS_BUCKET}/${object_key}"
|
||||||
|
}
|
||||||
|
|
||||||
|
collect_journals() {
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "[DRY-RUN] skip .rustfs.sys object reads."
|
||||||
|
echo "dry_run=true" > "$SUMMARY_TXT"
|
||||||
|
return 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
local task_prefix result_prefix
|
||||||
|
task_prefix="$(journal_key_prefix "$TASK_PREFIX")"
|
||||||
|
result_prefix="$(journal_key_prefix "$RESULT_PREFIX")"
|
||||||
|
|
||||||
|
local task_objs
|
||||||
|
local result_objs
|
||||||
|
local object_name
|
||||||
|
local task_key
|
||||||
|
local raw
|
||||||
|
local record_task_job record_result_job
|
||||||
|
local expected_job
|
||||||
|
expected_job="$JOB_ID"
|
||||||
|
|
||||||
|
local -A task_bucket task_object task_version task_storage task_queued task_job task_path
|
||||||
|
local -A result_status result_reason result_completed result_job result_path
|
||||||
|
local -A task_seen
|
||||||
|
local -A result_seen
|
||||||
|
|
||||||
|
task_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${task_prefix}" | jq -r 'select(.type == "file") | .key')"
|
||||||
|
result_objs="$("${MC_BIN}" --json ls --recursive "${MC_ALIAS}/${SYS_BUCKET}/${result_prefix}" | jq -r 'select(.type == "file") | .key')"
|
||||||
|
|
||||||
|
while IFS= read -r object_name; do
|
||||||
|
[[ -z "$object_name" ]] && continue
|
||||||
|
task_key="${object_name##*/}"
|
||||||
|
task_key="${task_key%.json}"
|
||||||
|
if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then
|
||||||
|
echo "WARN: ignore invalid task object name: ${object_name}" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
raw="$(read_json_file "$object_name")"
|
||||||
|
if ! record_task_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then
|
||||||
|
echo "WARN: parse error task object: ${object_name}" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ -z "$record_task_job" ]]; then
|
||||||
|
echo "WARN: task object missing record.job_id: ${object_name}" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
task_seen["$task_key"]="1"
|
||||||
|
task_bucket["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.bucket // empty' )"
|
||||||
|
task_object["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.object // empty' )"
|
||||||
|
task_version["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.version_id // empty' )"
|
||||||
|
task_storage["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.storage_class // empty' )"
|
||||||
|
task_queued["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.queued_at_unix_nanos // empty' )"
|
||||||
|
task_job["$task_key"]="$(printf '%s' "$record_task_job")"
|
||||||
|
task_path["$task_key"]="$object_name"
|
||||||
|
printf '%s,%s,%s,%s,%s,%s,%s\n' \
|
||||||
|
"$(csv_escape "$task_key")" \
|
||||||
|
"$(csv_escape "${task_bucket[$task_key]}")" \
|
||||||
|
"$(csv_escape "${task_object[$task_key]}")" \
|
||||||
|
"$(csv_escape "${task_version[$task_key]}")" \
|
||||||
|
"$(csv_escape "${task_storage[$task_key]}")" \
|
||||||
|
"$(csv_escape "${task_queued[$task_key]}")" \
|
||||||
|
"$(csv_escape "$object_name")" >> "$TASKS_CSV"
|
||||||
|
done <<< "$task_objs"
|
||||||
|
|
||||||
|
while IFS= read -r object_name; do
|
||||||
|
[[ -z "$object_name" ]] && continue
|
||||||
|
task_key="${object_name##*/}"
|
||||||
|
task_key="${task_key%.json}"
|
||||||
|
if ! [[ "$task_key" =~ ^[0-9a-f]{64}$ ]]; then
|
||||||
|
echo "WARN: ignore invalid result object name: ${object_name}" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
raw="$(read_json_file "$object_name")"
|
||||||
|
if ! record_result_job="$(printf '%s' "$raw" | jq -r '.record.job_id // empty' 2>/dev/null)"; then
|
||||||
|
echo "WARN: parse error result object: ${object_name}" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ -z "$record_result_job" ]]; then
|
||||||
|
echo "WARN: result object missing record.job_id: ${object_name}" >&2
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
result_seen["$task_key"]="1"
|
||||||
|
result_status["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.result // .result // empty' )"
|
||||||
|
result_reason["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.failure_reason // empty' )"
|
||||||
|
result_completed["$task_key"]="$(printf '%s' "$raw" | jq -r '.record.completed_at_unix_nanos // empty' )"
|
||||||
|
result_job["$task_key"]="$(printf '%s' "$record_result_job")"
|
||||||
|
result_path["$task_key"]="$object_name"
|
||||||
|
printf '%s,%s,%s,%s,%s\n' \
|
||||||
|
"$(csv_escape "$task_key")" \
|
||||||
|
"$(csv_escape "${result_status[$task_key]}")" \
|
||||||
|
"$(csv_escape "${result_reason[$task_key]}")" \
|
||||||
|
"$(csv_escape "${result_completed[$task_key]}")" \
|
||||||
|
"$(csv_escape "$object_name")" >> "$RESULTS_CSV"
|
||||||
|
done <<< "$result_objs"
|
||||||
|
|
||||||
|
local task_count=0
|
||||||
|
local result_count=0
|
||||||
|
local missing_count=0
|
||||||
|
local mismatch_count=0
|
||||||
|
local completed_count=0
|
||||||
|
local failed_count=0
|
||||||
|
local task_job_id_match
|
||||||
|
local result_job_id_match
|
||||||
|
local task_record_job
|
||||||
|
local result_record_job
|
||||||
|
local -A failure_by_reason=()
|
||||||
|
|
||||||
|
task_count="${#task_seen[@]}"
|
||||||
|
result_count="${#result_seen[@]}"
|
||||||
|
|
||||||
|
for task_key in "${!task_seen[@]}"; do
|
||||||
|
local result=""
|
||||||
|
local reason=""
|
||||||
|
local task_has_result="false"
|
||||||
|
if [[ -n "${result_seen[$task_key]+x}" ]]; then
|
||||||
|
task_has_result="true"
|
||||||
|
result="${result_status[$task_key]}"
|
||||||
|
reason="${result_reason[$task_key]}"
|
||||||
|
if [[ "${result}" == "Completed" ]]; then
|
||||||
|
completed_count=$((completed_count + 1))
|
||||||
|
elif [[ "${result}" == "TierFailure" ]]; then
|
||||||
|
failed_count=$((failed_count + 1))
|
||||||
|
reason="${reason:-Unknown}"
|
||||||
|
failure_by_reason["$reason"]=$(( ${failure_by_reason["$reason"]:-0} + 1 ))
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
missing_count=$((missing_count + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
task_job_id_match="false"
|
||||||
|
result_job_id_match="false"
|
||||||
|
|
||||||
|
task_record_job="${task_job[$task_key]-}"
|
||||||
|
if [[ -n "$task_record_job" && "$(normalize_uuid "$task_record_job")" == "$expected_job" ]]; then
|
||||||
|
task_job_id_match="true"
|
||||||
|
fi
|
||||||
|
result_record_job="${result_job[$task_key]-}"
|
||||||
|
if [[ -n "$result_record_job" && "$(normalize_uuid "$result_record_job")" == "$expected_job" ]]; then
|
||||||
|
result_job_id_match="true"
|
||||||
|
fi
|
||||||
|
if [[ "$task_job_id_match" != "true" ]] || [[ -n "${result_seen[$task_key]+x}" && "$result_job_id_match" != "true" ]]; then
|
||||||
|
mismatch_count=$((mismatch_count + 1))
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf '%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s\n' \
|
||||||
|
"$(csv_escape "$task_key")" \
|
||||||
|
"$(csv_escape "${task_bucket[$task_key]}")" \
|
||||||
|
"$(csv_escape "${task_object[$task_key]}")" \
|
||||||
|
"$(csv_escape "${task_version[$task_key]}")" \
|
||||||
|
"$(csv_escape "${task_storage[$task_key]}")" \
|
||||||
|
"$(csv_escape "$task_has_result")" \
|
||||||
|
"$(csv_escape "${result_status[$task_key]}")" \
|
||||||
|
"$(csv_escape "${result_reason[$task_key]}")" \
|
||||||
|
"$(csv_escape "$task_job_id_match")" \
|
||||||
|
"$(csv_escape "$result_job_id_match")" \
|
||||||
|
"$(csv_escape "${task_path[$task_key]}")" \
|
||||||
|
"$(csv_escape "${result_path[$task_key]-}")" >> "$RECONCILE_CSV"
|
||||||
|
done
|
||||||
|
|
||||||
|
local orphan_count=0
|
||||||
|
for task_key in "${!result_seen[@]}"; do
|
||||||
|
if [[ -z "${task_seen[$task_key]+x}" ]]; then
|
||||||
|
orphan_count=$((orphan_count + 1))
|
||||||
|
printf 'orphan_result_task_key=%s\n' "$task_key" >> "$SUMMARY_TXT"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "job_id=$JOB_ID"
|
||||||
|
echo "sys_bucket=$SYS_BUCKET"
|
||||||
|
echo "expected_task_prefix=$task_prefix"
|
||||||
|
echo "expected_result_prefix=$result_prefix"
|
||||||
|
echo "task_count=${task_count}"
|
||||||
|
echo "result_count=${result_count}"
|
||||||
|
echo "missing_result_count=${missing_count}"
|
||||||
|
echo "job_id_mismatch_count=${mismatch_count}"
|
||||||
|
echo "orphan_result_count=${orphan_count}"
|
||||||
|
echo "completed_result_count=${completed_count}"
|
||||||
|
echo "failed_result_count=${failed_count}"
|
||||||
|
for reason in "${!failure_by_reason[@]}"; do
|
||||||
|
echo "result_failure_${reason// /_}=${failure_by_reason[$reason]}"
|
||||||
|
done
|
||||||
|
} > "$SUMMARY_TXT"
|
||||||
|
|
||||||
|
{
|
||||||
|
echo
|
||||||
|
echo "## journal summary"
|
||||||
|
echo "- tasks: ${task_count}"
|
||||||
|
echo "- results: ${result_count}"
|
||||||
|
echo "- missing result for task: ${missing_count}"
|
||||||
|
echo "- orphaned result rows: ${orphan_count}"
|
||||||
|
echo "- completed result: ${completed_count}"
|
||||||
|
echo "- failed result: ${failed_count}"
|
||||||
|
} >> "$SUMMARY_TXT"
|
||||||
|
}
|
||||||
|
|
||||||
|
build_command_notes() {
|
||||||
|
{
|
||||||
|
echo "# Run snippets (to use manually)"
|
||||||
|
echo
|
||||||
|
echo "curl -sS ${ENDPOINT}/rustfs/admin/v3/metrics?types=${METRIC_TYPES}&n=${METRIC_SAMPLES} | jq '.aggregated.scanner.lifecycle_transition'"
|
||||||
|
echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token <token> --sys-bucket ${SYS_BUCKET}"
|
||||||
|
echo "scripts/manual_transition_journal_audit.sh --endpoint ${ENDPOINT} --job-id ${JOB_ID} --admin-token <token> --sys-bucket ${SYS_BUCKET} --out-dir ${OUT_DIR}"
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "# Add --mc-endpoint, --mc-access-key, --mc-secret-key when writing to .rustfs.sys."
|
||||||
|
fi
|
||||||
|
} > "$COMMANDS_TXT"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
parse_args "$@"
|
||||||
|
if [[ "$SHOW_HELP" == true ]]; then
|
||||||
|
usage
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
require_bash4
|
||||||
|
validate_args
|
||||||
|
require_cmd "$MC_BIN"
|
||||||
|
require_cmd curl
|
||||||
|
require_cmd jq
|
||||||
|
require_cmd rg
|
||||||
|
setup_output
|
||||||
|
setup_mc_alias
|
||||||
|
|
||||||
|
build_command_notes
|
||||||
|
run_worker_logs
|
||||||
|
echo
|
||||||
|
run_admin_events
|
||||||
|
echo
|
||||||
|
run_tier_operation_failures
|
||||||
|
echo
|
||||||
|
run_admin_job
|
||||||
|
echo
|
||||||
|
run_metrics
|
||||||
|
echo
|
||||||
|
collect_journals
|
||||||
|
echo "Artifacts: $OUT_DIR"
|
||||||
|
echo " - $TASKS_CSV"
|
||||||
|
echo " - $RESULTS_CSV"
|
||||||
|
echo " - $RECONCILE_CSV"
|
||||||
|
echo " - $SUMMARY_TXT"
|
||||||
|
echo " - $COMMANDS_TXT"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
+291
@@ -0,0 +1,291 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Copyright 2026 RustFS Team
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
|
|
||||||
|
ENDPOINT=""
|
||||||
|
ADMIN_TOKEN=""
|
||||||
|
OUT_DIR=""
|
||||||
|
PHASE_MATRIX="request-only:100:0:30,canary:90:10:120,mixed:50:50:240,full-rollout:0:100:360,rollback:100:0:30"
|
||||||
|
CONCURRENCIES="8,16,32"
|
||||||
|
OBJECT_COUNTS="5k,20k,50k"
|
||||||
|
JOB_BUCKET="manual-transition"
|
||||||
|
JOB_PREFIX="journal-mixed-rollout"
|
||||||
|
READ_RATIO="90"
|
||||||
|
RUN_ADMIN_CHECKS=true
|
||||||
|
DRY_RUN=false
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/manual_transition_mixed_rollout_matrix.sh --endpoint <admin-api> [options]
|
||||||
|
|
||||||
|
Required:
|
||||||
|
--endpoint Admin API base, e.g. https://127.0.0.1:9000
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
--admin-token Bearer token for admin calls
|
||||||
|
--phase-matrix Comma-separated phase spec: name:old_pct:new_pct:duration_min
|
||||||
|
--concurrencies Comma-separated concurrency list
|
||||||
|
--object-counts Comma-separated queue-size list (for plan rows)
|
||||||
|
--job-bucket Manual transition source bucket for sample commands
|
||||||
|
--job-prefix Prefix for generated job notes
|
||||||
|
--read-ratio Target read ratio percentage for mixed workload
|
||||||
|
--no-admin-checks Skip generated admin check commands
|
||||||
|
--out-dir Output dir
|
||||||
|
--dry-run
|
||||||
|
--help
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
scripts/manual_transition_mixed_rollout_matrix.sh \
|
||||||
|
--endpoint https://127.0.0.1:9000 \
|
||||||
|
--admin-token "$TOKEN" \
|
||||||
|
--phase-matrix "request-only:100:0:30,canary:90:10:120,full:0:100:240"
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
arg_value() {
|
||||||
|
local flag="$1"
|
||||||
|
local value="${2:-}"
|
||||||
|
if [[ -z "$value" || "$value" == --* ]]; then
|
||||||
|
echo "ERROR: missing value for $flag" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
require_cmd() {
|
||||||
|
local cmd="$1"
|
||||||
|
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: command not found: $cmd" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--phase-matrix) PHASE_MATRIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--concurrencies) CONCURRENCIES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--object-counts) OBJECT_COUNTS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--job-bucket) JOB_BUCKET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--job-prefix) JOB_PREFIX="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--read-ratio) READ_RATIO="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;;
|
||||||
|
--dry-run) DRY_RUN=true; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: unknown arg: $1" >&2
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
trim() {
|
||||||
|
echo "$1" | awk '{$1=$1;print}'
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_phase_row() {
|
||||||
|
local spec="$1"
|
||||||
|
local name old_ratio new_ratio duration_min
|
||||||
|
|
||||||
|
name="$(echo "$spec" | cut -d':' -f1)"
|
||||||
|
old_ratio="$(echo "$spec" | cut -d':' -f2)"
|
||||||
|
new_ratio="$(echo "$spec" | cut -d':' -f3)"
|
||||||
|
duration_min="$(echo "$spec" | cut -d':' -f4)"
|
||||||
|
name="$(trim "$name")"
|
||||||
|
old_ratio="$(trim "$old_ratio")"
|
||||||
|
new_ratio="$(trim "$new_ratio")"
|
||||||
|
duration_min="$(trim "$duration_min")"
|
||||||
|
|
||||||
|
if [[ -z "$name" || -z "$old_ratio" || -z "$new_ratio" || -z "$duration_min" ]]; then
|
||||||
|
echo "ERROR: invalid phase spec: $spec" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$old_ratio" =~ ^[0-9]+$ && "$new_ratio" =~ ^[0-9]+$ && "$duration_min" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "ERROR: phase ratio/duration must be integers: $spec" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "$name|$old_ratio|$new_ratio|$duration_min"
|
||||||
|
}
|
||||||
|
|
||||||
|
gate_for_phase() {
|
||||||
|
case "$1" in
|
||||||
|
request-only)
|
||||||
|
echo "No mixed-version errors; unknown failures should stay 0 in job report; log ratio must trend down."
|
||||||
|
;;
|
||||||
|
canary)
|
||||||
|
echo "old/new mismatch in worker results = 0; no Unknown job failures."
|
||||||
|
;;
|
||||||
|
mixed)
|
||||||
|
echo "request-only + canary acceptance gates both pass for 2+ windows, then proceed."
|
||||||
|
;;
|
||||||
|
full-rollout|full)
|
||||||
|
echo "full canary-to-full transition completed, no tier_failure in report, worker_result mismatch = 0."
|
||||||
|
;;
|
||||||
|
rollback)
|
||||||
|
echo "all rollout checks reverse cleanly; queued -> completed in job terminal state."
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo "run admin job status + journal reconcile + metric delta sanity check."
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
admin_check_cmd() {
|
||||||
|
local phase="$1"
|
||||||
|
local job_id_ref="$2"
|
||||||
|
local metric_types="$3"
|
||||||
|
local metric_samples="$4"
|
||||||
|
local metric_path
|
||||||
|
local job_url
|
||||||
|
local metric_url
|
||||||
|
local job_filter='.report, .queue_snapshot, .failure_reason'
|
||||||
|
local metric_filter='.aggregated.scanner.lifecycle_transition'
|
||||||
|
|
||||||
|
if [[ "$RUN_ADMIN_CHECKS" != true ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
metric_path="/rustfs/admin/v3/metrics?types=${metric_types}&n=${metric_samples}"
|
||||||
|
job_url="${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id_ref}"
|
||||||
|
metric_url="${ENDPOINT%/}${metric_path}"
|
||||||
|
|
||||||
|
{
|
||||||
|
printf '# check-%s\n' "$phase"
|
||||||
|
printf 'curl -sS'
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\""
|
||||||
|
fi
|
||||||
|
printf ' -X GET "%s" | jq '\''%s'\''\n' "$job_url" "$job_filter"
|
||||||
|
printf 'curl -sS'
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
printf " -H \"Authorization: Bearer \${ADMIN_TOKEN}\""
|
||||||
|
fi
|
||||||
|
printf ' -X GET "%s" | jq '\''%s'\''\n' "$metric_url" "$metric_filter"
|
||||||
|
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref"
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
printf " --admin-token \"\${ADMIN_TOKEN}\""
|
||||||
|
fi
|
||||||
|
printf ' --sys-bucket .rustfs.sys\n'
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
run_rows() {
|
||||||
|
local matrix_csv="${OUT_DIR}/mixed_rollout_matrix.csv"
|
||||||
|
local run_script="${OUT_DIR}/run_phase_commands.sh"
|
||||||
|
local checklist="${OUT_DIR}/mixed_rollout_checklist.md"
|
||||||
|
local command_manifest="${OUT_DIR}/run_phase_notes.txt"
|
||||||
|
local admin_check
|
||||||
|
local admin_check_csv
|
||||||
|
local job_id_ref
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "phase,old_pct,new_pct,duration_min,concurrency,object_count,read_ratio,gate,admin_check"
|
||||||
|
} > "$matrix_csv"
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "#!/usr/bin/env bash"
|
||||||
|
echo "set -euo pipefail"
|
||||||
|
echo ""
|
||||||
|
echo "# generated matrix commands: fill in JOB_ID after each manual transition run"
|
||||||
|
echo ""
|
||||||
|
} > "$run_script"
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "# mixed-version rollout command template"
|
||||||
|
echo "Endpoint: ${ENDPOINT}"
|
||||||
|
echo "Bucket: ${JOB_BUCKET}"
|
||||||
|
echo "Prefix: ${JOB_PREFIX}"
|
||||||
|
echo "Read ratio: ${READ_RATIO}%"
|
||||||
|
echo ""
|
||||||
|
} > "$checklist"
|
||||||
|
|
||||||
|
while IFS=',' read -r phase_spec; do
|
||||||
|
[[ -z "$phase_spec" ]] && continue
|
||||||
|
parsed="$(parse_phase_row "$phase_spec")"
|
||||||
|
IFS='|' read -r phase_name old_ratio new_ratio duration_min <<< "$parsed"
|
||||||
|
|
||||||
|
gate="$(gate_for_phase "$phase_name")"
|
||||||
|
IFS=',' read -r -a counts <<< "$OBJECT_COUNTS"
|
||||||
|
IFS=',' read -r -a concs <<< "$CONCURRENCIES"
|
||||||
|
for conc in "${concs[@]}"; do
|
||||||
|
conc="$(trim "$conc")"
|
||||||
|
[[ -z "$conc" ]] && continue
|
||||||
|
for count in "${counts[@]}"; do
|
||||||
|
count="$(trim "$count")"
|
||||||
|
[[ -z "$count" || -z "$conc" ]] && continue
|
||||||
|
|
||||||
|
job_id_ref="<JOB_ID_${phase_name}_c${conc}_q${count}>"
|
||||||
|
admin_check="$(admin_check_cmd "$phase_name" "$job_id_ref" 1 1 | tr '\n' ';')"
|
||||||
|
admin_check_csv="${admin_check//\"/\"\"}"
|
||||||
|
echo "${phase_name},${old_ratio},${new_ratio},${duration_min},${conc},${count},${READ_RATIO},\"${gate}\",\"${admin_check_csv}\"" >> "$matrix_csv"
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "# phase: ${phase_name} concurrency: ${conc} object_count: ${count}"
|
||||||
|
echo "# old:new = ${old_ratio}:${new_ratio}, duration=${duration_min}min"
|
||||||
|
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id_ref"
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
printf " --admin-token \"\${ADMIN_TOKEN}\""
|
||||||
|
fi
|
||||||
|
printf '\n'
|
||||||
|
echo ""
|
||||||
|
} >> "$run_script"
|
||||||
|
done
|
||||||
|
done
|
||||||
|
done < <(tr ',' '\n' <<< "$PHASE_MATRIX")
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "## Mixed-version rollout matrix artifacts"
|
||||||
|
echo ""
|
||||||
|
echo "- matrix: ${matrix_csv}"
|
||||||
|
echo "- commands: ${run_script}"
|
||||||
|
echo "- checklist: ${checklist}"
|
||||||
|
} > "$command_manifest"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
parse_args "$@"
|
||||||
|
if [[ -z "$ENDPOINT" ]]; then
|
||||||
|
echo "ERROR: --endpoint is required" >&2
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ -z "$OUT_DIR" ]]; then
|
||||||
|
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-mixed-rollout/$(date +%Y%m%dT%H%M%S)"
|
||||||
|
fi
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
require_cmd awk
|
||||||
|
run_rows
|
||||||
|
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "[DRY-RUN] generated files:"
|
||||||
|
fi
|
||||||
|
echo "Generated:"
|
||||||
|
echo " - ${OUT_DIR}/mixed_rollout_matrix.csv"
|
||||||
|
echo " - ${OUT_DIR}/run_phase_commands.sh"
|
||||||
|
echo " - ${OUT_DIR}/mixed_rollout_checklist.md"
|
||||||
|
echo " - ${OUT_DIR}/run_phase_notes.txt"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Executable
+262
@@ -0,0 +1,262 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
# Copyright 2026 RustFS Team
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
# you may not use this file except in compliance with the License.
|
||||||
|
# You may obtain a copy of the License at
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# software distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
# See the License for the specific language governing permissions and
|
||||||
|
# limitations under the License.
|
||||||
|
#
|
||||||
|
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||||
|
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
|
||||||
|
|
||||||
|
ENDPOINT=""
|
||||||
|
ADMIN_TOKEN=""
|
||||||
|
OUT_DIR=""
|
||||||
|
WINDOW_SPEC="nightly-2h:120:16:5000:balanced,nightly-12h:720:24:8000:write-heavy,nightly-24h:1440:32:12000:read-heavy"
|
||||||
|
SOAK_RATIOS="read-heavy:90:10,balanced:70:30,write-heavy:40:60"
|
||||||
|
WORKLOAD_SIZES="4KiB,1MiB,16MiB"
|
||||||
|
RUN_ADMIN_CHECKS=true
|
||||||
|
COST_BUDGET="$((24*1024))"
|
||||||
|
DRY_RUN=false
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
cat <<'USAGE'
|
||||||
|
Usage:
|
||||||
|
scripts/manual_transition_soak_matrix.sh --endpoint <admin-api> [options]
|
||||||
|
|
||||||
|
Required:
|
||||||
|
--endpoint Admin API base
|
||||||
|
|
||||||
|
Optional:
|
||||||
|
--admin-token Bearer token for admin calls
|
||||||
|
--window-spec CSV of <name:duration_min:concurrency:ops_per_minute:mix_name>
|
||||||
|
--workload-sizes Comma-separated object size set
|
||||||
|
--soak-ratios Comma-separated mix spec: label:read_pct:write_pct
|
||||||
|
--cost-budget Max budget units for each run, default 24576
|
||||||
|
--out-dir Output dir
|
||||||
|
--no-admin-checks Skip admin check command blocks
|
||||||
|
--dry-run
|
||||||
|
--help
|
||||||
|
USAGE
|
||||||
|
}
|
||||||
|
|
||||||
|
arg_value() {
|
||||||
|
local flag="$1"
|
||||||
|
local value="${2:-}"
|
||||||
|
if [[ -z "$value" || "$value" == --* ]]; then
|
||||||
|
echo "ERROR: missing value for $flag" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
printf '%s' "$value"
|
||||||
|
}
|
||||||
|
|
||||||
|
require_cmd() {
|
||||||
|
local cmd="$1"
|
||||||
|
if ! command -v "$cmd" >/dev/null 2>&1; then
|
||||||
|
echo "ERROR: command not found: $cmd" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_args() {
|
||||||
|
while [[ $# -gt 0 ]]; do
|
||||||
|
case "$1" in
|
||||||
|
--endpoint) ENDPOINT="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--admin-token) ADMIN_TOKEN="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--window-spec) WINDOW_SPEC="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--workload-sizes) WORKLOAD_SIZES="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--soak-ratios) SOAK_RATIOS="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--cost-budget) COST_BUDGET="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--out-dir) OUT_DIR="$(arg_value "$1" "${2:-}")"; shift 2 ;;
|
||||||
|
--no-admin-checks) RUN_ADMIN_CHECKS=false; shift ;;
|
||||||
|
--dry-run) DRY_RUN=true; shift ;;
|
||||||
|
-h|--help) usage; exit 0 ;;
|
||||||
|
*)
|
||||||
|
echo "ERROR: unknown arg: $1" >&2
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done
|
||||||
|
}
|
||||||
|
|
||||||
|
trim() {
|
||||||
|
echo "$1" | awk '{$1=$1;print}'
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_window() {
|
||||||
|
local spec="$1"
|
||||||
|
local name duration_min concurrency ops_per_min mix_name
|
||||||
|
name="$(echo "$spec" | cut -d':' -f1)"
|
||||||
|
duration_min="$(echo "$spec" | cut -d':' -f2)"
|
||||||
|
concurrency="$(echo "$spec" | cut -d':' -f3)"
|
||||||
|
ops_per_min="$(echo "$spec" | cut -d':' -f4)"
|
||||||
|
mix_name="$(echo "$spec" | cut -d':' -f5)"
|
||||||
|
name="$(trim "$name")"
|
||||||
|
duration_min="$(trim "$duration_min")"
|
||||||
|
concurrency="$(trim "$concurrency")"
|
||||||
|
ops_per_min="$(trim "$ops_per_min")"
|
||||||
|
mix_name="$(trim "$mix_name")"
|
||||||
|
if [[ -z "$name" || -z "$duration_min" || -z "$concurrency" || -z "$ops_per_min" || -z "$mix_name" ]]; then
|
||||||
|
echo "ERROR: invalid window spec: $spec" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$duration_min" =~ ^[0-9]+$ && "$concurrency" =~ ^[0-9]+$ && "$ops_per_min" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "ERROR: window duration/concurrency/ops must be integers: $spec" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "$name|$duration_min|$concurrency|$ops_per_min|$mix_name"
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_ratio() {
|
||||||
|
local spec="$1"
|
||||||
|
local name read_pct write_pct
|
||||||
|
name="$(echo "$spec" | cut -d':' -f1)"
|
||||||
|
read_pct="$(echo "$spec" | cut -d':' -f2)"
|
||||||
|
write_pct="$(echo "$spec" | cut -d':' -f3)"
|
||||||
|
name="$(trim "$name")"
|
||||||
|
read_pct="$(trim "$read_pct")"
|
||||||
|
write_pct="$(trim "$write_pct")"
|
||||||
|
if [[ -z "$name" || -z "$read_pct" || -z "$write_pct" ]]; then
|
||||||
|
echo "ERROR: invalid mix ratio: $spec" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if ! [[ "$read_pct" =~ ^[0-9]+$ && "$write_pct" =~ ^[0-9]+$ ]]; then
|
||||||
|
echo "ERROR: mix percentages must be integers: $spec" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
echo "$name|$read_pct|$write_pct"
|
||||||
|
}
|
||||||
|
|
||||||
|
resolve_mix() {
|
||||||
|
local name="$1"
|
||||||
|
local line
|
||||||
|
line="$(awk -v target="$name" -F: '{
|
||||||
|
if ($1 == target) { print $0; exit }
|
||||||
|
}' < <(tr ',' '\n' <<< "$SOAK_RATIOS"))"
|
||||||
|
if [[ -z "$line" ]]; then
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
parse_ratio "$line"
|
||||||
|
}
|
||||||
|
|
||||||
|
run_checks() {
|
||||||
|
local tag="$1"
|
||||||
|
local job_id="$2"
|
||||||
|
local token_hdr=""
|
||||||
|
if [[ -z "$job_id" ]]; then
|
||||||
|
job_id="<JOB_ID_${tag}>"
|
||||||
|
fi
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
token_hdr="-H \"Authorization: Bearer \${ADMIN_TOKEN}\" "
|
||||||
|
fi
|
||||||
|
if [[ "$RUN_ADMIN_CHECKS" != true ]]; then
|
||||||
|
return
|
||||||
|
fi
|
||||||
|
echo "# admin checks for ${tag}"
|
||||||
|
echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/ilm/transition/jobs/${job_id}\" | jq '.status, .report, .queue_snapshot, .failure_reason'"
|
||||||
|
echo "curl -sS ${token_hdr}-X GET \"${ENDPOINT%/}/rustfs/admin/v3/metrics?types=1&n=1\" | jq '.aggregated.scanner.lifecycle_transition'"
|
||||||
|
printf './scripts/manual_transition_journal_audit.sh --endpoint %s --job-id "%s"' "$ENDPOINT" "$job_id"
|
||||||
|
if [[ -n "$ADMIN_TOKEN" ]]; then
|
||||||
|
printf " --admin-token \"\${ADMIN_TOKEN}\""
|
||||||
|
fi
|
||||||
|
printf '\n'
|
||||||
|
}
|
||||||
|
|
||||||
|
write_matrix_files() {
|
||||||
|
local matrix_csv="${OUT_DIR}/nightly_soak_matrix.csv"
|
||||||
|
local run_matrix="${OUT_DIR}/run_soak_matrix.sh"
|
||||||
|
local notes="${OUT_DIR}/soak_notes.md"
|
||||||
|
local run_id=1
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "window,window_duration_min,concurrency,ops_per_min,mix_name,read_pct,write_pct,size,expected_ops,expected_run_id,run_check"
|
||||||
|
} > "$matrix_csv"
|
||||||
|
|
||||||
|
echo "#!/usr/bin/env bash" > "$run_matrix"
|
||||||
|
echo "set -euo pipefail" >> "$run_matrix"
|
||||||
|
echo "" >> "$run_matrix"
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "# Nightly/night soak matrix for manual transition stress"
|
||||||
|
echo "- endpoint: ${ENDPOINT}"
|
||||||
|
echo "- cost-budget: ${COST_BUDGET}"
|
||||||
|
echo ""
|
||||||
|
echo "Recommended gate for each row: no unknown failure_reason spikes, job result drift is 0, and queued/task/result counts reconcile."
|
||||||
|
echo ""
|
||||||
|
} > "$notes"
|
||||||
|
|
||||||
|
IFS=',' read -r -a sizes <<< "$WORKLOAD_SIZES"
|
||||||
|
while IFS=',' read -r window_spec; do
|
||||||
|
[[ -z "$window_spec" ]] && continue
|
||||||
|
parsed="$(parse_window "$window_spec")"
|
||||||
|
IFS='|' read -r window_name duration_min concurrency ops_per_min mix_name <<< "$parsed"
|
||||||
|
|
||||||
|
mix="$(resolve_mix "$mix_name" || true)"
|
||||||
|
if [[ -z "$mix" ]]; then
|
||||||
|
echo "ERROR: unknown mix name in window spec: ${window_name} -> ${mix_name}" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
IFS='|' read -r resolved_mix read_pct write_pct <<< "$mix"
|
||||||
|
|
||||||
|
for size in "${sizes[@]}"; do
|
||||||
|
size="$(trim "$size")"
|
||||||
|
[[ -z "$size" ]] && continue
|
||||||
|
expected_ops=$(( duration_min * ops_per_min ))
|
||||||
|
if (( expected_ops > COST_BUDGET )); then
|
||||||
|
budget_status="over-budget"
|
||||||
|
else
|
||||||
|
budget_status="within-budget"
|
||||||
|
fi
|
||||||
|
echo "${window_name},${duration_min},${concurrency},${ops_per_min},${resolved_mix},${read_pct},${write_pct},${size},${expected_ops},${run_id},${budget_status}" >> "$matrix_csv"
|
||||||
|
{
|
||||||
|
echo "# window=${window_name} size=${size} mix=${resolved_mix} concurrency=${concurrency} duration=${duration_min}m ops_per_min=${ops_per_min}"
|
||||||
|
echo "job_id=\"<JOB_ID_${window_name}_${size}_${concurrency}_${run_id}>\""
|
||||||
|
echo "read_pct=${read_pct}"
|
||||||
|
echo "write_pct=${write_pct}"
|
||||||
|
echo "expected_ops=${expected_ops}"
|
||||||
|
echo "budget_status=${budget_status}"
|
||||||
|
run_checks "${window_name}_${size}_${concurrency}_${run_id}" "<JOB_ID_${window_name}_${size}_${concurrency}_${run_id}>"
|
||||||
|
echo
|
||||||
|
} >> "$run_matrix"
|
||||||
|
run_id=$((run_id + 1))
|
||||||
|
done
|
||||||
|
done < <(tr ',' '\n' <<< "$WINDOW_SPEC")
|
||||||
|
|
||||||
|
{
|
||||||
|
echo "# run target:"
|
||||||
|
echo "bash ${run_matrix}"
|
||||||
|
} >> "$notes"
|
||||||
|
}
|
||||||
|
|
||||||
|
main() {
|
||||||
|
parse_args "$@"
|
||||||
|
if [[ -z "$ENDPOINT" ]]; then
|
||||||
|
echo "ERROR: --endpoint is required" >&2
|
||||||
|
usage
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
if [[ -z "$OUT_DIR" ]]; then
|
||||||
|
OUT_DIR="${PROJECT_ROOT}/target/manual-transition-nightly-soak/$(date +%Y%m%dT%H%M%S)"
|
||||||
|
fi
|
||||||
|
mkdir -p "$OUT_DIR"
|
||||||
|
require_cmd awk
|
||||||
|
write_matrix_files
|
||||||
|
if [[ "$DRY_RUN" == true ]]; then
|
||||||
|
echo "[DRY-RUN] generated matrix files:"
|
||||||
|
fi
|
||||||
|
echo "Generated:"
|
||||||
|
echo " - ${OUT_DIR}/nightly_soak_matrix.csv"
|
||||||
|
echo " - ${OUT_DIR}/run_soak_matrix.sh"
|
||||||
|
echo " - ${OUT_DIR}/soak_notes.md"
|
||||||
|
}
|
||||||
|
|
||||||
|
main "$@"
|
||||||
Reference in New Issue
Block a user