gsadmin/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-08-08 06:13:14 +00:00
Code Issues Packages Projects Releases Wiki Activity

test(internode): extend fallback and transition coverage (#5232)

Browse Source 
Add decode-error metrics for internode msgpack/json compatibility paths, extend the mixed fallback e2e assertions for transitioned multipart partNumber reads, and cover manual transition async status polling plus inactive-owner status behavior.

Co-authored-by: heihutu <heihutu@gmail.com>
This commit is contained in:
houseme
2026-07-25 21:26:23 +08:00
committed by GitHub
parent e73ed4f2a1
commit 5988606e68
9 changed files with 409 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files
rustfs/src/admin/handlers/ilm_transition.rs
+53 -10
View File
@@ -537,6 +537,15 @@ fn request_manual_transition_job_cancel(job_id: &str) -> Option<ManualTransition
Some(runtime.record.clone())
}
fn mark_manual_transition_job_owner_unknown(mut record: ManualTransitionJobRecord) -> ManualTransitionJobRecord {
if !record.status.is_terminal() {
record.status = ManualTransitionJobStatus::Unknown;
record.finished_at = Some(manual_transition_timestamp(OffsetDateTime::now_utc()));
record.failure_reason = Some("manual transition job owner is not active on this node".to_string());
}
record
}
fn remove_manual_transition_job(job_id: &str) {
let mut jobs = lock_manual_transition_jobs();
jobs.remove(job_id);
@@ -570,15 +579,10 @@ async fn load_manual_transition_job_record(store: Arc<ECStore>, job_id: &str) ->
if let Some(record) = in_memory_manual_transition_job_record(job_id) {
return Ok(record);
}
let Some(mut record) = read_manual_transition_job_record(store, job_id).await? else {
let Some(record) = read_manual_transition_job_record(store, job_id).await? else {
return Err(s3_error!(NoSuchKey, "manual transition job not found"));
};
if !record.status.is_terminal() {
record.status = ManualTransitionJobStatus::Unknown;
record.finished_at = Some(manual_transition_timestamp(OffsetDateTime::now_utc()));
record.failure_reason = Some("manual transition job owner is not active on this node".to_string());
}
Ok(record)
Ok(mark_manual_transition_job_owner_unknown(record))
}
async fn persist_manual_transition_job_update(store: Arc<ECStore>, record: &ManualTransitionJobRecord) -> bool {
@@ -762,10 +766,8 @@ impl Operation for ManualTransitionJobCancelHandler {
return Err(s3_error!(NoSuchKey, "manual transition job not found"));
};
if !record.status.is_terminal() {
record.status = ManualTransitionJobStatus::Unknown;
record.cancel_requested = true;
record.finished_at = Some(manual_transition_timestamp(OffsetDateTime::now_utc()));
record.failure_reason = Some("manual transition job owner is not active on this node".to_string());
record = mark_manual_transition_job_owner_unknown(record);
save_manual_transition_job_record(store, &record).await?;
}
record
@@ -1065,6 +1067,47 @@ mod tests {
assert!(cancel_token.is_cancelled());
}
#[test]
fn manual_transition_inactive_owner_marks_nonterminal_job_unknown() {
let (bucket, options, _run_mode) =
parse_manual_transition_query(Some("bucket=data&prefix=logs/&async=true")).expect("async query should parse");
let record = new_manual_transition_job_record(
"11111111-1111-4111-8111-111111111111".to_string(),
&bucket,
&options,
OffsetDateTime::UNIX_EPOCH,
);
let unknown = mark_manual_transition_job_owner_unknown(record);
assert_eq!(unknown.status, ManualTransitionJobStatus::Unknown);
assert!(unknown.finished_at.is_some());
assert_eq!(
unknown.failure_reason.as_deref(),
Some("manual transition job owner is not active on this node")
);
}
#[test]
fn manual_transition_inactive_owner_preserves_terminal_job() {
let (bucket, options, _run_mode) =
parse_manual_transition_query(Some("bucket=data&prefix=logs/&async=true")).expect("async query should parse");
let mut record = new_manual_transition_job_record(
"11111111-1111-4111-8111-111111111111".to_string(),
&bucket,
&options,
OffsetDateTime::UNIX_EPOCH,
);
record.status = ManualTransitionJobStatus::Completed;
record.finished_at = Some("1970-01-01T00:00:00Z".to_string());
let completed = mark_manual_transition_job_owner_unknown(record);
assert_eq!(completed.status, ManualTransitionJobStatus::Completed);
assert_eq!(completed.finished_at.as_deref(), Some("1970-01-01T00:00:00Z"));
assert_eq!(completed.failure_reason, None);
}
#[test]
fn manual_transition_handler_requires_set_tier_action() {
let src = include_str!("ilm_transition.rs");
rustfs/src/storage/rpc/node_service/disk.rs
+26 -5
View File
@@ -21,8 +21,9 @@ use crate::storage::storage_api::runtime_sources_consumer::runtime_sources;
use bytes::Bytes;
use rustfs_filemeta::FileInfo;
use rustfs_io_metrics::internode_metrics::{
INTERNODE_MSGPACK_DIRECTION_REQUEST, INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL,
INTERNODE_TRANSPORT_BACKEND_GRPC, global_internode_metrics,
INTERNODE_MSGPACK_CODEC_JSON, INTERNODE_MSGPACK_CODEC_MSGPACK, INTERNODE_MSGPACK_DIRECTION_REQUEST,
INTERNODE_OPERATION_GRPC_READ_ALL, INTERNODE_OPERATION_GRPC_WRITE_ALL, INTERNODE_TRANSPORT_BACKEND_GRPC,
global_internode_metrics,
};
use rustfs_protos::proto_gen::node_service::*;
use serde::de::DeserializeOwned;
@@ -41,14 +42,27 @@ fn decode_msgpack_or_json<T: DeserializeOwned>(
) -> std::result::Result<T, DiskError> {
if !binary.is_empty() {
let mut deserializer = rmp_serde::Deserializer::new(Cursor::new(binary));
return T::deserialize(&mut deserializer)
.map_err(|err| DiskError::other(format!("decode {value_name} msgpack failed: {err}")));
return T::deserialize(&mut deserializer).map_err(|err| {
global_internode_metrics().record_msgpack_json_decode_error(
INTERNODE_MSGPACK_DIRECTION_REQUEST,
value_name,
INTERNODE_MSGPACK_CODEC_MSGPACK,
);
DiskError::other(format!("decode {value_name} msgpack failed: {err}"))
});
}
// The msgpack payload was absent, so fall back to the JSON compatibility field. This branch
// must read zero across a release window before the redundant JSON fields can be dropped (P2).
global_internode_metrics().record_msgpack_json_fallback(INTERNODE_MSGPACK_DIRECTION_REQUEST, value_name);
serde_json::from_str(json).map_err(|err| DiskError::other(format!("decode {value_name} failed: {err}")))
serde_json::from_str(json).map_err(|err| {
global_internode_metrics().record_msgpack_json_decode_error(
INTERNODE_MSGPACK_DIRECTION_REQUEST,
value_name,
INTERNODE_MSGPACK_CODEC_JSON,
);
DiskError::other(format!("decode {value_name} failed: {err}"))
})
}
fn encode_msgpack<T: serde::Serialize>(value: &T, value_name: &str) -> std::result::Result<Vec<u8>, DiskError> {
@@ -1191,6 +1205,7 @@ mod tests {
};
use crate::storage::storage_api::ReadMultipleResp;
use crate::storage::storage_api::rpc_consumer::node_service::BatchReadVersionResp;
use rustfs_io_metrics::internode_metrics::global_internode_metrics;
use serde::{Deserialize, Serialize};
#[derive(Debug, PartialEq, Eq, Serialize, Deserialize)]
@@ -1314,18 +1329,24 @@ mod tests {
#[test]
fn decode_msgpack_or_json_fails_closed_on_corrupt_non_empty_msgpack() {
let before = global_internode_metrics().msgpack_json_decode_error_total_for_test();
let err = decode_msgpack_or_json::<SamplePayload>(b"not-msgpack", r#"{"name":"json","count":1}"#, "SamplePayload")
.expect_err("corrupt non-empty msgpack must not fall back to JSON");
let after = global_internode_metrics().msgpack_json_decode_error_total_for_test();
assert!(err.to_string().contains("decode SamplePayload msgpack failed"), "unexpected error: {err}");
assert!(after > before, "corrupt msgpack should increment decode-error metrics");
}
#[test]
fn decode_msgpack_or_json_reports_corrupt_json_item_when_msgpack_absent() {
let before = global_internode_metrics().msgpack_json_decode_error_total_for_test();
let err = decode_msgpack_or_json::<SamplePayload>(&[], "{not-json", "SamplePayload")
.expect_err("corrupt json item should fail in fallback branch");
let after = global_internode_metrics().msgpack_json_decode_error_total_for_test();
assert!(err.to_string().contains("decode SamplePayload failed"), "unexpected error: {err}");
assert!(after > before, "corrupt fallback JSON should increment decode-error metrics");
}
#[test]