mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-30 00:47:13 +00:00
refactor(protos): move compat manifest send-site assertions into owning crates (#6618)
refactor(protos): move internode compat manifest send-site assertions into owning crates Promotes the rolling-upgrade dual-write manifest from a test-only constant in rustfs-protos into the public rustfs_protos::compat_manifest module, moves the JSON-encoder send-site assertions into the crates that own the asserted sources (ecstore remote_disk.rs for requests, the rustfs binary node_service/disk.rs for responses), and splits the scanner Phase-0 overlap inventory so its heal- and ecstore-owned halves live in those crates. Adds a cross-crate include_str!/include! guard with fixture self-tests to scripts/check_layer_dependencies.sh so a library crate can never again read another crate's Rust source at compile time, and records the rule in docs/architecture/crate-boundaries.md. Part of rustfs/backlog#1884.
This commit is contained in:
Zhengchao An
+2
-217
@@ -12,6 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub mod compat_manifest;
|
||||
// SAFETY: `generated` is prost/tonic-generated protocol code. The allowance is
|
||||
// scoped to that module so generated internals do not relax lints elsewhere.
|
||||
#[allow(unsafe_code)]
|
||||
@@ -2628,176 +2629,7 @@ mod tests {
|
||||
assert_eq!(decoded.protocol_version, 0);
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)]
|
||||
struct CompatPayloadField {
|
||||
message: &'static str,
|
||||
json_field: &'static str,
|
||||
bin_field: &'static str,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
|
||||
enum RequestJsonPolicy {
|
||||
MsgpackOnlyEligible,
|
||||
AlwaysDualWriteUntilFallbackZero,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct RequestCompatSendSite {
|
||||
field: CompatPayloadField,
|
||||
json_encoder: &'static str,
|
||||
policy: RequestJsonPolicy,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
struct ResponseCompatSendSite {
|
||||
field: CompatPayloadField,
|
||||
json_encoder: &'static str,
|
||||
}
|
||||
|
||||
const REQUEST_COMPAT_SEND_SITES: &[RequestCompatSendSite] = &[
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "BatchReadVersionRequest",
|
||||
json_field: "batch_read_version_req",
|
||||
bin_field: "batch_read_version_req_bin",
|
||||
},
|
||||
json_encoder: "let batch_read_version_req = compat_json(&req)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "DeleteVersionRequest",
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = serde_json::to_string(&fi)?;",
|
||||
policy: RequestJsonPolicy::AlwaysDualWriteUntilFallbackZero,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "DeleteVersionRequest",
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let opts = serde_json::to_string(&opts)?;",
|
||||
policy: RequestJsonPolicy::AlwaysDualWriteUntilFallbackZero,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "DeleteVersionsRequest",
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let opts = match serde_json::to_string(&opts) {",
|
||||
policy: RequestJsonPolicy::AlwaysDualWriteUntilFallbackZero,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "DeleteVersionsRequest",
|
||||
json_field: "versions",
|
||||
bin_field: "versions_bin",
|
||||
},
|
||||
json_encoder: "versions_str.push(match serde_json::to_string(file_info_versions) {",
|
||||
policy: RequestJsonPolicy::AlwaysDualWriteUntilFallbackZero,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "ReadMultipleRequest",
|
||||
json_field: "read_multiple_req",
|
||||
bin_field: "read_multiple_req_bin",
|
||||
},
|
||||
json_encoder: "let read_multiple_req = compat_json(&req)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "ReadVersionRequest",
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let encoded_opts = compat_json(opts).and_then(|opts_str| encode_msgpack(opts).map(|opts_bin| (opts_str, opts_bin)));",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "RenameDataRequest",
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "UpdateMetadataRequest",
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "UpdateMetadataRequest",
|
||||
json_field: "opts",
|
||||
bin_field: "opts_bin",
|
||||
},
|
||||
json_encoder: "let opts_str = compat_json(&opts)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
RequestCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "WriteMetadataRequest",
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info = compat_json(&fi)?;",
|
||||
policy: RequestJsonPolicy::MsgpackOnlyEligible,
|
||||
},
|
||||
];
|
||||
|
||||
const RESPONSE_COMPAT_SEND_SITES: &[ResponseCompatSendSite] = &[
|
||||
ResponseCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "BatchReadVersionResponse",
|
||||
json_field: "batch_read_version_resps",
|
||||
bin_field: "batch_read_version_resps_bin",
|
||||
},
|
||||
json_encoder: "compat_response_json(batch_read_version_resp, request_decoded_from_msgpack)",
|
||||
},
|
||||
ResponseCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "ReadMultipleResponse",
|
||||
json_field: "read_multiple_resps",
|
||||
bin_field: "read_multiple_resps_bin",
|
||||
},
|
||||
json_encoder: "compat_response_json(read_multiple_resp, false)",
|
||||
},
|
||||
ResponseCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "ReadVersionResponse",
|
||||
json_field: "file_info",
|
||||
bin_field: "file_info_bin",
|
||||
},
|
||||
json_encoder: "let file_info_json = compat_response_json(&file_info, request_had_msgpack_payload);",
|
||||
},
|
||||
ResponseCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "ReadXLResponse",
|
||||
json_field: "raw_file_info",
|
||||
bin_field: "raw_file_info_bin",
|
||||
},
|
||||
json_encoder: "let raw_file_info_json = compat_response_json(&raw_file_info, false);",
|
||||
},
|
||||
ResponseCompatSendSite {
|
||||
field: CompatPayloadField {
|
||||
message: "RenameDataResponse",
|
||||
json_field: "rename_data_resp",
|
||||
bin_field: "rename_data_resp_bin",
|
||||
},
|
||||
json_encoder: "let rename_data_resp_json = compat_response_json(rename_data_resp, request_decoded_from_msgpack)",
|
||||
},
|
||||
];
|
||||
use crate::compat_manifest::{CompatPayloadField, REQUEST_COMPAT_SEND_SITES, RESPONSE_COMPAT_SEND_SITES, RequestJsonPolicy};
|
||||
|
||||
fn proto_bin_json_fields(message_suffix: &str) -> Vec<CompatPayloadField> {
|
||||
let proto = include_str!("node.proto");
|
||||
@@ -2837,13 +2669,6 @@ mod tests {
|
||||
fields
|
||||
}
|
||||
|
||||
fn production_source(source: &'static str, file_name: &str) -> &'static str {
|
||||
source
|
||||
.split("\n#[cfg(test)]\nmod tests")
|
||||
.next()
|
||||
.unwrap_or_else(|| panic!("{file_name} should contain production source before tests"))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_compat_send_site_manifest_covers_node_proto_bin_fields() {
|
||||
let mut manifest_fields = REQUEST_COMPAT_SEND_SITES
|
||||
@@ -2861,31 +2686,6 @@ mod tests {
|
||||
assert_eq!(manifest_fields, proto_bin_json_fields("Request"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_compat_send_site_manifest_pins_json_policy_and_encoder() {
|
||||
let source = production_source(include_str!("../../ecstore/src/cluster/rpc/remote_disk.rs"), "remote_disk.rs");
|
||||
let msgpack_only_eligible = REQUEST_COMPAT_SEND_SITES
|
||||
.iter()
|
||||
.filter(|send_site| send_site.policy == RequestJsonPolicy::MsgpackOnlyEligible)
|
||||
.count();
|
||||
let always_dual_write = REQUEST_COMPAT_SEND_SITES
|
||||
.iter()
|
||||
.filter(|send_site| send_site.policy == RequestJsonPolicy::AlwaysDualWriteUntilFallbackZero)
|
||||
.count();
|
||||
|
||||
assert_eq!(msgpack_only_eligible, 7);
|
||||
assert_eq!(always_dual_write, 4);
|
||||
for send_site in REQUEST_COMPAT_SEND_SITES {
|
||||
assert!(
|
||||
source.contains(send_site.json_encoder),
|
||||
"{}.{} must keep its manifest encoder: {}",
|
||||
send_site.field.message,
|
||||
send_site.field.json_field,
|
||||
send_site.json_encoder
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_compat_send_site_manifest_pins_exact_json_policies() {
|
||||
let mut policies = REQUEST_COMPAT_SEND_SITES
|
||||
@@ -2933,21 +2733,6 @@ mod tests {
|
||||
assert_eq!(manifest_fields, proto_bin_json_fields("Response"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn response_compat_send_site_manifest_pins_json_encoder() {
|
||||
let source = production_source(include_str!("../../../rustfs/src/storage/rpc/node_service/disk.rs"), "disk.rs");
|
||||
|
||||
for send_site in RESPONSE_COMPAT_SEND_SITES {
|
||||
assert!(
|
||||
source.contains(send_site.json_encoder),
|
||||
"{}.{} must keep its manifest encoder: {}",
|
||||
send_site.field.message,
|
||||
send_site.field.json_field,
|
||||
send_site.json_encoder
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn enforce_tls_generation_cache_bound_evicts_when_retained_entries_still_full() {
|
||||
let mut cache = HashMap::new();
|
||||
|
||||
Reference in New Issue
Block a user