From ff638d1140819f35fab3f6b2a586f27b7994cfa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=AE=89=E6=AD=A3=E8=B6=85?= Date: Fri, 19 Jun 2026 22:31:18 +0800 Subject: [PATCH] feat: add ops profiler extension schema (#3628) --- crates/extension-schema/src/lib.rs | 430 +++++++++++++++++- docs/architecture/migration-progress.md | 57 ++- .../profiling-numa-capability-inventory.md | 9 +- 3 files changed, 461 insertions(+), 35 deletions(-) diff --git a/crates/extension-schema/src/lib.rs b/crates/extension-schema/src/lib.rs index 2bf03f98b..bba4b0c43 100644 --- a/crates/extension-schema/src/lib.rs +++ b/crates/extension-schema/src/lib.rs @@ -19,6 +19,7 @@ use thiserror::Error; pub const EXTENSION_SCHEMA_VERSION: &str = "rustfs.extension-schema.v1"; pub const OPS_DIAGNOSTICS_CAPABILITY: &str = "ops.diagnostics.v1"; +pub const OPS_PROFILER_CAPABILITY: &str = "ops.profiler.v1"; pub const S3_POST_AUTH_HOOK_CAPABILITY: &str = "s3.hook.post_auth.v1"; #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -117,6 +118,79 @@ pub struct OpsDiagnosticsContract { pub requires_admin_action: bool, } +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpsProfilerContractMode { + CapabilityDescription, + ExecutionRequest, +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(transparent)] +pub struct OpsProfilerBackendName(String); + +impl OpsProfilerBackendName { + pub fn new(backend: impl Into) -> Self { + Self(backend.into()) + } + + pub fn as_str(&self) -> &str { + self.0.as_str() + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpsProfilerBackendStatus { + Enabled, + Disabled, + Unsupported, + Unknown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpsProfilerRedactionField { + Secret, + Token, + LocalPath, + Host, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum OpsProfilerTrustLevel { + RuntimeTrusted, + AdminTrusted, + ExtensionProvided, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OpsProfilerProvenance { + pub source: String, + pub collection_boundary: String, + pub trust_level: OpsProfilerTrustLevel, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OpsProfilerBackendCapability { + pub backend: OpsProfilerBackendName, + pub status: OpsProfilerBackendStatus, + pub supports_profile_export: bool, + pub redaction_required: Vec, + pub provenance: OpsProfilerProvenance, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct OpsProfilerContract { + pub mode: OpsProfilerContractMode, + pub backends: Vec, +} + #[derive(Debug, Error, PartialEq, Eq)] pub enum ExtensionSchemaError { #[error("extension schema at index {index} has an empty extension id")] @@ -178,6 +252,33 @@ pub enum ExtensionContractError { #[error("ops diagnostics contract must require an admin action")] OpsDiagnosticsMissingAdminAction, + + #[error("ops profiler contract must describe capabilities, not execution requests")] + OpsProfilerExecutionRequest, + + #[error("ops profiler contract must declare at least one backend")] + EmptyOpsProfilerBackends, + + #[error("ops profiler contract has an empty backend name")] + EmptyOpsProfilerBackend, + + #[error("ops profiler contract duplicates backend {backend}")] + DuplicateOpsProfilerBackend { backend: String }, + + #[error("ops profiler backend {backend} duplicates redaction field {field:?}")] + DuplicateOpsProfilerRedactionField { + backend: String, + field: OpsProfilerRedactionField, + }, + + #[error("ops profiler backend {backend} exports profiles without local path redaction")] + OpsProfilerMissingLocalPathRedaction { backend: String }, + + #[error("ops profiler backend {backend} has an empty provenance source")] + EmptyOpsProfilerProvenanceSource { backend: String }, + + #[error("ops profiler backend {backend} has an empty collection boundary")] + EmptyOpsProfilerCollectionBoundary { backend: String }, } pub fn validate_extension_schemas(schemas: &[ExtensionSchema]) -> Result<(), ExtensionSchemaError> { @@ -304,13 +405,69 @@ pub fn validate_ops_diagnostics_contract(contract: &OpsDiagnosticsContract) -> R Ok(()) } +pub fn validate_ops_profiler_contract(contract: &OpsProfilerContract) -> Result<(), ExtensionContractError> { + if contract.mode != OpsProfilerContractMode::CapabilityDescription { + return Err(ExtensionContractError::OpsProfilerExecutionRequest); + } + + if contract.backends.is_empty() { + return Err(ExtensionContractError::EmptyOpsProfilerBackends); + } + + let mut backends = BTreeSet::new(); + for backend in &contract.backends { + let backend_name = backend.backend.as_str().trim(); + if backend_name.is_empty() { + return Err(ExtensionContractError::EmptyOpsProfilerBackend); + } + + if !backends.insert(backend_name) { + return Err(ExtensionContractError::DuplicateOpsProfilerBackend { + backend: backend_name.to_string(), + }); + } + + let mut redaction_fields = BTreeSet::new(); + for field in &backend.redaction_required { + if !redaction_fields.insert(*field) { + return Err(ExtensionContractError::DuplicateOpsProfilerRedactionField { + backend: backend_name.to_string(), + field: *field, + }); + } + } + + if backend.supports_profile_export && !redaction_fields.contains(&OpsProfilerRedactionField::LocalPath) { + return Err(ExtensionContractError::OpsProfilerMissingLocalPathRedaction { + backend: backend_name.to_string(), + }); + } + + if backend.provenance.source.trim().is_empty() { + return Err(ExtensionContractError::EmptyOpsProfilerProvenanceSource { + backend: backend_name.to_string(), + }); + } + + if backend.provenance.collection_boundary.trim().is_empty() { + return Err(ExtensionContractError::EmptyOpsProfilerCollectionBoundary { + backend: backend_name.to_string(), + }); + } + } + + Ok(()) +} + #[cfg(test)] mod tests { use super::{ EXTENSION_SCHEMA_VERSION, ExtensionCapabilityRef, ExtensionContractError, ExtensionKind, ExtensionRuntimeBoundary, - ExtensionRuntimeContract, ExtensionSchema, ExtensionSchemaError, OPS_DIAGNOSTICS_CAPABILITY, OpsDiagnosticSurface, - OpsDiagnosticsContract, S3_POST_AUTH_HOOK_CAPABILITY, S3HookContract, S3HookPoint, validate_extension_schemas, - validate_ops_diagnostics_contract, validate_s3_hook_contract, + ExtensionRuntimeContract, ExtensionSchema, ExtensionSchemaError, OPS_DIAGNOSTICS_CAPABILITY, OPS_PROFILER_CAPABILITY, + OpsDiagnosticSurface, OpsDiagnosticsContract, OpsProfilerBackendCapability, OpsProfilerBackendName, + OpsProfilerBackendStatus, OpsProfilerContract, OpsProfilerContractMode, OpsProfilerProvenance, OpsProfilerRedactionField, + OpsProfilerTrustLevel, S3_POST_AUTH_HOOK_CAPABILITY, S3HookContract, S3HookPoint, validate_extension_schemas, + validate_ops_diagnostics_contract, validate_ops_profiler_contract, validate_s3_hook_contract, }; use serde_json::json; @@ -363,20 +520,36 @@ mod tests { #[test] fn validates_extension_schema_contracts() { - let schemas = [ExtensionSchema { - schema_version: EXTENSION_SCHEMA_VERSION.to_string(), - extension_id: "rustfs.ops.diagnostics".to_string(), - display_name: "Ops Diagnostics".to_string(), - provider: "rustfs".to_string(), - version: "1.0.0".to_string(), - kind: ExtensionKind::OpsDiagnostics, - runtime: ExtensionRuntimeContract { - api_version: "rustfs.extension.v1".to_string(), - boundary: ExtensionRuntimeBoundary::Builtin, + let schemas = [ + ExtensionSchema { + schema_version: EXTENSION_SCHEMA_VERSION.to_string(), + extension_id: "rustfs.ops.diagnostics".to_string(), + display_name: "Ops Diagnostics".to_string(), + provider: "rustfs".to_string(), + version: "1.0.0".to_string(), + kind: ExtensionKind::OpsDiagnostics, + runtime: ExtensionRuntimeContract { + api_version: "rustfs.extension.v1".to_string(), + boundary: ExtensionRuntimeBoundary::Builtin, + }, + capabilities: vec![ExtensionCapabilityRef::new(OPS_DIAGNOSTICS_CAPABILITY)], + disabled_by_default: false, }, - capabilities: vec![ExtensionCapabilityRef::new(OPS_DIAGNOSTICS_CAPABILITY)], - disabled_by_default: false, - }]; + ExtensionSchema { + schema_version: EXTENSION_SCHEMA_VERSION.to_string(), + extension_id: "rustfs.ops.profiler".to_string(), + display_name: "Ops Profiler".to_string(), + provider: "rustfs".to_string(), + version: "1.0.0".to_string(), + kind: ExtensionKind::OpsDiagnostics, + runtime: ExtensionRuntimeContract { + api_version: "rustfs.extension.v1".to_string(), + boundary: ExtensionRuntimeBoundary::Builtin, + }, + capabilities: vec![ExtensionCapabilityRef::new(OPS_PROFILER_CAPABILITY)], + disabled_by_default: false, + }, + ]; assert!(validate_extension_schemas(&schemas).is_ok()); } @@ -536,6 +709,231 @@ mod tests { assert_eq!(OPS_DIAGNOSTICS_CAPABILITY, "ops.diagnostics.v1"); } + fn profiler_backend( + backend: &str, + status: OpsProfilerBackendStatus, + supports_profile_export: bool, + ) -> OpsProfilerBackendCapability { + OpsProfilerBackendCapability { + backend: OpsProfilerBackendName::new(backend), + status, + supports_profile_export, + redaction_required: vec![ + OpsProfilerRedactionField::Secret, + OpsProfilerRedactionField::Token, + OpsProfilerRedactionField::LocalPath, + OpsProfilerRedactionField::Host, + ], + provenance: OpsProfilerProvenance { + source: "rustfs.profiling".to_string(), + collection_boundary: "rustfs-process".to_string(), + trust_level: OpsProfilerTrustLevel::RuntimeTrusted, + }, + } + } + + #[test] + fn validates_ops_profiler_contract_states_and_redaction() { + let contract = OpsProfilerContract { + mode: OpsProfilerContractMode::CapabilityDescription, + backends: vec![ + profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true), + profiler_backend("memory_pprof", OpsProfilerBackendStatus::Disabled, false), + profiler_backend("ebpf", OpsProfilerBackendStatus::Unsupported, false), + profiler_backend("future_kernel_profiler", OpsProfilerBackendStatus::Unknown, false), + ], + }; + + assert!(validate_ops_profiler_contract(&contract).is_ok()); + assert_eq!(OPS_PROFILER_CAPABILITY, "ops.profiler.v1"); + assert!( + contract + .backends + .iter() + .all(|backend| !backend.provenance.source.contains("token")) + ); + } + + #[test] + fn ops_profiler_schema_serializes_stable_json_shape() { + let contract = OpsProfilerContract { + mode: OpsProfilerContractMode::CapabilityDescription, + backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)], + }; + + let value = serde_json::to_value(contract).expect("ops profiler schema should serialize"); + + assert_eq!( + value, + json!({ + "mode": "capability_description", + "backends": [{ + "backend": "cpu_pprof", + "status": "enabled", + "supports_profile_export": true, + "redaction_required": ["secret", "token", "local_path", "host"], + "provenance": { + "source": "rustfs.profiling", + "collection_boundary": "rustfs-process", + "trust_level": "runtime_trusted" + } + }] + }) + ); + } + + #[test] + fn ops_profiler_schema_accepts_unknown_future_backend_names() { + let contract: OpsProfilerContract = serde_json::from_value(json!({ + "mode": "capability_description", + "backends": [{ + "backend": "vendor.future-profiler", + "status": "unknown", + "supports_profile_export": false, + "redaction_required": ["host"], + "provenance": { + "source": "extension.schema", + "collection_boundary": "read_only_inventory", + "trust_level": "unknown" + } + }] + })) + .expect("unknown future backend names should remain representable"); + + assert!(validate_ops_profiler_contract(&contract).is_ok()); + } + + #[test] + fn rejects_ops_profiler_execution_requests_and_empty_backends() { + let mut contract = OpsProfilerContract { + mode: OpsProfilerContractMode::ExecutionRequest, + backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)], + }; + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("execution requests are outside schema scope"), + ExtensionContractError::OpsProfilerExecutionRequest + ); + + contract.mode = OpsProfilerContractMode::CapabilityDescription; + contract.backends.clear(); + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("profiler capability must declare backends"), + ExtensionContractError::EmptyOpsProfilerBackends + ); + } + + #[test] + fn rejects_ops_profiler_missing_required_fields() { + let err = serde_json::from_value::(json!({ + "mode": "capability_description", + "backends": [{ + "backend": "cpu_pprof", + "status": "enabled", + "supports_profile_export": true, + "redaction_required": ["local_path"] + }] + })) + .expect_err("provenance is required"); + + assert!(err.to_string().contains("missing field `provenance`")); + } + + #[test] + fn rejects_ops_profiler_unknown_credential_fields() { + let err = serde_json::from_value::(json!({ + "mode": "capability_description", + "backends": [{ + "backend": "cpu_pprof", + "status": "enabled", + "supports_profile_export": true, + "redaction_required": ["local_path"], + "provenance": { + "source": "rustfs.profiling", + "collection_boundary": "rustfs-process", + "trust_level": "runtime_trusted", + "credentials": "not-allowed" + } + }] + })) + .expect_err("credential-bearing fields are outside the schema"); + + assert!(err.to_string().contains("unknown field `credentials`")); + } + + #[test] + fn rejects_ops_profiler_invalid_backend_and_redaction_shape() { + let mut contract = OpsProfilerContract { + mode: OpsProfilerContractMode::CapabilityDescription, + backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)], + }; + contract.backends[0].backend = OpsProfilerBackendName::new(" "); + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("backend name should be required"), + ExtensionContractError::EmptyOpsProfilerBackend + ); + + contract.backends[0] = profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true); + contract + .backends + .push(profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Disabled, false)); + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("duplicate backends should fail validation"), + ExtensionContractError::DuplicateOpsProfilerBackend { + backend: "cpu_pprof".to_string() + } + ); + + contract.backends.truncate(1); + contract.backends[0].redaction_required.push(OpsProfilerRedactionField::Host); + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("duplicate redaction fields should fail validation"), + ExtensionContractError::DuplicateOpsProfilerRedactionField { + backend: "cpu_pprof".to_string(), + field: OpsProfilerRedactionField::Host + } + ); + + contract.backends[0].redaction_required = vec![OpsProfilerRedactionField::Host]; + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("profile export requires local path redaction"), + ExtensionContractError::OpsProfilerMissingLocalPathRedaction { + backend: "cpu_pprof".to_string() + } + ); + } + + #[test] + fn rejects_ops_profiler_missing_provenance_boundary() { + let mut contract = OpsProfilerContract { + mode: OpsProfilerContractMode::CapabilityDescription, + backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)], + }; + contract.backends[0].provenance.source = " ".to_string(); + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("provenance source should be required"), + ExtensionContractError::EmptyOpsProfilerProvenanceSource { + backend: "cpu_pprof".to_string() + } + ); + + contract.backends[0].provenance.source = "rustfs.profiling".to_string(); + contract.backends[0].provenance.collection_boundary.clear(); + + assert_eq!( + validate_ops_profiler_contract(&contract).expect_err("collection boundary should be required"), + ExtensionContractError::EmptyOpsProfilerCollectionBoundary { + backend: "cpu_pprof".to_string() + } + ); + } + #[test] fn rejects_ops_diagnostics_without_admin_action_or_read_only_contract() { let mut contract = OpsDiagnosticsContract { diff --git a/docs/architecture/migration-progress.md b/docs/architecture/migration-progress.md index caf03cb52..31f4a9835 100644 --- a/docs/architecture/migration-progress.md +++ b/docs/architecture/migration-progress.md @@ -5,16 +5,15 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block ## Current Context - Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660) -- Branch: `overtrue/arch-final-object-boundary-prune` -- Baseline: stacked after `rustfs/rustfs#3622` - (`4300bf04f3a82ef4cc2605f417f89e7b4643b99e`). -- PR type for this branch: `consumer-migration` +- Branch: `overtrue/arch-ops-profiler-schema` +- Baseline: latest `origin/main` + (`71809ba02ea405c3a2f0e4cb82ff608915dd519c`). +- PR type for this branch: `contract` - Runtime behavior changes: none. -- Rust code changes: route heal and RustFS storage object reader, writer, - metadata, and options aliases through `rustfs_storage_api` associated types. -- CI/script changes: shrink the remaining external `rustfs_ecstore::object_api` - compatibility alias snapshot to empty. -- Docs changes: record the API-071 final object boundary prune slice. +- Rust code changes: define the `ops.profiler.v1` extension schema contract for + profiler capability reporting, backend state, redaction, and provenance. +- CI/script changes: none. +- Docs changes: record the X-012 profiler schema contract slice. ## Phase 0 Tasks @@ -1985,26 +1984,52 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block checks, formatting, migration guards, Rust risk scan, branch freshness check, and pre-commit quality gate. +- [x] `X-012` Define ops profiler extension schema contract. + - Do: add `ops.profiler.v1` capability DTOs for profiler backend status, + capability-description mode, profile export redaction requirements, and + provenance in the extension schema contract crate. + - Acceptance: disabled, unsupported, enabled, and unknown backend states are + representable; execution requests are rejected; profile export declarations + require local path redaction; provenance records source, collection + boundary, and trust level without credentials. + - Must preserve: no plugin execution, no sidecar startup, no profile route or + admin API behavior changes, no exporter/storage/object-path/telemetry + behavior changes, and no dependency edge from `extension-schema` to + implementation crates. + - Verification: extension schema check/tests, formatting, migration/layer + guards, diff hygiene, Rust risk scan, branch freshness check, and + pre-commit quality gate. + ## Next PRs -1. `pure-move`: continue larger lifecycle hook slices for optional runtime +1. `contract`: continue larger extension contract coverage for disabled and + unsupported runtime capability snapshots. +2. `pure-move`: continue larger lifecycle hook slices for optional runtime sidecars while preserving startup and shutdown ordering. -2. `consumer-migration`: continue larger cleanup slices with the - loss-prevention guards active for remaining ECStore compatibility contracts - that still have dedicated preservation coverage. ## Pre-Push Review Log | Expert | Status | Notes | |---|---|---| -| Quality/architecture | passed | R-020 moves profiling init/shutdown call sites behind `startup_profiling` hook functions without moving CPU/memory profiling implementation ownership or admin pprof APIs. | -| Migration preservation | passed | BOOT-006 still initializes profiling before trusted proxies/provider/TLS setup; STOP-004 still stops profiling after notifier/audit shutdown and before HTTP shutdown; profiling env flags, platform gates, target unsupported behavior, and cancellation-token ownership remain in `profiling.rs`. | -| Testing/verification | passed | Focused startup profiling hook tests, RustFS library check, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` validate the current R-020 slice. | +| Quality/architecture | passed | X-012 keeps the profiler schema in `rustfs-extension-schema`, uses explicit backend/status/redaction/provenance DTOs, and adds no new dependency edges or runtime abstractions. | +| Migration preservation | passed | Runtime profiling, admin pprof routes, exporters, storage paths, telemetry, sidecar startup, and plugin execution are untouched; the new contract only describes capabilities. | +| Testing/verification | passed | Extension schema tests cover enabled, disabled, unsupported, unknown, missing-field, redaction, provenance, and credential-field rejection; focused checks, guards, formatting, diff hygiene, and final pre-commit passed. | ## Verification Notes Passed before push: +- Issue #660 X-012 current slice: + - `cargo test -p rustfs-extension-schema`: passed. + - `cargo check -p rustfs-extension-schema`: passed. + - `./scripts/check_architecture_migration_rules.sh`: passed. + - `./scripts/check_layer_dependencies.sh`: passed. + - `cargo fmt --all --check`: passed. + - `git diff --check`: passed. + - Rust risk scan on changed Rust files: passed. + - `make pre-commit`: passed. + - Three-expert review: passed. + - Issue #660 R-020 current slice: - `cargo test -p rustfs --lib startup_profiling -- --nocapture`: passed. - `cargo check -p rustfs --lib`: passed. diff --git a/docs/architecture/profiling-numa-capability-inventory.md b/docs/architecture/profiling-numa-capability-inventory.md index 1fbac5334..682c22227 100644 --- a/docs/architecture/profiling-numa-capability-inventory.md +++ b/docs/architecture/profiling-numa-capability-inventory.md @@ -66,9 +66,12 @@ Future sidecars for profiling, eBPF, or NUMA must preserve these invariants: `X-012`: -- Add an optional profiling/eBPF extension point that can observe capability - status. -- Keep unsupported targets and disabled sidecars as no-op. +- Define the `ops.profiler.v1` extension schema for profiling capability + reporting, backend status, redaction requirements, and provenance. +- Keep the schema capability-only; it must not request profiler execution, + start sidecars, or change profile export behavior. +- Keep unsupported targets, disabled sidecars, and unknown future backends + representable as no-op capability states. `X-013`: