feat: add ops profiler extension schema (#3628)

This commit is contained in:
安正超
2026-06-19 22:31:18 +08:00
committed by GitHub
parent 71809ba0bb
commit ff638d1140
3 changed files with 461 additions and 35 deletions
+414 -16
View File
@@ -19,6 +19,7 @@ use thiserror::Error;
pub const EXTENSION_SCHEMA_VERSION: &str = "rustfs.extension-schema.v1";
pub const OPS_DIAGNOSTICS_CAPABILITY: &str = "ops.diagnostics.v1";
pub const OPS_PROFILER_CAPABILITY: &str = "ops.profiler.v1";
pub const S3_POST_AUTH_HOOK_CAPABILITY: &str = "s3.hook.post_auth.v1";
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
@@ -117,6 +118,79 @@ pub struct OpsDiagnosticsContract {
pub requires_admin_action: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OpsProfilerContractMode {
CapabilityDescription,
ExecutionRequest,
}
#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(transparent)]
pub struct OpsProfilerBackendName(String);
impl OpsProfilerBackendName {
pub fn new(backend: impl Into<String>) -> Self {
Self(backend.into())
}
pub fn as_str(&self) -> &str {
self.0.as_str()
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OpsProfilerBackendStatus {
Enabled,
Disabled,
Unsupported,
Unknown,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OpsProfilerRedactionField {
Secret,
Token,
LocalPath,
Host,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
#[serde(rename_all = "snake_case")]
pub enum OpsProfilerTrustLevel {
RuntimeTrusted,
AdminTrusted,
ExtensionProvided,
Unknown,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OpsProfilerProvenance {
pub source: String,
pub collection_boundary: String,
pub trust_level: OpsProfilerTrustLevel,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OpsProfilerBackendCapability {
pub backend: OpsProfilerBackendName,
pub status: OpsProfilerBackendStatus,
pub supports_profile_export: bool,
pub redaction_required: Vec<OpsProfilerRedactionField>,
pub provenance: OpsProfilerProvenance,
}
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct OpsProfilerContract {
pub mode: OpsProfilerContractMode,
pub backends: Vec<OpsProfilerBackendCapability>,
}
#[derive(Debug, Error, PartialEq, Eq)]
pub enum ExtensionSchemaError {
#[error("extension schema at index {index} has an empty extension id")]
@@ -178,6 +252,33 @@ pub enum ExtensionContractError {
#[error("ops diagnostics contract must require an admin action")]
OpsDiagnosticsMissingAdminAction,
#[error("ops profiler contract must describe capabilities, not execution requests")]
OpsProfilerExecutionRequest,
#[error("ops profiler contract must declare at least one backend")]
EmptyOpsProfilerBackends,
#[error("ops profiler contract has an empty backend name")]
EmptyOpsProfilerBackend,
#[error("ops profiler contract duplicates backend {backend}")]
DuplicateOpsProfilerBackend { backend: String },
#[error("ops profiler backend {backend} duplicates redaction field {field:?}")]
DuplicateOpsProfilerRedactionField {
backend: String,
field: OpsProfilerRedactionField,
},
#[error("ops profiler backend {backend} exports profiles without local path redaction")]
OpsProfilerMissingLocalPathRedaction { backend: String },
#[error("ops profiler backend {backend} has an empty provenance source")]
EmptyOpsProfilerProvenanceSource { backend: String },
#[error("ops profiler backend {backend} has an empty collection boundary")]
EmptyOpsProfilerCollectionBoundary { backend: String },
}
pub fn validate_extension_schemas(schemas: &[ExtensionSchema]) -> Result<(), ExtensionSchemaError> {
@@ -304,13 +405,69 @@ pub fn validate_ops_diagnostics_contract(contract: &OpsDiagnosticsContract) -> R
Ok(())
}
pub fn validate_ops_profiler_contract(contract: &OpsProfilerContract) -> Result<(), ExtensionContractError> {
if contract.mode != OpsProfilerContractMode::CapabilityDescription {
return Err(ExtensionContractError::OpsProfilerExecutionRequest);
}
if contract.backends.is_empty() {
return Err(ExtensionContractError::EmptyOpsProfilerBackends);
}
let mut backends = BTreeSet::new();
for backend in &contract.backends {
let backend_name = backend.backend.as_str().trim();
if backend_name.is_empty() {
return Err(ExtensionContractError::EmptyOpsProfilerBackend);
}
if !backends.insert(backend_name) {
return Err(ExtensionContractError::DuplicateOpsProfilerBackend {
backend: backend_name.to_string(),
});
}
let mut redaction_fields = BTreeSet::new();
for field in &backend.redaction_required {
if !redaction_fields.insert(*field) {
return Err(ExtensionContractError::DuplicateOpsProfilerRedactionField {
backend: backend_name.to_string(),
field: *field,
});
}
}
if backend.supports_profile_export && !redaction_fields.contains(&OpsProfilerRedactionField::LocalPath) {
return Err(ExtensionContractError::OpsProfilerMissingLocalPathRedaction {
backend: backend_name.to_string(),
});
}
if backend.provenance.source.trim().is_empty() {
return Err(ExtensionContractError::EmptyOpsProfilerProvenanceSource {
backend: backend_name.to_string(),
});
}
if backend.provenance.collection_boundary.trim().is_empty() {
return Err(ExtensionContractError::EmptyOpsProfilerCollectionBoundary {
backend: backend_name.to_string(),
});
}
}
Ok(())
}
#[cfg(test)]
mod tests {
use super::{
EXTENSION_SCHEMA_VERSION, ExtensionCapabilityRef, ExtensionContractError, ExtensionKind, ExtensionRuntimeBoundary,
ExtensionRuntimeContract, ExtensionSchema, ExtensionSchemaError, OPS_DIAGNOSTICS_CAPABILITY, OpsDiagnosticSurface,
OpsDiagnosticsContract, S3_POST_AUTH_HOOK_CAPABILITY, S3HookContract, S3HookPoint, validate_extension_schemas,
validate_ops_diagnostics_contract, validate_s3_hook_contract,
ExtensionRuntimeContract, ExtensionSchema, ExtensionSchemaError, OPS_DIAGNOSTICS_CAPABILITY, OPS_PROFILER_CAPABILITY,
OpsDiagnosticSurface, OpsDiagnosticsContract, OpsProfilerBackendCapability, OpsProfilerBackendName,
OpsProfilerBackendStatus, OpsProfilerContract, OpsProfilerContractMode, OpsProfilerProvenance, OpsProfilerRedactionField,
OpsProfilerTrustLevel, S3_POST_AUTH_HOOK_CAPABILITY, S3HookContract, S3HookPoint, validate_extension_schemas,
validate_ops_diagnostics_contract, validate_ops_profiler_contract, validate_s3_hook_contract,
};
use serde_json::json;
@@ -363,20 +520,36 @@ mod tests {
#[test]
fn validates_extension_schema_contracts() {
let schemas = [ExtensionSchema {
schema_version: EXTENSION_SCHEMA_VERSION.to_string(),
extension_id: "rustfs.ops.diagnostics".to_string(),
display_name: "Ops Diagnostics".to_string(),
provider: "rustfs".to_string(),
version: "1.0.0".to_string(),
kind: ExtensionKind::OpsDiagnostics,
runtime: ExtensionRuntimeContract {
api_version: "rustfs.extension.v1".to_string(),
boundary: ExtensionRuntimeBoundary::Builtin,
let schemas = [
ExtensionSchema {
schema_version: EXTENSION_SCHEMA_VERSION.to_string(),
extension_id: "rustfs.ops.diagnostics".to_string(),
display_name: "Ops Diagnostics".to_string(),
provider: "rustfs".to_string(),
version: "1.0.0".to_string(),
kind: ExtensionKind::OpsDiagnostics,
runtime: ExtensionRuntimeContract {
api_version: "rustfs.extension.v1".to_string(),
boundary: ExtensionRuntimeBoundary::Builtin,
},
capabilities: vec![ExtensionCapabilityRef::new(OPS_DIAGNOSTICS_CAPABILITY)],
disabled_by_default: false,
},
capabilities: vec![ExtensionCapabilityRef::new(OPS_DIAGNOSTICS_CAPABILITY)],
disabled_by_default: false,
}];
ExtensionSchema {
schema_version: EXTENSION_SCHEMA_VERSION.to_string(),
extension_id: "rustfs.ops.profiler".to_string(),
display_name: "Ops Profiler".to_string(),
provider: "rustfs".to_string(),
version: "1.0.0".to_string(),
kind: ExtensionKind::OpsDiagnostics,
runtime: ExtensionRuntimeContract {
api_version: "rustfs.extension.v1".to_string(),
boundary: ExtensionRuntimeBoundary::Builtin,
},
capabilities: vec![ExtensionCapabilityRef::new(OPS_PROFILER_CAPABILITY)],
disabled_by_default: false,
},
];
assert!(validate_extension_schemas(&schemas).is_ok());
}
@@ -536,6 +709,231 @@ mod tests {
assert_eq!(OPS_DIAGNOSTICS_CAPABILITY, "ops.diagnostics.v1");
}
fn profiler_backend(
backend: &str,
status: OpsProfilerBackendStatus,
supports_profile_export: bool,
) -> OpsProfilerBackendCapability {
OpsProfilerBackendCapability {
backend: OpsProfilerBackendName::new(backend),
status,
supports_profile_export,
redaction_required: vec![
OpsProfilerRedactionField::Secret,
OpsProfilerRedactionField::Token,
OpsProfilerRedactionField::LocalPath,
OpsProfilerRedactionField::Host,
],
provenance: OpsProfilerProvenance {
source: "rustfs.profiling".to_string(),
collection_boundary: "rustfs-process".to_string(),
trust_level: OpsProfilerTrustLevel::RuntimeTrusted,
},
}
}
#[test]
fn validates_ops_profiler_contract_states_and_redaction() {
let contract = OpsProfilerContract {
mode: OpsProfilerContractMode::CapabilityDescription,
backends: vec![
profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true),
profiler_backend("memory_pprof", OpsProfilerBackendStatus::Disabled, false),
profiler_backend("ebpf", OpsProfilerBackendStatus::Unsupported, false),
profiler_backend("future_kernel_profiler", OpsProfilerBackendStatus::Unknown, false),
],
};
assert!(validate_ops_profiler_contract(&contract).is_ok());
assert_eq!(OPS_PROFILER_CAPABILITY, "ops.profiler.v1");
assert!(
contract
.backends
.iter()
.all(|backend| !backend.provenance.source.contains("token"))
);
}
#[test]
fn ops_profiler_schema_serializes_stable_json_shape() {
let contract = OpsProfilerContract {
mode: OpsProfilerContractMode::CapabilityDescription,
backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
};
let value = serde_json::to_value(contract).expect("ops profiler schema should serialize");
assert_eq!(
value,
json!({
"mode": "capability_description",
"backends": [{
"backend": "cpu_pprof",
"status": "enabled",
"supports_profile_export": true,
"redaction_required": ["secret", "token", "local_path", "host"],
"provenance": {
"source": "rustfs.profiling",
"collection_boundary": "rustfs-process",
"trust_level": "runtime_trusted"
}
}]
})
);
}
#[test]
fn ops_profiler_schema_accepts_unknown_future_backend_names() {
let contract: OpsProfilerContract = serde_json::from_value(json!({
"mode": "capability_description",
"backends": [{
"backend": "vendor.future-profiler",
"status": "unknown",
"supports_profile_export": false,
"redaction_required": ["host"],
"provenance": {
"source": "extension.schema",
"collection_boundary": "read_only_inventory",
"trust_level": "unknown"
}
}]
}))
.expect("unknown future backend names should remain representable");
assert!(validate_ops_profiler_contract(&contract).is_ok());
}
#[test]
fn rejects_ops_profiler_execution_requests_and_empty_backends() {
let mut contract = OpsProfilerContract {
mode: OpsProfilerContractMode::ExecutionRequest,
backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
};
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("execution requests are outside schema scope"),
ExtensionContractError::OpsProfilerExecutionRequest
);
contract.mode = OpsProfilerContractMode::CapabilityDescription;
contract.backends.clear();
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("profiler capability must declare backends"),
ExtensionContractError::EmptyOpsProfilerBackends
);
}
#[test]
fn rejects_ops_profiler_missing_required_fields() {
let err = serde_json::from_value::<OpsProfilerContract>(json!({
"mode": "capability_description",
"backends": [{
"backend": "cpu_pprof",
"status": "enabled",
"supports_profile_export": true,
"redaction_required": ["local_path"]
}]
}))
.expect_err("provenance is required");
assert!(err.to_string().contains("missing field `provenance`"));
}
#[test]
fn rejects_ops_profiler_unknown_credential_fields() {
let err = serde_json::from_value::<OpsProfilerContract>(json!({
"mode": "capability_description",
"backends": [{
"backend": "cpu_pprof",
"status": "enabled",
"supports_profile_export": true,
"redaction_required": ["local_path"],
"provenance": {
"source": "rustfs.profiling",
"collection_boundary": "rustfs-process",
"trust_level": "runtime_trusted",
"credentials": "not-allowed"
}
}]
}))
.expect_err("credential-bearing fields are outside the schema");
assert!(err.to_string().contains("unknown field `credentials`"));
}
#[test]
fn rejects_ops_profiler_invalid_backend_and_redaction_shape() {
let mut contract = OpsProfilerContract {
mode: OpsProfilerContractMode::CapabilityDescription,
backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
};
contract.backends[0].backend = OpsProfilerBackendName::new(" ");
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("backend name should be required"),
ExtensionContractError::EmptyOpsProfilerBackend
);
contract.backends[0] = profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true);
contract
.backends
.push(profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Disabled, false));
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("duplicate backends should fail validation"),
ExtensionContractError::DuplicateOpsProfilerBackend {
backend: "cpu_pprof".to_string()
}
);
contract.backends.truncate(1);
contract.backends[0].redaction_required.push(OpsProfilerRedactionField::Host);
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("duplicate redaction fields should fail validation"),
ExtensionContractError::DuplicateOpsProfilerRedactionField {
backend: "cpu_pprof".to_string(),
field: OpsProfilerRedactionField::Host
}
);
contract.backends[0].redaction_required = vec![OpsProfilerRedactionField::Host];
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("profile export requires local path redaction"),
ExtensionContractError::OpsProfilerMissingLocalPathRedaction {
backend: "cpu_pprof".to_string()
}
);
}
#[test]
fn rejects_ops_profiler_missing_provenance_boundary() {
let mut contract = OpsProfilerContract {
mode: OpsProfilerContractMode::CapabilityDescription,
backends: vec![profiler_backend("cpu_pprof", OpsProfilerBackendStatus::Enabled, true)],
};
contract.backends[0].provenance.source = " ".to_string();
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("provenance source should be required"),
ExtensionContractError::EmptyOpsProfilerProvenanceSource {
backend: "cpu_pprof".to_string()
}
);
contract.backends[0].provenance.source = "rustfs.profiling".to_string();
contract.backends[0].provenance.collection_boundary.clear();
assert_eq!(
validate_ops_profiler_contract(&contract).expect_err("collection boundary should be required"),
ExtensionContractError::EmptyOpsProfilerCollectionBoundary {
backend: "cpu_pprof".to_string()
}
);
}
#[test]
fn rejects_ops_diagnostics_without_admin_action_or_read_only_contract() {
let mut contract = OpsDiagnosticsContract {
+41 -16
View File
@@ -5,16 +5,15 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
## Current Context
- Issue: [`rustfs/backlog#660`](https://github.com/rustfs/backlog/issues/660)
- Branch: `overtrue/arch-final-object-boundary-prune`
- Baseline: stacked after `rustfs/rustfs#3622`
(`4300bf04f3a82ef4cc2605f417f89e7b4643b99e`).
- PR type for this branch: `consumer-migration`
- Branch: `overtrue/arch-ops-profiler-schema`
- Baseline: latest `origin/main`
(`71809ba02ea405c3a2f0e4cb82ff608915dd519c`).
- PR type for this branch: `contract`
- Runtime behavior changes: none.
- Rust code changes: route heal and RustFS storage object reader, writer,
metadata, and options aliases through `rustfs_storage_api` associated types.
- CI/script changes: shrink the remaining external `rustfs_ecstore::object_api`
compatibility alias snapshot to empty.
- Docs changes: record the API-071 final object boundary prune slice.
- Rust code changes: define the `ops.profiler.v1` extension schema contract for
profiler capability reporting, backend state, redaction, and provenance.
- CI/script changes: none.
- Docs changes: record the X-012 profiler schema contract slice.
## Phase 0 Tasks
@@ -1985,26 +1984,52 @@ Status values: `[ ]` not started, `[~]` in progress, `[x]` complete, `[!]` block
checks, formatting, migration guards, Rust risk scan, branch freshness
check, and pre-commit quality gate.
- [x] `X-012` Define ops profiler extension schema contract.
- Do: add `ops.profiler.v1` capability DTOs for profiler backend status,
capability-description mode, profile export redaction requirements, and
provenance in the extension schema contract crate.
- Acceptance: disabled, unsupported, enabled, and unknown backend states are
representable; execution requests are rejected; profile export declarations
require local path redaction; provenance records source, collection
boundary, and trust level without credentials.
- Must preserve: no plugin execution, no sidecar startup, no profile route or
admin API behavior changes, no exporter/storage/object-path/telemetry
behavior changes, and no dependency edge from `extension-schema` to
implementation crates.
- Verification: extension schema check/tests, formatting, migration/layer
guards, diff hygiene, Rust risk scan, branch freshness check, and
pre-commit quality gate.
## Next PRs
1. `pure-move`: continue larger lifecycle hook slices for optional runtime
1. `contract`: continue larger extension contract coverage for disabled and
unsupported runtime capability snapshots.
2. `pure-move`: continue larger lifecycle hook slices for optional runtime
sidecars while preserving startup and shutdown ordering.
2. `consumer-migration`: continue larger cleanup slices with the
loss-prevention guards active for remaining ECStore compatibility contracts
that still have dedicated preservation coverage.
## Pre-Push Review Log
| Expert | Status | Notes |
|---|---|---|
| Quality/architecture | passed | R-020 moves profiling init/shutdown call sites behind `startup_profiling` hook functions without moving CPU/memory profiling implementation ownership or admin pprof APIs. |
| Migration preservation | passed | BOOT-006 still initializes profiling before trusted proxies/provider/TLS setup; STOP-004 still stops profiling after notifier/audit shutdown and before HTTP shutdown; profiling env flags, platform gates, target unsupported behavior, and cancellation-token ownership remain in `profiling.rs`. |
| Testing/verification | passed | Focused startup profiling hook tests, RustFS library check, migration/layer guards, formatting, diff hygiene, Rust risk scan, and full `make pre-commit` validate the current R-020 slice. |
| Quality/architecture | passed | X-012 keeps the profiler schema in `rustfs-extension-schema`, uses explicit backend/status/redaction/provenance DTOs, and adds no new dependency edges or runtime abstractions. |
| Migration preservation | passed | Runtime profiling, admin pprof routes, exporters, storage paths, telemetry, sidecar startup, and plugin execution are untouched; the new contract only describes capabilities. |
| Testing/verification | passed | Extension schema tests cover enabled, disabled, unsupported, unknown, missing-field, redaction, provenance, and credential-field rejection; focused checks, guards, formatting, diff hygiene, and final pre-commit passed. |
## Verification Notes
Passed before push:
- Issue #660 X-012 current slice:
- `cargo test -p rustfs-extension-schema`: passed.
- `cargo check -p rustfs-extension-schema`: passed.
- `./scripts/check_architecture_migration_rules.sh`: passed.
- `./scripts/check_layer_dependencies.sh`: passed.
- `cargo fmt --all --check`: passed.
- `git diff --check`: passed.
- Rust risk scan on changed Rust files: passed.
- `make pre-commit`: passed.
- Three-expert review: passed.
- Issue #660 R-020 current slice:
- `cargo test -p rustfs --lib startup_profiling -- --nocapture`: passed.
- `cargo check -p rustfs --lib`: passed.
@@ -66,9 +66,12 @@ Future sidecars for profiling, eBPF, or NUMA must preserve these invariants:
`X-012`:
- Add an optional profiling/eBPF extension point that can observe capability
status.
- Keep unsupported targets and disabled sidecars as no-op.
- Define the `ops.profiler.v1` extension schema for profiling capability
reporting, backend status, redaction requirements, and provenance.
- Keep the schema capability-only; it must not request profiler execution,
start sidecars, or change profile export behavior.
- Keep unsupported targets, disabled sidecars, and unknown future backends
representable as no-op capability states.
`X-013`: