mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-03 20:07:42 +00:00
feat(targets): gate sidecar runtime policy (#3388)
* feat(targets): gate sidecar runtime policy * fix(admin): add extension route policy specs
This commit is contained in:
@@ -22,7 +22,7 @@ use crate::manifest::{
|
||||
TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginRuntimeTransport,
|
||||
installable_target_marketplace_manifest,
|
||||
};
|
||||
use crate::runtime::sidecar::SidecarPluginRuntime;
|
||||
use crate::runtime::sidecar::{SidecarPluginRuntime, SidecarRuntimePolicy, SidecarRuntimeSafetyChecks};
|
||||
use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
@@ -76,8 +76,13 @@ pub fn example_external_webhook_plugin() -> ExampleInstallableTargetPlugin {
|
||||
};
|
||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", handshake);
|
||||
runtime
|
||||
.enable(base.plugin_id, TargetDomain::Notify)
|
||||
.expect("example sidecar plugin handshake should validate");
|
||||
.enable_with_policy(
|
||||
base.plugin_id,
|
||||
TargetDomain::Notify,
|
||||
&SidecarRuntimePolicy::verified_external(16, std::time::Duration::from_secs(5), 3),
|
||||
&SidecarRuntimeSafetyChecks::verified(0),
|
||||
)
|
||||
.expect("example sidecar plugin policy should validate");
|
||||
|
||||
ExampleInstallableTargetPlugin {
|
||||
manifest,
|
||||
|
||||
@@ -63,7 +63,7 @@ pub use runtime::{
|
||||
RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
|
||||
adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
|
||||
init_target_and_optionally_start_replay,
|
||||
sidecar::SidecarPluginRuntime,
|
||||
sidecar::{SidecarPluginRuntime, SidecarRuntimePolicy, SidecarRuntimeSafetyChecks},
|
||||
sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability},
|
||||
start_replay_worker,
|
||||
};
|
||||
|
||||
@@ -19,6 +19,66 @@ use std::time::Duration;
|
||||
|
||||
const DEFAULT_FAILURE_THRESHOLD: usize = 3;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SidecarRuntimePolicy {
|
||||
pub allow_external_sidecars: bool,
|
||||
pub require_sandbox: bool,
|
||||
pub require_provenance: bool,
|
||||
pub max_queue_depth: usize,
|
||||
pub operation_timeout: Duration,
|
||||
pub failure_threshold: usize,
|
||||
pub redact_error_details: bool,
|
||||
}
|
||||
|
||||
impl Default for SidecarRuntimePolicy {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
allow_external_sidecars: false,
|
||||
require_sandbox: true,
|
||||
require_provenance: true,
|
||||
max_queue_depth: 0,
|
||||
operation_timeout: Duration::from_secs(5),
|
||||
failure_threshold: DEFAULT_FAILURE_THRESHOLD,
|
||||
redact_error_details: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SidecarRuntimePolicy {
|
||||
pub fn verified_external(max_queue_depth: usize, operation_timeout: Duration, failure_threshold: usize) -> Self {
|
||||
Self {
|
||||
allow_external_sidecars: true,
|
||||
require_sandbox: true,
|
||||
require_provenance: true,
|
||||
max_queue_depth,
|
||||
operation_timeout,
|
||||
failure_threshold: failure_threshold.max(1),
|
||||
redact_error_details: true,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn failure_threshold(&self) -> usize {
|
||||
self.failure_threshold.max(1)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SidecarRuntimeSafetyChecks {
|
||||
pub sandboxed: bool,
|
||||
pub provenance_verified: bool,
|
||||
pub queue_depth: usize,
|
||||
}
|
||||
|
||||
impl SidecarRuntimeSafetyChecks {
|
||||
pub fn verified(queue_depth: usize) -> Self {
|
||||
Self {
|
||||
sandboxed: true,
|
||||
provenance_verified: true,
|
||||
queue_depth,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub struct SidecarPluginRuntime {
|
||||
@@ -58,6 +118,29 @@ impl SidecarPluginRuntime {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn enable_with_policy(
|
||||
&mut self,
|
||||
expected_plugin_id: &str,
|
||||
required_domain: TargetDomain,
|
||||
policy: &SidecarRuntimePolicy,
|
||||
safety_checks: &SidecarRuntimeSafetyChecks,
|
||||
) -> Result<(), String> {
|
||||
self.handshake.validate(expected_plugin_id)?;
|
||||
if !self.handshake.supported_domains.contains(&required_domain) {
|
||||
return Err(format!(
|
||||
"sidecar plugin {} does not support required domain {:?}",
|
||||
self.handshake.plugin_id, required_domain
|
||||
));
|
||||
}
|
||||
validate_runtime_policy(policy, safety_checks)?;
|
||||
|
||||
self.healthy = true;
|
||||
self.degraded_to_builtin = false;
|
||||
self.last_error = None;
|
||||
self.failure_count = 0;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn mark_unhealthy(&mut self) {
|
||||
self.healthy = false;
|
||||
}
|
||||
@@ -71,6 +154,19 @@ impl SidecarPluginRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn record_failure_with_policy(&mut self, policy: &SidecarRuntimePolicy, error: impl Into<String>) {
|
||||
self.failure_count = self.failure_count.saturating_add(1);
|
||||
self.healthy = false;
|
||||
self.last_error = Some(if policy.redact_error_details {
|
||||
"sidecar operation failed".to_string()
|
||||
} else {
|
||||
error.into()
|
||||
});
|
||||
if self.failure_count >= policy.failure_threshold() {
|
||||
self.degraded_to_builtin = true;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send_with_timeout(&mut self, operation_timeout: Duration, simulated_latency: Duration) -> Result<(), String> {
|
||||
if simulated_latency > operation_timeout {
|
||||
self.record_failure(format!(
|
||||
@@ -92,9 +188,28 @@ impl SidecarPluginRuntime {
|
||||
}
|
||||
}
|
||||
|
||||
fn validate_runtime_policy(policy: &SidecarRuntimePolicy, safety_checks: &SidecarRuntimeSafetyChecks) -> Result<(), String> {
|
||||
if !policy.allow_external_sidecars {
|
||||
return Err("external sidecar runtime is disabled by policy".to_string());
|
||||
}
|
||||
if policy.require_sandbox && !safety_checks.sandboxed {
|
||||
return Err("sidecar runtime requires sandbox isolation".to_string());
|
||||
}
|
||||
if policy.require_provenance && !safety_checks.provenance_verified {
|
||||
return Err("sidecar runtime requires verified provenance".to_string());
|
||||
}
|
||||
if safety_checks.queue_depth > policy.max_queue_depth {
|
||||
return Err(format!(
|
||||
"sidecar runtime queue depth {} exceeds policy bound {}",
|
||||
safety_checks.queue_depth, policy.max_queue_depth
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::SidecarPluginRuntime;
|
||||
use super::{SidecarPluginRuntime, SidecarRuntimePolicy, SidecarRuntimeSafetyChecks};
|
||||
use crate::TargetDomain;
|
||||
use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
|
||||
use std::time::Duration;
|
||||
@@ -124,6 +239,112 @@ mod tests {
|
||||
assert!(runtime.healthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_runtime_policy_rejects_external_activation_by_default() {
|
||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||
|
||||
let result = runtime.enable_with_policy(
|
||||
"external:webhook",
|
||||
TargetDomain::Notify,
|
||||
&SidecarRuntimePolicy::default(),
|
||||
&SidecarRuntimeSafetyChecks::verified(0),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result.as_ref().map_err(String::as_str),
|
||||
Err("external sidecar runtime is disabled by policy")
|
||||
);
|
||||
assert!(!runtime.healthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_runtime_policy_requires_sandbox_and_provenance() {
|
||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||
let policy = SidecarRuntimePolicy::verified_external(16, Duration::from_secs(5), 3);
|
||||
|
||||
let missing_sandbox = runtime.enable_with_policy(
|
||||
"external:webhook",
|
||||
TargetDomain::Notify,
|
||||
&policy,
|
||||
&SidecarRuntimeSafetyChecks {
|
||||
sandboxed: false,
|
||||
provenance_verified: true,
|
||||
queue_depth: 0,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
missing_sandbox.as_ref().map_err(String::as_str),
|
||||
Err("sidecar runtime requires sandbox isolation")
|
||||
);
|
||||
|
||||
let missing_provenance = runtime.enable_with_policy(
|
||||
"external:webhook",
|
||||
TargetDomain::Notify,
|
||||
&policy,
|
||||
&SidecarRuntimeSafetyChecks {
|
||||
sandboxed: true,
|
||||
provenance_verified: false,
|
||||
queue_depth: 0,
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
missing_provenance.as_ref().map_err(String::as_str),
|
||||
Err("sidecar runtime requires verified provenance")
|
||||
);
|
||||
assert!(!runtime.healthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_runtime_policy_enforces_queue_bound() {
|
||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||
let policy = SidecarRuntimePolicy::verified_external(2, Duration::from_secs(5), 3);
|
||||
|
||||
let result = runtime.enable_with_policy(
|
||||
"external:webhook",
|
||||
TargetDomain::Notify,
|
||||
&policy,
|
||||
&SidecarRuntimeSafetyChecks::verified(3),
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
result.as_ref().map_err(String::as_str),
|
||||
Err("sidecar runtime queue depth 3 exceeds policy bound 2")
|
||||
);
|
||||
assert!(!runtime.healthy);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_runtime_policy_allows_verified_external_activation() {
|
||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||
let policy = SidecarRuntimePolicy::verified_external(16, Duration::from_secs(5), 3);
|
||||
|
||||
runtime
|
||||
.enable_with_policy(
|
||||
"external:webhook",
|
||||
TargetDomain::Notify,
|
||||
&policy,
|
||||
&SidecarRuntimeSafetyChecks::verified(1),
|
||||
)
|
||||
.expect("verified sidecar runtime should enable");
|
||||
|
||||
assert!(runtime.healthy);
|
||||
assert_eq!(policy.failure_threshold(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_runtime_policy_redacts_failure_details() {
|
||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||
let policy = SidecarRuntimePolicy::verified_external(16, Duration::from_secs(5), 2);
|
||||
|
||||
runtime.record_failure_with_policy(&policy, "secret token leaked in transport error");
|
||||
runtime.record_failure_with_policy(&policy, "another secret error");
|
||||
|
||||
assert_eq!(runtime.last_error.as_deref(), Some("sidecar operation failed"));
|
||||
assert!(runtime.degraded_to_builtin);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sidecar_runtime_enable_rejects_domain_mismatch() {
|
||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||
|
||||
@@ -334,6 +334,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(HttpMethod::Put, "/rustfs/admin/v3/module-switches", CONFIG_UPDATE, RouteRiskLevel::High),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v4/extensions/catalog",
|
||||
SERVER_INFO,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v4/extensions/instances",
|
||||
GET_BUCKET_TARGET,
|
||||
RouteRiskLevel::Sensitive,
|
||||
),
|
||||
admin(
|
||||
HttpMethod::Get,
|
||||
"/rustfs/admin/v4/plugins/catalog",
|
||||
|
||||
Reference in New Issue
Block a user