mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-03 10:48:13 +00:00
feat(targets): gate sidecar runtime policy (#3388)
* feat(targets): gate sidecar runtime policy * fix(admin): add extension route policy specs
This commit is contained in:
@@ -22,7 +22,7 @@ use crate::manifest::{
|
|||||||
TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginRuntimeTransport,
|
TargetPluginExternalRuntimeContract, TargetPluginManifest, TargetPluginMarketplaceManifest, TargetPluginRuntimeTransport,
|
||||||
installable_target_marketplace_manifest,
|
installable_target_marketplace_manifest,
|
||||||
};
|
};
|
||||||
use crate::runtime::sidecar::SidecarPluginRuntime;
|
use crate::runtime::sidecar::{SidecarPluginRuntime, SidecarRuntimePolicy, SidecarRuntimeSafetyChecks};
|
||||||
use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
|
use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
@@ -76,8 +76,13 @@ pub fn example_external_webhook_plugin() -> ExampleInstallableTargetPlugin {
|
|||||||
};
|
};
|
||||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", handshake);
|
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", handshake);
|
||||||
runtime
|
runtime
|
||||||
.enable(base.plugin_id, TargetDomain::Notify)
|
.enable_with_policy(
|
||||||
.expect("example sidecar plugin handshake should validate");
|
base.plugin_id,
|
||||||
|
TargetDomain::Notify,
|
||||||
|
&SidecarRuntimePolicy::verified_external(16, std::time::Duration::from_secs(5), 3),
|
||||||
|
&SidecarRuntimeSafetyChecks::verified(0),
|
||||||
|
)
|
||||||
|
.expect("example sidecar plugin policy should validate");
|
||||||
|
|
||||||
ExampleInstallableTargetPlugin {
|
ExampleInstallableTargetPlugin {
|
||||||
manifest,
|
manifest,
|
||||||
|
|||||||
@@ -63,7 +63,7 @@ pub use runtime::{
|
|||||||
RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
|
RuntimeTargetHealthState, RuntimeTargetSnapshot, SharedTarget, TargetRuntimeManager, activate_targets_with_replay,
|
||||||
adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
|
adapter::{BuiltinPluginRuntimeAdapter, PluginRuntimeAdapter},
|
||||||
init_target_and_optionally_start_replay,
|
init_target_and_optionally_start_replay,
|
||||||
sidecar::SidecarPluginRuntime,
|
sidecar::{SidecarPluginRuntime, SidecarRuntimePolicy, SidecarRuntimeSafetyChecks},
|
||||||
sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability},
|
sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability},
|
||||||
start_replay_worker,
|
start_replay_worker,
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -19,6 +19,66 @@ use std::time::Duration;
|
|||||||
|
|
||||||
const DEFAULT_FAILURE_THRESHOLD: usize = 3;
|
const DEFAULT_FAILURE_THRESHOLD: usize = 3;
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||||
|
pub struct SidecarRuntimePolicy {
|
||||||
|
pub allow_external_sidecars: bool,
|
||||||
|
pub require_sandbox: bool,
|
||||||
|
pub require_provenance: bool,
|
||||||
|
pub max_queue_depth: usize,
|
||||||
|
pub operation_timeout: Duration,
|
||||||
|
pub failure_threshold: usize,
|
||||||
|
pub redact_error_details: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Default for SidecarRuntimePolicy {
|
||||||
|
fn default() -> Self {
|
||||||
|
Self {
|
||||||
|
allow_external_sidecars: false,
|
||||||
|
require_sandbox: true,
|
||||||
|
require_provenance: true,
|
||||||
|
max_queue_depth: 0,
|
||||||
|
operation_timeout: Duration::from_secs(5),
|
||||||
|
failure_threshold: DEFAULT_FAILURE_THRESHOLD,
|
||||||
|
redact_error_details: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SidecarRuntimePolicy {
|
||||||
|
pub fn verified_external(max_queue_depth: usize, operation_timeout: Duration, failure_threshold: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
allow_external_sidecars: true,
|
||||||
|
require_sandbox: true,
|
||||||
|
require_provenance: true,
|
||||||
|
max_queue_depth,
|
||||||
|
operation_timeout,
|
||||||
|
failure_threshold: failure_threshold.max(1),
|
||||||
|
redact_error_details: true,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn failure_threshold(&self) -> usize {
|
||||||
|
self.failure_threshold.max(1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||||
|
pub struct SidecarRuntimeSafetyChecks {
|
||||||
|
pub sandboxed: bool,
|
||||||
|
pub provenance_verified: bool,
|
||||||
|
pub queue_depth: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl SidecarRuntimeSafetyChecks {
|
||||||
|
pub fn verified(queue_depth: usize) -> Self {
|
||||||
|
Self {
|
||||||
|
sandboxed: true,
|
||||||
|
provenance_verified: true,
|
||||||
|
queue_depth,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
pub struct SidecarPluginRuntime {
|
pub struct SidecarPluginRuntime {
|
||||||
@@ -58,6 +118,29 @@ impl SidecarPluginRuntime {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn enable_with_policy(
|
||||||
|
&mut self,
|
||||||
|
expected_plugin_id: &str,
|
||||||
|
required_domain: TargetDomain,
|
||||||
|
policy: &SidecarRuntimePolicy,
|
||||||
|
safety_checks: &SidecarRuntimeSafetyChecks,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
self.handshake.validate(expected_plugin_id)?;
|
||||||
|
if !self.handshake.supported_domains.contains(&required_domain) {
|
||||||
|
return Err(format!(
|
||||||
|
"sidecar plugin {} does not support required domain {:?}",
|
||||||
|
self.handshake.plugin_id, required_domain
|
||||||
|
));
|
||||||
|
}
|
||||||
|
validate_runtime_policy(policy, safety_checks)?;
|
||||||
|
|
||||||
|
self.healthy = true;
|
||||||
|
self.degraded_to_builtin = false;
|
||||||
|
self.last_error = None;
|
||||||
|
self.failure_count = 0;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub fn mark_unhealthy(&mut self) {
|
pub fn mark_unhealthy(&mut self) {
|
||||||
self.healthy = false;
|
self.healthy = false;
|
||||||
}
|
}
|
||||||
@@ -71,6 +154,19 @@ impl SidecarPluginRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn record_failure_with_policy(&mut self, policy: &SidecarRuntimePolicy, error: impl Into<String>) {
|
||||||
|
self.failure_count = self.failure_count.saturating_add(1);
|
||||||
|
self.healthy = false;
|
||||||
|
self.last_error = Some(if policy.redact_error_details {
|
||||||
|
"sidecar operation failed".to_string()
|
||||||
|
} else {
|
||||||
|
error.into()
|
||||||
|
});
|
||||||
|
if self.failure_count >= policy.failure_threshold() {
|
||||||
|
self.degraded_to_builtin = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn send_with_timeout(&mut self, operation_timeout: Duration, simulated_latency: Duration) -> Result<(), String> {
|
pub fn send_with_timeout(&mut self, operation_timeout: Duration, simulated_latency: Duration) -> Result<(), String> {
|
||||||
if simulated_latency > operation_timeout {
|
if simulated_latency > operation_timeout {
|
||||||
self.record_failure(format!(
|
self.record_failure(format!(
|
||||||
@@ -92,9 +188,28 @@ impl SidecarPluginRuntime {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn validate_runtime_policy(policy: &SidecarRuntimePolicy, safety_checks: &SidecarRuntimeSafetyChecks) -> Result<(), String> {
|
||||||
|
if !policy.allow_external_sidecars {
|
||||||
|
return Err("external sidecar runtime is disabled by policy".to_string());
|
||||||
|
}
|
||||||
|
if policy.require_sandbox && !safety_checks.sandboxed {
|
||||||
|
return Err("sidecar runtime requires sandbox isolation".to_string());
|
||||||
|
}
|
||||||
|
if policy.require_provenance && !safety_checks.provenance_verified {
|
||||||
|
return Err("sidecar runtime requires verified provenance".to_string());
|
||||||
|
}
|
||||||
|
if safety_checks.queue_depth > policy.max_queue_depth {
|
||||||
|
return Err(format!(
|
||||||
|
"sidecar runtime queue depth {} exceeds policy bound {}",
|
||||||
|
safety_checks.queue_depth, policy.max_queue_depth
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::SidecarPluginRuntime;
|
use super::{SidecarPluginRuntime, SidecarRuntimePolicy, SidecarRuntimeSafetyChecks};
|
||||||
use crate::TargetDomain;
|
use crate::TargetDomain;
|
||||||
use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
|
use crate::runtime::sidecar_protocol::{SIDECAR_RUNTIME_PROTOCOL_VERSION, SidecarHandshake, SidecarPluginCapability};
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
@@ -124,6 +239,112 @@ mod tests {
|
|||||||
assert!(runtime.healthy);
|
assert!(runtime.healthy);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sidecar_runtime_policy_rejects_external_activation_by_default() {
|
||||||
|
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||||
|
|
||||||
|
let result = runtime.enable_with_policy(
|
||||||
|
"external:webhook",
|
||||||
|
TargetDomain::Notify,
|
||||||
|
&SidecarRuntimePolicy::default(),
|
||||||
|
&SidecarRuntimeSafetyChecks::verified(0),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
result.as_ref().map_err(String::as_str),
|
||||||
|
Err("external sidecar runtime is disabled by policy")
|
||||||
|
);
|
||||||
|
assert!(!runtime.healthy);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sidecar_runtime_policy_requires_sandbox_and_provenance() {
|
||||||
|
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||||
|
let policy = SidecarRuntimePolicy::verified_external(16, Duration::from_secs(5), 3);
|
||||||
|
|
||||||
|
let missing_sandbox = runtime.enable_with_policy(
|
||||||
|
"external:webhook",
|
||||||
|
TargetDomain::Notify,
|
||||||
|
&policy,
|
||||||
|
&SidecarRuntimeSafetyChecks {
|
||||||
|
sandboxed: false,
|
||||||
|
provenance_verified: true,
|
||||||
|
queue_depth: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
missing_sandbox.as_ref().map_err(String::as_str),
|
||||||
|
Err("sidecar runtime requires sandbox isolation")
|
||||||
|
);
|
||||||
|
|
||||||
|
let missing_provenance = runtime.enable_with_policy(
|
||||||
|
"external:webhook",
|
||||||
|
TargetDomain::Notify,
|
||||||
|
&policy,
|
||||||
|
&SidecarRuntimeSafetyChecks {
|
||||||
|
sandboxed: true,
|
||||||
|
provenance_verified: false,
|
||||||
|
queue_depth: 0,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
missing_provenance.as_ref().map_err(String::as_str),
|
||||||
|
Err("sidecar runtime requires verified provenance")
|
||||||
|
);
|
||||||
|
assert!(!runtime.healthy);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sidecar_runtime_policy_enforces_queue_bound() {
|
||||||
|
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||||
|
let policy = SidecarRuntimePolicy::verified_external(2, Duration::from_secs(5), 3);
|
||||||
|
|
||||||
|
let result = runtime.enable_with_policy(
|
||||||
|
"external:webhook",
|
||||||
|
TargetDomain::Notify,
|
||||||
|
&policy,
|
||||||
|
&SidecarRuntimeSafetyChecks::verified(3),
|
||||||
|
);
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
result.as_ref().map_err(String::as_str),
|
||||||
|
Err("sidecar runtime queue depth 3 exceeds policy bound 2")
|
||||||
|
);
|
||||||
|
assert!(!runtime.healthy);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sidecar_runtime_policy_allows_verified_external_activation() {
|
||||||
|
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||||
|
let policy = SidecarRuntimePolicy::verified_external(16, Duration::from_secs(5), 3);
|
||||||
|
|
||||||
|
runtime
|
||||||
|
.enable_with_policy(
|
||||||
|
"external:webhook",
|
||||||
|
TargetDomain::Notify,
|
||||||
|
&policy,
|
||||||
|
&SidecarRuntimeSafetyChecks::verified(1),
|
||||||
|
)
|
||||||
|
.expect("verified sidecar runtime should enable");
|
||||||
|
|
||||||
|
assert!(runtime.healthy);
|
||||||
|
assert_eq!(policy.failure_threshold(), 3);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn sidecar_runtime_policy_redacts_failure_details() {
|
||||||
|
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||||
|
let policy = SidecarRuntimePolicy::verified_external(16, Duration::from_secs(5), 2);
|
||||||
|
|
||||||
|
runtime.record_failure_with_policy(&policy, "secret token leaked in transport error");
|
||||||
|
runtime.record_failure_with_policy(&policy, "another secret error");
|
||||||
|
|
||||||
|
assert_eq!(runtime.last_error.as_deref(), Some("sidecar operation failed"));
|
||||||
|
assert!(runtime.degraded_to_builtin);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn sidecar_runtime_enable_rejects_domain_mismatch() {
|
fn sidecar_runtime_enable_rejects_domain_mismatch() {
|
||||||
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
let mut runtime = SidecarPluginRuntime::new("grpc://127.0.0.1:50051", notify_sidecar_handshake());
|
||||||
|
|||||||
@@ -334,6 +334,18 @@ pub const ADMIN_ROUTE_POLICY_SPECS: &[AdminRouteSpec] = &[
|
|||||||
RouteRiskLevel::Sensitive,
|
RouteRiskLevel::Sensitive,
|
||||||
),
|
),
|
||||||
admin(HttpMethod::Put, "/rustfs/admin/v3/module-switches", CONFIG_UPDATE, RouteRiskLevel::High),
|
admin(HttpMethod::Put, "/rustfs/admin/v3/module-switches", CONFIG_UPDATE, RouteRiskLevel::High),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/rustfs/admin/v4/extensions/catalog",
|
||||||
|
SERVER_INFO,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
|
admin(
|
||||||
|
HttpMethod::Get,
|
||||||
|
"/rustfs/admin/v4/extensions/instances",
|
||||||
|
GET_BUCKET_TARGET,
|
||||||
|
RouteRiskLevel::Sensitive,
|
||||||
|
),
|
||||||
admin(
|
admin(
|
||||||
HttpMethod::Get,
|
HttpMethod::Get,
|
||||||
"/rustfs/admin/v4/plugins/catalog",
|
"/rustfs/admin/v4/plugins/catalog",
|
||||||
|
|||||||
Reference in New Issue
Block a user