mirror of
https://github.com/rustfs/rustfs.git
synced 2026-08-01 19:12:14 +00:00
cad8246ffb
* test(kms): add a scripted loopback Vault for policy wiring tests A minimal HTTP/1.1 responder that serves canned Vault responses in order and records the method/path sequence, so wiring tests can assert exactly how many requests a code path performed (retries, read-confirm) without a live Vault server. * feat(kms): route Vault operations through the retry policy engine Wire every outbound vaultrs call in the KV2 and Transit backends through policy::execute, completing the wiring half of the operation policy work (the engine landed separately): - Reads (KV2 read/read_metadata/read_version/list, transit read/list/ encrypt/decrypt, health checks) run as ReadIdempotent: bounded retries with exponential backoff and jitter on 429, recoverable 5xx, and connection-level failures; 400/401/403/404 stay fatal. - Writes (KV2 set/CAS set/delete_metadata, transit create/update/rotate/ delete, metadata writes) run as MutatingNonIdempotent: exactly one attempt under the per-attempt timeout, never replayed. CAS conflicts in the rotation protocol pass through unchanged as the concurrency signal they are. - Each attempt takes a fresh credential snapshot, so a retry after a credential rotation uses the new token. - Read-confirm recovery for lost create responses: when a create finds an existing key that is exactly what it would have produced (same algorithm, enabled, usable material, and for request-level creates the same usage/description/tags), it reports the stored key as the create result instead of KeyAlreadyExists. Any divergence keeps failing. - Deletes treat already-deleted records as completed deletes (KV2 version records; transit metadata already did), so re-running an interrupted deletion converges. - A failed existence pre-check inside create now fails the create instead of falling through to a blind overwrite (fail closed). - The policy module sheds its allow(dead_code) now that it is wired. Wiring tests run against a scripted loopback Vault and assert request counts and endpoints for the retry, single-attempt, CAS-conflict, and read-confirm paths. Refs rustfs/backlog#1569 (part of rustfs/backlog#1562)
160 lines
5.9 KiB
Rust
160 lines
5.9 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//! Minimal scripted HTTP responder standing in for a Vault server.
|
|
//!
|
|
//! Wiring tests need to observe how many Vault requests a code path performs
|
|
//! (retries, read-confirm recovery) without a live Vault. The responder serves
|
|
//! one canned response per incoming request in order, closes the connection
|
|
//! after each response, and records the `METHOD /path` sequence for
|
|
//! assertions. It intentionally implements just enough HTTP/1.1 for the
|
|
//! `vaultrs` reqwest client: no keep-alive, no chunked bodies.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
use tokio::net::{TcpListener, TcpStream};
|
|
|
|
/// One canned HTTP response.
|
|
pub(crate) struct ScriptedResponse {
|
|
status: u16,
|
|
body: String,
|
|
}
|
|
|
|
impl ScriptedResponse {
|
|
/// A 200 response carrying `data` inside the standard Vault envelope.
|
|
pub(crate) fn ok(data: serde_json::Value) -> Self {
|
|
Self {
|
|
status: 200,
|
|
body: serde_json::json!({
|
|
"request_id": "scripted",
|
|
"lease_id": "",
|
|
"lease_duration": 0,
|
|
"renewable": false,
|
|
"data": data,
|
|
})
|
|
.to_string(),
|
|
}
|
|
}
|
|
|
|
/// An error response in Vault's `{"errors": [...]}` format.
|
|
pub(crate) fn error(status: u16, message: &str) -> Self {
|
|
Self {
|
|
status,
|
|
body: serde_json::json!({ "errors": [message] }).to_string(),
|
|
}
|
|
}
|
|
}
|
|
|
|
/// A scripted stand-in Vault listening on a loopback port.
|
|
pub(crate) struct ScriptedVault {
|
|
/// Base address (`http://127.0.0.1:port`) to point a Vault client at.
|
|
pub(crate) address: String,
|
|
requests: Arc<Mutex<Vec<String>>>,
|
|
}
|
|
|
|
impl ScriptedVault {
|
|
/// Bind a loopback listener and serve `responses` one per request.
|
|
///
|
|
/// Requests beyond the script get a 599 error so a test that under-scripts
|
|
/// fails loudly instead of hanging.
|
|
pub(crate) async fn serve(responses: Vec<ScriptedResponse>) -> Self {
|
|
let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind scripted vault listener");
|
|
let address = format!("http://{}", listener.local_addr().expect("scripted vault local addr"));
|
|
let requests = Arc::new(Mutex::new(Vec::new()));
|
|
let recorded = Arc::clone(&requests);
|
|
|
|
tokio::spawn(async move {
|
|
let mut responses = responses.into_iter();
|
|
loop {
|
|
let Ok((mut stream, _)) = listener.accept().await else {
|
|
return;
|
|
};
|
|
let Some(request_line) = read_request(&mut stream).await else {
|
|
continue;
|
|
};
|
|
recorded
|
|
.lock()
|
|
.expect("scripted vault request log poisoned")
|
|
.push(request_line);
|
|
let response = responses
|
|
.next()
|
|
.unwrap_or_else(|| ScriptedResponse::error(599, "scripted vault: script exhausted"));
|
|
let payload = format!(
|
|
"HTTP/1.1 {} Scripted\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}",
|
|
response.status,
|
|
response.body.len(),
|
|
response.body
|
|
);
|
|
let _ = stream.write_all(payload.as_bytes()).await;
|
|
let _ = stream.shutdown().await;
|
|
}
|
|
});
|
|
|
|
Self { address, requests }
|
|
}
|
|
|
|
/// The `METHOD /path` lines of every request served so far, in order.
|
|
pub(crate) fn requests(&self) -> Vec<String> {
|
|
self.requests.lock().expect("scripted vault request log poisoned").clone()
|
|
}
|
|
}
|
|
|
|
/// Read one HTTP/1.1 request (head plus content-length body) and return its
|
|
/// `METHOD /path` line. Draining the body before responding keeps the client
|
|
/// from seeing a connection reset while it is still writing.
|
|
async fn read_request(stream: &mut TcpStream) -> Option<String> {
|
|
let mut buffer = Vec::new();
|
|
let mut chunk = [0u8; 4096];
|
|
let head_end = loop {
|
|
if let Some(position) = buffer.windows(4).position(|window| window == b"\r\n\r\n") {
|
|
break position + 4;
|
|
}
|
|
let read = stream.read(&mut chunk).await.ok()?;
|
|
if read == 0 {
|
|
return None;
|
|
}
|
|
buffer.extend_from_slice(&chunk[..read]);
|
|
};
|
|
|
|
let head = String::from_utf8_lossy(&buffer[..head_end]).into_owned();
|
|
let mut lines = head.lines();
|
|
let request_line = lines.next()?;
|
|
let mut parts = request_line.split_whitespace();
|
|
let method = parts.next()?;
|
|
let path = parts.next()?;
|
|
// rustify appends a lone "?" when an endpoint has no query parameters;
|
|
// strip it so assertions can use the plain path.
|
|
let path = path.strip_suffix('?').unwrap_or(path);
|
|
|
|
let content_length: usize = lines
|
|
.filter_map(|line| {
|
|
let (name, value) = line.split_once(':')?;
|
|
name.eq_ignore_ascii_case("content-length")
|
|
.then(|| value.trim().parse().ok())?
|
|
})
|
|
.next()
|
|
.unwrap_or(0);
|
|
let mut remaining = content_length.saturating_sub(buffer.len() - head_end);
|
|
while remaining > 0 {
|
|
let read = stream.read(&mut chunk).await.ok()?;
|
|
if read == 0 {
|
|
break;
|
|
}
|
|
remaining = remaining.saturating_sub(read);
|
|
}
|
|
|
|
Some(format!("{method} {path}"))
|
|
}
|