mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-06 03:59:14 +00:00
a5bde8b0af
* feat(ecstore): add on-demand migration bucket config model Introduce OnDemandMigrationConfig (deny_unknown_fields, version 1) with typed validation, credential redaction, a secret-free Debug impl, and the OnceLock publish hook the runtime registers into. Exported through the api facade. * feat(ecstore): persist on-demand migration config in bucket metadata Store the config as a RustFS extension entry (on-demand-migration.json) with its update time in .metadata.bin, add the typed BucketMetadataSys accessor, and publish the config through the hook on every cache-install path alongside the durability sync. * refactor(ecstore): extract shared remote S3 client builder Move the aws_sdk_s3 client construction out of bucket_target_sys into bucket/remote_s3_client.rs: endpoint assembly, credential provider, path-style selection, custom CA / skip-TLS transports and the outbound SSRF gate now build from a neutral RemoteS3EndpointSpec so replication targets and the upcoming on-demand migration source client share one policy. Replication builds its client through From<&BucketTarget>; the gate keeps its relaxed semantics (private allowed, loopback only behind RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET) verbatim. The builder also gains optional connect/read timeouts and a User-Agent suffix interceptor, both unset for replication. Refs rustfs/backlog#2149 * feat(ecstore): add on-demand migration SourceClient Add bucket/on_demand_migration/source_client.rs on top of the shared remote S3 builder: HEAD, ranged streaming GET, ListObjectsV2 with source-prefix mapping, GetObjectTagging and an admin probe. Every request carries the x-rustfs-/x-minio-source-proxy-request anti-loop markers and a RustFS-OnDemandMigration/<version> User-Agent suffix; SSE-C source objects are rejected as unsupported. SourceError classifies SDK failures (not found, access denied, throttled, timeout, connect, server error) with retryability and a stable metrics label. Debug output redacts credentials. Refs rustfs/backlog#2149 * docs(operations): point outbound policy at shared remote S3 client builder * chore: integrate ODM-01 and ODM-02 as B1 base (fix facade merge) * feat(admin): add on-demand migration bucket admin API Add the management plane for On-Demand Migration (ODM-07, rustfs/backlog#2154): PUT/GET/DELETE /v3/on-demand-migration/{bucket}, PUT ?dry-run=true, and a GET .../status skeleton. - PUT authorizes SetBucketOnDemandMigration, checks the bucket, the RUSTFS_ON_DEMAND_MIGRATION_ENABLED switch and the license, validates the ODM-01 config against local endpoints and replication targets, probes the source with SourceClient::probe(), then persists through the incarnation gate and asks peers to reload. Responses carry the redacted config and a probe summary; probe failures name only the error class. - GET answers 404 NoSuchConfiguration when unset; DELETE is idempotent (204). - New AdminAction variants admin:SetBucketOnDemandMigration and admin:GetBucketOnDemandMigration, route policy matrix rows, registration and MinIO alias coverage, and a doc row for the extra handler gates. - rustfs-madmin gains on_demand_migration wire types and client methods; golden fixtures under crates/madmin/fixtures/on_demand_migration/ are asserted byte-for-byte by both the handler and the client tests. Anonymous sources still map to a 400 naming source.credentials until the runtime slice adds the credential-less path. * refactor(admin): route on-demand migration handler errors through the s3 facade
155 lines
5.7 KiB
Rust
155 lines
5.7 KiB
Rust
// Copyright 2024 RustFS Team
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
//! Test-only HTTP server shared by the admin client modules: one canned
|
|
//! response per connection, every request recorded for assertions.
|
|
|
|
use std::sync::{Arc, Mutex};
|
|
|
|
/// One recorded request, parsed off the wire with the minimum needed for
|
|
/// assertions: method, path, query, headers, body.
|
|
#[derive(Debug, Clone)]
|
|
pub(crate) struct RecordedRequest {
|
|
pub(crate) method: String,
|
|
pub(crate) path: String,
|
|
pub(crate) query: String,
|
|
pub(crate) headers: Vec<(String, String)>,
|
|
pub(crate) body: String,
|
|
}
|
|
|
|
impl RecordedRequest {
|
|
pub(crate) fn header(&self, name: &str) -> Option<String> {
|
|
self.headers
|
|
.iter()
|
|
.find(|(key, _)| key.eq_ignore_ascii_case(name))
|
|
.map(|(_, value)| value.clone())
|
|
}
|
|
}
|
|
|
|
/// Minimal HTTP/1.1 server: one canned response per connection, every
|
|
/// request recorded behind an `Arc<Mutex>`. Deliberately dependency-free —
|
|
/// the assertions only need the raw request bytes.
|
|
pub(crate) struct TestServer {
|
|
pub(crate) addr: std::net::SocketAddr,
|
|
requests: Arc<Mutex<Vec<RecordedRequest>>>,
|
|
}
|
|
|
|
impl TestServer {
|
|
pub(crate) async fn spawn(response_body: &'static str, status: u16) -> Self {
|
|
use tokio::io::{AsyncReadExt, AsyncWriteExt};
|
|
|
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0")
|
|
.await
|
|
.expect("bind ephemeral port");
|
|
let addr = listener.local_addr().expect("local addr");
|
|
let requests: Arc<Mutex<Vec<RecordedRequest>>> = Arc::new(Mutex::new(Vec::new()));
|
|
|
|
let recorded = requests.clone();
|
|
tokio::spawn(async move {
|
|
let reason = match status {
|
|
200 => "OK",
|
|
204 => "No Content",
|
|
400 => "Bad Request",
|
|
404 => "Not Found",
|
|
_ => "Forbidden",
|
|
};
|
|
let response = format!(
|
|
"HTTP/1.1 {status} {reason}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{response_body}",
|
|
response_body.len()
|
|
);
|
|
// Each request is a fresh connection (connection: close); a
|
|
// bounded loop serves every call a test makes while letting
|
|
// the task exit instead of lingering for the whole process.
|
|
for _ in 0..16 {
|
|
let Ok((mut stream, _)) = listener.accept().await else {
|
|
break;
|
|
};
|
|
let mut buffer = Vec::with_capacity(2048);
|
|
let mut chunk = [0u8; 2048];
|
|
// Read headers plus content-length body, or stop on close.
|
|
loop {
|
|
if let Some(end) = find_header_end(&buffer) {
|
|
let content_length = extract_content_length(&buffer[..end]);
|
|
if buffer.len() >= end + content_length {
|
|
break;
|
|
}
|
|
}
|
|
let n = match stream.read(&mut chunk).await {
|
|
Ok(0) | Err(_) => break,
|
|
Ok(n) => n,
|
|
};
|
|
buffer.extend_from_slice(&chunk[..n]);
|
|
if buffer.len() > 64 * 1024 {
|
|
break;
|
|
}
|
|
}
|
|
if let Some(request) = parse_request(&buffer) {
|
|
recorded.lock().expect("recorded lock").push(request);
|
|
}
|
|
let _ = stream.write_all(response.as_bytes()).await;
|
|
let _ = stream.shutdown().await;
|
|
}
|
|
});
|
|
|
|
Self { addr, requests }
|
|
}
|
|
|
|
pub(crate) fn recorded(&self) -> RecordedRequest {
|
|
self.requests
|
|
.lock()
|
|
.expect("recorded lock")
|
|
.last()
|
|
.cloned()
|
|
.expect("the client call must have produced one recorded request")
|
|
}
|
|
}
|
|
|
|
fn find_header_end(buffer: &[u8]) -> Option<usize> {
|
|
buffer.windows(4).position(|window| window == b"\r\n\r\n").map(|pos| pos + 4)
|
|
}
|
|
|
|
fn extract_content_length(headers: &[u8]) -> usize {
|
|
let text = String::from_utf8_lossy(headers).to_ascii_lowercase();
|
|
text.lines()
|
|
.find_map(|line| line.strip_prefix("content-length:"))
|
|
.and_then(|value| value.trim().parse().ok())
|
|
.unwrap_or(0)
|
|
}
|
|
|
|
fn parse_request(raw: &[u8]) -> Option<RecordedRequest> {
|
|
let end = find_header_end(raw)?;
|
|
let head = String::from_utf8_lossy(&raw[..end]);
|
|
let body = String::from_utf8_lossy(&raw[end..]).into_owned();
|
|
let mut lines = head.lines();
|
|
let request_line = lines.next()?;
|
|
let mut parts = request_line.split_whitespace();
|
|
let method = parts.next()?.to_string();
|
|
let target = parts.next()?.to_string();
|
|
let (path, query) = match target.split_once('?') {
|
|
Some((path, query)) => (path.to_string(), query.to_string()),
|
|
None => (target, String::new()),
|
|
};
|
|
let headers = lines
|
|
.filter_map(|line| line.split_once(':'))
|
|
.map(|(name, value)| (name.trim().to_string(), value.trim().to_string()))
|
|
.collect();
|
|
Some(RecordedRequest {
|
|
method,
|
|
path,
|
|
query,
|
|
headers,
|
|
body,
|
|
})
|
|
}
|