mirror of
https://github.com/rustfs/rustfs.git
synced 2026-09-07 12:35:54 +00:00
fix(odm): preserve source regressions after service relocation
Port the reviewed framing, native listing validation, and old-reader serialization fixes from 4d72989068f845f2a046a91b1afd537dcd1bfeda to the application-owned service without restoring the removed ECStore module. Preserve the current GCS HEAD/GET bucket proof, optional GCS feature, ListObjects v1 local pagination, and stored-config publication behavior. Validate Azure object absence at its operation boundary and ignore nonexistent provider error headers for GCS. Keep frozen compatibility fixtures within each owning crate. Related: rustfs/backlog#2303, rustfs/backlog#2306, rustfs/backlog#2307, rustfs/backlog#2308.
This commit is contained in:
overtrue
@@ -1 +1 @@
|
||||
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
|
||||
{"bucket":"photos","config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z"}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
|
||||
{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"sourceSecretKey123","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}}
|
||||
|
||||
@@ -1 +1 @@
|
||||
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":null},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
|
||||
{"bucket":"photos","dry_run":false,"config":{"version":1,"enabled":true,"source":{"provider":"minio","endpoint":"https://source.example.com:9000","region":"us-east-1","bucket":"legacy-photos","path_style":"auto","credentials":{"access_key":"AKIASOURCE","secret_key":"REDACTED","session_token":null},"tls":{"skip_verify":false,"ca_cert_pem":null}},"filter":{"prefix":null,"source_prefix":"photos/"},"policy":{"head":"proxy","range_get":"serve_and_backfill","source_error":"propagate","list_through":false,"respect_local_delete_marker":true,"preserve_etag":true,"copy_tags":false,"emit_events":true,"negative_cache_ttl_secs":30,"inline_max_bytes":16777216,"multipart_part_size_bytes":67108864,"max_concurrent_pulls":8,"pull_queue_capacity":1024,"source_timeout":{"connect_ms":5000,"first_byte_ms":15000,"idle_ms":30000},"bandwidth_limit_bytes_per_sec":null}},"updated_at":"2026-09-02T10:00:00Z","probe":{"reachable":true,"listable":true,"sample_key":"photos/2024/01.jpg"}}
|
||||
|
||||
@@ -0,0 +1,80 @@
|
||||
// Strict source reader frozen from e2a921bc1608823c8efec955d7463ab8350a8a01.
|
||||
// Wire declarations and credential Debug are copied verbatim; runtime methods are omitted.
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::fmt;
|
||||
|
||||
const REDACTED: &str = "REDACTED";
|
||||
|
||||
/// The external S3-compatible source bucket.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SourceConfig {
|
||||
pub provider: Provider,
|
||||
/// `http(s)://host[:port]` with no path or query. Optional only for
|
||||
/// [`Provider::Aws`], where it is derived from `region`.
|
||||
#[serde(default)]
|
||||
pub endpoint: Option<String>,
|
||||
pub region: String,
|
||||
pub bucket: String,
|
||||
#[serde(default)]
|
||||
pub path_style: PathStyle,
|
||||
/// `None` means anonymous access to a public source bucket.
|
||||
#[serde(default)]
|
||||
pub credentials: Option<SourceCredentials>,
|
||||
#[serde(default)]
|
||||
pub tls: TlsConfig,
|
||||
}
|
||||
|
||||
/// Source vendor family. `azure` is deliberately absent from this version.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum Provider {
|
||||
/// Generic S3-compatible endpoint.
|
||||
S3,
|
||||
Aws,
|
||||
Minio,
|
||||
Rustfs,
|
||||
R2,
|
||||
/// GCS XML interoperability API with HMAC keys.
|
||||
Gcs,
|
||||
}
|
||||
|
||||
/// Bucket addressing style. `auto` is resolved by the source client builder.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "lowercase")]
|
||||
pub enum PathStyle {
|
||||
#[default]
|
||||
Auto,
|
||||
Path,
|
||||
Virtual,
|
||||
}
|
||||
|
||||
/// Static credentials for the source. `Debug` never prints the secret or
|
||||
/// the session token.
|
||||
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct SourceCredentials {
|
||||
pub access_key: String,
|
||||
pub secret_key: String,
|
||||
#[serde(default)]
|
||||
pub session_token: Option<String>,
|
||||
}
|
||||
|
||||
impl fmt::Debug for SourceCredentials {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("SourceCredentials")
|
||||
.field("access_key", &self.access_key)
|
||||
.field("secret_key", &REDACTED)
|
||||
.field("session_token", &self.session_token.as_ref().map(|_| REDACTED))
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Default, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct TlsConfig {
|
||||
#[serde(default)]
|
||||
pub skip_verify: bool,
|
||||
#[serde(default)]
|
||||
pub ca_cert_pem: Option<String>,
|
||||
}
|
||||
@@ -85,10 +85,10 @@ pub struct OnDemandMigrationSource {
|
||||
#[serde(default)]
|
||||
pub tls: OnDemandMigrationTls,
|
||||
/// Required for `azure` and rejected for every other provider.
|
||||
#[serde(default)]
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub azure: Option<OnDemandMigrationAzure>,
|
||||
/// Required for `gcs_native` and rejected for every other provider.
|
||||
#[serde(default)]
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub gcs: Option<OnDemandMigrationGcs>,
|
||||
}
|
||||
|
||||
@@ -651,6 +651,10 @@ mod tests {
|
||||
use super::*;
|
||||
use crate::test_support::TestServer;
|
||||
|
||||
mod before_native_sources {
|
||||
include!("../fixtures/on_demand_migration/source_config_e2a.rs");
|
||||
}
|
||||
|
||||
const SET_REQUEST_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_request.json");
|
||||
const SET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/set_response.json");
|
||||
const GET_RESPONSE_FIXTURE: &str = include_str!("../fixtures/on_demand_migration/get_response.json");
|
||||
@@ -684,6 +688,20 @@ mod tests {
|
||||
assert_eq!(config.source.tls, OnDemandMigrationTls::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn s3_admin_writes_remain_readable_by_the_strict_pre_native_server() {
|
||||
for provider in ["s3", "aws", "minio", "rustfs", "r2", "gcs"] {
|
||||
let historical = SET_REQUEST_FIXTURE.replace("\"provider\":\"minio\"", &format!("\"provider\":\"{provider}\""));
|
||||
let config: OnDemandMigrationConfig = serde_json::from_str(&historical).expect("historical set request");
|
||||
let wire = serde_json::to_string(&config).expect("current admin set request");
|
||||
let actual: serde_json::Value = serde_json::from_str(&wire).expect("admin request JSON");
|
||||
let old_source: before_native_sources::SourceConfig = serde_json::from_value(actual["source"].clone())
|
||||
.expect("the strict e2a server must accept an ordinary S3 source from the new admin client");
|
||||
assert_eq!(serde_json::to_value(old_source).expect("old source wire"), actual["source"]);
|
||||
assert_eq!(wire, historical.trim(), "provider={provider}: preserve the historical request bytes");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn set_response_fixture_round_trips_and_is_redacted() {
|
||||
let response: OnDemandMigrationSetResponse = round_trip(SET_RESPONSE_FIXTURE);
|
||||
@@ -878,11 +896,11 @@ mod tests {
|
||||
for (label, json) in [
|
||||
(
|
||||
"azure",
|
||||
r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"},"gcs":null}"#,
|
||||
r#"{"provider":"azure","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":{"account":"legacyaccount","account_key":null,"sas_token":"sv=2021-08-06&sig=topsecret"}}"#,
|
||||
),
|
||||
(
|
||||
"gcs_native",
|
||||
r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"azure":null,"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
|
||||
r#"{"provider":"gcs_native","endpoint":null,"region":"auto","bucket":"legacy-photos","path_style":"auto","credentials":null,"tls":{"skip_verify":false,"ca_cert_pem":null},"gcs":{"service_account_json":"{\"type\":\"service_account\"}"}}"#,
|
||||
),
|
||||
] {
|
||||
let source: OnDemandMigrationSource = serde_json::from_str(json).unwrap_or_else(|err| panic!("{label}: {err}"));
|
||||
@@ -891,6 +909,16 @@ mod tests {
|
||||
json,
|
||||
"{label} must reproduce the server wire shape byte for byte"
|
||||
);
|
||||
let mut wire: serde_json::Value = serde_json::from_str(json).expect("native wire fixture");
|
||||
assert!(
|
||||
serde_json::from_value::<before_native_sources::SourceConfig>(wire.clone()).is_err(),
|
||||
"native provider names and fields still require an upgraded server"
|
||||
);
|
||||
wire[if label == "azure" { "gcs" } else { "azure" }] = serde_json::Value::Null;
|
||||
assert_eq!(
|
||||
serde_json::from_value::<OnDemandMigrationSource>(wire).expect("the prior explicit-null wire still decodes"),
|
||||
source
|
||||
);
|
||||
}
|
||||
|
||||
let azure = OnDemandMigrationAzure {
|
||||
@@ -945,6 +973,10 @@ mod tests {
|
||||
.is_some_and(|auth| auth.starts_with("AWS4-HMAC-SHA256"))
|
||||
);
|
||||
assert_eq!(request.body, SET_REQUEST_FIXTURE.trim(), "the body is the canonical config document");
|
||||
let body: serde_json::Value = serde_json::from_str(&request.body).expect("signed admin request JSON");
|
||||
let old_source: before_native_sources::SourceConfig = serde_json::from_value(body["source"].clone())
|
||||
.expect("the strict pre-native server must accept the actual signed PUT source");
|
||||
assert_eq!(old_source.provider, before_native_sources::Provider::Minio);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
Reference in New Issue
Block a user