diff --git a/crates/e2e_test/src/security_boundary_test.rs b/crates/e2e_test/src/security_boundary_test.rs index 0315422ff..b0d8533d7 100644 --- a/crates/e2e_test/src/security_boundary_test.rs +++ b/crates/e2e_test/src/security_boundary_test.rs @@ -245,11 +245,11 @@ async fn test_concurrent_object_operations() -> Result<(), Box Result<(), Box std::result::Result<(), AdminError> { self.ensure_generation_is_idle(tier_name)?; - let driver = match self.get_driver(tier_name).await { - Ok(driver) => driver, - Err(err) if err.code == ERR_TIER_NOT_FOUND.code => return Ok(()), - Err(err) => return Err(err), - }; + if force && !self.tiers.contains_key(tier_name) { + return Ok(()); + } if !force { + let driver = match self.get_driver(tier_name).await { + Ok(driver) => driver, + Err(err) if err.code == ERR_TIER_NOT_FOUND.code => return Ok(()), + Err(err) => return Err(err), + }; match driver.in_use().await { Err(err) => { let mut e = ERR_TIER_PERM_ERR.clone(); @@ -8819,8 +8822,9 @@ mod tests { #[tokio::test] async fn test_remove_unknown_tier_is_idempotent() { let mut mgr = empty_mgr(); - // Unknown tier: get_driver returns NotFound, which `remove` swallows. - mgr.remove("NOPE", false).await.expect("removing an unknown tier is a no-op"); + for force in [false, true] { + mgr.remove("NOPE", force).await.expect("removing an unknown tier is a no-op"); + } } #[tokio::test] @@ -8885,6 +8889,68 @@ mod tests { assert!(!mgr.driver_cache.contains_key("COLD-A")); } + #[tokio::test] + async fn test_remove_force_skips_uncached_driver_construction() { + let mut mgr = empty_mgr(); + let mut tier = build_s3_tier("COLD-A"); + tier.s3.as_mut().expect("S3 fixture must have a backend").endpoint = "https://169.254.169.254".to_string(); + mgr.tiers.insert("COLD-A".to_string(), tier); + assert!(mgr.driver_cache.is_empty()); + + let err = mgr + .remove("COLD-A", false) + .await + .expect_err("non-force removal must retain endpoint validation"); + assert_eq!(err.code, ERR_TIER_INVALID_CONFIG.code); + assert!(err.message.contains("tier endpoint is not allowed"), "{}", err.message); + assert!(mgr.tiers.contains_key("COLD-A")); + assert!(mgr.driver_cache.is_empty()); + + mgr.remove("COLD-A", true) + .await + .expect("force removal must not construct a driver for the forbidden endpoint"); + assert!(!mgr.tiers.contains_key("COLD-A")); + assert!(mgr.driver_cache.is_empty()); + } + + #[tokio::test] + async fn persisted_private_tiers_rebuild_uncached_drivers_without_rebinding() { + let endpoint = "https://192.168.10.200:9000"; + for provider in ["s3", "minio", "rustfs"] { + let tier: TierConfig = serde_json::from_value(serde_json::json!({ + "type": provider, + "name": "COLD-A", + (provider): { + "name": "COLD-A", + "endpoint": endpoint, + "accessKey": "access", + "secretKey": "secret", + "bucket": "tier-bucket", + "prefix": "archive", + "region": "us-east-1" + } + })) + .expect("private tier fixture must decode"); + let identity = tier_backend_identity(&tier).expect("private tier identity must encode"); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), tier); + let blob = encode_external_tiering_config_blob(&persisted).expect("private tier config must persist"); + let mut loaded = decode_external_tiering_config_blob(&blob).expect("private tier config must reload"); + assert!(loaded.driver_cache.is_empty(), "{provider} reload must start with a cold cache"); + + loaded + .get_driver("COLD-A") + .await + .expect("a persisted private tier must construct its driver without editing its endpoint"); + assert!(loaded.driver_cache.contains_key("COLD-A")); + let config = &loaded.tiers["COLD-A"]; + assert_eq!(config.name, "COLD-A"); + assert_eq!(tier_backend_identity(config).expect("reloaded identity must encode"), identity); + let wire = serde_json::to_value(config).expect("reloaded tier must serialize"); + assert_eq!(wire[provider]["endpoint"], endpoint, "{provider} endpoint must remain unchanged"); + } + } + #[tokio::test] async fn persisted_remove_preserves_force_semantics() { let mut candidate = empty_mgr(); @@ -16102,6 +16168,64 @@ mod tests { ); } + #[tokio::test] + async fn force_remove_uncached_invalid_endpoint_preserves_persisted_reference_proof() { + for has_free_version in [false, true] { + let store = Arc::new(CasConfigStore::default()); + let mut tier = build_rustfs_tier("COLD-A"); + tier.rustfs.as_mut().expect("RustFS fixture must have a backend").endpoint = "https://169.254.169.254".to_string(); + let identity = tier_backend_identity(&tier).expect("forbidden endpoint must retain a valid backend identity"); + let mut persisted = empty_mgr(); + persisted.tiers.insert("COLD-A".to_string(), tier.clone_with_credentials()); + persisted + .save_tiering_config_if_current(store.clone(), None) + .await + .expect("cold-cache remove fixture must persist"); + let (_, original_version) = load_tier_config_for_update(store.clone()) + .await + .expect("original config version must load"); + if has_free_version { + let mut object = transitioned_tier_object("photos", "free-version.jpg", "COLD-A", Some(identity)); + object.transitioned_object.status = "pending".to_string(); + object.transitioned_object.free_version = true; + store.add_listed_version(object); + } + let manager = TierConfigMgr::new(); + manager.write().await.tiers.insert("COLD-A".to_string(), tier); + assert!(manager.read().await.driver_cache.is_empty()); + + let result = TierConfigMgr::remove_and_save_with(&manager, store.clone(), "COLD-A", true).await; + if has_free_version { + match result.expect_err("a free-version reference must block force removal without constructing a driver") { + TierConfigUpdateError::Publish(err) => { + assert_eq!(err.code, ERR_TIER_BACKEND_IN_USE.code); + assert!(err.message.contains("photos/free-version.jpg"), "{}", err.message); + } + other => panic!("reference proof must surface as a publish error: {other:?}"), + } + } else { + result.expect("force removal without references must persist despite a forbidden endpoint"); + } + + { + let live = manager.read().await; + assert_eq!(live.tiers.contains_key("COLD-A"), has_free_version); + assert!(live.driver_cache.is_empty()); + } + let (loaded, current_version) = load_tier_config_for_update(store) + .await + .expect("post-removal config must reload"); + assert_eq!(loaded.tiers.contains_key("COLD-A"), has_free_version); + if has_free_version { + assert_eq!(current_version, original_version, "blocked removal must not save the candidate"); + assert_eq!( + tier_backend_identity(&loaded.tiers["COLD-A"]).expect("preserved tier identity must encode"), + identity + ); + } + } + } + #[tokio::test] async fn zero_reference_proof_blocks_remove_before_config_save() { let store = Arc::new(CasConfigStore::default()); @@ -17708,7 +17832,10 @@ mod tests { TierConfigMgr::update_candidate_with_config_lock( &manager, store, - TierCandidateMutation::Remove("COLD-A".to_string(), true), + // Non-force removal still validates the remote driver before + // entering reference proof; force removal intentionally skips + // that construction so an invalid endpoint can be removed. + TierCandidateMutation::Remove("COLD-A".to_string(), false), ), ), ), diff --git a/crates/ecstore/src/services/tier/warm_backend.rs b/crates/ecstore/src/services/tier/warm_backend.rs index 8fd400041..8ea706bd8 100644 --- a/crates/ecstore/src/services/tier/warm_backend.rs +++ b/crates/ecstore/src/services/tier/warm_backend.rs @@ -327,11 +327,9 @@ pub(crate) struct S3CompatibleWarmBackendParams<'a> { /// and metrics stay attributable. pub provider_tag: &'a str, /// SSRF guard run against the parsed endpoint once it's known to have a - /// host. Almost every provider passes [`rustfs_utils::egress::validate_outbound_url`] - /// unchanged; RustFS passes its own wrapper that adds a debug-only, - /// env-gated loopback exception for its e2e tier tests (see - /// rustfs/rustfs#6773) — the shared constructor stays the single call - /// site either way, so no provider can silently end up unvalidated. + /// host. Cloud providers use [`rustfs_utils::egress::validate_outbound_url`]; + /// MinIO allows private storage endpoints, and RustFS additionally wraps + /// that policy with its debug-only, env-gated loopback exception. pub validate_endpoint: fn(&url::Url) -> Result<(), rustfs_utils::egress::OutboundUrlError>, } @@ -2008,11 +2006,8 @@ mod tests { assert_eq!(err.to_string(), "Invalid endpoint URL: missing host"); } - /// Every migrated provider that uses `validate_outbound_url` directly (all - /// but RustFS, which injects its own debug-only, env-gated wrapper — see - /// rustfs/rustfs#6773) goes through this one construction path, so the - /// SSRF guard only needs to be pinned here rather than once per provider - /// file (see backlog#2040's migrate steps and rustfs/rustfs#6764). + /// Cloud providers retain the strict literal guard through the shared + /// constructor; self-hosted providers inject their own policy. #[tokio::test] async fn s3_compatible_backend_rejects_a_loopback_endpoint_before_any_network_setup() { let err = init_error(s3_compatible_params("https://127.0.0.1:9000"), "a loopback endpoint must be rejected").await; diff --git a/crates/ecstore/src/services/tier/warm_backend_minio.rs b/crates/ecstore/src/services/tier/warm_backend_minio.rs index 34657ef9a..299a47d6b 100644 --- a/crates/ecstore/src/services/tier/warm_backend_minio.rs +++ b/crates/ecstore/src/services/tier/warm_backend_minio.rs @@ -27,7 +27,7 @@ use crate::services::tier::{ warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl}; -use rustfs_utils::egress::validate_outbound_url; +use rustfs_utils::egress::validate_outbound_url_allow_private; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; @@ -48,7 +48,7 @@ impl WarmBackendMinIO { // `BucketLookupAuto`; pinning DNS here would break those deployments. bucket_lookup: BucketLookupType::BucketLookupAuto, provider_tag: "minio", - validate_endpoint: validate_outbound_url, + validate_endpoint: validate_outbound_url_allow_private, }) .await?, )) @@ -131,10 +131,7 @@ mod tests { use super::*; use crate::services::tier::tier_config::TierMinIO; - /// The SSRF guard itself is exercised once, generically, in - /// `warm_backend::tests` (see backlog#2040/backlog#2043 and - /// rustfs/rustfs#6764) — this test only pins that this provider's - /// production constructor really is wired through that shared path. + /// Pin the provider's policy wiring independently of the shared constructor. #[tokio::test] async fn new_rejects_loopback_endpoint_before_network_setup() { let conf = TierMinIO { @@ -151,4 +148,23 @@ mod tests { Err(err) => assert!(err.to_string().contains("not allowed")), } } + + #[tokio::test] + async fn issue_8717_private_endpoint_constructs_without_network_setup() { + for endpoint in ["http://192.168.10.200:9000", "https://[fd12:3456::1]:9443"] { + let conf = TierMinIO { + endpoint: endpoint.to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + let backend = WarmBackendMinIO::new(&conf, "tier") + .await + .expect("private MinIO tier should initialize offline"); + assert_eq!(backend.0.client.endpoint_url.as_str().trim_end_matches('/'), endpoint); + assert_eq!(backend.0.bucket, "tier-bucket"); + } + } } diff --git a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs index 9c1934a6d..0bec41b1f 100644 --- a/crates/ecstore/src/services/tier/warm_backend_rustfs.rs +++ b/crates/ecstore/src/services/tier/warm_backend_rustfs.rs @@ -27,7 +27,7 @@ use crate::services::tier::{ warm_backend_s3::WarmBackendS3, }; use rustfs_s3_client::transition_api::{BucketLookupType, ReadCloser, ReaderImpl}; -use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url}; +use rustfs_utils::egress::{OutboundUrlError, validate_outbound_url_allow_private}; const _MAX_PART_SIZE: i64 = 1024 * 1024 * 1024 * 5; const MIN_PART_SIZE: i64 = 1024 * 1024 * 128; @@ -43,7 +43,7 @@ fn validate_rustfs_tier_endpoint(url: &url::Url) -> Result<(), OutboundUrlError> } fn validate_rustfs_tier_endpoint_inner(url: &url::Url, allow_loopback: bool) -> Result<(), OutboundUrlError> { - match validate_outbound_url(url) { + match validate_outbound_url_allow_private(url) { Err(OutboundUrlError::ForbiddenHost { reason: "loopback address" | "loopback host", .. @@ -90,9 +90,8 @@ impl WarmBackendRustFS { // `BucketLookupAuto`; pinning DNS here would break those endpoints. bucket_lookup: BucketLookupType::BucketLookupAuto, provider_tag: "rustfs", - // Debug-only, env-gated loopback exception for this provider's own e2e tier - // tests (rustfs/rustfs#6773); every other provider passes plain - // `validate_outbound_url`. + // Private storage endpoints are allowed; loopback remains a debug-only, + // env-gated exception for this provider's own e2e tier tests. validate_endpoint: validate_rustfs_tier_endpoint, }) .await?, @@ -213,11 +212,27 @@ mod tests { } #[test] - fn loopback_opt_in_does_not_allow_other_private_endpoints() { - let loopback = url::Url::parse("https://127.0.0.1:9000").unwrap(); - assert!(validate_rustfs_tier_endpoint_inner(&loopback, true).is_ok()); + fn loopback_opt_in_preserves_private_storage_and_metadata_boundaries() { + let loopback = url::Url::parse("https://127.0.0.1:9000").expect("loopback URL should parse"); + let private = url::Url::parse("https://10.0.0.1:9000").expect("private URL should parse"); + for allow_loopback in [false, true] { + assert_eq!(validate_rustfs_tier_endpoint_inner(&loopback, allow_loopback).is_ok(), allow_loopback); + assert!(validate_rustfs_tier_endpoint_inner(&private, allow_loopback).is_ok()); + for endpoint in ["http://169.254.169.254", "http://[fd00:ec2::254]", "http://[fe80::1]"] { + let url = url::Url::parse(endpoint).expect("restricted URL should parse"); + assert!(validate_rustfs_tier_endpoint_inner(&url, allow_loopback).is_err(), "{endpoint}"); + } + } + } - let private = url::Url::parse("https://10.0.0.1:9000").unwrap(); - assert!(validate_rustfs_tier_endpoint_inner(&private, true).is_err()); + #[tokio::test] + async fn issue_8717_private_endpoint_constructs_without_network_setup() { + for endpoint in ["http://192.168.10.200:9000", "https://[fd12:3456::1]:9443"] { + let backend = WarmBackendRustFS::new(&rustfs_tier(endpoint), "tier") + .await + .expect("private RustFS tier should initialize offline"); + assert_eq!(backend.0.client.endpoint_url.as_str().trim_end_matches('/'), endpoint); + assert_eq!(backend.0.bucket, "bucket"); + } } } diff --git a/crates/ecstore/src/services/tier/warm_backend_s3.rs b/crates/ecstore/src/services/tier/warm_backend_s3.rs index 8303c1105..ab41b0fa5 100644 --- a/crates/ecstore/src/services/tier/warm_backend_s3.rs +++ b/crates/ecstore/src/services/tier/warm_backend_s3.rs @@ -40,7 +40,7 @@ use rustfs_s3_client::{ transition_api::{BucketLookupType, ObjectInfo, Options, TransitionClient, TransitionCore}, transition_api::{ReadCloser, ReaderImpl}, }; -use rustfs_utils::egress::validate_outbound_url; +use rustfs_utils::egress::validate_outbound_url_allow_private; use rustfs_utils::path::SLASH_SEPARATOR; use s3s::{S3ErrorCode, dto::BucketVersioningStatus}; @@ -102,7 +102,8 @@ impl WarmBackendS3 { return Err(std::io::Error::other(err.to_string())); } }; - validate_outbound_url(&u).map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; + validate_outbound_url_allow_private(&u) + .map_err(|err| std::io::Error::other(format!("tier endpoint is not allowed: {err}")))?; let has_web_identity_token_file = !conf.aws_role_web_identity_token_file.is_empty(); let has_role_arn = !conf.aws_role_arn.is_empty(); @@ -486,6 +487,25 @@ mod tests { assert_eq!(backend.client.endpoint_url.port(), Some(9443)); } + #[tokio::test] + async fn issue_8717_private_endpoint_constructs_without_network_setup() { + for endpoint in ["http://192.168.10.200:9000", "https://[fd12:3456::1]:9443"] { + let conf = TierS3 { + endpoint: endpoint.to_string(), + bucket: "tier-bucket".to_string(), + access_key: "access".to_string(), + secret_key: "secret".to_string(), + region: "us-east-1".to_string(), + ..Default::default() + }; + let backend = WarmBackendS3::new(&conf, "tier") + .await + .expect("private S3 tier should initialize offline"); + assert_eq!(backend.client.endpoint_url.as_str().trim_end_matches('/'), endpoint); + assert_eq!(backend.bucket, "tier-bucket"); + } + } + #[tokio::test] async fn overflowing_get_range_is_rejected_before_network_io() { let listener = match tokio::net::TcpListener::bind("127.0.0.1:0").await { diff --git a/crates/utils/src/egress.rs b/crates/utils/src/egress.rs index 8261d4f63..4f74c72c3 100644 --- a/crates/utils/src/egress.rs +++ b/crates/utils/src/egress.rs @@ -333,6 +333,29 @@ pub fn validate_outbound_url(url: &Url) -> Result<(), OutboundUrlError> { }) } +/// Apply the literal URL guard while allowing private storage endpoints. +/// +/// Known metadata services in IPv6 ULA space remain forbidden. This does not +/// resolve DNS or consult either outbound-origin or replication-loopback settings. +pub fn validate_outbound_url_allow_private(url: &Url) -> Result<(), OutboundUrlError> { + match validate_outbound_url(url) { + Err( + err @ OutboundUrlError::ForbiddenHost { + reason: "private address", + .. + }, + ) => { + let ip = normalized_host(url).and_then(|host| host.parse::().ok()); + if ip.is_some_and(|ip| !is_metadata_endpoint(ip)) { + Ok(()) + } else { + Err(err) + } + } + result => result, + } +} + fn validate_http_url_shape(url: &Url) -> Result<(), OutboundPolicyError> { if !matches!(url.scheme(), "http" | "https") { return Err(OutboundPolicyError::UnsupportedScheme { @@ -549,7 +572,10 @@ fn embedded_ipv4(v6: Ipv6Addr) -> Option { #[cfg(test)] mod tests { - use super::{OutboundPolicy, OutboundUrlError, find_outbound_dns_policy_rejection, validate_outbound_url}; + use super::{ + OutboundPolicy, OutboundUrlError, find_outbound_dns_policy_rejection, validate_outbound_url, + validate_outbound_url_allow_private, + }; use std::collections::HashMap; use std::net::SocketAddr; use url::Url; @@ -560,6 +586,64 @@ mod tests { assert!(validate_outbound_url(&url).is_ok()); } + #[test] + fn private_storage_endpoints_are_allowed_without_relaxing_the_global_guard() { + for endpoint in [ + "http://10.0.0.1:9000", + "http://172.16.0.1:9000", + "https://192.168.10.200:9443", + "http://[fd12:3456::1]:9000", + "http://[::ffff:192.168.10.200]:9000", + "http://[::192.168.10.200]:9000", + ] { + let url = Url::parse(endpoint).expect("private endpoint should parse"); + assert!(validate_outbound_url_allow_private(&url).is_ok(), "{endpoint}"); + assert!(validate_outbound_url(&url).is_err(), "global guard must still reject {endpoint}"); + } + } + + #[test] + fn private_storage_exception_preserves_forbidden_endpoints() { + for endpoint in [ + "http://localhost:9000", + "http://localhost.:9000", + "http://127.0.0.1:9000", + "http://[::1]:9000", + "http://[::ffff:127.0.0.1]:9000", + "http://169.254.1.1:9000", + "http://[fe80::1]:9000", + "http://0.0.0.0:9000", + "http://[::]:9000", + "http://169.254.169.254", + "http://[::ffff:169.254.169.254]", + "http://[::169.254.169.254]", + "http://[fd00:ec2::254]", + "http://[fd00:ec2::23]", + "http://[fd20:ce::254]", + "rustfs://", + ] { + let url = Url::parse(endpoint).expect("forbidden endpoint should parse"); + let original = validate_outbound_url(&url).expect_err("original guard must reject the fixture"); + assert_eq!( + validate_outbound_url_allow_private(&url).expect_err("private exception must retain the rejection"), + original, + "{endpoint}" + ); + } + } + + #[test] + fn private_storage_exception_preserves_public_endpoints() { + for endpoint in [ + "https://tier.example.com:9443", + "https://8.8.8.8", + "https://[2606:4700:4700::1111]", + ] { + let url = Url::parse(endpoint).expect("public endpoint should parse"); + assert!(validate_outbound_url_allow_private(&url).is_ok(), "{endpoint}"); + } + } + #[test] fn validate_outbound_url_rejects_localhost() { let url = Url::parse("https://localhost/webhook").expect("localhost URL should parse"); diff --git a/docs/operations/outbound-connection-policy.md b/docs/operations/outbound-connection-policy.md index b0a094cbe..bd5d8dae2 100644 --- a/docs/operations/outbound-connection-policy.md +++ b/docs/operations/outbound-connection-policy.md @@ -1,14 +1,14 @@ # Outbound Connection Policy **Use this when:** a webhook, audit target, OIDC provider, or object-lambda endpoint on a private or container network (Compose service names, `host.docker.internal`, RFC 1918 addresses) is not being reached, or you need to know which server-initiated connections RustFS restricts and how to allowlist one. -**Source of truth:** `crates/utils/src/egress.rs` (`OutboundPolicy`, `OutboundDnsResolver`, `validate_outbound_url`, `ENV_OUTBOUND_ALLOW_ORIGINS`). +**Source of truth:** `crates/utils/src/egress.rs` (`OutboundPolicy`, `OutboundDnsResolver`, `validate_outbound_url`, `validate_outbound_url_allow_private`, `ENV_OUTBOUND_ALLOW_ORIGINS`). RustFS validates every operator-configured outbound destination to close a server-side request forgery (SSRF) class. Two layers exist: | Layer | What it checks | Escape hatch | | --- | --- | --- | -| Literal URL check (`validate_outbound_url`) | Scheme is `http`/`https`; the host is not `localhost` or a loopback, private, shared, reserved, link-local, unspecified, or metadata address (IPv4-mapped and embedded IPv6 forms are classified by the embedded IPv4) | None | -| Full policy (`OutboundPolicy` + `OutboundDnsResolver`) | The literal check, plus re-validation of every address DNS returns on each new connection, so a hostname cannot be rebound to a restricted address after it was accepted | `RUSTFS_OUTBOUND_ALLOW_ORIGINS` for the loopback, private, shared, and reserved classes | +| Literal URL check (`validate_outbound_url`) | Host is not `localhost`, loopback, private, link-local, unspecified, or `169.254.169.254`; IPv4-mapped and IPv4-compatible IPv6 literals are classified by the embedded IPv4 | None | +| Full policy (`OutboundPolicy` + `OutboundDnsResolver`) | HTTP(S) URL without embedded credentials; additional shared, reserved, and known metadata address checks, plus re-validation of DNS answers on each new connection | `RUSTFS_OUTBOUND_ALLOW_ORIGINS` for the loopback, private, shared, and reserved classes | ## Which subsystem uses which layer @@ -21,11 +21,16 @@ RustFS validates every operator-configured outbound destination to close a serve | Bucket replication targets | Literal check, relaxed | Private addresses are always allowed; loopback only with `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET=true` (`crates/ecstore/src/bucket/remote_s3_client.rs` `validate_remote_endpoint`, shared with on-demand migration sources) | | On-demand migration sources | Literal check, relaxed | Same guard and same escape hatch as replication targets (see below) | | Site replication peers | Literal check | `rustfs/src/site_replication/mod.rs` | -| Tiering warm backends (S3, MinIO, RustFS, Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`; the RustFS provider adds a debug-only, env-gated loopback exception for e2e tests | +| Self-hosted tiering warm backends (S3, MinIO, RustFS) | Literal check, relaxed | `validate_outbound_url_allow_private` allows private storage addresses except known IPv6 metadata addresses; RustFS retains its own debug-only loopback exception | +| Cloud tiering warm backends (Azure, GCS, Aliyun, Tencent, Huawei, R2) | Literal check | `crates/ecstore/src/services/tier/warm_backend.rs` `validate_endpoint`; Wasabi additionally requires its canonical provider endpoint before using the S3 constructor | | Keystone `auth_url` | Literal check | `crates/keystone/src/config.rs` | The allowlist affects only the "Full policy" rows. A literal-check subsystem rejects a hostname that is itself a restricted IP literal, does not re-check what a hostname resolves to, and cannot be widened by `RUSTFS_OUTBOUND_ALLOW_ORIGINS`. +S3, MinIO, and RustFS tiers accept RFC 1918 and ordinary IPv6 ULA addresses without an environment switch. The private-address exception retains the literal guard's other rejections and does not allow the known IPv6 metadata endpoints `fd00:ec2::254`, `fd00:ec2::23`, or `fd20:ce::254`. It does not add DNS filtering or change the other subsystems' policies. `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET` has no effect on tiers. RustFS's `RUSTFS_TIER_RUSTFS_ALLOW_LOOPBACK_ENDPOINT` remains a debug-build-only exception; release builds reject literal loopback endpoints. + +The same tier endpoint check runs when constructing a driver from an existing configuration. Keep the original tier name and endpoint when recovering an upgrade that blocked a private IP; changing them can break existing destination bindings. Restoring the connection does not repair missing legacy cleanup metadata. See [tier free-version recovery](tier-ilm-debugging.md#free-version-recovery-controls). + On-demand migration sources share the replication egress guard exactly: the source endpoint an operator saves through `PUT /rustfs/admin/v3/on-demand-migration/{bucket}` goes through the same `validate_remote_endpoint` in `crates/ecstore/src/bucket/remote_s3_client.rs`, so private addresses are accepted (a source on the same private network is the normal migration case), loopback is refused unless `RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET=true`, and metadata, link-local and unspecified addresses stay blocked. The variable is named for replication because it is one switch over one shared guard; setting it for a single-host migration also relaxes loopback for replication targets, so prefer a real hostname or private address for the source and keep the switch for lab and e2e hosts. `RUSTFS_OUTBOUND_ALLOW_ORIGINS` does not apply here. Two migration-specific rules sit on top of the endpoint guard and are enforced by the config validator, not by the egress layer: a source that names this bucket on this deployment is rejected as a self-reference, and a source matching one of the bucket's own replication targets is rejected as a replication loop. See [on-demand-migration.md](on-demand-migration.md). ## Symptoms diff --git a/docs/operations/tier-ilm-debugging.md b/docs/operations/tier-ilm-debugging.md index a9a1e05a1..36c8fd2a4 100644 --- a/docs/operations/tier-ilm-debugging.md +++ b/docs/operations/tier-ilm-debugging.md @@ -34,6 +34,10 @@ An expired restored local copy can be cleaned up under Object Lock because the r ## Free-version recovery controls +S3, MinIO, and RustFS tiers can use private IP endpoints. After upgrading from a build that rejected those endpoints, keep the existing tier configuration and verify reads before checking whether pending cleanup records drain. A missing durable backend identity or an unknown remote version remains a separate cleanup blocker: beta.7 transitioned objects can lack these fields, and expiry on 1.0.1 only copies fields already present. Endpoint recovery does not backfill them. The legacy transitioned-object reconcile endpoint does not accept free-version records; retain the original metadata and backend/version evidence for a dedicated recovery procedure. + +Forced tier removal skips constructing the remote driver and probing the remote bucket, but still requires idle operations and a complete persisted-reference proof. Live transitioned objects, free versions, delete journals, and transition transactions continue to block removal. Do not delete those records to make a tier removable. + The dedicated free-version recovery loop is enabled by default and is independent of the data scanner and heal switches. Setting `RUSTFS_SCANNER_ENABLED=false` does not stop this repair loop. Set `RUSTFS_TIER_FREE_VERSION_RECOVERY_ENABLED=false` before process startup to disable only the dedicated persisted-marker walk. That setting does not disable lifecycle workers or prevent another scanner path from discovering a free version, and it can leave remote cleanup markers pending for longer, so use it as a break-glass pressure control rather than a cleanup mechanism. Enabled nodes compete for the independent `tier-free-version-recovery/leader.lock` namespace lock in the internal metadata bucket. The leader holds the renewable lock across pages, cooldowns, retries, and idle waits; followers only attempt ownership with a one-second acquisition budget and a random 15–30 second delay between attempts. This uses the existing pool 0 locker majority, so recovery pauses when that lock domain cannot form quorum even if another pool remains available. Recovery resumes when ownership can be acquired again. It does not depend on the main scanner's leader lock or enable switch.