fix(replication): allow loopback replication targets under an explicit test opt-in (#4725)

* fix(replication): allow loopback replication targets under an explicit test opt-in Commit 5c7c757a3 (#4712) activated the previously-dormant replication e2e suite (they had never run anywhere). All 9 fast tests then failed on main because the SSRF egress guard rejects the 127.0.0.1 targets the e2e harness configures: `target endpoint is not allowed: outbound URL host '127.0.0.1' is not allowed: loopback address`. The whole harness runs on loopback, so every replication test hit this before reaching its actual assertion. Loopback is a genuine SSRF vector and must stay rejected in production, so this does not relax the guard. Instead `validate_replication_target_endpoint` gains an off-by-default opt-in (`RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET`) that re-enables loopback targets (127.0.0.1 / ::1 / localhost) for single-host multi-instance dev and the e2e harness. Private addresses stay unconditionally allowed as before; the opt-in does not widen into link-local or the cloud metadata endpoint. The e2e harness sets the env for every server it spawns (single-node and cluster paths), overridable via extra_env. Verified end-to-end: all 9 previously-failing replication_extension_test smoke tests pass against a locally built binary. New unit tests in bucket_target_sys pin the matrix — public/private always allowed, loopback gated on the opt-in in both IP and hostname forms, and metadata/link-local still rejected even with the opt-in on. Refs: backlog#1147 Co-Authored-By: heihutu <heihutu@gmail.com> * test(replication): rename optin -> opt_in to satisfy typos check Pure rename of three unit-test function names; no behaviour change. Co-Authored-By: heihutu <heihutu@gmail.com> --------- Co-authored-by: heihutu <heihutu@gmail.com>
2026-08-19 02:56:18 +00:00 · 2026-07-11 15:49:39 +08:00
parent f63af3df63
commit 2ebe8e561b
4 changed files with 300 additions and 95 deletions
@@ -966,13 +966,37 @@ fn has_custom_ca_pem(target: &BucketTarget) -> bool {
    !target.ca_cert_pem.trim().is_empty()
 }

+/// Env opt-in that re-enables loopback replication targets. Loopback (`127.0.0.1`,
+/// `::1`, `localhost`) is a classic SSRF vector and stays rejected by default, but
+/// single-host multi-instance dev setups and the e2e harness legitimately replicate
+/// over loopback. Never set this in production.
+const ALLOW_LOOPBACK_REPLICATION_TARGET_ENV: &str = "RUSTFS_REPLICATION_ALLOW_LOOPBACK_TARGET";
+
+fn loopback_replication_targets_allowed() -> bool {
+    std::env::var(ALLOW_LOOPBACK_REPLICATION_TARGET_ENV)
+        .map(|v| v.eq_ignore_ascii_case("true") || v == "1")
+        .unwrap_or(false)
+}
+
 fn validate_replication_target_endpoint(url: &Url) -> Result<(), OutboundUrlError> {
+    validate_replication_target_endpoint_inner(url, loopback_replication_targets_allowed())
+}
+
+fn validate_replication_target_endpoint_inner(url: &Url, allow_loopback: bool) -> Result<(), OutboundUrlError> {
    match validate_outbound_url(url) {
        Ok(()) => Ok(()),
+        // Replication targets are trusted infrastructure the operator configures, and
+        // legitimately live on private networks, so private addresses are always allowed.
        Err(OutboundUrlError::ForbiddenHost {
            reason: "private address",
            ..
        }) => Ok(()),
+        // Loopback is far higher SSRF risk, so it is allowed only under the explicit,
+        // off-by-default opt-in above (single-host multi-instance / the e2e harness).
+        Err(OutboundUrlError::ForbiddenHost {
+            reason: "loopback address" | "loopback host",
+            ..
+        }) if allow_loopback => Ok(()),
        Err(err) => Err(err),
    }
 }
@@ -1906,6 +1930,76 @@ mod tests {
        assert!(!replication_target_versioning_enabled(None));
    }

+    fn parse_url(raw: &str) -> Url {
+        Url::parse(raw).expect("test URL should parse")
+    }
+
+    #[test]
+    fn replication_endpoint_always_allows_public_and_private() {
+        // Public hosts and private-network targets are allowed regardless of the
+        // loopback opt-in — replication commonly runs across trusted private infra.
+        for allow_loopback in [false, true] {
+            assert!(validate_replication_target_endpoint_inner(&parse_url("https://s3.example.com"), allow_loopback).is_ok());
+            assert!(validate_replication_target_endpoint_inner(&parse_url("http://10.0.0.5:9000"), allow_loopback).is_ok());
+            assert!(validate_replication_target_endpoint_inner(&parse_url("http://192.168.1.20"), allow_loopback).is_ok());
+        }
+    }
+
+    #[test]
+    fn replication_endpoint_rejects_loopback_without_opt_in() {
+        // Default (production) behaviour: loopback IP and localhost host both rejected.
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://127.0.0.1:9000"), false)
+            .expect_err("loopback IP must be rejected by default");
+        assert!(matches!(
+            err,
+            OutboundUrlError::ForbiddenHost {
+                reason: "loopback address",
+                ..
+            }
+        ));
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://localhost:9000"), false)
+            .expect_err("localhost must be rejected by default");
+        assert!(matches!(
+            err,
+            OutboundUrlError::ForbiddenHost {
+                reason: "loopback host",
+                ..
+            }
+        ));
+    }
+
+    #[test]
+    fn replication_endpoint_allows_loopback_with_opt_in() {
+        // e2e harness / single-host multi-instance: opt-in re-enables loopback in
+        // both IP (127.0.0.1, ::1) and hostname (localhost) forms.
+        assert!(validate_replication_target_endpoint_inner(&parse_url("http://127.0.0.1:9000"), true).is_ok());
+        assert!(validate_replication_target_endpoint_inner(&parse_url("http://[::1]:9000"), true).is_ok());
+        assert!(validate_replication_target_endpoint_inner(&parse_url("http://localhost:9000"), true).is_ok());
+    }
+
+    #[test]
+    fn replication_endpoint_opt_in_does_not_open_other_ssrf_targets() {
+        // The loopback opt-in must not widen into link-local / metadata endpoints.
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://169.254.169.254/latest/meta-data"), true)
+            .expect_err("metadata endpoint must stay rejected even with loopback opt-in");
+        assert!(matches!(
+            err,
+            OutboundUrlError::ForbiddenHost {
+                reason: "metadata endpoint",
+                ..
+            }
+        ));
+        let err = validate_replication_target_endpoint_inner(&parse_url("http://[fe80::1]:9000"), true)
+            .expect_err("link-local must stay rejected even with loopback opt-in");
+        assert!(matches!(
+            err,
+            OutboundUrlError::ForbiddenHost {
+                reason: "link-local address",
+                ..
+            }
+        ));
+    }
+
    #[test]
    fn remote_target_connection_error_display_redacts_access_key() {
        let err = BucketTargetError::RemoteTargetConnectionErr {
@@ -318,6 +318,20 @@ impl ReplicationResyncer {
                return Ok(());
            }

+            if state.resync_status == ResyncStatusType::ResyncCanceled && status != ResyncStatusType::ResyncCanceled {
+                debug!(
+                    event = EVENT_RESYNC_STATUS_UPDATE_SKIPPED,
+                    component = LOG_COMPONENT_ECSTORE,
+                    subsystem = LOG_SUBSYSTEM_REPLICATION_RESYNC,
+                    bucket = %opts.bucket,
+                    arn = %opts.arn,
+                    incoming_status = %status,
+                    reason = "canceled_status_is_terminal",
+                    "Skipped resync status update after cancellation"
+                );
+                return Ok(());
+            }
+
            if state.resync_id.is_empty() {
                state.resync_id = opts.resync_id.clone();
            }
@@ -339,7 +353,24 @@ impl ReplicationResyncer {
            (bucket_status.clone(), status_duration)
        };

-        save_resync_status(&opts.bucket, &bucket_status, obj_layer).await?;
+        save_resync_status(&opts.bucket, &bucket_status, obj_layer.clone()).await?;
+        if status != ResyncStatusType::ResyncCanceled {
+            let canceled_status = self
+                .status_map
+                .read()
+                .await
+                .get(&opts.bucket)
+                .filter(|current| {
+                    current.targets_map.get(&opts.arn).is_some_and(|target| {
+                        target.resync_id == opts.resync_id && target.resync_status == ResyncStatusType::ResyncCanceled
+                    })
+                })
+                .cloned();
+            if let Some(canceled_status) = canceled_status {
+                save_resync_status(&opts.bucket, &canceled_status, obj_layer).await?;
+                return Ok(());
+            }
+        }
        if let Some(stats) = runtime_sources::replication_stats() {
            stats.record_resync_status(&opts.bucket, status, status_duration).await;
        }