diff --git a/quinn-proto/src/config.rs b/quinn-proto/src/config.rs
index 411aa4818..3eb7bcbc2 100644
--- a/quinn-proto/src/config.rs
+++ b/quinn-proto/src/config.rs
@@ -50,7 +50,7 @@ pub struct TransportConfig {
     #[cfg(test)]
     pub(crate) deterministic_packet_numbers: bool,
 
-    pub(crate) congestion_controller_factory: Box<dyn congestion::ControllerFactory + Send + Sync>,
+    pub(crate) congestion_controller_factory: Arc<dyn congestion::ControllerFactory + Send + Sync>,
 
     pub(crate) enable_segmentation_offload: bool,
 }
@@ -293,9 +293,9 @@ impl TransportConfig {
     /// ```
     pub fn congestion_controller_factory(
         &mut self,
-        factory: impl congestion::ControllerFactory + Send + Sync + 'static,
+        factory: Arc<dyn congestion::ControllerFactory + Send + Sync + 'static>,
     ) -> &mut Self {
-        self.congestion_controller_factory = Box::new(factory);
+        self.congestion_controller_factory = factory;
         self
     }
 
@@ -349,7 +349,7 @@ impl Default for TransportConfig {
             #[cfg(test)]
             deterministic_packet_numbers: false,
 
-            congestion_controller_factory: Box::new(Arc::new(congestion::CubicConfig::default())),
+            congestion_controller_factory: Arc::new(congestion::CubicConfig::default()),
 
             enable_segmentation_offload: true,
         }
diff --git a/quinn-proto/src/congestion.rs b/quinn-proto/src/congestion.rs
index 41d24566d..ec6bde82c 100644
--- a/quinn-proto/src/congestion.rs
+++ b/quinn-proto/src/congestion.rs
@@ -2,6 +2,7 @@
 
 use crate::connection::RttEstimator;
 use std::any::Any;
+use std::sync::Arc;
 use std::time::Instant;
 
 mod bbr;
@@ -77,7 +78,7 @@ pub trait Controller: Send {
 /// Constructs controllers on demand
 pub trait ControllerFactory {
     /// Construct a fresh `Controller`
-    fn build(&self, now: Instant, current_mtu: u16) -> Box<dyn Controller>;
+    fn build(self: Arc<Self>, now: Instant, current_mtu: u16) -> Box<dyn Controller>;
 }
 
 const BASE_DATAGRAM_SIZE: u64 = 1200;
diff --git a/quinn-proto/src/congestion/bbr/mod.rs b/quinn-proto/src/congestion/bbr/mod.rs
index 07dde798d..a522f0b0f 100644
--- a/quinn-proto/src/congestion/bbr/mod.rs
+++ b/quinn-proto/src/congestion/bbr/mod.rs
@@ -522,9 +522,9 @@ impl Default for BbrConfig {
     }
 }
 
-impl ControllerFactory for Arc<BbrConfig> {
-    fn build(&self, _now: Instant, current_mtu: u16) -> Box<dyn Controller> {
-        Box::new(Bbr::new(self.clone(), current_mtu))
+impl ControllerFactory for BbrConfig {
+    fn build(self: Arc<Self>, _now: Instant, current_mtu: u16) -> Box<dyn Controller> {
+        Box::new(Bbr::new(self, current_mtu))
     }
 }
 
diff --git a/quinn-proto/src/congestion/cubic.rs b/quinn-proto/src/congestion/cubic.rs
index 7b6d41d6b..1df9ac4d7 100644
--- a/quinn-proto/src/congestion/cubic.rs
+++ b/quinn-proto/src/congestion/cubic.rs
@@ -259,8 +259,8 @@ impl Default for CubicConfig {
     }
 }
 
-impl ControllerFactory for Arc<CubicConfig> {
-    fn build(&self, now: Instant, current_mtu: u16) -> Box<dyn Controller> {
-        Box::new(Cubic::new(self.clone(), now, current_mtu))
+impl ControllerFactory for CubicConfig {
+    fn build(self: Arc<Self>, now: Instant, current_mtu: u16) -> Box<dyn Controller> {
+        Box::new(Cubic::new(self, now, current_mtu))
     }
 }
diff --git a/quinn-proto/src/congestion/new_reno.rs b/quinn-proto/src/congestion/new_reno.rs
index 0b085c377..264db0fb6 100644
--- a/quinn-proto/src/congestion/new_reno.rs
+++ b/quinn-proto/src/congestion/new_reno.rs
@@ -157,8 +157,8 @@ impl Default for NewRenoConfig {
     }
 }
 
-impl ControllerFactory for Arc<NewRenoConfig> {
-    fn build(&self, now: Instant, current_mtu: u16) -> Box<dyn Controller> {
-        Box::new(NewReno::new(self.clone(), now, current_mtu))
+impl ControllerFactory for NewRenoConfig {
+    fn build(self: Arc<Self>, now: Instant, current_mtu: u16) -> Box<dyn Controller> {
+        Box::new(NewReno::new(self, now, current_mtu))
     }
 }
diff --git a/quinn-proto/src/connection/mod.rs b/quinn-proto/src/connection/mod.rs
index ea9a2258a..801760f35 100644
--- a/quinn-proto/src/connection/mod.rs
+++ b/quinn-proto/src/connection/mod.rs
@@ -281,6 +281,7 @@ impl Connection {
                 config.initial_rtt,
                 config
                     .congestion_controller_factory
+                    .clone()
                     .build(now, config.get_initial_mtu()),
                 config.get_initial_mtu(),
                 config.min_mtu,
@@ -2873,6 +2874,7 @@ impl Connection {
                 self.config.initial_rtt,
                 self.config
                     .congestion_controller_factory
+                    .clone()
                     .build(now, self.config.get_initial_mtu()),
                 self.config.get_initial_mtu(),
                 self.config.min_mtu,