diff --git a/quinn/src/connection.rs b/quinn/src/connection.rs
index 43ef2f8d2..cf38a1f13 100644
--- a/quinn/src/connection.rs
+++ b/quinn/src/connection.rs
@@ -932,7 +932,7 @@ impl Clone for ConnectionRef {
 
 impl Drop for ConnectionRef {
     fn drop(&mut self) {
-        if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 0 {
+        if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 {
             return;
         }
 
diff --git a/quinn/src/endpoint.rs b/quinn/src/endpoint.rs
index f0fff2cb5..6bb7a5951 100644
--- a/quinn/src/endpoint.rs
+++ b/quinn/src/endpoint.rs
@@ -784,7 +784,7 @@ impl Clone for EndpointRef {
 
 impl Drop for EndpointRef {
     fn drop(&mut self) {
-        if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 0 {
+        if self.shared.ref_count.fetch_sub(1, Ordering::Relaxed) > 1 {
             return;
         }