diff --git a/quinn-proto/src/endpoint.rs b/quinn-proto/src/endpoint.rs
index 9293016f7..b2d619226 100644
--- a/quinn-proto/src/endpoint.rs
+++ b/quinn-proto/src/endpoint.rs
@@ -265,21 +265,14 @@ impl Endpoint {
             return None;
         }
 
-        if first_decode.has_long_header() {
-            if !first_decode.is_initial() {
-                debug!(
-                    "ignoring non-initial packet for unknown connection {}",
-                    dst_cid
-                );
-                return None;
-            }
+        if let Some(version) = first_decode.initial_version() {
             if datagram_len < MIN_INITIAL_SIZE as usize {
                 debug!("ignoring short initial for connection {}", dst_cid);
                 return None;
             }
 
             let crypto = match self.server_config.as_ref().unwrap().crypto.initial_keys(
-                first_decode.version().unwrap(),
+                version,
                 &dst_cid,
                 Side::Server,
             ) {
@@ -289,7 +282,7 @@ impl Endpoint {
                     // `EndpointConfig`.
                     debug!(
                         "ignoring initial packet version {:#x} unsupported by cryptographic layer",
-                        first_decode.version().unwrap()
+                        version
                     );
                     return None;
                 }
@@ -303,6 +296,12 @@ impl Endpoint {
                     None
                 }
             };
+        } else if first_decode.has_long_header() {
+            debug!(
+                "ignoring non-initial packet for unknown connection {}",
+                dst_cid
+            );
+            return None;
         }
 
         //
diff --git a/quinn-proto/src/packet.rs b/quinn-proto/src/packet.rs
index f935fc7f9..c7954c1b4 100644
--- a/quinn-proto/src/packet.rs
+++ b/quinn-proto/src/packet.rs
@@ -55,6 +55,13 @@ impl PartialDecode {
         self.buf.get_ref()
     }
 
+    pub(crate) fn initial_version(&self) -> Option<u32> {
+        match self.plain_header {
+            PlainHeader::Initial { version, .. } => Some(version),
+            _ => None,
+        }
+    }
+
     pub(crate) fn has_long_header(&self) -> bool {
         !matches!(self.plain_header, PlainHeader::Short { .. })
     }
@@ -96,10 +103,6 @@ impl PartialDecode {
         self.buf.get_ref().len()
     }
 
-    pub fn version(&self) -> Option<u32> {
-        self.plain_header.version()
-    }
-
     pub(crate) fn finish(
         self,
         header_crypto: Option<&dyn crypto::HeaderKey>,
@@ -596,14 +599,6 @@ impl PlainHeader {
             }
         }
     }
-
-    fn version(&self) -> Option<u32> {
-        use PlainHeader::*;
-        match *self {
-            Initial { version, .. } | Long { version, .. } | Retry { version, .. } => Some(version),
-            _ => None,
-        }
-    }
 }
 
 // An encoded packet number