diff --git a/quinn-proto/src/connection/mod.rs b/quinn-proto/src/connection/mod.rs
index 2df494b97..824cc9f2f 100644
--- a/quinn-proto/src/connection/mod.rs
+++ b/quinn-proto/src/connection/mod.rs
@@ -1595,6 +1595,15 @@ impl Connection {
         debug_assert!(self.side.is_server());
         let len = packet.header_data.len() + packet.payload.len();
         self.path.total_recvd = len as u64;
+        match self.state {
+            State::Handshake(ref mut state) => match packet.header {
+                Header::Initial { ref token, .. } => {
+                    state.token = Some(token.clone());
+                }
+                _ => unreachable!("first packet must be an Initial packet"),
+            },
+            _ => unreachable!("first packet must be delivered in Handshake state"),
+        }
 
         self.on_packet_authenticated(
             now,
@@ -1916,6 +1925,18 @@ impl Connection {
                     trace!("dropping short packet during handshake");
                     return;
                 } else {
+                    if let Header::Initial { ref token, .. } = packet.header {
+                        if let State::Handshake(ref hs) = self.state {
+                            if self.side.is_server() && Some(token) != hs.token.as_ref() {
+                                // Clients must send the same retry token in every Initial. Initial
+                                // packets can be spoofed, so we discard rather than killing the
+                                // connection.
+                                warn!("discarding Initial with invalid retry token");
+                                return;
+                            }
+                        }
+                    }
+
                     if !self.state.is_closed() {
                         let spin = match packet.header {
                             Header::Short { spin, .. } => spin,
@@ -3214,8 +3235,6 @@ mod state {
         /// Always set for servers
         pub rem_cid_set: bool,
         /// Stateless retry token, if the peer has provided one
-        ///
-        /// Only set for clients
         pub token: Option<Bytes>,
         /// First cryptographic message
         ///
diff --git a/quinn-proto/src/connection/packet_builder.rs b/quinn-proto/src/connection/packet_builder.rs
index 0630c3dd7..02f81694d 100644
--- a/quinn-proto/src/connection/packet_builder.rs
+++ b/quinn-proto/src/connection/packet_builder.rs
@@ -39,6 +39,7 @@ impl PacketBuilder {
         conn: &mut Connection,
         version: u32,
     ) -> Option<PacketBuilder> {
+        let is_client = conn.side().is_client();
         // Initiate key update if we're approaching the confidentiality limit
         let confidentiality_limit = conn.spaces[space_id]
             .crypto
@@ -107,7 +108,9 @@ impl PacketBuilder {
                 src_cid: conn.handshake_cid,
                 dst_cid: conn.rem_cids.active(),
                 token: match conn.state {
-                    State::Handshake(ref state) => state.token.clone().unwrap_or_else(Bytes::new),
+                    State::Handshake(ref state) if is_client => {
+                        state.token.clone().unwrap_or_else(Bytes::new)
+                    }
                     _ => Bytes::new(),
                 },
                 number,