dev networking, remote dhcp, due to double route bug

hive/daemon/daemon.go

@@ -1855,7 +1855,7 @@ func (d *Daemon) StartInstance(instance *vm.VM) error {
 					bindIP = "127.0.0.1"
 				}
 				instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{
-					Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort),
+					Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22,restrict=on", bindIP, sshDebugPort),
 				})
 				devMac := generateDevMAC(instance.ID)
 				instance.Config.Devices = append(instance.Config.Devices, vm.Device{

hive/daemon/daemon_handlers_instance.go

@@ -264,6 +264,12 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) {
 	// Launch all instances (volumes and VMs)
 	var successCount int
 	for _, instance := range instances {
+		// Pre-compute dev MAC so cloud-init can generate per-interface netplan
+		// that suppresses the default route on the dev/hostfwd NIC.
+		if d.config.Daemon.DevNetworking && instance.ENIId != "" {
+			instance.DevMAC = generateDevMAC(instance.ID)
+		}
+
 		// Prepare the root volume, cloud-init, EFI drives via NBD (AMI clone to new volume)
 		volumeInfos, err := d.instanceService.GenerateVolumes(runInstancesInput, instance)
 		if err != nil {

hive/handlers/ec2/instance/service_impl.go

@@ -64,11 +64,9 @@ instance-id: {{.InstanceID}}
 local-hostname: {{.Hostname}}
 `
 
-// cloudInitNetworkConfig enables DHCP on all NICs. Without this, Ubuntu cloud
-// images only configure the first NIC and the guest may have no IPv4 connectivity.
-// This is essential for DEV_NETWORKING dual-NIC setups (TAP + hostfwd) and
-// also ensures OVN DHCP works reliably on the TAP interface.
-const cloudInitNetworkConfig = `network:
+// cloudInitNetworkConfigWildcard enables DHCP on all NICs via wildcard match.
+// Used when there's no dual-NIC setup (non-VPC or VPC without DEV_NETWORKING).
+const cloudInitNetworkConfigWildcard = `network:
   version: 2
   ethernets:
     allnics:
@@ -78,6 +76,38 @@ const cloudInitNetworkConfig = `network:
       dhcp-identifier: mac
 `
 
+// generateNetworkConfig produces the cloud-init network-config for the instance.
+//
+// When both eniMAC and devMAC are provided (VPC + DEV_NETWORKING), it generates
+// per-interface config that suppresses the default route on the dev/hostfwd NIC.
+// Without this, both NICs get DHCP default routes at equal metric, causing
+// nondeterministic routing — SSH via hostfwd times out and VPC traffic leaks.
+//
+// The dev NIC still gets an IP via DHCP (needed for hostfwd port forwarding)
+// but dhcp4-overrides prevents it from installing routes or DNS.
+func generateNetworkConfig(eniMAC, devMAC string) string {
+	if eniMAC == "" || devMAC == "" {
+		return cloudInitNetworkConfigWildcard
+	}
+	return fmt.Sprintf(`network:
+  version: 2
+  ethernets:
+    vpc0:
+      match:
+        macaddress: "%s"
+      dhcp4: true
+      dhcp-identifier: mac
+    dev0:
+      match:
+        macaddress: "%s"
+      dhcp4: true
+      dhcp-identifier: mac
+      dhcp4-overrides:
+        use-routes: false
+        use-dns: false
+`, eniMAC, devMAC)
+}
+
 type CloudInitData struct {
 	Username            string
 	SSHKey              string
@@ -527,7 +557,7 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu
 	}
 
 	// Create the cloud-init ISO
-	err = s.createCloudInitISO(input, instance.ID, instance.AccountID, cloudInitVb)
+	err = s.createCloudInitISO(input, instance, cloudInitVb)
 	if err != nil {
 		return err
 	}
@@ -554,7 +584,7 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu
 }
 
 // createCloudInitISO generates the cloud-init ISO image
-func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, instanceId string, accountID string, cloudInitVb *viperblock.VB) error {
+func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, instance *vm.VM, cloudInitVb *viperblock.VB) error {
 	// Create ISO writer
 	writer, err := iso9660.NewWriter()
 	if err != nil {
@@ -564,7 +594,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 	defer writer.Cleanup()
 
 	// Generate instance metadata
-	hostname := generateHostname(instanceId)
+	hostname := generateHostname(instance.ID)
 
 	// Retrieve SSH pubkey from S3
 	keyName := ""
@@ -572,7 +602,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 		keyName = *input.KeyName
 	}
 
-	keyPath := fmt.Sprintf("keys/%s/%s", accountID, keyName)
+	keyPath := fmt.Sprintf("keys/%s/%s", instance.AccountID, keyName)
 	result, err := s.objectStore.GetObject(&awss3.GetObjectInput{
 		Bucket: aws.String(s.config.Predastore.Bucket),
 		Key:    aws.String(keyPath),
@@ -615,7 +645,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 
 	// Add meta-data
 	metaData := CloudInitMetaData{
-		InstanceID: instanceId,
+		InstanceID: instance.ID,
 		Hostname:   hostname,
 	}
 
@@ -633,8 +663,10 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 		return errors.New(awserrors.ErrorServerInternal)
 	}
 
-	// Add network-config (enables DHCP on all NICs)
-	err = writer.AddFile(strings.NewReader(cloudInitNetworkConfig), "network-config")
+	// Add network-config: per-interface when VPC+dev (suppresses dev default route),
+	// wildcard DHCP otherwise.
+	networkConfig := generateNetworkConfig(instance.ENIMac, instance.DevMAC)
+	err = writer.AddFile(strings.NewReader(networkConfig), "network-config")
 	if err != nil {
 		slog.Error("failed to add network-config file", "err", err)
 		return errors.New(awserrors.ErrorServerInternal)

hive/handlers/ec2/instance/service_impl_test.go

@@ -394,10 +394,48 @@ func TestMockInstanceService(t *testing.T) {
 	assert.Equal(t, "subnet-abc123", *inst.SubnetId)
 }
 
-func TestCloudInitNetworkConfig(t *testing.T) {
-	assert.Contains(t, cloudInitNetworkConfig, "version: 2")
-	assert.Contains(t, cloudInitNetworkConfig, "dhcp4: true")
-	assert.Contains(t, cloudInitNetworkConfig, "dhcp-identifier: mac")
+func TestCloudInitNetworkConfigWildcard(t *testing.T) {
+	// No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING)
+	cfg := generateNetworkConfig("", "")
+	assert.Contains(t, cfg, "version: 2")
+	assert.Contains(t, cfg, "dhcp4: true")
+	assert.Contains(t, cfg, "dhcp-identifier: mac")
+	assert.Contains(t, cfg, `name: "en*"`, "wildcard should match all en* interfaces")
+	assert.NotContains(t, cfg, "use-routes")
+}
+
+func TestCloudInitNetworkConfigDualNIC(t *testing.T) {
+	eniMAC := "02:00:00:61:ef:c2"
+	devMAC := "02:de:00:60:83:0d"
+
+	cfg := generateNetworkConfig(eniMAC, devMAC)
+
+	// Both MACs present in config
+	assert.Contains(t, cfg, eniMAC)
+	assert.Contains(t, cfg, devMAC)
+
+	// VPC NIC gets normal DHCP (with default route)
+	assert.Contains(t, cfg, "vpc0:")
+	assert.Contains(t, cfg, "dev0:")
+
+	// Dev NIC has route/DNS suppressed
+	assert.Contains(t, cfg, "use-routes: false")
+	assert.Contains(t, cfg, "use-dns: false")
+
+	// No wildcard match — per-interface only
+	assert.NotContains(t, cfg, `name: "en*"`)
+}
+
+func TestCloudInitNetworkConfigPartialMAC(t *testing.T) {
+	// Only ENI MAC (VPC without dev) → wildcard
+	cfg := generateNetworkConfig("02:00:00:61:ef:c2", "")
+	assert.Contains(t, cfg, `name: "en*"`)
+	assert.NotContains(t, cfg, "use-routes")
+
+	// Only dev MAC (shouldn't happen, but defensive) → wildcard
+	cfg = generateNetworkConfig("", "02:de:00:60:83:0d")
+	assert.Contains(t, cfg, `name: "en*"`)
+	assert.NotContains(t, cfg, "use-routes")
 }
 
 func TestCloudInitVolumeNamePerInstance(t *testing.T) {

hive/vm/vm.go

@@ -67,6 +67,10 @@ type VM struct {
 	// VPC networking: ENI attached to this instance (set by RunInstances when VPC mode is active)
 	ENIId  string `json:"eni_id,omitempty"`
 	ENIMac string `json:"eni_mac,omitempty"`
+
+	// DevMAC is the MAC for the dev/hostfwd NIC (DEV_NETWORKING mode).
+	// Set before cloud-init ISO generation so netplan can suppress its default route.
+	DevMAC string `json:"dev_mac,omitempty"`
 }
 
 // ResetNodeLocalState zeroes out fields that are specific to the daemon node