diff --git a/hive/daemon/daemon.go b/hive/daemon/daemon.go index 2880df2f1..6aceb0169 100644 --- a/hive/daemon/daemon.go +++ b/hive/daemon/daemon.go @@ -1855,7 +1855,7 @@ func (d *Daemon) StartInstance(instance *vm.VM) error { bindIP = "127.0.0.1" } instance.Config.NetDevs = append(instance.Config.NetDevs, vm.NetDev{ - Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22", bindIP, sshDebugPort), + Value: fmt.Sprintf("user,id=dev0,hostfwd=tcp:%s:%s-:22,restrict=on", bindIP, sshDebugPort), }) devMac := generateDevMAC(instance.ID) instance.Config.Devices = append(instance.Config.Devices, vm.Device{ diff --git a/hive/daemon/daemon_handlers_instance.go b/hive/daemon/daemon_handlers_instance.go index ac2b6f75f..0db6900f0 100644 --- a/hive/daemon/daemon_handlers_instance.go +++ b/hive/daemon/daemon_handlers_instance.go @@ -264,6 +264,12 @@ func (d *Daemon) handleEC2RunInstances(msg *nats.Msg) { // Launch all instances (volumes and VMs) var successCount int for _, instance := range instances { + // Pre-compute dev MAC so cloud-init can generate per-interface netplan + // that suppresses the default route on the dev/hostfwd NIC. + if d.config.Daemon.DevNetworking && instance.ENIId != "" { + instance.DevMAC = generateDevMAC(instance.ID) + } + // Prepare the root volume, cloud-init, EFI drives via NBD (AMI clone to new volume) volumeInfos, err := d.instanceService.GenerateVolumes(runInstancesInput, instance) if err != nil { diff --git a/hive/handlers/ec2/instance/service_impl.go b/hive/handlers/ec2/instance/service_impl.go index f8a68846c..c12e56ac1 100644 --- a/hive/handlers/ec2/instance/service_impl.go +++ b/hive/handlers/ec2/instance/service_impl.go @@ -64,11 +64,9 @@ instance-id: {{.InstanceID}} local-hostname: {{.Hostname}} ` -// cloudInitNetworkConfig enables DHCP on all NICs. Without this, Ubuntu cloud -// images only configure the first NIC and the guest may have no IPv4 connectivity. -// This is essential for DEV_NETWORKING dual-NIC setups (TAP + hostfwd) and -// also ensures OVN DHCP works reliably on the TAP interface. -const cloudInitNetworkConfig = `network: +// cloudInitNetworkConfigWildcard enables DHCP on all NICs via wildcard match. +// Used when there's no dual-NIC setup (non-VPC or VPC without DEV_NETWORKING). +const cloudInitNetworkConfigWildcard = `network: version: 2 ethernets: allnics: @@ -78,6 +76,38 @@ const cloudInitNetworkConfig = `network: dhcp-identifier: mac ` +// generateNetworkConfig produces the cloud-init network-config for the instance. +// +// When both eniMAC and devMAC are provided (VPC + DEV_NETWORKING), it generates +// per-interface config that suppresses the default route on the dev/hostfwd NIC. +// Without this, both NICs get DHCP default routes at equal metric, causing +// nondeterministic routing — SSH via hostfwd times out and VPC traffic leaks. +// +// The dev NIC still gets an IP via DHCP (needed for hostfwd port forwarding) +// but dhcp4-overrides prevents it from installing routes or DNS. +func generateNetworkConfig(eniMAC, devMAC string) string { + if eniMAC == "" || devMAC == "" { + return cloudInitNetworkConfigWildcard + } + return fmt.Sprintf(`network: + version: 2 + ethernets: + vpc0: + match: + macaddress: "%s" + dhcp4: true + dhcp-identifier: mac + dev0: + match: + macaddress: "%s" + dhcp4: true + dhcp-identifier: mac + dhcp4-overrides: + use-routes: false + use-dns: false +`, eniMAC, devMAC) +} + type CloudInitData struct { Username string SSHKey string @@ -527,7 +557,7 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu } // Create the cloud-init ISO - err = s.createCloudInitISO(input, instance.ID, instance.AccountID, cloudInitVb) + err = s.createCloudInitISO(input, instance, cloudInitVb) if err != nil { return err } @@ -554,7 +584,7 @@ func (s *InstanceServiceImpl) prepareCloudInitVolume(input *ec2.RunInstancesInpu } // createCloudInitISO generates the cloud-init ISO image -func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, instanceId string, accountID string, cloudInitVb *viperblock.VB) error { +func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, instance *vm.VM, cloudInitVb *viperblock.VB) error { // Create ISO writer writer, err := iso9660.NewWriter() if err != nil { @@ -564,7 +594,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i defer writer.Cleanup() // Generate instance metadata - hostname := generateHostname(instanceId) + hostname := generateHostname(instance.ID) // Retrieve SSH pubkey from S3 keyName := "" @@ -572,7 +602,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i keyName = *input.KeyName } - keyPath := fmt.Sprintf("keys/%s/%s", accountID, keyName) + keyPath := fmt.Sprintf("keys/%s/%s", instance.AccountID, keyName) result, err := s.objectStore.GetObject(&awss3.GetObjectInput{ Bucket: aws.String(s.config.Predastore.Bucket), Key: aws.String(keyPath), @@ -615,7 +645,7 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i // Add meta-data metaData := CloudInitMetaData{ - InstanceID: instanceId, + InstanceID: instance.ID, Hostname: hostname, } @@ -633,8 +663,10 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i return errors.New(awserrors.ErrorServerInternal) } - // Add network-config (enables DHCP on all NICs) - err = writer.AddFile(strings.NewReader(cloudInitNetworkConfig), "network-config") + // Add network-config: per-interface when VPC+dev (suppresses dev default route), + // wildcard DHCP otherwise. + networkConfig := generateNetworkConfig(instance.ENIMac, instance.DevMAC) + err = writer.AddFile(strings.NewReader(networkConfig), "network-config") if err != nil { slog.Error("failed to add network-config file", "err", err) return errors.New(awserrors.ErrorServerInternal) diff --git a/hive/handlers/ec2/instance/service_impl_test.go b/hive/handlers/ec2/instance/service_impl_test.go index 74e64c9f7..1c306c802 100644 --- a/hive/handlers/ec2/instance/service_impl_test.go +++ b/hive/handlers/ec2/instance/service_impl_test.go @@ -394,10 +394,48 @@ func TestMockInstanceService(t *testing.T) { assert.Equal(t, "subnet-abc123", *inst.SubnetId) } -func TestCloudInitNetworkConfig(t *testing.T) { - assert.Contains(t, cloudInitNetworkConfig, "version: 2") - assert.Contains(t, cloudInitNetworkConfig, "dhcp4: true") - assert.Contains(t, cloudInitNetworkConfig, "dhcp-identifier: mac") +func TestCloudInitNetworkConfigWildcard(t *testing.T) { + // No MACs → wildcard config (non-VPC or VPC without DEV_NETWORKING) + cfg := generateNetworkConfig("", "") + assert.Contains(t, cfg, "version: 2") + assert.Contains(t, cfg, "dhcp4: true") + assert.Contains(t, cfg, "dhcp-identifier: mac") + assert.Contains(t, cfg, `name: "en*"`, "wildcard should match all en* interfaces") + assert.NotContains(t, cfg, "use-routes") +} + +func TestCloudInitNetworkConfigDualNIC(t *testing.T) { + eniMAC := "02:00:00:61:ef:c2" + devMAC := "02:de:00:60:83:0d" + + cfg := generateNetworkConfig(eniMAC, devMAC) + + // Both MACs present in config + assert.Contains(t, cfg, eniMAC) + assert.Contains(t, cfg, devMAC) + + // VPC NIC gets normal DHCP (with default route) + assert.Contains(t, cfg, "vpc0:") + assert.Contains(t, cfg, "dev0:") + + // Dev NIC has route/DNS suppressed + assert.Contains(t, cfg, "use-routes: false") + assert.Contains(t, cfg, "use-dns: false") + + // No wildcard match — per-interface only + assert.NotContains(t, cfg, `name: "en*"`) +} + +func TestCloudInitNetworkConfigPartialMAC(t *testing.T) { + // Only ENI MAC (VPC without dev) → wildcard + cfg := generateNetworkConfig("02:00:00:61:ef:c2", "") + assert.Contains(t, cfg, `name: "en*"`) + assert.NotContains(t, cfg, "use-routes") + + // Only dev MAC (shouldn't happen, but defensive) → wildcard + cfg = generateNetworkConfig("", "02:de:00:60:83:0d") + assert.Contains(t, cfg, `name: "en*"`) + assert.NotContains(t, cfg, "use-routes") } func TestCloudInitVolumeNamePerInstance(t *testing.T) { diff --git a/hive/vm/vm.go b/hive/vm/vm.go index b62ff7df9..1207a03ea 100644 --- a/hive/vm/vm.go +++ b/hive/vm/vm.go @@ -67,6 +67,10 @@ type VM struct { // VPC networking: ENI attached to this instance (set by RunInstances when VPC mode is active) ENIId string `json:"eni_id,omitempty"` ENIMac string `json:"eni_mac,omitempty"` + + // DevMAC is the MAC for the dev/hostfwd NIC (DEV_NETWORKING mode). + // Set before cloud-init ISO generation so netplan can suppress its default route. + DevMAC string `json:"dev_mac,omitempty"` } // ResetNodeLocalState zeroes out fields that are specific to the daemon node