gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-23 12:47:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(ddil): tunnel guest SSH through host node

Browse Source 
EIP from MapPublicIpOnLaunch sits on the cluster's external pool subnet,
which is OVN-routed — reachable from any cluster node, not from the
off-cluster runner sitting on the DDIL_NODES WAN subnet.

Restore the host-jump pattern: SSH the hosting node (already known from
findHost), dial publicIP:22 over that connection, run a guest SSH
handshake over the tunnel. Symmetric with how shell e2e reaches guests
from inside a node.

Continues mulga-siv-57.
This commit is contained in:
joshsiv-mulga
2026-05-11 14:59:19 +10:00
parent b3f1b678e0
commit d2cd38fa76
1 changed files with 51 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/e2e/ddil/harness/witness.go
+51 -17
View File
@@ -35,13 +35,14 @@ var cloudInitWitness []byte
// tests/e2e/run-multinode-e2e.sh's AWSGW_PORT.
const gatewayPort = 9999
// Witness bundles the AWS SDK client, guest SSH credentials, and AMI
// Witness bundles the AWS SDK client, guest/host SSH credentials, and AMI
// selection needed to launch and interrogate counter VMs. One Witness is
// shared across the scenarios in a suite.
type Witness struct {
ec2 *ec2.EC2
cluster *Cluster
ssh SSH
hostSigner ssh.Signer
guestSigner ssh.Signer
guestUser string
ami string
@@ -93,6 +94,11 @@ func NewWitness(cluster *Cluster, transport SSH) (*Witness, error) {
return nil, fmt.Errorf("ddil harness: aws session: %w", err)
}
hostSigner, err := loadSigner(cluster.SSHKeyPath)
if err != nil {
return nil, fmt.Errorf("ddil harness: host ssh key: %w", err)
}
guestKeyPath := os.Getenv("DDIL_GUEST_SSH_KEY")
if guestKeyPath == "" {
return nil, errors.New("ddil harness: NewWitness requires DDIL_GUEST_SSH_KEY (path to the private key paired with DDIL_WITNESS_KEY_NAME's registered material)")
@@ -106,6 +112,7 @@ func NewWitness(cluster *Cluster, transport SSH) (*Witness, error) {
ec2: ec2.New(sess),
cluster: cluster,
ssh: transport,
hostSigner: hostSigner,
guestSigner: guestSigner,
guestUser: envDefault("DDIL_GUEST_SSH_USER", "ubuntu"),
ami: os.Getenv("DDIL_WITNESS_AMI"),
@@ -185,7 +192,7 @@ func LaunchWitnessVM(ctx context.Context, w *Witness, host Node) (*WitnessVM, er
_ = w.terminate(ctx, id)
return nil, err
}
baseline, err := readCounter(ctx, w, publicIP)
baseline, err := readCounter(ctx, w, placed, publicIP)
if err != nil {
// The counter service may not have started yet on a freshly
// booted guest; a zero baseline is the expected steady state
@@ -211,10 +218,10 @@ func LaunchWitnessVM(ctx context.Context, w *Witness, host Node) (*WitnessVM, er
return nil, fmt.Errorf("ddil harness: LaunchWitnessVM: %w after %d attempts", lastErr, maxPlacementAttempts)
}
// ReadCounter SSHes the guest at its public IP and returns the current
// /var/lib/counter value.
// ReadCounter SSHes the guest at its public IP (tunnelled through the
// hosting cluster node) and returns the current /var/lib/counter value.
func (v *WitnessVM) ReadCounter(ctx context.Context) (int, error) {
return readCounter(ctx, v.w, v.PublicIP)
return readCounter(ctx, v.w, v.HostNode, v.PublicIP)
}
// Terminate asks EC2 to shut the witness down. Scenarios call this from
@@ -416,26 +423,33 @@ func (w *Witness) terminate(ctx context.Context, id string) error {
return nil
}
// readCounter SSHes the witness at publicIP:22 directly and reads
// /var/lib/counter. The cluster's MapPublicIpOnLaunch wires the EIP from
// the external pool, which sits on the same WAN subnet as DDIL_NODES, so
// the runner reaches it without a host relay.
func readCounter(ctx context.Context, w *Witness, publicIP string) (int, error) {
// readCounter tunnels SSH through host to publicIP:22 and reads
// /var/lib/counter. The cluster's external pool (MapPublicIpOnLaunch) sits
// on an OVN-managed subnet that isn't routable from the off-cluster runner;
// the host node, however, reaches it via OVN's gateway router. Using the
// VM's hosting node as the jump keeps the relay inside the cluster.
func readCounter(ctx context.Context, w *Witness, host Node, publicIP string) (int, error) {
hostClient, err := dialHost(ctx, host, w.cluster.SSHUser, w.hostSigner)
if err != nil {
return 0, err
}
defer func() { _ = hostClient.Close() }()
guestAddr := net.JoinHostPort(publicIP, "22")
tunnel, err := hostClient.DialContext(ctx, "tcp", guestAddr)
if err != nil {
return 0, fmt.Errorf("ddil harness: tunnel %s → %s: %w", host.Name, guestAddr, err)
}
guestCfg := &ssh.ClientConfig{
User: w.guestUser,
Auth: []ssh.AuthMethod{ssh.PublicKeys(w.guestSigner)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // ephemeral test VM
Timeout: 10 * time.Second,
}
d := net.Dialer{Timeout: guestCfg.Timeout}
conn, err := d.DialContext(ctx, "tcp", guestAddr)
c, chans, reqs, err := ssh.NewClientConn(tunnel, guestAddr, guestCfg)
if err != nil {
return 0, fmt.Errorf("ddil harness: dial guest %s: %w", guestAddr, err)
}
c, chans, reqs, err := ssh.NewClientConn(conn, guestAddr, guestCfg)
if err != nil {
_ = conn.Close()
_ = tunnel.Close()
return 0, fmt.Errorf("ddil harness: guest ssh handshake on %s: %w", guestAddr, err)
}
guestClient := ssh.NewClient(c, chans, reqs)
@@ -462,6 +476,26 @@ func readCounter(ctx context.Context, w *Witness, publicIP string) (int, error)
return n, nil
}
func dialHost(ctx context.Context, node Node, user string, signer ssh.Signer) (*ssh.Client, error) {
cfg := &ssh.ClientConfig{
User: user,
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // tofu-cluster test hosts
Timeout: 10 * time.Second,
}
d := net.Dialer{Timeout: cfg.Timeout}
conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(node.Addr, "22"))
if err != nil {
return nil, fmt.Errorf("ddil harness: dial host %s: %w", node.Name, err)
}
c, chans, reqs, err := ssh.NewClientConn(conn, node.Addr, cfg)
if err != nil {
_ = conn.Close()
return nil, fmt.Errorf("ddil harness: host ssh handshake %s: %w", node.Name, err)
}
return ssh.NewClient(c, chans, reqs), nil
}
func loadSigner(path string) (ssh.Signer, error) {
b, err := os.ReadFile(path)
if err != nil {