Files
spinifex/tests/e2e/harness/ssh.go
T
joshsiv-mulga 06ca05d611 refactor(e2e): hoist generic DDIL harness primitives to shared tests/e2e/harness
Step 1.5 of docs/development/improvements/e2e-go-harness.md. Promotes
Cluster/Node/SSH/DaemonClient/LocalStateRevision/ResetAllNodes/Witness
into the shared package so the LB, multinode, and BM scenarios in
steps 4-5 can build on the same primitives instead of duplicating SSH
+ cluster plumbing.

Moves
- ddil/harness/{cluster,ssh,daemon_client,state,reset,witness}.go
  -> harness/ (plus the witness testdata embed dir).
- ddil/harness/{fault,netem,profiles,snapshot}.go -> ddil/fault/ as a
  new package; fault.PartitionNode/HealNode/KillNATS/StartNATS/
  RestartDaemonOnly/ApplyNetem/ClearNetem/LinkProfile*/TakeSnapshot
  now take harness.Cluster / harness.Node / harness.SSH explicitly.
- ddil/harness/retry.go::Run retry-with-reset extracted as
  harness.RetryWithReset in harness/wait.go; ddil/harness/quarantine.go
  keeps the DDIL-only quarantine wrapper that calls into it.
- WaitForMode moved from fault.go to harness/daemon_client.go since
  it polls daemon state and has no fault-injection concern.
- The pre-hoist harness/ssh.go (PeerSSH, subprocess ssh) is preserved
  as harness/peer_ssh.go so the lb_test suite keeps working unchanged;
  the typed x/crypto/ssh client from DDIL now owns harness/ssh.go.
- New harness/util.go exports ShellQuote (was unexported shellQuote in
  netem.go) so both ddil/fault and reset.go/witness.go can use it.

Env var rename
- SPINIFEX_NODES / SPINIFEX_SSH_USER / SPINIFEX_SSH_KEY replace
  DDIL_NODES / DDIL_SSH_USER / DDIL_SSH_KEY. Old names still honoured
  with a stderr deprecation note (envWithLegacy) so the DDIL CI keeps
  passing until .github/workflows/e2e-ddil.yml drops the old names.
  DDIL_QUARANTINED stays DDIL-specific.

DDIL parity preserved — scenarios still call ddilh.Run for quarantine
semantics; the underlying retry-with-reset is the same code path,
just shared.

Closes mulga-siv-74.
2026-05-18 16:10:12 +10:00

129 lines
3.5 KiB
Go

//go:build e2e
package harness
import (
"bytes"
"context"
"fmt"
"net"
"os"
"time"
"golang.org/x/crypto/ssh"
)
// SSH is the transport scenarios use to run commands on a remote Node.
//
// It is an interface so dry-run mode and unit tests can stub it without
// opening real network connections. Run returns the command's combined
// stdout+stderr and a non-nil error if the remote command exited non-zero or
// the session failed.
type SSH interface {
Run(ctx context.Context, node Node, cmd string) ([]byte, error)
Close() error
}
// sshClient is the production SSH transport, backed by
// golang.org/x/crypto/ssh. One client is opened per (user, key, node) tuple
// on first use and cached for the harness lifetime.
type sshClient struct {
user string
signer ssh.Signer
conns map[string]*ssh.Client
connectDur time.Duration
runDur time.Duration
}
var _ SSH = (*sshClient)(nil)
// NewSSH returns an SSH transport configured from a Cluster.
//
// The private key at cluster.SSHKeyPath must be in a format understood by
// ssh.ParsePrivateKey (OpenSSH or PEM). Host keys are not verified — the
// harness runs against short-lived test infrastructure where TOFU would add
// noise and no security benefit. Do not reuse this transport for production
// access.
func NewSSH(cluster *Cluster) (SSH, error) {
keyBytes, err := os.ReadFile(cluster.SSHKeyPath)
if err != nil {
return nil, fmt.Errorf("e2e harness: read ssh key %s: %w", cluster.SSHKeyPath, err)
}
signer, err := ssh.ParsePrivateKey(keyBytes)
if err != nil {
return nil, fmt.Errorf("e2e harness: parse ssh key %s: %w", cluster.SSHKeyPath, err)
}
return &sshClient{
user: cluster.SSHUser,
signer: signer,
conns: make(map[string]*ssh.Client),
connectDur: 10 * time.Second,
runDur: 2 * time.Minute,
}, nil
}
func (s *sshClient) dial(node Node) (*ssh.Client, error) {
if c, ok := s.conns[node.Addr]; ok {
return c, nil
}
cfg := &ssh.ClientConfig{
User: s.user,
Auth: []ssh.AuthMethod{ssh.PublicKeys(s.signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // test harness, not production
Timeout: s.connectDur,
}
c, err := ssh.Dial("tcp", net.JoinHostPort(node.Addr, "22"), cfg)
if err != nil {
return nil, fmt.Errorf("e2e harness: ssh dial %s: %w", node.Addr, err)
}
s.conns[node.Addr] = c
return c, nil
}
func (s *sshClient) Run(ctx context.Context, node Node, cmd string) ([]byte, error) {
client, err := s.dial(node)
if err != nil {
return nil, err
}
session, err := client.NewSession()
if err != nil {
return nil, fmt.Errorf("e2e harness: ssh new session %s: %w", node.Name, err)
}
defer func() { _ = session.Close() }()
var out bytes.Buffer
session.Stdout = &out
session.Stderr = &out
// Propagate context cancellation by signalling the session. ssh.Session
// has no Kill; Signal(KILL) is the portable way to interrupt a running
// command.
done := make(chan struct{})
defer close(done)
go func() {
select {
case <-ctx.Done():
_ = session.Signal(ssh.SIGKILL)
_ = session.Close()
case <-done:
}
}()
if err := session.Run(cmd); err != nil {
return out.Bytes(), fmt.Errorf("e2e harness: ssh run on %s (%q): %w\noutput: %s",
node.Name, cmd, err, out.String())
}
return out.Bytes(), nil
}
func (s *sshClient) Close() error {
var firstErr error
for addr, c := range s.conns {
if err := c.Close(); err != nil && firstErr == nil {
firstErr = fmt.Errorf("e2e harness: ssh close %s: %w", addr, err)
}
delete(s.conns, addr)
}
return firstErr
}