mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-23 04:36:40 +00:00
06ca05d611
Step 1.5 of docs/development/improvements/e2e-go-harness.md. Promotes
Cluster/Node/SSH/DaemonClient/LocalStateRevision/ResetAllNodes/Witness
into the shared package so the LB, multinode, and BM scenarios in
steps 4-5 can build on the same primitives instead of duplicating SSH
+ cluster plumbing.
Moves
- ddil/harness/{cluster,ssh,daemon_client,state,reset,witness}.go
-> harness/ (plus the witness testdata embed dir).
- ddil/harness/{fault,netem,profiles,snapshot}.go -> ddil/fault/ as a
new package; fault.PartitionNode/HealNode/KillNATS/StartNATS/
RestartDaemonOnly/ApplyNetem/ClearNetem/LinkProfile*/TakeSnapshot
now take harness.Cluster / harness.Node / harness.SSH explicitly.
- ddil/harness/retry.go::Run retry-with-reset extracted as
harness.RetryWithReset in harness/wait.go; ddil/harness/quarantine.go
keeps the DDIL-only quarantine wrapper that calls into it.
- WaitForMode moved from fault.go to harness/daemon_client.go since
it polls daemon state and has no fault-injection concern.
- The pre-hoist harness/ssh.go (PeerSSH, subprocess ssh) is preserved
as harness/peer_ssh.go so the lb_test suite keeps working unchanged;
the typed x/crypto/ssh client from DDIL now owns harness/ssh.go.
- New harness/util.go exports ShellQuote (was unexported shellQuote in
netem.go) so both ddil/fault and reset.go/witness.go can use it.
Env var rename
- SPINIFEX_NODES / SPINIFEX_SSH_USER / SPINIFEX_SSH_KEY replace
DDIL_NODES / DDIL_SSH_USER / DDIL_SSH_KEY. Old names still honoured
with a stderr deprecation note (envWithLegacy) so the DDIL CI keeps
passing until .github/workflows/e2e-ddil.yml drops the old names.
DDIL_QUARANTINED stays DDIL-specific.
DDIL parity preserved — scenarios still call ddilh.Run for quarantine
semantics; the underlying retry-with-reset is the same code path,
just shared.
Closes mulga-siv-74.
129 lines
3.5 KiB
Go
129 lines
3.5 KiB
Go
//go:build e2e
|
|
|
|
package harness
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"net"
|
|
"os"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
)
|
|
|
|
// SSH is the transport scenarios use to run commands on a remote Node.
|
|
//
|
|
// It is an interface so dry-run mode and unit tests can stub it without
|
|
// opening real network connections. Run returns the command's combined
|
|
// stdout+stderr and a non-nil error if the remote command exited non-zero or
|
|
// the session failed.
|
|
type SSH interface {
|
|
Run(ctx context.Context, node Node, cmd string) ([]byte, error)
|
|
Close() error
|
|
}
|
|
|
|
// sshClient is the production SSH transport, backed by
|
|
// golang.org/x/crypto/ssh. One client is opened per (user, key, node) tuple
|
|
// on first use and cached for the harness lifetime.
|
|
type sshClient struct {
|
|
user string
|
|
signer ssh.Signer
|
|
conns map[string]*ssh.Client
|
|
connectDur time.Duration
|
|
runDur time.Duration
|
|
}
|
|
|
|
var _ SSH = (*sshClient)(nil)
|
|
|
|
// NewSSH returns an SSH transport configured from a Cluster.
|
|
//
|
|
// The private key at cluster.SSHKeyPath must be in a format understood by
|
|
// ssh.ParsePrivateKey (OpenSSH or PEM). Host keys are not verified — the
|
|
// harness runs against short-lived test infrastructure where TOFU would add
|
|
// noise and no security benefit. Do not reuse this transport for production
|
|
// access.
|
|
func NewSSH(cluster *Cluster) (SSH, error) {
|
|
keyBytes, err := os.ReadFile(cluster.SSHKeyPath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("e2e harness: read ssh key %s: %w", cluster.SSHKeyPath, err)
|
|
}
|
|
signer, err := ssh.ParsePrivateKey(keyBytes)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("e2e harness: parse ssh key %s: %w", cluster.SSHKeyPath, err)
|
|
}
|
|
return &sshClient{
|
|
user: cluster.SSHUser,
|
|
signer: signer,
|
|
conns: make(map[string]*ssh.Client),
|
|
connectDur: 10 * time.Second,
|
|
runDur: 2 * time.Minute,
|
|
}, nil
|
|
}
|
|
|
|
func (s *sshClient) dial(node Node) (*ssh.Client, error) {
|
|
if c, ok := s.conns[node.Addr]; ok {
|
|
return c, nil
|
|
}
|
|
cfg := &ssh.ClientConfig{
|
|
User: s.user,
|
|
Auth: []ssh.AuthMethod{ssh.PublicKeys(s.signer)},
|
|
HostKeyCallback: ssh.InsecureIgnoreHostKey(), //nolint:gosec // test harness, not production
|
|
Timeout: s.connectDur,
|
|
}
|
|
c, err := ssh.Dial("tcp", net.JoinHostPort(node.Addr, "22"), cfg)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("e2e harness: ssh dial %s: %w", node.Addr, err)
|
|
}
|
|
s.conns[node.Addr] = c
|
|
return c, nil
|
|
}
|
|
|
|
func (s *sshClient) Run(ctx context.Context, node Node, cmd string) ([]byte, error) {
|
|
client, err := s.dial(node)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
session, err := client.NewSession()
|
|
if err != nil {
|
|
return nil, fmt.Errorf("e2e harness: ssh new session %s: %w", node.Name, err)
|
|
}
|
|
defer func() { _ = session.Close() }()
|
|
|
|
var out bytes.Buffer
|
|
session.Stdout = &out
|
|
session.Stderr = &out
|
|
|
|
// Propagate context cancellation by signalling the session. ssh.Session
|
|
// has no Kill; Signal(KILL) is the portable way to interrupt a running
|
|
// command.
|
|
done := make(chan struct{})
|
|
defer close(done)
|
|
go func() {
|
|
select {
|
|
case <-ctx.Done():
|
|
_ = session.Signal(ssh.SIGKILL)
|
|
_ = session.Close()
|
|
case <-done:
|
|
}
|
|
}()
|
|
|
|
if err := session.Run(cmd); err != nil {
|
|
return out.Bytes(), fmt.Errorf("e2e harness: ssh run on %s (%q): %w\noutput: %s",
|
|
node.Name, cmd, err, out.String())
|
|
}
|
|
return out.Bytes(), nil
|
|
}
|
|
|
|
func (s *sshClient) Close() error {
|
|
var firstErr error
|
|
for addr, c := range s.conns {
|
|
if err := c.Close(); err != nil && firstErr == nil {
|
|
firstErr = fmt.Errorf("e2e harness: ssh close %s: %w", addr, err)
|
|
}
|
|
delete(s.conns, addr)
|
|
}
|
|
return firstErr
|
|
}
|