mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-19 10:52:04 +00:00
Julian Sommer
80 lines
3.1 KiB
Go
80 lines
3.1 KiB
Go
//go:build e2e
|
|
|
|
// Package fault provides DDIL-only fault-injection helpers — iptables-based
|
|
// peer partitioning, NATS/daemon lifecycle toggles, netem link-condition
|
|
// shaping, and instance-state snapshots used to assert that nothing rewound
|
|
// across a partition/heal cycle. Generic cluster/SSH/daemon primitives live
|
|
// in github.com/mulgadc/spinifex/tests/e2e/harness.
|
|
package fault
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"github.com/mulgadc/spinifex/tests/e2e/harness"
|
|
)
|
|
|
|
// PartitionNode isolates target from peers with per-peer iptables DROP rules (INPUT+OUTPUT).
|
|
// A post-partition SSH echo confirms the orchestrator's control-plane path was not severed;
|
|
// rules are additive so PartitionNode composes with existing non-DDIL firewall state.
|
|
func PartitionNode(ctx context.Context, ssh harness.SSH, target harness.Node, peers []harness.Node) error {
|
|
if len(peers) == 0 {
|
|
return fmt.Errorf("ddil fault: PartitionNode %s: no peers supplied", target.Name)
|
|
}
|
|
|
|
var lines []string
|
|
for _, p := range peers {
|
|
lines = append(lines,
|
|
fmt.Sprintf("sudo iptables -I INPUT -s %s -j DROP", harness.ShellQuote(p.Addr)),
|
|
fmt.Sprintf("sudo iptables -I OUTPUT -d %s -j DROP", harness.ShellQuote(p.Addr)),
|
|
)
|
|
}
|
|
cmd := strings.Join(lines, " && ")
|
|
if _, err := ssh.Run(ctx, target, cmd); err != nil {
|
|
return fmt.Errorf("ddil fault: partition %s: %w", target.Name, err)
|
|
}
|
|
|
|
// Confirm orchestrator SSH still works; if not, the orchestrator shares a peer IP.
|
|
if _, err := ssh.Run(ctx, target, "echo ok"); err != nil {
|
|
return fmt.Errorf("ddil fault: partition %s severed orchestrator SSH "+
|
|
"(orchestrator may share peer IPs): %w", target.Name, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// HealNode flushes and deletes all iptables rules on target, reversing a
|
|
// prior PartitionNode. Idempotent: safe on a node with no rules installed.
|
|
func HealNode(ctx context.Context, ssh harness.SSH, target harness.Node) error {
|
|
if _, err := ssh.Run(ctx, target, "sudo iptables -F && sudo iptables -X"); err != nil {
|
|
return fmt.Errorf("ddil fault: heal %s: %w", target.Name, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// KillNATS stops spinifex-nats without touching spinifex-daemon (unlike systemctl stop
|
|
// spinifex.target which brings both down).
|
|
func KillNATS(ctx context.Context, ssh harness.SSH, node harness.Node) error {
|
|
if _, err := ssh.Run(ctx, node, "sudo systemctl stop spinifex-nats"); err != nil {
|
|
return fmt.Errorf("ddil fault: kill nats on %s: %w", node.Name, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// StartNATS starts spinifex-nats on node. Paired with KillNATS.
|
|
func StartNATS(ctx context.Context, ssh harness.SSH, node harness.Node) error {
|
|
if _, err := ssh.Run(ctx, node, "sudo systemctl start spinifex-nats"); err != nil {
|
|
return fmt.Errorf("ddil fault: start nats on %s: %w", node.Name, err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// RestartDaemonOnly restarts spinifex-daemon without touching spinifex-nats,
|
|
// exercising the daemon-without-NATS startup path.
|
|
func RestartDaemonOnly(ctx context.Context, ssh harness.SSH, node harness.Node) error {
|
|
if _, err := ssh.Run(ctx, node, "sudo systemctl restart spinifex-daemon"); err != nil {
|
|
return fmt.Errorf("ddil fault: restart daemon on %s: %w", node.Name, err)
|
|
}
|
|
return nil
|
|
}
|