mirror of
https://github.com/mulgadc/spinifex.git
synced 2026-10-15 17:11:30 +00:00
0823bf8c16
* feat: add OVSDB RAFT cluster primitives to setup-ovn.sh Add --db-cluster-local-addr / --db-cluster-remote-addr flags so a management node can run the OVN NB/SB databases as a native OVSDB RAFT cluster instead of a single non-clustered ovn-central. When a local addr is given, write a shared OVN_CTL_OPTS to /etc/default/ovn-central (NB RAFT 6643 / SB RAFT 6644) and restart the per-DB units directly, since the ovn-central aggregator is ExecStart=/bin/true and won't restart its children. Empty remote addr creates the cluster (init node); a set remote addr joins it. NB/SB client set-connection runs only on the create node (RAFT replicates it cluster-wide). The single-node --management path is unchanged when no cluster addrs are supplied, so dev boxes and existing single-node clusters keep working. --ovn-remote already passes through verbatim, so a comma-separated SB endpoint list works for compute nodes; documented in usage. * feat: OVN client multi-endpoint failover for RAFT clusters Accept a comma-separated OVN NB/SB endpoint list so vpcd can target a clustered DB. config.ParseEndpoints splits the list (trimming whitespace, dropping empties); live.go Connect passes one libovsdb WithEndpoint per endpoint, so the client tries them in order, fails over on disconnect, and follows RAFT leader redirects. Single-endpoint strings still yield one endpoint, so single-node dev is unchanged. ovn-sbctl --db= already accepts the raw comma list, so the SB CLI consumers (discoverChassis, gateway_claim) need no change. * feat: write clustered OVN NB/SB endpoints across the RAFT quorum spx admin init/join now emit comma-separated OVN NB/SB endpoint lists for the first three quorum nodes (formation.BuildOVNDBAddrs) instead of loopback or a single leader address, so vpcd fails over across the cluster. firstboot defers OVN bring-up to the provisioning controller when it owns formation (SkipFormation): a standalone single-node .db would otherwise block the controller's clustered create/join. * test: add multinode OVN RAFT cluster + failover e2e; document clustered topology ovn_raft_test.go asserts NB/SB cluster/status reports a 3-server quorum with one leader on every DB node, then stops the NB leader's ovn-central and proves an NB write still lands via a surviving endpoint. Docs: vpc-networking gains the multi-endpoint vpcd config and a RAFT cluster-status debugging section; network-connections lists the 6643/6644 RAFT ports.
107 lines
3.7 KiB
Go
107 lines
3.7 KiB
Go
//go:build e2e
|
|
|
|
package multinode
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
// Top-level Test* wrappers, each delegating to a runX function. Parallel tests
|
|
// share the read-only singleton trio; sequential tests mutate cluster state
|
|
// (stop/start nodes, predastore, EIP pool) and are ordered so the cluster is
|
|
// fully restabilised before GuestSSH probes every trio member.
|
|
|
|
// TestMultinodePreflight runs sequentially first to initialise the package fixture singleton.
|
|
func TestMultinodePreflight(t *testing.T) {
|
|
runPreflight(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// Baseline tests run before needInstanceTrio so the default SG/subnet/route table
|
|
// are in pristine state. Both own dedicated SGs and self-cleaning instances.
|
|
func TestMultinodeDefaultSGReachabilityBaseline(t *testing.T) {
|
|
runMultinodeDefaultSGReachabilityBaseline(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
func TestMultinodeSameSGCrossHostComms(t *testing.T) {
|
|
runMultinodeSameSGCrossHostComms(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
func TestMultinodeClusterHealth(t *testing.T) {
|
|
t.Parallel()
|
|
runClusterHealth(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
func TestMultinodeInstanceDistribution(t *testing.T) {
|
|
t.Parallel()
|
|
runInstanceDistribution(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeJetStreamReplicas is read-only over NATS; parallel so it resumes
|
|
// after the sequential node-failure/recovery tests have restabilised the cluster.
|
|
func TestMultinodeJetStreamReplicas(t *testing.T) {
|
|
t.Parallel()
|
|
runJetStreamReplicas(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeVolumeLifecycle is sequential: touches predastore state.
|
|
func TestMultinodeVolumeLifecycle(t *testing.T) {
|
|
runVolumeLifecycle(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeVolumeDurability is sequential and declared before CrossNodeOps
|
|
// (which destabilises trio[0]'s sshd) so its guest-SSH probes hit settled VMs.
|
|
// Touches predastore state.
|
|
func TestMultinodeVolumeDurability(t *testing.T) {
|
|
runVolumeDurability(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeCrossNodeGateway is sequential: asserts a stable per-gateway instance count
|
|
// that concurrent launches/terminates would break.
|
|
func TestMultinodeCrossNodeGateway(t *testing.T) {
|
|
runCrossNodeGateway(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeCrossNodeOps is sequential: stops/starts trio[0]; GuestSSH is declared
|
|
// after this so it probes the trio only after the cluster has restabilised.
|
|
func TestMultinodeCrossNodeOps(t *testing.T) {
|
|
runCrossNodeOps(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
func TestMultinodeNodeFailure(t *testing.T) {
|
|
runNodeFailure(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
func TestMultinodeNodeRecovery(t *testing.T) {
|
|
runNodeRecovery(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeOVNRaft is sequential: stops ovn-central on the NB leader to
|
|
// prove DB failover, restoring it in cleanup before later tests run.
|
|
func TestMultinodeOVNRaft(t *testing.T) {
|
|
runOVNRaft(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeSpread is sequential after NodeRecovery; owns 10.100.0.0/16 + EIP pool.
|
|
func TestMultinodeSpread(t *testing.T) {
|
|
runSpread(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeGuestSSH is sequential and declared last so it runs after the cluster has
|
|
// fully restabilised. CrossNodeOps stops/starts trio[0] and only waits for "running" — not
|
|
// sshd — so GuestSSH must not probe until the guest has settled.
|
|
func TestMultinodeGuestSSH(t *testing.T) {
|
|
runGuestSSH(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeVPCNetworking owns 10.200.0.0/16 (no EIP use); safe to run in parallel.
|
|
func TestMultinodeVPCNetworking(t *testing.T) {
|
|
t.Parallel()
|
|
runVPCNetworking(t, requireMultiNodeFixture(t))
|
|
}
|
|
|
|
// TestMultinodeIPSec is read-only over SSH; launches no AWS resources.
|
|
func TestMultinodeIPSec(t *testing.T) {
|
|
t.Parallel()
|
|
runIPSec(t, requireMultiNodeFixture(t))
|
|
}
|