feat: improved certificate authority with auto-discovery, CA download, and cloud-init injection

Auto-discover all node IPs (WAN, mgmt, VPC) and hostname for server cert SANs — fixes TLS failures when accessing Spinifex via non-localhost IPs. Adds CA download endpoint (/api/ca.pem) with install instructions on the login page. Injects CA cert into guest VM cloud-init so EC2 instances trust Spinifex services without --insecure. Adds 'spx admin cert renew' command for cert regeneration after IP changes. Includes certificate validation E2E test suite (run-cert-e2e.sh) with system trust store, SAN, and in-guest TLS verification across single, pseudo multi-node, and real multi-node modes. Closes: mulga-777, mulga-778, mulga-779, mulga-780, mulga-781 Plan: docs/development/feature/cert-manager.md
2026-10-19 19:01:42 +00:00 · 2026-03-28 14:29:07 +10:00
parent 15f88caee7
commit 5426dc1bb0
19 changed files with 1022 additions and 88 deletions
@@ -72,7 +72,7 @@ jobs:
          SSH_KEY=$(echo "$INVENTORY" | jq -r '.ssh_key_path')
          SSH_KEY="${SSH_KEY/#\~/$HOME}"
          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${WAN_IP} \
-            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh'
+            'cd ~/Development/mulga/spinifex && ./tests/e2e/run-e2e.sh && ./tests/e2e/run-cert-e2e.sh'

      - name: Teardown
        if: always()
@@ -207,7 +207,7 @@ jobs:
          done

          ssh -o StrictHostKeyChecking=no -i "$SSH_KEY" tf-user@${NODE1_IP} \
-            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS}"
+            "cd ~/Development/mulga/spinifex && ./tests/e2e/run-multinode-e2e.sh ${NODE_IPS} && ./tests/e2e/run-cert-e2e.sh --multinode ${NODE_IPS}"

      - name: Teardown
        if: always()
@@ -158,6 +158,22 @@ var accountListCmd = &cobra.Command{
 	Run:   runAccountList,
 }

+var certCmd = &cobra.Command{
+	Use:   "cert",
+	Short: "Manage TLS certificates",
+	Long:  `Manage TLS certificates for the Spinifex platform.`,
+}
+
+var certRenewCmd = &cobra.Command{
+	Use:   "renew",
+	Short: "Regenerate the server certificate with current IPs",
+	Long: `Regenerate the server certificate signed by the existing CA.
+The new certificate will include all current network interface IPs
+and the machine hostname in its SANs. Use this after adding a new
+network interface or changing IP addresses.`,
+	Run: runCertRenew,
+}
+
 /*
 CLI ideas

@@ -195,6 +211,11 @@ func init() {
 	adminCmd.AddCommand(accountCmd)
 	accountCmd.AddCommand(accountCreateCmd)
 	accountCmd.AddCommand(accountListCmd)
+
+	adminCmd.AddCommand(certCmd)
+	certCmd.AddCommand(certRenewCmd)
+	certRenewCmd.Flags().StringSlice("extra-ip", nil, "Additional IP addresses to include in SANs")
+	certRenewCmd.Flags().StringSlice("extra-dns", nil, "Additional DNS names to include in SANs")
 	accountCreateCmd.Flags().String("name", "", "Account name (required)")
 	accountCreateCmd.MarkFlagRequired("name")

@@ -1768,6 +1789,33 @@ func runAccountList(cmd *cobra.Command, args []string) {
 	}
 }

+func runCertRenew(cmd *cobra.Command, _ []string) {
+	configDir, _ := cmd.Root().Flags().GetString("config-dir")
+
+	// Verify CA files exist.
+	caCertPath := filepath.Join(configDir, "ca.pem")
+	caKeyPath := filepath.Join(configDir, "ca.key")
+	if !admin.FileExists(caCertPath) || !admin.FileExists(caKeyPath) {
+		fmt.Fprintf(os.Stderr, "Error: CA files not found in %s\nRun 'spx admin init' first.\n", configDir)
+		os.Exit(1)
+	}
+
+	extraIPs, _ := cmd.Flags().GetStringSlice("extra-ip")
+	extraDNS, _ := cmd.Flags().GetStringSlice("extra-dns")
+
+	serverCertPath := filepath.Join(configDir, "server.pem")
+	serverKeyPath := filepath.Join(configDir, "server.key")
+
+	if err := admin.GenerateSignedCertWithDNS(serverCertPath, serverKeyPath, caCertPath, caKeyPath, extraIPs, extraDNS); err != nil {
+		fmt.Fprintf(os.Stderr, "Error regenerating server certificate: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Println("✅ Server certificate regenerated with current IPs and hostname")
+	fmt.Printf("   Certificate: %s\n", serverCertPath)
+	fmt.Println("\n⚠️  Restart awsgw and daemon services to pick up the new certificate.")
+}
+
 // writeBootstrapResult holds the admin credentials so callers can
 // write them to ~/.aws/credentials instead of the system credentials.
 type writeBootstrapResult struct {
@@ -256,4 +256,46 @@ fi

 aws ec2 describe-images

-echo "Reset complete, fresh AMI imported, proceed to creating instances"
+# --- Launch a smoke-test instance ---
+echo "Launching smoke-test instance..."
+
+# Pick instance type based on CPU vendor
+if grep -q 'AuthenticAMD' /proc/cpuinfo; then
+    INSTANCE_TYPE="t3a.small"
+else
+    INSTANCE_TYPE="t3.small"
+fi
+
+# Get the AMI we just imported
+AMI_ID=$(aws ec2 describe-images --query "Images[0].ImageId" --output text)
+if [ -z "$AMI_ID" ] || [ "$AMI_ID" = "None" ]; then
+    echo "❌ No AMI found, skipping instance launch"
+    exit 0
+fi
+
+# Find a public subnet (first available)
+SUBNET_ID=$(aws ec2 describe-subnets \
+    --filters "Name=map-public-ip-on-launch,Values=true" \
+    --query "Subnets[0].SubnetId" --output text 2>/dev/null)
+# Fallback: just grab the first subnet if no public-tagged one exists
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    SUBNET_ID=$(aws ec2 describe-subnets --query "Subnets[0].SubnetId" --output text)
+fi
+if [ -z "$SUBNET_ID" ] || [ "$SUBNET_ID" = "None" ]; then
+    echo "❌ No subnet found, skipping instance launch"
+    exit 0
+fi
+
+echo "  AMI: $AMI_ID"
+echo "  Instance type: $INSTANCE_TYPE"
+echo "  Subnet: $SUBNET_ID"
+
+INSTANCE_ID=$(aws ec2 run-instances \
+    --image-id "$AMI_ID" \
+    --instance-type "$INSTANCE_TYPE" \
+    --key-name spinifex-key \
+    --subnet-id "$SUBNET_ID" \
+    --count 1 \
+    --query 'Instances[0].InstanceId' --output text)
+
+echo "✅ Reset complete — instance $INSTANCE_ID launched ($INSTANCE_TYPE)"
@@ -427,53 +427,109 @@ func GenerateCACert(caCertPath, caKeyPath string) error {
 	return nil
 }

+// DiscoverLocalIPs enumerates all non-loopback network interface addresses on
+// this machine and returns them as strings. Link-local addresses are excluded.
+func DiscoverLocalIPs() []string {
+	ifaces, err := net.Interfaces()
+	if err != nil {
+		slog.Warn("failed to enumerate network interfaces", "error", err)
+		return nil
+	}
+
+	seen := make(map[string]struct{})
+	var ips []string
+
+	for _, iface := range ifaces {
+		// Skip loopback and down interfaces.
+		if iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
+			continue
+		}
+		addrs, err := iface.Addrs()
+		if err != nil {
+			slog.Debug("failed to get addresses for interface", "iface", iface.Name, "error", err)
+			continue
+		}
+		for _, addr := range addrs {
+			var ip net.IP
+			switch v := addr.(type) {
+			case *net.IPNet:
+				ip = v.IP
+			case *net.IPAddr:
+				ip = v.IP
+			}
+			if ip == nil || ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() {
+				continue
+			}
+			s := ip.String()
+			if _, ok := seen[s]; !ok {
+				seen[s] = struct{}{}
+				ips = append(ips, s)
+			}
+		}
+	}
+	return ips
+}
+
+// DiscoverHostname returns the system hostname suitable for use as a DNS SAN.
+// Returns an empty string if the hostname cannot be determined or is "localhost".
+func DiscoverHostname() string {
+	hostname, err := os.Hostname()
+	if err != nil {
+		slog.Warn("failed to determine system hostname", "error", err)
+		return ""
+	}
+	if hostname == "" || hostname == "localhost" {
+		return ""
+	}
+	return hostname
+}
+
 // GenerateSignedCert generates a server certificate signed by the CA.
 // extraIPs are additional IP addresses to include in the certificate's SANs.
+// All non-loopback interface IPs on the local machine are automatically included.
 func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIPs ...string) error {
-	// Load CA certificate
+	return GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, nil)
+}
+
+// GenerateSignedCertWithDNS generates a server certificate signed by the CA.
+// All non-loopback interface IPs and the machine hostname are automatically
+// included. extraIPs and extraDNS allow adding additional SANs.
+func GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath string, extraIPs, extraDNS []string) error {
 	caCertPEM, err := os.ReadFile(caCertPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA cert: %w", err)
 	}
-
 	caCertBlock, _ := pem.Decode(caCertPEM)
 	if caCertBlock == nil {
 		return fmt.Errorf("failed to decode CA cert PEM")
 	}
-
 	caCert, err := x509.ParseCertificate(caCertBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA cert: %w", err)
 	}

-	// Load CA private key
 	caKeyPEM, err := os.ReadFile(caKeyPath)
 	if err != nil {
 		return fmt.Errorf("failed to read CA key: %w", err)
 	}
-
 	caKeyBlock, _ := pem.Decode(caKeyPEM)
 	if caKeyBlock == nil {
 		return fmt.Errorf("failed to decode CA key PEM")
 	}
-
 	caPrivateKey, err := x509.ParsePKCS8PrivateKey(caKeyBlock.Bytes)
 	if err != nil {
 		return fmt.Errorf("failed to parse CA private key: %w", err)
 	}
-
 	caRSAKey, ok := caPrivateKey.(*rsa.PrivateKey)
 	if !ok {
 		return fmt.Errorf("CA key is not RSA")
 	}

-	// Generate server private key
 	serverPrivateKey, err := rsa.GenerateKey(rand.Reader, 4096)
 	if err != nil {
 		return fmt.Errorf("failed to generate server private key: %w", err)
 	}

-	// Create server certificate template
 	notBefore := time.Now()
 	notAfter := notBefore.Add(365 * 24 * time.Hour) // 1 year for server certs

@@ -482,20 +538,51 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		return fmt.Errorf("failed to generate serial number: %w", err)
 	}

-	// Build IP list: localhost IPs + any extra IPs (e.g., bind IP for multi-node)
+	// Build IP list: localhost IPs + auto-discovered interface IPs + explicit extras.
+	seen := map[string]struct{}{"127.0.0.1": {}, "::1": {}}
 	ipAddresses := []net.IP{net.ParseIP("127.0.0.1"), net.ParseIP("::1")}
+	addIP := func(s string) {
+		if s == "" || s == "0.0.0.0" {
+			return
+		}
+		if _, ok := seen[s]; ok {
+			return
+		}
+		if parsed := net.ParseIP(s); parsed != nil {
+			seen[s] = struct{}{}
+			ipAddresses = append(ipAddresses, parsed)
+		}
+	}
+	for _, ip := range DiscoverLocalIPs() {
+		addIP(ip)
+	}
 	for _, ip := range extraIPs {
-		if ip != "" && ip != "127.0.0.1" && ip != "::1" && ip != "0.0.0.0" {
-			if parsed := net.ParseIP(ip); parsed != nil {
-				ipAddresses = append(ipAddresses, parsed)
-			}
+		addIP(ip)
+	}
+
+	// Build DNS names: always include localhost, plus hostname if available.
+	dnsSeen := map[string]struct{}{"localhost": {}}
+	dnsNames := []string{"localhost"}
+	if hostname := DiscoverHostname(); hostname != "" {
+		if _, ok := dnsSeen[hostname]; !ok {
+			dnsSeen[hostname] = struct{}{}
+			dnsNames = append(dnsNames, hostname)
+		}
+	}
+	for _, dns := range extraDNS {
+		if dns == "" {
+			continue
+		}
+		if _, ok := dnsSeen[dns]; !ok {
+			dnsSeen[dns] = struct{}{}
+			dnsNames = append(dnsNames, dns)
 		}
 	}

 	serverTemplate := x509.Certificate{
 		SerialNumber: serialNumber,
 		Subject: pkix.Name{
-			CommonName:   "localhost",
+			CommonName:   "Spinifex Server",
 			Organization: []string{"Spinifex Platform"},
 		},
 		NotBefore:             notBefore,
@@ -503,39 +590,33 @@ func GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath string, extraIP
 		KeyUsage:              x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature,
 		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
 		BasicConstraintsValid: true,
-		DNSNames:              []string{"localhost"},
+		DNSNames:              dnsNames,
 		IPAddresses:           ipAddresses,
 	}

-	// Sign the server certificate with the CA
 	serverDerBytes, err := x509.CreateCertificate(rand.Reader, &serverTemplate, caCert, &serverPrivateKey.PublicKey, caRSAKey)
 	if err != nil {
 		return fmt.Errorf("failed to create server certificate: %w", err)
 	}

-	// Write server certificate to file
 	certOut, err := os.Create(certPath)
 	if err != nil {
 		return fmt.Errorf("failed to create cert file: %w", err)
 	}
 	defer certOut.Close()
-
 	if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: serverDerBytes}); err != nil {
 		return fmt.Errorf("failed to write cert: %w", err)
 	}

-	// Write server private key to file
 	keyOut, err := os.OpenFile(keyPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
 	if err != nil {
 		return fmt.Errorf("failed to create key file: %w", err)
 	}
 	defer keyOut.Close()
-
 	privBytes, err := x509.MarshalPKCS8PrivateKey(serverPrivateKey)
 	if err != nil {
 		return fmt.Errorf("failed to marshal private key: %w", err)
 	}
-
 	if err := pem.Encode(keyOut, &pem.Block{Type: "PRIVATE KEY", Bytes: privBytes}); err != nil {
 		return fmt.Errorf("failed to write key: %w", err)
 	}
@@ -335,7 +335,7 @@ func TestGenerateSignedCert(t *testing.T) {
 		cert, err := x509.ParseCertificate(block.Bytes)
 		require.NoError(t, err)
 		assert.False(t, cert.IsCA)
-		assert.Equal(t, "localhost", cert.Subject.CommonName)
+		assert.Equal(t, "Spinifex Server", cert.Subject.CommonName)
 		assert.Contains(t, cert.DNSNames, "localhost")

 		hasLoopback := false
@@ -346,6 +346,9 @@ func TestGenerateSignedCert(t *testing.T) {
 		}
 		assert.True(t, hasLoopback)

+		// Auto-discovered IPs should be present (at least loopback + any interface IPs).
+		assert.GreaterOrEqual(t, len(cert.IPAddresses), 2)
+
 		// Verify against CA
 		caCertPEM, _ := os.ReadFile(caCertPath)
 		caBlock, _ := pem.Decode(caCertPEM)
@@ -392,7 +395,19 @@ func TestGenerateSignedCert(t *testing.T) {
 		block, _ := pem.Decode(certPEM)
 		cert, _ := x509.ParseCertificate(block.Bytes)

-		assert.Len(t, cert.IPAddresses, 2)
+		// Should have loopback (127.0.0.1, ::1) + auto-discovered interface IPs.
+		// Duplicates (127.0.0.1, ::1) and specials (0.0.0.0, "") must not add extras.
+		// Count unique IPs from a clean run without extras.
+		baseDir := t.TempDir()
+		baseCert := filepath.Join(baseDir, "server.pem")
+		baseKey := filepath.Join(baseDir, "server.key")
+		require.NoError(t, GenerateSignedCert(baseCert, baseKey, caCertPath, caKeyPath))
+		basePEM, _ := os.ReadFile(baseCert)
+		baseBlock, _ := pem.Decode(basePEM)
+		baseParsed, _ := x509.ParseCertificate(baseBlock.Bytes)
+
+		assert.Equal(t, len(baseParsed.IPAddresses), len(cert.IPAddresses),
+			"passing duplicate/special IPs should not add extra entries")
 	})

 	t.Run("InvalidCACert", func(t *testing.T) {
@@ -406,6 +421,178 @@ func TestGenerateSignedCert(t *testing.T) {
 	})
 }

+func TestDiscoverLocalIPs(t *testing.T) {
+	t.Parallel()
+	ips := DiscoverLocalIPs()
+	// Should find at least one non-loopback IP on any machine with a network interface.
+	assert.NotEmpty(t, ips, "expected at least one non-loopback IP")
+
+	for _, ip := range ips {
+		parsed := net.ParseIP(ip)
+		require.NotNil(t, parsed, "returned IP should be parseable: %s", ip)
+		assert.False(t, parsed.IsLoopback(), "should not include loopback: %s", ip)
+		assert.False(t, parsed.IsLinkLocalUnicast(), "should not include link-local: %s", ip)
+	}
+
+	// No duplicates.
+	seen := make(map[string]struct{})
+	for _, ip := range ips {
+		_, dup := seen[ip]
+		assert.False(t, dup, "duplicate IP: %s", ip)
+		seen[ip] = struct{}{}
+	}
+}
+
+func TestDiscoverHostname(t *testing.T) {
+	t.Parallel()
+	hostname := DiscoverHostname()
+	// We can't assert a specific value, but if non-empty it shouldn't be "localhost".
+	if hostname != "" {
+		assert.NotEqual(t, "localhost", hostname)
+	}
+}
+
+func TestGenerateSignedCertWithDNS_DedupDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Pass "localhost" as extra DNS — should not duplicate the built-in entry.
+	// Also pass empty strings and duplicates — should be ignored.
+	extraDNS := []string{"localhost", "", "example.local", "example.local", ""}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	// Count occurrences of "localhost" — must be exactly 1.
+	localhostCount := 0
+	exampleCount := 0
+	for _, dns := range cert.DNSNames {
+		if dns == "localhost" {
+			localhostCount++
+		}
+		if dns == "example.local" {
+			exampleCount++
+		}
+	}
+	assert.Equal(t, 1, localhostCount, "localhost should appear exactly once")
+	assert.Equal(t, 1, exampleCount, "example.local should appear exactly once")
+	assert.Contains(t, cert.DNSNames, "example.local")
+}
+
+func TestGenerateSignedCertWithDNS_NilSlices(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	// Both slices nil — should still produce a valid cert with defaults.
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, nil, nil))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.GreaterOrEqual(t, len(cert.IPAddresses), 2, "should have at least loopback IPs")
+}
+
+func TestGenerateSignedCertWithDNS_InvalidCA(t *testing.T) {
+	t.Parallel()
+	dir := t.TempDir()
+
+	// Missing CA cert file.
+	err := GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "missing-ca.pem"), filepath.Join(dir, "missing-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), "failed to read CA cert")
+
+	// Corrupt CA cert.
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.pem"), []byte("not-pem"), 0600))
+	require.NoError(t, os.WriteFile(filepath.Join(dir, "bad-ca.key"), []byte("not-pem"), 0600))
+	err = GenerateSignedCertWithDNS(
+		filepath.Join(dir, "s.pem"), filepath.Join(dir, "s.key"),
+		filepath.Join(dir, "bad-ca.pem"), filepath.Join(dir, "bad-ca.key"),
+		nil, nil,
+	)
+	assert.Error(t, err)
+}
+
+func TestGenerateSignedCertWithDNS_ExtraDNS(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+
+	extraIPs := []string{"10.0.0.42"}
+	extraDNS := []string{"spinifex.local", "node1.spinifex.local"}
+	require.NoError(t, GenerateSignedCertWithDNS(certPath, keyPath, caCertPath, caKeyPath, extraIPs, extraDNS))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	assert.Contains(t, cert.DNSNames, "spinifex.local")
+	assert.Contains(t, cert.DNSNames, "node1.spinifex.local")
+
+	hasExtraIP := false
+	for _, ip := range cert.IPAddresses {
+		if ip.Equal(net.ParseIP("10.0.0.42")) {
+			hasExtraIP = true
+		}
+	}
+	assert.True(t, hasExtraIP, "cert should contain extra IP 10.0.0.42")
+
+	info, _ := os.Stat(keyPath)
+	assert.Equal(t, os.FileMode(0600), info.Mode().Perm())
+}
+
+func TestGenerateSignedCert_IncludesHostname(t *testing.T) {
+	t.Parallel()
+	caDir := t.TempDir()
+	caCertPath := filepath.Join(caDir, "ca.pem")
+	caKeyPath := filepath.Join(caDir, "ca.key")
+	require.NoError(t, GenerateCACert(caCertPath, caKeyPath))
+
+	dir := t.TempDir()
+	certPath := filepath.Join(dir, "server.pem")
+	keyPath := filepath.Join(dir, "server.key")
+	require.NoError(t, GenerateSignedCert(certPath, keyPath, caCertPath, caKeyPath))
+
+	certPEM, _ := os.ReadFile(certPath)
+	block, _ := pem.Decode(certPEM)
+	cert, _ := x509.ParseCertificate(block.Bytes)
+
+	assert.Contains(t, cert.DNSNames, "localhost")
+	hostname := DiscoverHostname()
+	if hostname != "" {
+		assert.Contains(t, cert.DNSNames, hostname,
+			"cert DNS SANs should include the machine hostname")
+	}
+}
+
 func TestGenerateSelfSignedCert_CreatesValidCert(t *testing.T) {
 	t.Parallel()
 	dir := t.TempDir()
@@ -8,6 +8,7 @@ import (
 	"io"
 	"log/slog"
 	"os"
+	"path/filepath"
 	"strings"
 	"text/template"
 	"time"
@@ -51,6 +52,13 @@ ssh_pwauth: true
 hostname: {{.Hostname}}
 manage_etc_hosts: true

+{{if .CACertPEM}}
+ca_certs:
+  trusted:
+    - |
+{{.CACertPEM}}
+{{end}}
+
 {{if .UserDataCloudConfig}}
 {{.UserDataCloudConfig}}
 {{end}}
@@ -122,6 +130,7 @@ type CloudInitData struct {
 	Hostname            string
 	UserDataCloudConfig string
 	UserDataScript      string
+	CACertPEM           string
 }

 type CloudInitMetaData struct {
@@ -633,10 +642,34 @@ func (s *InstanceServiceImpl) createCloudInitISO(input *ec2.RunInstancesInput, i
 		}
 	}

+	// Read CA certificate for injection into guest cloud-init.
+	// Derive the config directory: BaseDir (e.g. ~/spinifex/spinifex/) sits one
+	// level below the data root; the CA cert is at <data-root>/config/ca.pem.
+	var caCertPEM string
+	dataRoot := filepath.Dir(strings.TrimSuffix(s.config.BaseDir, "/"))
+	caCertPath := filepath.Join(dataRoot, "config", "ca.pem")
+	if caBytes, err := os.ReadFile(caCertPath); err == nil {
+		// Indent each line by 6 spaces for YAML block scalar in ca_certs.trusted.
+		var indented strings.Builder
+		for line := range strings.SplitSeq(string(caBytes), "\n") {
+			if line != "" {
+				indented.WriteString("      ")
+				indented.WriteString(line)
+				indented.WriteByte('\n')
+			}
+		}
+		caCertPEM = indented.String()
+	} else if os.IsNotExist(err) {
+		slog.Warn("CA cert not found, guest VMs will not trust Spinifex services", "path", caCertPath)
+	} else {
+		slog.Error("failed to read CA cert for guest cloud-init injection", "path", caCertPath, "error", err)
+	}
+
 	userData := CloudInitData{
-		Username: "ec2-user",
-		SSHKey:   string(sshKey),
-		Hostname: hostname,
+		Username:  "ec2-user",
+		SSHKey:    string(sshKey),
+		Hostname:  hostname,
+		CACertPEM: caCertPEM,
 	}

 	// Decode and classify user-data from RunInstances (base64-encoded).
@@ -172,9 +172,10 @@ func TestRunInstance_UniqueIDs(t *testing.T) {

 func TestCloudInitTemplateRendering(t *testing.T) {
 	tests := []struct {
-		name     string
-		data     CloudInitData
-		contains []string
+		name        string
+		data        CloudInitData
+		contains    []string
+		notContains []string
 	}{
 		{
 			name: "Basic SSH key and hostname",
@@ -218,6 +219,40 @@ func TestCloudInitTemplateRendering(t *testing.T) {
 				"echo hello",
 			},
 		},
+		{
+			name: "With CA certificate PEM",
+			data: CloudInitData{
+				Username: "ec2-user",
+				SSHKey:   "ssh-rsa AAAA...",
+				Hostname: "spinifex-vm-ca-test",
+				CACertPEM: "      -----BEGIN CERTIFICATE-----\n" +
+					"      MIIFazCCA1OgAwIBAgIUAbcdefg1234567890ABCDEFG=\n" +
+					"      -----END CERTIFICATE-----\n",
+			},
+			contains: []string{
+				"ca_certs:",
+				"trusted:",
+				"-----BEGIN CERTIFICATE-----",
+				"-----END CERTIFICATE-----",
+				"MIIFazCCA1OgAwIBAgIUAbcdefg1234567890ABCDEFG=",
+			},
+		},
+		{
+			name: "Without CA certificate PEM",
+			data: CloudInitData{
+				Username: "ec2-user",
+				SSHKey:   "ssh-rsa AAAA...",
+				Hostname: "spinifex-vm-no-ca",
+			},
+			contains: []string{
+				"#cloud-config",
+				"spinifex-vm-no-ca",
+			},
+			notContains: []string{
+				"ca_certs:",
+				"trusted:",
+			},
+		},
 	}

 	for _, tt := range tests {
@@ -231,6 +266,9 @@ func TestCloudInitTemplateRendering(t *testing.T) {
 			for _, s := range tt.contains {
 				assert.Contains(t, rendered, s)
 			}
+			for _, s := range tt.notContains {
+				assert.NotContains(t, rendered, s)
+			}
 		})
 	}
 }
@@ -0,0 +1 @@
+import{a as e,r as t}from"./compiler-runtime.js";import{u as n}from"./utils.js";import{R as r,i,o as a}from"./button.js";var o=e(t()),s=[];function c(e){o.useEffect(e,s)}var l=null;globalThis.requestAnimationFrame;var u=new class{callbacks=[];callbacksCount=0;nextId=1;startId=1;isScheduled=!1;tick=e=>{this.isScheduled=!1;let t=this.callbacks,n=this.callbacksCount;if(this.callbacks=[],this.callbacksCount=0,this.startId=this.nextId,n>0)for(let n=0;n<t.length;n+=1)t[n]?.(e)};request(e){let t=this.nextId;return this.nextId+=1,this.callbacks.push(e),this.callbacksCount+=1,this.isScheduled||=(requestAnimationFrame(this.tick),!0),t}cancel(e){let t=e-this.startId;t<0||t>=this.callbacks.length||(this.callbacks[t]=null,--this.callbacksCount)}},d=class e{static create(){return new e}static request(e){return u.request(e)}static cancel(e){return u.cancel(e)}currentId=l;request(e){this.cancel(),this.currentId=u.request(()=>{this.currentId=l,e()})}cancel=()=>{this.currentId!==l&&(u.cancel(this.currentId),this.currentId=l)};disposeEffect=()=>this.cancel};function f(){let e=r(d.create).current;return c(e.disposeEffect),e}function p(e){return e==null?e:`current`in e?e.current:e}var m=function(e){return e.startingStyle=`data-starting-style`,e.endingStyle=`data-ending-style`,e}({}),h={[m.startingStyle]:``},g={[m.endingStyle]:``},_={transitionStatus(e){return e===`starting`?h:e===`ending`?g:null}},v=e(n());function y(e,t=!1,n=!0){let r=f();return a((i,a=null)=>{r.cancel();function o(){v.flushSync(i)}let s=p(e);if(s==null)return;let c=s;if(typeof c.getAnimations!=`function`||globalThis.BASE_UI_ANIMATIONS_DISABLED)i();else{function e(){let e=m.startingStyle;if(!c.hasAttribute(e)){r.request(i);return}let t=new MutationObserver(()=>{c.hasAttribute(e)||(t.disconnect(),i())});t.observe(c,{attributes:!0,attributeFilter:[e]}),a?.addEventListener(`abort`,()=>t.disconnect(),{once:!0})}function i(){Promise.all(c.getAnimations().map(e=>e.finished)).then(()=>{a?.aborted||o()}).catch(()=>{let e=c.getAnimations();if(n){if(a?.aborted)return;o()}else e.length>0&&e.some(e=>e.pending||e.playState!==`finished`)&&i()})}if(t){e();return}r.request(i)}})}function b(e){let{enabled:t=!0,open:n,ref:r,onComplete:i}=e,s=a(i),c=y(r,n,!1);o.useEffect(()=>{if(!t)return;let e=new AbortController;return c(s,e.signal),()=>{e.abort()}},[t,n,s,c])}function x(e,t=!1,n=!1){let[r,a]=o.useState(e&&t?`idle`:void 0),[s,c]=o.useState(e);return e&&!s&&(c(!0),a(`starting`)),!e&&s&&r!==`ending`&&!n&&a(`ending`),!e&&!s&&r===`ending`&&a(void 0),i(()=>{if(!e&&s&&r!==`ending`&&n){let e=d.request(()=>{a(`ending`)});return()=>{d.cancel(e)}}},[e,s,r,n]),i(()=>{if(!e||t)return;let n=d.request(()=>{a(void 0)});return()=>{d.cancel(n)}},[t,e]),i(()=>{if(!e||!t)return;e&&s&&r!==`idle`&&a(`starting`);let n=d.request(()=>{a(`idle`)});return()=>{d.cancel(n)}},[t,e,s,a,r]),o.useMemo(()=>({mounted:s,setMounted:c,transitionStatus:r}),[s,r])}export{_ as a,f as c,m as i,c as l,b as n,p as o,y as r,d as s,x as t};
@@ -18,6 +18,7 @@
    <link rel="modulepreload" crossorigin href="/assets/utils.js">
    <link rel="modulepreload" crossorigin href="/assets/button.js">
    <link rel="modulepreload" crossorigin href="/assets/createBaseUIEventDetails.js">
+    <link rel="modulepreload" crossorigin href="/assets/useTransitionStatus.js">
    <link rel="modulepreload" crossorigin href="/assets/useStore.js">
    <link rel="modulepreload" crossorigin href="/assets/useOpenInteractionType.js">
    <link rel="modulepreload" crossorigin href="/assets/separator.js">
@@ -6,6 +6,11 @@ import { useForm } from "react-hook-form"

 import { Button } from "@/components/ui/button"
 import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
+import {
+  Collapsible,
+  CollapsibleContent,
+  CollapsibleTrigger,
+} from "@/components/ui/collapsible"
 import {
  Field,
  FieldError,
@@ -61,52 +66,118 @@ function LoginPage() {

  return (
    <div className="flex flex-1 items-center justify-center">
-      <Card className="w-full max-w-sm">
-        <CardHeader>
-          <CardTitle>AWS Credentials</CardTitle>
-        </CardHeader>
-        <CardContent>
-          {authError && (
-            <p className="mb-4 rounded-md bg-destructive/10 p-3 text-sm text-destructive">
-              {authError}
-            </p>
-          )}
-          <form onSubmit={handleSubmit(onSubmit)}>
-            <FieldGroup>
-              <Field>
-                <FieldTitle>
-                  <label htmlFor="accessKeyId">Access Key ID</label>
-                </FieldTitle>
-                <Input
-                  aria-invalid={!!errors.accessKeyId}
-                  autoComplete="username"
-                  id="accessKeyId"
-                  placeholder="AKIAIOSFODNN7EXAMPLE"
-                  {...register("accessKeyId")}
-                />
-                <FieldError errors={[errors.accessKeyId]} />
-              </Field>
-              <Field>
-                <FieldTitle>
-                  <label htmlFor="secretAccessKey">Secret Access Key</label>
-                </FieldTitle>
-                <Input
-                  aria-invalid={!!errors.secretAccessKey}
-                  autoComplete="current-password"
-                  id="secretAccessKey"
-                  placeholder="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
-                  type="password"
-                  {...register("secretAccessKey")}
-                />
-                <FieldError errors={[errors.secretAccessKey]} />
-              </Field>
-              <Button className="w-full" disabled={isSubmitting} type="submit">
-                {isSubmitting ? "Signing in..." : "Sign In"}
-              </Button>
-            </FieldGroup>
-          </form>
-        </CardContent>
-      </Card>
+      <div className="flex w-full max-w-sm flex-col gap-4">
+        <Card>
+          <CardHeader>
+            <CardTitle>AWS Credentials</CardTitle>
+          </CardHeader>
+          <CardContent>
+            {authError && (
+              <p className="mb-4 rounded-md bg-destructive/10 p-3 text-sm text-destructive">
+                {authError}
+              </p>
+            )}
+            <form onSubmit={handleSubmit(onSubmit)}>
+              <FieldGroup>
+                <Field>
+                  <FieldTitle>
+                    <label htmlFor="accessKeyId">Access Key ID</label>
+                  </FieldTitle>
+                  <Input
+                    aria-invalid={!!errors.accessKeyId}
+                    autoComplete="username"
+                    id="accessKeyId"
+                    placeholder="AKIAIOSFODNN7EXAMPLE"
+                    {...register("accessKeyId")}
+                  />
+                  <FieldError errors={[errors.accessKeyId]} />
+                </Field>
+                <Field>
+                  <FieldTitle>
+                    <label htmlFor="secretAccessKey">Secret Access Key</label>
+                  </FieldTitle>
+                  <Input
+                    aria-invalid={!!errors.secretAccessKey}
+                    autoComplete="current-password"
+                    id="secretAccessKey"
+                    placeholder="wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"
+                    type="password"
+                    {...register("secretAccessKey")}
+                  />
+                  <FieldError errors={[errors.secretAccessKey]} />
+                </Field>
+                <Button
+                  className="w-full"
+                  disabled={isSubmitting}
+                  type="submit"
+                >
+                  {isSubmitting ? "Signing in..." : "Sign In"}
+                </Button>
+              </FieldGroup>
+            </form>
+          </CardContent>
+        </Card>
+
+        <CertificateInstallBanner />
+      </div>
    </div>
  )
 }
+
+function CertificateInstallBanner() {
+  return (
+    <Card className="border-muted bg-muted/30">
+      <CardContent className="pt-4 pb-4">
+        <div className="flex flex-col gap-3">
+          <div>
+            <p className="text-sm font-medium">
+              Install Local Certificate Authority
+            </p>
+            <p className="text-xs text-muted-foreground">
+              To avoid browser certificate warnings, download and install the
+              Spinifex CA as a trusted root on your OS.
+            </p>
+          </div>
+          <a
+            className="inline-flex h-8 w-full items-center justify-center rounded-md border border-input bg-background px-3 text-sm font-medium hover:bg-accent hover:text-accent-foreground"
+            download="spinifex-ca.pem"
+            href="/api/ca.pem"
+          >
+            Download CA Certificate
+          </a>
+          <Collapsible>
+            <CollapsibleTrigger className="cursor-pointer text-xs text-muted-foreground underline-offset-4 hover:underline">
+              Installation instructions
+            </CollapsibleTrigger>
+            <CollapsibleContent>
+              <div className="mt-2 flex flex-col gap-2 text-xs text-muted-foreground">
+                <div>
+                  <p className="font-medium text-foreground">macOS</p>
+                  <code className="mt-1 block rounded bg-muted p-2 text-[11px] break-all">
+                    sudo security add-trusted-cert -d -r trustRoot -k
+                    /Library/Keychains/System.keychain spinifex-ca.pem
+                  </code>
+                </div>
+                <div>
+                  <p className="font-medium text-foreground">Windows</p>
+                  <p>
+                    Double-click the file &rarr; Install Certificate &rarr;
+                    Local Machine &rarr; Trusted Root Certification Authorities
+                  </p>
+                </div>
+                <div>
+                  <p className="font-medium text-foreground">Linux</p>
+                  <code className="mt-1 block rounded bg-muted p-2 text-[11px] break-all">
+                    sudo cp spinifex-ca.pem
+                    /usr/local/share/ca-certificates/spinifex-ca.crt && sudo
+                    update-ca-certificates
+                  </code>
+                </div>
+              </div>
+            </CollapsibleContent>
+          </Collapsible>
+        </div>
+      </CardContent>
+    </Card>
+  )
+}
@@ -146,11 +146,27 @@ func (svc *Service) launchService() error {
 		return fmt.Errorf("key file not found: %s", svc.Config.TLSKey)
 	}

+	// Derive CA cert path from server cert directory.
+	caCertPath := filepath.Join(filepath.Dir(svc.Config.TLSCert), "ca.pem")
+
 	// Serve static files from embedded filesystem
 	fileServer := http.FileServer(http.FS(contentFS))

 	// SPA handler: try to serve the file, fallback to index.html
 	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// Serve CA certificate download at /api/ca.pem.
+		if r.URL.Path == "/api/ca.pem" {
+			if _, err := os.Stat(caCertPath); os.IsNotExist(err) {
+				slog.Warn("CA certificate requested but not found", "path", caCertPath)
+				http.Error(w, "CA certificate not yet generated. Run 'spx admin init' to create it.", http.StatusNotFound)
+				return
+			}
+			w.Header().Set("Content-Type", "application/x-pem-file")
+			w.Header().Set("Content-Disposition", `attachment; filename="spinifex-ca.pem"`)
+			http.ServeFile(w, r, caCertPath)
+			return
+		}
+
 		// Clean the path
 		path := strings.TrimPrefix(r.URL.Path, "/")
 		if path == "" {
@@ -0,0 +1,355 @@
+#!/bin/bash
+# run-cert-e2e.sh — Certificate validation E2E tests.
+#
+# Validates that server certificates contain the correct SANs and that
+# TLS connections succeed without --insecure when using the Spinifex CA.
+#
+# Works in three modes:
+#   1. Single-node:        ./run-cert-e2e.sh
+#   2. Pseudo multi-node:  ./run-cert-e2e.sh --pseudo-multinode
+#   3. Real multi-node:    ./run-cert-e2e.sh --multinode <node1-ip> <node2-ip> <node3-ip>
+#
+# Prerequisites:
+#   - Cluster already bootstrapped and running (services up, certs generated)
+#   - CA cert installed in system trust store (bootstrap.sh step 8)
+#   - openssl and curl available
+#   - For pseudo multi-node: simulated IPs configured (10.11.12.1-3)
+
+set -euo pipefail
+
+# --- Configuration ---
+
+AWSGW_PORT=9999
+UI_PORT=3000
+PASSED=0
+FAILED=0
+TOTAL=0
+
+# --- Parse arguments ---
+
+MODE="single"
+NODE_IPS=()
+
+case "${1:-}" in
+    --pseudo-multinode)
+        MODE="pseudo"
+        NODE_IPS=("10.11.12.1" "10.11.12.2" "10.11.12.3")
+        ;;
+    --multinode)
+        shift
+        MODE="multinode"
+        NODE_IPS=("$@")
+        if [ ${#NODE_IPS[@]} -lt 2 ]; then
+            echo "Usage: $0 --multinode <node1-ip> <node2-ip> [node3-ip ...]"
+            exit 1
+        fi
+        ;;
+    "")
+        MODE="single"
+        # Single-node: discover all IPs that should be in the cert.
+        NODE_IPS=("127.0.0.1")
+        # Also add the machine's non-loopback IPs.
+        while IFS= read -r ip; do
+            NODE_IPS+=("$ip")
+        done < <(ip -4 addr show scope global | grep -oP 'inet \K[\d.]+')
+        ;;
+    *)
+        echo "Unknown option: $1"
+        echo "Usage: $0 [--pseudo-multinode | --multinode <ip1> <ip2> ...]"
+        exit 1
+        ;;
+esac
+
+# --- Helpers ---
+
+pass() {
+    PASSED=$((PASSED + 1))
+    TOTAL=$((TOTAL + 1))
+    echo "  PASS: $1"
+}
+
+fail() {
+    FAILED=$((FAILED + 1))
+    TOTAL=$((TOTAL + 1))
+    echo "  FAIL: $1"
+}
+
+# Resolve the Spinifex config CA cert path (not the system copy).
+resolve_ca_cert() {
+    for candidate in \
+        "$HOME/spinifex/config/ca.pem" \
+        "$HOME/node1/config/ca.pem"; do
+        if [ -f "$candidate" ]; then
+            echo "$candidate"
+            return
+        fi
+    done
+    echo ""
+}
+
+# --- Locate CA cert ---
+
+CA_CERT=$(resolve_ca_cert)
+if [ -z "$CA_CERT" ]; then
+    echo "ERROR: Cannot find Spinifex CA certificate"
+    exit 1
+fi
+
+SYSTEM_CA_PATH="/usr/local/share/ca-certificates/spinifex-ca.crt"
+
+echo "Using CA cert: $CA_CERT"
+echo "Mode: $MODE"
+echo "Node IPs: ${NODE_IPS[*]}"
+echo ""
+
+# --- Test 1: Verify cert SANs contain expected IPs ---
+
+echo "=== Test 1: Certificate SAN validation ==="
+
+for ip in "${NODE_IPS[@]}"; do
+    # Determine which node's cert to check.
+    case "$MODE" in
+        single)
+            CERT_PATH="$HOME/spinifex/config/server.pem"
+            ;;
+        pseudo)
+            # Each pseudo node has its own config dir.
+            NODE_NUM="${ip##*.}"  # Extract last octet (1, 2, 3)
+            CERT_PATH="$HOME/node${NODE_NUM}/config/server.pem"
+            ;;
+        multinode)
+            # For real multi-node, fetch cert via openssl s_client.
+            CERT_PATH=""
+            ;;
+    esac
+
+    if [ -n "$CERT_PATH" ] && [ -f "$CERT_PATH" ]; then
+        # Check SAN IPs in the local cert file.
+        SANS=$(openssl x509 -in "$CERT_PATH" -text -noout 2>/dev/null | grep -A1 "Subject Alternative Name" || true)
+        if echo "$SANS" | grep -q "$ip"; then
+            pass "Cert at $CERT_PATH contains IP SAN $ip"
+        else
+            fail "Cert at $CERT_PATH missing IP SAN $ip (SANs: $SANS)"
+        fi
+    else
+        # Fetch cert over the wire and check SANs.
+        SANS=$(echo | openssl s_client -connect "$ip:$AWSGW_PORT" -servername "$ip" 2>/dev/null \
+            | openssl x509 -text -noout 2>/dev/null \
+            | grep -A1 "Subject Alternative Name" || true)
+        if echo "$SANS" | grep -q "$ip"; then
+            pass "Cert served by $ip:$AWSGW_PORT contains IP SAN $ip"
+        else
+            fail "Cert served by $ip:$AWSGW_PORT missing IP SAN $ip (SANs: $SANS)"
+        fi
+    fi
+done
+
+# --- Test 2: Verify hostname is in DNS SANs ---
+
+echo ""
+echo "=== Test 2: Hostname DNS SAN validation ==="
+
+HOSTNAME=$(hostname)
+if [ "$HOSTNAME" != "localhost" ] && [ -n "$HOSTNAME" ]; then
+    case "$MODE" in
+        single)
+            CERT_PATH="$HOME/spinifex/config/server.pem"
+            ;;
+        pseudo)
+            CERT_PATH="$HOME/node1/config/server.pem"
+            ;;
+        multinode)
+            CERT_PATH=""
+            ;;
+    esac
+
+    if [ -n "$CERT_PATH" ] && [ -f "$CERT_PATH" ]; then
+        SANS=$(openssl x509 -in "$CERT_PATH" -text -noout 2>/dev/null | grep -A1 "Subject Alternative Name" || true)
+        if echo "$SANS" | grep -qi "$HOSTNAME"; then
+            pass "Cert contains DNS SAN for hostname '$HOSTNAME'"
+        else
+            fail "Cert missing DNS SAN for hostname '$HOSTNAME' (SANs: $SANS)"
+        fi
+    else
+        echo "  SKIP: Cannot check hostname SAN for remote nodes"
+    fi
+else
+    echo "  SKIP: Hostname is 'localhost' or empty"
+fi
+
+# --- Test 3: TLS connection succeeds with explicit CA cert (no --insecure) ---
+
+echo ""
+echo "=== Test 3: TLS handshake with explicit CA trust (no --insecure) ==="
+
+for ip in "${NODE_IPS[@]}"; do
+    HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
+        --cacert "$CA_CERT" \
+        --connect-timeout 5 \
+        "https://$ip:$AWSGW_PORT/" 2>/dev/null || echo "000")
+
+    if [ "$HTTP_CODE" != "000" ]; then
+        pass "TLS handshake OK for https://$ip:$AWSGW_PORT (HTTP $HTTP_CODE)"
+    else
+        fail "TLS handshake FAILED for https://$ip:$AWSGW_PORT"
+    fi
+done
+
+# --- Test 4: System trust store — CA installed and trusted ---
+
+echo ""
+echo "=== Test 4: System trust store validation ==="
+
+# 4a: Check the CA file exists in the system cert directory.
+if [ -f "$SYSTEM_CA_PATH" ]; then
+    pass "CA cert installed at $SYSTEM_CA_PATH"
+else
+    fail "CA cert NOT found at $SYSTEM_CA_PATH"
+fi
+
+# 4b: Verify the system copy matches the config copy (not stale).
+if [ -f "$SYSTEM_CA_PATH" ]; then
+    SYSTEM_HASH=$(openssl x509 -in "$SYSTEM_CA_PATH" -fingerprint -noout 2>/dev/null || echo "")
+    CONFIG_HASH=$(openssl x509 -in "$CA_CERT" -fingerprint -noout 2>/dev/null || echo "")
+    if [ -n "$SYSTEM_HASH" ] && [ "$SYSTEM_HASH" = "$CONFIG_HASH" ]; then
+        pass "System CA fingerprint matches config CA"
+    else
+        fail "System CA fingerprint mismatch (system: $SYSTEM_HASH, config: $CONFIG_HASH)"
+    fi
+fi
+
+# 4c: Verify curl works WITHOUT --cacert by relying on system trust store.
+# This confirms update-ca-certificates ran and the CA is in the trust bundle.
+TEST_IP="${NODE_IPS[0]}"
+HTTP_CODE=$(curl -s -o /dev/null -w "%{http_code}" \
+    --connect-timeout 5 \
+    "https://$TEST_IP:$AWSGW_PORT/" 2>/dev/null || echo "000")
+
+if [ "$HTTP_CODE" != "000" ]; then
+    pass "System trust store: curl https://$TEST_IP:$AWSGW_PORT succeeds without --cacert (HTTP $HTTP_CODE)"
+else
+    fail "System trust store: curl https://$TEST_IP:$AWSGW_PORT FAILED without --cacert — CA not in system bundle?"
+fi
+
+# --- Test 5: openssl s_client verify with CA ---
+
+echo ""
+echo "=== Test 5: openssl s_client verify ==="
+
+for ip in "${NODE_IPS[@]}"; do
+    VERIFY_OUTPUT=$(echo | openssl s_client \
+        -CAfile "$CA_CERT" \
+        -connect "$ip:$AWSGW_PORT" \
+        -verify_return_error 2>&1 || true)
+
+    if echo "$VERIFY_OUTPUT" | grep -q "Verify return code: 0"; then
+        pass "openssl verify OK for $ip:$AWSGW_PORT"
+    else
+        VERIFY_CODE=$(echo "$VERIFY_OUTPUT" | grep "Verify return code:" | head -1 || echo "unknown")
+        fail "openssl verify FAILED for $ip:$AWSGW_PORT ($VERIFY_CODE)"
+    fi
+done
+
+# --- Test 6: CA download endpoint (UI) ---
+
+echo ""
+echo "=== Test 6: CA download endpoint ==="
+
+# Try UI endpoint on first node.
+UI_IP="${NODE_IPS[0]}"
+if [ "$MODE" = "single" ]; then
+    UI_IP="127.0.0.1"
+fi
+
+DOWNLOAD_OUTPUT=$(curl -s --cacert "$CA_CERT" --connect-timeout 5 \
+    "https://$UI_IP:$UI_PORT/api/ca.pem" 2>/dev/null || echo "")
+
+if echo "$DOWNLOAD_OUTPUT" | grep -q "BEGIN CERTIFICATE"; then
+    # Verify downloaded cert matches the CA cert we're using.
+    DOWNLOADED_HASH=$(echo "$DOWNLOAD_OUTPUT" | openssl x509 -fingerprint -noout 2>/dev/null || echo "")
+    LOCAL_HASH=$(openssl x509 -in "$CA_CERT" -fingerprint -noout 2>/dev/null || echo "")
+
+    if [ -n "$DOWNLOADED_HASH" ] && [ "$DOWNLOADED_HASH" = "$LOCAL_HASH" ]; then
+        pass "CA download from https://$UI_IP:$UI_PORT/api/ca.pem matches local CA"
+    else
+        fail "CA download fingerprint mismatch (downloaded: $DOWNLOADED_HASH, local: $LOCAL_HASH)"
+    fi
+else
+    # UI may not be running in all E2E modes (pseudo-multinode disables UI).
+    echo "  SKIP: UI not reachable at https://$UI_IP:$UI_PORT (not running or wrong IP)"
+fi
+
+# --- Test 7: Cert CN and issuer ---
+
+echo ""
+echo "=== Test 7: Certificate metadata ==="
+
+case "$MODE" in
+    single) CERT_PATH="$HOME/spinifex/config/server.pem" ;;
+    pseudo) CERT_PATH="$HOME/node1/config/server.pem" ;;
+    *) CERT_PATH="" ;;
+esac
+
+if [ -n "$CERT_PATH" ] && [ -f "$CERT_PATH" ]; then
+    CN=$(openssl x509 -in "$CERT_PATH" -subject -noout 2>/dev/null || echo "")
+    ISSUER=$(openssl x509 -in "$CERT_PATH" -issuer -noout 2>/dev/null || echo "")
+
+    if echo "$CN" | grep -q "Spinifex Server"; then
+        pass "Server cert CN = 'Spinifex Server'"
+    else
+        fail "Server cert CN is not 'Spinifex Server' (got: $CN)"
+    fi
+
+    if echo "$ISSUER" | grep -q "Spinifex Local CA"; then
+        pass "Server cert issued by 'Spinifex Local CA'"
+    else
+        fail "Server cert issuer is not 'Spinifex Local CA' (got: $ISSUER)"
+    fi
+
+    # Check expiry is within 1 year.
+    EXPIRY=$(openssl x509 -in "$CERT_PATH" -enddate -noout 2>/dev/null | sed 's/notAfter=//')
+    if [ -n "$EXPIRY" ]; then
+        pass "Server cert expiry: $EXPIRY"
+    else
+        fail "Could not read cert expiry"
+    fi
+else
+    echo "  SKIP: No local cert path for multinode mode"
+fi
+
+# --- Test 8: Cross-node cert isolation (multi-node only) ---
+
+if [ "$MODE" != "single" ] && [ ${#NODE_IPS[@]} -ge 2 ]; then
+    echo ""
+    echo "=== Test 8: Cross-node cert isolation ==="
+
+    for ip in "${NODE_IPS[@]}"; do
+        # Each node's cert should contain its own IP but served certs should all
+        # chain to the same CA.
+        CERT_ISSUER=$(echo | openssl s_client -connect "$ip:$AWSGW_PORT" 2>/dev/null \
+            | openssl x509 -issuer -noout 2>/dev/null || echo "")
+
+        if echo "$CERT_ISSUER" | grep -q "Spinifex Local CA"; then
+            pass "Node $ip cert chains to shared Spinifex CA"
+        else
+            fail "Node $ip cert has unexpected issuer: $CERT_ISSUER"
+        fi
+    done
+fi
+
+# --- Summary ---
+
+echo ""
+echo "========================================"
+echo "Certificate E2E Results"
+echo "========================================"
+echo "  Total:  $TOTAL"
+echo "  Passed: $PASSED"
+echo "  Failed: $FAILED"
+echo ""
+
+if [ "$FAILED" -gt 0 ]; then
+    echo "CERTIFICATE E2E TESTS FAILED"
+    exit 1
+fi
+echo "ALL CERTIFICATE E2E TESTS PASSED"
@@ -2844,6 +2844,59 @@ if [ "$PUB_SSH_READY" = true ]; then
    else
        echo "WARN: Outbound internet test inconclusive (may depend on WAN gateway)"
    fi
+    # Verify cloud-init injected the Spinifex CA into the guest trust store.
+    # The guest should be able to curl Spinifex services (awsgw, predastore)
+    # WITHOUT --insecure, proving the CA cert was installed via ca_certs module.
+    echo ""
+    echo "Phase 8b Step 2b: In-guest TLS trust (cloud-init CA injection)"
+
+    # Test 1: Guest can reach awsgw via HTTPS without --insecure
+    set +e
+    GUEST_TLS_AWSGW=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+        -o ConnectTimeout=5 -o BatchMode=yes -i "test-key-1.pem" \
+        ec2-user@"$PUB_IP" "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 https://${GATEWAY_HOST}:9999/ 2>/dev/null || echo 000" 2>/dev/null)
+    set -e
+
+    if [ -n "$GUEST_TLS_AWSGW" ] && [ "$GUEST_TLS_AWSGW" != "000" ]; then
+        echo "PASS: Guest VM trusts awsgw TLS (HTTP $GUEST_TLS_AWSGW) — cloud-init CA injection works"
+    else
+        echo "FAIL: Guest VM cannot reach awsgw via HTTPS without --insecure (got: $GUEST_TLS_AWSGW)"
+        echo "  This means cloud-init did not inject the Spinifex CA into the guest trust store."
+        echo "  Checking guest user-data for ca_certs block..."
+        ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+            -o BatchMode=yes -i "test-key-1.pem" \
+            ec2-user@"$PUB_IP" "cat /var/lib/cloud/instance/user-data.txt 2>/dev/null | head -30" 2>/dev/null || true
+        exit 1
+    fi
+
+    # Test 2: Guest can reach predastore S3 via HTTPS without --insecure
+    set +e
+    GUEST_TLS_S3=$(ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+        -o ConnectTimeout=5 -o BatchMode=yes -i "test-key-1.pem" \
+        ec2-user@"$PUB_IP" "curl -s -o /dev/null -w '%{http_code}' --connect-timeout 5 https://${GATEWAY_HOST}:8443/ 2>/dev/null || echo 000" 2>/dev/null)
+    set -e
+
+    if [ -n "$GUEST_TLS_S3" ] && [ "$GUEST_TLS_S3" != "000" ]; then
+        echo "PASS: Guest VM trusts predastore S3 TLS (HTTP $GUEST_TLS_S3)"
+    else
+        echo "WARN: Guest VM cannot reach predastore S3 at https://${GATEWAY_HOST}:8443 (got: $GUEST_TLS_S3)"
+        echo "  Predastore may use a separate self-signed cert (QUIC) — this is expected for now."
+    fi
+
+    # Test 3: Verify the ca_certs block exists in guest user-data
+    set +e
+    ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR \
+        -o BatchMode=yes -i "test-key-1.pem" \
+        ec2-user@"$PUB_IP" "sudo grep -q ca_certs /var/lib/cloud/instance/user-data.txt" 2>/dev/null
+    GREP_EXIT=$?
+    set -e
+
+    if [ "$GREP_EXIT" -eq 0 ]; then
+        echo "PASS: Guest user-data contains ca_certs block"
+    else
+        echo "FAIL: Guest user-data does NOT contain ca_certs block"
+        exit 1
+    fi
 else
    echo "WARN: SSH via public IP $PUB_IP not reachable (macvlan isolation or bridge not ready)"
    echo "Falling back to API-only verification (OVN state already validated above)"
@@ -1308,6 +1308,14 @@ fi

 echo "  Instance relaunch + terminate after restart passed"

+# Run certificate validation E2E while services are still up (before cleanup trap).
+echo ""
+echo "========================================"
+echo "Running Certificate E2E Tests"
+echo "========================================"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+"${SCRIPT_DIR}/run-cert-e2e.sh" --pseudo-multinode
+
 echo ""
 echo "========================================"
 echo "Multi-Node E2E Tests Completed Successfully"
				`@@ -0,0 +1 @@`
				import{a as e,r as t}from"./compiler-runtime.js";import{u as n}from"./utils.js";import{R as r,i,o as a}from"./button.js";var o=e(t()),s=[];function c(e){o.useEffect(e,s)}var l=null;globalThis.requestAnimationFrame;var u=new class{callbacks=[];callbacksCount=0;nextId=1;startId=1;isScheduled=!1;tick=e=>{this.isScheduled=!1;let t=this.callbacks,n=this.callbacksCount;if(this.callbacks=[],this.callbacksCount=0,this.startId=this.nextId,n>0)for(let n=0;n<t.length;n+=1)t[n]?.(e)};request(e){let t=this.nextId;return this.nextId+=1,this.callbacks.push(e),this.callbacksCount+=1,this.isScheduled\|\|=(requestAnimationFrame(this.tick),!0),t}cancel(e){let t=e-this.startId;t<0\|\|t>=this.callbacks.length\|\|(this.callbacks[t]=null,--this.callbacksCount)}},d=class e{static create(){return new e}static request(e){return u.request(e)}static cancel(e){return u.cancel(e)}currentId=l;request(e){this.cancel(),this.currentId=u.request(()=>{this.currentId=l,e()})}cancel=()=>{this.currentId!==l&&(u.cancel(this.currentId),this.currentId=l)};disposeEffect=()=>this.cancel};function f(){let e=r(d.create).current;return c(e.disposeEffect),e}function p(e){return e==null?e:`current`in e?e.current:e}var m=function(e){return e.startingStyle=`data-starting-style`,e.endingStyle=`data-ending-style`,e}({}),h={[m.startingStyle]:``},g={[m.endingStyle]:``},_={transitionStatus(e){return e===`starting`?h:e===`ending`?g:null}},v=e(n());function y(e,t=!1,n=!0){let r=f();return a((i,a=null)=>{r.cancel();function o(){v.flushSync(i)}let s=p(e);if(s==null)return;let c=s;if(typeof c.getAnimations!=`function`\|\|globalThis.BASE_UI_ANIMATIONS_DISABLED)i();else{function e(){let e=m.startingStyle;if(!c.hasAttribute(e)){r.request(i);return}let t=new MutationObserver(()=>{c.hasAttribute(e)\|\|(t.disconnect(),i())});t.observe(c,{attributes:!0,attributeFilter:[e]}),a?.addEventListener(`abort`,()=>t.disconnect(),{once:!0})}function i(){Promise.all(c.getAnimations().map(e=>e.finished)).then(()=>{a?.aborted\|\|o()}).catch(()=>{let e=c.getAnimations();if(n){if(a?.aborted)return;o()}else e.length>0&&e.some(e=>e.pending\|\|e.playState!==`finished`)&&i()})}if(t){e();return}r.request(i)}})}function b(e){let{enabled:t=!0,open:n,ref:r,onComplete:i}=e,s=a(i),c=y(r,n,!1);o.useEffect(()=>{if(!t)return;let e=new AbortController;return c(s,e.signal),()=>{e.abort()}},[t,n,s,c])}function x(e,t=!1,n=!1){let[r,a]=o.useState(e&&t?`idle`:void 0),[s,c]=o.useState(e);return e&&!s&&(c(!0),a(`starting`)),!e&&s&&r!==`ending`&&!n&&a(`ending`),!e&&!s&&r===`ending`&&a(void 0),i(()=>{if(!e&&s&&r!==`ending`&&n){let e=d.request(()=>{a(`ending`)});return()=>{d.cancel(e)}}},[e,s,r,n]),i(()=>{if(!e\|\|t)return;let n=d.request(()=>{a(void 0)});return()=>{d.cancel(n)}},[t,e]),i(()=>{if(!e\|\|!t)return;e&&s&&r!==`idle`&&a(`starting`);let n=d.request(()=>{a(`idle`)});return()=>{d.cancel(n)}},[t,e,s,a,r]),o.useMemo(()=>({mounted:s,setMounted:c,transitionStatus:r}),[s,r])}export{_ as a,f as c,m as i,c as l,b as n,p as o,y as r,d as s,x as t};