gsadmin/spinifex
1
0
Fork 0
mirror of https://github.com/mulgadc/spinifex.git synced 2026-10-15 17:11:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

ISO tweaks and doc template

Browse Source
This commit is contained in:
Ben Duncan
2026-04-16 15:38:56 +10:00
parent 402a04e1b9
commit d8924a2ecc
4 changed files with 474 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/installer/firstboot/firstboot.go
+32 -8
View File
@@ -82,14 +82,15 @@ trap 'systemctl disable spinifex-firstboot.service' EXIT
# Set hostname
hostnamectl set-hostname %s
# Run setup.sh with the embedded tarball (air-gapped, idempotent).
# This creates service users, directories, sudoers rules, installs and enables
# the systemd units — the same path as the online curl|bash installer.
# Apt deps and AWS CLI are pre-installed in the squashfs so both are skipped.
INSTALL_SPINIFEX_TARBALL=/opt/spinifex/spinifex.tar.gz \
INSTALL_SPINIFEX_SKIP_APT=1 \
INSTALL_SPINIFEX_SKIP_NEWGRP=1 \
bash /usr/local/share/spinifex/setup.sh
# NOTE: We do NOT invoke /usr/local/share/spinifex/setup.sh here. The ISO
# rootfs (scripts/iso-builder/build/build-rootfs.sh) already pre-stages
# everything setup.sh would do at install time: spx binary, viperblock
# nbdkit plugin, service users (spinifex-{nats,gw,daemon,storage,viperblock,
# vpcd,ui}), data directories with per-service ownership, sudoers rules,
# systemd units, and tmpfiles.d entries. Calling setup.sh would also fail
# in the air-gapped boot environment because no tarball is staged at
# /opt/spinifex/spinifex.tar.gz — the firstboot would never complete and
# every spinifex-* service depending on it would refuse to start.
# Pre-start OVS and OVN central so their databases are initialised before
# setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes
@@ -116,6 +117,29 @@ done
# Cluster formation — capture credentials to file for display on console.
%s 2>&1
# Fix ownership of files spx admin init created. spx runs as root (no
# SUDO_USER under systemd), so /etc/spinifex/{spinifex.toml,master.key} and
# any per-service files written under /var/lib/spinifex/* land as root:root.
# Re-chown them so each service can read/write its own subtree at
# Type=notify start. The directories themselves were pre-created with the
# correct ownership in build-rootfs.sh — these chowns only affect the
# runtime-generated config and key files inside them.
chown root:spinifex /etc/spinifex && chmod 750 /etc/spinifex
find /etc/spinifex -type f -exec chmod 640 {} \;
chown -R spinifex-gw:spinifex /var/lib/spinifex/awsgw
chown -R spinifex-daemon:spinifex /var/lib/spinifex/spinifex
chown -R spinifex-nats:spinifex /var/lib/spinifex/nats
chown -R spinifex-storage:spinifex /var/lib/spinifex/predastore
chown -R spinifex-viperblock:spinifex /var/lib/spinifex/viperblock
chown -R spinifex-vpcd:spinifex /var/lib/spinifex/vpcd
# awsgw looks for master.key at <BaseDir>/config/master.key. In production
# BaseDir is /var/lib/spinifex/awsgw/ (set by SPINIFEX_BASE_DIR), but the
# key lives in /etc/spinifex/. Symlink so both paths resolve to the same
# file. spx admin init writes the key, so this must run after it.
mkdir -p /var/lib/spinifex/awsgw/config
ln -sf /etc/spinifex/master.key /var/lib/spinifex/awsgw/config/master.key
# Copy AWS credentials to the spinifex user's home directory.
# spx admin init runs with HOME=/root (set by the systemd unit), so credentials
# land in /root/.aws/. Copy them to the spinifex user's home so the operator
cmd/installer/install/install.go
+129 -34
View File
@@ -85,14 +85,50 @@ func partitionDisk(disk string) error {
// p1: 1MiB BIOS Boot Partition — required for grub-install i386-pc on GPT
// p2: 512MiB EFI System Partition
// p3: remainder as root (ext4)
return run("parted", "--script", disk,
if err := run("parted", "--script", disk,
"mklabel", "gpt",
"mkpart", "bios_boot", "1MiB", "2MiB",
"set", "1", "bios_grub", "on",
"mkpart", "ESP", "fat32", "2MiB", "514MiB",
"set", "2", "esp", "on",
"mkpart", "root", "ext4", "514MiB", "100%",
)
); err != nil {
return err
}
// Force the kernel to re-read the partition table and wait for udev to
// create the partition device nodes. Without this, mkfs.fat in the next
// step may race and fail with "No such file or directory" on /dev/sda2 —
// the kernel has accepted the new layout but /dev hasn't been populated
// yet. Trixie's udev seems slower at this than Bookworm's was.
return waitForPartitions(disk)
}
// waitForPartitions ensures the EFI and root partition device nodes exist
// after parted creates them. It runs partprobe (kernel re-read) and
// udevadm settle (wait for queued events), then polls /dev for the files.
func waitForPartitions(disk string) error {
// Best-effort: partprobe failure isn't fatal — udev may still pick up
// the change from the BLKRRPART ioctl that parted itself issued.
if err := run("partprobe", disk); err != nil {
slog.Warn("partprobe failed, continuing", "disk", disk, "err", err)
}
if err := run("udevadm", "settle", "--timeout=10"); err != nil {
slog.Warn("udevadm settle failed, continuing", "err", err)
}
efi, root := partitionPaths(disk)
deadline := time.Now().Add(15 * time.Second)
for _, part := range []string{efi, root} {
for {
if _, err := os.Stat(part); err == nil {
break
}
if time.Now().After(deadline) {
return fmt.Errorf("partition device %s did not appear within timeout — kernel/udev did not pick up new partition table", part)
}
time.Sleep(200 * time.Millisecond)
}
}
return nil
}
func formatPartitions(disk string) error {
@@ -178,6 +214,14 @@ func installSpinifex(cfg *Config) error {
// /usr/local/bin/ — no binary copy needed. Regenerate machine-specific
// identity files so each installed node is unique.
// Bind-mount /dev, /proc, /sys into the chroot so PAM (chpasswd),
// systemd-machine-id-setup, and other chroot commands work correctly.
// Trixie's PAM requires /proc and /dev/urandom for password hashing.
if err := bindChrootMounts(); err != nil {
return err
}
defer unbindChrootMounts()
// Fresh machine-id (required by systemd and dbus).
machineIDPath := filepath.Join(mountRoot, "etc/machine-id")
_ = os.Remove(machineIDPath)
@@ -203,24 +247,29 @@ func installSpinifex(cfg *Config) error {
return err
}
// Set root password via chpasswd inside the chroot.
// Set root + spinifex passwords. We invoke chpasswd from the LIVE
// installer (not via `chroot`), passing the target root with -R and
// forcing -c YESCRYPT. This deliberately bypasses PAM:
// * `chroot ... chpasswd` uses /etc/pam.d/chpasswd → common-password →
// pam_unix.so with the "obscure" option, which can return
// "Authentication token manipulation error" inside a chroot for
// reasons that are awkward to diagnose (audit subsystem, locked
// shadow entries from useradd, etc.).
// * -c YESCRYPT tells chpasswd to hash locally with libcrypt and write
// directly to <root>/etc/shadow — no PAM stack involved. YESCRYPT
// matches Trixie's ENCRYPT_METHOD so subsequent logins use the same
// algorithm.
// * -R <root> opens the target's passwd/shadow directly; no bind
// mounts of /dev/urandom or /proc are needed for the password step.
if cfg.RootPassword != "" {
cmd := exec.Command("chroot", mountRoot, "chpasswd")
cmd.Stdin = strings.NewReader("root:" + cfg.RootPassword)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
if err := setShadowPassword("root", cfg.RootPassword); err != nil {
return fmt.Errorf("set root password: %w", err)
}
// Set the spinifex login user password. The spinifex account is the
// default interactive login on the node (autologin on console, SSH
// access). Root SSH is disabled, so this is the sole remote entry point.
cmd = exec.Command("chroot", mountRoot, "chpasswd")
cmd.Stdin = strings.NewReader("spinifex:" + cfg.RootPassword)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
if err := cmd.Run(); err != nil {
// The spinifex account is the default interactive login on the
// node (console + SSH). Root SSH is disabled, so this is the sole
// remote entry point. The user itself is created at rootfs build
// time (build-rootfs.sh) — here we just set its password.
if err := setShadowPassword("spinifex", cfg.RootPassword); err != nil {
return fmt.Errorf("set spinifex password: %w", err)
}
}
@@ -250,6 +299,13 @@ func installSpinifex(cfg *Config) error {
return err
}
// Mask dhcpcd.service so only ifupdown controls DHCP on the bridges.
// Trixie's dhcpcd-base ships a standalone dhcpcd.service that races with
// ifupdown's own dhcpcd invocations, causing duplicate leases or IP flapping.
if err := maskSystemdUnit(mountRoot, "dhcpcd.service"); err != nil {
slog.Warn("installSpinifex: failed to mask dhcpcd.service", "err", err)
}
return systemd.WriteNetworkingDropIn(mountRoot)
}
@@ -416,24 +472,11 @@ GRUB_BACKGROUND=/boot/grub/splash.png
}
// update-grub (grub-mkconfig) runs inside the installed system's chroot and
// needs /dev, /proc, and /sys to probe devices. Bind-mount them in and
// unmount regardless of outcome.
chrootMounts := []string{"dev", "proc", "sys"}
for _, m := range chrootMounts {
dst := filepath.Join(mountRoot, m)
if err := os.MkdirAll(dst, 0o755); err != nil {
return fmt.Errorf("create chroot mountpoint /%s: %w", m, err)
}
if err := run("mount", "--bind", "/"+m, dst); err != nil {
return fmt.Errorf("bind-mount /%s into chroot: %w", m, err)
}
// needs /dev, /proc, and /sys to probe devices.
if err := bindChrootMounts(); err != nil {
return err
}
defer func() {
// Unmount in reverse order; ignore errors (best-effort cleanup).
for i := len(chrootMounts) - 1; i >= 0; i-- {
_ = run("umount", filepath.Join(mountRoot, chrootMounts[i]))
}
}()
defer unbindChrootMounts()
return run("chroot", mountRoot, "update-grub")
}
@@ -445,6 +488,10 @@ func installCACert(cfg *Config) error {
if err := os.WriteFile(certPath, []byte(cfg.CACert), 0o644); err != nil {
return err
}
if err := bindChrootMounts(); err != nil {
return err
}
defer unbindChrootMounts()
return run("chroot", mountRoot, "update-ca-certificates")
}
@@ -563,6 +610,54 @@ func copySplashImage(root string) {
}
}
// maskSystemdUnit creates a symlink to /dev/null for the given unit, which is
// the standard way to permanently disable a unit so systemd will never start it.
func maskSystemdUnit(root, unit string) error {
unitPath := filepath.Join(root, "etc/systemd/system", unit)
_ = os.Remove(unitPath)
return os.Symlink("/dev/null", unitPath)
}
// setShadowPassword sets a Unix password on the installed system without
// going through PAM. See the long comment in installSpinifex for the
// rationale.
func setShadowPassword(user, password string) error {
cmd := exec.Command("chpasswd", "-c", "YESCRYPT", "-R", mountRoot)
cmd.Stdin = strings.NewReader(user + ":" + password)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
return cmd.Run()
}
// chrootMountPaths lists virtual filesystems to bind-mount into the chroot.
// Order matters: unbind in reverse.
var chrootMountPaths = []string{"dev", "proc", "sys"}
// bindChrootMounts bind-mounts /dev, /proc, and /sys into the installed rootfs
// so chroot commands (chpasswd, systemd-machine-id-setup, update-grub) can
// access hardware, process info, and entropy sources. Idempotent — already-
// mounted paths are skipped.
func bindChrootMounts() error {
for _, m := range chrootMountPaths {
dst := filepath.Join(mountRoot, m)
if err := os.MkdirAll(dst, 0o755); err != nil {
return fmt.Errorf("create chroot mountpoint /%s: %w", m, err)
}
if err := run("mount", "--bind", "/"+m, dst); err != nil {
return fmt.Errorf("bind-mount /%s into chroot: %w", m, err)
}
}
return nil
}
// unbindChrootMounts unmounts the virtual filesystems in reverse order.
// Errors are logged but not returned — this is best-effort cleanup.
func unbindChrootMounts() {
for i := len(chrootMountPaths) - 1; i >= 0; i-- {
_ = run("umount", filepath.Join(mountRoot, chrootMountPaths[i]))
}
}
func run(name string, args ...string) error {
cmd := exec.Command(name, args...)
cmd.Stdout = os.Stdout
cmd/installer/main.go
+12 -3
View File
@@ -58,10 +58,19 @@ func main() {
}
// detectTTY returns the TTY device path the installer should attach to.
// It reads SPINIFEX_CONSOLE from the kernel cmdline (/proc/cmdline) so that
// spinifex-init can direct the installer to the correct console (ttyS0 for
// serial, tty1 for VGA). Falls back to tty1.
// It first checks the SPINIFEX_CONSOLE environment variable (set by
// spinifex-init.sh after console fallback logic), then falls back to parsing
// /proc/cmdline. spinifex-init.sh may override the GRUB-set value (e.g. when
// GRUB says tty1 but no framebuffer exists, it falls back to ttyS0 and
// exports the corrected value).
func detectTTY() string {
// Prefer the env var — spinifex-init.sh sets this after resolving the
// actual console (including headless/framebuffer fallback).
if val := os.Getenv("SPINIFEX_CONSOLE"); val != "" && val != "auto" {
return "/dev/" + val
}
// Fall back to kernel cmdline for direct invocation without spinifex-init.
data, err := os.ReadFile("/proc/cmdline")
if err == nil {
for param := range strings.FieldsSeq(string(data)) {
docs/install-vm-eval/README.md
+301
View File
@@ -0,0 +1,301 @@
# Installing Spinifex in a VM (MacOS)
For evaluation / quick POC use only.
Spinifex is designed for bare-metal hardware, edge nodes and data-centre use. However, follow this guide to install Spinifex on a MacOS VM to demonstrate the capabilities (dev/testing only).
Note this tutorial is for Apple Silicon (ARM64)
## UTM installation
For this tutorial download UTM.app on Mac to simplify the VM installation.
* UTM - [https://mac.getutm.app](https://mac.getutm.app)
## Download Debian ISO
Next, download the Debian ISO (ARM64, Apple Silicon)
* Debian 13.4 - [https://cdimage.debian.org/debian-cd/current/arm64/iso-cd/debian-13.4.0-arm64-netinst.iso](https://cdimage.debian.org/debian-cd/current/arm64/iso-cd/debian-13.4.0-arm64-netinst.iso)
## Login
Using the VM console, login using the `spinifex` user and your specified password. A few additional steps are required via the console, before you can SSH into the machine for easier administration.
## Install prerequisites
For a minimal Debian install additional dependencies are required prior to the Spinifex installation:
```
su root
apt update
apt install sudo curl bridge-utils
```
### Enable sudoers
Enable sudo for `spinifex` user as the root user:
```
su root
echo "spinifex ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/spinifex
```
## Networking
Spinifex requires a Linux bridge for networking to correctly function, given EC2 instances and resources can obtain an IP address from the upstream DHCP server or a static public IP address range you specify.
This is a requirement step for valid installation.
View the existing `/etc/network/interfaces` for the network configuration.
```sh
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface
allow-hotplug enp0s1
iface enp0s1 inet dhcp
# This is an autoconfigured IPv6 interface
iface enp0s1 inet6 auto
```
Edit `/etc/network/interfaces` note replace `enp0s1` with your network adapter.
```sh
# This file describes the network interfaces available on your system
# and how to activate them. For more information, see interfaces(5).
source /etc/network/interfaces.d/*
# The loopback network interface
auto lo
iface lo inet loopback
# The primary network interface - toggle to manual
allow-hotplug enp0s1
iface enp0s1 inet manual
# This is an autoconfigured IPv6 interface - toggle to manual
iface enp0s1 inet6 manual
# Spinifex Config
# WAN bridge (Spinifex EC2 instances connect here)
# VMs (EC2) / containers <-> br-wan <-> enp0s1 <-> external network
auto br-wan
iface br-wan inet dhcp
bridge_ports enp0s1
bridge_stp off
bridge_fd 0
bridge_maxwait 0
```
Walkthrough of changes:
* Adds eno3 into the bridge
* eno3 becomes a slave interface
* All traffic goes through br-wan
* Spinifex EC2 (VMs) in a public subnet, obtain an IP address from the WAN upstream
### Restart networking
Make sure you are logged into the console, not via SSH for these steps. Otherwise you will lose networking to the instance.
Replace `enp0s1` with your network adapter.
```bash
ifdown enp0s1
systemctl restart networking
```
Next, retrieve the IP address of the new `br-wan` bridge from your upstream DHCP server.
```bash
ip a show dev enp0s1
```
```bash
6: br-wan: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc noqueue state UP group default qlen 1000
link/ether 12:c2:bc:f3:f6:7b brd ff:ff:ff:ff:ff:ff
inet 192.168.0.249/24 brd 192.168.0.255 scope global dynamic noprefixroute br-wan
valid_lft 86103sec preferred_lft 75303sec
inet6 fd0d:9d60:d09b:0:ee77:630d:3237:9a10/64 scope global dynamic mngtmpaddr noprefixroute
valid_lft 86282sec preferred_lft 86282sec
inet6 fe80::85c0:f2aa:f0bf:e951/64 scope link
valid_lft forever preferred_lft forever
```
Note the IP `192.168.0.249`
# Spinifex Installation
## Launch installer
Next, optionally SSH into your instance (using the IP above) or continue using the console.
Replace with your IP below.
```
ssh spinifex@<your-ip>
```
As the `spinifex` user launch the binary installer:
```bash
su spinifex
curl -fsSL https://install.mulgadc.com | bash
```
```bash
[INFO] Spinifex installer
[INFO] Detected OS: Debian GNU/Linux 13 (trixie)
[INFO] Detected architecture: aarch64 (arm64)
[INFO] Installing system dependencies...
[INFO] System dependencies installed
[INFO] Installing AWS CLI v2...
[INFO] AWS CLI installed: aws-cli/2.34.30 Python/3.14.4 Linux/6.12.74+deb13+1-arm64 exe/aarch64.debian.13
[INFO] Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui})
/etc/sudoers.d/spinifex-network: parsed OK
[INFO] Scoped sudoers rules installed for spinifex-daemon and spinifex-vpcd
[INFO] Downloading Spinifex (latest channel, arm64)...
[INFO] Verifying checksum...
[INFO] Checksum verified
[INFO] Extracting...
[INFO] Installing files...
[INFO] /usr/local/bin/spx
[INFO] /usr/lib/aarch64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so
[INFO] /usr/local/share/spinifex/setup-ovn.sh
[INFO] Creating directories...
[INFO] Generated /etc/spinifex/systemd.env
[INFO] /var/lib/spinifex/predastore-start.sh
[INFO] /var/lib/spinifex/wait-for-nats.sh
[INFO] Fixing file ownership for privilege separation...
[INFO] File ownership updated
[INFO] Installing systemd units...
[INFO] /etc/systemd/system/spinifex-awsgw.service
[INFO] /etc/systemd/system/spinifex-daemon.service
[INFO] /etc/systemd/system/spinifex-nats.service
[INFO] /etc/systemd/system/spinifex-predastore.service
[INFO] /etc/systemd/system/spinifex.target
[INFO] /etc/systemd/system/spinifex-ui.service
[INFO] /etc/systemd/system/spinifex-viperblock.service
[INFO] /etc/systemd/system/spinifex-vpcd.service
Created symlink '/etc/systemd/system/multi-user.target.wants/spinifex.target' → '/etc/systemd/system/spinifex.target'.
[INFO] Systemd units installed and enabled (per-service users)
[INFO] Fresh install detected, skipping migrations
[INFO] Logrotate config installed
============================================
Spinifex installed successfully
============================================
Version: spinifex v1.1.0 (b7b6390) linux/arm64
Architecture: arm64
Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}
Binary: /usr/local/bin/spx
Config: /etc/spinifex/
Data: /var/lib/spinifex/
Logs: /var/log/spinifex/
```
## Setup OVN networking
Since the WAN interface is already a bridge (e.g. br-wan) run the Openvswitch (OVN) install script:
```bash
sudo /usr/local/share/spinifex/setup-ovn.sh --management
```
Output:
```bash
Auto-detected Linux bridge: br-wan (default route)
Will create OVS bridge br-ext + veth pair to link them
Auto-detected encap IP: 192.168.0.249
Auto-detected chassis ID: chassis-spinifex
=== Spinifex OVN Compute Node Setup ===
Management node: true
WAN bridge: br-ext (veth)
Linux bridge: br-wan (linked via veth pair)
OVN Remote (SB): tcp:127.0.0.1:6642
Encap IP: 192.168.0.249
Chassis ID: chassis-spinifex
...
```
## Spinifex Initialize
```bash
sudo spx admin init --node node1 --nodes 1
```
## Start services
```bash
sudo systemctl start spinifex.target
```
## Verify installation
Connect to the local AWS gateway and issue a sample command using the AWS CLI tool, all running on your local infrastructure, no cloud required.
```bash
export AWS_PROFILE=spinifex
aws ec2 describe-instance-types
```
> ***Note:*** The system may require ~30 seconds to initalise prior to accepting connections for the command to complete above.
Output:
```json
{
"InstanceTypes": [
{
"InstanceType": "t4g.nano",
"CurrentGeneration": true,
"SupportedRootDeviceTypes": [
"ebs"
],
"SupportedVirtualizationTypes": [
"hvm"
],
"Hypervisor": "kvm",
"ProcessorInfo": {
"SupportedArchitectures": [
"arm64"
]
},
"VCpuInfo": {
"DefaultVCpus": 2
},
"MemoryInfo": {
"SizeInMiB": 512
},
"PlacementGroupInfo": {
"SupportedStrategies": [
"cluster",
"spread"
]
},
"BurstablePerformanceSupported": true
},
]
}
```
# Installation Complete
Congratulations! 🥳
Your installation of Spinifex is now complete on a VM running on a Mac.
# Next steps