diff --git a/cmd/installer/firstboot/firstboot.go b/cmd/installer/firstboot/firstboot.go index 85f73e3c7..b5e8e2b41 100644 --- a/cmd/installer/firstboot/firstboot.go +++ b/cmd/installer/firstboot/firstboot.go @@ -82,14 +82,15 @@ trap 'systemctl disable spinifex-firstboot.service' EXIT # Set hostname hostnamectl set-hostname %s -# Run setup.sh with the embedded tarball (air-gapped, idempotent). -# This creates service users, directories, sudoers rules, installs and enables -# the systemd units — the same path as the online curl|bash installer. -# Apt deps and AWS CLI are pre-installed in the squashfs so both are skipped. -INSTALL_SPINIFEX_TARBALL=/opt/spinifex/spinifex.tar.gz \ -INSTALL_SPINIFEX_SKIP_APT=1 \ -INSTALL_SPINIFEX_SKIP_NEWGRP=1 \ -bash /usr/local/share/spinifex/setup.sh +# NOTE: We do NOT invoke /usr/local/share/spinifex/setup.sh here. The ISO +# rootfs (scripts/iso-builder/build/build-rootfs.sh) already pre-stages +# everything setup.sh would do at install time: spx binary, viperblock +# nbdkit plugin, service users (spinifex-{nats,gw,daemon,storage,viperblock, +# vpcd,ui}), data directories with per-service ownership, sudoers rules, +# systemd units, and tmpfiles.d entries. Calling setup.sh would also fail +# in the air-gapped boot environment because no tarball is staged at +# /opt/spinifex/spinifex.tar.gz — the firstboot would never complete and +# every spinifex-* service depending on it would refuse to start. # Pre-start OVS and OVN central so their databases are initialised before # setup-ovn.sh runs. On physical hardware, first-boot DB initialisation takes @@ -116,6 +117,29 @@ done # Cluster formation — capture credentials to file for display on console. %s 2>&1 +# Fix ownership of files spx admin init created. spx runs as root (no +# SUDO_USER under systemd), so /etc/spinifex/{spinifex.toml,master.key} and +# any per-service files written under /var/lib/spinifex/* land as root:root. +# Re-chown them so each service can read/write its own subtree at +# Type=notify start. The directories themselves were pre-created with the +# correct ownership in build-rootfs.sh — these chowns only affect the +# runtime-generated config and key files inside them. +chown root:spinifex /etc/spinifex && chmod 750 /etc/spinifex +find /etc/spinifex -type f -exec chmod 640 {} \; +chown -R spinifex-gw:spinifex /var/lib/spinifex/awsgw +chown -R spinifex-daemon:spinifex /var/lib/spinifex/spinifex +chown -R spinifex-nats:spinifex /var/lib/spinifex/nats +chown -R spinifex-storage:spinifex /var/lib/spinifex/predastore +chown -R spinifex-viperblock:spinifex /var/lib/spinifex/viperblock +chown -R spinifex-vpcd:spinifex /var/lib/spinifex/vpcd + +# awsgw looks for master.key at /config/master.key. In production +# BaseDir is /var/lib/spinifex/awsgw/ (set by SPINIFEX_BASE_DIR), but the +# key lives in /etc/spinifex/. Symlink so both paths resolve to the same +# file. spx admin init writes the key, so this must run after it. +mkdir -p /var/lib/spinifex/awsgw/config +ln -sf /etc/spinifex/master.key /var/lib/spinifex/awsgw/config/master.key + # Copy AWS credentials to the spinifex user's home directory. # spx admin init runs with HOME=/root (set by the systemd unit), so credentials # land in /root/.aws/. Copy them to the spinifex user's home so the operator diff --git a/cmd/installer/install/install.go b/cmd/installer/install/install.go index 6e4603431..a8f47b4c2 100644 --- a/cmd/installer/install/install.go +++ b/cmd/installer/install/install.go @@ -85,14 +85,50 @@ func partitionDisk(disk string) error { // p1: 1MiB BIOS Boot Partition — required for grub-install i386-pc on GPT // p2: 512MiB EFI System Partition // p3: remainder as root (ext4) - return run("parted", "--script", disk, + if err := run("parted", "--script", disk, "mklabel", "gpt", "mkpart", "bios_boot", "1MiB", "2MiB", "set", "1", "bios_grub", "on", "mkpart", "ESP", "fat32", "2MiB", "514MiB", "set", "2", "esp", "on", "mkpart", "root", "ext4", "514MiB", "100%", - ) + ); err != nil { + return err + } + // Force the kernel to re-read the partition table and wait for udev to + // create the partition device nodes. Without this, mkfs.fat in the next + // step may race and fail with "No such file or directory" on /dev/sda2 — + // the kernel has accepted the new layout but /dev hasn't been populated + // yet. Trixie's udev seems slower at this than Bookworm's was. + return waitForPartitions(disk) +} + +// waitForPartitions ensures the EFI and root partition device nodes exist +// after parted creates them. It runs partprobe (kernel re-read) and +// udevadm settle (wait for queued events), then polls /dev for the files. +func waitForPartitions(disk string) error { + // Best-effort: partprobe failure isn't fatal — udev may still pick up + // the change from the BLKRRPART ioctl that parted itself issued. + if err := run("partprobe", disk); err != nil { + slog.Warn("partprobe failed, continuing", "disk", disk, "err", err) + } + if err := run("udevadm", "settle", "--timeout=10"); err != nil { + slog.Warn("udevadm settle failed, continuing", "err", err) + } + efi, root := partitionPaths(disk) + deadline := time.Now().Add(15 * time.Second) + for _, part := range []string{efi, root} { + for { + if _, err := os.Stat(part); err == nil { + break + } + if time.Now().After(deadline) { + return fmt.Errorf("partition device %s did not appear within timeout — kernel/udev did not pick up new partition table", part) + } + time.Sleep(200 * time.Millisecond) + } + } + return nil } func formatPartitions(disk string) error { @@ -178,6 +214,14 @@ func installSpinifex(cfg *Config) error { // /usr/local/bin/ — no binary copy needed. Regenerate machine-specific // identity files so each installed node is unique. + // Bind-mount /dev, /proc, /sys into the chroot so PAM (chpasswd), + // systemd-machine-id-setup, and other chroot commands work correctly. + // Trixie's PAM requires /proc and /dev/urandom for password hashing. + if err := bindChrootMounts(); err != nil { + return err + } + defer unbindChrootMounts() + // Fresh machine-id (required by systemd and dbus). machineIDPath := filepath.Join(mountRoot, "etc/machine-id") _ = os.Remove(machineIDPath) @@ -203,24 +247,29 @@ func installSpinifex(cfg *Config) error { return err } - // Set root password via chpasswd inside the chroot. + // Set root + spinifex passwords. We invoke chpasswd from the LIVE + // installer (not via `chroot`), passing the target root with -R and + // forcing -c YESCRYPT. This deliberately bypasses PAM: + // * `chroot ... chpasswd` uses /etc/pam.d/chpasswd → common-password → + // pam_unix.so with the "obscure" option, which can return + // "Authentication token manipulation error" inside a chroot for + // reasons that are awkward to diagnose (audit subsystem, locked + // shadow entries from useradd, etc.). + // * -c YESCRYPT tells chpasswd to hash locally with libcrypt and write + // directly to /etc/shadow — no PAM stack involved. YESCRYPT + // matches Trixie's ENCRYPT_METHOD so subsequent logins use the same + // algorithm. + // * -R opens the target's passwd/shadow directly; no bind + // mounts of /dev/urandom or /proc are needed for the password step. if cfg.RootPassword != "" { - cmd := exec.Command("chroot", mountRoot, "chpasswd") - cmd.Stdin = strings.NewReader("root:" + cfg.RootPassword) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { + if err := setShadowPassword("root", cfg.RootPassword); err != nil { return fmt.Errorf("set root password: %w", err) } - - // Set the spinifex login user password. The spinifex account is the - // default interactive login on the node (autologin on console, SSH - // access). Root SSH is disabled, so this is the sole remote entry point. - cmd = exec.Command("chroot", mountRoot, "chpasswd") - cmd.Stdin = strings.NewReader("spinifex:" + cfg.RootPassword) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - if err := cmd.Run(); err != nil { + // The spinifex account is the default interactive login on the + // node (console + SSH). Root SSH is disabled, so this is the sole + // remote entry point. The user itself is created at rootfs build + // time (build-rootfs.sh) — here we just set its password. + if err := setShadowPassword("spinifex", cfg.RootPassword); err != nil { return fmt.Errorf("set spinifex password: %w", err) } } @@ -250,6 +299,13 @@ func installSpinifex(cfg *Config) error { return err } + // Mask dhcpcd.service so only ifupdown controls DHCP on the bridges. + // Trixie's dhcpcd-base ships a standalone dhcpcd.service that races with + // ifupdown's own dhcpcd invocations, causing duplicate leases or IP flapping. + if err := maskSystemdUnit(mountRoot, "dhcpcd.service"); err != nil { + slog.Warn("installSpinifex: failed to mask dhcpcd.service", "err", err) + } + return systemd.WriteNetworkingDropIn(mountRoot) } @@ -416,24 +472,11 @@ GRUB_BACKGROUND=/boot/grub/splash.png } // update-grub (grub-mkconfig) runs inside the installed system's chroot and - // needs /dev, /proc, and /sys to probe devices. Bind-mount them in and - // unmount regardless of outcome. - chrootMounts := []string{"dev", "proc", "sys"} - for _, m := range chrootMounts { - dst := filepath.Join(mountRoot, m) - if err := os.MkdirAll(dst, 0o755); err != nil { - return fmt.Errorf("create chroot mountpoint /%s: %w", m, err) - } - if err := run("mount", "--bind", "/"+m, dst); err != nil { - return fmt.Errorf("bind-mount /%s into chroot: %w", m, err) - } + // needs /dev, /proc, and /sys to probe devices. + if err := bindChrootMounts(); err != nil { + return err } - defer func() { - // Unmount in reverse order; ignore errors (best-effort cleanup). - for i := len(chrootMounts) - 1; i >= 0; i-- { - _ = run("umount", filepath.Join(mountRoot, chrootMounts[i])) - } - }() + defer unbindChrootMounts() return run("chroot", mountRoot, "update-grub") } @@ -445,6 +488,10 @@ func installCACert(cfg *Config) error { if err := os.WriteFile(certPath, []byte(cfg.CACert), 0o644); err != nil { return err } + if err := bindChrootMounts(); err != nil { + return err + } + defer unbindChrootMounts() return run("chroot", mountRoot, "update-ca-certificates") } @@ -563,6 +610,54 @@ func copySplashImage(root string) { } } +// maskSystemdUnit creates a symlink to /dev/null for the given unit, which is +// the standard way to permanently disable a unit so systemd will never start it. +func maskSystemdUnit(root, unit string) error { + unitPath := filepath.Join(root, "etc/systemd/system", unit) + _ = os.Remove(unitPath) + return os.Symlink("/dev/null", unitPath) +} + +// setShadowPassword sets a Unix password on the installed system without +// going through PAM. See the long comment in installSpinifex for the +// rationale. +func setShadowPassword(user, password string) error { + cmd := exec.Command("chpasswd", "-c", "YESCRYPT", "-R", mountRoot) + cmd.Stdin = strings.NewReader(user + ":" + password) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +// chrootMountPaths lists virtual filesystems to bind-mount into the chroot. +// Order matters: unbind in reverse. +var chrootMountPaths = []string{"dev", "proc", "sys"} + +// bindChrootMounts bind-mounts /dev, /proc, and /sys into the installed rootfs +// so chroot commands (chpasswd, systemd-machine-id-setup, update-grub) can +// access hardware, process info, and entropy sources. Idempotent — already- +// mounted paths are skipped. +func bindChrootMounts() error { + for _, m := range chrootMountPaths { + dst := filepath.Join(mountRoot, m) + if err := os.MkdirAll(dst, 0o755); err != nil { + return fmt.Errorf("create chroot mountpoint /%s: %w", m, err) + } + if err := run("mount", "--bind", "/"+m, dst); err != nil { + return fmt.Errorf("bind-mount /%s into chroot: %w", m, err) + } + } + return nil +} + +// unbindChrootMounts unmounts the virtual filesystems in reverse order. +// Errors are logged but not returned — this is best-effort cleanup. +func unbindChrootMounts() { + for i := len(chrootMountPaths) - 1; i >= 0; i-- { + _ = run("umount", filepath.Join(mountRoot, chrootMountPaths[i])) + } +} + func run(name string, args ...string) error { cmd := exec.Command(name, args...) cmd.Stdout = os.Stdout diff --git a/cmd/installer/main.go b/cmd/installer/main.go index a5610f370..469506342 100644 --- a/cmd/installer/main.go +++ b/cmd/installer/main.go @@ -58,10 +58,19 @@ func main() { } // detectTTY returns the TTY device path the installer should attach to. -// It reads SPINIFEX_CONSOLE from the kernel cmdline (/proc/cmdline) so that -// spinifex-init can direct the installer to the correct console (ttyS0 for -// serial, tty1 for VGA). Falls back to tty1. +// It first checks the SPINIFEX_CONSOLE environment variable (set by +// spinifex-init.sh after console fallback logic), then falls back to parsing +// /proc/cmdline. spinifex-init.sh may override the GRUB-set value (e.g. when +// GRUB says tty1 but no framebuffer exists, it falls back to ttyS0 and +// exports the corrected value). func detectTTY() string { + // Prefer the env var — spinifex-init.sh sets this after resolving the + // actual console (including headless/framebuffer fallback). + if val := os.Getenv("SPINIFEX_CONSOLE"); val != "" && val != "auto" { + return "/dev/" + val + } + + // Fall back to kernel cmdline for direct invocation without spinifex-init. data, err := os.ReadFile("/proc/cmdline") if err == nil { for param := range strings.FieldsSeq(string(data)) { diff --git a/docs/install-vm-eval/README.md b/docs/install-vm-eval/README.md new file mode 100644 index 000000000..361d88fb4 --- /dev/null +++ b/docs/install-vm-eval/README.md @@ -0,0 +1,301 @@ +# Installing Spinifex in a VM (MacOS) + +For evaluation / quick POC use only. + +Spinifex is designed for bare-metal hardware, edge nodes and data-centre use. However, follow this guide to install Spinifex on a MacOS VM to demonstrate the capabilities (dev/testing only). + +Note this tutorial is for Apple Silicon (ARM64) + +## UTM installation + +For this tutorial download UTM.app on Mac to simplify the VM installation. + +* UTM - [https://mac.getutm.app](https://mac.getutm.app) + + + +## Download Debian ISO + +Next, download the Debian ISO (ARM64, Apple Silicon) + +* Debian 13.4 - [https://cdimage.debian.org/debian-cd/current/arm64/iso-cd/debian-13.4.0-arm64-netinst.iso](https://cdimage.debian.org/debian-cd/current/arm64/iso-cd/debian-13.4.0-arm64-netinst.iso) + +## Login + +Using the VM console, login using the `spinifex` user and your specified password. A few additional steps are required via the console, before you can SSH into the machine for easier administration. + +## Install prerequisites + +For a minimal Debian install additional dependencies are required prior to the Spinifex installation: + +``` +su root +apt update +apt install sudo curl bridge-utils +``` + +### Enable sudoers + +Enable sudo for `spinifex` user as the root user: + +``` +su root +echo "spinifex ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/spinifex +``` + +## Networking + +Spinifex requires a Linux bridge for networking to correctly function, given EC2 instances and resources can obtain an IP address from the upstream DHCP server or a static public IP address range you specify. + +This is a requirement step for valid installation. + +View the existing `/etc/network/interfaces` for the network configuration. + +```sh +# This file describes the network interfaces available on your system +# and how to activate them. For more information, see interfaces(5). + +source /etc/network/interfaces.d/* + +# The loopback network interface +auto lo +iface lo inet loopback + +# The primary network interface +allow-hotplug enp0s1 +iface enp0s1 inet dhcp +# This is an autoconfigured IPv6 interface +iface enp0s1 inet6 auto +``` + +Edit `/etc/network/interfaces` note replace `enp0s1` with your network adapter. + +```sh +# This file describes the network interfaces available on your system +# and how to activate them. For more information, see interfaces(5). + +source /etc/network/interfaces.d/* + +# The loopback network interface +auto lo +iface lo inet loopback + +# The primary network interface - toggle to manual +allow-hotplug enp0s1 +iface enp0s1 inet manual +# This is an autoconfigured IPv6 interface - toggle to manual +iface enp0s1 inet6 manual + +# Spinifex Config +# WAN bridge (Spinifex EC2 instances connect here) +# VMs (EC2) / containers <-> br-wan <-> enp0s1 <-> external network +auto br-wan +iface br-wan inet dhcp + bridge_ports enp0s1 + bridge_stp off + bridge_fd 0 + bridge_maxwait 0 +``` + +Walkthrough of changes: + +* Adds eno3 into the bridge +* eno3 becomes a slave interface +* All traffic goes through br-wan +* Spinifex EC2 (VMs) in a public subnet, obtain an IP address from the WAN upstream + +### Restart networking + +Make sure you are logged into the console, not via SSH for these steps. Otherwise you will lose networking to the instance. + +Replace `enp0s1` with your network adapter. + +```bash +ifdown enp0s1 +systemctl restart networking +``` + +Next, retrieve the IP address of the new `br-wan` bridge from your upstream DHCP server. + +```bash +ip a show dev enp0s1 +``` + +```bash +6: br-wan: mtu 1500 qdisc noqueue state UP group default qlen 1000 + link/ether 12:c2:bc:f3:f6:7b brd ff:ff:ff:ff:ff:ff + inet 192.168.0.249/24 brd 192.168.0.255 scope global dynamic noprefixroute br-wan + valid_lft 86103sec preferred_lft 75303sec + inet6 fd0d:9d60:d09b:0:ee77:630d:3237:9a10/64 scope global dynamic mngtmpaddr noprefixroute + valid_lft 86282sec preferred_lft 86282sec + inet6 fe80::85c0:f2aa:f0bf:e951/64 scope link + valid_lft forever preferred_lft forever +``` + +Note the IP `192.168.0.249` + +# Spinifex Installation + +## Launch installer + +Next, optionally SSH into your instance (using the IP above) or continue using the console. + +Replace with your IP below. + +``` +ssh spinifex@ +``` + +As the `spinifex` user launch the binary installer: + +```bash +su spinifex +curl -fsSL https://install.mulgadc.com | bash +``` + +```bash +[INFO] Spinifex installer + +[INFO] Detected OS: Debian GNU/Linux 13 (trixie) +[INFO] Detected architecture: aarch64 (arm64) +[INFO] Installing system dependencies... +[INFO] System dependencies installed +[INFO] Installing AWS CLI v2... +[INFO] AWS CLI installed: aws-cli/2.34.30 Python/3.14.4 Linux/6.12.74+deb13+1-arm64 exe/aarch64.debian.13 +[INFO] Service users created (spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui}) +/etc/sudoers.d/spinifex-network: parsed OK +[INFO] Scoped sudoers rules installed for spinifex-daemon and spinifex-vpcd +[INFO] Downloading Spinifex (latest channel, arm64)... +[INFO] Verifying checksum... +[INFO] Checksum verified +[INFO] Extracting... +[INFO] Installing files... +[INFO] /usr/local/bin/spx +[INFO] /usr/lib/aarch64-linux-gnu/nbdkit/plugins/nbdkit-viperblock-plugin.so +[INFO] /usr/local/share/spinifex/setup-ovn.sh +[INFO] Creating directories... +[INFO] Generated /etc/spinifex/systemd.env +[INFO] /var/lib/spinifex/predastore-start.sh +[INFO] /var/lib/spinifex/wait-for-nats.sh +[INFO] Fixing file ownership for privilege separation... +[INFO] File ownership updated +[INFO] Installing systemd units... +[INFO] /etc/systemd/system/spinifex-awsgw.service +[INFO] /etc/systemd/system/spinifex-daemon.service +[INFO] /etc/systemd/system/spinifex-nats.service +[INFO] /etc/systemd/system/spinifex-predastore.service +[INFO] /etc/systemd/system/spinifex.target +[INFO] /etc/systemd/system/spinifex-ui.service +[INFO] /etc/systemd/system/spinifex-viperblock.service +[INFO] /etc/systemd/system/spinifex-vpcd.service +Created symlink '/etc/systemd/system/multi-user.target.wants/spinifex.target' → '/etc/systemd/system/spinifex.target'. +[INFO] Systemd units installed and enabled (per-service users) +[INFO] Fresh install detected, skipping migrations +[INFO] Logrotate config installed + +============================================ + Spinifex installed successfully +============================================ + + Version: spinifex v1.1.0 (b7b6390) linux/arm64 + Architecture: arm64 + Service users: spinifex-{nats,gw,daemon,storage,viperblock,vpcd,ui} + Binary: /usr/local/bin/spx + Config: /etc/spinifex/ + Data: /var/lib/spinifex/ + Logs: /var/log/spinifex/ +``` + +## Setup OVN networking + +Since the WAN interface is already a bridge (e.g. br-wan) run the Openvswitch (OVN) install script: + +```bash +sudo /usr/local/share/spinifex/setup-ovn.sh --management +``` + +Output: + +```bash +Auto-detected Linux bridge: br-wan (default route) + Will create OVS bridge br-ext + veth pair to link them +Auto-detected encap IP: 192.168.0.249 +Auto-detected chassis ID: chassis-spinifex +=== Spinifex OVN Compute Node Setup === + Management node: true + WAN bridge: br-ext (veth) + Linux bridge: br-wan (linked via veth pair) + OVN Remote (SB): tcp:127.0.0.1:6642 + Encap IP: 192.168.0.249 + Chassis ID: chassis-spinifex +... +``` + +## Spinifex Initialize + +```bash +sudo spx admin init --node node1 --nodes 1 +``` + +## Start services + +```bash +sudo systemctl start spinifex.target +``` + +## Verify installation + +Connect to the local AWS gateway and issue a sample command using the AWS CLI tool, all running on your local infrastructure, no cloud required. + +```bash +export AWS_PROFILE=spinifex +aws ec2 describe-instance-types +``` + +> ***Note:*** The system may require ~30 seconds to initalise prior to accepting connections for the command to complete above. + +Output: + +```json +{ + "InstanceTypes": [ + { + "InstanceType": "t4g.nano", + "CurrentGeneration": true, + "SupportedRootDeviceTypes": [ + "ebs" + ], + "SupportedVirtualizationTypes": [ + "hvm" + ], + "Hypervisor": "kvm", + "ProcessorInfo": { + "SupportedArchitectures": [ + "arm64" + ] + }, + "VCpuInfo": { + "DefaultVCpus": 2 + }, + "MemoryInfo": { + "SizeInMiB": 512 + }, + "PlacementGroupInfo": { + "SupportedStrategies": [ + "cluster", + "spread" + ] + }, + "BurstablePerformanceSupported": true + }, + ] +} +``` + +# Installation Complete + +Congratulations! 🥳 + +Your installation of Spinifex is now complete on a VM running on a Mac. + +# Next steps