diff --git a/Makefile b/Makefile index 7f8eb7012..b1873a661 100644 --- a/Makefile +++ b/Makefile @@ -296,9 +296,30 @@ ansible-dev-reset: ansible-dev-deploy: cd scripts/ansible && ansible-playbook playbooks/dev-deploy.yml -.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race test-actions test-harness manifest-check diff-coverage bench run \ +ansible-dev-status: + cd scripts/ansible && ansible-playbook playbooks/dev-status.yml + +ansible-dev-logs: + cd scripts/ansible && ansible-playbook playbooks/dev-logs.yml + +# Snapshot / restore require an explicit -e snapshot_name=. The +# wrapper passes ANSIBLE_EXTRA through so devs can run +# `make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge'`. +ansible-dev-snapshot: + cd scripts/ansible && ansible-playbook playbooks/dev-snapshot.yml $(ANSIBLE_EXTRA) + +ansible-dev-restore: + cd scripts/ansible && ansible-playbook playbooks/dev-restore.yml $(ANSIBLE_EXTRA) + +ansible-dev-version: + cd scripts/ansible && ansible-playbook playbooks/dev-version.yml + +ansible-dev-vpc: + cd scripts/ansible && ansible-playbook playbooks/dev-vpc.yml $(ANSIBLE_EXTRA) + +.PHONY: build build-ui build-installer build-lb-agent build-system-image build-microvm-image install-microvm go_build go_run preflight test test-cover test-race diff-coverage bench run test-actions test-harness manifest-check diff-coverage bench run \ deploy reinstall clean \ install-system install-go install-aws quickinstall \ lint fix govulncheck \ distro distro-amd64 distro-arm64 distro-clean \ - ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy + ansible-dev-preflight ansible-dev-teardown ansible-dev-install ansible-dev-reset ansible-dev-deploy ansible-dev-status ansible-dev-logs ansible-dev-snapshot ansible-dev-restore ansible-dev-version ansible-dev-vpc diff --git a/scripts/ansible/README.md b/scripts/ansible/README.md index 7e3ae7c86..2b53c367f 100644 --- a/scripts/ansible/README.md +++ b/scripts/ansible/README.md @@ -32,6 +32,12 @@ inventory and roles relative to CWD. | `playbooks/dev-install.yml` | Build + install + init + smoketest on clean box | `dev-install.sh` | | `playbooks/dev-reset.yml` | Capture settings → teardown → build → install → init → smoketest | `reset-dev-env.sh` (full) | | `playbooks/dev-deploy.yml` | Rebuild + swap binaries/microvm artifacts + restart (no setup.sh, no smoketest) | `make deploy` | +| `playbooks/dev-status.yml` | Read-only health snapshot (units, ports, OVN/OVS, gateway, counts, drift) | none | +| `playbooks/dev-logs.yml` | Dump journald + `/var/log/spinifex/*` + status/networking/versions into a tarball | none | +| `playbooks/dev-snapshot.yml` | Quiesce spinifex+OVN+OVS, tar state paths to a named bundle, restart | none | +| `playbooks/dev-restore.yml` | Stop services, wipe + untar a named bundle, refresh CA, restart | none | +| `playbooks/dev-version.yml` | Build-vs-install drift report (spx, plugin, microvm) + sub-repo HEADs | none | +| `playbooks/dev-vpc.yml` | Idempotent dev VPC fixture (VPC + public subnet + IGW + default route); `-e dev_vpc_state=absent` tears down | partial overlap with `scripts/iac/examples/*` | Upcoming (not yet implemented): @@ -46,6 +52,12 @@ ansible-playbook playbooks/dev-teardown.yml ansible-playbook playbooks/dev-install.yml ansible-playbook playbooks/dev-reset.yml ansible-playbook playbooks/dev-deploy.yml +ansible-playbook playbooks/dev-status.yml +ansible-playbook playbooks/dev-logs.yml +ansible-playbook playbooks/dev-snapshot.yml -e snapshot_name=before-merge +ansible-playbook playbooks/dev-restore.yml -e snapshot_name=before-merge +ansible-playbook playbooks/dev-version.yml +ansible-playbook playbooks/dev-vpc.yml ``` Or via `make` (from `spinifex/`): @@ -56,6 +68,13 @@ make ansible-dev-teardown make ansible-dev-install make ansible-dev-reset make ansible-dev-deploy +make ansible-dev-status +make ansible-dev-logs +make ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name=before-merge' +make ansible-dev-restore ANSIBLE_EXTRA='-e snapshot_name=before-merge' +make ansible-dev-version +make ansible-dev-vpc +make ansible-dev-vpc ANSIBLE_EXTRA='-e dev_vpc_state=absent' ``` ### When to use which @@ -64,6 +83,11 @@ make ansible-dev-deploy - Iterate on Go code, microvm initramfs, lb-agent → `ansible-dev-deploy` (fast) - Changed systemd units, helper scripts, logrotate, setup.sh → `ansible-dev-reset` (slow, full rebuild) - Need a clean slate without reinstall → `ansible-dev-teardown` +- "Is my dev box healthy?" → `ansible-dev-status` (read-only, never mutates) +- Filing a bug / capturing the state of a broken box → `ansible-dev-logs` (writes `/tmp/spinifex-logs--.tar.gz`; override `-e logs_since=10min` or `-e logs_include_dmesg=false`) +- Checkpoint before a risky op (branch switch, viperblock surgery, ovn restart cycle) → `ansible-dev-snapshot ANSIBLE_EXTRA='-e snapshot_name='`; roll back via `ansible-dev-restore` instead of a full `dev-reset`. Bundles live in `/var/lib/spinifex-snapshots/.tar.gz` (root-owned, survives `dev-teardown`). +- "Do I need to `ansible-dev-deploy`?" → `ansible-dev-version` (drift-only check; faster than `dev-status`) +- Want a known-good VPC/subnet for manual EC2 testing without writing tofu → `ansible-dev-vpc` (smoketest will then find and use it automatically). Tear down with `ANSIBLE_EXTRA='-e dev_vpc_state=absent'`. ## Variable overrides @@ -90,6 +114,23 @@ pool range, gateway, prefix-len, nat gateway_ip from the existing `/etc/spinifex/spinifex.toml` before teardown and replays them into `init`. Use `-e` to override. +## Helper scripts + +`scripts/aws-as-spinifex.sh` and `scripts/spx-as-spinifex.sh` wrap `aws` +and `spx` invocations in `sg spinifex -c '...'` so they run with the +`spinifex` supplementary group active. Both helpers preset +`AWS_PROFILE=spinifex`. All AWS-calling tasks in the roles use these +via `spinifex_aws_cli` / `spinifex_spx_cli` from `vars/defaults.yml`. + +Background: `/etc/spinifex/ca.pem` lives under `drwxr-x--- root:spinifex`, +so the AWS CLI needs `spinifex` group membership to read the CA bundle. +Ansible's `become_user` is a no-op when the target user equals the +calling user (no `sudo -u` invoked → no `initgroups(3)` → supplementary +groups inherited from the operator's shell). On a fresh login the +operator's shell would have `spinifex` (added by `setup.sh`); on shells +that predate the group add it does not. The helpers sidestep this +entirely. See bead `mulga-siv-92`. + ## Leak catalog `roles/teardown/README.md` tracks every known state source. When teardown diff --git a/scripts/ansible/playbooks/dev-logs.yml b/scripts/ansible/playbooks/dev-logs.yml new file mode 100644 index 000000000..2461fd749 --- /dev/null +++ b/scripts/ansible/playbooks/dev-logs.yml @@ -0,0 +1,17 @@ +--- +# Capture spinifex + ovn/ovs + qemu/nbdkit logs from journald and +# /var/log/spinifex/ into a single tarball for bug reports. +# Tail mode is intentionally not provided — use `journalctl -u 'spinifex-*' -f` +# directly; ansible is the wrong tool for follow. +- name: Dev logs — dump journald + varlog + status into a bundle + hosts: localhost + gather_facts: true + vars_files: + - ../vars/defaults.yml + vars: + logs_since: "1h" + logs_output_dir: "/tmp" + logs_include_dmesg: true + + roles: + - logs diff --git a/scripts/ansible/playbooks/dev-restore.yml b/scripts/ansible/playbooks/dev-restore.yml new file mode 100644 index 000000000..1742cd4ef --- /dev/null +++ b/scripts/ansible/playbooks/dev-restore.yml @@ -0,0 +1,14 @@ +--- +# Restore a dev-snapshot bundle: stop services, wipe the snapshot path +# set, untar the bundle, refresh CA trust, restart. +# +# Required: -e snapshot_name= +# Optional: -e snapshot_dir=/some/path +- name: Dev restore — wipe + untar snapshot + restart + hosts: localhost + gather_facts: true + vars_files: + - ../vars/defaults.yml + + roles: + - restore diff --git a/scripts/ansible/playbooks/dev-snapshot.yml b/scripts/ansible/playbooks/dev-snapshot.yml new file mode 100644 index 000000000..04875e073 --- /dev/null +++ b/scripts/ansible/playbooks/dev-snapshot.yml @@ -0,0 +1,17 @@ +--- +# Quiesce spinifex + ovn + ovs, tar /etc/spinifex + /var/lib/spinifex + +# /var/lib/ovn + /etc/openvswitch/conf.db into a named bundle, restart. +# Use to checkpoint a working state before a risky operation (branch +# switch, viperblock surgery, ovn-controller restart cycle). Roll back +# via dev-restore without a full dev-reset. +# +# Required: -e snapshot_name= +# Optional: -e snapshot_dir=/some/path +- name: Dev snapshot — quiesce + tar state + restart + hosts: localhost + gather_facts: true + vars_files: + - ../vars/defaults.yml + + roles: + - snapshot diff --git a/scripts/ansible/playbooks/dev-status.yml b/scripts/ansible/playbooks/dev-status.yml new file mode 100644 index 000000000..45a4e2d1e --- /dev/null +++ b/scripts/ansible/playbooks/dev-status.yml @@ -0,0 +1,13 @@ +--- +# Read-only health snapshot for a Spinifex dev node. Never mutates state. +# Run any time the operator wants to know "is my box healthy?" — pairs with +# dev-preflight (run before install) as the post-install / running-system +# diagnostic. +- name: Dev status — health snapshot (read-only) + hosts: localhost + gather_facts: true + vars_files: + - ../vars/defaults.yml + + roles: + - status diff --git a/scripts/ansible/playbooks/dev-version.yml b/scripts/ansible/playbooks/dev-version.yml new file mode 100644 index 000000000..8698af643 --- /dev/null +++ b/scripts/ansible/playbooks/dev-version.yml @@ -0,0 +1,11 @@ +--- +# Detailed build-vs-install drift report. Faster than dev-status (no AWS, +# OVN, or port probing). Answers "do I need to ansible-dev-deploy?". +- name: Dev version — build vs install drift (read-only) + hosts: localhost + gather_facts: false + vars_files: + - ../vars/defaults.yml + + roles: + - version diff --git a/scripts/ansible/playbooks/dev-vpc.yml b/scripts/ansible/playbooks/dev-vpc.yml new file mode 100644 index 000000000..3981dbb9b --- /dev/null +++ b/scripts/ansible/playbooks/dev-vpc.yml @@ -0,0 +1,14 @@ +--- +# Idempotent dev-mode VPC fixture: spinifex-dev-vpc + public subnet + +# IGW + default route. Hooks into smoketest's launch path (which finds +# the public subnet automatically). Re-runs are no-ops once converged. +# +# Default state: present. Tear down with -e dev_vpc_state=absent. +- name: Dev VPC — fixture VPC/subnet/IGW/route (idempotent) + hosts: localhost + gather_facts: false + vars_files: + - ../vars/defaults.yml + + roles: + - vpc diff --git a/scripts/ansible/roles/deploy/meta/main.yml b/scripts/ansible/roles/deploy/meta/main.yml new file mode 100644 index 000000000..8e4aa0695 --- /dev/null +++ b/scripts/ansible/roles/deploy/meta/main.yml @@ -0,0 +1,7 @@ +--- +galaxy_info: + role_name: deploy + description: In-place binary + microvm artifact swap for a running Spinifex node (parity with `make deploy`). + license: BSD-3-Clause + min_ansible_version: "2.14" +dependencies: [] diff --git a/scripts/ansible/roles/deploy/tasks/main.yml b/scripts/ansible/roles/deploy/tasks/main.yml index 3c24d890d..1153b7faf 100644 --- a/scripts/ansible/roles/deploy/tasks/main.yml +++ b/scripts/ansible/roles/deploy/tasks/main.yml @@ -127,11 +127,7 @@ state: started - name: deploy — wait for awsgw to respond - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex - ansible.builtin.command: aws ec2 describe-regions + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions" register: deploy_awsgw retries: 15 delay: 2 diff --git a/scripts/ansible/roles/init/tasks/main.yml b/scripts/ansible/roles/init/tasks/main.yml index 6e2b91311..ff3f48b52 100644 --- a/scripts/ansible/roles/init/tasks/main.yml +++ b/scripts/ansible/roles/init/tasks/main.yml @@ -63,11 +63,7 @@ # to NATS + daemons after that. Bash start-dev.sh has internal wait loops; # systemd path does not. Poll a benign call until gateway is fully ready. - name: init — wait for AWS gateway readiness - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex - ansible.builtin.command: aws ec2 describe-regions + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions" register: init_gw_ready until: init_gw_ready.rc == 0 retries: 30 diff --git a/scripts/ansible/roles/logs/meta/main.yml b/scripts/ansible/roles/logs/meta/main.yml new file mode 100644 index 000000000..72b3aaa47 --- /dev/null +++ b/scripts/ansible/roles/logs/meta/main.yml @@ -0,0 +1,7 @@ +--- +galaxy_info: + role_name: logs + description: Dump spinifex + ovn + system logs into a tarball for bug reports. + license: BSD-3-Clause + min_ansible_version: "2.14" +dependencies: [] diff --git a/scripts/ansible/roles/logs/tasks/main.yml b/scripts/ansible/roles/logs/tasks/main.yml new file mode 100644 index 000000000..38cea5572 --- /dev/null +++ b/scripts/ansible/roles/logs/tasks/main.yml @@ -0,0 +1,153 @@ +--- +# Collect a self-contained log bundle for bug reports. NEVER mutates running +# state — only reads journalctl + /var/log/spinifex/* + status output, then +# tars into logs_output_dir. Errors on missing inputs are non-fatal so a +# half-broken node still produces a bundle. + +- name: logs — resolve staging directory + ansible.builtin.set_fact: + logs_stage_dir: "/tmp/spinifex-logs-{{ ansible_facts['hostname'] }}-{{ ansible_facts['date_time']['iso8601_basic_short'] }}" + logs_bundle_path: "{{ logs_output_dir }}/spinifex-logs-{{ ansible_facts['hostname'] }}-{{ ansible_facts['date_time']['iso8601_basic_short'] }}.tar.gz" + +- name: logs — create staging tree + ansible.builtin.file: + path: "{{ logs_stage_dir }}/{{ item }}" + state: directory + mode: "0755" + loop: + - journal + - varlog + - status + - kernel + +- name: logs — capture journalctl for spinifex-* units + become: true + ansible.builtin.shell: | + journalctl --no-pager --output=short-iso \ + -u 'spinifex-*' --since {{ logs_since | quote }} \ + > {{ logs_stage_dir }}/journal/spinifex.log 2>&1 || true + changed_when: true + +- name: logs — capture journalctl for ovn-* + openvswitch-switch + become: true + ansible.builtin.shell: | + journalctl --no-pager --output=short-iso \ + -u 'ovn-*' -u openvswitch-switch \ + --since {{ logs_since | quote }} \ + > {{ logs_stage_dir }}/journal/ovn-ovs.log 2>&1 || true + changed_when: true + +- name: logs — capture journalctl for nbdkit + qemu (best-effort match) + become: true + ansible.builtin.shell: | + journalctl --no-pager --output=short-iso \ + --since {{ logs_since | quote }} \ + _COMM=qemu-system-x86_64 + _COMM=qemu-system-aarch64 + _COMM=nbdkit \ + > {{ logs_stage_dir }}/journal/qemu-nbdkit.log 2>&1 || true + changed_when: true + +- name: logs — copy /var/log/spinifex/* + become: true + ansible.builtin.shell: | + if [ -d {{ spinifex_log_dir | quote }} ]; then + cp -a {{ spinifex_log_dir | quote }}/. {{ logs_stage_dir | quote }}/varlog/ 2>/dev/null || true + fi + changed_when: true + +- name: logs — systemctl status snapshot (spinifex-*) + become: true + ansible.builtin.shell: | + systemctl status 'spinifex-*' --no-pager --all \ + > {{ logs_stage_dir }}/status/spinifex-units.txt 2>&1 || true + systemctl status ovn-controller ovn-central openvswitch-switch \ + --no-pager > {{ logs_stage_dir }}/status/ovn-ovs-units.txt 2>&1 || true + systemctl list-units 'spinifex-*' --all --no-legend --plain \ + > {{ logs_stage_dir }}/status/spinifex-list.txt 2>&1 || true + changed_when: true + +- name: logs — networking snapshot (ip, ss, ovs) + become: true + ansible.builtin.shell: | + ip -d -s link show > {{ logs_stage_dir }}/status/ip-link.txt 2>&1 || true + ip addr show > {{ logs_stage_dir }}/status/ip-addr.txt 2>&1 || true + ip route show table all > {{ logs_stage_dir }}/status/ip-route.txt 2>&1 || true + ss -tlnp > {{ logs_stage_dir }}/status/ss-tlnp.txt 2>&1 || true + if command -v ovs-vsctl >/dev/null 2>&1; then + ovs-vsctl show > {{ logs_stage_dir }}/status/ovs-show.txt 2>&1 || true + ovs-vsctl list-br > {{ logs_stage_dir }}/status/ovs-br.txt 2>&1 || true + fi + if command -v ovn-nbctl >/dev/null 2>&1; then + ovn-nbctl show > {{ logs_stage_dir }}/status/ovn-nb.txt 2>&1 || true + ovn-sbctl show > {{ logs_stage_dir }}/status/ovn-sb.txt 2>&1 || true + fi + changed_when: true + +- name: logs — kernel ring buffer + become: true + ansible.builtin.shell: | + dmesg --time-format=iso > {{ logs_stage_dir }}/kernel/dmesg.txt 2>&1 || true + changed_when: true + when: logs_include_dmesg | bool + +- name: logs — capture spx + plugin versions + become: true + ansible.builtin.shell: | + { + echo '## spx version' + /usr/local/bin/spx version 2>&1 | grep -vE '^[0-9]{4}/' || true + echo + echo '## nbdkit plugin sha256' + pdir=$(nbdkit --dump-config 2>/dev/null | awk -F= '/^plugindir=/{print $2; exit}') + [ -n "$pdir" ] && sha256sum "$pdir/nbdkit-viperblock-plugin.so" 2>/dev/null || true + echo + echo '## microvm artifacts' + sha256sum /usr/share/spinifex/microvm/* 2>/dev/null || true + echo + echo '## kernel + os' + uname -a + cat /etc/os-release 2>/dev/null || true + } > {{ logs_stage_dir }}/status/versions.txt 2>&1 || true + changed_when: true + +- name: logs — create bundle tarball + become: true + ansible.builtin.shell: | + tar -C {{ logs_stage_dir | dirname | quote }} \ + -czf {{ logs_bundle_path | quote }} \ + {{ logs_stage_dir | basename | quote }} + changed_when: true + +- name: logs — make bundle owned by invoking user + become: true + ansible.builtin.file: + path: "{{ logs_bundle_path }}" + owner: "{{ spinifex_invoking_user }}" + group: "{{ spinifex_invoking_user }}" + mode: "0640" + +- name: logs — remove staging tree + become: true + ansible.builtin.file: + path: "{{ logs_stage_dir }}" + state: absent + +- name: logs — bundle size + ansible.builtin.stat: + path: "{{ logs_bundle_path }}" + register: logs_bundle_stat + +- name: logs — summary + ansible.builtin.debug: + msg: | + Log bundle written: + path : {{ logs_bundle_path }} + size : {{ (logs_bundle_stat.stat.size | int / 1024) | round(1) }} KiB + since : {{ logs_since }} + Contents: + journal/ spinifex-*, ovn-*+openvswitch-switch, qemu/nbdkit + varlog/ copy of {{ spinifex_log_dir }} + status/ systemctl status, ip/ss/ovs/ovn show, versions + kernel/ dmesg{{ '' if logs_include_dmesg else ' (skipped via -e logs_include_dmesg=false)' }} + + Attach to bug report. For live tail use: + journalctl -u 'spinifex-*' -f diff --git a/scripts/ansible/roles/restore/meta/main.yml b/scripts/ansible/roles/restore/meta/main.yml new file mode 100644 index 000000000..30285dc4d --- /dev/null +++ b/scripts/ansible/roles/restore/meta/main.yml @@ -0,0 +1,7 @@ +--- +galaxy_info: + role_name: restore + description: Restore a spinifex+ovn+ovs snapshot bundle, refresh CA trust, restart. + license: BSD-3-Clause + min_ansible_version: "2.14" +dependencies: [] diff --git a/scripts/ansible/roles/restore/tasks/main.yml b/scripts/ansible/roles/restore/tasks/main.yml new file mode 100644 index 000000000..862e39a4c --- /dev/null +++ b/scripts/ansible/roles/restore/tasks/main.yml @@ -0,0 +1,140 @@ +--- +# Inverse of the snapshot role: stop services, wipe the same path set, +# untar the named bundle, refresh CA trust (because /etc/spinifex/ca.pem +# came from the snapshot but the system trust store didn't), restart. + +- name: restore — require snapshot_name + ansible.builtin.assert: + that: + - snapshot_name is defined + - snapshot_name | length > 0 + - snapshot_name is match('^[A-Za-z0-9._-]+$') + fail_msg: "snapshot_name required (alphanumeric/._-): -e snapshot_name=" + +- name: restore — resolve bundle path + ansible.builtin.set_fact: + snapshot_bundle_path: "{{ snapshot_dir }}/{{ snapshot_name }}.tar.gz" + +- name: restore — assert bundle exists + become: true + ansible.builtin.stat: + path: "{{ snapshot_bundle_path }}" + register: restore_bundle_stat + +- name: restore — fail if missing + ansible.builtin.assert: + that: restore_bundle_stat.stat.exists + fail_msg: | + Snapshot bundle missing: {{ snapshot_bundle_path }}. + Available bundles: + ls -lh {{ snapshot_dir }}/*.tar.gz 2>/dev/null + +- name: restore — stop services in dependency order + become: true + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: "{{ snapshot_services_stop_order }}" + failed_when: false + +- name: restore — kill lingering QEMU VMs + become: true + ansible.builtin.command: "pkill -x {{ item }}" + loop: "{{ spinifex_qemu_processes }}" + changed_when: false + failed_when: false + +- name: restore — wait for QEMU exit + become: true + ansible.builtin.shell: | + set -e + for i in $(seq 1 {{ teardown_process_wait_seconds }}); do + alive=0 + for proc in {{ spinifex_qemu_processes | join(' ') }}; do + if pgrep -x "$proc" >/dev/null 2>&1; then alive=1; break; fi + done + if [ "$alive" = "0" ]; then exit 0; fi + sleep 1 + done + exit 1 + changed_when: false + +- name: restore — wipe target paths (only those the snapshot will replace) + become: true + ansible.builtin.file: + path: "{{ item }}" + state: absent + loop: "{{ snapshot_paths }}" + +- name: restore — recreate parent dirs that tar will refill + become: true + ansible.builtin.file: + path: "{{ item }}" + state: directory + owner: root + group: root + mode: "0755" + loop: + - "{{ ovn_db_dir }}" + - /etc/openvswitch + +- name: restore — extract bundle (preserve permissions/ACLs) + become: true + ansible.builtin.command: + cmd: >- + tar --acls --xattrs -xzpf {{ snapshot_bundle_path | quote }} + -C / + changed_when: true + +- name: restore — refresh spinifex CA in system trust + become: true + ansible.builtin.stat: + path: "{{ spinifex_ca_src }}" + register: restore_ca_stat + +- name: restore — copy CA back to trust path + become: true + ansible.builtin.copy: + src: "{{ spinifex_ca_src }}" + dest: "{{ spinifex_ca_trust_path }}" + mode: "0644" + remote_src: true + when: restore_ca_stat.stat.exists + +- name: restore — rebuild CA bundle + become: true + ansible.builtin.command: update-ca-certificates + changed_when: true + when: restore_ca_stat.stat.exists + +- name: restore — daemon-reload (covers any unit file delta in the snapshot) + become: true + ansible.builtin.systemd: + daemon_reload: true + +- name: restore — restart services in reverse order + become: true + ansible.builtin.systemd: + name: "{{ item }}" + state: started + loop: "{{ snapshot_services_stop_order | reverse | list }}" + failed_when: false + +- name: restore — wait for AWS gateway readiness + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions" + register: restore_gw_ready + retries: 30 + delay: 2 + until: restore_gw_ready.rc == 0 + changed_when: false + failed_when: false + +- name: restore — summary + ansible.builtin.debug: + msg: | + Restore complete: + snapshot : {{ snapshot_name }} + bundle : {{ snapshot_bundle_path }} + paths : {{ snapshot_paths | join(', ') }} + CA : {{ 'refreshed' if restore_ca_stat.stat.exists else 'not in bundle — system trust unchanged' }} + gateway : {{ 'ready' if restore_gw_ready.rc == 0 else 'NOT ready after restart — investigate' }} diff --git a/scripts/ansible/roles/smoketest/tasks/main.yml b/scripts/ansible/roles/smoketest/tasks/main.yml index ea0d62719..d944bc36a 100644 --- a/scripts/ansible/roles/smoketest/tasks/main.yml +++ b/scripts/ansible/roles/smoketest/tasks/main.yml @@ -16,12 +16,8 @@ creates: "{{ smoketest_user_home }}/.ssh/spinifex-key" - name: smoketest — import SSH key to spinifex - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex ansible.builtin.command: >- - aws ec2 import-key-pair + {{ spinifex_aws_cli }} ec2 import-key-pair --key-name spinifex-key --public-key-material fileb://{{ smoketest_user_home }}/.ssh/spinifex-key.pub register: smoketest_keypair @@ -51,12 +47,8 @@ register: smoketest_local_img - name: smoketest — import AMI from local file - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex ansible.builtin.command: >- - spx admin images import + {{ spinifex_spx_cli }} admin images import --file {{ smoketest_local_image_path }} --distro {{ spinifex_smoketest_distro }} --version {{ spinifex_smoketest_version }} @@ -75,12 +67,8 @@ when: smoketest_local_img.stat.exists - name: smoketest — import AMI by name (fallback) - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex ansible.builtin.command: >- - spx admin images import + {{ spinifex_spx_cli }} admin images import --name {{ spinifex_smoketest_distro }}-{{ spinifex_smoketest_version }}-{{ smoketest_arch }} register: smoketest_import_remote until: >- @@ -105,11 +93,7 @@ smoketest_instance_type: "{{ 't3a.small' if smoketest_amd.rc == 0 else 't3.small' }}" - name: smoketest — find AMI ID - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex - ansible.builtin.command: aws ec2 describe-images --query "Images[0].ImageId" --output text + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-images --query Images[0].ImageId --output text" register: smoketest_ami until: smoketest_ami.rc == 0 retries: 10 @@ -117,12 +101,8 @@ changed_when: false - name: smoketest — find public subnet (best-effort) - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex ansible.builtin.command: >- - aws ec2 describe-subnets + {{ spinifex_aws_cli }} ec2 describe-subnets --filters Name=map-public-ip-on-launch,Values=true --query Subnets[0].SubnetId --output text @@ -131,11 +111,7 @@ failed_when: false - name: smoketest — fallback to first subnet (best-effort) - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex - ansible.builtin.command: aws ec2 describe-subnets --query "Subnets[0].SubnetId" --output text + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-subnets --query Subnets[0].SubnetId --output text" register: smoketest_subnet_any changed_when: false failed_when: false @@ -161,12 +137,8 @@ when: smoketest_subnet_id | trim == '' - name: smoketest — launch instance - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex ansible.builtin.command: >- - aws ec2 run-instances + {{ spinifex_aws_cli }} ec2 run-instances --image-id {{ smoketest_ami.stdout }} --instance-type {{ smoketest_instance_type }} --key-name spinifex-key @@ -182,12 +154,8 @@ when: smoketest_subnet_id | trim != '' - name: smoketest — wait for instance running (proves launch path works) - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex ansible.builtin.command: >- - aws ec2 describe-instances + {{ spinifex_aws_cli }} ec2 describe-instances --instance-ids {{ smoketest_instance.stdout }} --query Reservations[0].Instances[0].State.Name --output text @@ -203,12 +171,8 @@ - smoketest_instance.stdout | trim != '' - name: smoketest — terminate instance (cleanup) - become: true - become_user: "{{ spinifex_invoking_user }}" - environment: - AWS_PROFILE: spinifex ansible.builtin.command: >- - aws ec2 terminate-instances + {{ spinifex_aws_cli }} ec2 terminate-instances --instance-ids {{ smoketest_instance.stdout }} register: smoketest_terminate until: smoketest_terminate.rc == 0 diff --git a/scripts/ansible/roles/snapshot/meta/main.yml b/scripts/ansible/roles/snapshot/meta/main.yml new file mode 100644 index 000000000..2c415c4a1 --- /dev/null +++ b/scripts/ansible/roles/snapshot/meta/main.yml @@ -0,0 +1,7 @@ +--- +galaxy_info: + role_name: snapshot + description: Quiesce spinifex + ovn + ovs, tar state paths, restart. + license: BSD-3-Clause + min_ansible_version: "2.14" +dependencies: [] diff --git a/scripts/ansible/roles/snapshot/tasks/main.yml b/scripts/ansible/roles/snapshot/tasks/main.yml new file mode 100644 index 000000000..5458c6542 --- /dev/null +++ b/scripts/ansible/roles/snapshot/tasks/main.yml @@ -0,0 +1,154 @@ +--- +# Snapshot a quiesced spinifex node: stop spinifex.target → ovn → ovs in order, +# tar /etc/spinifex + /var/lib/spinifex + /var/lib/ovn + /etc/openvswitch/conf.db, +# restart in reverse. Lets devs checkpoint a working state before risky ops +# (branch switch, viperblock surgery, ovn-controller restart cycle) and roll +# back via dev-restore without a full ~10-min dev-reset. +# +# Caveat: /var/lib/spinifex contains viperblock volumes — snapshot size scales +# with VM disks. Default gzip; switch to a faster compressor by hand if needed. + +- name: snapshot — require snapshot_name + ansible.builtin.assert: + that: + - snapshot_name is defined + - snapshot_name | length > 0 + - snapshot_name is match('^[A-Za-z0-9._-]+$') + fail_msg: "snapshot_name required (alphanumeric/._-): -e snapshot_name=" + +- name: snapshot — resolve bundle path + ansible.builtin.set_fact: + snapshot_bundle_path: "{{ snapshot_dir }}/{{ snapshot_name }}.tar.gz" + +- name: snapshot — refuse to overwrite existing bundle + ansible.builtin.stat: + path: "{{ snapshot_bundle_path }}" + register: snapshot_existing + +- name: snapshot — assert not already present + ansible.builtin.assert: + that: not snapshot_existing.stat.exists + fail_msg: | + Snapshot {{ snapshot_bundle_path }} already exists. Pick another + name, or `sudo rm` the existing bundle first. + +- name: snapshot — ensure snapshot dir exists + become: true + ansible.builtin.file: + path: "{{ snapshot_dir }}" + state: directory + owner: root + group: root + mode: "0750" + +- name: snapshot — stop services in dependency order + become: true + ansible.builtin.systemd: + name: "{{ item }}" + state: stopped + loop: "{{ snapshot_services_stop_order }}" + failed_when: false + +- name: snapshot — kill lingering QEMU VMs + become: true + ansible.builtin.command: "pkill -x {{ item }}" + loop: "{{ spinifex_qemu_processes }}" + changed_when: false + failed_when: false + +- name: snapshot — wait for QEMU exit (up to {{ teardown_process_wait_seconds }}s) + become: true + ansible.builtin.shell: | + set -e + for i in $(seq 1 {{ teardown_process_wait_seconds }}); do + alive=0 + for proc in {{ spinifex_qemu_processes | join(' ') }}; do + if pgrep -x "$proc" >/dev/null 2>&1; then alive=1; break; fi + done + if [ "$alive" = "0" ]; then exit 0; fi + sleep 1 + done + echo "QEMU still running after {{ teardown_process_wait_seconds }}s" >&2 + exit 1 + changed_when: false + +- name: snapshot — verify which paths actually exist (skip missing silently) + ansible.builtin.stat: + path: "{{ item }}" + register: snapshot_path_stats + loop: "{{ snapshot_paths }}" + loop_control: + label: "{{ item }}" + +- name: snapshot — assemble list of present paths + ansible.builtin.set_fact: + snapshot_present_paths: >- + {{ snapshot_path_stats.results + | selectattr('stat.exists') | map(attribute='item') | list }} + +- name: snapshot — assert at least one path present + ansible.builtin.assert: + that: snapshot_present_paths | length > 0 + fail_msg: | + None of the snapshot paths exist: + {{ snapshot_paths | join(', ') }} + Has spinifex ever been installed on this node? + +- name: snapshot — create bundle (tar czf with absolute paths) + become: true + ansible.builtin.command: + cmd: >- + tar --acls --xattrs -czpf {{ snapshot_bundle_path | quote }} + {{ snapshot_present_paths | map('quote') | join(' ') }} + changed_when: true + +- name: snapshot — record manifest inside the bundle dir for traceability + become: true + ansible.builtin.copy: + dest: "{{ snapshot_dir }}/{{ snapshot_name }}.manifest" + mode: "0640" + content: | + snapshot_name: {{ snapshot_name }} + created_at: {{ ansible_facts['date_time']['iso8601'] }} + hostname: {{ ansible_facts['hostname'] }} + paths: + {% for p in snapshot_present_paths %} - {{ p }} + {% endfor %} + services_quiesced: + {% for s in snapshot_services_stop_order %} - {{ s }} + {% endfor %} + +- name: snapshot — restart services in reverse order + become: true + ansible.builtin.systemd: + name: "{{ item }}" + state: started + loop: "{{ snapshot_services_stop_order | reverse | list }}" + failed_when: false + +- name: snapshot — wait for AWS gateway readiness + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions" + register: snapshot_gw_ready + retries: 30 + delay: 2 + until: snapshot_gw_ready.rc == 0 + changed_when: false + failed_when: false + +- name: snapshot — bundle size + become: true + ansible.builtin.stat: + path: "{{ snapshot_bundle_path }}" + register: snapshot_bundle_stat + +- name: snapshot — summary + ansible.builtin.debug: + msg: | + Snapshot complete: + name : {{ snapshot_name }} + path : {{ snapshot_bundle_path }} + size : {{ (snapshot_bundle_stat.stat.size | int / 1024 / 1024) | round(1) }} MiB + paths : {{ snapshot_present_paths | join(', ') }} + gateway : {{ 'ready' if snapshot_gw_ready.rc == 0 else 'NOT ready after restart — investigate' }} + + Restore with: ansible-playbook playbooks/dev-restore.yml -e snapshot_name={{ snapshot_name }} diff --git a/scripts/ansible/roles/status/meta/main.yml b/scripts/ansible/roles/status/meta/main.yml new file mode 100644 index 000000000..a3abdbf04 --- /dev/null +++ b/scripts/ansible/roles/status/meta/main.yml @@ -0,0 +1,7 @@ +--- +galaxy_info: + role_name: status + description: Read-only health snapshot for a Spinifex dev node (doctor). + license: BSD-3-Clause + min_ansible_version: "2.14" +dependencies: [] diff --git a/scripts/ansible/roles/status/tasks/main.yml b/scripts/ansible/roles/status/tasks/main.yml new file mode 100644 index 000000000..2e37b03ea --- /dev/null +++ b/scripts/ansible/roles/status/tasks/main.yml @@ -0,0 +1,259 @@ +--- +# Read-only health snapshot. NEVER mutates state. Every probe must use +# `changed_when: false` and `failed_when: false` so the role always reaches +# the summary task even when the node is half-broken — surfacing brokenness +# is the whole point. + +- name: status — spinifex.target activation + ansible.builtin.command: systemctl is-active spinifex.target + register: status_target + changed_when: false + failed_when: false + +- name: status — list spinifex-* units (active + failed) + ansible.builtin.shell: | + systemctl list-units 'spinifex-*' --all --no-legend --plain \ + --output=json 2>/dev/null || \ + systemctl list-units 'spinifex-*' --all --no-legend --plain + register: status_units + changed_when: false + failed_when: false + +- name: status — failed spinifex-* units (one line per) + ansible.builtin.shell: | + systemctl list-units 'spinifex-*' --state=failed --no-legend --plain \ + | awk '{print $1}' + register: status_failed_units + changed_when: false + failed_when: false + +- name: status — port listeners (which process holds each spinifex port) + ansible.builtin.shell: | + ss -tlnpH 'sport = :{{ item.port }}' | head -1 + register: status_ports + loop: "{{ spinifex_ports }}" + changed_when: false + failed_when: false + loop_control: + label: "{{ item.service }} ({{ item.port }})" + +- name: status — OVS tooling present? + ansible.builtin.command: which ovs-vsctl + register: status_ovs_which + changed_when: false + failed_when: false + +- name: status — OVS bridges + become: true + ansible.builtin.command: ovs-vsctl list-br + register: status_ovs_brs + changed_when: false + failed_when: false + when: status_ovs_which.rc == 0 + +- name: status — br-int exists? + become: true + ansible.builtin.command: ovs-vsctl br-exists br-int + register: status_brint + changed_when: false + failed_when: false + when: status_ovs_which.rc == 0 + +- name: status — ovn-controller active? + ansible.builtin.command: systemctl is-active ovn-controller + register: status_ovn_ctrl + changed_when: false + failed_when: false + +- name: status — ovn-central active? + ansible.builtin.command: systemctl is-active ovn-central + register: status_ovn_central + changed_when: false + failed_when: false + +- name: status — OVN NB/SB DB files present? + become: true + ansible.builtin.stat: + path: "{{ item }}" + register: status_ovn_dbs + loop: + - "{{ ovn_nb_db }}" + - "{{ ovn_sb_db }}" + loop_control: + label: "{{ item }}" + +- name: status — spinifex.toml present? + become: true + ansible.builtin.stat: + path: "{{ spinifex_config_file }}" + register: status_config + +- name: status — CA trust present? + become: true + ansible.builtin.stat: + path: "{{ spinifex_ca_trust_path }}" + register: status_ca + +- name: status — AWS gateway round-trip (describe-regions) + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-regions --output text" + register: status_awsgw + changed_when: false + failed_when: false + +- name: status — installed spx binary + become: true + ansible.builtin.stat: + path: /usr/local/bin/spx + checksum_algorithm: sha256 + register: status_spx_installed + +- name: status — built spx binary + ansible.builtin.stat: + path: "{{ spinifex_root }}/bin/spx" + checksum_algorithm: sha256 + register: status_spx_built + +- name: status — spx version (installed) + ansible.builtin.shell: | + /usr/local/bin/spx version 2>/dev/null | grep -vE '^[0-9]{4}/' | head -1 + register: status_spx_version + changed_when: false + failed_when: false + when: status_spx_installed.stat.exists + +- name: status — installed nbdkit plugin + become: true + ansible.builtin.shell: | + set -e + pdir=$(nbdkit --dump-config | awk -F= '/^plugindir=/{print $2; exit}') + [ -f "$pdir/nbdkit-viperblock-plugin.so" ] && \ + sha256sum "$pdir/nbdkit-viperblock-plugin.so" | awk '{print $1}' + register: status_plugin_installed_sha + changed_when: false + failed_when: false + +- name: status — built nbdkit plugin + ansible.builtin.stat: + path: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so" + checksum_algorithm: sha256 + register: status_plugin_built + +- name: status — microvm artifacts (built) + ansible.builtin.stat: + path: "{{ spinifex_root }}/build/microvm/{{ item }}" + checksum_algorithm: sha256 + register: status_microvm_built + loop: + - vmlinuz + - initramfs.cpio.gz + +- name: status — microvm artifacts (installed) + become: true + ansible.builtin.stat: + path: /usr/share/spinifex/microvm/{{ item }} + checksum_algorithm: sha256 + register: status_microvm_installed + loop: + - vmlinuz + - initramfs.cpio.gz + +- name: status — running QEMU VMs + ansible.builtin.shell: pgrep -fa 'qemu-system-' | wc -l + register: status_qemu_count + changed_when: false + failed_when: false + +- name: status — count AMIs + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-images --query length(Images) --output text" + register: status_ami_count + changed_when: false + failed_when: false + +- name: status — count keypairs + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-key-pairs --query length(KeyPairs) --output text" + register: status_key_count + changed_when: false + failed_when: false + +- name: status — count volumes + ansible.builtin.command: "{{ spinifex_aws_cli }} ec2 describe-volumes --query length(Volumes) --output text" + register: status_vol_count + changed_when: false + failed_when: false + +- name: status — count running instances + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-instances + --filters Name=instance-state-name,Values=running + --query length(Reservations[].Instances[]) --output text + register: status_instance_count + changed_when: false + failed_when: false + +- name: status — compute drift facts + ansible.builtin.set_fact: + status_spx_drift: >- + {{ status_spx_installed.stat.exists and status_spx_built.stat.exists and + (status_spx_installed.stat.checksum | default('')) != + (status_spx_built.stat.checksum | default('')) }} + status_plugin_drift: >- + {{ status_plugin_built.stat.exists and + (status_plugin_installed_sha.stdout | default('') | trim) != + (status_plugin_built.stat.checksum | default('')) }} + status_microvm_built_present: "{{ status_microvm_built.results + | map(attribute='stat.exists') | min }}" + +- name: status — render port lines + ansible.builtin.set_fact: + status_port_lines: >- + {{ status_port_lines | default([]) + + [('%-24s' | format(item.item.service ~ ' (' ~ item.item.port ~ ')')) + ~ ': ' + ~ ('listening' if (item.stdout | length) > 0 else 'free')] }} + loop: "{{ status_ports.results }}" + loop_control: + label: "{{ item.item.service }}" + +- name: status — assemble summary + ansible.builtin.set_fact: + status_summary_lines: >- + {{ + [ + '── Spinifex dev-node status ──────────────────────────────────────────────', + 'target : ' ~ (status_target.stdout | trim), + 'failed units : ' ~ (status_failed_units.stdout_lines | length) + ~ ((' → ' ~ (status_failed_units.stdout_lines | join(', '))) if (status_failed_units.stdout_lines | length) > 0 else ''), + '── Ports ──' + ] + + status_port_lines + + [ + '── Networking ──', + 'ovs-vsctl : ' ~ ('present' if status_ovs_which.rc == 0 else 'MISSING'), + 'ovs bridges : ' ~ ((status_ovs_brs.stdout_lines | default([]) | join(', ')) or '(none)'), + 'br-int : ' ~ ('present' if (status_brint.rc | default(1)) == 0 else 'MISSING'), + 'ovn-controller : ' ~ (status_ovn_ctrl.stdout | trim), + 'ovn-central : ' ~ (status_ovn_central.stdout | trim), + 'ovn NB DB : ' ~ ('present' if status_ovn_dbs.results[0].stat.exists else 'absent'), + 'ovn SB DB : ' ~ ('present' if status_ovn_dbs.results[1].stat.exists else 'absent'), + '── Config / trust ──', + 'spinifex.toml : ' ~ ('present' if status_config.stat.exists else 'MISSING') ~ ' (' ~ spinifex_config_file ~ ')', + 'CA trust : ' ~ ('present' if status_ca.stat.exists else 'MISSING') ~ ' (' ~ spinifex_ca_trust_path ~ ')', + '── AWS gateway ──', + 'describe-regions : ' ~ ('OK' if status_awsgw.rc == 0 else 'FAIL (rc=' ~ status_awsgw.rc ~ ')'), + 'AMIs : ' ~ (status_ami_count.stdout | default('?') | trim), + 'keypairs : ' ~ (status_key_count.stdout | default('?') | trim), + 'volumes : ' ~ (status_vol_count.stdout | default('?') | trim), + 'running instances: ' ~ (status_instance_count.stdout | default('?') | trim), + 'running QEMU : ' ~ (status_qemu_count.stdout | default('?') | trim), + '── Versions / drift ──', + 'spx (installed) : ' ~ ((status_spx_version.stdout | default('') | trim) or '(not installed)'), + 'spx drift : ' ~ ('YES (built ≠ installed) — run ansible-dev-deploy' if status_spx_drift else 'no'), + 'plugin drift : ' ~ ('YES (built ≠ installed) — run ansible-dev-deploy' if status_plugin_drift else 'no'), + 'microvm built : ' ~ ('present' if status_microvm_built_present else 'not built on this branch'), + '──────────────────────────────────────────────────────────────────────────' + ] + }} + +- name: status — summary + ansible.builtin.debug: + msg: "{{ status_summary_lines }}" diff --git a/scripts/ansible/roles/version/meta/main.yml b/scripts/ansible/roles/version/meta/main.yml new file mode 100644 index 000000000..780fdbf08 --- /dev/null +++ b/scripts/ansible/roles/version/meta/main.yml @@ -0,0 +1,7 @@ +--- +galaxy_info: + role_name: version + description: Detailed build-vs-install drift report for spinifex artifacts. + license: BSD-3-Clause + min_ansible_version: "2.14" +dependencies: [] diff --git a/scripts/ansible/roles/version/tasks/main.yml b/scripts/ansible/roles/version/tasks/main.yml new file mode 100644 index 000000000..3c794c816 --- /dev/null +++ b/scripts/ansible/roles/version/tasks/main.yml @@ -0,0 +1,161 @@ +--- +# Detailed build-vs-install drift report for the four artifacts that +# `ansible-dev-deploy` swaps: spx, the nbdkit viperblock plugin, and the +# two microvm artifacts (kernel + initramfs). Pure read-only; faster +# than dev-status (no AWS / OVN / port probing). + +- name: version — built spx + ansible.builtin.stat: + path: "{{ spinifex_root }}/bin/spx" + checksum_algorithm: sha256 + register: ver_spx_built + +- name: version — installed spx + become: true + ansible.builtin.stat: + path: /usr/local/bin/spx + checksum_algorithm: sha256 + register: ver_spx_installed + +- name: version — installed spx version string + ansible.builtin.shell: | + /usr/local/bin/spx version 2>/dev/null | grep -vE '^[0-9]{4}/' | head -1 + register: ver_spx_version + changed_when: false + failed_when: false + when: ver_spx_installed.stat.exists + +- name: version — built plugin + ansible.builtin.stat: + path: "{{ mulga_root }}/viperblock/lib/nbdkit-viperblock-plugin.so" + checksum_algorithm: sha256 + register: ver_plugin_built + +- name: version — nbdkit plugin dir + ansible.builtin.shell: | + nbdkit --dump-config 2>/dev/null | awk -F= '/^plugindir=/{print $2; exit}' + register: ver_plugindir + changed_when: false + failed_when: false + +- name: version — installed plugin + become: true + ansible.builtin.stat: + path: "{{ ver_plugindir.stdout | trim }}/nbdkit-viperblock-plugin.so" + checksum_algorithm: sha256 + register: ver_plugin_installed + when: ver_plugindir.stdout | trim | length > 0 + +- name: version — built microvm artifacts + ansible.builtin.stat: + path: "{{ spinifex_root }}/build/microvm/{{ item }}" + checksum_algorithm: sha256 + register: ver_microvm_built + loop: + - vmlinuz + - initramfs.cpio.gz + +- name: version — installed microvm artifacts + become: true + ansible.builtin.stat: + path: /usr/share/spinifex/microvm/{{ item }} + checksum_algorithm: sha256 + register: ver_microvm_installed + loop: + - vmlinuz + - initramfs.cpio.gz + +- name: version — sub-repo HEAD short-rev (spinifex) + ansible.builtin.command: git -C {{ spinifex_root | quote }} rev-parse --short HEAD + register: ver_spinifex_head + changed_when: false + failed_when: false + +- name: version — sub-repo HEAD short-rev (viperblock) + ansible.builtin.command: git -C {{ mulga_root }}/viperblock rev-parse --short HEAD + register: ver_viperblock_head + changed_when: false + failed_when: false + +- name: version — sub-repo HEAD short-rev (predastore) + ansible.builtin.command: git -C {{ mulga_root }}/predastore rev-parse --short HEAD + register: ver_predastore_head + changed_when: false + failed_when: false + +- name: version — assemble per-artifact rows + ansible.builtin.set_fact: + ver_artifact_rows: >- + {{ + [ + ('%-18s built=%s installed=%s drift=%s' | format( + 'spx', + (ver_spx_built.stat.checksum | default('-') | truncate(13, true, '')) + if ver_spx_built.stat.exists else 'MISSING', + (ver_spx_installed.stat.checksum | default('-') | truncate(13, true, '')) + if ver_spx_installed.stat.exists else 'MISSING', + ('YES' if ver_spx_built.stat.exists and ver_spx_installed.stat.exists + and ver_spx_built.stat.checksum != ver_spx_installed.stat.checksum + else 'no') + )), + ('%-18s built=%s installed=%s drift=%s' | format( + 'plugin (nbdkit)', + (ver_plugin_built.stat.checksum | default('-') | truncate(13, true, '')) + if ver_plugin_built.stat.exists else 'MISSING', + (ver_plugin_installed.stat.checksum | default('-') | truncate(13, true, '')) + if (ver_plugin_installed.stat | default({})).get('exists', false) else 'MISSING', + ('YES' if ver_plugin_built.stat.exists + and (ver_plugin_installed.stat | default({})).get('exists', false) + and ver_plugin_built.stat.checksum != ver_plugin_installed.stat.checksum + else 'no') + )), + ('%-18s built=%s installed=%s drift=%s' | format( + 'microvm vmlinuz', + (ver_microvm_built.results[0].stat.checksum | default('-') | truncate(13, true, '')) + if ver_microvm_built.results[0].stat.exists else 'MISSING', + (ver_microvm_installed.results[0].stat.checksum | default('-') | truncate(13, true, '')) + if ver_microvm_installed.results[0].stat.exists else 'MISSING', + ('YES' if ver_microvm_built.results[0].stat.exists + and ver_microvm_installed.results[0].stat.exists + and ver_microvm_built.results[0].stat.checksum + != ver_microvm_installed.results[0].stat.checksum + else 'no') + )), + ('%-18s built=%s installed=%s drift=%s' | format( + 'microvm initramfs', + (ver_microvm_built.results[1].stat.checksum | default('-') | truncate(13, true, '')) + if ver_microvm_built.results[1].stat.exists else 'MISSING', + (ver_microvm_installed.results[1].stat.checksum | default('-') | truncate(13, true, '')) + if ver_microvm_installed.results[1].stat.exists else 'MISSING', + ('YES' if ver_microvm_built.results[1].stat.exists + and ver_microvm_installed.results[1].stat.exists + and ver_microvm_built.results[1].stat.checksum + != ver_microvm_installed.results[1].stat.checksum + else 'no') + )) + ] + }} + +- name: version — compute any-drift flag + ansible.builtin.set_fact: + ver_any_drift: "{{ 'YES' in (ver_artifact_rows | join(' ')) }}" + +- name: version — summary + ansible.builtin.debug: + msg: >- + {{ + [ + '── Spinifex build vs install ─────────────────────────────────────────────', + 'spx (installed): ' ~ ((ver_spx_version.stdout | default('') | trim) or '(not installed)') + ] + + ver_artifact_rows + + [ + '── Sub-repo HEADs ──', + 'spinifex : ' ~ ((ver_spinifex_head.stdout | default('') | trim) or '(not a git repo)'), + 'viperblock : ' ~ ((ver_viperblock_head.stdout | default('') | trim) or '(missing)'), + 'predastore : ' ~ ((ver_predastore_head.stdout | default('') | trim) or '(missing)'), + '──────────────────────────────────────────────────────────────────────────', + ('DRIFT detected — run: make ansible-dev-deploy' if ver_any_drift + else 'No drift; built = installed for all artifacts.') + ] + }} diff --git a/scripts/ansible/roles/vpc/meta/main.yml b/scripts/ansible/roles/vpc/meta/main.yml new file mode 100644 index 000000000..cc90f9d83 --- /dev/null +++ b/scripts/ansible/roles/vpc/meta/main.yml @@ -0,0 +1,7 @@ +--- +galaxy_info: + role_name: vpc + description: Idempotent dev-mode VPC + public subnet + IGW + default route fixture. + license: BSD-3-Clause + min_ansible_version: "2.14" +dependencies: [] diff --git a/scripts/ansible/roles/vpc/tasks/absent.yml b/scripts/ansible/roles/vpc/tasks/absent.yml new file mode 100644 index 000000000..2f72e21fd --- /dev/null +++ b/scripts/ansible/roles/vpc/tasks/absent.yml @@ -0,0 +1,128 @@ +--- +# Tear down the dev VPC fixture. Deletes in reverse dependency order: +# route-table associations, route table, subnet, IGW (detach + delete), +# VPC. Each step is best-effort so a partially-converged fixture still +# converges to "gone" on re-run. + +- name: vpc — find VPC by Name tag + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-vpcs + --filters Name=tag:Name,Values={{ dev_vpc_name }} + --query Vpcs[0].VpcId --output text + register: absent_vpc + changed_when: false + +- name: vpc — nothing to do + ansible.builtin.debug: + msg: "VPC tagged Name={{ dev_vpc_name }} not found; nothing to delete." + when: absent_vpc.stdout | trim in ['', 'None'] + +- name: vpc — short-circuit when no VPC + ansible.builtin.meta: end_play + when: absent_vpc.stdout | trim in ['', 'None'] + +- name: vpc — set VPC id fact + ansible.builtin.set_fact: + absent_vpc_id: "{{ absent_vpc.stdout | trim }}" + +- name: vpc — find subnet + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-subnets + --filters Name=vpc-id,Values={{ absent_vpc_id }} + Name=tag:Name,Values={{ dev_vpc_name }}-public + --query Subnets[0].SubnetId --output text + register: absent_subnet + changed_when: false + +# Enumerate ALL non-main route tables in the VPC (main RT can't be +# deleted; it goes when the VPC does). Catches orphans from earlier +# runs that landed before the create-tags workaround. +- name: vpc — find all non-main route tables in VPC + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-route-tables + --filters Name=vpc-id,Values={{ absent_vpc_id }} + --query RouteTables[?!(Associations[?Main==`true`])].RouteTableId + --output text + register: absent_rt_list + changed_when: false + +- name: vpc — split RT id list + ansible.builtin.set_fact: + absent_rt_ids: "{{ absent_rt_list.stdout.split() | reject('equalto', '') | list }}" + +- name: vpc — find all subnet/RT associations to disassociate + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-route-tables + --route-table-ids {{ item }} + --query RouteTables[0].Associations[?SubnetId!=null].RouteTableAssociationId + --output text + register: absent_assocs + loop: "{{ absent_rt_ids }}" + changed_when: false + failed_when: false + +- name: vpc — disassociate subnets from route tables + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 disassociate-route-table + --association-id {{ item }} + loop: "{{ absent_assocs.results + | map(attribute='stdout') + | map('default', '') + | map('split') + | flatten + | reject('equalto', '') + | list }}" + changed_when: true + failed_when: false + +- name: vpc — delete all non-main route tables + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 delete-route-table + --route-table-id {{ item }} + loop: "{{ absent_rt_ids }}" + changed_when: true + failed_when: false + +- name: vpc — delete subnet + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 delete-subnet + --subnet-id {{ absent_subnet.stdout | trim }} + when: absent_subnet.stdout | trim not in ['', 'None'] + changed_when: true + failed_when: false + +- name: vpc — find IGW + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-internet-gateways + --filters Name=attachment.vpc-id,Values={{ absent_vpc_id }} + --query InternetGateways[0].InternetGatewayId --output text + register: absent_igw + changed_when: false + +- name: vpc — detach IGW + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 detach-internet-gateway + --internet-gateway-id {{ absent_igw.stdout | trim }} + --vpc-id {{ absent_vpc_id }} + when: absent_igw.stdout | trim not in ['', 'None'] + changed_when: true + failed_when: false + +- name: vpc — delete IGW + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 delete-internet-gateway + --internet-gateway-id {{ absent_igw.stdout | trim }} + when: absent_igw.stdout | trim not in ['', 'None'] + changed_when: true + failed_when: false + +- name: vpc — delete VPC + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 delete-vpc + --vpc-id {{ absent_vpc_id }} + changed_when: true + failed_when: false + +- name: vpc — summary (absent) + ansible.builtin.debug: + msg: "Dev VPC {{ dev_vpc_name }} (was {{ absent_vpc_id }}) torn down." diff --git a/scripts/ansible/roles/vpc/tasks/main.yml b/scripts/ansible/roles/vpc/tasks/main.yml new file mode 100644 index 000000000..5258f3bd4 --- /dev/null +++ b/scripts/ansible/roles/vpc/tasks/main.yml @@ -0,0 +1,3 @@ +--- +- name: vpc — dispatch by state + ansible.builtin.include_tasks: "{{ dev_vpc_state }}.yml" diff --git a/scripts/ansible/roles/vpc/tasks/present.yml b/scripts/ansible/roles/vpc/tasks/present.yml new file mode 100644 index 000000000..275c7def9 --- /dev/null +++ b/scripts/ansible/roles/vpc/tasks/present.yml @@ -0,0 +1,209 @@ +--- +# Idempotent create. Every step describes-then-creates: re-running on an +# already-converged VPC produces zero changes. Tag everything with +# Name={{ dev_vpc_name }} so a single tag query finds the whole graph. + +- name: vpc — find existing VPC by Name tag + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-vpcs + --filters Name=tag:Name,Values={{ dev_vpc_name }} + --query Vpcs[0].VpcId --output text + register: vpc_existing + changed_when: false + +- name: vpc — create VPC if missing + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 create-vpc + --cidr-block {{ dev_vpc_cidr }} + --tag-specifications + ResourceType=vpc,Tags=[{Key=Name,Value={{ dev_vpc_name }}}] + --query Vpc.VpcId --output text + register: vpc_created + when: vpc_existing.stdout | trim in ['', 'None'] + changed_when: true + +- name: vpc — set VPC id fact + ansible.builtin.set_fact: + dev_vpc_id: >- + {{ vpc_created.stdout | default('') | trim + if (vpc_existing.stdout | trim in ['', 'None']) + else vpc_existing.stdout | trim }} + +- name: vpc — assert VPC id resolved + ansible.builtin.assert: + that: dev_vpc_id | length > 0 + fail_msg: "VPC id could not be resolved after describe/create" + +- name: vpc — find existing IGW attached to this VPC + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-internet-gateways + --filters Name=attachment.vpc-id,Values={{ dev_vpc_id }} + --query InternetGateways[0].InternetGatewayId --output text + register: vpc_igw_existing + changed_when: false + +- name: vpc — create IGW if missing + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 create-internet-gateway + --tag-specifications + ResourceType=internet-gateway,Tags=[{Key=Name,Value={{ dev_vpc_name }}-igw}] + --query InternetGateway.InternetGatewayId --output text + register: vpc_igw_created + when: vpc_igw_existing.stdout | trim in ['', 'None'] + changed_when: true + +- name: vpc — set IGW id fact + ansible.builtin.set_fact: + dev_vpc_igw_id: >- + {{ vpc_igw_created.stdout | default('') | trim + if (vpc_igw_existing.stdout | trim in ['', 'None']) + else vpc_igw_existing.stdout | trim }} + +- name: vpc — attach IGW to VPC if needed + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 attach-internet-gateway + --internet-gateway-id {{ dev_vpc_igw_id }} + --vpc-id {{ dev_vpc_id }} + register: vpc_igw_attach + when: vpc_igw_existing.stdout | trim in ['', 'None'] + changed_when: true + failed_when: + - vpc_igw_attach.rc | default(0) != 0 + - "'Resource.AlreadyAssociated' not in (vpc_igw_attach.stderr | default(''))" + +- name: vpc — find existing subnet by Name tag + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-subnets + --filters Name=vpc-id,Values={{ dev_vpc_id }} + Name=tag:Name,Values={{ dev_vpc_name }}-public + --query Subnets[0].SubnetId --output text + register: vpc_subnet_existing + changed_when: false + +- name: vpc — create subnet if missing + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 create-subnet + --vpc-id {{ dev_vpc_id }} + --cidr-block {{ dev_vpc_subnet_cidr }} + --availability-zone {{ dev_vpc_subnet_az }} + --tag-specifications + ResourceType=subnet,Tags=[{Key=Name,Value={{ dev_vpc_name }}-public}] + --query Subnet.SubnetId --output text + register: vpc_subnet_created + when: vpc_subnet_existing.stdout | trim in ['', 'None'] + changed_when: true + +- name: vpc — set subnet id fact + ansible.builtin.set_fact: + dev_vpc_subnet_id: >- + {{ vpc_subnet_created.stdout | default('') | trim + if (vpc_subnet_existing.stdout | trim in ['', 'None']) + else vpc_subnet_existing.stdout | trim }} + +- name: vpc — ensure subnet has map-public-ip-on-launch + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 modify-subnet-attribute + --subnet-id {{ dev_vpc_subnet_id }} + --map-public-ip-on-launch + register: vpc_subnet_mapip + changed_when: vpc_subnet_existing.stdout | trim in ['', 'None'] + failed_when: false + +# Try tag-based lookup first. If the gateway didn't tag a previously +# created RT (spinifex gateway silently drops --tag-specifications on +# create-route-table AND no-ops create-tags for route tables — see +# mulga-siv-94; works correctly for subnets), fall back to "RT +# associated with our subnet" which doesn't need tags. +- name: vpc — find existing route table by Name tag + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-route-tables + --filters Name=vpc-id,Values={{ dev_vpc_id }} + Name=tag:Name,Values={{ dev_vpc_name }}-public + --query RouteTables[0].RouteTableId --output text + register: vpc_rt_existing + changed_when: false + +- name: vpc — fallback lookup by subnet association + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 describe-route-tables + --filters Name=association.subnet-id,Values={{ dev_vpc_subnet_id }} + --query RouteTables[0].RouteTableId --output text + register: vpc_rt_byassoc + changed_when: false + when: vpc_rt_existing.stdout | trim in ['', 'None'] + +- name: vpc — create route table if neither lookup found one + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 create-route-table + --vpc-id {{ dev_vpc_id }} + --query RouteTable.RouteTableId --output text + register: vpc_rt_created + when: + - vpc_rt_existing.stdout | trim in ['', 'None'] + - (vpc_rt_byassoc.stdout | default('') | trim) in ['', 'None'] + changed_when: true + +- name: vpc — set route-table id fact + ansible.builtin.set_fact: + dev_vpc_rt_id: >- + {%- if (vpc_rt_existing.stdout | trim) not in ['', 'None'] -%} + {{ vpc_rt_existing.stdout | trim }} + {%- elif (vpc_rt_byassoc.stdout | default('') | trim) not in ['', 'None'] -%} + {{ vpc_rt_byassoc.stdout | default('') | trim }} + {%- else -%} + {{ vpc_rt_created.stdout | default('') | trim }} + {%- endif -%} + +- name: vpc — assert route-table id resolved + ansible.builtin.assert: + that: dev_vpc_rt_id | length > 0 + fail_msg: "Route table id could not be resolved after describe/create" + +# Spinifex gateway dropped --tag-specifications on the create call; +# apply via create-tags so re-runs can find the RT via tag filter. +- name: vpc — tag route table (workaround for create-tag-specifications gap) + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 create-tags + --resources {{ dev_vpc_rt_id }} + --tags Key=Name,Value={{ dev_vpc_name }}-public + changed_when: false + failed_when: false + +- name: vpc — add default route via IGW (ignore RouteAlreadyExists) + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 create-route + --route-table-id {{ dev_vpc_rt_id }} + --destination-cidr-block 0.0.0.0/0 + --gateway-id {{ dev_vpc_igw_id }} + register: vpc_route_add + changed_when: + - vpc_route_add.rc == 0 + - "'RouteAlreadyExists' not in (vpc_route_add.stderr | default(''))" + failed_when: + - vpc_route_add.rc != 0 + - "'RouteAlreadyExists' not in (vpc_route_add.stderr | default(''))" + +- name: vpc — associate subnet with route table (ignore already-associated) + ansible.builtin.command: >- + {{ spinifex_aws_cli }} ec2 associate-route-table + --route-table-id {{ dev_vpc_rt_id }} + --subnet-id {{ dev_vpc_subnet_id }} + register: vpc_assoc + changed_when: + - vpc_assoc.rc == 0 + - "'Resource.AlreadyAssociated' not in (vpc_assoc.stderr | default(''))" + failed_when: + - vpc_assoc.rc != 0 + - "'Resource.AlreadyAssociated' not in (vpc_assoc.stderr | default(''))" + +- name: vpc — summary (present) + ansible.builtin.debug: + msg: | + Dev VPC ready: + VPC : {{ dev_vpc_id }} ({{ dev_vpc_cidr }}) + IGW : {{ dev_vpc_igw_id }} + Subnet : {{ dev_vpc_subnet_id }} ({{ dev_vpc_subnet_cidr }} @ {{ dev_vpc_subnet_az }}, public) + Route table : {{ dev_vpc_rt_id }} + + Smoketest now finds this subnet via its existing + Name=map-public-ip-on-launch,Values=true filter. diff --git a/scripts/ansible/scripts/aws-as-spinifex.sh b/scripts/ansible/scripts/aws-as-spinifex.sh new file mode 100755 index 000000000..59b0fffe2 --- /dev/null +++ b/scripts/ansible/scripts/aws-as-spinifex.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Run `aws` CLI inside a shell that has the `spinifex` supplementary group +# active, with AWS_PROFILE=spinifex preset. Works around mulga-siv-92: +# ansible's `become` is a no-op when the target user equals the calling +# user, so `sudo -u` is never invoked and supplementary groups are not +# re-initialised. Without the `spinifex` group, /etc/spinifex/ca.pem (the +# AWS endpoint CA bundle) is unreadable and aws fails with SSL Errno 13. +# +# Usage from ansible tasks: +# ansible.builtin.command: +# cmd: "{{ playbook_dir }}/../scripts/aws-as-spinifex.sh ec2 describe-regions" +# +# All args are forwarded to `aws`. Quoting is preserved via printf %q so +# things like `--query 'length(Images)'` reach aws intact. +set -euo pipefail + +if [[ $# -eq 0 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +cmd="AWS_PROFILE=spinifex aws" +for arg in "$@"; do + cmd+=" $(printf '%q' "$arg")" +done + +exec sg spinifex -c "$cmd" diff --git a/scripts/ansible/scripts/spx-as-spinifex.sh b/scripts/ansible/scripts/spx-as-spinifex.sh new file mode 100755 index 000000000..8d8ab3538 --- /dev/null +++ b/scripts/ansible/scripts/spx-as-spinifex.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Run `spx` inside a shell that has the `spinifex` supplementary group +# active. Works around mulga-siv-92 (see aws-as-spinifex.sh for details). +# All args are forwarded to /usr/local/bin/spx via printf %q so flag +# values with spaces or shell metachars survive intact. +set -euo pipefail + +if [[ $# -eq 0 ]]; then + echo "usage: $0 " >&2 + exit 64 +fi + +cmd="AWS_PROFILE=spinifex /usr/local/bin/spx" +for arg in "$@"; do + cmd+=" $(printf '%q' "$arg")" +done + +exec sg spinifex -c "$cmd" diff --git a/scripts/ansible/vars/defaults.yml b/scripts/ansible/vars/defaults.yml index 8d275dae9..ad2dad7c5 100644 --- a/scripts/ansible/vars/defaults.yml +++ b/scripts/ansible/vars/defaults.yml @@ -76,9 +76,45 @@ teardown_process_wait_seconds: 30 spinifex_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../..') }}" mulga_root: "{{ lookup('pipe', 'readlink -f ' ~ playbook_dir ~ '/../../../..') }}" +# Wrappers around `aws` and `spx` that re-enter a shell with the +# `spinifex` supplementary group active. Needed because ansible's +# `become_user` is a no-op when target user equals current user, so +# supplementary groups are never re-initialised and /etc/spinifex/ca.pem +# (drwxr-x--- root:spinifex) is unreadable. See mulga-siv-92. +spinifex_aws_cli: "{{ playbook_dir }}/../scripts/aws-as-spinifex.sh" +spinifex_spx_cli: "{{ playbook_dir }}/../scripts/spx-as-spinifex.sh" + +# dev-vpc fixture. Deterministic VPC + public subnet + IGW + default +# route. Idempotent: re-running detects existing resources by Name tag. +# Used by smoketest's launch path (which looks for any +# map-public-ip-on-launch=true subnet). +dev_vpc_name: spinifex-dev-vpc +dev_vpc_cidr: 10.99.0.0/16 +dev_vpc_subnet_cidr: 10.99.1.0/24 +dev_vpc_subnet_az: "{{ spinifex_az }}" +dev_vpc_state: present # present | absent + # Build output. build_tarball_path: /tmp/spinifex-local.tar.gz +# Snapshot / restore. Snapshots quiesce the full spinifex+ovn+ovs stack, +# tar these paths, then restart services. Restore wipes + untars + restarts. +# Default snapshot dir is root-owned so it survives `dev-teardown` +# (teardown only removes the four spinifex_*_dir paths above). +snapshot_dir: /var/lib/spinifex-snapshots +snapshot_paths: + - "{{ spinifex_etc_dir }}" + - "{{ spinifex_data_dir }}" + - "{{ ovn_db_dir }}" + - /etc/openvswitch/conf.db +# Order matters: stop spinifex first (depends on OVN/OVS), then OVN, then OVS. +# Restart in reverse. +snapshot_services_stop_order: + - spinifex.target + - ovn-controller + - ovn-central + - openvswitch-switch + # Smoketest image — parameterised so tofu examples can share the same base. spinifex_smoketest_distro: debian spinifex_smoketest_version: "13"