diff --git a/.github/workflows/dev-release.yml b/.github/workflows/dev-release.yml index 88dd9bdeb..fd7a16dcc 100644 --- a/.github/workflows/dev-release.yml +++ b/.github/workflows/dev-release.yml @@ -75,7 +75,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.27.0" + go-version-file: spinifex/go.mod # The tag carries the describe string so the commit is recoverable from the # version alone, and the -dev suffix is what release.yml's own convention @@ -167,6 +167,10 @@ jobs: # Kept rather than unbounded: the Worker reads the first page of releases to # resolve the channel, so enough prereleases would eventually push the real # ones off it, and each build carries two assets. + # + # Twenty, not five. A pinned dev tag is how an external evaluator installs a + # specific build, and five is about a week of them -- short enough that the tag + # someone was given stops resolving while they are still using it. - name: Prune old dev prereleases if: steps.version.outputs.exists != 'true' env: @@ -174,7 +178,7 @@ jobs: run: | set -euo pipefail gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \ - --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[5:] | .[].tag_name' \ + --jq 'map(select(.prerelease and (.tag_name | endswith("-dev")))) | .[20:] | .[].tag_name' \ > stale.txt while read -r tag; do [ -n "$tag" ] || continue diff --git a/.github/workflows/e2e-oci-nightly.yml b/.github/workflows/e2e-cloudvendor-nightly.yml similarity index 68% rename from .github/workflows/e2e-oci-nightly.yml rename to .github/workflows/e2e-cloudvendor-nightly.yml index 220399d52..03d14cf04 100644 --- a/.github/workflows/e2e-oci-nightly.yml +++ b/.github/workflows/e2e-cloudvendor-nightly.yml @@ -1,4 +1,4 @@ -name: E2E OCI Nightly +name: E2E Cloud Vendor Nightly # Dispatched by the same Cloudflare Worker that drives e2e-nightly.yml, once per # ref, rather than by a `schedule:` here. GitHub registers a schedule only from @@ -28,6 +28,12 @@ on: required: false type: boolean default: false + auth: + description: "How nodes reach the OCI API: instance_principal (needs the tenancy dynamic group) or api_key (needs the credential hook)" + required: false + default: instance_principal + type: choice + options: [instance_principal, api_key] no_external_pool: description: "Form without the OCI allocator. Skips every workbook needing a public address." required: false @@ -59,12 +65,17 @@ jobs: # Three topologies including a bare-metal build, each with its own install, # formation and workbook pass. The sweep still runs if this fires. timeout-minutes: 300 - env: - # Outside github.workspace on purpose. actions/checkout runs git clean -ffdx, - # which on a persistent runner would delete the Terraform state of a killed - # run before the next sweep could read it, leaving instances nothing can find. - OCI_STATE_ROOT: ${{ runner.workspace }}/oci-state steps: + # Set here rather than in a job-level env block: the runner context is not + # available at job level, only from a step onwards, and a workflow that + # references it there is rejected as invalid before any job starts. + # + # Outside github.workspace on purpose. The checkout wipes untracked files, so + # on a persistent runner it would delete the Terraform state of a killed run + # before the next sweep could read it, leaving instances nothing can find. + - name: Put the Terraform state outside the checkout + run: echo "OCI_STATE_ROOT=$RUNNER_WORKSPACE/oci-state" >> "$GITHUB_ENV" + - uses: actions/checkout@v7 with: path: spinifex @@ -121,12 +132,20 @@ jobs: OCI_TOPOLOGIES: ${{ inputs.topologies }} OCI_SOURCE: ${{ inputs.source }} OCI_CHANNEL: ${{ inputs.channel }} - # 1 unless the dispatch asked to form without the allocator, which is the - # only way to run before the tenancy's dynamic group exists. The run then - # records the allocator and the workbooks as SKIPPED rather than passing - # them, so a green run with this set is not a claim about guest networking. - OCI_INSTANCE_PRINCIPAL: ${{ inputs.no_external_pool && '0' || '1' }} + # Independent of the pool, deliberately. These were one switch, which made + # the combination we actually need -- an API key with the allocator running + # -- impossible to ask for, so public-IP allocation went untested. + OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }} + # Skipping the pool records the allocator and every public-address workbook + # as SKIPPED, so a green run with this set is not a claim about guest + # networking. It exists for a formation-only check, not as a way past a + # missing credential. OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }} + # Only an API-key run needs one: an instance principal is the node's own + # certificate, so there is nothing to hand over. Unset is not an error, and + # the allocator gate then fails rather than passing a node that cannot + # authenticate. + OCI_CREDENTIAL_HOOK: ${{ inputs.auth == 'api_key' && vars.OCI_CREDENTIAL_HOOK || '' }} OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci OCI_KEEP_ON_FAIL: ${{ inputs.keep_on_fail && '1' || '0' }} # Through the environment, never interpolated into the script below: a @@ -141,6 +160,29 @@ jobs: ./e2e-oci.sh 2>&1 | tee "$RUNNER_TEMP/e2e-oci.log" exit "${PIPESTATUS[0]}" + # Before the sweep, which destroys the state the tables are read from, and + # always(): a run whose verdict is a failure is exactly the one somebody + # wants a table for. The successful path writes the same tables itself, so + # this step is for the run that did not get there. + # + # The markdown lives in the script rather than here. A heredoc in a YAML + # string is the one place in this repository nothing can lint or run. + - name: Publish the result tables + if: always() + working-directory: spinifex/scripts/terraform/oci-spx + env: + OCI_REGION: ${{ vars.OCI_REGION || 'ap-sydney-1' }} + OCI_ARTIFACT_DIR: ${{ runner.temp }}/e2e-oci + OCI_TOPOLOGIES: ${{ inputs.topologies }} + OCI_SOURCE: ${{ inputs.source }} + OCI_CHANNEL: ${{ inputs.channel }} + SPX_REF: ${{ github.ref_name }} + # The same two the run step passes. The header states what was and was not + # tested, so a summary rendered without them would claim more than the run. + OCI_INSTANCE_PRINCIPAL: ${{ inputs.auth == 'api_key' && '0' || '1' }} + OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }} + run: ./e2e-oci.sh --summary-only + # always(), and before the artifact upload: an OCI bare-metal host left # running by a cancelled or timed-out job is the most expensive way this # workflow can fail. The sweep works from the Terraform state files, so it @@ -178,7 +220,14 @@ jobs: ${{ runner.temp }}/e2e-oci.log ${{ runner.temp }}/e2e-oci/*.log ${{ runner.temp }}/e2e-oci/verdict.txt + ${{ runner.temp }}/e2e-oci/summary.md ${{ runner.temp }}/e2e-oci-sweep/*.log ${{ runner.workspace }}/oci-state/.validate-*/*.log ${{ runner.workspace }}/oci-state/.validate-*/*.txt + ${{ runner.workspace }}/oci-state/.validate-*/*.tsv + # The hook is the operator's own executable and the one thing known + # about it is that it handles a private key. Whatever it printed is not + # ours to publish in a downloadable artifact, so it is excluded rather + # than trusted to have been quiet. + !${{ runner.workspace }}/oci-state/.validate-*/credential-hook.log retention-days: 14 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 311ced8f9..44e8781fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -46,7 +46,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v7 with: - go-version: "1.27.0" + go-version-file: spinifex/go.mod - name: Build microVM artifacts (lb-agent + kernel + initramfs) run: cd spinifex && make build-microvm-image diff --git a/Makefile b/Makefile index da9150060..a68316f56 100644 --- a/Makefile +++ b/Makefile @@ -167,7 +167,7 @@ preflight: # reason: between them they install the software, wire OVN, erase the state and # remove the install, so a shellcheck warning in one is an outage or a wrong # deletion, not a style note. -NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/node-reset.sh scripts/uninstall-spx.sh +NODE_SCRIPTS := scripts/setup.sh scripts/setup-ovn.sh scripts/install-node.sh scripts/smoke-test.sh scripts/node-reset.sh scripts/uninstall-spx.sh test-build-scripts: @echo -e "\n....Running build/scripts/**/*_test.sh...." diff --git a/docs/oci-integration/README.md b/docs/oci-integration/README.md index 331ef5524..5792021c6 100644 --- a/docs/oci-integration/README.md +++ b/docs/oci-integration/README.md @@ -24,292 +24,92 @@ resources: > Run the AWS surface — EC2, EBS, S3, VPC — inside your own OCI tenancy, with guests that get real, publicly reachable addresses through OCI's API. +Spinifex brings core AWS services to hardware you control: a node runs the Spinifex daemon, QEMU/KVM for guests, OVN for VPC networking, [Predastore](https://github.com/mulgadc/predastore) for S3 and [Viperblock](https://github.com/mulgadc/viperblock) for EBS, all behind a SigV4 endpoint the ordinary AWS SDKs and CLI talk to unmodified. + +This guide runs that stack on OCI. **Terraform does the whole deployment** — the VCN, the block volumes, both VNICs per node, the install, the cluster formation and the OCI public-address allocator — so [Deploy](#deploy) is four steps and one command, and everything after it is reference. One node is a working install; three is the minimum that survives losing a node. Every command below was run against a live OCI tenancy. + ## Table of Contents -- [Overview](#overview) -- [Support status](#support-status) - - [What happens when a node fails](#what-happens-when-a-node-fails) -- [Why run Spinifex on OCI](#why-run-spinifex-on-oci) -- [What OCI changes](#what-oci-changes) -- [How it fits together](#how-it-fits-together) -- [Prerequisites](#prerequisites) -- [Deploying](#deploying) - - [Step 1. Set your Terraform inputs](#step-1-set-your-terraform-inputs) - - [Step 2. Build the infrastructure](#step-2-build-the-infrastructure) - - [Step 3. Install Spinifex and set up OVN](#step-3-install-spinifex-and-set-up-ovn) - - [Step 4. Form the cluster](#step-4-form-the-cluster) - - [Step 5. Configure Spinifex for OCI, start and verify](#step-5-configure-spinifex-for-oci-start-and-verify) - - [Step 6. Set up your cluster](#step-6-set-up-your-cluster) - - [Step 7. Run your existing Terraform against the cluster](#step-7-run-your-existing-terraform-against-the-cluster) - - [Step 8. Oracle Linux as a guest image](#step-8-oracle-linux-as-a-guest-image) +- [Deploy](#deploy) + - [What you need](#what-you-need) + - [1. Get the configuration](#1-get-the-configuration) + - [2. Choose one node or three](#2-choose-one-node-or-three) + - [3. Write your Terraform inputs](#3-write-your-terraform-inputs) + - [4. Run the deploy](#4-run-the-deploy) + - [What a successful run looks like](#what-a-successful-run-looks-like) + - [If a stage fails](#if-a-stage-fails) + - [Removing the deployment](#removing-the-deployment) - [Verify end to end](#verify-end-to-end) +- [Next steps](#next-steps) +- [Reference](#reference) + - [Why run Spinifex on OCI](#why-run-spinifex-on-oci) + - [Sizing](#sizing) + - [How it fits together](#how-it-fits-together) + - [Credentials — an instance principal, or an API key](#credentials--an-instance-principal-or-an-api-key) + - [The IAM policy](#the-iam-policy) + - [Quotas](#quotas) + - [The `oci` CLI](#the-oci-cli) + - [What Terraform builds, per node](#what-terraform-builds-per-node) + - [Every Terraform variable](#every-terraform-variable) - [Troubleshooting](#troubleshooting) -- [Limits in this version](#limits-in-this-version) +- [Deploying without Terraform](#deploying-without-terraform) --- -## Overview +## Deploy -Spinifex is an open-source infrastructure platform that brings core AWS services to bare-metal, edge and on-prem environments. A node runs the Spinifex daemon and CLI, QEMU/KVM for guests, OVN/Open vSwitch for VPC networking, [Predastore](https://github.com/mulgadc/predastore) for S3-compatible object storage and [Viperblock](https://github.com/mulgadc/viperblock) for EBS-compatible block storage — and serves them all behind a SigV4 endpoint that the ordinary AWS SDKs and CLI talk to unmodified. +### What you need -This guide runs that same stack on Oracle Cloud Infrastructure. Everything in [Single-Node Install](/docs/install) and [Multi-Node Install](/docs/install-multi-node) still applies: one node is a working install, three is the minimum for a cluster that can lose a node, and formation, storage and OVN behave exactly as they do on hardware you own. What changes is the layer underneath — an OCI VCN is not an Ethernet segment, so the external datapath is wired differently, and public addresses come from OCI's API rather than from a range you write in a config file. - -**Read this guide end to end and you have a working cluster.** [Deploying](#deploying) is one sequence of eight steps, from an empty compartment to a node that can launch an instance with a publicly reachable address, and then to the Oracle Linux guest image to put on it. It builds the infrastructure with Terraform, then installs and forms Spinifex the same way [Single-Node Install](/docs/install) and [Multi-Node Install](/docs/install-multi-node) do — those steps are inlined here, so you do not need to read three documents at once. The only choice to make is one node or three, and it is one line in Step 1. - -Every command and every output below was run against a live OCI tenancy, on both paths. - -## Support status - -**What is written here was proven on one shape, in one region, on one guest OS.** Three nodes of `VM.Standard.E6.Flex` in `ap-sydney-1`, Ubuntu hosts, routed NAT, OCI reserved public IPs over secondary private IPs, with a single-node pass of the same guide beside it. Nothing about the integration is region-specific or shape-specific by design, but nothing else has been run, so treat another region, a bare-metal shape or a different host image as untested rather than unsupported — and record the shape, region, fault domains, quotas, kernel and `spx version` when you reproduce it, because that is the only way a later difference can be attributed. - -**OCI is not in the nightly test matrix.** The routed-NAT datapath this integration depends on is guarded nightly, on three nodes, on our own hardware — the behaviour is covered, the cloud underneath it is not. So an OCI-specific regression would be found by the next person to run this guide rather than by CI. That is a real gap and it is tracked; it is not a reason to avoid the integration, but it is a reason not to read "proven" as "continuously proven". - -### What happens when a node fails - -A three-node cluster recovers a guest from a node that has stopped heartbeating, onto a survivor, with no operator action. There is no setting for it: an instance whose host is gone is down either way, and the only thing a switch could buy is leaving it down. It is inert below three nodes, because a single node has nowhere to move a guest to. - -The cluster is deliberately slow to believe a node has gone. A daemon takes no recovery action until it has seen every peer live or five minutes have passed, so a coordinated restart does not look like three simultaneous host failures; a node has to be seen stale across two passes fifteen seconds apart; and the guest's volume lease has to lapse before anything else may open it. Expect a recovery to take minutes, not seconds. - -| Condition | What the cluster does | What you see | -| --- | --- | --- | -| A node stops heartbeating and a survivor can reach its own object store | A survivor claims the instance and relaunches it | The same instance ID, the same volume and the same private address on a new host. `DescribeInstanceStatus` reports the move, as it would on EC2 | -| The volume is still leased by the node that had it | Retries every 15–90 seconds until the lease lapses | Recovery takes about a lease TTL longer. The guest never runs in two places | -| A survivor's own object store is not answering | That node stands down and lets the others try | If the object store is broken everywhere, nothing moves and nothing claims to have moved. An instance that no node could have run stays where it is | -| No survivor has capacity | Each survivor tries in its own order, with backoff | After the attempt budget the instance is stopped carrying `Server.InsufficientInstanceCapacity`, one `StartInstances` from running | -| Anything else — a torn volume, an OCI API denial, a launch that will not finish | Retries with doubling backoff, bounded at twelve attempts, roughly an hour and a half | The instance is stopped carrying `Server.HostRecoveryFailed` and the last error. It is never left looping with nothing visible to you | -| A planned whole-cluster restart | The settle window suppresses recovery while nodes return | Guests are not migrated out from under a deploy | - -**The public address follows the guest, and it keeps its identity while it does.** An OCI public address is a reserved public IP object that OCI 1:1-NATs onto a secondary private IP on a VNIC, and the guest only ever sees the private half. Recovery moves the private half between VNICs in the same subnet — one OCI call, server-side, keeping the OCID — so the reserved public IP is never released, never reallocated and never renumbered. The customer's address is the same address before and after. What does change is which node OCI delivers it to, and each node claims its own guests' addresses on a fifteen-second pass, so there is a window after the guest is running in which it is not yet reachable on its public address. Private VPC addresses have no such window: they are node-independent. - -**What this is not.** A fenced guest is stopped locally, and the volume lease is what stops it — it is not a write barrier the storage backend enforces. A node that loses contact with the cluster gives its volume up on a clock it can evaluate without reaching anything, which bounds the exposure to one renewal interval rather than to the lease TTL, and a survivor does not open the volume until the lease has lapsed. But a write the losing node had already issued is not refused by the object store, and the failure to stop the local writer is reported rather than corrected. So the guarantee is "one running guest, and a bounded, logged window" — not "a second writer is impossible". If your workload cannot tolerate that, it wants a replicated datastore rather than a recovered instance. - -## Why run Spinifex on OCI - -**Lower cost for the same shape of workload.** OCI's compute and block storage are priced well below the large US clouds for equivalent OCPU and IOPS, and Spinifex turns one large instance into a whole EC2/EBS/S3 surface — so a tenant's twenty small instances are twenty QEMU guests on hardware you are billed for once, not twenty billable cloud instances. - -**Egress stops being the line item that decides the architecture.** Oracle publishes a large monthly outbound-transfer allowance per tenancy and a per-GB rate beyond it that is roughly an order of magnitude below the majors — check the current pricing page rather than a number in a document, but the gap is the reason this integration exists. Traffic *between* guests never leaves the host or, on a cluster, never leaves the VCN: it is Geneve over the private plane, which is not billed as internet egress at all. - -**Portability — the workload stops being AWS-shaped and starts being yours.** The API your tenants code against is Spinifex's, so the same Terraform, the same SDK calls and the same AMIs run on OCI today, on bare metal in your own rack tomorrow, and at an edge site after that. Moving off AWS becomes a deployment decision rather than a rewrite, and moving *again* costs nothing extra. - -**You keep the parts of OCI that are worth keeping.** Bare-metal shapes, fault domains, block volumes with a chosen performance tier, and real public addresses out of OCI's own pool. Spinifex does not hide the cloud underneath it — it registers each guest's address with OCI so that guest is genuinely reachable on the internet, not behind a shared NAT. - -## What OCI changes +| On your workstation | Why | +| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | +| Terraform or OpenTofu, `git`, Python 3 | Python is used only by a helper that is standard library only, so there is nothing to `pip install` | +| An SSH keypair at `~/.ssh/oci-spx` | Created in step 1. The deploy refuses to start without it rather than building instances you cannot log in to | +| An OCI compartment **OCID** | This deploys into a compartment that already exists. Creating one needs tenancy-root rights the API user must not have | +| OCI API credentials in `~/.oci/config` | For Terraform itself, to build the infrastructure | +| A way for **the nodes** to call OCI | Separate from the above, and the one prerequisite worth reading first: an [instance principal or an API key](#credentials--an-instance-principal-or-an-api-key) | > [!IMPORTANT] -> **Read this section before you provision anything.** OCI enforces addressing inside its virtual network in a way that makes one of Spinifex's three external modes unusable, and the choice is baked in at `spx admin init`. Getting it wrong produces a cluster that looks entirely healthy and drops every guest packet. +> **That last row is the one that bites.** Terraform's credentials build the infrastructure; the nodes need their own credentials at runtime to allocate public addresses. A node with none forms, passes every health check, and then refuses every launch that wants a public address with `InsufficientAddressCapacity`, naming no cause outside its own journal. Decide between the two now — it changes one argument in step 4. -Three properties of an OCI virtual network shape everything below. +### 1. Get the configuration -**OCI enforces the source IP per VNIC.** A packet leaving the instance with a source address that is not a registered private-IP object on that VNIC is dropped by the hypervisor. Measured, not inferred: from the same NIC with the same MAC, `ping -I 8.8.8.8` gets 0% loss and `ping -I 8.8.8.8` gets 100%. So an external address pool cannot be a range you write in a config file — **every address has to be registered with OCI before it works**, and registering it is exactly what this integration does. - -> Pinging the VCN router (`10.200.0.1`) proves nothing. It answers from any source address at ~0.08 ms because the hypervisor replies locally. Always test egress past the subnet. - -**A VNIC accepts one MAC; it does not learn MACs.** Every inbound frame arrives addressed to the VNIC's own MAC, whatever address the packet is for. Spinifex's default external datapath advertises each address by ARPing from a per-VPC router MAC onto a shared L2 segment, which needs a segment that learns MACs. OCI has none. Enforcement sits in the hypervisor on VM shapes and in the SmartNIC on bare metal, so there is no guest-side workaround. - -**A public IP is a second object, and it never appears on the wire.** An OCI public address is a RESERVED public IP that OCI 1:1-NATs upstream onto a **secondary private IP** on the VNIC. The instance only ever sees the private half. Spinifex handles this internally — AWS-facing APIs report the public address, the datapath uses the private one — but it explains why `ip addr` on the host never shows a customer's public IP, and why that is correct rather than a fault. - -### The consequence: routed mode is mandatory - -| External mode | What goes on the uplink | On OCI | -| --- | --- | --- | -| `pool` (`UplinkModePhysical`, `UplinkModeVeth`) | Per-VPC gateway MAC plus a per-rule external MAC, ARPed onto the segment | **Does not work.** Egress is dropped (wrong source MAC); ingress never reaches OVN (the frame is addressed to the VNIC MAC, which no router port owns). | -| `nat` (`UplinkModeRouted`) | Nothing. The host forwards over an RFC 6598 transit veth and sets no external MAC | **Works.** One MAC identity, and registered source addresses. | - -Spinifex **refuses** `source = "oci"` on a pool unless the node is in `nat` mode, and the refusal names this reason. See [VPC Networking → External modes](/docs/vpc-networking) for what each mode does off OCI. - -### Sizing - -**Bare metal is the recommendation.** `BM.Standard.E2.64` or similar gives you the hardware directly — no nested virtualisation, full NIC performance, and no hypervisor between your guests and the wire. - -| | Minimum | Recommended | -| --- | --- | --- | -| **Shape** | `VM.Standard.E6.Flex` | Bare metal, e.g. `BM.Standard.E2.64` | -| **OCPU** | 8 (16 vCPU) | 32+ | -| **RAM** | 32 GB | 128 GB+ | -| **Data volume** | 256 GiB block volume | 1 TB+ NVMe-backed | -| **VNICs** | 2 | 2 | -| **Nodes** | 1 | 3 — see [Cluster sizing](/docs/install-multi-node#cluster-sizing) | - -**Verify nested virtualisation before anything else on a VM shape.** Oracle's nested-KVM guidance names `VM.Standard3.Flex` (Intel) and `VM.Standard.E5.Flex` (AMD), and states Ampere shapes do not support it. `VM.Standard.E6.Flex` is **not** in that list but was measured working on 2026-09-24 (EPYC 9J45, `/dev/kvm` present, `svm` on all threads, `kvm_amd nested=1`). Treat an unlisted shape as unproven until you have checked: +The Terraform configuration and the deploy driver live in the Spinifex repository: ```bash -# If /dev/kvm is absent, stop — change shape, do not continue. -ls -l /dev/kvm -grep -oE 'vmx|svm' /proc/cpuinfo | sort -u -cat /sys/module/kvm_amd/parameters/nested 2>/dev/null \ - || cat /sys/module/kvm_intel/parameters/nested +git clone https://github.com/mulgadc/spinifex.git +cd spinifex +git switch dev +cd scripts/terraform/oci-spx ``` -**OCI has no Debian image.** Ubuntu LTS is the tested path — Ubuntu 26.04 LTS on kernel `7.0.0-1009-oracle` is what this integration was built and proven against. Anything in the tooling that assumes Debian is a known gap, not a supported configuration. - ---- - -## How it fits together - -### Single node — the whole AWS surface inside one OCI instance - -One bare-metal instance runs the control plane and every tenant guest. Customers reach the AWS APIs on the node's own address; their instances reach the internet through addresses registered on the node's second VNIC. - -

- Single node on OCI — the instance inside the VCN public subnet, both VNICs, br-wan, the routed-NAT transit veth, the control plane, tenant VPCs and the iSCSI data volume -

- -Two things in that picture are the whole integration. The guest's private address (`10.0.1.4`) never leaves the host — OVN and the host route both hold it. The address OCI delivers to is a **secondary private IP registered on VNIC 1**, appearing on `br-wan`, and the customer's public address (`150.230.13.131`) is a reserved public IP that OCI 1:1-NATs onto it upstream. `describe-instances` reports the public one; every datapath object holds the private one, which is why the two never appear together on the host. - -The third is storage. Every guest disk, every S3 object and the JetStream state all live under `/var/lib/spinifex`, which is an **OCI block volume attached over iSCSI**, not the boot disk. Size it for the guests you intend to run; a boot volume that quietly ends up carrying the stack is the most common way an install on OCI fails later rather than immediately. - -### Three nodes — the same surface, distributed - -Each node registers the addresses for **its own** guests on **its own** second VNIC, and the allocator runs per node, so nothing has to know another node's OCIDs. When a guest moves — a stop/start that lands elsewhere, or a host failure — the node it lands on claims the address's private half from the old VNIC through a single OCI call. The OCID and the public half are untouched, so the customer's address never changes. - -

- Three nodes on OCI — three instances inside the VCN public subnet, one per fault domain, each with its own VNICs, addresses and iSCSI volume, joined east-west by Geneve, NATS, predastore and the OVN raft -

- -The overlay, the object shards and the OVN databases all cross **VNIC 0**, inside the VCN. Size that plane, not the external one: guest-to-guest traffic between nodes is Geneve over the VCN, and it is the link every distributed layer shares. +```bash +# The deploy installs this on every node and logs in with it. Any type OpenSSH +# supports is fine; it must be at this path, or named with --ssh-public-key. +ssh-keygen -t ed25519 -f ~/.ssh/oci-spx -N '' -C spinifex-oci +``` > [!NOTE] -> **The Spinifex region is your own naming and has nothing to do with the OCI region.** `ap-southeast-2` in `spinifex.toml` beside `ap-sydney-1` in OCI is correct and expected. Do not read one as evidence about the other. +> **`git switch dev` is needed until the next release ships.** OCI support is not in `v1.21.0`, so the configuration you are about to run is not on `main` yet. That ref decides the _Terraform and the driver_ only — which Spinifex build lands on the nodes is a separate choice, made by `--version` in step 4. ---- +### 2. Choose one node or three -## Prerequisites +**This is the only architectural decision, and it is one line in step 3.** -Both paths need the same three things: an OCI compartment you can write to, an API key for Spinifex to allocate addresses with, and enough quota to hand out the addresses you plan to use. +| | Single node | Three nodes | +| ------------------------------ | -------------------------------------------- | ------------------------------------------------------------------------------------------------ | +| **`node_count`** | `1` | `3` | +| **Survives losing a node** | No | Yes — NATS keeps quorum, predastore RS(2,1) keeps the data readable, the OVN raft keeps a leader | +| **Public addresses available** | 64 — the per-VNIC secondary private IP limit | 192, three VNICs' worth, still bounded by the regional reserved-public-IP quota | +| **Fault domains** | One | Three, one per node, chosen by Terraform | +| **Use it for** | Evaluation, a lab, an edge site with one box | Anything you would be unhappy to lose | -### Credentials — v1 uses an operator-provisioned API key +Two is not an option worth taking: it doubles the cost of a single node and gives you a cluster that cannot form a quorum. See [Cluster sizing](/docs/install-multi-node#cluster-sizing) and [Sizing](#sizing) for shapes. -**Instance principal is the better design and is not what v1 uses.** The instance certificate is served at `/opc/v2/identity/cert.pem` and the Go SDK can authenticate as the instance with no key material on disk — but it still needs a dynamic group and a policy authorising it, and on the reference tenancy it authenticated without being authorised for anything. **v1 requires an operator-provisioned API key**, deliberately, to keep the setup a single documented path. +### 3. Write your Terraform inputs + +One hand-edited file, in the directory you are already in. Every variable has a default that builds a working single node, so only `compartment_ocid` is strictly required: ```bash -# On your workstation, not the node. -openssl genrsa -out ~/.oci/oci_api_key.pem 2048 -chmod 600 ~/.oci/oci_api_key.pem -openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem -openssl rsa -pubout -outform DER -in ~/.oci/oci_api_key.pem \ - | openssl md5 -c # this is the fingerprint -``` - -Upload the public key to the user under **Identity → Users → API Keys**, and record the user OCID, the fingerprint and the tenancy OCID. - -### The IAM policy - -Spinifex calls exactly nine operations. Grant no more than these: - -| Operation | Why | -| --- | --- | -| `CreatePrivateIp` | Register the secondary private IP the datapath carries | -| `DeletePrivateIp` | Release it | -| `GetPrivateIp` | Poll for `AVAILABLE` after an asynchronous assign | -| `ListPrivateIps` | The startup reconcile, and the affinity pass that moves an address between nodes | -| `UpdatePrivateIp` | Reassign a private IP to this node's VNIC when a guest moves here | -| `CreatePublicIp` | Create the RESERVED public IP | -| `DeletePublicIp` | Release it | -| `GetPublicIp` / `GetPublicIpByPrivateIpId` | Poll lifecycle state, and reconcile a private IP back to its public half | -| `UpdatePublicIp` | Attach and detach the public IP from a private IP | - -The matching policy, scoped to the one compartment Spinifex allocates in: - -```text -Allow group SpinifexOperators to manage private-ips in compartment -Allow group SpinifexOperators to manage public-ips in compartment -Allow group SpinifexOperators to read vnics in compartment -Allow group SpinifexOperators to read subnets in compartment -Allow group SpinifexOperators to read vcns in compartment -``` - -**Scope it to a compartment, not the tenancy.** `manage public-ips` at tenancy level lets a compromised node consume the whole regional reserved-public-IP quota. - -**Do not grant `manage instances` or `manage vnics`.** Spinifex never creates, attaches or deletes a VNIC — it only adds addresses to a VNIC that already exists. A grant that allows VNIC lifecycle is strictly more than the integration can use. - -### Quotas to check before you size a deployment - -Two limits bound how many public addresses a deployment can hand out, and they do not scale the same way: - -| Quota | Default | Scope | Notes | -| --- | --- | --- | --- | -| **Reserved public IPs** | 50 | Per region, whole tenancy | Shared by every node, so it does **not** grow with node count. This is the wall you hit first. | -| **Secondary private IPs per VNIC** | 64 | Per VNIC, so per node | Not raisable. Three nodes gives you 192 — this is the limit that does scale. | - -A raise on the first is a support request, not a setting: Console → Governance & Administration → Limits, Quotas and Usage → "Request a service limit increase". Cite the exact limit name from `oci limits definition list`, not a name from documentation. - -```bash -oci limits definition list --service-name vcn --compartment-id -oci limits value list --service-name vcn --compartment-id -oci limits resource-availability get --service-name vcn \ - --limit-name --compartment-id -``` - -**These calls need a grant the node does not have, and must not be given.** Limits and usage are tenancy-level, which cannot be compartment-scoped the way the policy above insists the node's is. Create a **second, read-only principal** for an operator or a reporting job and keep it off every node: - -```text -Allow group SpinifexReporting to read limits in tenancy -Allow group SpinifexReporting to read usage-reports in tenancy -``` - -The second verb is what Cost Analysis and `oci usage-api usage-summary request-summarized-usages` read. Adding either to `SpinifexOperators` would hand every node tenancy-wide read, which is exactly what the node policy refuses. - -### The `oci` CLI - -**The `oci` CLI is not on the Ubuntu image**, and `apt` has no `python3-oci-cli` package. Install it into a virtualenv on your workstation, and on the node if you want to inspect allocations there: - -```bash -sudo apt-get update && sudo apt-get install -y python3-venv -python3 -m venv ~/.oci-cli-venv -~/.oci-cli-venv/bin/pip install --upgrade pip oci-cli -sudo ln -sf ~/.oci-cli-venv/bin/oci /usr/local/bin/oci -oci --version -``` - -**What the CLI is and is not for.** Spinifex's daemon uses the OCI **Go SDK** for every allocation — the CLI is not on that path. You need it for bootstrap and for operational inspection. Do not build automation that shells out to it during an allocation; allocations run inside a request budget that a forked Python process will not respect. - ---- - -## Deploying - -**Terraform builds the infrastructure; the standard Spinifex installer builds the node.** That split is deliberate — formation on OCI is the same path prod and bare metal take, and diverging it for one cloud would mean two formation paths to keep correct. - -There is one sequence of eight steps below, and the only thing that differs between a single node and a cluster is `node_count` in Step 1 and which variant of Steps 4 and 5 you follow. Decide now: - -| | Single node | Three nodes | -| --- | --- | --- | -| **`node_count`** | `1` | `3` | -| **Survives losing a node** | No | Yes — NATS keeps quorum, predastore RS(2,1) keeps the data readable, the OVN raft keeps a leader | -| **Public addresses available** | 64 — the per-VNIC secondary private IP limit | 192, three VNICs' worth, still bounded by the regional reserved-public-IP quota | -| **Fault domains** | One | Three, one per node, chosen by Terraform | -| **Use it for** | Evaluation, a lab, an edge site with one box | Anything you would be unhappy to lose | - -Two is not an option worth taking: it doubles the cost of a single node and gives you a cluster that cannot form a quorum. See [Cluster sizing](/docs/install-multi-node#cluster-sizing). - -The configuration lives in [`scripts/terraform/oci-spx`](../../scripts/terraform/oci-spx/README.md), which has the resource-by-resource architecture. - -### What Terraform builds, per node - -| Resource | Detail | -| --- | --- | -| `oci_core_instance` | One per `node_count`, round-robined across **fault domains** so three nodes land on three sets of racks rather than wherever placement puts them | -| Primary VNIC | Public IP, `hostname_label`, the host plane — SSH, the advertise address, the Geneve endpoint | -| `oci_core_vnic_attachment` | A **second VNIC** in the same subnet. Every Spinifex external address becomes a secondary private IP here at runtime | -| `oci_core_volume` | A block volume per node, `data_volume_size_in_gbs` at `data_volume_vpus_per_gb` | -| `oci_core_volume_attachment` | **`attachment_type = "iscsi"`**, with the Oracle Cloud Agent's Block Volume Management plugin enabled so the node logs the iSCSI session in itself | -| cloud-init | Partitions, formats, mounts and `fstab`s the volume; builds `br-wan` over the second VNIC; opens the service ports | - -**The data volume is iSCSI, and that is the detail that bites.** OCI presents the volume as an iSCSI target reachable at `169.254.2.2:3260` — attaching it in the API does not put a block device on the host. `is_agent_auto_iscsi_login_enabled` makes the Oracle Cloud Agent do the login, and because it does that *asynchronously*, the device is not there when cloud-init first runs. The mount script waits up to ten minutes for it, formats it **only if `blkid` reports no filesystem** (so a re-run cannot erase a populated volume), and writes an fstab entry by UUID: - -```text -UUID= /var/lib/spinifex ext4 defaults,_netdev,nofail 0 2 -``` - -`_netdev` is what orders the mount after the network, and therefore after `iscsid` has a session. `nofail` is what keeps a missing volume from wedging the boot. Mounting `/var/lib/spinifex` *itself* rather than somewhere else and symlinking is deliberate: viperblock, predastore and JetStream all land on the volume with no configuration pointing anywhere unusual. - ---- - -## Step 1. Set your Terraform inputs - -Prerequisites: Terraform, Python 3, OCI credentials in `~/.oci/config`, and an SSH public key. The Python helper builds and uses the configuration's own `.venv`. - -```bash -cd scripts/terraform/oci-spx cat > terraform.auto.tfvars <<'EOF' compartment_ocid = "ocid1.compartment.oc1..aaaa..." deployment_name = "spinifex" @@ -330,463 +130,114 @@ node_client_cidr_allow_list = ["203.0.113.10/32"] EOF ``` -The variables worth knowing, all of which have defaults that build a working single node: - -| Variable | Default | What it decides | -| --- | --- | --- | -| `compartment_ocid` | — | **Required.** An OCID, not a name: this deploys into a compartment that already exists, because creating one needs tenancy-root rights the API user must not have | -| `deployment_name` | `spinifex` | Prefixes every resource and becomes the VCN `dns_label`, so 1–15 lowercase alphanumerics starting with a letter | -| `node_count` | `1` | Nodes, 1–25. `1` for a single node, `3` for a cluster. Each gets its own volume, its own second VNIC and its own fault domain | -| `compute_shape` | `VM.Standard.E6.Flex` | **`BM.*` for bare metal, `VM.*.Flex` for a VM.** A fixed bare-metal shape carries its own size, so `shape_config` is emitted only when the name contains `Flex` — set a BM shape and the two sizing variables below are ignored rather than rejected | -| `compute_ocpus` | `8` | OCPUs on a flex shape. Validated at ≥ 8 (16 vCPU), the minimum recommended for a node | -| `compute_memory_in_gbs` | `32` | GiB on a flex shape. Validated at ≥ 32 | -| `data_volume_size_in_gbs` | `256` | Size of each node's data volume. This is where guest disks, S3 objects and JetStream live — size it for the guests, not for the control plane | -| `data_volume_vpus_per_gb` | `120` | Performance tier, 0–120 VPUs/GB. 120 is Ultra High Performance; 10 is Balanced. Higher is more IOPS and more cost | -| `data_mountpoint` | `/var/lib/spinifex` | Where cloud-init mounts it. Change only if you know why | -| `vcn_cidr` | `10.200.0.0/22` | VCN address space. A `/22` leaves room for the secondary private IPs Spinifex registers per node | -| `public_subnet_cidr` | `10.200.0.0/23` | The subnet every node sits in, **on both VNICs**. Nodes serve the UI, the S3 gate and the AWS gateway directly, so they cannot live in a private subnet | -| `node_client_cidr_allow_list` | `["0.0.0.0/0"]` | Who may reach SSH and the service ports from the internet. **Change this** | -| `node_service_ports` | `[3000, 8443, 9999]` | Console, predastore gate, AWS gateway. SSH is opened separately | -| `ubuntu_version` | `26.04` | Canonical platform image version. The newest image for the shape and region is resolved at plan time rather than pinned to a regional OCID | -| `wan_bridge_name` | `br-wan` | The Linux bridge built over the second VNIC | -| `wan_bridge_mtu` | `9000` | The OCI VCN carries 9000 | +Add `instance_principal = "adopt"` if you went that way in [Credentials](#credentials--an-instance-principal-or-an-api-key). [Every Terraform variable](#every-terraform-variable) has the full list. > [!WARNING] -> **`node_client_cidr_allow_list` defaults to `0.0.0.0/0`.** Nodes sit in a public subnet with public addresses because they serve the UI, the S3 gate and the AWS gateway directly — there is no bastion. Restrict this before any real use. +> **`node_client_cidr_allow_list` defaults to `0.0.0.0/0`.** Nodes sit in a public subnet with public addresses because they serve the UI, the S3 gate and the AWS gateway directly — there is no bastion. Restrict it to your own address before any real use, as the example above does. -> [!NOTE] -> **Bare metal needs the Terraform to place the external VNIC, and that is now automatic.** Set `compute_shape = "BM.Standard.E2.64"` and the same config builds it: the two flex sizing variables stop applying and the shape's own CPU and memory take over. Nested virtualisation stops being a question, because there is no nesting. -> -> Two steps are *not* identical to a VM, and both are handled for you rather than being yours to work around. On bare metal the OS owns the physical ports, so the first VNIC on a port **is** the port and a second one sharing it is a tagged overlay with no Linux device at all — the external VNIC is therefore attached at `nic_index = 1`, making it the first VNIC on the second port and so untagged and present at boot, exactly as on a VM. And OCI makes a bare-metal volume attachment multipath, where the Oracle Cloud Agent will not log the session in unless multipathd is running but probes the unit name Debian retired; cloud-init installs the alias it looks for. The device then appears as `/dev/mapper/mpatha` rather than a plain disk. -> -> Proved on `BM.Standard.E2.64` on 2026-10-01: both VNICs untagged, `br-wan` built unattended with egress past the subnet, the volume mounted, and guests running. The external-address path on bare metal is **not** yet proved, because it needs the credentials in Step 5. +### 4. Run the deploy -## Step 2. Build the infrastructure +One command builds the infrastructure, installs Spinifex, forms the cluster, configures OCI public addressing and verifies the result: ```bash -KEY=~/.ssh/oci-spx.pub -python3 scripts/oci_env.py --ssh-public-key-path "$KEY" -- terraform init -python3 scripts/oci_env.py --ssh-public-key-path "$KEY" -- terraform plan -out=oci-spx.tfplan +./validate-topology.sh \ + --topology vm-single \ + --version v1.21.0-70-g1743b602-dev-1-g66bc1172-dev \ + --credential-hook ~/.spinifex/oci-credential-hook.sh \ + --keep ``` -Read the saved plan, then apply **that exact plan** rather than re-planning: +About ten minutes for a single node, printing each stage as it finishes. The four arguments: -```bash -python3 scripts/oci_env.py --ssh-public-key-path "$KEY" -- terraform apply oci-spx.tfplan -``` +| Argument | What to pass | +| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--topology` | `vm-single` for one VM, `vm-multi` for a three-node cluster, `bm` for one bare-metal host. Required, with no default, so a command cannot be aimed at the wrong one by omission | +| `--version` | The Spinifex release to install on the nodes. **Use the tag above until the next release ships**, because OCI support is not in the current stable release yet | +| `--credential-hook` | Only when nodes authenticate with an API key — see [Credentials](#credentials--an-instance-principal-or-an-api-key). With an instance principal, replace it with `--instance-principal` | +| `--keep` | **This is what makes it a deployment rather than a test.** Without it the driver destroys everything at the end, which is correct for CI and not what you want here | -**The apply ends with the addresses you need for every remaining step.** With `node_count = 3` it prints twenty-two resources and this: +Also worth knowing: + +- `--skip-workload` stops after verifying the cluster. Without it the driver launches real guests on public addresses and tears them down again, which is the only check that proves end-to-end connectivity rather than inferring it. +- `--dry-run` prints the plan and the per-node steps and changes nothing. +- `--ssh-public-key` / `--ssh-private-key` if your keypair is not at `~/.ssh/oci-spx`. + +> [!TIP] +> Prefer `--version ` over `--channel dev`. Both install a prerelease, but `--channel dev` resolves through a GitHub API call that is rate limited per source address, and when it trips it returns a 404 that looks exactly like a missing release. A tag is a plain redirect and has no such failure mode. + +### What a successful run looks like + +The stages, in order, and all of them have to appear: ```text -Apply complete! Resources: 22 added, 0 changed, 0 destroyed. - -Outputs: - -compartment_name = "mulgadc" -compartment_ocid = "ocid1.compartment.oc1..aaaaaaaa6nkk...ztta" -hosts_file = </`: -Read `nodes` twice, because the two addresses in each entry are used for different things and confusing them wastes an afternoon: +| Stage | Log | +| ----------------------------------------------- | ---------------------------------------- | +| Building the infrastructure | `apply.log` | +| Installing Spinifex | `install-.log` | +| Forming the cluster | `form.log` | +| Installing the credential, configuring the pool | `credential-hook.log`, `pool-.log` | +| Verifying public addressing | `allocator-.log` | -- **`public_ip`** is how *you* reach the node — SSH, the console on `:3000`, the AWS gateway on `:9999`. It is on the primary VNIC and Terraform assigned it. -- **`private_ip`** is what goes in `--bind`, `--cluster-bind`, `--encap-ip` and `--db-cluster-*-addr` in Steps 4 and 5. **Every cluster flag takes the private address**, because the mesh runs inside the VCN and never over the public one. -- **`fault_domain`** is not cosmetic. Three nodes in three fault domains is what makes a three-node cluster survive a rack, and it is chosen here, not later. +The last one is the log worth knowing about, because public addressing is the part with the most moving pieces. If the allocator never comes up, the driver captures the daemon journal, the pool as configured, the bridge and OCI MAC addresses, and whether each credential file exists and is readable — never the credentials themselves — before anything is torn down. [Troubleshooting](#troubleshooting) reads those. -`terraform output` reprints all of this at any time, and `terraform output -json nodes` is the machine-readable form. +Add `--keep-on-fail` to leave a failed deployment running so you can log in and look. **It keeps billing** until you destroy it. -> [!WARNING] -> Never commit the state file, a plan file, or key material. `.gitignore` covers `terraform.tfstate*`, `*.tfplan`, `*.auto.tfvars`, `terraform.tfvars` and `*.pem`. - -## Step 3. Install Spinifex and set up OVN - -Spinifex installs the same way here as anywhere else. Run this **on every node**: +### Removing the deployment ```bash -curl -fsSL https://install.mulgadc.com | bash +./validate-topology.sh --topology vm-single --destroy-only ``` -That puts the binary, the systemd units and the helper scripts in place and starts nothing. Then wire OVN, which is where the single-node and multi-node paths differ. +It destroys whatever that topology's Terraform state holds, so it can only remove what this configuration created. -Two flags are the same on every node and on both paths, and both are decisions you cannot change afterwards without re-doing this step: +--- -- **`--management` is load bearing.** Omitting it takes the compute-node branch, which stops *and disables* `ovn-central` — silently, leaving a node with no record of what it was meant to be. -- **`--nat-uplink`, not `--wan-bridge=br-wan`.** The two are mutually exclusive, and only `--nat-uplink` creates the `spx-nat` transit veth that routed mode runs on. `--wan-bridge` takes the veth-to-`br-ext` branch and deletes `spx-nat` on the way, after which `host.Routed.EnsureUplinkPort` refuses with *"not on OVS — run setup-ovn.sh --nat-uplink"*. - -**That does not make `br-wan` redundant.** It is still doing two jobs, neither of which involves OVS: - -1. It **names the VNIC** for the OCI allocator. `oci_vnic_iface = "br-wan"` in Step 6's pool config is resolved to a VNIC OCID **by MAC**, on the node it runs on — which is what lets one identical config file be correct on all three nodes. -2. It **holds the addresses.** Every external address Spinifex registers becomes a secondary private IP on that VNIC, appears on `br-wan`, and gets its own source-routing rule into table 200. The guests' own traffic reaches it through `spx-nat` and the host's `spinifex-nat-egress` masquerade. - -So a correct OCI node has `br-wan` as a **Linux** bridge with no OVS port at all, and `br-ext` as an OVS bridge whose ports are `spx-nat-ovs` and the patch to `br-int`. That asymmetry is the design, not a half-finished install. - -> If you suspect an OVS problem on this host, read `ovs-vsctl --columns=name,error list Interface` and the vswitchd log. **Never trust `ovs-vsctl`'s exit status** — it exits 0 even when the datapath rejected the device. - -### Single node +## Verify end to end ```bash -sudo /usr/local/share/spinifex/setup-ovn.sh --management --nat-uplink +export AWS_PROFILE=spinifex + +aws ec2 allocate-address +aws ec2 run-instances --image-id --instance-type t3.micro --key-name \ + --subnet-id --associate-public-ip-address +aws ec2 describe-instances --query \ + 'Reservations[].Instances[].[InstanceId,PrivateIpAddress,PublicIpAddress,State.Name]' \ + --output text ``` -### Three nodes - -Servers 1 to 3 run a clustered **OVN database**, so VPC networking survives losing any one of them. **Node 1 creates the cluster and must finish first**; nodes 2 and 3 then join it. `--recreate-db` appears in all three because `ovn-central` starts a standalone database when the package installs, and a clustered one can only be created from scratch. - -Every address below is a **private** IP from the Terraform `nodes` output. +Then reach the guest on its public address — **from your workstation, not from the node**: ```bash -# Run on every node, with the values from your own apply. -export SPINIFEX_NODE1=10.200.1.249 -export SPINIFEX_NODE2=10.200.0.201 -export SPINIFEX_NODE3=10.200.1.79 -export SPINIFEX_OVN_REMOTE=tcp:$SPINIFEX_NODE1:6642,tcp:$SPINIFEX_NODE2:6642,tcp:$SPINIFEX_NODE3:6642 +ping -c 3 +ssh -i ubuntu@ hostname ``` -`--ovn-remote` is the whole member list, and **it is the same on all three nodes** — do not trim it to the peers. It is what `ovn-northd` dials, and an OVSDB RAFT follower does not forward writes: it answers *not cluster leader, trying another server*. A northd pointed at one member therefore works only while it happens to share a node with the database leader, and stops translating Northbound intent into Southbound flows the moment leadership moves. Nothing about that is visible from `systemctl` — every service stays `active` — and the symptom appears much later as a `terraform apply` hung on `aws_internet_gateway.igw: Still creating...` forever. +> [!IMPORTANT] +> **Probe a public address from outside the VCN, not from a node.** Two things make a node a bad vantage point, and they are indistinguishable from each other and from a broken datapath: `node_client_cidr_allow_list` may not admit the node's own public address, and a VCN may not reflect its public addresses back into itself at all. Either way a healthy cluster fails the probe, so a timeout from a node is not evidence about the guest. Private addresses are the separate question and do work from inside. -**Node 1 — create the cluster:** - -```bash -sudo /usr/local/share/spinifex/setup-ovn.sh \ - --management --nat-uplink \ - --db-cluster-local-addr=$SPINIFEX_NODE1 \ - --ovn-remote=$SPINIFEX_OVN_REMOTE \ - --recreate-db \ - --encap-ip=$SPINIFEX_NODE1 -``` - -**Nodes 2 and 3 — join it,** after node 1 reports `=== OVN compute node setup complete ===`: - -```bash -# on node 2 -sudo /usr/local/share/spinifex/setup-ovn.sh \ - --management --nat-uplink \ - --db-cluster-local-addr=$SPINIFEX_NODE2 \ - --db-cluster-remote-addr=$SPINIFEX_NODE1 \ - --ovn-remote=$SPINIFEX_OVN_REMOTE \ - --recreate-db \ - --encap-ip=$SPINIFEX_NODE2 - -# on node 3 -sudo /usr/local/share/spinifex/setup-ovn.sh \ - --management --nat-uplink \ - --db-cluster-local-addr=$SPINIFEX_NODE3 \ - --db-cluster-remote-addr=$SPINIFEX_NODE1 \ - --ovn-remote=$SPINIFEX_OVN_REMOTE \ - --recreate-db \ - --encap-ip=$SPINIFEX_NODE3 -``` - -Each run ends with its own verification block. Node 1 reports `chassis count: 1`; nodes 2 and 3 report `0`, because a chassis is registered by `ovn-controller` a moment later and the script checks immediately. That is not a failure — confirm the real answer from node 1 once all three are done: - -```bash -sudo ovn-appctl -t /var/run/ovn/ovnnb_db.ctl cluster/status OVN_Northbound -sudo ovn-sbctl show -``` - -```text -Name: OVN_Northbound -Cluster ID: 7c7d (7c7d6d29-8243-4f82-8b4e-2848dbd5942e) -Server ID: 5d95 (5d952726-6ab5-4188-8eb9-b70acf37c374) -Address: tcp:10.200.1.249:6643 -Status: cluster member -Role: leader -Term: 1 -``` - -`Status: cluster member` with one leader, and a `Chassis` entry in `ovn-sbctl show` for each of the three. If `cluster/status` says the database is standalone, `--db-cluster-local-addr` did not reach it — re-run that node. - -Then confirm northd is actually translating, which `cluster/status` cannot tell you. Run this on any node: - -```bash -echo "NB $(sudo ovn-nbctl --no-leader-only get NB_Global . nb_cfg) / SB $(sudo ovn-sbctl --no-leader-only get SB_Global . nb_cfg)" -``` - -The two numbers must be equal, or within one of each other on a busy cluster. A gap that does not close within a few seconds means the active `ovn-northd` cannot write to the Northbound leader, and `/var/log/ovn/ovn-northd.log` will be looping on `clustered database server is not cluster leader; trying another server`. The fix is the `--ovn-remote` list above. - -## Step 4. Form the cluster - -Everything here is Spinifex's own formation, unchanged from bare metal except for two flags that OCI requires: - -- **`--external-mode=nat` is mandatory.** See [the consequence](#the-consequence-routed-mode-is-mandatory). `pool` mode produces a cluster that passes every health check and drops every guest packet. -- **`--ipsec=false`:** `openvswitch-ipsec` is masked on the OCI Ubuntu image while `spx admin init` defaults the flag to true. A cluster formed with it on will not bring its overlay up. - -Both are read from node 1 only. Joining nodes inherit the external mode, the transit pool and the IPsec posture, so they are chosen once. - -### Single node - -```bash -sudo spx admin init --node node1 --nodes 1 \ - --region ap-southeast-2 --az ap-southeast-2a \ - --external-mode=nat --ipsec=false -``` - -It finishes in a few seconds: - -```text -📡 External networking: nat (routed; no public pool configured yet) - Transit: 100.127.0.0/24 via spx-nat-host (host masquerades out any uplink) -✅ Created: spinifex.toml -🔧 Configuring AWS credentials... Profile: spinifex -✅ Host DNS: spx3.net + compute.internal -> 10.200.0.252:53 (northstar) -🎉 Spinifex initialization complete! - Advertise IP: 10.200.0.252 -``` - -"no public pool configured yet" is expected — the OCI pool is Step 6. - -### Three nodes - -Init and join run **concurrently**: init blocks until every node has joined. Start node 1, take the token from its output, then run both joins while it is still waiting. - -`--force` is in every command so the sequence is identical whichever way you installed. On a node Terraform just built there is nothing to lose either way; on a node that has been in service, joining discards its master key and orphans every volume sealed under it, and `--force` is the confirmation for that. - -**Node 1 — initialize:** - -```bash -sudo spx admin init --force \ - --node node1 --nodes 3 \ - --bind $SPINIFEX_NODE1 --cluster-bind $SPINIFEX_NODE1 \ - --port 4432 --region ap-southeast-2 --az ap-southeast-2a \ - --external-mode=nat --ipsec=false -``` - -It generates the keys and the CA, then stops and waits, printing the token: - -```text -📡 Formation server started on 10.200.1.249:4432 - Waiting for 2 more node(s) to join... - Token expires in 30m0s - - Other nodes should run: - sudo spx admin join --host 10.200.1.249:4432 --token spx_join_YHRL6y_yVP0E8Tc1 --node --bind -``` - -Take the **token** from that output, but run the commands below rather than the line it prints — they add `--force` and `--cluster-bind`. - -**Nodes 2 and 3 — join,** while init is still waiting: - -```bash -# on node 2 -sudo spx admin join --force \ - --node node2 --bind $SPINIFEX_NODE2 --cluster-bind $SPINIFEX_NODE2 \ - --host $SPINIFEX_NODE1:4432 --token \ - --region ap-southeast-2 --az ap-southeast-2a - -# on node 3 -sudo spx admin join --force \ - --node node3 --bind $SPINIFEX_NODE3 --cluster-bind $SPINIFEX_NODE3 \ - --host $SPINIFEX_NODE1:4432 --token \ - --region ap-southeast-2 --az ap-southeast-2a -``` - -Each join answers with the membership it now sees: - -```text -🎉 Node successfully joined cluster! - Cluster: spinifex (3 nodes) - Bind: 10.200.0.201 Advertise: 10.200.0.201 Loopback: 127.0.0.1 - Nodes: - - node2 (bind=10.200.0.201 advertise=10.200.0.201) - - node3 (bind=10.200.1.79 advertise=10.200.1.79) - - node1 (bind=10.200.1.249 advertise=10.200.1.249) -``` - -and node 1 unblocks: - -```text -🎉 Cluster formation complete! - Cluster: spinifex (3 nodes) - Region: ap-southeast-2 -``` - -Confirm the joiners inherited the OCI-critical settings before going on — this is cheap and catches a node that formed against the wrong leader: - -```bash -sudo grep -E 'external_mode|ipsec_enabled' /etc/spinifex/spinifex.toml -``` - -```text -ipsec_enabled = false -external_mode = "nat" -``` - -The join token expires 30 minutes after init; `--token-ttl 2h` if provisioning is slower than that. - -## Step 5. Configure Spinifex for OCI, start and verify - -**This is the step that is genuinely OCI-specific**, and it is identical on one node or three. Do all of it on **every** node before starting anything. - -### Discover the OCIDs - -```bash -# Instance, compartment and both VNICs, straight from the instance metadata. -curl -sH 'Authorization: Bearer Oracle' http://169.254.169.254/opc/v2/instance/ \ - | python3 -c 'import json,sys; d=json.load(sys.stdin); print("compartment:", d["compartmentId"]); print("instance:", d["id"])' - -curl -sH 'Authorization: Bearer Oracle' http://169.254.169.254/opc/v2/vnics/ \ - | python3 -c 'import json,sys -for v in json.load(sys.stdin): - print(v["macAddr"], v["vnicId"], v["subnetCidrBlock"], v.get("privateIp"))' -``` - -```text -compartment: ocid1.compartment.oc1..aaaaaaaa6nkk...ztta -instance: ocid1.instance.oc1.ap-sydney-1.anzxsljr6eq5...tkxdq - -02:00:17:01:9C:22 ocid1.vnic.oc1.ap-sydney-1.abzxsljrnat7...y6vw5q 10.200.0.0/23 10.200.0.252 -02:00:17:04:1A:57 ocid1.vnic.oc1.ap-sydney-1.abzxsljrjd7z...czgvua 10.200.0.0/23 10.200.1.233 -``` - -**You only need the compartment OCID for the config below** — the VNIC is resolved at runtime. Match a VNIC by MAC against `ip -br link show br-wan` if you want to confirm which is which; Oracle's VNIC ordering is not a documented contract, so "the second one" is not a selector. - -### Choose how the node authenticates - -Two routes. **Instance principal is the better one**: each node authenticates with the certificate its own metadata service serves, so there is no key material on any node, nothing to rotate, and nothing to copy as you add nodes. It needs a dynamic group and an IAM policy, which only a tenancy admin can create — once, after which every node and every rebuild inherits them. The `oci-spx` Terraform creates both behind `enable_instance_principal = true`; its README has the command and what the policy grants. - -With that done, the whole of the next section is skipped and the pool below carries `oci_auth = "instance_principal"` instead of the two `oci_config_*` keys. Setting both is refused at config load, because the certificate is already the credential and a key file beside it would be the half silently ignored. - -**An API key is the fallback**, and the default, because it needs nothing from a tenancy admin. The cost is that the key goes on every node by hand, and a node you forget fails only when a guest asks for an address. - -### Install the credentials on the node - -Skip this entirely under instance principal. - -**The daemon cannot read `~/.oci/`.** Its systemd unit sets `ProtectHome=yes`, so a config under any home directory is invisible to it however the permissions read. Credentials go under `/etc/spinifex/`, on every node: - -```bash -sudo install -d -o root -g spinifex -m 0750 /etc/spinifex/oci -sudo install -o root -g spinifex -m 0640 ~/.oci/oci_api_key.pem /etc/spinifex/oci/oci_api_key.pem -sudo tee /etc/spinifex/oci/config >/dev/null <<'EOF' -[spinifex] -user=ocid1.user.oc1..aaaa... -fingerprint=aa:bb:cc:... -key_file=/etc/spinifex/oci/oci_api_key.pem -tenancy=ocid1.tenancy.oc1..aaaa... -region=ap-sydney-1 -EOF -sudo chown root:spinifex /etc/spinifex/oci/config -sudo chmod 0640 /etc/spinifex/oci/config -``` - -```text -drwxr-x--- 2 root spinifex 4096 Sep 26 03:48 . --rw-r----- 1 root spinifex 303 Sep 26 03:48 config --rw-r----- 1 root spinifex 1715 Sep 26 03:48 oci_api_key.pem -``` - -The profile name — `spinifex` here — is what `oci_config_profile` names below. Keep a copy at `~/.oci/config` too if you want the `oci` CLI on the node; that is a different consumer and it is not on any Spinifex code path. - -### Configure the OCI pool and the IMDS remap - -Both go in `/etc/spinifex/spinifex.toml`, on every node, with the **same text on each** — nothing here is per-node: - -```toml -[network] -external_mode = "nat" -imds_host_meta_ip = "169.254.42.254" -imds_host_dns_ip = "169.254.42.253" - -# Leave the nat-transit pool init wrote exactly as it is, and add this one. -[[network.external_pools]] -name = "oci-public" -source = "oci" -oci_compartment_id = "ocid1.compartment.oc1..aaaa..." -oci_vnic_iface = "br-wan" -oci_config_file = "/etc/spinifex/oci/config" -oci_config_profile = "spinifex" -dns_servers = ["169.254.169.253"] -``` - -Under instance principal the last three lines of that block become one: - -```toml -oci_auth = "instance_principal" -dns_servers = ["169.254.169.253"] -``` - -The Terraform stages whichever of the two is correct at `/etc/spinifex/oci/external-pool.toml` on each node, so this is an append rather than something to type. - -Notes on the keys: - -- **Exactly one of `oci_vnic_id` and `oci_vnic_iface`**, never both — the config is rejected if you set both or neither. Two that disagreed would send allocations to a VNIC the datapath is not on, and OCI would drop the traffic without a word. -- **`oci_vnic_iface` is the right choice on a cluster**, because it resolves through instance metadata **by MAC** on the node it runs on. Naming either `br-wan` or the physical interface resolves to the same VNIC. Use `oci_vnic_id` only when you want to pin a specific VNIC on a single node. -- `oci_compartment_id` is the only other required key. `oci_subnet_id` is optional and defaults to the VNIC's own subnet; set it only when you want private IPs from a different subnet. `oci_config_file` defaults to `~/.oci/config` and `oci_config_profile` to `DEFAULT` — **on a node both need setting**, because the daemon cannot read a home directory. Neither is valid under `oci_auth = "instance_principal"`. -- **`oci_auth` is `config_file` by default** and takes `instance_principal` for the keyless route. Name it explicitly rather than relying on a default you have forgotten: a typo is rejected at config load, which is better than a node quietly falling back to a key file nobody installed. -- `oci_public_ip_pool` takes a BYOIP pool OCID. Accepted today so BYOIP is a config change later rather than a code change; see [Limits](#limits-in-this-version). -- `range_start`, `range_end`, `gw_lrp_range_*`, `bind_bridge` and `dhcp_mac` are **rejected** on an OCI pool. OCI owns the addresses; a range you wrote would be fiction. -- Leave the `nat-transit` pool alone. In routed mode the per-VPC gateway router addresses come from the RFC 6598 transit range, not from OCI — **only public addresses handed to guests consume an OCI address.** Fifty VPCs and three Elastic IPs cost three reserved public IPs, not fifty-three. -- **The IMDS pair is required on OCI**, and it is set both keys or neither — a half-configured pair is ignored and the node behaves as if the remap were off. [The next section](#the-169254169254-collision) is why it exists; `169.254.42.254` / `169.254.42.253` is the tested choice. - -### Start and verify - -On **every** node: - -```bash -sudo systemctl start spinifex.target -``` - -Then, from any one of them: - -```bash -sudo spx get nodes -``` - -```text -NAME | STATUS | ROLES | IP | REGION | AZ | UPTIME | VMs | SERVICES -node1 | Ready | nats:follower | 10.200.1.249 | ap-southeast-2 | ap-southeast-2a | 0m | 0 | nats,predastore,viperblock,daemon,awsgw,vpcd,ui -node2 | Ready | nats:follower | 10.200.0.201 | ap-southeast-2 | ap-southeast-2a | 0m | 0 | nats,predastore,viperblock,daemon,awsgw,vpcd,ui -node3 | Ready | nats:leader | 10.200.1.79 | ap-southeast-2 | ap-southeast-2a | 0m | 0 | nats,predastore,viperblock,daemon,awsgw,vpcd,ui -``` - -Every node listed, all `Ready`, exactly one `nats:leader`, and the same `SERVICES` on each. `spx top nodes` shows pooled capacity and what the cluster can launch right now; if it looks like one server rather than three, the others never joined. - -**Confirm the OCI allocator came up**, which `spx get nodes` cannot tell you: +**Confirm the OCI allocator came up**, which `spx get nodes` cannot tell you. Run this on the node: ```bash sudo journalctl -u spinifex-vpcd --since -5m | grep -i ocinet @@ -803,22 +254,30 @@ sudo journalctl -u spinifex-vpcd --since -5m | grep -i ocinet > [!NOTE] > On a cold cluster start you may instead see `OCI allocator reconcile failed … nats: no responders available for request` on some nodes. That is vpcd racing JetStream's KV at boot; the startup reconcile is skipped and not retried. It is harmless on a new cluster where nothing has been allocated, and it is tracked — restart `spinifex-vpcd` on that node to run it. The `resolved the external VNIC` line above is still the one that decides whether allocation works. -## Step 6. Set up your cluster +And from inside the guest — note that IMDS is **IMDSv2, so it needs a token**; an untokened request correctly returns 401: + +```bash +TOK=$(curl -s -X PUT http://169.254.169.254/latest/api/token \ + -H 'X-aws-ec2-metadata-token-ttl-seconds: 120') +curl -s -H "X-aws-ec2-metadata-token: $TOK" \ + http://169.254.169.254/latest/meta-data/instance-id +traceroute -n 8.8.8.8 +``` + +--- + +## Next steps + +### Set up your cluster The cluster is running, but it holds nothing yet — no machine images, no networks, no instances. Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance. It ends by arming the [host firewall](/docs/host-firewall), which is also where you re-arm it if you turned it off to form the cluster. ---- - -## Step 7. Run your existing Terraform against the cluster +### Run your existing Terraform against the cluster This is the part worth pausing on. Everything above builds infrastructure **on** OCI using the OCI provider. Everything from here uses the **AWS** provider — unmodified, from the OpenTofu or Terraform registry — pointed at your own cluster. The same `aws_vpc`, `aws_instance`, `aws_db_instance`, `aws_ecs_service` and `aws_eks_cluster` resources a team already has in git apply against Spinifex on an OCI tenancy, with no rewriting and no OCI-specific module. -That is the claim, so it is measured rather than asserted. What follows is a run of the workbooks we ship against a three-node Spinifex cluster on OCI VMs. - -### Pointing the provider at your cluster - Three things differ from a provider block aimed at AWS, and only three: ```hcl @@ -842,48 +301,35 @@ provider "aws" { Credentials come from the `[spinifex]` profile that `spx admin init` writes into `~/.aws/credentials` on node 1. -> **Use the node's private address, not its OCI reserved public IP.** The node certificate carries no SAN for the public address, because that address is never on the wire — OCI NATs it to a private one. An `aws` or `terraform` call to `https://:9999` fails TLS verification with *hostname doesn't match*. Until that is fixed, run Terraform from a node, or from anything else inside the VCN. +> **Use the node's private address, not its OCI reserved public IP.** The node certificate carries no SAN for the public address, because that address is never on the wire — OCI NATs it to a private one. An `aws` or `terraform` call to `https://:9999` fails TLS verification with _hostname doesn't match_. Until that is fixed, run Terraform from a node, or from anything else inside the VCN. -### What we tested, and what happened +That portability is a claim, so it is measured rather than asserted. Each workbook below is a full `apply`, a functional assertion against the thing it built, and a `destroy`, measured on 2026-09-26 against a three-node cluster of `VM.Standard.E6.Flex` instances in `ap-sydney-1` running `spinifex v1.20.0-113`. -Measured on 2026-09-26 against a three-node cluster of `VM.Standard.E6.Flex` instances in `ap-sydney-1`, running `spinifex v1.20.0-113`. Each workbook is a full `apply`, a functional assertion against the thing it built, and a `destroy`. +**This is a snapshot, and the current answer is the nightly.** `e2e-cloudvendor-nightly` runs the same workbooks on a single VM, on three VMs and on bare metal, and publishes a table per topology on its own run page. Read that for the build you are installing rather than taking this table as current. -| Workbook | What it exercises | Result | -| --- | --- | --- | -| `nginx-webserver` | VPC, subnet, IGW, security group, key pair, EC2 instance, public IP; HTTP 200 from the internet | **Passed** (86s) | -| `bastion-private-subnet` | Public and private subnets, NAT egress, a bastion, SSH hop to a private host | **Passed** (117s) | -| `rds-quickstart` | DB subnet group, parameter group, `aws_db_instance` (PostgreSQL), client instance, connection | **Passed** (296s) | -| `ecs-quickstart` | ECS cluster, task definition, service, container instances, ALB with a healthy target | **Passed** (166s) | -| `eks-quickstart` | EKS control plane, managed node group, `kubectl` against the cluster, nodes `Ready` | **Passed** (259s) | -| `nginx-alb` | Application Load Balancer across two subnets, two backends, health checks | **Passed** (123s) | -| `s3-webapp` | S3 bucket, IAM role and instance profile, IMDS-fetched credentials, upload from the guest | **Failed** — see below | -| `demo-app` | Container image for the EKS workbooks, pushed to ECR | **Passed** | -| `eks-https-ingress` | Private-subnet workers behind a NAT gateway, LBC addon, ACM certificate, HTTPS Ingress | **Passed** (252s) | -| `eks-gitops-argocd` | Private-subnet workers, Argo CD addon, EBS-CSI PersistentVolume, GitOps sync | **Passed** (225s) | +| Workbook | What it exercises | Result | +| ------------------------ | ----------------------------------------------------------------------------------------------- | ----------------- | +| `nginx-webserver` | VPC, subnet, IGW, security group, key pair, EC2 instance, public IP; HTTP 200 from the internet | **Passed** (86s) | +| `bastion-private-subnet` | Public and private subnets, NAT egress, a bastion, SSH hop to a private host | **Passed** (117s) | +| `rds-quickstart` | DB subnet group, parameter group, `aws_db_instance` (PostgreSQL), client instance, connection | **Passed** (296s) | +| `ecs-quickstart` | ECS cluster, task definition, service, container instances, ALB with a healthy target | **Passed** (166s) | +| `eks-quickstart` | EKS control plane, managed node group, `kubectl` against the cluster, nodes `Ready` | **Passed** (259s) | +| `nginx-alb` | Application Load Balancer across two subnets, two backends, health checks | **Passed** (123s) | +| `s3-webapp` | S3 bucket, IAM role and instance profile, IMDS-fetched credentials, upload from the guest | **Passed** | +| `demo-app` | Container image for the EKS workbooks, pushed to ECR | **Passed** | +| `eks-https-ingress` | Private-subnet workers behind a NAT gateway, LBC addon, ACM certificate, HTTPS Ingress | **Passed** (252s) | +| `eks-gitops-argocd` | Private-subnet workers, Argo CD addon, EBS-CSI PersistentVolume, GitOps sync | **Passed** (225s) | -Nine of the ten pass. RDS, ECS and EKS are the answer to the question this section exists to ask: an EKS control plane and a managed node group come up, `kubectl get nodes` reports them `Ready`, a PostgreSQL instance accepts connections, and an ECS service runs behind a load balancer with a healthy target. None of it knows it is running on someone else's cloud. - -The last two are the harder network. Their workers sit in private subnets and reach the cluster endpoint and ECR through a NAT gateway, so a worker registering `Ready` is the proof that the whole NAT path works — which on OCI it did not until three defects in the gateway's address handling were fixed. If those two pass and `eks-quickstart` also passes, NAT egress is sound. - -`s3-webapp` fails on the read-back rather than the create: the AWS provider issues an **S3 Control** `ListTagsForResource` for the bucket, an endpoint nothing serves, and the SDK builds its hostname by prefixing the account ID onto the host — which cannot resolve against an IP. The bucket, the IAM role and the instance are all created correctly first. This is not OCI-specific — it fails the same way on bare metal — and it is a known defect we are fixing. Until then, either pin the AWS provider to 5.x or set `skip_requesting_account_id = true` in the provider block, and buckets apply cleanly. - -Two things bit during the run that are worth knowing before you hit them, neither of which is a reason the workbooks do not work: - -- A `spinifex-daemon` restart while a node group is creating is terminal for that node group — `NodeCreationFailure: create interrupted by daemon restart`, with no retry. -- A `terraform destroy` immediately followed by a `terraform apply` of the same EKS cluster name can fail with `ResourceInUseException` while `DescribeCluster` and `ListClusters` both report the cluster gone, so there is nothing to wait on. Leave a minute between the two. - ---- - -## Step 8. Oracle Linux as a guest image +### Oracle Linux as a guest image Spinifex ships four Oracle Linux entries in its image catalog, so a customer on OCI can run the same distro their Oracle support contract covers. They import exactly like any other AMI — nothing about them is OCI-specific, and they run equally well on bare metal: -| Catalog name | Release | Arch | Kernel | -| --- | --- | --- | --- | -| `oracle-10.1-x86_64` | Oracle Linux 10.1 | x86_64 | UEK 8 | -| `oracle-10.1-arm64` | Oracle Linux 10.1 | arm64 | UEK 8 | -| `oracle-9.8-x86_64` | Oracle Linux 9.8 | x86_64 | UEK 7 | -| `oracle-9.8-arm64` | Oracle Linux 9.8 | arm64 | UEK 7 | +| Catalog name | Release | Arch | Kernel | +| -------------------- | ----------------- | ------ | ------ | +| `oracle-10.1-x86_64` | Oracle Linux 10.1 | x86_64 | UEK 8 | +| `oracle-10.1-arm64` | Oracle Linux 10.1 | arm64 | UEK 8 | +| `oracle-9.8-x86_64` | Oracle Linux 9.8 | x86_64 | UEK 7 | +| `oracle-9.8-arm64` | Oracle Linux 9.8 | arm64 | UEK 7 | ```bash sudo spx admin images import --name oracle-10.1-x86_64 --config /etc/spinifex/spinifex.toml @@ -892,77 +338,227 @@ aws ec2 describe-images --query 'Images[].[Name,State,BootMode]' --output text All four boot **UEFI**, so launch them into a shape that boots UEFI — a BIOS-only instance type will not come up, and the failure looks like a hung boot rather than a rejected image. -### The login user is `cloud-user`, not `opc` +**The login user is `cloud-user`, not `opc`**, and this is the one that catches people: ```bash ssh -i path/to/key.pem cloud-user@ ``` -**This is the one that catches people.** `opc` is the default user on the images Oracle publishes *into OCI itself*; these are the generic **KVM cloud images** from `yum.oracle.com`, whose cloud-init `default_user` is `cloud-user`. Verified on both releases: `opc`, `oracle` and `ec2-user` are all refused. The Spinifex UI's instance detail page shows the right user per AMI, so read it there rather than guessing from the distro. - -### Why these four pin a checksum digest inline - -Every other catalog entry verifies against the publisher's sums file. Oracle ships none — the digests are published as HTML on `yum.oracle.com/oracle-linux-templates.html` — so these four carry `ChecksumDigest` in `spinifex/utils/images.go` instead. That is safe **only** because each URL names an immutable build (`b291`, `b293`, `b178`, `b182`) and Oracle never moves one. A new point release is a new URL and a new digest, never an edit to an existing entry. - -On arm64 Oracle publishes a `-kvm-cloud-` build alongside a plain `-kvm-` one. The catalog takes `-kvm-cloud-`: it is the one with cloud-init, and so the one that matches x86_64's `-kvm-`. The plain arm64 build imports fine and then has no datasource, which surfaces as an instance with no key and no metadata rather than as an import error. - -### SELinux is enforcing - -Both releases ship `SELinux: enforcing`, unlike the Debian and Ubuntu images. That is the upstream default and Spinifex does not change it. It does not affect networking, IMDS or cloud-init — all verified working — but a workload that has only ever run on Debian may meet it for the first time here. +`opc` is the default user on the images Oracle publishes _into OCI itself_; these are the generic **KVM cloud images** from `yum.oracle.com`, whose cloud-init `default_user` is `cloud-user`. Verified on both releases: `opc`, `oracle` and `ec2-user` are all refused. The Spinifex UI's instance detail page shows the right user per AMI, so read it there rather than guessing from the distro. --- ---- +## Reference -## Verify end to end +### Why run Spinifex on OCI + +**Lower cost for the same shape of workload.** OCI's compute and block storage are priced well below the large US clouds for equivalent OCPU and IOPS, and Spinifex turns one large instance into a whole EC2/EBS/S3 surface — so a tenant's twenty small instances are twenty QEMU guests on hardware you are billed for once, not twenty billable cloud instances. + +**Egress stops being the line item that decides the architecture.** Oracle publishes a large monthly outbound-transfer allowance per tenancy and a per-GB rate beyond it that is roughly an order of magnitude below the majors — check the current pricing page rather than a number in a document, but the gap is the reason this integration exists. Traffic _between_ guests never leaves the host or, on a cluster, never leaves the VCN: it is Geneve over the private plane, which is not billed as internet egress at all. + +**Portability — the workload stops being AWS-shaped and starts being yours.** The API your tenants code against is Spinifex's, so the same Terraform, the same SDK calls and the same AMIs run on OCI today, on bare metal in your own rack tomorrow, and at an edge site after that. Moving off AWS becomes a deployment decision rather than a rewrite, and moving _again_ costs nothing extra. + +**You keep the parts of OCI that are worth keeping.** Bare-metal shapes, fault domains, block volumes with a chosen performance tier, and real public addresses out of OCI's own pool. Spinifex does not hide the cloud underneath it — it registers each guest's address with OCI so that guest is genuinely reachable on the internet, not behind a shared NAT. + +### Sizing + +**Bare metal is the recommendation.** `BM.Standard.E2.64` or similar gives you the hardware directly — no nested virtualisation, full NIC performance, and no hypervisor between your guests and the wire. If bare metal is out of scope, a single `VM.Standard.E6.Flex` is the recommendation for testing and development, or three of them for production. + +| | Minimum | Recommended | +| --------------- | --------------------- | ----------------------------------------------------------------- | +| **Shape** | `VM.Standard.E6.Flex` | Bare metal, e.g. `BM.Standard.E2.64` | +| **OCPU** | 8 (16 vCPU) | 32+ | +| **RAM** | 32 GB | 128 GB+ | +| **Data volume** | 256 GiB block volume | 1 TB+ NVMe-backed | +| **VNICs** | 2 | 2 | +| **Nodes** | 1 | 3 — see [Cluster sizing](/docs/install-multi-node#cluster-sizing) | + +### How it fits together + +Everything in [Single-Node Install](/docs/install) and [Multi-Node Install](/docs/install-multi-node) still applies: formation, storage and OVN behave exactly as they do on hardware you own. What changes is the layer underneath — an OCI VCN is not an Ethernet segment, so the external datapath is wired differently, and public addresses come from OCI's API rather than from a range you write in a config file. + +**Single node — the whole AWS surface inside one OCI instance.** One instance runs the control plane and every tenant guest. Customers reach the AWS APIs on the node's own address; their instances reach the internet through addresses registered on the node's second VNIC. + +

+ Single node on OCI — the instance inside the VCN public subnet, both VNICs, br-wan, the routed-NAT transit veth, the control plane, tenant VPCs and the iSCSI data volume +

+ +Two things in that picture are the whole integration. The guest's private address (`10.0.1.4`) never leaves the host — OVN and the host route both hold it. The address OCI delivers to is a **secondary private IP registered on VNIC 1**, appearing on `br-wan`, and the customer's public address (`150.230.13.131`) is a reserved public IP that OCI 1:1-NATs onto it upstream. `describe-instances` reports the public one; every datapath object holds the private one, which is why the two never appear together on the host. + +The third is storage. Every guest disk, every S3 object and the JetStream state all live under `/var/lib/spinifex`, which is an **OCI block volume attached over iSCSI**, not the boot disk. Size it for the guests you intend to run; a boot volume that quietly ends up carrying the stack is the most common way an install on OCI fails later rather than immediately. + +**Three nodes — the same surface, distributed.** Each node registers the addresses for **its own** guests on **its own** second VNIC, and the allocator runs per node, so nothing has to know another node's OCIDs. When a guest moves — a stop/start that lands elsewhere, or a host failure — the node it lands on claims the address's private half from the old VNIC through a single OCI call. The OCID and the public half are untouched, so the customer's address never changes. + +

+ Three nodes on OCI — three instances inside the VCN public subnet, one per fault domain, each with its own VNICs, addresses and iSCSI volume, joined east-west by Geneve, NATS, predastore and the OVN raft +

+ +The overlay, the object shards and the OVN databases all cross **VNIC 0**, inside the VCN. Size that plane, not the external one: guest-to-guest traffic between nodes is Geneve over the VCN, and it is the link every distributed layer shares. + +> [!NOTE] +> **The Spinifex region is your own naming and has nothing to do with the OCI region.** `ap-southeast-2` in `spinifex.toml` beside `ap-sydney-1` in OCI is correct and expected. Do not read one as evidence about the other. + +### Credentials — an instance principal, or an API key + +The allocator needs OCI credentials at runtime, and there are two ways to give it them. + +**An instance principal is the better one, and it is a one-time setup step.** The instance certificate is served at `/opc/v2/identity/cert.pem` and the Go SDK authenticates as the instance with no key material on disk — but it must also be _authorised_, by a dynamic group and a policy. On the reference tenancy it authenticated without being authorised for anything, which is the silent failure above. Those two resources live at the tenancy root, so creating them needs a tenancy admin once; every deployment and rebuild afterwards needs only a compartment-scoped user: ```bash -export AWS_PROFILE= - -aws ec2 allocate-address -aws ec2 run-instances --image-id --instance-type t3.micro --key-name \ - --subnet-id --associate-public-ip-address -aws ec2 describe-instances --query \ - 'Reservations[].Instances[].[InstanceId,PrivateIpAddress,PublicIpAddress,State.Name]' \ - --output text +cd scripts/terraform/oci-spx +./setup-identity.sh --dry-run # always first +./setup-identity.sh # needs a tenancy-admin OCI profile ``` -Then, from the host: +Then set `instance_principal = "adopt"` in your tfvars and pass `--instance-principal` to the deploy. The dynamic group matches on `instance.compartment.id`, so it covers every node you ever build in that compartment and nothing needs updating when nodes are replaced. + +**An API key is the fallback**, and the default, because it needs nothing from a tenancy admin — which makes it the only option when the compartment belongs to someone else's tenancy. The cost is key material on disk, with a rotation obligation, on every node: ```bash -ping -c 3 -ssh -i ubuntu@ hostname +# On your workstation, not the node. +openssl genrsa -out ~/.oci/oci_api_key.pem 2048 +chmod 600 ~/.oci/oci_api_key.pem +openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem +openssl rsa -pubout -outform DER -in ~/.oci/oci_api_key.pem \ + | openssl md5 -c # this is the fingerprint ``` -And from inside the guest — note that IMDS is **IMDSv2, so it needs a token**; an untokened request correctly returns 401: +Upload the public key to the user under **Identity → Users → API Keys**, and record the user OCID, the fingerprint and the tenancy OCID. + +Each node then needs **two** files, and a node with only the first is the silent failure — it forms, looks healthy, and cannot allocate: + +| Path | Mode | Contents | +| ----------------------------------- | -------------------- | ------------------------------------------------------------------------ | +| `/etc/spinifex/oci/oci_api_key.pem` | `0640 root:spinifex` | The private key | +| `/etc/spinifex/oci/config` | `0640 root:spinifex` | An ordinary OCI SDK config, profile `[spinifex]`, naming the key by path | + +Nothing creates the second one for you, and its profile name must be `spinifex` to match `oci_config_profile` in the pool: + +```ini +[spinifex] +user=ocid1.user.oc1..aaaa... +fingerprint=12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef +tenancy=ocid1.tenancy.oc1..aaaa... +region=ap-sydney-1 +key_file=/etc/spinifex/oci/oci_api_key.pem +``` + +**Put that in a credential hook and the deploy installs it on every node.** The hook is an executable of yours, run as `hook ...` after formation and before the pool is configured, which is the only window where a node has `/etc/spinifex` but has not started the allocator. It is yours rather than ours deliberately: a credential belongs to whoever owns it, and one rendered into user-data or Terraform state is readable from instance metadata for the life of the instance — on a host that runs other people's guests. + +### The IAM policy + +Spinifex calls exactly nine operations. Grant no more than these: + +| Operation | Why | +| ------------------------------------------ | -------------------------------------------------------------------------------- | +| `CreatePrivateIp` | Register the secondary private IP the datapath carries | +| `DeletePrivateIp` | Release it | +| `GetPrivateIp` | Poll for `AVAILABLE` after an asynchronous assign | +| `ListPrivateIps` | The startup reconcile, and the affinity pass that moves an address between nodes | +| `UpdatePrivateIp` | Reassign a private IP to this node's VNIC when a guest moves here | +| `CreatePublicIp` | Create the RESERVED public IP | +| `DeletePublicIp` | Release it | +| `GetPublicIp` / `GetPublicIpByPrivateIpId` | Poll lifecycle state, and reconcile a private IP back to its public half | +| `UpdatePublicIp` | Attach and detach the public IP from a private IP | + +The matching policy, scoped to the one compartment Spinifex allocates in: + +```text +Allow group SpinifexOperators to manage private-ips in compartment +Allow group SpinifexOperators to manage public-ips in compartment +Allow group SpinifexOperators to read vnics in compartment +Allow group SpinifexOperators to read subnets in compartment +Allow group SpinifexOperators to read vcns in compartment +``` + +**Scope it to a compartment, not the tenancy.** `manage public-ips` at tenancy level lets a compromised node consume the whole regional reserved-public-IP quota. + +**Do not grant `manage instances` or `manage vnics`.** Spinifex never creates, attaches or deletes a VNIC — it only adds addresses to a VNIC that already exists. A grant that allows VNIC lifecycle is strictly more than the integration can use. + +### Quotas + +Two limits bound how many public addresses a deployment can hand out, and they do not scale the same way: + +| Quota | Default | Scope | Notes | +| ---------------------------------- | ------- | ------------------------- | ---------------------------------------------------------------------------------------------- | +| **Reserved public IPs** | 50 | Per region, whole tenancy | Shared by every node, so it does **not** grow with node count. This is the wall you hit first. | +| **Secondary private IPs per VNIC** | 64 | Per VNIC, so per node | Not raisable. Three nodes gives you 192 — this is the limit that does scale. | + +A raise on the first is a support request, not a setting: Console → Governance & Administration → Limits, Quotas and Usage → "Request a service limit increase". Cite the exact limit name from `oci limits definition list`, not a name from documentation. ```bash -TOK=$(curl -s -X PUT http://169.254.169.254/latest/api/token \ - -H 'X-aws-ec2-metadata-token-ttl-seconds: 120') -curl -s -H "X-aws-ec2-metadata-token: $TOK" \ - http://169.254.169.254/latest/meta-data/instance-id -traceroute -n 8.8.8.8 +oci limits definition list --service-name vcn --compartment-id +oci limits value list --service-name vcn --compartment-id +oci limits resource-availability get --service-name vcn \ + --limit-name --compartment-id ``` -A healthy topology is three hops before OCI's network: the OVN router port, the transit gateway, then upstream. +**These calls need a grant the node does not have, and must not be given.** Limits and usage are tenancy-level, which cannot be compartment-scoped the way the policy above insists the node's is. Create a **second, read-only principal** for an operator or a reporting job and keep it off every node: -On a cluster, prove an address survives a move. `scripts/terraform/oci-spx/eip-move-test.sh` does the whole sequence — it asserts that an auto-assigned address is returned to OCI on stop and a different one comes back, then associates an Elastic IP, forces the guest to start on another node, and checks OCI's own record of which VNIC carries the private half: +```text +Allow group SpinifexReporting to read limits in tenancy +Allow group SpinifexReporting to read usage-reports in tenancy +``` + +The second verb is what Cost Analysis and `oci usage-api usage-summary request-summarized-usages` read. Adding either to `SpinifexOperators` would hand every node tenancy-wide read, which is exactly what the node policy refuses. + +### The `oci` CLI + +**The `oci` CLI is not on the Ubuntu image**, and `apt` has no `python3-oci-cli` package. Nothing in [Deploy](#deploy) needs it; install it when you want to inspect allocations by hand: ```bash -INSTANCE=i-... ./scripts/terraform/oci-spx/eip-move-test.sh +sudo apt-get update && sudo apt-get install -y python3-venv +python3 -m venv ~/.oci-cli-venv +~/.oci-cli-venv/bin/pip install --upgrade pip oci-cli +sudo ln -sf ~/.oci-cli-venv/bin/oci /usr/local/bin/oci +oci --version ``` -### What a correct node looks like +**What the CLI is and is not for.** Spinifex's daemon uses the OCI **Go SDK** for every allocation — the CLI is not on that path. You need it for bootstrap and for operational inspection. Do not build automation that shells out to it during an allocation; allocations run inside a request budget that a forked Python process will not respect. -Worth knowing so the asymmetry does not read as a fault: +### What Terraform builds, per node -```bash -sudo ovn-nbctl lr-nat-list # dnat_and_snat holds the PRIVATE address -aws ec2 describe-instances ... # reports the PUBLIC address -ip route show | grep spx-nat-host # route to the PRIVATE address +| Resource | Detail | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `oci_core_instance` | One per `node_count`, round-robined across **fault domains** so three nodes land on three sets of racks rather than wherever placement puts them | +| Primary VNIC | Public IP, `hostname_label`, the host plane — SSH, the advertise address, the Geneve endpoint | +| `oci_core_vnic_attachment` | A **second VNIC** in the same subnet. Every Spinifex external address becomes a secondary private IP here at runtime | +| `oci_core_volume` | A block volume per node, `data_volume_size_in_gbs` at `data_volume_vpus_per_gb` | +| `oci_core_volume_attachment` | **`attachment_type = "iscsi"`**, with the Oracle Cloud Agent's Block Volume Management plugin enabled so the node logs the iSCSI session in itself | +| cloud-init | Partitions, formats, mounts and `fstab`s the volume; builds `br-wan` over the second VNIC; opens the service ports | + +The resource-by-resource architecture is in [`scripts/terraform/oci-spx`](../../scripts/terraform/oci-spx/README.md). + +**The data volume is iSCSI, and that is the detail that bites.** OCI presents the volume as an iSCSI target reachable at `169.254.2.2:3260` — attaching it in the API does not put a block device on the host. `is_agent_auto_iscsi_login_enabled` makes the Oracle Cloud Agent do the login, and because it does that _asynchronously_, the device is not there when cloud-init first runs. The mount script waits up to ten minutes for it, formats it **only if `blkid` reports no filesystem** (so a re-run cannot erase a populated volume), and writes an fstab entry by UUID: + +```text +UUID= /var/lib/spinifex ext4 defaults,_netdev,nofail 0 2 ``` -**These disagreeing is the design.** The public address is 1:1-NATed by OCI upstream and never touches the wire, so every datapath object holds the private half while every AWS-facing record holds the public half. If OVN showed the public address, the rule would be matching something that never arrives. +`_netdev` is what orders the mount after the network, and therefore after `iscsid` has a session. `nofail` is what keeps a missing volume from wedging the boot. Mounting `/var/lib/spinifex` _itself_ rather than somewhere else and symlinking is deliberate: viperblock, predastore and JetStream all land on the volume with no configuration pointing anywhere unusual. + +### Every Terraform variable + +All of these have defaults that build a working single node, except the first: + +| Variable | Default | What it decides | +| ----------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `compartment_ocid` | — | **Required.** An OCID, not a name: this deploys into a compartment that already exists, because creating one needs tenancy-root rights the API user must not have | +| `deployment_name` | `spinifex` | Prefixes every resource and becomes the VCN `dns_label`, so 1–15 lowercase alphanumerics starting with a letter | +| `node_count` | `1` | Nodes, 1–25. `1` for a single node, `3` for a cluster. Each gets its own volume, its own second VNIC and its own fault domain | +| `compute_shape` | `VM.Standard.E6.Flex` | **`BM.*` for bare metal, `VM.*.Flex` for a VM.** A fixed bare-metal shape carries its own size, so `shape_config` is emitted only when the name contains `Flex` — set a BM shape and the two sizing variables below are ignored rather than rejected | +| `compute_ocpus` | `8` | OCPUs on a flex shape. Validated at ≥ 8 (16 vCPU), the minimum recommended for a node | +| `compute_memory_in_gbs` | `32` | GiB on a flex shape. Validated at ≥ 32 | +| `data_volume_size_in_gbs` | `256` | Size of each node's data volume. This is where guest disks, S3 objects and JetStream live — size it for the guests, not for the control plane | +| `data_volume_vpus_per_gb` | `120` | Performance tier, 0–120 VPUs/GB. 120 is Ultra High Performance; 10 is Balanced. Higher is more IOPS and more cost | +| `data_mountpoint` | `/var/lib/spinifex` | Where cloud-init mounts it. Change only if you know why | +| `vcn_cidr` | `10.200.0.0/22` | VCN address space. A `/22` leaves room for the secondary private IPs Spinifex registers per node | +| `public_subnet_cidr` | `10.200.0.0/23` | The subnet every node sits in, **on both VNICs**. Nodes serve the UI, the S3 gate and the AWS gateway directly, so they cannot live in a private subnet | +| `node_client_cidr_allow_list` | `["0.0.0.0/0"]` | Who may reach SSH and the service ports from the internet. **Change this** | +| `node_service_ports` | `[3000, 8443, 9999]` | Console, predastore gate, AWS gateway. SSH is opened separately | +| `ubuntu_version` | `26.04` | Canonical platform image version. The newest image for the shape and region is resolved at plan time rather than pinned to a regional OCID | +| `wan_bridge_name` | `br-wan` | The Linux bridge built over the second VNIC | +| `wan_bridge_mtu` | `9000` | The OCI VCN carries 9000 | --- @@ -982,12 +578,12 @@ ip route show | grep spx-nat-host # route to the PRIVATE address **Instance launches but the public address is unreachable.** In order: -1. `sudo ovn-nbctl lr-nat-list ` — is there a `dnat_and_snat` row, and does it hold the *private* address? +1. `sudo ovn-nbctl lr-nat-list ` — is there a `dnat_and_snat` row, and does it hold the _private_ address? 2. `ip route get ` — does it route via the transit veth? 3. `sudo iptables -S FORWARD | grep spinifex-eip-ingress` — both directions present? 4. `oci network private-ip list --vnic-id ` — is the private address actually registered with OCI, **and on this node's VNIC**? If not, nothing else matters. -**Egress works from one address and not another on the same NIC.** That is [source-IP enforcement](#what-oci-changes), and it means the address is not a registered private-IP object on the VNIC. It is never a routing problem on the host. +**Egress works from one address and not another on the same NIC.** OCI enforces the source address per VNIC, so this means the address is not a registered private-IP object on it. It is never a routing problem on the host. **An address went dark after a guest moved node.** The affinity pass should have claimed it within 15 seconds. Check the new node's daemon journal for `ocinet affinity`, then confirm with step 4 above which VNIC OCI thinks holds the private half. @@ -1023,7 +619,7 @@ Three answers are worth knowing in advance, because each looks wrong and is not: **Guests boot but cloud-init hangs on metadata**, or the host stops answering its own. OCI uses `169.254.169.254` for its instance metadata, and so does every AWS-compatible guest — the host and the guests want the same address. -Spinifex resolves this by moving the *host* side of the per-guest metadata link to a different link-local pair, so guests keep asking `169.254.169.254` exactly as they do on AWS. [Step 5](#step-5-configure-spinifex-for-oci-start-and-verify) sets `imds_host_meta_ip` and `imds_host_dns_ip` for you. **Set both or neither** — a half-configured pair is ignored and the node behaves as if the remap were off, which is what this symptom looks like. +Spinifex resolves this by moving the _host_ side of the per-guest metadata link to a different link-local pair, so guests keep asking `169.254.169.254` exactly as they do on AWS. [The deploy](#4-run-the-deploy) sets `imds_host_meta_ip` and `imds_host_dns_ip` for you. **Set both or neither** — a half-configured pair is ignored and the node behaves as if the remap were off, which is what this symptom looks like. Check it after your first guest launches: @@ -1038,20 +634,18 @@ ip -br addr show | grep ime- # holds 169.254.42.x, not .169.x --- -## Limits in this version +## Deploying without Terraform -- **Operator-provisioned API key**, not instance principal. The key is on each node's disk and its rotation is your responsibility. -- **One VNIC per pool.** A pool cannot grow past the per-VNIC secondary private IP limit of 64. Three nodes give you three pools' worth; a single node is capped at 64 addresses. -- **IPv4 only.** IPv6 on OCI is materially simpler — no pools, prefixes assign directly to VCNs — but is not implemented. -- **No BYOIP.** The integration accepts a public IP pool OCID in config so BYOIP becomes a config change rather than a code change, but the RIR validation and Oracle's own validation window both precede any use of it. -- **OCI Block Volumes are not an EBS provider.** They back `/var/lib/spinifex` and therefore viperblock, which captures most of the performance benefit, but there is no native OCI block provider. -- **A recovered guest is briefly unreachable on its public address.** The address is never renumbered, but the node it is delivered to is corrected by a background pass every fifteen seconds rather than by the recovery itself. Private VPC addresses are unaffected. [Support status](#support-status) has the detail. -- **Terraform builds infrastructure, not Spinifex.** Formation is still the standard install path, by design: it is the same path prod and bare metal take, and diverging it for one cloud would mean two formation paths to keep correct. +**Terraform automates every step of a normal install, and that is the point of it.** The `oci-spx` configuration builds the VCN, the subnets, the gateways, the block volumes and both VNICs per node, then renders the cloud-init that installs Spinifex and sets up OVN — so the sequence below happens without anyone typing it, in the same order every time, on one node or three. -### What is no longer a limit +If you need to install by hand — an existing OCI tenancy you cannot run Terraform against, an unsupported shape, or a stage you are debugging — the install itself is not OCI-specific. Follow the standard guides: -Recorded because earlier versions of this guide said otherwise: +- [Installing Spinifex](../install/install/README.md) for a single node. +- [Multi-Node Installation](../install/install-multi-node/README.md) for a cluster, which covers `spx admin init` on the leader and `spx admin join` on the rest. -- **Multi-node works.** The `--nodes=1` cap on routed mode is gone, three-node formation and guest-to-guest traffic are proven on OCI, and the routed-NAT datapath is guarded nightly on three nodes — on our own hardware, not on OCI. See [Support status](#support-status). -- **Public addresses are not pinned to one node.** Each node allocates on its own VNIC, and an affinity pass reassigns an address's private half to whichever node the guest is on — one OCI call, keeping the OCID, so the customer's public address never changes. No node needs another node's VNIC OCID, and `oci_vnic_iface` is correct on every node. -- **Addresses are released when an instance stops.** An auto-assigned address goes back to the pool on stop and the reserved public IP object is deleted, so a stopped fleet stops costing you addresses. +Four things about OCI are not in those guides, and each has its own section above: + +- **The external pool** is `source = "oci"`, which needs the instance principal or API key from [Credentials](#credentials--an-instance-principal-or-an-api-key) and the [IAM policy](#the-iam-policy). Without it a node forms, passes every health check, and then refuses every launch that wants a public address. +- **`br-wan` stays a Linux bridge** owned by netplan, linked to OVS `br-ext` by a veth pair. Pass `setup-ovn.sh --wan-bridge=br-wan` and it detects this itself; the NIC never becomes an OVS port, so the VNIC keeps its MAC identity. See [How it fits together](#how-it-fits-together). +- **The block volume must be mounted with `_netdev`**, because it arrives over iSCSI and the mount is otherwise attempted before a session exists. +- **IMDS needs the 169.254 remap** described under [Guests cannot reach instance metadata](#guests-cannot-reach-instance-metadata), since the guest's metadata address collides with OCI's own. diff --git a/go.mod b/go.mod index 44b8fe953..f7440d1c9 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/charmbracelet/bubbles v1.0.0 github.com/charmbracelet/bubbletea v1.3.10 github.com/charmbracelet/lipgloss v1.1.0 - github.com/containerd/containerd/v2 v2.4.0 + github.com/containerd/containerd/v2 v2.4.1 github.com/distribution/reference v0.6.0 github.com/go-chi/chi/v5 v5.3.2 github.com/golang-jwt/jwt/v5 v5.3.1 diff --git a/go.sum b/go.sum index 772fa382f..8ca859ee1 100644 --- a/go.sum +++ b/go.sum @@ -111,8 +111,8 @@ github.com/containerd/console v1.0.5 h1:R0ymNeydRqH2DmakFNdmjR2k0t7UPuiOV/N/27/q github.com/containerd/console v1.0.5/go.mod h1:YynlIjWYF8myEu6sdkwKIvGQq+cOckRm6So2avqoYAk= github.com/containerd/containerd/api v1.12.0 h1:kuQm82SbDrCuO4n7hf2L8zsBtZLuympyq5X/VotfX2A= github.com/containerd/containerd/api v1.12.0/go.mod h1:EBcSzoi9Vl18cdODaXUCskf3D2NT8lsSXeZJnU5jIUc= -github.com/containerd/containerd/v2 v2.4.0 h1:mZLYWkPAgF4tfTjinClrRfvUxmxHxg1PQL7cjrrpTtk= -github.com/containerd/containerd/v2 v2.4.0/go.mod h1:gHZz+v5y8mGt9grF3ynuaa3r6bXLmghBZWTAftHAUtg= +github.com/containerd/containerd/v2 v2.4.1 h1:DUx/ZJN7cEu0WuzHClDB+68H/bqMEH5pWoEjf0ae4hc= +github.com/containerd/containerd/v2 v2.4.1/go.mod h1:vgLdtvl3prFk1d3ZVqQcsDD5Q9IKM+vAIdU54U85w+I= github.com/containerd/continuity v0.5.0 h1:7a85HZpCSs+1Zps0Ee3DPSuAWY+0SJM1JNM51nlEVDg= github.com/containerd/continuity v0.5.0/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= diff --git a/scripts/setup.sh b/scripts/setup.sh index 33a9a8d75..4cd059dcc 100755 --- a/scripts/setup.sh +++ b/scripts/setup.sh @@ -898,6 +898,17 @@ install_files() { info " /usr/local/share/spinifex/setup.sh" fi + # Formation. Shipping these means a multi-node cluster can be formed from a + # `curl | bash` install with no checkout, which is what a cloud operator has. + # smoke-test.sh comes too: install-node.sh scp's it from its own directory for + # the post-formation verification, so without it formation fails at the last step. + for formation in install-node.sh smoke-test.sh; do + if [ -f "$EXTRACT_DIR/$formation" ]; then + $SUDO install -m 0755 "$EXTRACT_DIR/$formation" "/usr/local/share/spinifex/$formation" + info " /usr/local/share/spinifex/$formation" + fi + done + # Teardown scripts. An install delivered by `curl | bash` with no way to # undo it is a gap in the product, not just in our tooling, and node-reset.sh # ships alongside because uninstall-spx.sh delegates its state teardown to it diff --git a/scripts/terraform/oci-spx/.e2e-oci-20261001T180429Z/verdict.txt b/scripts/terraform/oci-spx/.e2e-oci-20261001T180429Z/verdict.txt deleted file mode 100644 index e69de29bb..000000000 diff --git a/scripts/terraform/oci-spx/.gitignore b/scripts/terraform/oci-spx/.gitignore index 96ec21cb4..bb5276a90 100644 --- a/scripts/terraform/oci-spx/.gitignore +++ b/scripts/terraform/oci-spx/.gitignore @@ -10,3 +10,6 @@ terraform.tfvars *.pem *.log .validate-*/ +# Per-run artifacts and the identity state, which is not a topology's to destroy. +.e2e-oci-*/ +.identity/ diff --git a/scripts/terraform/oci-spx/README.md b/scripts/terraform/oci-spx/README.md index b434b6df0..1bd925c9e 100644 --- a/scripts/terraform/oci-spx/README.md +++ b/scripts/terraform/oci-spx/README.md @@ -209,14 +209,27 @@ node_client_cidr_allow_list = ["203.0.113.10/32"] The allocator needs OCI credentials at runtime, and there are two ways to give it them. A node with neither forms, passes every health check, and then refuses every launch that wants a public address with `InsufficientAddressCapacity` — the cause appears only in the node's journal, so this is worth getting right before first start. -**Instance principal is the better one, and it is what `enable_instance_principal = true` sets up.** Each node authenticates with the certificate its own metadata service serves, so no key material exists on any node, there is nothing to rotate, and nothing sensitive reaches Terraform state. The cost is a dynamic group and a policy, which are tenancy-root resources — so the apply needs a tenancy-admin principal, which is why it is off by default and why the rest of this configuration deliberately creates nothing at tenancy root. Create them once, by hand or with a privileged principal, and every later node and rebuild inherits them: +**Instance principal is the better one.** Each node authenticates with the certificate its own metadata service serves, so no key material exists on any node, there is nothing to rotate, and nothing sensitive reaches Terraform state. The cost is a dynamic group and a policy, which are tenancy-root resources — so creating them needs a tenancy-admin principal, which is why the rest of this configuration deliberately creates nothing at tenancy root. + +`instance_principal` has three values, because using an instance principal and being allowed to create one are different rights: + +| Value | Pool auth | Creates the dynamic group and policy | Credential needed | +| --- | --- | --- | --- | +| `off` (default) | API key file | No | Compartment-scoped | +| `adopt` | `instance_principal` | No — assumes they exist | Compartment-scoped | +| `create` | `instance_principal` | Yes | **Tenancy admin** | + +Create them once per tenancy, then every deployment and rebuild afterwards uses `adopt`: ```bash -python3 scripts/oci_env.py --ssh-public-key-path -- \ - terraform apply -var enable_instance_principal=true \ - -target oci_identity_dynamic_group.nodes -target oci_identity_policy.nodes +./setup-identity.sh --dry-run # always first +./setup-identity.sh ``` +`setup-identity.sh` targets only those two resources and keeps them in `.identity/terraform.tfstate`, separate from every topology's state. That separation is load-bearing: `validate-topology.sh` destroys its own state at the end of each run, so holding tenancy resources there would let a nightly teardown delete the tenancy's policy. + +**`adopt` references the dynamic group by nothing at all.** Its matching rule is `instance.compartment.id`, so it covers every instance in the compartment and names no OCID — which is why adopting needs no read on an identity resource and no tenancy rights. The cost is that a missing policy is invisible at apply time: the node forms, passes every health check, and then refuses every launch wanting a public address. The allocator gate in `validate-topology.sh` is what catches that, by requiring `resolved the external VNIC` in each node's `spinifex-vpcd` journal. + The policy grants three verbs in one compartment — `use vnics`, `manage private-ips`, `manage public-ips` — which is exactly what allocating an external address does and nothing more. **An API key is the fallback**, and the default because it needs nothing from a tenancy admin. Grant that user only the operations Spinifex performs on addresses and nothing else; it is not a tenancy admin, and broader rights widen the blast radius of a node compromise for no benefit. The key then has to be installed on **every** node, by hand, under `/etc/spinifex/oci/` — not a home directory, because the daemon's unit sets `ProtectHome=yes`: @@ -226,6 +239,22 @@ sudo install -d -o root -g spinifex -m 0750 /etc/spinifex/oci sudo install -o root -g spinifex -m 0640 ~/.oci/oci_api_key.pem /etc/spinifex/oci/oci_api_key.pem ``` +**The key is half of it.** `oci_config_file` names an ordinary OCI SDK config, and nothing creates that file, so a node with only the PEM forms and then fails every allocation. The profile must be `spinifex`, to match `oci_config_profile`, and the four identifiers are the ones already in the tfvars: + +```bash +sudo tee /etc/spinifex/oci/config >/dev/null <<'CONF' +[spinifex] +user=ocid1.user.oc1.. +fingerprint= +tenancy=ocid1.tenancy.oc1.. +region=ap-sydney-1 +key_file=/etc/spinifex/oci/oci_api_key.pem +CONF +sudo chown root:spinifex /etc/spinifex/oci/config && sudo chmod 0640 /etc/spinifex/oci/config +``` + +For repeated runs, `validate-topology.sh --credential-hook` does both steps on every node. The hook is an operator-owned executable rather than anything in this repository, because a credential rendered into user-data or Terraform state is readable from instance metadata for the life of the instance, and these hosts run other people's guests. Instance principal needs no hook at all, which is the reason to prefer it wherever a dynamic group can be created. + Either way, Terraform stages the matching pool block at `/etc/spinifex/oci/external-pool.toml` on each node, with `oci_auth` set to match. Append it to `/etc/spinifex/spinifex.toml` after `spx admin init` and restart `spinifex.target`. Under instance principal that file holds no secret at all, which is the point of it. ## Validating a topology end to end diff --git a/scripts/terraform/oci-spx/compute.tf b/scripts/terraform/oci-spx/compute.tf index e83cd6ae4..031fad7b9 100644 --- a/scripts/terraform/oci-spx/compute.tf +++ b/scripts/terraform/oci-spx/compute.tf @@ -45,6 +45,11 @@ locals { # after the boot disk (mulga-poc has oraclevda1 pointing at sdb1). data_device = "/dev/oracleoci/oraclevdb" + # adopt and create differ only in who creates the dynamic group and policy; a node + # authenticates identically either way, because the matching rule covers the whole + # compartment and so names no instance. + use_instance_principal = var.instance_principal != "off" + # The allocator's pool, staged on each node for spx admin init to pick up. Under # instance principal there is no oci_config_file key, because naming one beside a # certificate credential is a second source of truth that config load rejects. @@ -56,7 +61,7 @@ locals { "source = \"oci\"", "oci_compartment_id = \"${var.compartment_ocid}\"", "oci_vnic_iface = \"${var.wan_bridge_name}\"", - ], var.enable_instance_principal ? [ + ], local.use_instance_principal ? [ "oci_auth = \"instance_principal\"", ] : [ "oci_config_file = \"/etc/spinifex/oci/config\"", diff --git a/scripts/terraform/oci-spx/e2e-oci.sh b/scripts/terraform/oci-spx/e2e-oci.sh index 027a02819..df64da3ca 100755 --- a/scripts/terraform/oci-spx/e2e-oci.sh +++ b/scripts/terraform/oci-spx/e2e-oci.sh @@ -9,6 +9,10 @@ # SPX_REF Ref to prove. Default: the spinifex checkout's current branch. # OCI_TOPOLOGIES Space-separated. Default: "vm-single vm-multi bm". # OCI_SOURCE tree (build SPX_REF) or release (the published installer). +# OCI_VERSION With OCI_SOURCE=release, install this exact release tag and +# ignore OCI_CHANNEL. Preferred for anything repeatable: a tag +# resolves by redirect, while the dev channel resolves through a +# rate-limited GitHub API call that 404s when it trips. # OCI_CHANNEL With OCI_SOURCE=release, which published channel to install: # latest (default) or dev, the newest prerelease. dev is the # path our cloud contacts are given, so it is worth testing. @@ -24,6 +28,10 @@ # workbook are then recorded SKIPPED, never PASS, so a green # run with this set is not a claim about guest networking. # OCI_SSH_PUBLIC_KEY / OCI_SSH_PRIVATE_KEY Paths. Default ~/.ssh/oci-spx[.pub]. +# OCI_CREDENTIAL_HOOK Executable run on each topology after formation and +# before the pool, as "hook ...". An API-key +# deployment installs its credential here; instance principal +# needs none, so it is skipped unless the pool is key-based. # OCI_ARTIFACT_DIR Where logs and the verdict land. Default ./.e2e-oci-. # OCI_KEEP_ON_FAIL 1 to leave a failed topology up for inspection. Off by # default: an OCI bare-metal host left overnight is expensive. @@ -42,6 +50,7 @@ MULGA_ROOT="${MULGA_ROOT:-$SPINIFEX_ROOT/..}" TOPOLOGIES="${OCI_TOPOLOGIES:-vm-single vm-multi bm}" SOURCE="${OCI_SOURCE:-tree}" CHANNEL="${OCI_CHANNEL:-latest}" +INSTALL_VERSION="${OCI_VERSION:-}" REF="${SPX_REF:-$(git -C "$SPINIFEX_ROOT" rev-parse --abbrev-ref HEAD)}" SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}" SSH_PRIVATE_KEY="${OCI_SSH_PRIVATE_KEY:-$HOME/.ssh/oci-spx}" @@ -52,6 +61,7 @@ ARTIFACT_DIR="${OCI_ARTIFACT_DIR:-$HERE/.e2e-oci-$(date -u +%Y%m%dT%H%M%SZ)}" export OCI_STATE_ROOT="${OCI_STATE_ROOT:-$HERE}" STATE_ROOT="$OCI_STATE_ROOT" SWEEP_ONLY=0 +SUMMARY_ONLY=0 DRY_RUN=0 log() { printf '[e2e-oci] %s\n' "$*"; } @@ -62,11 +72,13 @@ die() { usage() { cat >&2 < "$VERDICT" +# The run page is the evidence, and its readers are not us: someone assessing +# whether Spinifex works on their cloud should get the answer without opening a +# log or knowing what this harness is. Markdown on stdout as well as to the step +# summary, so a workstation run shows the same thing CI publishes. +mark() { + case "$1" in + PASS) printf '✅ PASS' ;; + FAIL) printf '❌ FAIL' ;; + SKIPPED) printf '⏭️ SKIPPED' ;; + *) printf 'ℹ️ %s' "$1" ;; + esac +} + +summary() { + printf '## Spinifex on OCI — %s\n\n' "$REF" + # Said, not implied. A tree build and a published release answer different + # questions, and a table headed "published release" over a build of somebody's + # branch is the one error here that would mislead a reader who trusted it. + if [ "$SOURCE" = tree ]; then + printf 'Built from this ref, region `%s`.\n\n' "${OCI_REGION:-ap-sydney-1}" + else + printf 'Published release `%s`, region `%s`.\n\n' \ + "${INSTALL_VERSION:-$CHANNEL channel}" "${OCI_REGION:-ap-sydney-1}" + fi + + # In the header, not left to a SKIPPED row further down. A run with no allocator + # is not evidence about public addressing, and that is precisely the claim a + # reader of a green table would otherwise take from it. + if [ "${OCI_NO_EXTERNAL_POOL:-0}" = 1 ]; then + printf '> **No external address pool.** Public-address allocation and every workbook needing a public address were skipped, so nothing here speaks to guest ingress.\n\n' + elif [ "${OCI_INSTANCE_PRINCIPAL:-1}" = 1 ]; then + printf 'Allocator authenticated as the instance principal.\n\n' + else + printf 'Allocator authenticated with an API key.\n\n' + fi + + local topology results workbooks gate status detail name secs + for topology in $TOPOLOGIES; do + results="$STATE_ROOT/.validate-$topology/results.tsv" + workbooks="$STATE_ROOT/.validate-$topology/workbooks.tsv" + printf '### `%s`\n\n' "$topology" + # A topology the loop never reached is said so rather than omitted: a + # reader takes an absent row for a passing one. + if [ ! -s "$results" ]; then + printf 'Did not run.\n\n' + continue + fi + printf '| Gate | Result | Detail |\n| --- | --- | --- |\n' + while IFS=$'\t' read -r gate status detail; do + [ -n "$gate" ] || continue + printf '| %s | %s | %s |\n' "$gate" "$(mark "$status")" "$detail" + done < "$results" + printf '\n' + if [ ! -s "$workbooks" ]; then + printf 'No published workbooks ran.\n\n' + continue + fi + printf '| Terraform workbook | Result | Duration |\n| --- | --- | --- |\n' + while IFS=$'\t' read -r name status secs; do + [ -n "$name" ] || continue + printf '| `%s` | %s | %ss |\n' "$name" "$(mark "$status")" "$secs" + done < "$workbooks" + printf '\n' + done +} + +# GITHUB_STEP_SUMMARY is per step, so appending from both the run and the dedicated +# publish step would put two copies of the tables on the run page. One owner: the run +# writes the file and prints it, and only --summary-only publishes it. +emit_summary() { + summary | tee "$ARTIFACT_DIR/summary.md" +} + +publish_summary() { + emit_summary + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$ARTIFACT_DIR/summary.md" >> "$GITHUB_STEP_SUMMARY" + fi +} + # A topology destroys itself from its own EXIT trap, so this only catches a run # the trap never reached: a killed process, a runner reclaimed mid-job, a reboot. # It works from the state files rather than by name filter, so it can only ever @@ -127,6 +222,14 @@ sys.exit(0 if any(r["instances"] for r in state.get("resources", [])) else 1) [ "$found" = 0 ] && log "sweep: nothing left behind" } +# Before the sweep, because the sweep is what destroys the state the tables are +# read from. This is the path a cancelled job takes, so it has to work on whatever +# the run got through rather than assuming a complete set. +if [ "$SUMMARY_ONLY" = 1 ]; then + publish_summary + exit 0 +fi + # Run the sweep before the topologies as well as after: a leftover cluster from a # killed run is both a bill and a second set of resources with our names on it. log "artifacts in $ARTIFACT_DIR" @@ -178,7 +281,14 @@ for topology in $TOPOLOGIES; do args+=(--instance-principal) fi [ -n "$DISTRO_TARBALL" ] && args+=(--distro "$DISTRO_TARBALL" --setup-sh "$ARTIFACT_DIR/setup.sh") - [ "$SOURCE" = release ] && args+=(--channel "$CHANNEL") + if [ "$SOURCE" = release ]; then + if [ -n "$INSTALL_VERSION" ]; then + args+=(--version "$INSTALL_VERSION") + else + args+=(--channel "$CHANNEL") + fi + fi + [ -n "${OCI_CREDENTIAL_HOOK:-}" ] && args+=(--credential-hook "$OCI_CREDENTIAL_HOOK") [ "${OCI_KEEP_ON_FAIL:-0}" = 1 ] && args+=(--keep-on-fail) [ -n "${WORKBOOKS+x}" ] && args+=(--workbooks "$WORKBOOKS") @@ -201,8 +311,11 @@ done # case that costs money, and the verdict should say so rather than the log. sweep +emit_summary + + echo -log "=== $REF on OCI ($SOURCE build$([ "$SOURCE" = release ] && echo ", $CHANNEL channel")) ===" +log "=== $REF on OCI ($SOURCE build$([ "$SOURCE" = release ] && { [ -n "$INSTALL_VERSION" ] && echo ", $INSTALL_VERSION" || echo ", $CHANNEL channel"; })) ===" cat "$VERDICT" if [ "$RUN_RC" = 0 ] && ! grep -q FAIL "$VERDICT"; then log "PASS" diff --git a/scripts/terraform/oci-spx/instance-principal.tf b/scripts/terraform/oci-spx/instance-principal.tf index 47767ed03..fd9641941 100644 --- a/scripts/terraform/oci-spx/instance-principal.tf +++ b/scripts/terraform/oci-spx/instance-principal.tf @@ -3,15 +3,15 @@ # node forms and passes every health check, then refuses every launch that wants a # public address with InsufficientAddressCapacity and no stated cause. # -# Off by default, and deliberately: a dynamic group and a policy are tenancy-root -# resources, so this needs rights the compartment-scoped user the rest of this -# configuration runs as does not have and should not be given. Turn it on with a -# principal that holds them, or create both by hand once -- the OCIDs are stable -# and nothing here is per-node. +# Only instance_principal = "create" builds these, and deliberately: a dynamic +# group and a policy are tenancy-root resources, so creating them needs rights the +# compartment-scoped user the rest of this configuration runs as does not have and +# should not be given. Create them once with a tenancy admin, then every later +# deployment uses "adopt", which references nothing and so needs no such rights. resource "oci_identity_dynamic_group" "nodes" { provider = oci.home - count = var.enable_instance_principal ? 1 : 0 + count = var.instance_principal == "create" ? 1 : 0 compartment_id = var.tenancy_ocid name = "${var.deployment_name}-nodes" description = "Spinifex nodes in the ${var.deployment_name} deployment, authenticating as instance principals." @@ -26,7 +26,7 @@ resource "oci_identity_dynamic_group" "nodes" { # these three verbs cover; nothing here grants compute, storage or identity. resource "oci_identity_policy" "nodes" { provider = oci.home - count = var.enable_instance_principal ? 1 : 0 + count = var.instance_principal == "create" ? 1 : 0 compartment_id = var.compartment_ocid name = "${var.deployment_name}-nodes-network" description = "Allows Spinifex nodes to manage the private and public IPs backing external addresses." diff --git a/scripts/terraform/oci-spx/setup-identity.sh b/scripts/terraform/oci-spx/setup-identity.sh new file mode 100755 index 000000000..c41143e5a --- /dev/null +++ b/scripts/terraform/oci-spx/setup-identity.sh @@ -0,0 +1,86 @@ +#!/bin/bash +# setup-identity.sh -- create the dynamic group and policy that let Spinifex nodes +# authenticate to the OCI API as themselves, so no API key is installed on any node. +# +# Run once per tenancy, with a tenancy-admin principal. Every deployment afterwards +# passes instance_principal = "adopt", which references these by nothing at all: the +# matching rule covers the whole compartment, so it names no instance and needs no +# OCID from here. +# +# Its own state file, deliberately. validate-topology.sh destroys its state at the +# end of every run, and these two resources outlive every topology -- holding them +# in a per-topology state would let a nightly teardown delete the tenancy's policy. +set -uo pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +STATE_DIR="${OCI_STATE_ROOT:-$HERE}/.identity" +SSH_PUBLIC_KEY="${OCI_SSH_PUBLIC_KEY:-$HOME/.ssh/oci-spx.pub}" +DESTROY=0 +DRY_RUN=0 + +log() { printf '[setup-identity] %s\n' "$*"; } +die() { + printf '[setup-identity] ERROR: %s\n' "$*" >&2 + exit 1 +} + +usage() { + cat >&2 </dev/null || die "terraform init failed" + +if [ "$DESTROY" = 1 ]; then + log "destroying the dynamic group and policy" + tf destroy -auto-approve -var 'instance_principal=create' \ + -state "$STATE_DIR/terraform.tfstate" "${targets[@]}" \ + || die "destroy failed" + log "removed. Any node configured for instance principal can no longer allocate an address." + exit 0 +fi + +args=(-var 'instance_principal=create' -state "$STATE_DIR/terraform.tfstate" "${targets[@]}") + +if [ "$DRY_RUN" = 1 ]; then + log "dry run: planning the dynamic group and policy only" + tf plan "${args[@]}" || die "plan failed" + exit 0 +fi + +log "creating the dynamic group and policy (tenancy-root resources)" +tf apply -auto-approve "${args[@]}" || die "apply failed -- a tenancy-admin principal is required" + +log "done. State in $STATE_DIR/terraform.tfstate -- keep it, it is the only record." +log "Deployments can now use instance_principal = \"adopt\" with a compartment-scoped user." diff --git a/scripts/terraform/oci-spx/validate-topology.sh b/scripts/terraform/oci-spx/validate-topology.sh index e3b7ceda4..ba74a16bd 100755 --- a/scripts/terraform/oci-spx/validate-topology.sh +++ b/scripts/terraform/oci-spx/validate-topology.sh @@ -21,12 +21,14 @@ SKIP_POOL=0 DRY_RUN=0 DESTROY_ONLY=0 INSTANCE_PRINCIPAL=0 +CREDENTIAL_HOOK="${OCI_CREDENTIAL_HOOK:-}" DISTRO="" SETUP_SH="" # Empty means the driver's own default list. Unset is distinguishable from empty, # so --workbooks "" can deliberately mean "run none". WORKBOOKS_SET=0 CHANNEL=latest +INSTALL_VERSION="" WORKBOOKS="" # Shapes and counts per topology. Named here rather than passed in, because the @@ -62,9 +64,16 @@ workbook against it, then destroys everything. --instance-principal Configure the pool with oci_auth="instance_principal" instead of a key file. Needs the dynamic group and policy to exist already; see instance-principal.tf. + --credential-hook PATH Executable run after formation, before the pool, as + "hook ...". Where an API-key deployment + installs its credential. Default \$OCI_CREDENTIAL_HOOK. --workbooks LIST Space-separated workbooks for the published driver to run on the cluster. Empty means run none; omitted means the driver's own default list. + --version TAG Install this exact release tag. Preferred over + --channel dev for anything repeatable: a tag resolves + by redirect, while the dev channel resolves through a + rate-limited GitHub API call that 404s when it trips. --distro PATH Install this distro tarball instead of the published release, so what is proved is the ref it was built from. --setup-sh PATH setup.sh to pair with --distro. Both or neither. @@ -87,9 +96,11 @@ while [ $# -gt 0 ]; do --ssh-private-key) SSH_PRIVATE_KEY="${2:?}"; shift 2 ;; --workbooks) WORKBOOKS_SET=1; WORKBOOKS="${2-}"; shift 2 ;; --channel) CHANNEL="${2-}"; shift 2 ;; + --version) INSTALL_VERSION="${2:?}"; shift 2 ;; --distro) DISTRO="${2:?}"; shift 2 ;; --setup-sh) SETUP_SH="${2:?}"; shift 2 ;; --instance-principal) INSTANCE_PRINCIPAL=1; shift ;; + --credential-hook) CREDENTIAL_HOOK="${2:?}"; shift 2 ;; --no-external-pool) SKIP_POOL=1; SKIP_WORKLOAD=1; shift ;; --skip-workload) SKIP_WORKLOAD=1; shift ;; --keep) KEEP=1; shift ;; @@ -112,6 +123,10 @@ if [ -n "$DISTRO" ] || [ -n "$SETUP_SH" ]; then [ -r "$SETUP_SH" ] || die "--setup-sh is not readable: '$SETUP_SH'" fi +# adopt, never create: this script's state is destroyed at the end of every run, and +# the dynamic group and policy outlive every topology. setup-identity.sh owns them. +PRINCIPAL_MODE=$([ "$INSTANCE_PRINCIPAL" = 1 ] && echo adopt || echo off) + SHAPE="${TOPO_SHAPE[$TOPOLOGY]}" NODES="${TOPO_NODES[$TOPOLOGY]}" # Outside the checkout on a persistent runner, because actions/checkout runs @@ -122,6 +137,12 @@ mkdir -p "$STATE_ROOT" STATE_DIR="$STATE_ROOT/.validate-$TOPOLOGY" RESULTS="$STATE_DIR/results.txt" +# Tab-separated siblings of the human block, for the run-page tables. Written here +# rather than parsed out of the log later: the status of a gate is then a field +# this script sets, not a regex over prose that reads fine and matches wrong. +RESULTS_TSV="$STATE_DIR/results.tsv" +WORKBOOKS_TSV="$STATE_DIR/workbooks.tsv" + # A separate state directory per topology, so two topologies can be built from one # checkout without one destroying the other's instances. tf() { @@ -135,7 +156,7 @@ tf_state=(-state "$STATE_DIR/terraform.tfstate") tf_vars=( -var "compute_shape=$SHAPE" -var "node_count=$NODES" - -var "enable_instance_principal=$([ "$INSTANCE_PRINCIPAL" = 1 ] && echo true || echo false)" + -var "instance_principal=$PRINCIPAL_MODE" "${tf_state[@]}" ) @@ -148,7 +169,23 @@ ssh_node() { -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15 "ubuntu@$host" "$@" } -record() { printf '%s\n' "$*" >> "$RESULTS"; } +# record [detail] +record() { + local gate="$1" status="$2" detail="${3-}" + printf '%s: %s%s\n' "$gate" "$status" "${detail:+ $detail}" >> "$RESULTS" + printf '%s\t%s\t%s\n' "$gate" "$status" "$detail" >> "$RESULTS_TSV" +} + +# The driver's per-workbook lines, which are go test's own shape because +# go-junit-report consumes them downstream. A workbook the driver never reached +# is absent from its log and so absent here, which the summary renders as a row +# rather than dropping — a missing row reads as a pass. +record_workbooks() { + local log="$1" + [ -r "$log" ] || return 0 + sed -n 's/^--- \(PASS\|FAIL\): TestTofuWorkbook_\([A-Za-z0-9_]*\) (\([0-9]*\)\.[0-9]*s)$/\2\t\1\t\3/p' \ + "$log" | tr '_' '-' > "$WORKBOOKS_TSV" +} # Teardown is the last assertion, not cleanup: a workbook or a topology that # cannot be destroyed is a defect, and it has been one before. @@ -157,10 +194,10 @@ destroy_topology() { log "destroying" if tf destroy -auto-approve -no-color "${tf_vars[@]}" > "$STATE_DIR/destroy.log" 2>&1; then DESTROY_RC=0 - record "destroy: PASS" + record destroy PASS else DESTROY_RC=1 - record "destroy: FAIL (see $STATE_DIR/destroy.log)" + record destroy FAIL "see $STATE_DIR/destroy.log" log "TEARDOWN FAILED — resources may still be billing. $STATE_DIR/destroy.log" fi } @@ -206,9 +243,20 @@ if [ "$DESTROY_ONLY" = 1 ]; then fi : > "$RESULTS" +: > "$RESULTS_TSV" +: > "$WORKBOOKS_TSV" log "$SHAPE, $NODES node(s), state in $STATE_DIR" +# Checked here, not where the hook is run: nothing about it depends on the apply, and +# the run is otherwise forty minutes and a bare-metal bill from discovering that the +# path in a CI variable does not exist on this runner. +if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then + [ -n "$CREDENTIAL_HOOK" ] \ + || die "this deployment authenticates with an API key and no credential hook is set, so no node could reach the OCI API; set --credential-hook or OCI_CREDENTIAL_HOOK, or use an instance principal" + [ -x "$CREDENTIAL_HOOK" ] || die "credential hook is not executable: $CREDENTIAL_HOOK" +fi + # The directory's own terraform.tfstate belongs to hand-driven runs, not to a # topology, and an instance left in it is both a bill and a name that collides # with ours in the console. Say so rather than letting it be a surprise. @@ -249,7 +297,7 @@ tf init -input=false > "$STATE_DIR/init.log" 2>&1 || die "terraform init failed; log "building" tf apply -auto-approve -no-color "${tf_vars[@]}" > "$STATE_DIR/apply.log" 2>&1 \ || die "terraform apply failed; see $STATE_DIR/apply.log" -record "build: PASS" +record build PASS mapfile -t HOSTS < <(tf output -raw "${tf_state[@]}" hosts_file) \ || die "could not read the hosts_file output from $STATE_DIR/terraform.tfstate" @@ -283,7 +331,7 @@ for host in "${HOSTS[@]}"; do || die "cloud-init did not converge on $host; see $STATE_DIR/cloudinit-$host.txt" log "$host cloud-init: $(head -1 "$STATE_DIR/cloudinit-$host.txt")" done -record "cloud-init units: PASS" +record "cloud-init units" PASS # With --distro the bytes are the ref under test; without it they are the published # release, which answers a different question -- what a customer following the @@ -298,7 +346,7 @@ for host in "${HOSTS[@]}"; do # cloud-init being finished does not mean apt is: the apt-daily timers and # unattended-upgrades run on their own schedule and hold the dpkg lock, which # the installer then fails on. Wait for the lock rather than fight it. - ssh_node "$host" "DISTRO_NAME='$(basename "${DISTRO:-}")' SETUP_NAME='$(basename "${SETUP_SH:-}")' CHANNEL='$CHANNEL' bash -s" \ + ssh_node "$host" "DISTRO_NAME='$(basename "${DISTRO:-}")' SETUP_NAME='$(basename "${SETUP_SH:-}")' CHANNEL='$CHANNEL' INSTALL_VERSION='$INSTALL_VERSION' bash -s" \ > "$STATE_DIR/install-$host.log" 2>&1 <<'REMOTE' \ || die "install failed on $host; see $STATE_DIR/install-$host.log" set -e @@ -318,13 +366,23 @@ else # The real customer path, including the checksum step a local tarball skips. # The environment variable rather than --channel: the installer served by # install.mulgadc.com is the latest release's, so it predates the flag. - curl -sfL https://install.mulgadc.com | sudo env INSTALL_SPINIFEX_CHANNEL="$CHANNEL" bash + # + # A tag goes in as VERSION, not CHANNEL. Only the dev channel resolves through + # an unauthenticated GitHub API call, which is rate limited per source address + # and returns 404 once it trips -- indistinguishable from a missing asset. A + # tagged path is a plain redirect, so it does not have that failure mode. + if [ -n "$INSTALL_VERSION" ]; then + curl -sfL https://install.mulgadc.com | sudo env INSTALL_SPINIFEX_VERSION="$INSTALL_VERSION" bash + else + curl -sfL https://install.mulgadc.com | sudo env INSTALL_SPINIFEX_CHANNEL="$CHANNEL" bash + fi fi sudo /usr/local/share/spinifex/setup-ovn.sh --management --nat-uplink REMOTE - log "$host installed: $(ssh_node "$host" 'spx version' 2>&1 | head -1)" + INSTALLED_VERSION="$(ssh_node "$host" 'spx version' 2>&1 | head -1)" + log "$host installed: $INSTALLED_VERSION" done -record "install: PASS" +record install PASS "$INSTALLED_VERSION" # A single node is its own documented path: install-node.sh refuses fewer than two # hosts, because there is nothing to join. Both branches use the flags the guide @@ -351,7 +409,21 @@ else --yes > "$STATE_DIR/form.log" 2>&1 \ || die "formation failed; see $STATE_DIR/form.log" fi -record "formation: PASS" +record formation PASS + +# Runs after formation and before the pool is configured, which is the only window +# where a node has /etc/spinifex but has not yet started the allocator. Nothing in +# this repository knows what it does: an API-key deployment needs a credential on +# each node, and a credential belongs to the operator, not to a checked-in script. +# Skipped under instance principal, which needs no handoff at all. +if [ "$SKIP_POOL" != 1 ] && [ "$PRINCIPAL_MODE" = off ]; then + log "running the credential hook" + # Arguments, not a file: the hook is told where the nodes are and how to reach + # them, and decides for itself what to put there. + "$CREDENTIAL_HOOK" "$SSH_PRIVATE_KEY" "${HOSTS[@]}" > "$STATE_DIR/credential-hook.log" 2>&1 \ + || die "the credential hook failed; see $STATE_DIR/credential-hook.log" + log "credential hook ok" +fi # The IMDS remap and the pool are both set-before-first-start, and the remap is the # half that is invisible when missing: without it the cloud's metadata service is @@ -362,45 +434,90 @@ for host in "${HOSTS[@]}"; do || die "pool configuration failed on $host; see $STATE_DIR/pool-$host.log" set -e sudo cp /etc/spinifex/spinifex.toml /etc/spinifex/spinifex.toml.bak-prepool +# subn, not sub: with no [network] section to match, sub rewrites the file unchanged +# and exits 0, leaving the remap silently unapplied. That is the half of this that is +# invisible when missing, so the substitution count is checked rather than assumed. grep -q imds_host_meta_ip /etc/spinifex/spinifex.toml || sudo python3 - <<"PY" -import re +import re, sys path = "/etc/spinifex/spinifex.toml" -text = open(path).read() -text = re.sub(r"(?m)^\[network\]$", - "[network]\nimds_host_meta_ip = \"169.254.42.254\"\nimds_host_dns_ip = \"169.254.42.253\"", - text, count=1) +text, n = re.subn(r"(?m)^\[network\]$", + "[network]\nimds_host_meta_ip = \"169.254.42.254\"\nimds_host_dns_ip = \"169.254.42.253\"", + open(path).read(), count=1) +if n != 1: + sys.exit("no [network] section in %s, so the IMDS remap was not applied" % path) open(path, "w").write(text) PY if [ "${SKIP_POOL:-0}" != 1 ]; then grep -q 'name = "oci-public"' /etc/spinifex/spinifex.toml \ || sudo tee -a /etc/spinifex/spinifex.toml < /etc/spinifex/oci/external-pool.toml >/dev/null fi -sudo spx config validate --config /etc/spinifex/spinifex.toml 2>/dev/null || true +# spx has no config subcommand, so the `spx config validate` that used to be here +# could only ever fail, and was swallowed. Parsing the file is the check that was +# wanted: both edits above are textual, and a broken result would otherwise surface +# as every service failing to start with nothing saying why. +sudo python3 -c 'import sys, tomllib; tomllib.load(open(sys.argv[1], "rb"))' \ + /etc/spinifex/spinifex.toml \ + || { echo "spinifex.toml is not valid TOML after the IMDS remap and the pool append"; exit 1; } sudo systemctl restart spinifex.target REMOTE done +# Everything needed to tell the three allocator faults apart, and deliberately no +# credential: the config file and the PEM are reported as mode and size only, which +# distinguishes absent from unreadable without copying either into a log that is +# attached to a CI run. +allocator_diagnostics() { + local host="$1" + ssh_node "$host" ' + echo "=== spinifex-daemon: the allocator (ocinet) lives here ===" + sudo journalctl -u spinifex-daemon --since -20min --no-pager | grep -i "ocinet\|external VNIC\|allocator\|oci" | tail -40 + echo "=== spinifex-vpcd: consumes the addresses, does not allocate them ===" + sudo journalctl -u spinifex-vpcd --since -20min --no-pager | tail -40 + echo "=== external_pools as configured ===" + sudo sed -n "/\[\[network.external_pools\]\]/,\$p" /etc/spinifex/spinifex.toml + echo "=== credential files (mode and size, never content) ===" + sudo stat -c "%n %A %s bytes owner=%U:%G" \ + /etc/spinifex/oci/config /etc/spinifex/oci/oci_api_key.pem 2>&1 + echo "=== the bridge MAC the allocator matches on ===" + ip -br link show + echo "=== MACs IMDS reports for this instance VNICs ===" + curl -sf -H "Authorization: Bearer Oracle" \ + http://169.254.169.254/opc/v2/vnics/ | tr "," "\n" | grep -i "macAddr\|privateIp" || \ + echo "IMDS unreachable -- check the remap, Spinifex claims 169.254.169.254 itself" + ' 2>&1 +} + # resolved the external VNIC is the line that proves the credential works, the # compartment is right and br-wan's MAC matched a real VNIC. A node missing it # accepts allocate-address and then fails it. if [ "$SKIP_POOL" = 1 ]; then log "--no-external-pool: no allocator to check" - record "oci allocator: SKIPPED" + record "oci allocator" SKIPPED "no external pool configured" else log "checking the allocator came up" for host in "${HOSTS[@]}"; do found=0 for _ in $(seq 1 30); do - if ssh_node "$host" "sudo journalctl -u spinifex-vpcd --since -10min --no-pager | grep -q 'resolved the external VNIC'" 2>/dev/null; then + # spinifex-daemon, not spinifex-vpcd: the allocator is built in the + # daemon, and vpcd only consumes the addresses it hands out. Gating on + # vpcd's journal failed a node whose allocator was working. + if ssh_node "$host" "sudo journalctl -u spinifex-daemon --since -10min --no-pager | grep -q 'resolved the external VNIC'" 2>/dev/null; then found=1 break fi sleep 10 done - [ "$found" = 1 ] || die "$host never logged 'resolved the external VNIC'; the OCI allocator is not up, so no guest can get a public address" + if [ "$found" != 1 ]; then + # Before the teardown, because the teardown is what destroys the only + # copy. Three unrelated faults land here -- a credential the SDK would + # not load, a bridge MAC matching no VNIC, and a pool vpcd never read -- + # and they are indistinguishable from the missing log line alone. + allocator_diagnostics "$host" > "$STATE_DIR/allocator-$host.log" 2>&1 || true + die "$host never logged 'resolved the external VNIC'; the OCI allocator is not up, so no guest can get a public address. Diagnostics: $STATE_DIR/allocator-$host.log" + fi log "$host allocator ready" done - record "oci allocator: PASS" + record "oci allocator" PASS fi # Parse the STATUS column, never grep for Ready: NotReady contains it, and that @@ -421,11 +538,11 @@ for _ in $(seq 1 30); do done cat "$STATE_DIR/nodes.txt" [ "$ready" = "$NODES" ] || die "expected $NODES Ready node(s), got $ready; see $STATE_DIR/nodes.txt" -record "membership: PASS ($ready Ready)" +record membership PASS "$ready Ready" if [ "$SKIP_WORKLOAD" = 1 ]; then log "--skip-workload: stopping before the workbook" - record "workbook: SKIPPED" + record workbooks SKIPPED "--skip-workload" exit 0 fi @@ -458,14 +575,58 @@ ssh_node "${HOSTS[0]}" ' workbook_env="WORKBOOK_DIR=\$HOME/workbooks" [ "$WORKBOOKS_SET" = 1 ] && workbook_env="$workbook_env $(printf 'WORKBOOKS=%q' "$WORKBOOKS")" +# Whether a node can reach a public address inside its own VCN. Informational: the +# remedy below is right either way, because a customer reaches a guest from outside. +# +# Needs two hosts. With one, HOSTS[0] and HOSTS[-1] are the same address and the node +# connects to itself, which always succeeds and measures nothing — it reported +# "present" on vm-single and bm for exactly that reason before this guard. +# +# A negative result is only interpretable when the security list admits the source, +# so the record says which it was: narrowing node_client_cidr_allow_list makes a +# node's own public address a non-permitted source, and the refusal that follows +# looks identical to a VCN that does not hairpin. +if [ "${#HOSTS[@]}" -lt 2 ]; then + record "vcn hairpin" SKIPPED "needs two nodes; one node can only probe itself" +elif ssh_node "${HOSTS[0]}" \ + "bash -c 'exec 3<>/dev/tcp/${HOSTS[${#HOSTS[@]}-1]}/22' 2>/dev/null"; then + record "vcn hairpin" INFO "present: a node reached another node's public address" +else + record "vcn hairpin" INFO "not reached, which is a VCN without hairpin or a security list that excludes the source" +fi + +# Remote dynamic forward: the node gets a SOCKS5 proxy on this port whose egress +# is this host, outside the VCN. It gives the driver the customer's vantage point +# for public addresses without moving the driver off the node, where its ovn-nbctl, +# journalctl and spx diagnostics have to run. The subnet's own allow list does not +# need a new entry: ssh from here already works, and its rules are per-CIDR for +# every protocol. +PUBLIC_PROXY_PORT="${PUBLIC_PROXY_PORT:-1080}" +workbook_env="$workbook_env E2E_PUBLIC_PROXY=127.0.0.1:$PUBLIC_PROXY_PORT" + +# ExitOnForwardFailure, so a port already in use stops the run here instead of +# handing the driver a proxy that is not there. Keepalives because this one +# connection carries the proxy for the whole suite, and a dropped forward would +# read as every remaining public address going dark at once. +# # if !, not a $? read after the fact: under set -e a failing ssh never reaches the # next line, which is the one that prints the driver's own diagnostics. -if ssh_node "${HOSTS[0]}" "chmod +x ~/run-tofu-examples-e2e.sh; $workbook_env ~/run-tofu-examples-e2e.sh" \ +if ssh -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + -o LogLevel=ERROR -o BatchMode=yes -o ConnectTimeout=15 \ + -o ServerAliveInterval=30 -o ServerAliveCountMax=6 \ + -o ExitOnForwardFailure=yes -R "$PUBLIC_PROXY_PORT" "ubuntu@${HOSTS[0]}" \ + "chmod +x ~/run-tofu-examples-e2e.sh; $workbook_env ~/run-tofu-examples-e2e.sh" \ > "$STATE_DIR/workbooks.log" 2>&1; then + record_workbooks "$STATE_DIR/workbooks.log" passed="$(grep -c '^--- PASS' "$STATE_DIR/workbooks.log" || true)" ran="$(grep -c '^=== RUN' "$STATE_DIR/workbooks.log" || true)" - record "workbooks: PASS ($passed of $ran)" + record workbooks PASS "$passed of $ran" else + # Before die, so a failed suite still gets its per-workbook table: which four + # passed is most of what the fifth failing means. + record_workbooks "$STATE_DIR/workbooks.log" + failed="$(grep -c '^--- FAIL' "$STATE_DIR/workbooks.log" || true)" + record workbooks FAIL "$failed failed, see $STATE_DIR/workbooks.log" tail -60 "$STATE_DIR/workbooks.log" grep -E '^--- (PASS|FAIL)' "$STATE_DIR/workbooks.log" | sed 's/^/ /' || true die "the published workbooks failed on $TOPOLOGY; see $STATE_DIR/workbooks.log" diff --git a/scripts/terraform/oci-spx/variables.tf b/scripts/terraform/oci-spx/variables.tf index 6302e4a20..3650bfe6f 100644 --- a/scripts/terraform/oci-spx/variables.tf +++ b/scripts/terraform/oci-spx/variables.tf @@ -54,12 +54,18 @@ variable "deployment_name" { error_message = "deployment_name must be 1-15 lowercase alphanumeric characters starting with a letter, because it is used as the VCN dns_label." } } -# Needs a tenancy-admin principal, which is why it is opt-in. See -# instance-principal.tf for what the policy grants and why it is narrow. -variable "enable_instance_principal" { - description = "Create the dynamic group and policy that let nodes authenticate as themselves, so no OCI API key is installed on any node." - type = bool - default = false +# Three values rather than a bool, because using an instance principal and being +# allowed to create one are different rights. See instance-principal.tf for what +# the policy grants and why it is narrow. +variable "instance_principal" { + description = "off: nodes use an API key. adopt: nodes authenticate as themselves against a dynamic group and policy that already exist. create: also create those two, which needs a tenancy-admin principal." + type = string + default = "off" + + validation { + condition = contains(["off", "adopt", "create"], var.instance_principal) + error_message = "instance_principal must be off, adopt or create." + } } variable "vcn_cidr" { diff --git a/tests/e2e/run-tofu-examples-e2e.sh b/tests/e2e/run-tofu-examples-e2e.sh index 8e9f76979..2bc5a7462 100755 --- a/tests/e2e/run-tofu-examples-e2e.sh +++ b/tests/e2e/run-tofu-examples-e2e.sh @@ -25,10 +25,60 @@ OPENTOFU_VERSION="${OPENTOFU_VERSION:-1.11.5}" WAN_IP=$(ip -4 route get 1.1.1.1 2>/dev/null | awk '{print $7; exit}') SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 -o LogLevel=ERROR) +# A guest's public address, reached from where a customer would reach it. Empty +# means this host is already outside the cluster's own network, which is true of +# every hypervisor runner. Set it to host:port for a SOCKS5 proxy whose egress is +# off-network when the host is not: a cloud may refuse a public address from inside +# its own network, or admit only listed sources, and either fails a healthy cluster +# from here however well the datapath works. +PUBLIC_PROXY="${E2E_PUBLIC_PROXY:-}" + CURRENT_WORKBOOK="" log() { echo "[$(date +%H:%M:%S)] $*"; } +# curl, from outside the cluster's network when PUBLIC_PROXY says it has to be. +# socks5h, not socks5: the proxy resolves, so a name only the far side knows +# still works. +curl_public() { + if [ -n "$PUBLIC_PROXY" ]; then + curl --proxy "socks5h://${PUBLIC_PROXY}" "$@" + return + fi + curl "$@" +} + +# PUBLIC_PROXY is for public addresses only, so this is deliberately not in +# SSH_OPTS itself: a hop added to every ssh would also divert the bastion's +# inner ssh to a private address, which no proxy can reach. +ssh_public() { + if [ -n "$PUBLIC_PROXY" ]; then + ssh -o "ProxyCommand=nc -X 5 -x ${PUBLIC_PROXY} %h %p" "$@" + return + fi + ssh "$@" +} + +# Fail here rather than letting every assertion time out in turn: a proxy that +# is named and does not work is indistinguishable, from the far end, from a +# cluster that is not serving. +check_public_proxy() { + [ -n "$PUBLIC_PROXY" ] || return 0 + command -v nc >/dev/null 2>&1 || { + sudo apt-get install -y -qq netcat-openbsd >/dev/null 2>&1 || { + log "public-address probes need nc (netcat-openbsd) for ssh over SOCKS" + return 1 + } + } + # Something off-network that is not ours, so the answer is about the proxy + # and not about the cluster. + if ! curl_public -sf -o /dev/null --max-time 20 https://api.github.com/; then + log "public-address probes go through SOCKS ${PUBLIC_PROXY}, which cannot reach the internet" + return 1 + fi + log "public-address probes go through SOCKS ${PUBLIC_PROXY} (outside this host's network)" +} + install_tofu() { command -v tofu >/dev/null 2>&1 && return 0 local arch @@ -110,7 +160,7 @@ trap cleanup EXIT wait_for_ssh() { local key="$1" host="$2" for _ in $(seq 1 30); do - if ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" true 2>/dev/null; then + if ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" true 2>/dev/null; then return 0 fi sleep 5 @@ -124,7 +174,7 @@ wait_for_http_200() { local attempts=$((budget / 5)) for _ in $(seq 1 "$attempts"); do local status - status=$(curl -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null || echo "000") + status=$(curl_public -sk -o /dev/null -w '%{http_code}' --max-time 5 "$url" 2>/dev/null || echo "000") if [ "$status" = "200" ]; then return 0 fi @@ -163,7 +213,7 @@ assert_bastion_private_subnet() { } # sshd accepts before cloud-init finishes writing ~/.ssh/bastion-demo.pem. # Wait up to 180s for user_data to drop the key before hopping. - if ! ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" \ + if ! ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" \ 'for _ in $(seq 1 36); do [ -s ~/.ssh/bastion-demo.pem ] && exit 0; sleep 5; done; exit 1'; then log " bastion: ~/.ssh/bastion-demo.pem never appeared (cloud-init stalled?)" return 1 @@ -172,8 +222,8 @@ assert_bastion_private_subnet() { local attempt for attempt in $(seq 1 30); do - if ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} true" 2>/dev/null; then - ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} id" | grep -q '^uid=' + if ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} true" 2>/dev/null; then + ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${bastion}" "${inner_ssh} id" | grep -q '^uid=' return $? fi sleep 5 @@ -251,10 +301,10 @@ assert_rds_quickstart() { # cloud-init installs psql and writes .pgpass after sshd starts. Bound the # wait so a stalled guest still produces diagnostics within the cell budget. - if ! ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \ + if ! ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \ 'timeout 300 cloud-init status --wait >/dev/null && command -v psql >/dev/null && test -s "$HOME/.pgpass"'; then log " rds-quickstart: client bootstrap incomplete after 300s" - ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \ + ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \ 'cloud-init status --long; sudo tail -n 40 /var/log/cloud-init-output.log' 2>&1 | \ sed "s|^| |" || true return 1 @@ -266,7 +316,7 @@ assert_rds_quickstart() { "CREATE TABLE IF NOT EXISTS smoke (id int primary key, note text);" \ "INSERT INTO smoke VALUES (1, 'nightly') ON CONFLICT (id) DO UPDATE SET note = 'nightly';" \ "SELECT note FROM smoke WHERE id = 1;" | - ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \ + ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${ip}" \ 'bash -lc "psql -v ON_ERROR_STOP=1 -tA"' 2>&1); then log " rds-quickstart: SQL round-trip failed" printf '%s\n' "$query_output" | sed 's|^| |' @@ -292,7 +342,7 @@ dump_s3_webapp_guest() { fi chmod 600 "$key" 2>/dev/null || true log "--- s3-webapp guest: cloud-init status + output log + unit status ---" - ssh "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" ' + ssh_public "${SSH_OPTS[@]}" -i "$key" "ubuntu@${host}" ' echo "== cloud-init status =="; cloud-init status --long 2>/dev/null || true echo "== /var/log/cloud-init-output.log (tail 80) ==" sudo tail -n 80 /var/log/cloud-init-output.log 2>/dev/null || true @@ -322,14 +372,14 @@ assert_s3_webapp() { tmp=$(mktemp) echo "nightly-smoke" > "$tmp" - if ! curl -sf -F "file=@${tmp};filename=${sentinel}" "http://${ip}/upload" >/dev/null; then + if ! curl_public -sf -F "file=@${tmp};filename=${sentinel}" "http://${ip}/upload" >/dev/null; then rm -f "$tmp" log " s3-webapp: upload via app failed (IMDS -> STS -> S3 PutObject?)" return 1 fi rm -f "$tmp" - curl -sf "http://${ip}/" | grep -q "$sentinel" || return 1 + curl_public -sf "http://${ip}/" | grep -q "$sentinel" || return 1 # The bucket carries tags, and a clean plan over them is the gate: the AWS # provider reads them through S3 Control at https://{account}.{endpoint}, @@ -659,6 +709,7 @@ run_workbook() { # --- Main --- install_tofu || { log "tofu install failed"; exit 1; } +check_public_proxy || { log "the public-address vantage point is unusable" ; exit 1; } require_rds_image || { log "required RDS AMI is unavailable"; exit 1; } INSTANCE_TYPE=$(detect_instance_type)