diff --git a/.github/workflows/e2e-cloudvendor-nightly.yml b/.github/workflows/e2e-cloudvendor-nightly.yml index 03d14cf04..3338b2083 100644 --- a/.github/workflows/e2e-cloudvendor-nightly.yml +++ b/.github/workflows/e2e-cloudvendor-nightly.yml @@ -183,6 +183,43 @@ jobs: OCI_NO_EXTERNAL_POOL: ${{ inputs.no_external_pool && '1' || '0' }} run: ./e2e-oci.sh --summary-only + # The sweep below deletes the per-topology state directories, so anything + # still only in one of them at that point is unrecoverable -- and a failing + # workbook writes its assertion output to exactly there. Copying into the + # artifact directory first is what makes a failure diagnosable by whoever + # reads the run page, rather than only on the runner that produced it. + - name: Collect the per-topology logs + if: always() + run: | + set -uo pipefail + dest="$RUNNER_TEMP/e2e-oci" + mkdir -p "$dest" + # The topologies this run drove, taken from the per-topology log the + # harness writes rather than from the dispatch input: a run that names + # none gets the harness's own default, and reading that back here would + # mean a second copy of the list to disagree with the first. A state + # directory left by an earlier run is then not collected, so the artifact + # cannot show a topology this run never attempted. + topologies="" + for log in "$dest"/*.log; do + [ -f "$log" ] || continue + name="${log##*/}"; name="${name%.log}" + [ -d "$OCI_STATE_ROOT/.validate-$name" ] && topologies="$topologies $name" + done + echo "collecting:${topologies:- nothing}" + for topology in $topologies; do + for file in "$OCI_STATE_ROOT/.validate-$topology"/*.log \ + "$OCI_STATE_ROOT/.validate-$topology"/*.txt \ + "$OCI_STATE_ROOT/.validate-$topology"/*.tsv; do + [ -f "$file" ] || continue + # The hook handles a private key and the one thing known about an + # operator's executable is that it is not ours to publish. + case "${file##*/}" in credential-hook.log) continue ;; esac + cp -- "$file" "$dest/$topology-${file##*/}" + done + done + ls -l "$dest" + # always(), and before the artifact upload: an OCI bare-metal host left # running by a cancelled or timed-out job is the most expensive way this # workflow can fail. The sweep works from the Terraform state files, so it @@ -207,10 +244,10 @@ jobs: if: always() run: rm -f ~/.ssh/oci-spx ~/.ssh/oci-spx.pub - # Named files rather than a directory glob, for two reasons. The artifact - # directory also holds the distro tarball and the test archive, which are - # hundreds of megabytes and reproducible from the ref; and the per-topology - # state directories hold terraform.tfstate, which is never an artifact. + # Named extensions rather than a directory glob: the artifact directory also + # holds the distro tarball and the test archive, which are hundreds of + # megabytes and reproducible from the ref. Everything here is a file the + # collect step above chose, so terraform.tfstate cannot reach an artifact. - name: Upload the run if: always() uses: actions/upload-artifact@v4 @@ -221,13 +258,7 @@ jobs: ${{ runner.temp }}/e2e-oci/*.log ${{ runner.temp }}/e2e-oci/verdict.txt ${{ runner.temp }}/e2e-oci/summary.md + ${{ runner.temp }}/e2e-oci/*.txt + ${{ runner.temp }}/e2e-oci/*.tsv ${{ runner.temp }}/e2e-oci-sweep/*.log - ${{ runner.workspace }}/oci-state/.validate-*/*.log - ${{ runner.workspace }}/oci-state/.validate-*/*.txt - ${{ runner.workspace }}/oci-state/.validate-*/*.tsv - # The hook is the operator's own executable and the one thing known - # about it is that it handles a private key. Whatever it printed is not - # ours to publish in a downloadable artifact, so it is excluded rather - # than trusted to have been quiet. - !${{ runner.workspace }}/oci-state/.validate-*/credential-hook.log retention-days: 14 diff --git a/docs/oci-integration/README.md b/docs/oci-integration/README.md index 5792021c6..7bb0f0dad 100644 --- a/docs/oci-integration/README.md +++ b/docs/oci-integration/README.md @@ -10,10 +10,10 @@ tags: - terraform - cluster resources: + - title: "OCI Architecture and Operations" + url: "/docs/oci-integration/architecture" - title: "Multi-Node Install" url: "/docs/install-multi-node" - - title: "VPC Networking" - url: "/docs/vpc-networking" - title: "Host Firewall" url: "/docs/host-firewall" - title: "Spinifex Repository" @@ -26,54 +26,38 @@ resources: Spinifex brings core AWS services to hardware you control: a node runs the Spinifex daemon, QEMU/KVM for guests, OVN for VPC networking, [Predastore](https://github.com/mulgadc/predastore) for S3 and [Viperblock](https://github.com/mulgadc/viperblock) for EBS, all behind a SigV4 endpoint the ordinary AWS SDKs and CLI talk to unmodified. -This guide runs that stack on OCI. **Terraform does the whole deployment** — the VCN, the block volumes, both VNICs per node, the install, the cluster formation and the OCI public-address allocator — so [Deploy](#deploy) is four steps and one command, and everything after it is reference. One node is a working install; three is the minimum that survives losing a node. Every command below was run against a live OCI tenancy. +**Terraform does the whole deployment** — the VCN, the block volumes, both VNICs per node, the install, the cluster formation and the OCI public-address allocator. Six steps below, one command, about ten minutes for a single node. + +Everything underneath — the architecture, the IAM policy, the quotas, every variable, the troubleshooting — is in [Architecture and Operations](./architecture.md). ## Table of Contents -- [Deploy](#deploy) - - [What you need](#what-you-need) - - [1. Get the configuration](#1-get-the-configuration) - - [2. Choose one node or three](#2-choose-one-node-or-three) - - [3. Write your Terraform inputs](#3-write-your-terraform-inputs) - - [4. Run the deploy](#4-run-the-deploy) - - [What a successful run looks like](#what-a-successful-run-looks-like) - - [If a stage fails](#if-a-stage-fails) - - [Removing the deployment](#removing-the-deployment) -- [Verify end to end](#verify-end-to-end) +- [What you need](#what-you-need) +- [1. Get the configuration](#1-get-the-configuration) +- [2. Give the nodes an OCI credential](#2-give-the-nodes-an-oci-credential) +- [3. Choose one node or three](#3-choose-one-node-or-three) +- [4. Write your Terraform inputs](#4-write-your-terraform-inputs) +- [5. Deploy](#5-deploy) +- [6. Check it worked](#6-check-it-worked) +- [Harden it before production](#harden-it-before-production) - [Next steps](#next-steps) -- [Reference](#reference) - - [Why run Spinifex on OCI](#why-run-spinifex-on-oci) - - [Sizing](#sizing) - - [How it fits together](#how-it-fits-together) - - [Credentials — an instance principal, or an API key](#credentials--an-instance-principal-or-an-api-key) - - [The IAM policy](#the-iam-policy) - - [Quotas](#quotas) - - [The `oci` CLI](#the-oci-cli) - - [What Terraform builds, per node](#what-terraform-builds-per-node) - - [Every Terraform variable](#every-terraform-variable) -- [Troubleshooting](#troubleshooting) -- [Deploying without Terraform](#deploying-without-terraform) +- [Removing the deployment](#removing-the-deployment) +- [If something goes wrong](#if-something-goes-wrong) --- -## Deploy +## What you need -### What you need +| | | +| --- | --- | +| **An OCI compartment OCID** | This deploys into a compartment that already exists. Creating one needs tenancy-root rights the API user must not have | +| **OCI credentials in `~/.oci/config`** | For Terraform, to build the infrastructure. `oci setup config` writes one, or write it by hand | +| **An OCI API key for the nodes** | Separate from the above, and step 2. Without it a node forms, looks healthy, and cannot give any guest a public address | +| **Terraform or OpenTofu, `git`, Python 3** | On your workstation. The Python helper is standard library only, so there is nothing to `pip install` | -| On your workstation | Why | -| ----------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------- | -| Terraform or OpenTofu, `git`, Python 3 | Python is used only by a helper that is standard library only, so there is nothing to `pip install` | -| An SSH keypair at `~/.ssh/oci-spx` | Created in step 1. The deploy refuses to start without it rather than building instances you cannot log in to | -| An OCI compartment **OCID** | This deploys into a compartment that already exists. Creating one needs tenancy-root rights the API user must not have | -| OCI API credentials in `~/.oci/config` | For Terraform itself, to build the infrastructure | -| A way for **the nodes** to call OCI | Separate from the above, and the one prerequisite worth reading first: an [instance principal or an API key](#credentials--an-instance-principal-or-an-api-key) | +Check two quotas before you start, because both refuse at apply time rather than at plan time: **reserved public IPs** (50 per region, tenancy-wide) and the **compute limit for your shape**, which on a new tenancy is often zero for bare metal. [Quotas](./architecture.md#quotas) has the commands. -> [!IMPORTANT] -> **That last row is the one that bites.** Terraform's credentials build the infrastructure; the nodes need their own credentials at runtime to allocate public addresses. A node with none forms, passes every health check, and then refuses every launch that wants a public address with `InsufficientAddressCapacity`, naming no cause outside its own journal. Decide between the two now — it changes one argument in step 4. - -### 1. Get the configuration - -The Terraform configuration and the deploy driver live in the Spinifex repository: +## 1. Get the configuration ```bash git clone https://github.com/mulgadc/spinifex.git @@ -82,101 +66,128 @@ git switch dev cd scripts/terraform/oci-spx ``` +> [!IMPORTANT] +> **`git switch dev` is needed today.** OCI support is not in the current release, so the deploy driver is not on `main` yet. From the next release onward `main` carries it and you can skip that line. This chooses the _Terraform and the driver_ only — which Spinifex build lands on the nodes is step 5. + +Then make the keypair the deploy installs on every node and logs in with. It refuses to start without one, rather than building instances you cannot reach: + ```bash -# The deploy installs this on every node and logs in with it. Any type OpenSSH -# supports is fine; it must be at this path, or named with --ssh-public-key. ssh-keygen -t ed25519 -f ~/.ssh/oci-spx -N '' -C spinifex-oci ``` -> [!NOTE] -> **`git switch dev` is needed until the next release ships.** OCI support is not in `v1.21.0`, so the configuration you are about to run is not on `main` yet. That ref decides the _Terraform and the driver_ only — which Spinifex build lands on the nodes is a separate choice, made by `--version` in step 4. +Terraform reads your OCI credential from `~/.oci/config`, taking the tenancy, the user, the fingerprint, the key and **the region** from the profile — so the region you deploy into is the profile's, not something you set in step 4. A config holding a single `[DEFAULT]` profile, which is what `oci setup config` writes, needs nothing further. With more than one profile, name the one you mean: -### 2. Choose one node or three +```bash +./validate-topology.sh --oci-profile mycorp ... # or export OCI_CLI_PROFILE=mycorp +``` -**This is the only architectural decision, and it is one line in step 3.** +A name you pass that the config does not hold is an error listing the profiles it does hold. It never falls back, because the failure mode worth preventing is building a cluster in whichever tenancy happened to be first. -| | Single node | Three nodes | -| ------------------------------ | -------------------------------------------- | ------------------------------------------------------------------------------------------------ | -| **`node_count`** | `1` | `3` | -| **Survives losing a node** | No | Yes — NATS keeps quorum, predastore RS(2,1) keeps the data readable, the OVN raft keeps a leader | -| **Public addresses available** | 64 — the per-VNIC secondary private IP limit | 192, three VNICs' worth, still bounded by the regional reserved-public-IP quota | -| **Fault domains** | One | Three, one per node, chosen by Terraform | -| **Use it for** | Evaluation, a lab, an edge site with one box | Anything you would be unhappy to lose | +## 2. Give the nodes an OCI credential -Two is not an option worth taking: it doubles the cost of a single node and gives you a cluster that cannot form a quorum. See [Cluster sizing](/docs/install-multi-node#cluster-sizing) and [Sizing](#sizing) for shapes. +Spinifex calls the OCI API at runtime to register each guest's public address. Create an API key for that, if you do not already have one: -### 3. Write your Terraform inputs +```bash +openssl genrsa -out ~/.oci/oci_api_key.pem 2048 +chmod 600 ~/.oci/oci_api_key.pem +openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem +openssl rsa -pubout -outform DER -in ~/.oci/oci_api_key.pem | openssl md5 -c # the fingerprint +``` -One hand-edited file, in the directory you are already in. Every variable has a default that builds a working single node, so only `compartment_ocid` is strictly required: +Upload the public key under **Identity → Users → API Keys**, and grant it the [ten operations Spinifex uses](./architecture.md#the-iam-policy) in your compartment — no more. + +**The deploy installs the credential on each node by calling a small script of yours**, so no key material ever goes into user-data or Terraform state, where it would be readable from instance metadata for the life of the instance. Write one that puts these two files on each host it is given: + +| Path | Mode | Contents | +| --- | --- | --- | +| `/etc/spinifex/oci/oci_api_key.pem` | `0640 root:spinifex` | The private key | +| `/etc/spinifex/oci/config` | `0640 root:spinifex` | An OCI SDK config, profile **`[spinifex]`**, naming the key by path | + +The profile name must be `spinifex`. It is called as `your-hook ...` — the first argument is the private key to reach the nodes with, the rest are the hosts — and it must exit non-zero if any host failed, so a missing credential stops the deploy rather than surfacing later as a launch that cannot get an address. [Credentials](./architecture.md#credentials--an-api-key-or-an-instance-principal) has the config file's exact contents. + +`chmod +x` it. The deploy checks that before it builds anything, so a hook that is missing or not executable costs you a few seconds rather than forty minutes and a bare-metal bill. + +**Nothing in this repository ships that script, deliberately.** A credential belongs to whoever owns it, so the hook is yours to write and yours to keep. + +> [!TIP] +> If you are a tenancy admin you can skip key files entirely and authenticate the nodes as the instance itself. It needs a dynamic group and a policy created once at the tenancy root — see [instance principal](./architecture.md#an-instance-principal--no-key-material-but-it-needs-a-tenancy-admin). The API key path above is the one that works in any tenancy, including a compartment someone allocated to you. + +## 3. Choose one node or three + +**This is the only architectural decision, and it is one line in step 4.** + +| | Single node | Three nodes | +| --- | --- | --- | +| **`node_count`** | `1` | `3` | +| **Survives losing a node** | No | Yes — NATS keeps quorum, predastore RS(2,1) keeps the data readable, the OVN raft keeps a leader | +| **Public addresses available** | 64 | 192, three VNICs' worth, still bounded by the regional quota | +| **Fault domains** | One | Three, one per node, chosen by Terraform | +| **Use it for** | Evaluation, a lab, an edge site with one box | Anything you would be unhappy to lose | + +Two is not worth taking: it doubles the cost of a single node and gives you a cluster that cannot form a quorum. [Sizing](./architecture.md#sizing) covers shapes; bare metal is the recommendation, and `VM.Standard.E6.Flex` is the minimum. + +The pool has no configured size — Spinifex asks OCI for an address when a guest needs one. **Two ceilings bind it and neither is ours**: 64 secondary private IPs per VNIC, which is the 64 above and is not raisable, and the regional reserved-public-IP quota, which is tenancy-wide and shared with everything else you run on OCI. The second is the one you will hit first. + +## 4. Write your Terraform inputs + +One file, in the directory you are already in. Only `compartment_ocid` is required — every other variable has a default that builds a working single node: ```bash cat > terraform.auto.tfvars <<'EOF' -compartment_ocid = "ocid1.compartment.oc1..aaaa..." -deployment_name = "spinifex" +compartment_ocid = "ocid1.compartment.oc1..aaaa..." +deployment_name = "spinifex" # Shape and size. Bare metal is preferred; a flex VM is what the quota allows. -compute_shape = "VM.Standard.E6.Flex" -compute_ocpus = 8 # OCI counts an OCPU as a full core -compute_memory_in_gbs = 32 +compute_shape = "VM.Standard.E6.Flex" +compute_ocpus = 8 # OCI counts an OCPU as a full core +compute_memory_in_gbs = 32 # 1 for a single node, 3 for a cluster. This is the only line that chooses. -node_count = 3 +node_count = 1 # The block volume behind /var/lib/spinifex, attached over iSCSI. -data_volume_size_in_gbs = 256 -data_volume_vpus_per_gb = 120 # 120 = Ultra High Performance - -node_client_cidr_allow_list = ["203.0.113.10/32"] +data_volume_size_in_gbs = 256 +data_volume_vpus_per_gb = 120 # 120 = Ultra High Performance EOF ``` -Add `instance_principal = "adopt"` if you went that way in [Credentials](#credentials--an-instance-principal-or-an-api-key). [Every Terraform variable](#every-terraform-variable) has the full list. +[Every Terraform variable](./architecture.md#every-terraform-variable) has the full list. -> [!WARNING] -> **`node_client_cidr_allow_list` defaults to `0.0.0.0/0`.** Nodes sit in a public subnet with public addresses because they serve the UI, the S3 gate and the AWS gateway directly — there is no bastion. Restrict it to your own address before any real use, as the example above does. +> [!NOTE] +> **Leave `node_client_cidr_allow_list` at its `0.0.0.0/0` default for now.** It is an OCI security-list rule covering the whole public subnet, and your guests' public addresses cross it too — narrowing it to your own address cuts internet access to every guest, not just to the node. [Harden it before production](#harden-it-before-production) is where you lock the node down, in the layer that can tell the two apart. -### 4. Run the deploy - -One command builds the infrastructure, installs Spinifex, forms the cluster, configures OCI public addressing and verifies the result: +## 5. Deploy ```bash ./validate-topology.sh \ --topology vm-single \ - --version v1.21.0-70-g1743b602-dev-1-g66bc1172-dev \ + --channel dev \ --credential-hook ~/.spinifex/oci-credential-hook.sh \ --keep ``` -About ten minutes for a single node, printing each stage as it finishes. The four arguments: +One command builds the infrastructure, installs Spinifex, forms the cluster, configures OCI public addressing, verifies it, then launches real guests on public addresses and tears those down again. -| Argument | What to pass | -| ------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `--topology` | `vm-single` for one VM, `vm-multi` for a three-node cluster, `bm` for one bare-metal host. Required, with no default, so a command cannot be aimed at the wrong one by omission | -| `--version` | The Spinifex release to install on the nodes. **Use the tag above until the next release ships**, because OCI support is not in the current stable release yet | -| `--credential-hook` | Only when nodes authenticate with an API key — see [Credentials](#credentials--an-instance-principal-or-an-api-key). With an instance principal, replace it with `--instance-principal` | -| `--keep` | **This is what makes it a deployment rather than a test.** Without it the driver destroys everything at the end, which is correct for CI and not what you want here | +| Argument | What to pass | +| --- | --- | +| `--topology` | `vm-single` for one VM, `vm-multi` for three, `bm` for one bare-metal host. Required, with no default, so a command cannot be aimed at the wrong one by omission | +| `--channel` | **`dev` until the next release ships**, because OCI support is not in the current stable release yet. It resolves to the newest pre-release at the moment you run it. `latest` is the default and becomes the right answer once a release carries OCI support | +| `--credential-hook` | Your script from step 2. With an instance principal, replace it with `--instance-principal` | +| `--keep` | **This is what makes it a deployment rather than a test.** Without it the driver destroys everything at the end, which is right for CI and not what you want here | -Also worth knowing: +Add `--skip-workload` to stop once the cluster is verified, without launching the validation guests. Add `--dry-run` to print the plan and change nothing. -- `--skip-workload` stops after verifying the cluster. Without it the driver launches real guests on public addresses and tears them down again, which is the only check that proves end-to-end connectivity rather than inferring it. -- `--dry-run` prints the plan and the per-node steps and changes nothing. -- `--ssh-public-key` / `--ssh-private-key` if your keypair is not at `~/.ssh/oci-spx`. +> [!NOTE] +> **If the install step fails to download, pass a tag instead of the channel.** `dev` is the one channel that resolves through an unauthenticated GitHub API call, so it can be rate-limited and then 404s. Take the newest tag from the [releases page](https://github.com/mulgadc/spinifex/releases) and swap `--channel dev` for `--version `, which resolves by redirect and cannot trip the same limit. That is also what to use for a repeatable deployment, where you want the build pinned rather than current. -> [!TIP] -> Prefer `--version ` over `--channel dev`. Both install a prerelease, but `--channel dev` resolves through a GitHub API call that is rate limited per source address, and when it trips it returns a 404 that looks exactly like a missing release. A tag is a plain redirect and has no such failure mode. +## 6. Check it worked -### What a successful run looks like - -The stages, in order, and all of them have to appear: +The stages print as they finish, and all of these have to appear: ```text -[validate-vm-single] VM.Standard.E6.Flex, 1 node(s), state in .validate-vm-single [validate-vm-single] building -[validate-vm-single] hosts: 150.230.13.131 [validate-vm-single] waiting for cloud-init on each node -[validate-vm-single] 150.230.13.131 cloud-init: status: done -[validate-vm-single] installing Spinifex on each node (published release) -[validate-vm-single] 150.230.13.131 installed: spinifex v1.21.0-70-g1743b602-dev +[validate-vm-single] installing Spinifex on each node [validate-vm-single] initializing the single node [validate-vm-single] running the credential hook [validate-vm-single] configuring the IMDS remap and the external pool @@ -186,35 +197,9 @@ The stages, in order, and all of them have to appear: [validate-vm-single] --keep: leaving the infrastructure up, no verdict recorded ``` -**`allocator ready` is the line to look for.** Everything above it would also appear on a node that cannot allocate a public address, and that failure is otherwise silent until the first launch. `--keep` records no verdict by design: a deployment left running has not been proved destroyable, so the driver declines to call it a pass. +**`allocator ready` is the line to look for.** Every line above it also appears on a node that cannot allocate a public address, and that failure is otherwise silent until a guest asks for one. -### If a stage fails - -The driver stops at the first failure, names the log, and leaves the rest alone. Logs are all under `.validate-/`: - -| Stage | Log | -| ----------------------------------------------- | ---------------------------------------- | -| Building the infrastructure | `apply.log` | -| Installing Spinifex | `install-.log` | -| Forming the cluster | `form.log` | -| Installing the credential, configuring the pool | `credential-hook.log`, `pool-.log` | -| Verifying public addressing | `allocator-.log` | - -The last one is the log worth knowing about, because public addressing is the part with the most moving pieces. If the allocator never comes up, the driver captures the daemon journal, the pool as configured, the bridge and OCI MAC addresses, and whether each credential file exists and is readable — never the credentials themselves — before anything is torn down. [Troubleshooting](#troubleshooting) reads those. - -Add `--keep-on-fail` to leave a failed deployment running so you can log in and look. **It keeps billing** until you destroy it. - -### Removing the deployment - -```bash -./validate-topology.sh --topology vm-single --destroy-only -``` - -It destroys whatever that topology's Terraform state holds, so it can only remove what this configuration created. - ---- - -## Verify end to end +Then prove it the way a customer would. On the node, the `[spinifex]` profile that `spx admin init` wrote into `~/.aws/credentials` is the one to use: ```bash export AWS_PROFILE=spinifex @@ -223,11 +208,10 @@ aws ec2 allocate-address aws ec2 run-instances --image-id --instance-type t3.micro --key-name \ --subnet-id --associate-public-ip-address aws ec2 describe-instances --query \ - 'Reservations[].Instances[].[InstanceId,PrivateIpAddress,PublicIpAddress,State.Name]' \ - --output text + 'Reservations[].Instances[].[InstanceId,PublicIpAddress,State.Name]' --output text ``` -Then reach the guest on its public address — **from your workstation, not from the node**: +Then reach the guest **from your workstation, not from the node**: ```bash ping -c 3 @@ -235,50 +219,42 @@ ssh -i ubuntu@ hostname ``` > [!IMPORTANT] -> **Probe a public address from outside the VCN, not from a node.** Two things make a node a bad vantage point, and they are indistinguishable from each other and from a broken datapath: `node_client_cidr_allow_list` may not admit the node's own public address, and a VCN may not reflect its public addresses back into itself at all. Either way a healthy cluster fails the probe, so a timeout from a node is not evidence about the guest. Private addresses are the separate question and do work from inside. +> **Probe a public address from outside the VCN.** A node is a bad vantage point for its own network's public addresses, so a timeout from a node is not evidence about the guest. Private addresses are the separate question and do work from inside. -**Confirm the OCI allocator came up**, which `spx get nodes` cannot tell you. Run this on the node: +## Harden it before production + +The deployment comes up usable, not locked down. Three layers filter traffic and they are easy to confuse — [Firewalls, in all three layers](./architecture.md#firewalls-in-all-three-layers) explains which is which. The short version: + +**1. Arm the Spinifex host firewall.** It is installed but not armed on this path. Arming it gives you the proper policy: the public plane open, the cluster plane scoped to peers only, and SSH scoped to a list you own. ```bash -sudo journalctl -u spinifex-vpcd --since -5m | grep -i ocinet +# On every node. +sudo env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on ``` -```text -"msg":"ocinet resolved the external VNIC from its interface","pool":"oci-public","iface":"br-wan", - "vnic_id":"ocid1.vnic.oc1.ap-sydney-1.abzxsljrjwlf...kimuq","private_ip":"10.200.1.31","subnet":"10.200.0.0/23" -"msg":"OCI allocator ready","pool":"oci-public","collected":0,"stale_bindings":0,"skipped":1 -``` - -**`resolved the external VNIC` is the line that matters** — it proves the credentials work, the compartment is right, and `br-wan`'s MAC matched a real VNIC. A node missing it will accept `allocate-address` and fail it. - -> [!NOTE] -> On a cold cluster start you may instead see `OCI allocator reconcile failed … nats: no responders available for request` on some nodes. That is vpcd racing JetStream's KV at boot; the startup reconcile is skipped and not retried. It is harmless on a new cluster where nothing has been allocated, and it is tracked — restart `spinifex-vpcd` on that node to run it. The `resolved the external VNIC` line above is still the one that decides whether allocation works. - -And from inside the guest — note that IMDS is **IMDSv2, so it needs a token**; an untokened request correctly returns 401: +**2. Narrow SSH.** It is open to the world until you do — that comes from the Ubuntu image's own rules, not from ours. Edit the one file the installer creates and never overwrites: ```bash -TOK=$(curl -s -X PUT http://169.254.169.254/latest/api/token \ - -H 'X-aws-ec2-metadata-token-ttl-seconds: 120') -curl -s -H "X-aws-ec2-metadata-token: $TOK" \ - http://169.254.169.254/latest/meta-data/instance-id -traceroute -n 8.8.8.8 +# /etc/spinifex/firewall/custom.nft +define trusted_ssh_peers = { 203.0.113.10/32 } ``` ---- +```bash +sudo nft -c -f /etc/spinifex/firewall/spinifex.nft # check, then apply +sudo /usr/local/lib/spinifex/spinifex-firewall-apply +``` + +**3. Decide about the console on port 3000.** The S3 gate (8443) and the AWS gateway (9999) are the product's public surface and should stay public. The web console need not be. + +**Leave the OCI security list open.** Guest public addresses cross it, so narrowing `node_client_cidr_allow_list` breaks your tenants rather than protecting the node — the host firewall above is the layer that can tell a node's listener from a guest's. + +Full detail, including which port serves what and why forming a cluster starts by turning the policy off, is in [Host Firewall](/docs/host-firewall). ## Next steps -### Set up your cluster +**Set up your cluster.** It is running but holds nothing yet — no images, no networks, no instances. [Setting Up Your Cluster](/docs/setting-up-your-cluster) imports an AMI, creates a key pair and a VPC, and launches your first instance. -The cluster is running, but it holds nothing yet — no machine images, no networks, no instances. - -Continue to [Setting Up Your Cluster](/docs/setting-up-your-cluster) to import an AMI, create an SSH key pair, create a VPC with a public subnet, and launch your first instance. It ends by arming the [host firewall](/docs/host-firewall), which is also where you re-arm it if you turned it off to form the cluster. - -### Run your existing Terraform against the cluster - -This is the part worth pausing on. Everything above builds infrastructure **on** OCI using the OCI provider. Everything from here uses the **AWS** provider — unmodified, from the OpenTofu or Terraform registry — pointed at your own cluster. The same `aws_vpc`, `aws_instance`, `aws_db_instance`, `aws_ecs_service` and `aws_eks_cluster` resources a team already has in git apply against Spinifex on an OCI tenancy, with no rewriting and no OCI-specific module. - -Three things differ from a provider block aimed at AWS, and only three: +**Run your existing Terraform against it.** Everything above builds infrastructure _on_ OCI with the OCI provider. From here you use the **AWS** provider, unmodified from the registry, pointed at your own cluster — the same `aws_vpc`, `aws_instance`, `aws_db_instance`, `aws_ecs_service` and `aws_eks_cluster` resources a team already has in git. Three things differ, and only three: ```hcl provider "aws" { @@ -299,353 +275,36 @@ provider "aws" { } ``` -Credentials come from the `[spinifex]` profile that `spx admin init` writes into `~/.aws/credentials` on node 1. +> **Use the node's private address, not its public one.** The node certificate carries no SAN for the public address, because that address is never on the wire — OCI NATs it to a private one. A call to `https://:9999` fails TLS verification with _hostname doesn't match_. Run Terraform from a node, or from anything else inside the VCN. -> **Use the node's private address, not its OCI reserved public IP.** The node certificate carries no SAN for the public address, because that address is never on the wire — OCI NATs it to a private one. An `aws` or `terraform` call to `https://:9999` fails TLS verification with _hostname doesn't match_. Until that is fixed, run Terraform from a node, or from anything else inside the VCN. +The [Terraform workbooks](../terraform-workbooks/) we ship are the worked examples, and `e2e-cloudvendor-nightly` runs them on a single VM, on three VMs and on bare metal, publishing a table per topology on its own run page. Read that for the build you are installing. -That portability is a claim, so it is measured rather than asserted. Each workbook below is a full `apply`, a functional assertion against the thing it built, and a `destroy`, measured on 2026-09-26 against a three-node cluster of `VM.Standard.E6.Flex` instances in `ap-sydney-1` running `spinifex v1.20.0-113`. - -**This is a snapshot, and the current answer is the nightly.** `e2e-cloudvendor-nightly` runs the same workbooks on a single VM, on three VMs and on bare metal, and publishes a table per topology on its own run page. Read that for the build you are installing rather than taking this table as current. - -| Workbook | What it exercises | Result | -| ------------------------ | ----------------------------------------------------------------------------------------------- | ----------------- | -| `nginx-webserver` | VPC, subnet, IGW, security group, key pair, EC2 instance, public IP; HTTP 200 from the internet | **Passed** (86s) | -| `bastion-private-subnet` | Public and private subnets, NAT egress, a bastion, SSH hop to a private host | **Passed** (117s) | -| `rds-quickstart` | DB subnet group, parameter group, `aws_db_instance` (PostgreSQL), client instance, connection | **Passed** (296s) | -| `ecs-quickstart` | ECS cluster, task definition, service, container instances, ALB with a healthy target | **Passed** (166s) | -| `eks-quickstart` | EKS control plane, managed node group, `kubectl` against the cluster, nodes `Ready` | **Passed** (259s) | -| `nginx-alb` | Application Load Balancer across two subnets, two backends, health checks | **Passed** (123s) | -| `s3-webapp` | S3 bucket, IAM role and instance profile, IMDS-fetched credentials, upload from the guest | **Passed** | -| `demo-app` | Container image for the EKS workbooks, pushed to ECR | **Passed** | -| `eks-https-ingress` | Private-subnet workers behind a NAT gateway, LBC addon, ACM certificate, HTTPS Ingress | **Passed** (252s) | -| `eks-gitops-argocd` | Private-subnet workers, Argo CD addon, EBS-CSI PersistentVolume, GitOps sync | **Passed** (225s) | - -### Oracle Linux as a guest image - -Spinifex ships four Oracle Linux entries in its image catalog, so a customer on OCI can run the same distro their Oracle support contract covers. They import exactly like any other AMI — nothing about them is OCI-specific, and they run equally well on bare metal: - -| Catalog name | Release | Arch | Kernel | -| -------------------- | ----------------- | ------ | ------ | -| `oracle-10.1-x86_64` | Oracle Linux 10.1 | x86_64 | UEK 8 | -| `oracle-10.1-arm64` | Oracle Linux 10.1 | arm64 | UEK 8 | -| `oracle-9.8-x86_64` | Oracle Linux 9.8 | x86_64 | UEK 7 | -| `oracle-9.8-arm64` | Oracle Linux 9.8 | arm64 | UEK 7 | +**Oracle Linux guests.** Four Oracle Linux images are in the catalog, so you can run the distro your Oracle support contract covers: ```bash sudo spx admin images import --name oracle-10.1-x86_64 --config /etc/spinifex/spinifex.toml -aws ec2 describe-images --query 'Images[].[Name,State,BootMode]' --output text ``` -All four boot **UEFI**, so launch them into a shape that boots UEFI — a BIOS-only instance type will not come up, and the failure looks like a hung boot rather than a rejected image. +`oracle-10.1-x86_64`, `oracle-10.1-arm64`, `oracle-9.8-x86_64` and `oracle-9.8-arm64`. All four boot **UEFI**, so launch them into a shape that boots UEFI or the failure looks like a hung boot rather than a rejected image. **Log in as `cloud-user`, not `opc`** — these are the generic KVM cloud images from `yum.oracle.com`, and `opc`, `oracle` and `ec2-user` are all refused. The UI's instance detail page shows the right user per AMI. -**The login user is `cloud-user`, not `opc`**, and this is the one that catches people: +## Removing the deployment ```bash -ssh -i path/to/key.pem cloud-user@ +./validate-topology.sh --topology vm-single --destroy-only ``` -`opc` is the default user on the images Oracle publishes _into OCI itself_; these are the generic **KVM cloud images** from `yum.oracle.com`, whose cloud-init `default_user` is `cloud-user`. Verified on both releases: `opc`, `oracle` and `ec2-user` are all refused. The Spinifex UI's instance detail page shows the right user per AMI, so read it there rather than guessing from the distro. +It destroys whatever that topology's Terraform state holds, so it can only remove what this configuration created. ---- +## If something goes wrong -## Reference +The deploy stops at the first failure and names the log it wrote, all under `.validate-/`. Add `--keep-on-fail` to leave a failed deployment up so you can log in and look — **it keeps billing** until you run `--destroy-only`. -### Why run Spinifex on OCI +[Troubleshooting](./architecture.md#troubleshooting) covers the symptoms worth knowing in advance, and several of them look like a different fault than they are: -**Lower cost for the same shape of workload.** OCI's compute and block storage are priced well below the large US clouds for equivalent OCPU and IOPS, and Spinifex turns one large instance into a whole EC2/EBS/S3 surface — so a tenant's twenty small instances are twenty QEMU guests on hardware you are billed for once, not twenty billable cloud instances. +- [If a deploy stage fails](./architecture.md#if-a-deploy-stage-fails) — which log holds the answer +- [A guest's public address is unreachable](./architecture.md#a-guests-public-address-is-unreachable) +- [A node that was fine and then was not](./architecture.md#a-node-that-was-fine-and-then-was-not) — one command answers all of it +- [Guests cannot reach instance metadata](./architecture.md#guests-cannot-reach-instance-metadata) — OCI and AWS guests want the same address +- [Addresses and quotas](./architecture.md#addresses-and-quotas) — including why a detached address still bills -**Egress stops being the line item that decides the architecture.** Oracle publishes a large monthly outbound-transfer allowance per tenancy and a per-GB rate beyond it that is roughly an order of magnitude below the majors — check the current pricing page rather than a number in a document, but the gap is the reason this integration exists. Traffic _between_ guests never leaves the host or, on a cluster, never leaves the VCN: it is Geneve over the private plane, which is not billed as internet egress at all. - -**Portability — the workload stops being AWS-shaped and starts being yours.** The API your tenants code against is Spinifex's, so the same Terraform, the same SDK calls and the same AMIs run on OCI today, on bare metal in your own rack tomorrow, and at an edge site after that. Moving off AWS becomes a deployment decision rather than a rewrite, and moving _again_ costs nothing extra. - -**You keep the parts of OCI that are worth keeping.** Bare-metal shapes, fault domains, block volumes with a chosen performance tier, and real public addresses out of OCI's own pool. Spinifex does not hide the cloud underneath it — it registers each guest's address with OCI so that guest is genuinely reachable on the internet, not behind a shared NAT. - -### Sizing - -**Bare metal is the recommendation.** `BM.Standard.E2.64` or similar gives you the hardware directly — no nested virtualisation, full NIC performance, and no hypervisor between your guests and the wire. If bare metal is out of scope, a single `VM.Standard.E6.Flex` is the recommendation for testing and development, or three of them for production. - -| | Minimum | Recommended | -| --------------- | --------------------- | ----------------------------------------------------------------- | -| **Shape** | `VM.Standard.E6.Flex` | Bare metal, e.g. `BM.Standard.E2.64` | -| **OCPU** | 8 (16 vCPU) | 32+ | -| **RAM** | 32 GB | 128 GB+ | -| **Data volume** | 256 GiB block volume | 1 TB+ NVMe-backed | -| **VNICs** | 2 | 2 | -| **Nodes** | 1 | 3 — see [Cluster sizing](/docs/install-multi-node#cluster-sizing) | - -### How it fits together - -Everything in [Single-Node Install](/docs/install) and [Multi-Node Install](/docs/install-multi-node) still applies: formation, storage and OVN behave exactly as they do on hardware you own. What changes is the layer underneath — an OCI VCN is not an Ethernet segment, so the external datapath is wired differently, and public addresses come from OCI's API rather than from a range you write in a config file. - -**Single node — the whole AWS surface inside one OCI instance.** One instance runs the control plane and every tenant guest. Customers reach the AWS APIs on the node's own address; their instances reach the internet through addresses registered on the node's second VNIC. - -

- Single node on OCI — the instance inside the VCN public subnet, both VNICs, br-wan, the routed-NAT transit veth, the control plane, tenant VPCs and the iSCSI data volume -

- -Two things in that picture are the whole integration. The guest's private address (`10.0.1.4`) never leaves the host — OVN and the host route both hold it. The address OCI delivers to is a **secondary private IP registered on VNIC 1**, appearing on `br-wan`, and the customer's public address (`150.230.13.131`) is a reserved public IP that OCI 1:1-NATs onto it upstream. `describe-instances` reports the public one; every datapath object holds the private one, which is why the two never appear together on the host. - -The third is storage. Every guest disk, every S3 object and the JetStream state all live under `/var/lib/spinifex`, which is an **OCI block volume attached over iSCSI**, not the boot disk. Size it for the guests you intend to run; a boot volume that quietly ends up carrying the stack is the most common way an install on OCI fails later rather than immediately. - -**Three nodes — the same surface, distributed.** Each node registers the addresses for **its own** guests on **its own** second VNIC, and the allocator runs per node, so nothing has to know another node's OCIDs. When a guest moves — a stop/start that lands elsewhere, or a host failure — the node it lands on claims the address's private half from the old VNIC through a single OCI call. The OCID and the public half are untouched, so the customer's address never changes. - -

- Three nodes on OCI — three instances inside the VCN public subnet, one per fault domain, each with its own VNICs, addresses and iSCSI volume, joined east-west by Geneve, NATS, predastore and the OVN raft -

- -The overlay, the object shards and the OVN databases all cross **VNIC 0**, inside the VCN. Size that plane, not the external one: guest-to-guest traffic between nodes is Geneve over the VCN, and it is the link every distributed layer shares. - -> [!NOTE] -> **The Spinifex region is your own naming and has nothing to do with the OCI region.** `ap-southeast-2` in `spinifex.toml` beside `ap-sydney-1` in OCI is correct and expected. Do not read one as evidence about the other. - -### Credentials — an instance principal, or an API key - -The allocator needs OCI credentials at runtime, and there are two ways to give it them. - -**An instance principal is the better one, and it is a one-time setup step.** The instance certificate is served at `/opc/v2/identity/cert.pem` and the Go SDK authenticates as the instance with no key material on disk — but it must also be _authorised_, by a dynamic group and a policy. On the reference tenancy it authenticated without being authorised for anything, which is the silent failure above. Those two resources live at the tenancy root, so creating them needs a tenancy admin once; every deployment and rebuild afterwards needs only a compartment-scoped user: - -```bash -cd scripts/terraform/oci-spx -./setup-identity.sh --dry-run # always first -./setup-identity.sh # needs a tenancy-admin OCI profile -``` - -Then set `instance_principal = "adopt"` in your tfvars and pass `--instance-principal` to the deploy. The dynamic group matches on `instance.compartment.id`, so it covers every node you ever build in that compartment and nothing needs updating when nodes are replaced. - -**An API key is the fallback**, and the default, because it needs nothing from a tenancy admin — which makes it the only option when the compartment belongs to someone else's tenancy. The cost is key material on disk, with a rotation obligation, on every node: - -```bash -# On your workstation, not the node. -openssl genrsa -out ~/.oci/oci_api_key.pem 2048 -chmod 600 ~/.oci/oci_api_key.pem -openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem -openssl rsa -pubout -outform DER -in ~/.oci/oci_api_key.pem \ - | openssl md5 -c # this is the fingerprint -``` - -Upload the public key to the user under **Identity → Users → API Keys**, and record the user OCID, the fingerprint and the tenancy OCID. - -Each node then needs **two** files, and a node with only the first is the silent failure — it forms, looks healthy, and cannot allocate: - -| Path | Mode | Contents | -| ----------------------------------- | -------------------- | ------------------------------------------------------------------------ | -| `/etc/spinifex/oci/oci_api_key.pem` | `0640 root:spinifex` | The private key | -| `/etc/spinifex/oci/config` | `0640 root:spinifex` | An ordinary OCI SDK config, profile `[spinifex]`, naming the key by path | - -Nothing creates the second one for you, and its profile name must be `spinifex` to match `oci_config_profile` in the pool: - -```ini -[spinifex] -user=ocid1.user.oc1..aaaa... -fingerprint=12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef -tenancy=ocid1.tenancy.oc1..aaaa... -region=ap-sydney-1 -key_file=/etc/spinifex/oci/oci_api_key.pem -``` - -**Put that in a credential hook and the deploy installs it on every node.** The hook is an executable of yours, run as `hook ...` after formation and before the pool is configured, which is the only window where a node has `/etc/spinifex` but has not started the allocator. It is yours rather than ours deliberately: a credential belongs to whoever owns it, and one rendered into user-data or Terraform state is readable from instance metadata for the life of the instance — on a host that runs other people's guests. - -### The IAM policy - -Spinifex calls exactly nine operations. Grant no more than these: - -| Operation | Why | -| ------------------------------------------ | -------------------------------------------------------------------------------- | -| `CreatePrivateIp` | Register the secondary private IP the datapath carries | -| `DeletePrivateIp` | Release it | -| `GetPrivateIp` | Poll for `AVAILABLE` after an asynchronous assign | -| `ListPrivateIps` | The startup reconcile, and the affinity pass that moves an address between nodes | -| `UpdatePrivateIp` | Reassign a private IP to this node's VNIC when a guest moves here | -| `CreatePublicIp` | Create the RESERVED public IP | -| `DeletePublicIp` | Release it | -| `GetPublicIp` / `GetPublicIpByPrivateIpId` | Poll lifecycle state, and reconcile a private IP back to its public half | -| `UpdatePublicIp` | Attach and detach the public IP from a private IP | - -The matching policy, scoped to the one compartment Spinifex allocates in: - -```text -Allow group SpinifexOperators to manage private-ips in compartment -Allow group SpinifexOperators to manage public-ips in compartment -Allow group SpinifexOperators to read vnics in compartment -Allow group SpinifexOperators to read subnets in compartment -Allow group SpinifexOperators to read vcns in compartment -``` - -**Scope it to a compartment, not the tenancy.** `manage public-ips` at tenancy level lets a compromised node consume the whole regional reserved-public-IP quota. - -**Do not grant `manage instances` or `manage vnics`.** Spinifex never creates, attaches or deletes a VNIC — it only adds addresses to a VNIC that already exists. A grant that allows VNIC lifecycle is strictly more than the integration can use. - -### Quotas - -Two limits bound how many public addresses a deployment can hand out, and they do not scale the same way: - -| Quota | Default | Scope | Notes | -| ---------------------------------- | ------- | ------------------------- | ---------------------------------------------------------------------------------------------- | -| **Reserved public IPs** | 50 | Per region, whole tenancy | Shared by every node, so it does **not** grow with node count. This is the wall you hit first. | -| **Secondary private IPs per VNIC** | 64 | Per VNIC, so per node | Not raisable. Three nodes gives you 192 — this is the limit that does scale. | - -A raise on the first is a support request, not a setting: Console → Governance & Administration → Limits, Quotas and Usage → "Request a service limit increase". Cite the exact limit name from `oci limits definition list`, not a name from documentation. - -```bash -oci limits definition list --service-name vcn --compartment-id -oci limits value list --service-name vcn --compartment-id -oci limits resource-availability get --service-name vcn \ - --limit-name --compartment-id -``` - -**These calls need a grant the node does not have, and must not be given.** Limits and usage are tenancy-level, which cannot be compartment-scoped the way the policy above insists the node's is. Create a **second, read-only principal** for an operator or a reporting job and keep it off every node: - -```text -Allow group SpinifexReporting to read limits in tenancy -Allow group SpinifexReporting to read usage-reports in tenancy -``` - -The second verb is what Cost Analysis and `oci usage-api usage-summary request-summarized-usages` read. Adding either to `SpinifexOperators` would hand every node tenancy-wide read, which is exactly what the node policy refuses. - -### The `oci` CLI - -**The `oci` CLI is not on the Ubuntu image**, and `apt` has no `python3-oci-cli` package. Nothing in [Deploy](#deploy) needs it; install it when you want to inspect allocations by hand: - -```bash -sudo apt-get update && sudo apt-get install -y python3-venv -python3 -m venv ~/.oci-cli-venv -~/.oci-cli-venv/bin/pip install --upgrade pip oci-cli -sudo ln -sf ~/.oci-cli-venv/bin/oci /usr/local/bin/oci -oci --version -``` - -**What the CLI is and is not for.** Spinifex's daemon uses the OCI **Go SDK** for every allocation — the CLI is not on that path. You need it for bootstrap and for operational inspection. Do not build automation that shells out to it during an allocation; allocations run inside a request budget that a forked Python process will not respect. - -### What Terraform builds, per node - -| Resource | Detail | -| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | -| `oci_core_instance` | One per `node_count`, round-robined across **fault domains** so three nodes land on three sets of racks rather than wherever placement puts them | -| Primary VNIC | Public IP, `hostname_label`, the host plane — SSH, the advertise address, the Geneve endpoint | -| `oci_core_vnic_attachment` | A **second VNIC** in the same subnet. Every Spinifex external address becomes a secondary private IP here at runtime | -| `oci_core_volume` | A block volume per node, `data_volume_size_in_gbs` at `data_volume_vpus_per_gb` | -| `oci_core_volume_attachment` | **`attachment_type = "iscsi"`**, with the Oracle Cloud Agent's Block Volume Management plugin enabled so the node logs the iSCSI session in itself | -| cloud-init | Partitions, formats, mounts and `fstab`s the volume; builds `br-wan` over the second VNIC; opens the service ports | - -The resource-by-resource architecture is in [`scripts/terraform/oci-spx`](../../scripts/terraform/oci-spx/README.md). - -**The data volume is iSCSI, and that is the detail that bites.** OCI presents the volume as an iSCSI target reachable at `169.254.2.2:3260` — attaching it in the API does not put a block device on the host. `is_agent_auto_iscsi_login_enabled` makes the Oracle Cloud Agent do the login, and because it does that _asynchronously_, the device is not there when cloud-init first runs. The mount script waits up to ten minutes for it, formats it **only if `blkid` reports no filesystem** (so a re-run cannot erase a populated volume), and writes an fstab entry by UUID: - -```text -UUID= /var/lib/spinifex ext4 defaults,_netdev,nofail 0 2 -``` - -`_netdev` is what orders the mount after the network, and therefore after `iscsid` has a session. `nofail` is what keeps a missing volume from wedging the boot. Mounting `/var/lib/spinifex` _itself_ rather than somewhere else and symlinking is deliberate: viperblock, predastore and JetStream all land on the volume with no configuration pointing anywhere unusual. - -### Every Terraform variable - -All of these have defaults that build a working single node, except the first: - -| Variable | Default | What it decides | -| ----------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `compartment_ocid` | — | **Required.** An OCID, not a name: this deploys into a compartment that already exists, because creating one needs tenancy-root rights the API user must not have | -| `deployment_name` | `spinifex` | Prefixes every resource and becomes the VCN `dns_label`, so 1–15 lowercase alphanumerics starting with a letter | -| `node_count` | `1` | Nodes, 1–25. `1` for a single node, `3` for a cluster. Each gets its own volume, its own second VNIC and its own fault domain | -| `compute_shape` | `VM.Standard.E6.Flex` | **`BM.*` for bare metal, `VM.*.Flex` for a VM.** A fixed bare-metal shape carries its own size, so `shape_config` is emitted only when the name contains `Flex` — set a BM shape and the two sizing variables below are ignored rather than rejected | -| `compute_ocpus` | `8` | OCPUs on a flex shape. Validated at ≥ 8 (16 vCPU), the minimum recommended for a node | -| `compute_memory_in_gbs` | `32` | GiB on a flex shape. Validated at ≥ 32 | -| `data_volume_size_in_gbs` | `256` | Size of each node's data volume. This is where guest disks, S3 objects and JetStream live — size it for the guests, not for the control plane | -| `data_volume_vpus_per_gb` | `120` | Performance tier, 0–120 VPUs/GB. 120 is Ultra High Performance; 10 is Balanced. Higher is more IOPS and more cost | -| `data_mountpoint` | `/var/lib/spinifex` | Where cloud-init mounts it. Change only if you know why | -| `vcn_cidr` | `10.200.0.0/22` | VCN address space. A `/22` leaves room for the secondary private IPs Spinifex registers per node | -| `public_subnet_cidr` | `10.200.0.0/23` | The subnet every node sits in, **on both VNICs**. Nodes serve the UI, the S3 gate and the AWS gateway directly, so they cannot live in a private subnet | -| `node_client_cidr_allow_list` | `["0.0.0.0/0"]` | Who may reach SSH and the service ports from the internet. **Change this** | -| `node_service_ports` | `[3000, 8443, 9999]` | Console, predastore gate, AWS gateway. SSH is opened separately | -| `ubuntu_version` | `26.04` | Canonical platform image version. The newest image for the shape and region is resolved at plan time rather than pinned to a regional OCID | -| `wan_bridge_name` | `br-wan` | The Linux bridge built over the second VNIC | -| `wan_bridge_mtu` | `9000` | The OCI VCN carries 9000 | - ---- - -## Troubleshooting - -### Addresses and quotas - -**Spinifex enforces no limit of its own.** It attempts the allocation and reports what OCI says, so the ceiling you hit is always your tenancy's, never a number baked into the software. A quota is raised with Oracle, not reconfigured here, and there is nothing to restart afterwards. - -**`InsufficientAddressCapacity` on `allocate-address` is what exhaustion looks like.** It is the AWS error code for "out of addresses" and Spinifex returns it for either quota, mapped from OCI's own refusal. On a cluster the regional one is the likelier of the two. - -**A detached reserved public IP still bills.** `disassociate-address` returns the address to `AVAILABLE` without deleting it, exactly as an unassociated AWS Elastic IP behaves, and it keeps consuming your regional quota as well as your bill. `release-address` is what frees both. - -**A stopped instance costs nothing for its auto-assigned address.** Stopping an instance returns its auto-assigned address to the pool and deletes the reserved public IP object behind it, as AWS does; the start that follows creates a new one. An **Elastic IP** is yours and survives the stop untouched — that is the whole distinction between the two, and on OCI it is also the difference between a stopped fleet that bills for addresses and one that does not. - -### Datapath - -**Instance launches but the public address is unreachable.** In order: - -1. `sudo ovn-nbctl lr-nat-list ` — is there a `dnat_and_snat` row, and does it hold the _private_ address? -2. `ip route get ` — does it route via the transit veth? -3. `sudo iptables -S FORWARD | grep spinifex-eip-ingress` — both directions present? -4. `oci network private-ip list --vnic-id ` — is the private address actually registered with OCI, **and on this node's VNIC**? If not, nothing else matters. - -**Egress works from one address and not another on the same NIC.** OCI enforces the source address per VNIC, so this means the address is not a registered private-IP object on it. It is never a routing problem on the host. - -**An address went dark after a guest moved node.** The affinity pass should have claimed it within 15 seconds. Check the new node's daemon journal for `ocinet affinity`, then confirm with step 4 above which VNIC OCI thinks holds the private half. - -### A node that was fine and then was not - -Terraform's cloud-init does the host jobs on each node — the iSCSI volume, `br-wan` and its source routing, the firewall. When one of them did not take, the symptom arrives much later and rarely looks like a networking or storage problem. This one command answers all of it: - -```bash -ssh -i ~/.ssh/oci-spx ubuntu@ ' - cloud-init status # done - sudo iscsiadm -m session # a session to 169.254.2.2:3260 - findmnt /var/lib/spinifex # mounted by UUID, with _netdev - ip -br addr show br-wan # UP, second VNIC address as a /32 - ip -br addr show enp1s0 # the bridge MEMBER holds NO address - ip route show default # still on the PRIMARY interface - ls -l /dev/kvm # present -' -``` - -Three answers are worth knowing in advance, because each looks wrong and is not: - -- **`br-wan` holds a `/32` and the default route is on the other interface.** That is how two VNICs share one subnet without the second stealing the first one's traffic, and it is what satisfies OCI's per-VNIC source check. `br-wan`'s address appears in `ip rule` instead of in the default route. -- **`br-wan`'s address is not one you chose.** It is the second VNIC's own private IP, assigned by OCI, and not predictable before the apply. -- **`enp1s0` carrying an address is the one real fault here.** The kernel derives an on-link route from it that beats the primary VNIC's, so every packet to another node leaves the wrong VNIC and OCI drops it. The node still forms a cluster and still reports `Ready` — formation uses outbound connections — and it surfaces later as `predastore … the stripe is short one holder` on the first AMI import. `sudo /usr/local/sbin/spinifex-setup-wan-bridge` repairs it in place. - -### Host - -**`RunInstances` returns `ServerInternal`.** Check the daemon journal for the real error — `journalctl -u spinifex-daemon --since -10m`. An OCI allocation failure surfaces this way. - -**The stack ends up on the boot disk and the node fills.** The data volume was mounted without `_netdev` and lost the race with `iscsid`. `findmnt /var/lib/spinifex` is the check; `ls` cannot tell the difference. - -### Guests cannot reach instance metadata - -**Guests boot but cloud-init hangs on metadata**, or the host stops answering its own. OCI uses `169.254.169.254` for its instance metadata, and so does every AWS-compatible guest — the host and the guests want the same address. - -Spinifex resolves this by moving the _host_ side of the per-guest metadata link to a different link-local pair, so guests keep asking `169.254.169.254` exactly as they do on AWS. [The deploy](#4-run-the-deploy) sets `imds_host_meta_ip` and `imds_host_dns_ip` for you. **Set both or neither** — a half-configured pair is ignored and the node behaves as if the remap were off, which is what this symptom looks like. - -Check it after your first guest launches: - -```bash -ip route get 169.254.169.254 # your real NIC, not lo -curl -sH 'Authorization: Bearer Oracle' -o /dev/null -w '%{http_code}\n' \ - http://169.254.169.254/opc/v2/instance/ # 200 -ip -br addr show | grep ime- # holds 169.254.42.x, not .169.x -``` - -`ip route get 169.254.169.254` returning `dev lo` is the signature of a missing remap. The same signature can appear after a guest terminates, from a leftover `ime-` endpoint still holding the address; with the remap configured it is harmless, and removing the stale OVS port clears it. - ---- - -## Deploying without Terraform - -**Terraform automates every step of a normal install, and that is the point of it.** The `oci-spx` configuration builds the VCN, the subnets, the gateways, the block volumes and both VNICs per node, then renders the cloud-init that installs Spinifex and sets up OVN — so the sequence below happens without anyone typing it, in the same order every time, on one node or three. - -If you need to install by hand — an existing OCI tenancy you cannot run Terraform against, an unsupported shape, or a stage you are debugging — the install itself is not OCI-specific. Follow the standard guides: - -- [Installing Spinifex](../install/install/README.md) for a single node. -- [Multi-Node Installation](../install/install-multi-node/README.md) for a cluster, which covers `spx admin init` on the leader and `spx admin join` on the rest. - -Four things about OCI are not in those guides, and each has its own section above: - -- **The external pool** is `source = "oci"`, which needs the instance principal or API key from [Credentials](#credentials--an-instance-principal-or-an-api-key) and the [IAM policy](#the-iam-policy). Without it a node forms, passes every health check, and then refuses every launch that wants a public address. -- **`br-wan` stays a Linux bridge** owned by netplan, linked to OVS `br-ext` by a veth pair. Pass `setup-ovn.sh --wan-bridge=br-wan` and it detects this itself; the NIC never becomes an OVS port, so the VNIC keeps its MAC identity. See [How it fits together](#how-it-fits-together). -- **The block volume must be mounted with `_netdev`**, because it arrives over iSCSI and the mount is otherwise attempted before a session exists. -- **IMDS needs the 169.254 remap** described under [Guests cannot reach instance metadata](#guests-cannot-reach-instance-metadata), since the guest's metadata address collides with OCI's own. +Installing by hand instead of with Terraform is [Deploying without Terraform](./architecture.md#deploying-without-terraform). diff --git a/docs/oci-integration/architecture.md b/docs/oci-integration/architecture.md new file mode 100644 index 000000000..281b2f705 --- /dev/null +++ b/docs/oci-integration/architecture.md @@ -0,0 +1,465 @@ +--- +title: "Spinifex on Oracle Cloud — Architecture and Operations" +seoTitle: "Spinifex on OCI: architecture, IAM, quotas and troubleshooting — Spinifex Docs" +description: "How Spinifex is wired on OCI — both VNICs, the iSCSI data volume, the public-address allocator — with the IAM policy, the quotas, every Terraform variable and the troubleshooting." +category: "Install" +tags: + - install + - oci + - oracle cloud + - terraform + - architecture + - troubleshooting +resources: + - title: "Spinifex on Oracle Cloud" + url: "/docs/oci-integration" + - title: "Host Firewall" + url: "/docs/host-firewall" + - title: "VPC Networking" + url: "/docs/vpc-networking" + - title: "Multi-Node Install" + url: "/docs/install-multi-node" +--- + +# Spinifex on OCI — Architecture and Operations + +> The reference half of the OCI integration. [Spinifex on Oracle Cloud](/docs/oci-integration) is the step-by-step deployment; everything here is what sits underneath it, plus the troubleshooting. + +## Table of Contents + +- [Why run Spinifex on OCI](#why-run-spinifex-on-oci) +- [Sizing](#sizing) +- [How it fits together](#how-it-fits-together) +- [Credentials — an API key, or an instance principal](#credentials--an-api-key-or-an-instance-principal) +- [The IAM policy](#the-iam-policy) +- [Quotas](#quotas) +- [Firewalls, in all three layers](#firewalls-in-all-three-layers) +- [What Terraform builds, per node](#what-terraform-builds-per-node) +- [Every Terraform variable](#every-terraform-variable) +- [The `oci` CLI](#the-oci-cli) +- [Troubleshooting](#troubleshooting) +- [Deploying without Terraform](#deploying-without-terraform) + +--- + +## Why run Spinifex on OCI + +**Lower cost for the same shape of workload.** OCI's compute and block storage are priced well below the large US clouds for equivalent OCPU and IOPS, and Spinifex turns one large instance into a whole EC2/EBS/S3 surface — so a tenant's twenty small instances are twenty QEMU guests on hardware you are billed for once, not twenty billable cloud instances. + +**Egress stops being the line item that decides the architecture.** Oracle publishes a large monthly outbound-transfer allowance per tenancy and a per-GB rate beyond it that is roughly an order of magnitude below the majors — check the current pricing page rather than a number in a document, but the gap is the reason this integration exists. Traffic _between_ guests never leaves the host or, on a cluster, never leaves the VCN: it is Geneve over the private plane, which is not billed as internet egress at all. + +**Portability — the workload stops being AWS-shaped and starts being yours.** The API your tenants code against is Spinifex's, so the same Terraform, the same SDK calls and the same AMIs run on OCI today, on bare metal in your own rack tomorrow, and at an edge site after that. Moving off AWS becomes a deployment decision rather than a rewrite, and moving _again_ costs nothing extra. + +**You keep the parts of OCI that are worth keeping.** Bare-metal shapes, fault domains, block volumes with a chosen performance tier, and real public addresses out of OCI's own pool. Spinifex does not hide the cloud underneath it — it registers each guest's address with OCI so that guest is genuinely reachable on the internet, not behind a shared NAT. + +## Sizing + +**Bare metal is the recommendation.** `BM.Standard.E2.64` or similar gives you the hardware directly — no nested virtualisation, full NIC performance, and no hypervisor between your guests and the wire. If bare metal is out of scope, a single `VM.Standard.E6.Flex` is the recommendation for testing and development, or three of them for production. + +| | Minimum | Recommended | +| --------------- | --------------------- | ----------------------------------------------------------------- | +| **Shape** | `VM.Standard.E6.Flex` | Bare metal, e.g. `BM.Standard.E2.64` | +| **OCPU** | 8 (16 vCPU) | 32+ | +| **RAM** | 32 GB | 128 GB+ | +| **Data volume** | 256 GiB block volume | 1 TB+ NVMe-backed | +| **VNICs** | 2 | 2 | +| **Nodes** | 1 | 3 — see [Cluster sizing](/docs/install-multi-node#cluster-sizing) | + +## How it fits together + +Everything in [Single-Node Install](/docs/install) and [Multi-Node Install](/docs/install-multi-node) still applies: formation, storage and OVN behave exactly as they do on hardware you own. What changes is the layer underneath — an OCI VCN is not an Ethernet segment, so the external datapath is wired differently, and public addresses come from OCI's API rather than from a range you write in a config file. + +### Single node — the whole AWS surface inside one OCI instance + +One instance runs the control plane and every tenant guest. Customers reach the AWS APIs on the node's own address; their instances reach the internet through addresses registered on the node's second VNIC. + +

+ Single node on OCI — the instance inside the VCN public subnet, both VNICs, br-wan, the routed-NAT transit veth, the control plane, tenant VPCs and the iSCSI data volume +

+ +Two things in that picture are the whole integration. The guest's private address (`10.0.1.4`) never leaves the host — OVN and the host route both hold it. The address OCI delivers to is a **secondary private IP registered on VNIC 1**, appearing on `br-wan`, and the customer's public address (`150.230.13.131`) is a reserved public IP that OCI 1:1-NATs onto it upstream. `describe-instances` reports the public one; every datapath object holds the private one, which is why the two never appear together on the host. + +The third is storage. Every guest disk, every S3 object and the JetStream state all live under `/var/lib/spinifex`, which is an **OCI block volume attached over iSCSI**, not the boot disk. Size it for the guests you intend to run; a boot volume that quietly ends up carrying the stack is the most common way an install on OCI fails later rather than immediately. + +### Three nodes — the same surface, distributed + +Each node registers the addresses for **its own** guests on **its own** second VNIC, and the allocator runs per node, so nothing has to know another node's OCIDs. When a guest moves — a stop/start that lands elsewhere, or a host failure — the node it lands on claims the address's private half from the old VNIC through a single OCI call. The OCID and the public half are untouched, so the customer's address never changes. + +

+ Three nodes on OCI — three instances inside the VCN public subnet, one per fault domain, each with its own VNICs, addresses and iSCSI volume, joined east-west by Geneve, NATS, predastore and the OVN raft +

+ +The overlay, the object shards and the OVN databases all cross **VNIC 0**, inside the VCN. Size that plane, not the external one: guest-to-guest traffic between nodes is Geneve over the VCN, and it is the link every distributed layer shares. + +> [!NOTE] +> **The Spinifex region is your own naming and has nothing to do with the OCI region.** `ap-southeast-2` in `spinifex.toml` beside `ap-sydney-1` in OCI is correct and expected. Do not read one as evidence about the other. + +## Credentials — an API key, or an instance principal + +The allocator needs OCI credentials at runtime. **A node with none forms, passes every health check, and then refuses every launch that wants a public address** with `InsufficientAddressCapacity`, naming no cause outside its own journal. + +### An API key — the documented path + +It needs nothing from a tenancy admin, which also makes it the only option when the compartment belongs to someone else's tenancy. The cost is key material on disk, with a rotation obligation, on every node: + +```bash +# On your workstation, not the node. +openssl genrsa -out ~/.oci/oci_api_key.pem 2048 +chmod 600 ~/.oci/oci_api_key.pem +openssl rsa -pubout -in ~/.oci/oci_api_key.pem -out ~/.oci/oci_api_key_public.pem +openssl rsa -pubout -outform DER -in ~/.oci/oci_api_key.pem \ + | openssl md5 -c # this is the fingerprint +``` + +Upload the public key to the user under **Identity → Users → API Keys**, and record the user OCID, the fingerprint and the tenancy OCID. + +Each node then needs **two** files, and a node with only the first is the silent failure above — it forms, looks healthy, and cannot allocate: + +| Path | Mode | Contents | +| ----------------------------------- | -------------------- | ------------------------------------------------------------------------ | +| `/etc/spinifex/oci/oci_api_key.pem` | `0640 root:spinifex` | The private key | +| `/etc/spinifex/oci/config` | `0640 root:spinifex` | An ordinary OCI SDK config, profile `[spinifex]`, naming the key by path | + +Nothing creates the second one for you, and its profile name must be `spinifex` to match `oci_config_profile` in the pool: + +```ini +[spinifex] +user=ocid1.user.oc1..aaaa... +fingerprint=12:34:56:78:90:ab:cd:ef:12:34:56:78:90:ab:cd:ef +tenancy=ocid1.tenancy.oc1..aaaa... +region=ap-sydney-1 +key_file=/etc/spinifex/oci/oci_api_key.pem +``` + +**Put that in a credential hook and the deploy installs it on every node.** The hook is an executable of yours, run as `hook ...` after formation and before the pool is configured, which is the only window where a node has `/etc/spinifex` but has not started the allocator. It is yours rather than ours deliberately: a credential belongs to whoever owns it, and one rendered into user-data or Terraform state is readable from instance metadata for the life of the instance — on a host that runs other people's guests. + +### An instance principal — no key material, but it needs a tenancy admin + +The instance certificate is served at `/opc/v2/identity/cert.pem` and the Go SDK authenticates as the instance with no key on disk. It must also be _authorised_, by a dynamic group and a policy, and **both are tenancy-root resources** — `oci_identity_dynamic_group` is created with `compartment_id` set to the tenancy OCID. A compartment-scoped user cannot create them, and the attempt fails with `404-NotAuthorizedOrNotFound` on `CreateDynamicGroup`, which reads like a misconfiguration and is not one. + +```bash +cd scripts/terraform/oci-spx +./setup-identity.sh --dry-run # always first +./setup-identity.sh # needs a tenancy-admin OCI profile +``` + +Then set `instance_principal = "adopt"` in your tfvars and pass `--instance-principal` to the deploy instead of `--credential-hook`. The dynamic group matches on `instance.compartment.id`, so it covers every node you ever build in that compartment and nothing needs updating when nodes are replaced. + +**This path is not yet proved end to end.** Every validated run so far used an API key, because the reference tenancy is an allocated child compartment with no tenancy-root rights. On a tenancy where you are admin it should be the better choice; treat the first run as the test. + +## The IAM policy + +Spinifex calls exactly ten operations, all of them in `spinifex/cloud/oci/client.go`. Grant no more than these: + +| Operation | Why | +| ------------------------------------------ | -------------------------------------------------------------------------------- | +| `CreatePrivateIp` | Register the secondary private IP the datapath carries | +| `DeletePrivateIp` | Release it | +| `GetPrivateIp` | Poll for `AVAILABLE` after an asynchronous assign | +| `ListPrivateIps` | The startup reconcile, and the affinity pass that moves an address between nodes | +| `UpdatePrivateIp` | Reassign a private IP to this node's VNIC when a guest moves here | +| `CreatePublicIp` | Create the RESERVED public IP | +| `DeletePublicIp` | Release it | +| `GetPublicIp` / `GetPublicIpByPrivateIpId` | Poll lifecycle state, and reconcile a private IP back to its public half | +| `UpdatePublicIp` | Attach and detach the public IP from a private IP | + +The matching policy, scoped to the one compartment Spinifex allocates in: + +```text +Allow group SpinifexOperators to use vnics in compartment +Allow group SpinifexOperators to manage private-ips in compartment +Allow group SpinifexOperators to manage public-ips in compartment +``` + +These are the same three statements `instance-principal.tf` grants the dynamic group, which is the only version of this policy that has been exercised. Keep them identical: the two credential paths differ in who is authorised, never in what. + +> [!IMPORTANT] +> **`use vnics` is required, and `read vnics` is not enough.** Registering a secondary private IP is an operation *on a VNIC*, so OCI checks `VNIC_ASSIGN` as well as `PRIVATE_IP_CREATE`, and `VNIC_ASSIGN` lives in the `use` verb. With only `read`, the node forms and looks healthy and every allocation fails `NotAuthorizedOrNotFound` — which reads as a wrong compartment OCID rather than a missing verb. + +**Scope it to a compartment, not the tenancy.** `manage public-ips` at tenancy level lets a compromised node consume the whole regional reserved-public-IP quota. + +**Do not grant `manage vnics` or `manage instances`.** Spinifex never creates, attaches or deletes a VNIC, and never touches an instance — it only adds addresses to a VNIC that already exists. `use` permits that; `manage` permits VNIC lifecycle, which is strictly more than the integration can use. + +## Quotas + +Two limits bound how many public addresses a deployment can hand out, and they do not scale the same way: + +| Quota | Default | Scope | Notes | +| ---------------------------------- | ------- | ------------------------- | ---------------------------------------------------------------------------------------------- | +| **Reserved public IPs** | 50 | Per region, whole tenancy | Shared by every node, so it does **not** grow with node count. This is the wall you hit first. | +| **Secondary private IPs per VNIC** | 64 | Per VNIC, so per node | Not raisable. Three nodes gives you 192 — this is the limit that does scale. | + +**On a new tenancy, check the compute limits too.** A fresh account commonly carries a service limit of zero for bare-metal shapes and a low one for flex VMs, so `BM.Standard.E2.64` may be unavailable until you request it. That refusal arrives at apply time, not at plan time. + +A raise is a support request, not a setting: Console → Governance & Administration → Limits, Quotas and Usage → "Request a service limit increase". Cite the exact limit name from `oci limits definition list`, not a name from documentation. + +```bash +oci limits definition list --service-name vcn --compartment-id +oci limits value list --service-name vcn --compartment-id +oci limits resource-availability get --service-name vcn \ + --limit-name --compartment-id +``` + +**These calls need a grant the node does not have, and must not be given.** Limits and usage are tenancy-level, which cannot be compartment-scoped the way the policy above insists the node's is. Create a **second, read-only principal** for an operator or a reporting job and keep it off every node: + +```text +Allow group SpinifexReporting to read limits in tenancy +Allow group SpinifexReporting to read usage-reports in tenancy +``` + +The second verb is what Cost Analysis and `oci usage-api usage-summary request-summarized-usages` read. Adding either to `SpinifexOperators` would hand every node tenancy-wide read, which is exactly what the node policy refuses. + +## Firewalls, in all three layers + +Three independent things filter traffic to an OCI deployment, and they are easy to mistake for one. Nearly every "my port is closed" report on OCI is the wrong layer being adjusted. + +| Layer | What it guards | Who configures it | +| --- | --- | --- | +| **OCI security list** | The whole public subnet, including the addresses guests are reached on | `node_client_cidr_allow_list` | +| **The host firewall** | The node's own listeners — console, S3 gate, AWS gateway, SSH, the cluster plane | `node_service_ports`, then the Spinifex nft policy | +| **OVN security groups** | Each guest, per AWS security-group rules | Your tenants, through the EC2 API | + +### The OCI security list is deliberately open, and should stay that way + +```hcl +# network.tf, security_list.public +ingress_security_rules { protocol = "all", source = var.vcn_cidr } # intra-VCN +dynamic "ingress_security_rules" { ... source = each allow-list CIDR } # protocol = "all" +``` + +A guest's public address is a **secondary private IP on VNIC 1, in the public subnet**, so inbound traffic to a guest crosses this security list. Narrowing `node_client_cidr_allow_list` to your own office `/32` therefore cuts internet ingress to **every guest**, not just to the node — a tenant who opens 443 on their own security group finds it does nothing, with no error anywhere naming the cause. + +That is also why the rule is `protocol = "all"` rather than the three service ports: enumerating ports here would mean a Terraform change for every customer port. + +**So leave it at `0.0.0.0/0` for any deployment that serves guests**, and do the restricting in the host firewall, which is the layer that can tell the node's own listeners apart from a guest's. Narrow the allow list only for a private lab with no public guest services. + +The separate intra-VCN rule exists so that narrowing the allow list can never cut Geneve, OVN, NATS or predastore between nodes. + +### The host firewall is what you harden + +Terraform's cloud-init runs `open-service-ports.sh`, which opens all protocols from the peer CIDR plus `node_service_ports` (`3000 8443 9999`) to any source, in the Ubuntu image's **iptables**. SSH is already open to the world from the image's own INPUT rule, and nothing in that script narrows it. + +The Spinifex nft policy is installed but **not armed** on this path, because the installer arms it only when asked. Arming it is what gives you the proper policy — public plane, peer-scoped cluster plane, and an SSH scope you own: + +```bash +# On each node. Arms the policy the ISO installer arms by default. +sudo env SETUP_STAGES=firewall /usr/local/share/spinifex/setup.sh --firewall on +``` + +Then narrow SSH in the one file `setup.sh` creates and never overwrites: + +```bash +# /etc/spinifex/firewall/custom.nft — yours, left unchanged by upgrades. +define trusted_ssh_peers = { 203.0.113.10/32 } +``` + +```bash +sudo nft -c -f /etc/spinifex/firewall/spinifex.nft # check before applying +sudo /usr/local/lib/spinifex/spinifex-firewall-apply +``` + +What the armed policy allows, and why each port is where it is, is in [Host Firewall](/docs/host-firewall) — including [Narrowing SSH access](/docs/host-firewall#narrowing-ssh-access) and the rule that **forming or expanding a cluster starts by turning the policy off**, because nodes mid-formation are not yet each other's peers. + +Two OCI-specific notes on top of that guide: + +- **Northstar's DNS on 53 is in the armed policy's public plane, but `node_service_ports` does not include it.** Until the nft policy is armed, a node serving a public zone does not answer on 53 from the internet. +- **Never flush iptables on an OCI node.** The image's `InstanceServices` chain in OUTPUT is what permits iSCSI to `169.254.2.0/24:3260` and the metadata service; clearing it takes the data volume and IMDS with it. + +## What Terraform builds, per node + +| Resource | Detail | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------- | +| `oci_core_instance` | One per `node_count`, round-robined across **fault domains** so three nodes land on three sets of racks rather than wherever placement puts them | +| Primary VNIC | Public IP, `hostname_label`, the host plane — SSH, the advertise address, the Geneve endpoint | +| `oci_core_vnic_attachment` | A **second VNIC** in the same subnet. Every Spinifex external address becomes a secondary private IP here at runtime | +| `oci_core_volume` | A block volume per node, `data_volume_size_in_gbs` at `data_volume_vpus_per_gb` | +| `oci_core_volume_attachment` | **`attachment_type = "iscsi"`**, with the Oracle Cloud Agent's Block Volume Management plugin enabled so the node logs the iSCSI session in itself | +| cloud-init | Partitions, formats, mounts and `fstab`s the volume; builds `br-wan` over the second VNIC; opens the service ports | + +The resource-by-resource architecture is in [`scripts/terraform/oci-spx`](../../scripts/terraform/oci-spx/README.md). + +**The data volume is iSCSI, and that is the detail that bites.** OCI presents the volume as an iSCSI target reachable at `169.254.2.2:3260` — attaching it in the API does not put a block device on the host. `is_agent_auto_iscsi_login_enabled` makes the Oracle Cloud Agent do the login, and because it does that _asynchronously_, the device is not there when cloud-init first runs. The mount script waits up to ten minutes for it, formats it **only if `blkid` reports no filesystem** (so a re-run cannot erase a populated volume), and writes an fstab entry by UUID: + +```text +UUID= /var/lib/spinifex ext4 defaults,_netdev,nofail 0 2 +``` + +`_netdev` is what orders the mount after the network, and therefore after `iscsid` has a session. `nofail` is what keeps a missing volume from wedging the boot. Mounting `/var/lib/spinifex` _itself_ rather than somewhere else and symlinking is deliberate: viperblock, predastore and JetStream all land on the volume with no configuration pointing anywhere unusual. + +## Every Terraform variable + +All of these have defaults that build a working single node, except the first: + +| Variable | Default | What it decides | +| ----------------------------- | --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `compartment_ocid` | — | **Required.** An OCID, not a name: this deploys into a compartment that already exists, because creating one needs tenancy-root rights the API user must not have | +| `deployment_name` | `spinifex` | Prefixes every resource and becomes the VCN `dns_label`, so 1–15 lowercase alphanumerics starting with a letter | +| `node_count` | `1` | Nodes, 1–25. `1` for a single node, `3` for a cluster. Each gets its own volume, its own second VNIC and its own fault domain | +| `compute_shape` | `VM.Standard.E6.Flex` | **`BM.*` for bare metal, `VM.*.Flex` for a VM.** A fixed bare-metal shape carries its own size, so `shape_config` is emitted only when the name contains `Flex` — set a BM shape and the two sizing variables below are ignored rather than rejected | +| `compute_ocpus` | `8` | OCPUs on a flex shape. Validated at ≥ 8 (16 vCPU), the minimum recommended for a node | +| `compute_memory_in_gbs` | `32` | GiB on a flex shape. Validated at ≥ 32 | +| `data_volume_size_in_gbs` | `256` | Size of each node's data volume. This is where guest disks, S3 objects and JetStream live — size it for the guests, not for the control plane | +| `data_volume_vpus_per_gb` | `120` | Performance tier, 0–120 VPUs/GB. 120 is Ultra High Performance; 10 is Balanced. Higher is more IOPS and more cost | +| `data_mountpoint` | `/var/lib/spinifex` | Where cloud-init mounts it. Change only if you know why | +| `vcn_cidr` | `10.200.0.0/22` | VCN address space. A `/22` leaves room for the secondary private IPs Spinifex registers per node | +| `public_subnet_cidr` | `10.200.0.0/23` | The subnet every node sits in, **on both VNICs**. Nodes serve the UI, the S3 gate and the AWS gateway directly, so they cannot live in a private subnet | +| `node_client_cidr_allow_list` | `["0.0.0.0/0"]` | OCI security-list ingress, **all protocols**, for the whole public subnet. Guest public addresses cross it too — see [Firewalls](#firewalls-in-all-three-layers) before narrowing it | +| `node_service_ports` | `[3000, 8443, 9999]` | Console, predastore gate, AWS gateway, opened in the host's iptables by cloud-init. SSH comes from the image's own rule | +| `instance_principal` | unset | `"adopt"` to authenticate the allocator as the instance rather than with a key file. Needs the tenancy dynamic group and policy to exist | +| `ubuntu_version` | `26.04` | Canonical platform image version. The newest image for the shape and region is resolved at plan time rather than pinned to a regional OCID | +| `wan_bridge_name` | `br-wan` | The Linux bridge built over the second VNIC | +| `wan_bridge_mtu` | `9000` | The OCI VCN carries 9000 | + +**Six more exist and are not yours to set**: `tenancy_ocid`, `user_ocid`, `fingerprint`, `private_key`, `private_key_path` and `region`. `scripts/oci_env.py` fills them as `TF_VAR_*` from the profile in `~/.oci/config` on every `terraform` call the driver makes, so **the region you deploy into is the profile's region** and the credential never reaches a `.tfvars` file. Setting them by hand works and is how a run with no config file at all would have to work, but then two sources disagree about which tenancy this is. + +The profile is resolved as `--oci-profile`, then `$OCI_CLI_PROFILE`, then the reference tenancy, then `DEFAULT`. A name passed explicitly and absent from the config is an error that lists the profiles present; only an unasked-for name falls through. CI takes a different route entirely — `OCI_TENANCY_OCID`, `OCI_USER_OCID`, `OCI_FINGERPRINT`, `OCI_PRIVATE_KEY` and `OCI_REGION` in the environment, so a runner writes no key to disk — and a partial set there is an error rather than a fallback, because half a credential means a secret failed to reach the job. + +## The `oci` CLI + +**The `oci` CLI is not on the Ubuntu image**, and `apt` has no `python3-oci-cli` package. Nothing in the deployment needs it; install it when you want to inspect allocations by hand: + +```bash +sudo apt-get update && sudo apt-get install -y python3-venv +python3 -m venv ~/.oci-cli-venv +~/.oci-cli-venv/bin/pip install --upgrade pip oci-cli +sudo ln -sf ~/.oci-cli-venv/bin/oci /usr/local/bin/oci +oci --version +``` + +**What the CLI is and is not for.** Spinifex's daemon uses the OCI **Go SDK** for every allocation — the CLI is not on that path. You need it for bootstrap and for operational inspection. Do not build automation that shells out to it during an allocation; allocations run inside a request budget that a forked Python process will not respect. + +--- + +## Troubleshooting + +### If a deploy stage fails + +The driver stops at the first failure, names the log, and leaves the rest alone. Logs are all under `.validate-/`: + +| Stage | Log | +| ----------------------------------------------- | ---------------------------------------- | +| Building the infrastructure | `apply.log` | +| Installing Spinifex | `install-.log` | +| Forming the cluster | `form.log` | +| Installing the credential, configuring the pool | `credential-hook.log`, `pool-.log` | +| Verifying public addressing | `allocator-.log` | +| The Terraform workbooks | `workbooks.log`, `workbooks.tsv` | + +The allocator log is the one worth knowing about, because public addressing is the part with the most moving pieces. If the allocator never comes up, the driver captures the daemon journal, the pool as configured, the bridge and OCI MAC addresses, and whether each credential file exists and is readable — never the credentials themselves — before anything is torn down. + +Add `--keep-on-fail` to leave a failed deployment running so you can log in and look. **It keeps billing** until you run `--destroy-only`. + +### Addresses and quotas + +**Spinifex enforces no limit of its own.** It attempts the allocation and reports what OCI says, so the ceiling you hit is always your tenancy's, never a number baked into the software. A quota is raised with Oracle, not reconfigured here, and there is nothing to restart afterwards. + +**`InsufficientAddressCapacity` on `allocate-address` is what exhaustion looks like.** It is the AWS error code for "out of addresses" and Spinifex returns it for either quota, mapped from OCI's own refusal. On a cluster the regional one is the likelier of the two. + +**A detached reserved public IP still bills.** `disassociate-address` returns the address to `AVAILABLE` without deleting it, exactly as an unassociated AWS Elastic IP behaves, and it keeps consuming your regional quota as well as your bill. `release-address` is what frees both. + +**A stopped instance costs nothing for its auto-assigned address.** Stopping an instance returns its auto-assigned address to the pool and deletes the reserved public IP object behind it, as AWS does; the start that follows creates a new one. An **Elastic IP** is yours and survives the stop untouched — that is the whole distinction between the two, and on OCI it is also the difference between a stopped fleet that bills for addresses and one that does not. + +### A guest's public address is unreachable + +**Check where you are probing from first.** A node is a bad vantage point for a public address, and two independent things make it so: the security list may not admit the node's own public address as a source, and a VCN may or may not reflect its public addresses back into itself. Either way a healthy cluster fails the probe. Test from outside the VCN, which is where a customer is. + +Then, in order: + +1. `sudo ovn-nbctl lr-nat-list ` — is there a `dnat_and_snat` row, and does it hold the _private_ address? +2. `ip route get ` — does it route via the transit veth? +3. `sudo iptables -S FORWARD | grep spinifex-eip-ingress` — both directions present? +4. `oci network private-ip list --vnic-id ` — is the private address actually registered with OCI, **and on this node's VNIC**? If not, nothing else matters. + +**Egress works from one address and not another on the same NIC.** OCI enforces the source address per VNIC, so this means the address is not a registered private-IP object on it. It is never a routing problem on the host. + +**An address went dark after a guest moved node.** The affinity pass should have claimed it within 15 seconds. Check the new node's daemon journal for `ocinet affinity`, then confirm with step 4 above which VNIC OCI thinks holds the private half. + +### A node that was fine and then was not + +Terraform's cloud-init does the host jobs on each node — the iSCSI volume, `br-wan` and its source routing, the firewall. When one of them did not take, the symptom arrives much later and rarely looks like a networking or storage problem. This one command answers all of it: + +```bash +ssh -i ~/.ssh/oci-spx ubuntu@ ' + cloud-init status # done + sudo iscsiadm -m session # a session to 169.254.2.2:3260 + findmnt /var/lib/spinifex # mounted by UUID, with _netdev + ip -br addr show br-wan # UP, second VNIC address as a /32 + ip -br addr show enp1s0 # the bridge MEMBER holds NO address + ip route show default # still on the PRIMARY interface + ls -l /dev/kvm # present +' +``` + +Three answers are worth knowing in advance, because each looks wrong and is not: + +- **`br-wan` holds a `/32` and the default route is on the other interface.** That is how two VNICs share one subnet without the second stealing the first one's traffic, and it is what satisfies OCI's per-VNIC source check. `br-wan`'s address appears in `ip rule` instead of in the default route. +- **`br-wan`'s address is not one you chose.** It is the second VNIC's own private IP, assigned by OCI, and not predictable before the apply. +- **`enp1s0` carrying an address is the one real fault here.** The kernel derives an on-link route from it that beats the primary VNIC's, so every packet to another node leaves the wrong VNIC and OCI drops it. The node still forms a cluster and still reports `Ready` — formation uses outbound connections — and it surfaces later as `predastore … the stripe is short one holder` on the first AMI import. `sudo /usr/local/sbin/spinifex-setup-wan-bridge` repairs it in place. + +### The allocator + +**Confirm it came up**, which `spx get nodes` cannot tell you: + +```bash +sudo journalctl -u spinifex-vpcd --since -5m | grep -i ocinet +``` + +```text +"msg":"ocinet resolved the external VNIC from its interface","pool":"oci-public","iface":"br-wan", + "vnic_id":"ocid1.vnic.oc1.ap-sydney-1.abzxsljrjwlf...kimuq","private_ip":"10.200.1.31","subnet":"10.200.0.0/23" +"msg":"OCI allocator ready","pool":"oci-public","collected":0,"stale_bindings":0,"skipped":1 +``` + +**`resolved the external VNIC` is the line that matters** — it proves the credentials work, the compartment is right, and `br-wan`'s MAC matched a real VNIC. A node missing it will accept `allocate-address` and fail it. + +On a cold cluster start you may instead see `OCI allocator reconcile failed … nats: no responders available for request` on some nodes. That is vpcd racing JetStream's KV at boot; the startup reconcile is skipped and not retried. It is harmless on a new cluster where nothing has been allocated, and it is tracked — restart `spinifex-vpcd` on that node to run it. + +### Host + +**`RunInstances` returns `ServerInternal`.** Check the daemon journal for the real error — `journalctl -u spinifex-daemon --since -10m`. An OCI allocation failure surfaces this way. + +**The stack ends up on the boot disk and the node fills.** The data volume was mounted without `_netdev` and lost the race with `iscsid`. `findmnt /var/lib/spinifex` is the check; `ls` cannot tell the difference. + +### Guests cannot reach instance metadata + +**Guests boot but cloud-init hangs on metadata**, or the host stops answering its own. OCI uses `169.254.169.254` for its instance metadata, and so does every AWS-compatible guest — the host and the guests want the same address. + +Spinifex resolves this by moving the _host_ side of the per-guest metadata link to a different link-local pair, so guests keep asking `169.254.169.254` exactly as they do on AWS. The deploy sets `imds_host_meta_ip` and `imds_host_dns_ip` for you. **Set both or neither** — a half-configured pair is ignored and the node behaves as if the remap were off, which is what this symptom looks like. + +Check it after your first guest launches: + +```bash +ip route get 169.254.169.254 # your real NIC, not lo +curl -sH 'Authorization: Bearer Oracle' -o /dev/null -w '%{http_code}\n' \ + http://169.254.169.254/opc/v2/instance/ # 200 +ip -br addr show | grep ime- # holds 169.254.42.x, not .169.x +``` + +`ip route get 169.254.169.254` returning `dev lo` is the signature of a missing remap. The same signature can appear after a guest terminates, from a leftover `ime-` endpoint still holding the address; with the remap configured it is harmless, and removing the stale OVS port clears it. + +### An OVS problem, when `ovs-vsctl` says nothing is wrong + +`ovs-vsctl` exits 0 even when a bridge could not be instantiated. Read the error column and the log, never the exit status: + +```bash +sudo ovs-vsctl --columns=name,error list Interface +sudo journalctl -u ovs-vswitchd --since -10m +``` + +--- + +## Deploying without Terraform + +**Terraform automates every step of a normal install, and that is the point of it.** The `oci-spx` configuration builds the VCN, the subnets, the gateways, the block volumes and both VNICs per node, then renders the cloud-init that installs Spinifex and sets up OVN — so the sequence below happens without anyone typing it, in the same order every time, on one node or three. + +If you need to install by hand — an existing OCI tenancy you cannot run Terraform against, an unsupported shape, or a stage you are debugging — the install itself is not OCI-specific. Follow the standard guides: + +- [Installing Spinifex](../install/install/README.md) for a single node. +- [Multi-Node Installation](../install/install-multi-node/README.md) for a cluster, which covers `spx admin init` on the leader and `spx admin join` on the rest. + +Four things about OCI are not in those guides, and each has its own section above: + +- **The external pool** is `source = "oci"`, which needs the API key or instance principal from [Credentials](#credentials--an-api-key-or-an-instance-principal) and the [IAM policy](#the-iam-policy). Without it a node forms, passes every health check, and then refuses every launch that wants a public address. +- **`br-wan` stays a Linux bridge** owned by netplan, linked to OVS `br-ext` by a veth pair. Pass `setup-ovn.sh --wan-bridge=br-wan` and it detects this itself; the NIC never becomes an OVS port, so the VNIC keeps its MAC identity. See [How it fits together](#how-it-fits-together). +- **The block volume must be mounted with `_netdev`**, because it arrives over iSCSI and the mount is otherwise attempted before a session exists. +- **IMDS needs the 169.254 remap** described under [Guests cannot reach instance metadata](#guests-cannot-reach-instance-metadata), since the guest's metadata address collides with OCI's own. diff --git a/scripts/terraform/oci-spx/scripts/oci_env.py b/scripts/terraform/oci-spx/scripts/oci_env.py index 431ec9108..d77376a58 100755 --- a/scripts/terraform/oci-spx/scripts/oci_env.py +++ b/scripts/terraform/oci-spx/scripts/oci_env.py @@ -25,21 +25,61 @@ import sys REPOSITORY = Path(__file__).resolve().parent.parent REQUESTED_PROFILE = "apacanzset03child03" FALLBACK_PROFILE = "apacanzset03child3" +# The OCI CLI's own name for "the profile when none is named", so a config written +# by `oci setup config` works with no flag. Tried only when nothing was asked for +# explicitly, which keeps a named profile a hard requirement rather than a hint. +DEFAULT_PROFILE = "DEFAULT" + + +def default_profile() -> str: + """The profile to read when the caller named none. + + OCI_CLI_PROFILE is the OCI CLI's own variable, so an operator who has already + selected a profile for `oci` does not select it twice. The reference tenancy's + name remains the last resort, which is what every run of ours uses. + """ + return os.environ.get("OCI_CLI_PROFILE") or REQUESTED_PROFILE + + +def has_profile(parser: configparser.RawConfigParser, name: str) -> bool: + """Whether the config holds this profile. + + DEFAULT needs its own test: configparser treats it as the fallback for every + other section rather than as a section, so has_section and sections() both + deny it exists even when `oci setup config` wrote nothing else. + """ + if name == DEFAULT_PROFILE: + return bool(parser.defaults()) + return parser.has_section(name) + + +def profile_names(parser: configparser.RawConfigParser) -> list[str]: + names = list(parser.sections()) + if parser.defaults(): + names.insert(0, DEFAULT_PROFILE) + return names def load_profile(config_path: Path, requested_profile: str) -> tuple[str, dict[str, str]]: parser = configparser.RawConfigParser() parser.read(config_path) profile = requested_profile - if not parser.has_section(profile): - if profile == REQUESTED_PROFILE and parser.has_section(FALLBACK_PROFILE): - print( - f"Profile {REQUESTED_PROFILE} not found; using {FALLBACK_PROFILE}.", - file=sys.stderr, - ) - profile = FALLBACK_PROFILE + if not has_profile(parser, profile): + # Only an unasked-for profile falls back. A name the caller passed and the + # config does not hold is an error, because the alternative is building a + # cluster in whichever tenancy DEFAULT happens to point at. + implicit = profile == default_profile() + for candidate in (FALLBACK_PROFILE, DEFAULT_PROFILE) if implicit else (): + if has_profile(parser, candidate): + print(f"Profile {profile} not found; using {candidate}.", file=sys.stderr) + profile = candidate + break else: - raise ValueError(f"OCI profile {profile!r} was not found in {config_path}") + raise ValueError( + f"OCI profile {profile!r} was not found in {config_path}; " + f"profiles present: {', '.join(profile_names(parser)) or 'none'}. " + "Name one with --profile or OCI_CLI_PROFILE." + ) values = {key: value.strip() for key, value in parser.items(profile)} required = ("tenancy", "user", "fingerprint", "key_file", "region") @@ -166,7 +206,11 @@ def terraform_environment( def main() -> int: arg_parser = argparse.ArgumentParser() - arg_parser.add_argument("--profile", default=REQUESTED_PROFILE) + arg_parser.add_argument( + "--profile", + default=default_profile(), + help="Profile in ~/.oci/config. Defaults to $OCI_CLI_PROFILE, then the reference tenancy, then DEFAULT.", + ) arg_parser.add_argument("--region") arg_parser.add_argument( "--ssh-public-key-path", diff --git a/scripts/terraform/oci-spx/validate-topology.sh b/scripts/terraform/oci-spx/validate-topology.sh index ba74a16bd..cfcc861a4 100755 --- a/scripts/terraform/oci-spx/validate-topology.sh +++ b/scripts/terraform/oci-spx/validate-topology.sh @@ -22,6 +22,9 @@ DRY_RUN=0 DESTROY_ONLY=0 INSTANCE_PRINCIPAL=0 CREDENTIAL_HOOK="${OCI_CREDENTIAL_HOOK:-}" +# Empty means oci_env.py chooses, which is $OCI_CLI_PROFILE, then the reference +# tenancy, then DEFAULT. Named here so a deployment can say which tenancy it is in. +OCI_PROFILE="${OCI_PROFILE:-}" DISTRO="" SETUP_SH="" # Empty means the driver's own default list. Unset is distinguishable from empty, @@ -67,6 +70,10 @@ workbook against it, then destroys everything. --credential-hook PATH Executable run after formation, before the pool, as "hook ...". Where an API-key deployment installs its credential. Default \$OCI_CREDENTIAL_HOOK. + --oci-profile NAME Profile in ~/.oci/config that Terraform builds with. + Default \$OCI_PROFILE, else \$OCI_CLI_PROFILE, else the + reference tenancy, else DEFAULT. A name that is passed + and absent is an error, never a fallback. --workbooks LIST Space-separated workbooks for the published driver to run on the cluster. Empty means run none; omitted means the driver's own default list. @@ -101,6 +108,7 @@ while [ $# -gt 0 ]; do --setup-sh) SETUP_SH="${2:?}"; shift 2 ;; --instance-principal) INSTANCE_PRINCIPAL=1; shift ;; --credential-hook) CREDENTIAL_HOOK="${2:?}"; shift 2 ;; + --oci-profile) OCI_PROFILE="${2:?}"; shift 2 ;; --no-external-pool) SKIP_POOL=1; SKIP_WORKLOAD=1; shift ;; --skip-workload) SKIP_WORKLOAD=1; shift ;; --keep) KEEP=1; shift ;; @@ -146,7 +154,8 @@ WORKBOOKS_TSV="$STATE_DIR/workbooks.tsv" # A separate state directory per topology, so two topologies can be built from one # checkout without one destroying the other's instances. tf() { - python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" -- \ + python3 "$HERE/scripts/oci_env.py" --ssh-public-key-path "$SSH_PUBLIC_KEY" \ + ${OCI_PROFILE:+--profile "$OCI_PROFILE"} -- \ terraform -chdir="$HERE" "$@" } @@ -170,6 +179,22 @@ ssh_node() { } # record [detail] +# The node's own account of a failed run. A workbook's log says which API call +# failed; only the journal says why, and the nodes are destroyed minutes later. +# Warnings first because that is where a swallowed error surfaces, then a bounded +# tail for context -- unbounded, a multi-hour suite's journal dwarfs the artifact. +capture_journals() { + local host + for host in "${HOSTS[@]}"; do + ssh_node "$host" ' + echo "=== spinifex, warning and above ===" + sudo journalctl -u "spinifex-*" --since -4h --priority=warning --no-pager | tail -2000 + echo "=== spinifex, all priorities, last 2000 lines ===" + sudo journalctl -u "spinifex-*" --since -4h --no-pager | tail -2000 + ' > "$STATE_DIR/journal-$host.log" 2>&1 || log "could not collect the journal from $host" + done +} + record() { local gate="$1" status="$2" detail="${3-}" printf '%s: %s%s\n' "$gate" "$status" "${detail:+ $detail}" >> "$RESULTS" @@ -246,6 +271,14 @@ fi : > "$RESULTS_TSV" : > "$WORKBOOKS_TSV" +# Last run's logs go before this one's first line, so what is left in here always +# describes the run that is starting. Without this a topology keeps logs that read +# as current, and anything collecting the directory publishes them as this run's. +# Named globs rather than a find: the tfstate is how the sweep finds hosts to +# destroy, so what is removed here has to be readable at a glance. +# :? so an unset STATE_DIR stops the shell rather than expanding to /*.log. +rm -f "${STATE_DIR:?}"/*.log "${STATE_DIR:?}"/nodes.txt "${STATE_DIR:?}"/cloudinit-*.txt + log "$SHAPE, $NODES node(s), state in $STATE_DIR" # Checked here, not where the hook is run: nothing about it depends on the apply, and @@ -561,19 +594,27 @@ scp -i "$SSH_PRIVATE_KEY" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev || die "could not copy the workbooks" # Every AMI the workbooks need, imported one at a time so predastore is not asked -# to absorb parallel uploads. rds-quickstart is the one that needs an appliance. +# to absorb parallel uploads. Three are appliances rather than distros: RDS, ECS +# and EKS each boot their own, found by a spinifex:managed-by tag and never by name. log "importing the images the workbooks need" ssh_node "${HOSTS[0]}" ' set -e - for img in ubuntu-26.04-x86_64 spinifex-rds-postgres; do + for img in ubuntu-26.04-x86_64 spinifex-rds-postgres spinifex-ecs-node spinifex-eks-node; do sudo spx admin images import --name "$img" --config /etc/spinifex/spinifex.toml >/dev/null done ' > "$STATE_DIR/images.log" 2>&1 || die "image import failed; see $STATE_DIR/images.log" -# WORKBOOKS unset leaves the driver on its own default list, which is the list the -# nightly judges every other platform by. -workbook_env="WORKBOOK_DIR=\$HOME/workbooks" -[ "$WORKBOOKS_SET" = 1 ] && workbook_env="$workbook_env $(printf 'WORKBOOKS=%q' "$WORKBOOKS")" +# Every workbook that is a workbook. The shared driver's own default is five, which +# leaves ECS and all three EKS variants untested on every platform -- their +# assertions exist and nothing was running them. +# +# demo-app is absent because it is not a workbook: it has no .tf at all, being the +# container image the EKS workbooks' nested workloads/ modules deploy. Listing it +# here would fail on a missing root module rather than test anything. +OCI_WORKBOOKS="nginx-alb bastion-private-subnet nginx-webserver s3-webapp rds-quickstart" +OCI_WORKBOOKS="$OCI_WORKBOOKS ecs-quickstart eks-quickstart eks-https-ingress eks-gitops-argocd" +[ "$WORKBOOKS_SET" = 1 ] || WORKBOOKS="$OCI_WORKBOOKS" +workbook_env="WORKBOOK_DIR=\$HOME/workbooks $(printf 'WORKBOOKS=%q' "$WORKBOOKS")" # Whether a node can reach a public address inside its own VCN. Informational: the # remedy below is right either way, because a customer reaches a guest from outside. @@ -627,6 +668,7 @@ else record_workbooks "$STATE_DIR/workbooks.log" failed="$(grep -c '^--- FAIL' "$STATE_DIR/workbooks.log" || true)" record workbooks FAIL "$failed failed, see $STATE_DIR/workbooks.log" + capture_journals tail -60 "$STATE_DIR/workbooks.log" grep -E '^--- (PASS|FAIL)' "$STATE_DIR/workbooks.log" | sed 's/^/ /' || true die "the published workbooks failed on $TOPOLOGY; see $STATE_DIR/workbooks.log"