From ba12b60baa9fa5fb031ca0c014e6c5c814ca4943 Mon Sep 17 00:00:00 2001 From: Paul Lorenz Date: Wed, 12 Aug 2026 11:14:56 -0400 Subject: [PATCH] Add in-place upgrade smoke test. Fixes #4199 - adds a fablab model (zititest/models/upgrade-test) that stands up a multi-region HA controller cluster, routers, and loop4 sim traffic clients (SDK, ERT, ZET, tunneler) and drives repeatable in-place upgrade/downgrade iterations, re-bootstrapping in place each iteration - exercises HA cluster disruption and recovery (snapshot restore, node rejoin) between iterations and validates steady-state traffic and expected terminators after each disruption - honors the raft restartSelf setting on migration-snapshot restore, so a restored controller restarts itself instead of exiting, and promotes RestartController to a package-level function - re-sends loop4 run-scenario requests to sims that reconnect mid-scenario, tracking each scenario's expected client set explicitly and ignoring results from clients outside it, so post-disruption client reconnection no longer stalls validation - closes leaked loop4 sim-control connections on error and on supersession - rotates component logs and pins the log-pipe binary to the local build, and drops router debug logging, to keep long iterating runs disk-bounded - excludes the doc/unsettled/ scratch area for in-progress design docs from version control --- .github/workflows/upgrade-test.yml | 151 +++++ .gitignore | 1 + controller/controller.go | 1 + controller/network/network.go | 22 +- controller/raft/fsm.go | 4 +- zititest/go.mod | 2 + zititest/go.sum | 2 + zititest/models/upgrade-test/README.md | 275 ++++++++ zititest/models/upgrade-test/bootstrap.go | 126 ++++ .../models/upgrade-test/clusterupgrade.go | 316 +++++++++ .../models/upgrade-test/configs/ctrl.yml.tmpl | 113 ++++ .../configs/loop4-backend-host.yml.tmpl | 11 + .../configs/loop4-client.yml.tmpl | 42 ++ .../configs/loop4-sdk-host.yml.tmpl | 13 + .../configs/loop4-transport-dialer.yml.tmpl | 30 + .../upgrade-test/configs/router.yml.tmpl | 78 +++ zititest/models/upgrade-test/ctrlmem.go | 435 ++++++++++++ zititest/models/upgrade-test/ctrlmem_test.go | 104 +++ zititest/models/upgrade-test/ha.go | 195 ++++++ zititest/models/upgrade-test/main.go | 640 ++++++++++++++++++ zititest/models/upgrade-test/oidc.go | 219 ++++++ zititest/models/upgrade-test/start.go | 59 ++ zititest/models/upgrade-test/steadystate.go | 389 +++++++++++ zititest/models/upgrade-test/upgrade.go | 217 ++++++ zititest/ziti-traffic-test/loop4/dialer.go | 15 + .../loop4/remoteControlled.go | 219 +++++- .../loop4/remoteController.go | 162 ++++- zititest/zitilab/component_controller.go | 11 +- zititest/zitilab/component_loop4_sim.go | 13 +- zititest/zitilab/component_router.go | 9 +- .../zitilab/component_ziti_edge_tunnel.go | 6 +- zititest/zitilab/component_ziti_tunnel.go | 21 +- .../runlevel/5_operation/client_metrics.go | 52 ++ zititest/zitilab/stageziti/stageziti.go | 79 ++- zititest/zitilab/validations/terminators.go | 105 ++- 35 files changed, 4070 insertions(+), 67 deletions(-) create mode 100644 .github/workflows/upgrade-test.yml create mode 100644 zititest/models/upgrade-test/README.md create mode 100644 zititest/models/upgrade-test/bootstrap.go create mode 100644 zititest/models/upgrade-test/clusterupgrade.go create mode 100644 zititest/models/upgrade-test/configs/ctrl.yml.tmpl create mode 100644 zititest/models/upgrade-test/configs/loop4-backend-host.yml.tmpl create mode 100644 zititest/models/upgrade-test/configs/loop4-client.yml.tmpl create mode 100644 zititest/models/upgrade-test/configs/loop4-sdk-host.yml.tmpl create mode 100644 zititest/models/upgrade-test/configs/loop4-transport-dialer.yml.tmpl create mode 100644 zititest/models/upgrade-test/configs/router.yml.tmpl create mode 100644 zititest/models/upgrade-test/ctrlmem.go create mode 100644 zititest/models/upgrade-test/ctrlmem_test.go create mode 100644 zititest/models/upgrade-test/ha.go create mode 100644 zititest/models/upgrade-test/main.go create mode 100644 zititest/models/upgrade-test/oidc.go create mode 100644 zititest/models/upgrade-test/start.go create mode 100644 zititest/models/upgrade-test/steadystate.go create mode 100644 zititest/models/upgrade-test/upgrade.go diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml new file mode 100644 index 000000000..287b6f4f3 --- /dev/null +++ b/.github/workflows/upgrade-test.yml @@ -0,0 +1,151 @@ +name: Upgrade Test + +on: + workflow_dispatch: + inputs: + zetVersion: + description: 'ziti-edge-tunnel version for every ZET client and host (blank uses the model default)' + required: false + default: '' + fromVersion: + description: 'version the system starts on and is upgraded from (blank uses the model default)' + required: false + default: '' + toVersion: + description: 'version to upgrade to, and the version a ref build is stamped with (blank uses the model default)' + required: false + default: '' + toVersionRef: + description: 'git ref on openziti/ziti to build toVersion from; blank uses the model default' + required: false + default: '' + nextVersion: + description: 'second upgrade hop, applied to the whole cluster after toVersion; blank uses the model default, "none" skips the phase' + required: false + default: '' + nextVersionRef: + description: 'git ref on openziti/ziti to build nextVersion from; blank uses the model default' + required: false + default: '' + clusterUpgradeMode: + description: 'how the cluster moves to nextVersion' + required: false + type: choice + options: + - '' + - rolling + - all-at-once + default: '' + +# one upgrade test at a time: each run provisions a multi-region AWS topology +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +env: + GOFLAGS: "-trimpath" + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_DEFAULT_REGION: "us-east-2" + gh_ci_key: ${{ secrets.GH_CI_KEY }} + BUILD_NUMBER: ${{ format('{0}-{1}-{2}', github.run_id, github.run_number, github.run_attempt) }} + +jobs: + upgrade-test: + name: Fablab Upgrade Test + if: github.repository_owner == 'openziti' + runs-on: ubuntu-24.04 + timeout-minutes: 180 + + steps: + - name: Git Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 + + - name: Install Go + uses: ./.github/actions/setup-go + + - name: Install Ziti CI + uses: openziti/ziti-ci@v1 + + - name: Install Terraform CLI + uses: hashicorp/setup-terraform@v4 + with: + terraform_version: ~1.5 + + - name: Build + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ziti_ci_gpg_key: ${{ secrets.ZITI_CI_GPG_KEY }} + ziti_ci_gpg_key_id: ${{ secrets.ZITI_CI_GPG_KEY_ID }} + shell: bash + run: | + $(go env GOPATH)/bin/ziti-ci configure-git + $(go env GOPATH)/bin/ziti-ci generate-build-info common/version/info_generated.go version + pushd zititest && go mod tidy && go install -tags all ./... && popd + go install -tags=all,tests ./... + + - name: Create Test Environment + env: + # inputs reach the script through the environment, never interpolated into it + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + ZET_VERSION: ${{ inputs.zetVersion }} + FROM_VERSION: ${{ inputs.fromVersion }} + TO_VERSION: ${{ inputs.toVersion }} + TO_VERSION_REF: ${{ inputs.toVersionRef }} + NEXT_VERSION: ${{ inputs.nextVersion }} + NEXT_VERSION_REF: ${{ inputs.nextVersionRef }} + CLUSTER_UPGRADE_MODE: ${{ inputs.clusterUpgradeMode }} + shell: bash + run: | + echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" + + # pass only what was supplied, so the model's own defaults stay authoritative + LABELS="environment=gh-upgrade-test" + if [ -n "$ZET_VERSION" ]; then LABELS="$LABELS,zetVersion=$ZET_VERSION"; fi + if [ -n "$FROM_VERSION" ]; then LABELS="$LABELS,fromVersion=$FROM_VERSION"; fi + if [ -n "$TO_VERSION" ]; then LABELS="$LABELS,toVersion=$TO_VERSION"; fi + if [ -n "$TO_VERSION_REF" ]; then LABELS="$LABELS,toVersionRef=$TO_VERSION_REF"; fi + # "none" is how a run switches the optional second hop off, since a blank input means + # "use the model default" and the model defaults it on + if [ "$NEXT_VERSION" = "none" ]; then + LABELS="$LABELS,nextVersion=" + elif [ -n "$NEXT_VERSION" ]; then + LABELS="$LABELS,nextVersion=$NEXT_VERSION" + fi + if [ -n "$NEXT_VERSION_REF" ]; then LABELS="$LABELS,nextVersionRef=$NEXT_VERSION_REF"; fi + if [ -n "$CLUSTER_UPGRADE_MODE" ]; then LABELS="$LABELS,clusterUpgradeMode=$CLUSTER_UPGRADE_MODE"; fi + echo "using labels: $LABELS" + + INSTANCE="upgrade-test-${GITHUB_RUN_NUMBER}" + echo "INSTANCE=${INSTANCE}" >> "$GITHUB_ENV" + $(go env GOPATH)/bin/upgrade-test create -d "${INSTANCE}" -n "${INSTANCE}" -l "$LABELS" + $(go env GOPATH)/bin/upgrade-test up + + - name: Run Upgrade Iteration + shell: bash + run: | + $(go env GOPATH)/bin/upgrade-test exec testIteration + + - name: Create Logs Archive + if: always() + shell: bash + run: | + $(go env GOPATH)/bin/upgrade-test get files '*' "./logs/{{ .Id }}/" ./logs + + - name: Upload artifacts + uses: actions/upload-artifact@v7 + if: always() + with: + name: logs-upgrade-test-${{ github.run_id }} + path: logs/ + compression-level: 7 + retention-days: 5 + + - name: Tear down Test Environment + timeout-minutes: 30 + if: always() + shell: bash + run: | + $(go env GOPATH)/bin/upgrade-test dispose diff --git a/.gitignore b/.gitignore index b045ea0bf..65e073ec7 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,7 @@ go.work.sum .github/workflows/localtest.yml doc/ha/pki doc/ha/data +doc/unsettled/ simple-transfer-*/ etc/endpoints etc/endpoints.yml diff --git a/controller/controller.go b/controller/controller.go index d0663bbbe..e767a130e 100644 --- a/controller/controller.go +++ b/controller/controller.go @@ -301,6 +301,7 @@ func NewController(cfg *config.Config, versionProvider versions.VersionProvider) if n, err := network.NewNetwork(c, appEnv); err == nil { c.network = n + n.SetRestartSelfOnSnapshot(c.config.Raft != nil && c.config.Raft.RestartSelf) } else { return nil, err } diff --git a/controller/network/network.go b/controller/network/network.go index 88459a659..7d789376b 100644 --- a/controller/network/network.go +++ b/controller/network/network.go @@ -55,6 +55,7 @@ import ( "github.com/openziti/ziti/v2/controller/event" "github.com/openziti/ziti/v2/controller/idgen" "github.com/openziti/ziti/v2/controller/model" + "github.com/openziti/ziti/v2/controller/raft" "github.com/openziti/ziti/v2/controller/storage/boltz" "github.com/openziti/ziti/v2/controller/storage/objectz" "github.com/openziti/ziti/v2/controller/xt" @@ -114,6 +115,10 @@ type Network struct { config Config + // restartSelfOnSnapshot, when true, restarts this controller in place after a snapshot restore + // (e.g. a node joining the cluster) instead of exiting and relying on an external process manager. + restartSelfOnSnapshot bool + Inspections *InspectionsManager RouterMessaging *RouterMessaging inspectionTargets concurrenz.CopyOnWriteSlice[InspectTarget] @@ -1494,7 +1499,15 @@ func (network *Network) RestoreSnapshot(cmd *command.SyncSnapshotCommand, index } time.AfterFunc(5*time.Second, func() { - log.Info("database restore requires controller restart. exiting...") + if network.restartSelfOnSnapshot { + log.Info("database restore requires controller restart, restarting...") + // RestartController returns only when the restart failed; on success it replaces this + // process and never comes back. + err := raft.RestartController() + log.WithError(err).Error("failed to restart controller after snapshot restore, exiting...") + } else { + log.Info("database restore requires controller restart. exiting...") + } os.Exit(0) }) @@ -1524,6 +1537,13 @@ func (network *Network) ensureRaftIndex(index uint64) error { }) } +// SetRestartSelfOnSnapshot controls whether the controller restarts itself after a snapshot restore +// (true) or exits expecting an external restart (false). It mirrors the raft restartSelfOnSnapshot +// setting and is applied by the owning controller after construction. +func (network *Network) SetRestartSelfOnSnapshot(v bool) { + network.restartSelfOnSnapshot = v +} + func (network *Network) AddInspectTarget(target InspectTarget) { network.inspectionTargets.Append(target) } diff --git a/controller/raft/fsm.go b/controller/raft/fsm.go index cff98aed0..76a44a242 100644 --- a/controller/raft/fsm.go +++ b/controller/raft/fsm.go @@ -443,7 +443,7 @@ func (self *BoltDbFsm) Restore(snapshot io.ReadCloser) error { time.AfterFunc(5*time.Second, func() { if self.restartSelf { log.Info("restored snapshot to initialized system, restart required, restarting now") - if err = self.RestartController(); err != nil { + if err = RestartController(); err != nil { log.WithError(err).Error("failed to restart controller, exiting now") os.Exit(0) } @@ -464,7 +464,7 @@ func (self *BoltDbFsm) Restore(snapshot io.ReadCloser) error { // RestartController starts a new controller process with the same parameters and exits the current process. // This is useful when the controller needs to be restarted after applying a snapshot or other configuration changes. -func (self *BoltDbFsm) RestartController() error { +func RestartController() error { log := pfxlog.Logger() // Get the current executable path diff --git a/zititest/go.mod b/zititest/go.mod index 30f0f4532..56b3b0d2c 100644 --- a/zititest/go.mod +++ b/zititest/go.mod @@ -23,6 +23,7 @@ require ( github.com/openziti/foundation/v2 v2.0.100 github.com/openziti/identity v1.0.140 github.com/openziti/metrics v1.4.5 + github.com/openziti/sdk-golang/acquire v0.3.0 github.com/openziti/sdk-golang/v2 v2.0.0-pre4 github.com/openziti/transport/v2 v2.0.220 github.com/openziti/ziti/v2 v2.0.3 @@ -236,6 +237,7 @@ require ( go4.org v0.0.0-20260112195520-a5071408f32f // indirect golang.org/x/crypto v0.55.0 // indirect golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect + golang.org/x/mod v0.39.0 // indirect golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect diff --git a/zititest/go.sum b/zititest/go.sum index f87ce7cb6..b0e570461 100644 --- a/zititest/go.sum +++ b/zititest/go.sum @@ -619,6 +619,8 @@ github.com/openziti/metrics v1.4.5 h1:p51HYSQyqaDafizPWluhYjU/rVfY52snER8/BHpfPr github.com/openziti/metrics v1.4.5/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg= github.com/openziti/runzmd v1.0.92 h1:dnv+luFxYrv3DY1FPfLsFO9/d21PijhLiAHehJHGPyA= github.com/openziti/runzmd v1.0.92/go.mod h1:818zJed1POFnZPR2DndES9/7VS3RMByKHH2/MEw2KfA= +github.com/openziti/sdk-golang/acquire v0.3.0 h1:wNi3JaXUoSGxissurb9SUObLOtkanNQNJiZXro6Wup0= +github.com/openziti/sdk-golang/acquire v0.3.0/go.mod h1:fM5a2tGvDdeUxiq3NT2vU4qCSmWniIItSEQV4O6vJuo= github.com/openziti/sdk-golang/v2 v2.0.0-pre4 h1:/3a0/mGYczTRFH+h4NYn2d+xWJl8xVCHU1g2XJcAMng= github.com/openziti/sdk-golang/v2 v2.0.0-pre4/go.mod h1:WYPHgevyOIHFrRLpeftzBAYOvd9YAJOxSNW3g+ORzIg= github.com/openziti/secretstream v0.1.52 h1:UL8bshHGpXRLt7wjIeyvySXsmoBNoRsumyTsAh+O8ns= diff --git a/zititest/models/upgrade-test/README.md b/zititest/models/upgrade-test/README.md new file mode 100644 index 000000000..034d80e99 --- /dev/null +++ b/zititest/models/upgrade-test/README.md @@ -0,0 +1,275 @@ +# upgrade-test model + +## Goal + +Stand up a full smoke-style OpenZiti topology on a parameterized "from" version using legacy +authentication, confirm traffic is flowing, then walk the system through a sequence of in-place +upgrades while asserting that traffic keeps flowing throughout: + +1. Start on the "from" version (default: latest 1.6.x) with a single standalone controller, legacy auth. +2. Upgrade the controller to the "to" version (default: v2.0.3). This auto-enables OIDC. +3. Upgrade the routers to the "to" version. +4. Convert the controller to HA (single-node cluster). +5. Add two more controller nodes to form a 3-node cluster. +6. Optionally upgrade the whole cluster again, to the "next" version (default: 2.1 built from `main`). + +The ZET (ziti-edge-tunnel), ERT (edge-router-tunneler), and ziti tunnel clients and hosts stay up +across the entire sequence. The whole run is repeatable in a loop so we can flush out timing errors +and race conditions. + +## Topology + +All AWS hosts are provisioned up front. Adding AWS instances mid-run is avoided; the two extra +controllers simply sit idle until the "add nodes" phase. + +Controllers: +- `ctrl1` - runs from the start. +- `ctrl2`, `ctrl3` - provisioned but not started until the add-nodes phase. + +Routers (same as smoke): +- `router-east-1` - ERT client. +- `router-east-2` - initiator. +- `router-west` - ERT host. + +Clients and hosts that must stay up across the whole sequence, each present as a **pair** so that +one instance can be restarted while the other keeps serving (see OIDC verification). Each pair is named +`-stable` and `-restart`, and carries a `stable` or `restart` tag so the restart and +PID-persistence actions can select the whole set by role: +- `loop4-client-stable` / `-restart` - Go SDK sim dialer. +- `loop4-sdk-host-stable` / `-restart` - Go SDK sim listener (binds the `loop-sdk` service). +- `ziti-edge-tunnel-client-stable` / `-restart` and `ziti-edge-tunnel-host-stable` / `-restart` - ZET, the C SDK tunneler. +- `ziti-tunnel-client-stable` / `-restart` and `ziti-tunnel-host-stable` / `-restart` - Go SDK tunneler. + +The two instances of each pair co-locate on the same host as each other, so pairing adds components +but no AWS instances. Process filters keep co-located instances individually controllable: ZET by its +config id, ziti-tunnel by `--cli-agent-alias`, and loop4 once its process filter includes the config id +(a small, backward-compatible change to `Loop4SimType`). The paired hosts binding the same service +simply create two terminators, so restarting one leaves the other serving. + +Support components (single, not paired): plain-TCP loop4 backends that the tunneler hosts forward their +loop service to (see Liveness). `loop4-ert-backend` sits on the ERT host; the ZET and ziti-tunnel hosts +share a box and a single `loop4-tunnel-backend`. These have no ziti identity, so they are not part of the +OIDC story. + +## Version parameterization + +Both ends are fully parameterized (version X -> version Y), defaulting to latest 1.6.x -> v2.0.3. + +- `fromVersion` / `toVersion` model variables drive the controller and router binaries. +- `zetVersion` for ZET, covering both its client and host instances. ZET is not upgraded mid-run; it + stays up the whole time. Parameterizing it lets us sweep ZET versions across separate runs. Default is + the version shipped with the current stable Windows desktop edge client: `v1.18.0` (bundled in Ziti + Desktop Edge for Windows release 2.11.2.5, the latest stable release; note 2.11.2.7 bundles v1.18.2 + but is a pre-release). +- `nextVersion` is an optional second hop: once the 3-node cluster is up on `toVersion`, the whole + cluster is upgraded again. `nextVersionSource` / `nextVersionRef` work exactly like their `toVersion` + counterparts. Setting `nextVersion` to the empty string skips the phase and its two gates entirely. +- The existing per-component `_version` label knob still works for one-off overrides. + +Both the "from" and "to" binaries are pre-staged on every host during distribution. Because binaries +are versioned and coexist on-host (`ziti-`, `ziti-edge-tunnel-`), every upgrade is +just `SetVersion(newVersion)` followed by a restart, and a downgrade (for reset) is the same in reverse. +No mid-run binary rsync is needed. `ControllerType.Start` resolves the binary from the component's +current `Version` at start time, so a version swap plus restart is sufficient. + +The Go SDK clients (`ziti-tunnel-*`, `loop4-*`) start on `fromVersion` and stay there. That is +deliberately the interesting compatibility case for the OIDC question below. + +## Config strategy + +One controller config works for both binaries. It has no explicit OIDC section, so the 2.0 binary +auto-binds the `edge-oidc` API on its first restart (`ensureOidcOnClientApiServer` in +`controller/controller.go`). That auto-bind is the "upgrade forces OIDC to become available" event, and +it happens with no config change. + +For the HA conversion, the controller config is re-rendered to add the `cluster:` directive and rsynced +to the host before restart. The standalone binary then auto-migrates the bbolt DB into raft and comes up +as a single-node HA cluster. + +Router controller-endpoint handling after the HA conversion (whether routers need all three controller +addresses in config or auto-learn them from the cluster) is verified during implementation. + +## Liveness + +We adopt the `circuit-test` harness: `SimServices` plus remote-controlled loop4 clients with +`iterations: -1` (run forever, auto-reconnect after restarts), streaming success/failure metrics every +five seconds. + +To drive traffic through the ZET and ziti-tunnel hosting paths (not just the pure-SDK path), we define +services whose terminators are the ZET host and the ziti-tunnel host, each backed by a plain-TCP loop4 +listener behind the tunnel: + +``` +loop4 dialer (SDK) -> ziti service -> ZET / ziti-tunnel host -> TCP -> loop4 listener +``` + +The circuit-test model already uses loop4 this way, so this is a proven pattern. We also keep a +pure-SDK-hosted service and an ERT-hosted service, so the sim continuously exercises all four hosting +flavors. + +Quiescent-vs-churn validation (no failures when idle, tolerated during restarts). Rather than correlate +raw 5-second metric samples with churn windows, validation is expressed in terms of the sim's discrete +scenario runs (the remote-controlled pattern), which is far less flake-prone. After each disruptive step: + +- Recovery gate: require N consecutive clean scenario runs (zero errors) achieved within a recovery + timeout (default ~1 minute). This tolerates the transient failures expected while a controller or + router restarts. +- Stability gate: once recovered, require scenario runs to continue clean for a stability window + (default ~2 minutes) before advancing to the next phase. +- N, the recovery timeout, and the stability window are all tunable knobs. + +The baseline (phase 1) and the final validation (phase 6) use the same clean-runs gate with no preceding +disruption. + +## Phase sequence + +The main orchestration action runs these phases in order. + +0. Bootstrap on `fromVersion`: standalone `ctrl1`, routers, all clients and hosts enrolled; start; wait + ready. +1. Baseline: start the continuous sim; assert clean steady-state. Traffic is flowing on legacy auth. +2. Upgrade controller to `toVersion`: stop `ctrl1`, `SetVersion`, restart (same standalone config, so + OIDC auto-binds). Assert recovery and clean steady-state, then run the restart-one-of-each OIDC + verification (see below). +3. Rolling router upgrade to `toVersion`: one router at a time (stop, `SetVersion`, start, wait) so + traffic keeps flowing through the others. Assert recovery after each, clean steady-state after all. +4. Convert controller to HA (single node): stop `ctrl1`, swap in the cluster-directive config, restart + so it auto-migrates to raft. Assert recovery and clean steady-state, then re-run the restart-one-of-each + verification. +5. Add two nodes: start `ctrl2` and `ctrl3`, run `ziti agent cluster add` for each, wait for a stable + three-voter quorum, update router (and possibly SDK client) controller lists if needed. Assert + recovery and clean steady-state, then re-run the restart-one-of-each verification. +6. Final validation: full clean steady-state. +7. Optional cluster upgrade to `nextVersion`: upgrade all three controllers, assert clean steady-state, + roll the routers, assert clean steady-state again. Skipped when `nextVersion` is empty. + + Not implemented: asserting that the *un-restarted* instance of each pair kept its original PID across + phases 1-5, which would prove the long-lived clients truly stayed up rather than having been quietly + restarted. The steady-state gate checks they are healthy, not that they are the same processes. + +## OIDC verification via client restart + +We verify OIDC by observing real clients rather than a synthetic probe. This is the only way to cover +ZET (the C SDK), and it exercises the exact auth path production clients use. + +The mechanism relies on two facts: + +- A long-lived Go SDK context caches the controller's advertised capabilities via `sync.Once` and never + re-reads them on reconnect (`edge-apis/client_edge_client.go`). So a client that stays up across the + 1.6 -> 2.0 upgrade keeps using its legacy session; only a freshly started client re-reads capabilities + and can switch to OIDC. +- The controller emits an `apiSession` event (`controller/event/api_session.go`) whose `type` field is + `legacy` or `jwt` (jwt is the token/OIDC session), keyed by `identity_id` and `ip_address`. We enable + this event with a file handler in `ctrl.yml.tmpl`, exactly like the entityChange/circuit/link/router + events already there. (The `authentication` event's `type` is the credential kind, `cert`/`updb`/`ext-jwt`, + not the session mechanism, so it does not distinguish legacy from OIDC; we use `apiSession`.) + +Because every client and host is a pair, after the controller upgrade we restart the `-restart` instance +of each pair, leaving the `-stable` instance up: + +- The `-stable` instance proves the client survives the upgrade on its existing legacy session, with no + re-auth and no traffic gap (its PID is checked for continuity in the final phase). +- The `-restart` instance is a fresh process, so it re-authenticates. We read the new `apiSession` + `created` event for that instance (matched by identity or source IP) and check its `type`. + +Expected results: freshly started Go SDK clients (loop4, ziti-tunnel) default to dynamic OIDC detection, +so they should come back as `jwt`. ZET (C SDK, `v1.18.0`) auto-switch behavior is genuinely uncertain; if +a restarted ZET comes back `legacy`, that is a real finding, and the `apiSession` event makes it visible +either way. So ZET's expected value is TBD by design, the test documents actual behavior. + +This replaces the earlier synthetic edge_apis probe and the standalone capability assertion: a restarted +client returning `jwt` already implies the controller advertises OIDC. + +Phases 4-5 (HA and multi-controller) are the predicted breaking point for the long-lived legacy siblings. +"They keep working" is the pass criterion. If a later phase breaks them, that triggers an in-scope +sdk-golang fix to make long-lived contexts re-detect OIDC / handle the cluster on reconnect. The test is +designed to surface exactly that. + +## Cluster upgrade and controller memory + +The optional second hop upgrades an established 3-node cluster rather than building one, which is the +scenario production operators actually run and the one [#4219](https://github.com/openziti/ziti/issues/4219) +reports a controller OOM in. `clusterUpgradeMode` picks how it happens: + +- `rolling` (default) restarts one node at a time, followers first and the leader last. Peers hold the + cluster in mixed-version read-only mode until the last node is done, which is the state #4219 blames. +- `all-at-once` restarts every node together, so no node ever sees a version mismatch. This is the + workaround the issue asks about, and it is also the control case: every node still cold-starts, so if + memory climbs here too, the mismatch is not the mechanism. + +No steady-state gate runs between nodes in rolling mode. The cluster is read-only while versions differ, +so anything needing a write would fail for reasons the phase is not testing. The gate runs once the whole +cluster is on `nextVersion`. + +Every controller host samples its controller's RSS once a second for the whole iteration, writing +`~/logs/-mem.csv`. The controller is located by its agent alias, the same discriminator fablab's +process filter uses, so sampling follows the process across an upgrade and records a zero while it is +down: a node in a crash loop shows up as a sawtooth rather than a gap. Peaks are reported per node after +the cluster upgrade and again at the end of the iteration. + +`ctrlMemory.heapDumpAtMb` captures a heap profile (`~/logs/-mem-.pprof`) the first time RSS +crosses it. That profile is the thing #4219 is missing and the one artifact that cannot be recovered after +the fact. The samples and profiles are under `~/logs`, so CI collects them with the rest of the logs. + +Two checks can fail the cluster upgrade, both disabled by setting them to zero: + +- `ctrlMemory.failAtRatio` (default 3) compares each controller's peak over the upgrade against its own + peak over the two minutes of settled traffic immediately before it. This is the one that bites at this + scale, and it is the shape #4219 reports: roughly 5x a 150-190 MiB baseline. The default is a first + guess, since a cold start legitimately overshoots a warm steady state; calibrate it against a run known + to be good. +- `ctrlMemory.failAtMb` (default 1024) is the absolute ceiling, sized for the reported failure. It is a + backstop, and is unlikely to be reached without a much larger data set. + +`fablab exec ctrlMemorySummary` prints one row per controller per iteration for the whole run, plus the +highest peak seen and where, so a finished run can be read back with one command rather than scrolled for. +`fablab exec reportCtrlMemory` is the narrower form, covering only the current iteration. + +Nothing is discarded between iterations. Samples accumulate in one file behind a `# start,` marker +that scopes each report to the current iteration, and the profile path carries the sampler's start time, +so iteration 4 cannot delete the profile iteration 3 captured. At a line a second the samples cost roughly +1.7 MB per controller per day. + +Scale caveat: this model runs a handful of routers, identities and services, so cold-start cost here is a +fraction of a production controller's. Expect the *shape* to be informative (does the upgraded node +diverge from its warm peers, and does it diverge in `all-at-once` too) rather than the magnitude. Chasing +the reported 1 GiB number would mean seeding a much larger data set first. + +## Iteration and reset + +A `testIteration` action runs `{reset -> full upgrade sequence}` once, and `fablab exec-loop` repeats it +for as long as asked. Rather than hand-rolling a surgical state wipe, reset leans on the existing fablab +machinery: wipe everything (controller DB and raft state, PKI/enrollment, cached tokens) and re-run +distribution and bootstrap from the `fromVersion` standalone configuration. This is more trustworthy +than trying to selectively clear state, and an incomplete wipe would produce cross-iteration +contamination that looks exactly like a race. + +Each iteration is fully independent, which is what flushes timing and race issues. SDK clients restart +per iteration because the wipe invalidates identities, but they stay up across all phases within an +iteration, which is what matters for catching mid-upgrade problems. A full downgrade-in-place without a +wipe is not possible, since the 2.0 DB schema cannot be read by the 1.6 binary. + +A bare `fablab exec testIteration` runs a single bounded pass; `fablab exec-loop testIteration` +is the opt-in form for long local soak runs and a time-boxed slot in the validation suite. + +## Longer-term direction + +The version pair is a parameter, but the model is currently shaped around one specific transition: +1.6.x -> 2.0.x. The goal is to run LTS -> current instead, so each release is exercised against the +version most deployments are actually upgrading from, and the pair moves as LTS moves. + +Two things stand in the way, both of which encode "this is the 1.6 -> 2.0 upgrade" rather than "this is +an upgrade": + +- The transition-specific workarounds. `restartZetWorkaround` restarts ziti-edge-tunnel because versions + at or below `zetRestartWorkaroundMaxVersion` cannot rebuild their edge sessions after the 2.0 JWT + session migration, and `reconcileStaleTerminators` runs only while `anyPreV2Router` holds, cleaning up + terminators that pre-2.0 routers drop without telling the controller. Both are gated on version so + they switch themselves off, but a different pair needs its own set, and there is no structure yet for + saying which workarounds belong to which transition. +- The phase sequence itself. The standalone -> HA conversion and the cluster-node join are steps in the + 1.6 -> 2.0 story specifically. An LTS -> current run where both ends are already HA would want a + different sequence, so the phases need to become selectable rather than a fixed list. + +Neither is large on its own, but together they are why the version defaults are not simply pointed at a +moving LTS target today. diff --git a/zititest/models/upgrade-test/bootstrap.go b/zititest/models/upgrade-test/bootstrap.go new file mode 100644 index 000000000..4c3861d47 --- /dev/null +++ b/zititest/models/upgrade-test/bootstrap.go @@ -0,0 +1,126 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "time" + + "github.com/openziti/fablab/kernel/lib/actions" + "github.com/openziti/fablab/kernel/lib/actions/component" + "github.com/openziti/fablab/kernel/lib/actions/host" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab" + zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/models" +) + +type bootstrapAction struct{} + +// newBootstrapAction builds the phase-1 bootstrap: a single standalone controller on the +// from-version, enrolled routers and identities, and the loop services that the sim dials +// across all four hosting flavors (SDK, ERT, ZET, ziti-tunnel). +func newBootstrapAction() model.ActionBinder { + action := &bootstrapAction{} + return action.bind +} + +func (a *bootstrapAction) bind(m *model.Model) model.Action { + workflow := actions.Workflow() + + // Stop every component first so a re-bootstrap starts from a clean slate. Otherwise sim clients + // and tunnelers left running from a prior iteration keep stale sessions (and an old-version + // controller/router process can survive), which fails the next baseline. + workflow.AddAction(component.StopInParallel("*", 25)) + + // Only ctrl1 runs in phase 1. ctrl2/ctrl3 are provisioned but idle until the add-nodes phase. + workflow.AddAction(component.StopInParallel(".ctrl", 15)) + workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*")) + workflow.AddAction(host.GroupExec("component.ctrl", 5, "rm -rf ./fablab/ctrldata ./fablab/ctrl.db")) + + workflow.AddAction(component.Exec("#ctrl1", zitilab.ControllerActionInitStandalone)) + workflow.AddAction(component.Start("#ctrl1")) + workflow.AddAction(edge.ControllerAvailable("#ctrl1", 30*time.Second)) + workflow.AddAction(edge.Login("#ctrl1")) + + workflow.AddAction(component.StopInParallel(models.EdgeRouterTag, 25)) + workflow.AddAction(edge.InitEdgeRouters(models.EdgeRouterTag, 2)) + workflow.AddAction(edge.InitIdentities(models.SdkAppTag, 2)) + + // Shared host.v1 config: every tunneler-hosted loop service forwards to a local loop4 + // listener on tcp:127.0.0.1:3456. + workflow.AddAction(zitilib_actions.Edge("create", "config", "loop-backend", "host.v1", ` + { + "address" : "localhost", + "port" : 3456, + "protocol" : "tcp" + }`)) + + // intercept.v1 for loop-zet: ZET is tproxy-only (no proxy-port mode), so it needs an intercept + // config to know the address/port to intercept. The Go ziti-tunnel and ERT clients use proxy mode + // with the port on the command line, so they need no intercept config. Use a hostname (not a bare + // IP): ZET's tproxy assigns an IP from its own DNS range for the hostname and intercepts that; a + // static IP isn't intercepted. The co-located loop4 dialer targets this hostname, which resolves + // through ZET's resolver into the intercept. + workflow.AddAction(zitilib_actions.Edge("create", "config", "loop-zet-intercept", "intercept.v1", ` + { + "addresses": ["loop-zet.ziti"], + "portRanges": [{ "low": 15391, "high": 15391 }], + "protocols": ["tcp"] + }`)) + + // One loop service per hosting flavor. + workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-sdk", "-a", "loop-svc,loop-sdk-host-svc")) + workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-ert", "-c", "loop-backend", "-a", "loop-svc,loop-ert-svc")) + workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-zet", "-c", "loop-backend,loop-zet-intercept", "-a", "loop-svc,loop-zet-svc")) + workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-ziti-tunnel", "-c", "loop-backend", "-a", "loop-svc,loop-zt-svc")) + + // Bind policies: each flavor's service is hosted by the matching host identity. + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-sdk-hosts", "Bind", "--service-roles", "#loop-sdk-host-svc", "--identity-roles", "#loop-sdk-host")) + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ert-hosts", "Bind", "--service-roles", "#loop-ert-svc", "--identity-roles", "#ert-host")) + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-zet-hosts", "Bind", "--service-roles", "#loop-zet-svc", "--identity-roles", "#zet-host")) + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ziti-tunnel-hosts", "Bind", "--service-roles", "#loop-zt-svc", "--identity-roles", "#ziti-tunnel-host")) + + // Dial policy: the sim client dials every loop service. + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-clients", "Dial", "--service-roles", "#loop-svc", "--identity-roles", "#loop-client")) + + // Dial policy: the ziti-tunnel proxy clients dial loop-ziti-tunnel, so a co-located loop4 dialer can + // push traffic through the tunneler's local proxy listener (exercises the ziti-tunnel client path). + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ziti-tunnel-clients", "Dial", "--service-roles", "#loop-zt-svc", "--identity-roles", "#ziti-tunnel-client")) + + // Dial policy: the ERT proxy client (router-east-1) dials loop-ert, so its proxy listener + a + // co-located loop4 dialer exercise the ERT client path. + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ert-clients", "Dial", "--service-roles", "#loop-ert-svc", "--identity-roles", "#ert-proxy-client")) + + // Dial policy: the ZET clients dial loop-zet so their tproxy intercept + a co-located loop4 dialer + // exercise the ZET client path. + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-zet-clients", "Dial", "--service-roles", "#loop-zet-svc", "--identity-roles", "#zet-client")) + + // Sim control-plane services (metrics reporting + scenario control), hosted by the + // sim-controller identity created during activation. + workflow.AddAction(zitilib_actions.Edge("create", "service", "metrics", "-a", "sim-services")) + workflow.AddAction(zitilib_actions.Edge("create", "service", "sim-control", "-a", "sim-services")) + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "sim-service-hosts", "Bind", "--service-roles", "#sim-services", "--identity-roles", "#sim-services-host")) + workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "sim-service-clients", "Dial", "--service-roles", "#sim-services", "--identity-roles", "#sim-services-client")) + + // Broad routing: every identity may use every edge router, and every service may be + // routed over every edge router. Simple and sufficient for the upgrade test. + workflow.AddAction(zitilib_actions.Edge("create", "edge-router-policy", "all-endpoints", "--edge-router-roles", "#all", "--identity-roles", "#all")) + workflow.AddAction(zitilib_actions.Edge("create", "service-edge-router-policy", "all-services", "--service-roles", "#all", "--edge-router-roles", "#all")) + + return workflow +} diff --git a/zititest/models/upgrade-test/clusterupgrade.go b/zititest/models/upgrade-test/clusterupgrade.go new file mode 100644 index 000000000..5e9cb48af --- /dev/null +++ b/zititest/models/upgrade-test/clusterupgrade.go @@ -0,0 +1,316 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "regexp" + "strings" + "time" + + "github.com/openziti/fablab/kernel/lib/tui" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/foundation/v2/netz" + "github.com/openziti/ziti/zititest/zitilab" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/chaos" +) + +// cluster upgrade modes, selected by the clusterUpgradeMode variable. +const ( + // clusterUpgradeRolling restarts one node at a time, so the cluster runs mixed versions until + // the last node is done. + clusterUpgradeRolling = "rolling" + // clusterUpgradeAllAtOnce restarts every node together, so no node sees a version mismatch but + // every node pays its cold-start cost simultaneously. + clusterUpgradeAllAtOnce = "all-at-once" +) + +const ( + // clusterPortUpTimeout bounds how long a restarted node may take to listen again. A node that + // keeps dying and restarting can flap the port open, so this is a floor on detection, not a + // health check; clusterSettleTimeout is what actually decides health. + clusterPortUpTimeout = 2 * time.Minute + // clusterSettleTimeout bounds how long the cluster may take to report every peer connected with + // a leader elected. A node stuck in a crash loop never gets there, which is how a node that + // cannot stay up surfaces as a failure rather than a hang. + clusterSettleTimeout = 3 * time.Minute +) + +// ansiEscape matches the terminal color codes the inspect command emits, which are not stripped when +// its output is captured over ssh on some platforms and would otherwise be taken for JSON. +var ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`) + +// ctrlPeer is the part of a controller's connected-peers inspection this model uses. +type ctrlPeer struct { + Id string `json:"id"` + IsLeader bool `json:"isLeader"` + IsConnected bool `json:"isConnected"` +} + +// inspectPeers returns c's own view of the cluster. The inspect request is made with c's own binary, +// so the agent client always matches the controller it is talking to even while a rolling upgrade has +// different nodes on different versions. +func inspectPeers(c *model.Component) ([]ctrlPeer, error) { + ctrlType, ok := c.Type.(*zitilab.ControllerType) + if !ok { + return nil, fmt.Errorf("component %s is not a controller", c.Id) + } + cmd := fmt.Sprintf("%s agent inspect -a %s connected-peers", ctrlType.GetBinaryPath(c), c.Id) + out, err := c.GetHost().ExecLogged(cmd) + if err != nil { + return nil, fmt.Errorf("failed to inspect peers on %s: %w", c.Id, err) + } + out = ansiEscape.ReplaceAllString(out, "") + start := strings.Index(out, "[") + end := strings.LastIndex(out, "]") + if start < 0 || end < start { + return nil, fmt.Errorf("no peer list in the inspect output from %s: %s", c.Id, strings.TrimSpace(out)) + } + var peers []ctrlPeer + if err := json.Unmarshal([]byte(out[start:end+1]), &peers); err != nil { + return nil, fmt.Errorf("unparsable peer list from %s: %w", c.Id, err) + } + return peers, nil +} + +// clusterUpgradeOrder returns ctrls with the leader last, the ordering a rolling upgrade wants: +// restarting a follower costs nothing, while restarting the leader forces an election. When no node +// reports a leader, or reports one under an id that is not a component id, the bootstrap node stands +// in: it seeded the cluster and the others joined it, so it is the node most likely to be leading. +func clusterUpgradeOrder(ctrls []*model.Component) []*model.Component { + leaderId := "" + for _, c := range ctrls { + peers, err := inspectPeers(c) + if err != nil { + continue + } + for _, p := range peers { + if p.IsLeader { + leaderId = p.Id + } + } + if leaderId != "" { + break + } + } + + var followers, leader []*model.Component + for _, c := range ctrls { + if c.Id == leaderId { + leader = append(leader, c) + } else { + followers = append(followers, c) + } + } + if len(leader) == 0 { + for i, c := range followers { + if c.HasTag("bootstrap-ctrl") { + followers = append(followers[:i], followers[i+1:]...) + leader = append(leader, c) + break + } + } + } + return append(followers, leader...) +} + +// checkClusterConnected reports whether every controller sees the full peer set connected with a +// leader elected. +func checkClusterConnected(ctrls []*model.Component) error { + for _, c := range ctrls { + peers, err := inspectPeers(c) + if err != nil { + return err + } + if len(peers) != len(ctrls) { + return fmt.Errorf("%s sees %d peers, expected %d", c.Id, len(peers), len(ctrls)) + } + hasLeader := false + for _, p := range peers { + if !p.IsConnected { + return fmt.Errorf("%s reports peer %s as not connected", c.Id, p.Id) + } + hasLeader = hasLeader || p.IsLeader + } + if !hasLeader { + return fmt.Errorf("%s reports no leader", c.Id) + } + } + return nil +} + +// waitForClusterConnected blocks until the cluster reports itself healthy or timeout elapses. +func waitForClusterConnected(ctrls []*model.Component, timeout time.Duration) error { + log := tui.ValidationLogger() + deadline := time.Now().Add(timeout) + var lastErr error + for { + lastErr = checkClusterConnected(ctrls) + if lastErr == nil { + log.Infof("cluster reports %d peers connected with a leader", len(ctrls)) + return nil + } + if time.Now().After(deadline) { + return fmt.Errorf("cluster did not settle within %s: %w", timeout, lastErr) + } + time.Sleep(5 * time.Second) + } +} + +// restartCtrlOnVersion swaps c's binary to version and restarts it, waiting for its cluster port to +// come back. The config is untouched: a cluster node's config does not depend on the ziti version. +func restartCtrlOnVersion(run model.Run, c *model.Component, version string) error { + ctrlType, ok := c.Type.(*zitilab.ControllerType) + if !ok { + return fmt.Errorf("component %s is not a controller", c.Id) + } + ctrlType.SetVersion(version) + + // RestartSelected waits for the old process to exit before starting the new one; a plain + // stop/start would race, since Start no-ops while the old process is still shutting down + if err := chaos.RestartSelected(run, 1, c); err != nil { + return err + } + hostPort := c.GetHost().PublicIp + ":6262" + if err := netz.WaitForPortActive(hostPort, clusterPortUpTimeout); err != nil { + return fmt.Errorf("%s cluster port did not come back after upgrading to %s: %w", c.Id, version, err) + } + return nil +} + +// rollingClusterUpgrade upgrades one node at a time, waiting for the cluster to settle before moving +// on, so a node that cannot survive the upgrade fails the phase at the node that broke. +func rollingClusterUpgrade(run model.Run, ctrls []*model.Component, version string) error { + log := tui.ValidationLogger() + for _, c := range clusterUpgradeOrder(ctrls) { + log.Infof("upgrading cluster node %s to %s", c.Id, version) + if err := restartCtrlOnVersion(run, c, version); err != nil { + return err + } + if err := waitForClusterConnected(ctrls, clusterSettleTimeout); err != nil { + return fmt.Errorf("cluster unhealthy after upgrading %s to %s: %w", c.Id, version, err) + } + } + return nil +} + +// allAtOnceClusterUpgrade restarts every node on the new version together, so the cluster never runs +// mixed versions. +func allAtOnceClusterUpgrade(run model.Run, ctrls []*model.Component, version string) error { + log := tui.ValidationLogger() + for _, c := range ctrls { + ctrlType, ok := c.Type.(*zitilab.ControllerType) + if !ok { + return fmt.Errorf("component %s is not a controller", c.Id) + } + ctrlType.SetVersion(version) + } + + log.Infof("restarting all %d cluster nodes on %s at once", len(ctrls), version) + if err := chaos.RestartSelected(run, len(ctrls), ctrls...); err != nil { + return err + } + for _, c := range ctrls { + hostPort := c.GetHost().PublicIp + ":6262" + if err := netz.WaitForPortActive(hostPort, clusterPortUpTimeout); err != nil { + return fmt.Errorf("%s cluster port did not come back after upgrading to %s: %w", c.Id, version, err) + } + } + return waitForClusterConnected(ctrls, clusterSettleTimeout) +} + +// upgradeClusterControllers upgrades the whole controller cluster from toVersion to nextVersion, +// reporting each node's peak memory over the upgrade. clusterUpgradeMode selects between a rolling +// upgrade and an all-at-once one. +// +// No steady-state gate runs between nodes in rolling mode. Peers hold the cluster read-only while a +// version mismatch exists, so anything needing a write would fail for reasons this phase is not +// testing; the gate runs once the whole cluster is on nextVersion. +func upgradeClusterControllers(run model.Run) error { + m := run.GetModel() + log := tui.ValidationLogger() + + nextVersion := m.GetStringVariableOr("nextVersion", "") + if nextVersion == "" { + log.Infof("nextVersion is not set, skipping the cluster upgrade") + return nil + } + + ctrls := ctrlComponents(m) + if len(ctrls) == 0 { + return fmt.Errorf("no controllers found to upgrade") + } + + mode := m.GetStringVariableOr("clusterUpgradeMode", clusterUpgradeRolling) + // the steady-state gate just ran, so the window immediately behind us is settled traffic on + // toVersion, which is the baseline each node's upgrade is measured against + upgradeStart := time.Now() + baselineStart := upgradeStart.Add(-ctrlMemBaselineWindow) + + var err error + switch mode { + case clusterUpgradeRolling: + err = rollingClusterUpgrade(run, ctrls, nextVersion) + case clusterUpgradeAllAtOnce: + err = allAtOnceClusterUpgrade(run, ctrls, nextVersion) + default: + return fmt.Errorf("unknown clusterUpgradeMode [%s], expected %s or %s", + mode, clusterUpgradeRolling, clusterUpgradeAllAtOnce) + } + + // report either way: a node that died on the way up is the case the samples exist to explain + if reportErr := reportCtrlMemoryUpgrade(run, baselineStart, upgradeStart, "cluster upgrade to "+nextVersion); reportErr != nil { + if err == nil { + return reportErr + } + log.WithError(reportErr).Warn("controller memory report after a failed cluster upgrade") + } + if err != nil { + return err + } + return edge.Login("#ctrl1").Execute(run) +} + +// upgradeClusterRouters rolls the routers onto nextVersion, completing the upgrade the controllers +// started. +func upgradeClusterRouters(run model.Run) error { + nextVersion := run.GetModel().GetStringVariableOr("nextVersion", "") + if nextVersion == "" { + tui.ValidationLogger().Infof("nextVersion is not set, skipping the router upgrade") + return nil + } + return upgradeRoutersTo(run, nextVersion) +} + +// upgradeToNextVersion runs the optional cluster upgrade phase: the controllers, then the routers, +// with a steady-state gate after each. The whole phase is skipped when nextVersion is unset, so an +// iteration that ends at toVersion does not pay for two extra gates. +func upgradeToNextVersion(run model.Run) error { + m := run.GetModel() + if m.GetStringVariableOr("nextVersion", "") == "" { + tui.ValidationLogger().Infof("nextVersion is not set, skipping the cluster upgrade phase") + return nil + } + return m.Exec(run, + "upgradeClusterControllers", // 3-node cluster toVersion -> nextVersion + "validateSteadyStateAfterDisruption", // cluster back, terminators + traffic recovered + "upgradeClusterRouters", // routers toVersion -> nextVersion, rolling one at a time + "validateSteadyStateAfterDisruption", // whole system on nextVersion and clean + ) +} diff --git a/zititest/models/upgrade-test/configs/ctrl.yml.tmpl b/zititest/models/upgrade-test/configs/ctrl.yml.tmpl new file mode 100644 index 000000000..785d80b8c --- /dev/null +++ b/zititest/models/upgrade-test/configs/ctrl.yml.tmpl @@ -0,0 +1,113 @@ +v: 3 + +{{if .Component.GetFlag "cluster"}} +cluster: + dataDir: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrldata + # the standalone->HA migration seeds raft from the source db via a snapshot, which requires a + # controller restart to apply; restart ourselves so no external restart is needed. + restartSelfOnSnapshot: true +{{if .Component.GetFlag "migrateDb"}} +# migrate the existing standalone bbolt db into raft on first cluster start (bootstrap node only) +db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrl.db +{{end}} +{{else}} +db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrl.db +routerDataModel: + enabled: true +{{end}} + +identity: + cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem + key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key + ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.chain.pem + +trustDomain: upgrade-test + +# the endpoint that routers will connect to the controller over. +ctrl: + listener: tls:0.0.0.0:6262 + options: + advertiseAddress: tls:{{ .Host.PublicIp }}:6262 + +events: + entityChangeEventsLogger: + subscriptions: + - type: entityChange + propagateAlways: true + handler: + type: file + format: json + path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/entity-change.log + + circuitEventsLogger: + subscriptions: + - type: circuit + handler: + type: file + format: json + path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/circuits.log + + linkEventsLogger: + subscriptions: + - type: link + handler: + type: file + format: json + path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/links.log + + routerEventsLogger: + subscriptions: + - type: router + handler: + type: file + format: json + path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/router.log + + # api-session events carry the session type (legacy vs jwt), keyed by identity and source IP. + # The upgrade test reads this log to confirm a restarted client re-authenticated via OIDC (jwt). + apiSessionEventsLogger: + subscriptions: + - type: apiSession + handler: + type: file + format: json + path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/api-sessions.log + +healthChecks: + boltCheck: + interval: 30s + timeout: 15s + initialDelay: 15s + +edge: + api: + sessionTimeout: 30m + address: {{ .Host.PublicIp }}:1280 + enrollment: + signingCert: + cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert + key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key + edgeIdentity: + duration: 5m + edgeRouter: + duration: 5m + +web: + - name: all-apis-localhost + bindPoints: + - interface: 0.0.0.0:1280 + address: {{ .Host.PublicIp }}:1280 + options: + idleTimeout: 5000ms + readTimeout: 5000ms + writeTimeout: 100000ms + minTLSVersion: TLS1.2 + maxTLSVersion: TLS1.3 + apis: + - binding: health-checks + - binding: fabric + - binding: edge-management + - binding: edge-client +{{if .Component.GetFlag "oidc"}} + - binding: edge-oidc +{{end}} diff --git a/zititest/models/upgrade-test/configs/loop4-backend-host.yml.tmpl b/zititest/models/upgrade-test/configs/loop4-backend-host.yml.tmpl new file mode 100644 index 000000000..08fc5ef36 --- /dev/null +++ b/zititest/models/upgrade-test/configs/loop4-backend-host.yml.tmpl @@ -0,0 +1,11 @@ +# Plain-TCP loop4 listener that sits behind a ZET or ziti-tunnel host. The tunneler binds +# the ziti service (loop-zet / loop-ziti-tunnel) and forwards it to localhost:3456, where +# this listener speaks the loop4 protocol back to the remote dialer. +connectors: + backend: + transport: + address: tcp:127.0.0.1:3456 + +workloads: + - name: loop-backend + connector: backend diff --git a/zititest/models/upgrade-test/configs/loop4-client.yml.tmpl b/zititest/models/upgrade-test/configs/loop4-client.yml.tmpl new file mode 100644 index 000000000..e98dac5f0 --- /dev/null +++ b/zititest/models/upgrade-test/configs/loop4-client.yml.tmpl @@ -0,0 +1,42 @@ +connectors: + default: + sdk: + identity_file: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/cfg/{{ .Component.Id }}.json + report_sdk_metrics: true + ha: {{ .Component.GetFlag "ha" }} + max_control_underlays: 1 + test_service: loop-sdk + +remoteControlled: + connector: default + service: sim-control + +metrics: + connector: default + service: metrics + interval: 5s + clientId: {{ .Component.Id }} + +# One workload per hosting flavor. Each dials the ziti service that is hosted by the +# corresponding host type (SDK, ERT, ZET, ziti-tunnel). A scenario run executes every +# workload once, so a clean run means every flavor's data path is currently healthy. +workloads: + - name: loop-sdk + service_name: loop-sdk + connector: default + {{ $.Model.MustVariable "livenessWorkload" }} + + - name: loop-ert + service_name: loop-ert + connector: default + {{ $.Model.MustVariable "livenessWorkload" }} + + - name: loop-zet + service_name: loop-zet + connector: default + {{ $.Model.MustVariable "livenessWorkload" }} + + - name: loop-ziti-tunnel + service_name: loop-ziti-tunnel + connector: default + {{ $.Model.MustVariable "livenessWorkload" }} diff --git a/zititest/models/upgrade-test/configs/loop4-sdk-host.yml.tmpl b/zititest/models/upgrade-test/configs/loop4-sdk-host.yml.tmpl new file mode 100644 index 000000000..04dabdeb2 --- /dev/null +++ b/zititest/models/upgrade-test/configs/loop4-sdk-host.yml.tmpl @@ -0,0 +1,13 @@ +# loop4 SDK listener: binds the loop-sdk ziti service directly via the Go SDK. Deployed as a pair +# so one instance can be restarted (to observe OIDC re-auth) while the other keeps a terminator up. +connectors: + default: + sdk: + identity_file: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/cfg/{{ .Component.Id }}.json + ha: {{ .Component.GetFlag "ha" }} + max_control_underlays: 1 + +workloads: + - name: loop-sdk + connector: default + service_name: loop-sdk diff --git a/zititest/models/upgrade-test/configs/loop4-transport-dialer.yml.tmpl b/zititest/models/upgrade-test/configs/loop4-transport-dialer.yml.tmpl new file mode 100644 index 000000000..e79613b9c --- /dev/null +++ b/zititest/models/upgrade-test/configs/loop4-transport-dialer.yml.tmpl @@ -0,0 +1,30 @@ +# A loop4 dialer that drives traffic through a co-located tunneler's local listener instead of +# dialing a ziti service directly. The 'default' SDK connector is only used for sim control-plane +# (scenario control + metrics); the workload runs over the 'viaTunnel' transport connector, which +# connects to the tunneler's local TCP port. That port forwards through the tunneler client into the +# ziti service and on to the loop4 backend, exercising the tunneler's client-side data path. +connectors: + default: + sdk: + identity_file: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/cfg/{{ .Component.Id }}.json + report_sdk_metrics: true + ha: {{ .Component.GetFlag "ha" }} + max_control_underlays: 1 + viaTunnel: + transport: + address: {{ .Component.MustStringVariable "loopTunnelAddress" }} + +remoteControlled: + connector: default + service: sim-control + +metrics: + connector: default + service: metrics + interval: 5s + clientId: {{ .Component.Id }} + +workloads: + - name: via-tunnel + connector: viaTunnel + {{ $.Model.MustVariable "livenessWorkload" }} diff --git a/zititest/models/upgrade-test/configs/router.yml.tmpl b/zititest/models/upgrade-test/configs/router.yml.tmpl new file mode 100644 index 000000000..001230061 --- /dev/null +++ b/zititest/models/upgrade-test/configs/router.yml.tmpl @@ -0,0 +1,78 @@ +{{$ssh_username := .Model.MustVariable "credentials.ssh.username"}} +{{$identity := .Component.Id}} +{{$router_ip := .Host.PublicIp}} + +v: 3 + +enableDebugOps: true + +{{if .Component.GetFlag "ha"}} +ha: + enabled: true +{{end}} + +identity: + cert: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-client.cert + server_cert: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.cert + key: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}.key + ca: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.chain.pem + +tls: + handshakeTimeout: 30s + +# Only the initial controller is listed here. Additional cluster members (added during the HA +# phase) are learned dynamically from the controller once it is running in cluster mode. +ctrl: + endpoints: {{ range $host := .Model.MustSelectHosts "component.bootstrap-ctrl" 1 }} + - tls:{{ $host.PublicIp }}:6262{{end}} + startupTimeout: 5m + +healthChecks: + ctrlPingCheck: + interval: 30s + timeout: 15s + initialDelay: 15s + +metrics: + reportInterval: 15s + messageQueueSize: 10 + +link: + listeners: + - binding: transport + bind: tls:0.0.0.0:6000 + advertise: tls:{{$router_ip}}:6000 + dialers: + - binding: transport + options: + connectTimeout: 30s + +listeners: +{{if .Component.HasTag "tunneler"}} + - binding: tunnel + options: +{{- if .Component.HasTag "ert-proxy-client"}} + # proxy mode: open a local TCP listener for loop-ert so a co-located loop4 dialer can push + # traffic through this ERT client (no tproxy/:53, so it can share a host). + mode: proxy + services: + - loop-ert:15390 +{{- else}} + mode: tproxy +{{- end}} +{{end}} + - binding: edge + address: tls:0.0.0.0:6262 + options: + advertise: {{ .Host.PublicIp }}:6262 + +edge: + csr: + country: US + province: NC + locality: Charlotte + organization: NetFoundry + organizationalUnit: Ziti + sans: + ip: + - {{ .Host.PublicIp }} diff --git a/zititest/models/upgrade-test/ctrlmem.go b/zititest/models/upgrade-test/ctrlmem.go new file mode 100644 index 000000000..7519d22d8 --- /dev/null +++ b/zititest/models/upgrade-test/ctrlmem.go @@ -0,0 +1,435 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/openziti/fablab/kernel/lib/tui" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab" +) + +// memSamplerScript records one "," line per second for a controller, and captures a +// heap profile the first time RSS crosses the soft threshold. The heap path is unique per sampler +// start, so the guard means one profile per iteration rather than one per host for the whole run. +// +// The controller is located by the agent alias fablab passes at start, which is the same +// discriminator its process filter uses and is independent of the binary version, so the sampler +// keeps following the process across an upgrade. A stopped or restarting controller records a zero +// rather than a gap, which is what makes a restart loop visible in the samples. +// +// The alias only ever appears inside this file, never on the sampler's own command line, so pgrep +// cannot match the sampler instead of the controller. The pid is recorded so the sampler can be +// stopped without a pattern match, for the same reason. +const memSamplerScript = `#!/bin/sh +samples=%s +heap=%s +ziti=%s +alias=%s +soft_kb=%d +echo $$ > %s +while true; do + pids=$(pgrep -f "cli-agent-alias $alias " 2>/dev/null | tr '\n' ',' | sed 's/,$//') + rss=0 + if [ -n "$pids" ]; then + rss=$(ps -o rss= -p "$pids" 2>/dev/null | awk '{s+=$1} END {print s+0}') + fi + echo "$(date +%%s),$rss" >> "$samples" + if [ "$soft_kb" -gt 0 ] && [ "$rss" -gt "$soft_kb" ] && [ ! -f "$heap" ]; then + "$ziti" agent pprof-heap -a "$alias" -o "$heap" >/dev/null 2>&1 + fi + sleep 1 +done +` + +// samplerStartMarker prefixes the line written when a sampler starts. It divides the accumulated +// samples into one window per iteration, so a report can scope itself without being told when the +// iteration began. +const samplerStartMarker = "# start," + +// ctrlMemBaselineWindow is how far back an upgrade report looks for the steady state to compare +// against. The steady-state gate runs immediately before the cluster upgrade, so this lands on +// settled traffic rather than on the tail of an earlier disruption. +const ctrlMemBaselineWindow = 2 * time.Minute + +// ctrlMemPath builds a per-controller path under the host's log directory, which is outside the kit +// and so survives the --delete rsync that resetToBaseline runs. +func ctrlMemPath(c *model.Component, suffix string) string { + return fmt.Sprintf("/home/%s/logs/%s-%s", c.GetHost().GetSshUser(), c.Id, suffix) +} + +// ctrlHeapPattern is where c's heap profiles land, one per sampler start. +func ctrlHeapPattern(c *model.Component) string { + return ctrlMemPath(c, "mem-*.pprof") +} + +// ctrlComponents returns the model's controllers sorted by id, so every phase that walks them uses +// the same order. +func ctrlComponents(m *model.Model) []*model.Component { + ctrls := m.SelectComponents(".ctrl") + sort.Slice(ctrls, func(i, j int) bool { return ctrls[i].Id < ctrls[j].Id }) + return ctrls +} + +// ctrlMemThresholdKb reads a MiB-valued model variable as KiB, the unit ps reports RSS in. A missing +// or unparsable value disables the threshold. +func ctrlMemThresholdKb(m *model.Model, name string) int64 { + mb, err := strconv.ParseInt(m.GetStringVariableOr(name, ""), 10, 64) + if err != nil { + return 0 + } + return mb * 1024 +} + +// ctrlMemRatio reads a multiplier-valued model variable. A missing or unparsable value, or anything +// at or below one, disables the check. +func ctrlMemRatio(m *model.Model, name string) float64 { + ratio, err := strconv.ParseFloat(m.GetStringVariableOr(name, ""), 64) + if err != nil || ratio <= 1 { + return 0 + } + return ratio +} + +// startCtrlMemorySamplers installs and starts the RSS sampler on every controller host, replacing any +// sampler left over from a previous pass. Sampling covers the whole iteration, not just the cluster +// upgrade, so a node's upgrade behavior can be compared against its own earlier steady state as well +// as against its peers. +// +// Nothing already recorded is discarded. The samples accumulate across iterations behind a start +// marker that scopes reports to the current one, and each start gets its own heap profile path, so an +// iteration cannot destroy the evidence an earlier one captured. +func startCtrlMemorySamplers(run model.Run) error { + m := run.GetModel() + softKb := ctrlMemThresholdKb(m, "ctrlMemory.heapDumpAtMb") + startedAt := time.Now().Unix() + log := tui.ValidationLogger() + + for _, c := range ctrlComponents(m) { + samples := ctrlMemPath(c, "mem.csv") + heap := ctrlMemPath(c, fmt.Sprintf("mem-%d.pprof", startedAt)) + scriptPath := ctrlMemSamplerScriptPath(c) + script := fmt.Sprintf(memSamplerScript, samples, heap, zitilab.GetZitiBinaryPath(c, ""), c.Id, softKb, + ctrlMemSamplerPidPath(c)) + + if err := stopCtrlMemorySampler(c); err != nil { + return err + } + if err := c.GetHost().SendData([]byte(script), scriptPath); err != nil { + return fmt.Errorf("failed to install the memory sampler on %s: %w", c.Id, err) + } + // Two commands, not one backgrounded list. Backgrounding a list makes bash fork a subshell + // that holds the ssh session's stdout while it waits for the sampler, and the sampler never + // exits, so the session never closes. A simple command with its own redirects does not. + // The marker is written first, so no sample of this iteration precedes it. + marker := fmt.Sprintf("echo '%s%d' >> %s", samplerStartMarker, startedAt, samples) + start := fmt.Sprintf("nohup sh %s >/dev/null 2>&1 &", scriptPath) + if err := c.GetHost().ExecLogOnlyOnError(marker, start); err != nil { + return fmt.Errorf("failed to start the memory sampler on %s: %w", c.Id, err) + } + log.Infof("controller memory sampler running on %s, samples in %s", c.Id, samples) + } + return nil +} + +// ctrlMemSamplerScriptPath is where c's sampler script is installed. +func ctrlMemSamplerScriptPath(c *model.Component) string { + return ctrlMemPath(c, "mem-sampler.sh") +} + +// ctrlMemSamplerPidPath is where c's sampler records its pid so it can be stopped again. +func ctrlMemSamplerPidPath(c *model.Component) string { + return ctrlMemPath(c, "mem-sampler.pid") +} + +// stopCtrlMemorySampler kills c's sampler if one is running, and is a no-op when none is. +// +// The pid comes from the file the sampler writes, never from a pattern match. ssh runs a command +// through a shell whose own command line is the whole command string, so a pkill pattern naming the +// sampler matches that shell too and the sampler stop kills its own session. The recorded pid is +// checked against the running process before signalling, since a pid left behind by a host reboot +// could by then belong to something else. +func stopCtrlMemorySampler(c *model.Component) error { + pidFile := ctrlMemSamplerPidPath(c) + cmd := fmt.Sprintf( + `pid=$(cat %s 2>/dev/null); if [ -n "$pid" ] && grep -qasF %s /proc/$pid/cmdline; then kill "$pid"; fi; rm -f %s; true`, + pidFile, ctrlMemSamplerScriptPath(c), pidFile) + return c.GetHost().ExecLogOnlyOnError(cmd) +} + +// stopCtrlMemorySamplers stops sampling on every controller host. Nothing calls this during a run; +// it is here so sampling can be paused on a live instance. +func stopCtrlMemorySamplers(run model.Run) error { + for _, c := range ctrlComponents(run.GetModel()) { + if err := stopCtrlMemorySampler(c); err != nil { + return err + } + } + return nil +} + +// memSample is one sampler reading. +type memSample struct { + ts int64 + rss int64 +} + +// ctrlMemSeries is everything one controller's sampler has recorded, with the start markers that +// divide it into one window per iteration. +type ctrlMemSeries struct { + c *model.Component + samples []memSample + starts []int64 +} + +// ctrlMemStats summarizes a controller's samples over some window. +type ctrlMemStats struct { + peakKb int64 + lastKb int64 + samples int +} + +// readCtrlMemSeries fetches and parses everything c's sampler has recorded. The whole file is read +// once so a caller can summarize several windows of it without going back over the network. +func readCtrlMemSeries(c *model.Component) (*ctrlMemSeries, error) { + out, err := c.GetHost().ExecLogged("cat " + ctrlMemPath(c, "mem.csv")) + if err != nil { + return nil, fmt.Errorf("failed to read memory samples for %s: %w", c.Id, err) + } + samples, starts := parseCtrlMemSeries(out) + return &ctrlMemSeries{c: c, samples: samples, starts: starts}, nil +} + +// parseCtrlMemSeries splits sampler output into samples and the timestamps of the start markers +// between them. Unparsable lines are skipped: the file is appended to by a shell loop that can be +// killed mid-write, so a torn final line is normal rather than a fault. +func parseCtrlMemSeries(out string) ([]memSample, []int64) { + var samples []memSample + var starts []int64 + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if value, isMarker := strings.CutPrefix(line, samplerStartMarker); isMarker { + if ts, err := strconv.ParseInt(value, 10, 64); err == nil { + starts = append(starts, ts) + } + continue + } + if sample, ok := parseMemSample(line); ok { + samples = append(samples, sample) + } + } + return samples, starts +} + +// parseMemSample parses one "," sampler line, reporting false for anything else. +func parseMemSample(line string) (memSample, bool) { + tsField, rssField, found := strings.Cut(strings.TrimSpace(line), ",") + if !found { + return memSample{}, false + } + ts, err := strconv.ParseInt(tsField, 10, 64) + if err != nil { + return memSample{}, false + } + rss, err := strconv.ParseInt(rssField, 10, 64) + if err != nil { + return memSample{}, false + } + return memSample{ts: ts, rss: rss}, true +} + +// statsIn summarizes the samples in [from, to). A zero from starts at the most recent sampler start, +// which scopes a report to the current iteration; a zero to runs to the newest sample. +func (self *ctrlMemSeries) statsIn(from, to time.Time) ctrlMemStats { + start := self.currentWindowStart() + if !from.IsZero() { + start = from.Unix() + } + end := int64(0) + if !to.IsZero() { + end = to.Unix() + } + + var stats ctrlMemStats + for _, sample := range self.samples { + if sample.ts < start || (end != 0 && sample.ts >= end) { + continue + } + stats.samples++ + stats.lastKb = sample.rss + if sample.rss > stats.peakKb { + stats.peakKb = sample.rss + } + } + return stats +} + +// currentWindowStart returns the timestamp of the most recent sampler start. A series with no marker +// yields a start that admits every sample, which is what a sampler predating the marker leaves behind. +func (self *ctrlMemSeries) currentWindowStart() int64 { + if len(self.starts) == 0 { + return 0 + } + return self.starts[len(self.starts)-1] +} + +// mib renders a KiB reading the way the reports talk about memory. +func mib(kb int64) int64 { + return kb / 1024 +} + +// reportCtrlMemory logs peak and current RSS for every controller over the window opening at since (a +// zero since means the current iteration), and fails when a controller's peak exceeds +// ctrlMemory.failAtMb. +func reportCtrlMemory(run model.Run, since time.Time, phase string) error { + m := run.GetModel() + log := tui.ValidationLogger() + failKb := ctrlMemThresholdKb(m, "ctrlMemory.failAtMb") + + var worstPeak int64 + var worst *model.Component + + for _, c := range ctrlComponents(m) { + series, err := readCtrlMemSeries(c) + if err != nil { + log.WithError(err).Warnf("no memory samples for %s", c.Id) + continue + } + stats := series.statsIn(since, time.Time{}) + if stats.samples == 0 { + log.Warnf("controller memory [%s] %s: no samples", phase, c.Id) + continue + } + log.Infof("controller memory [%s] %s: peak %d MiB, current %d MiB, %d samples", + phase, c.Id, mib(stats.peakKb), mib(stats.lastKb), stats.samples) + if stats.peakKb > worstPeak { + worstPeak, worst = stats.peakKb, c + } + } + + if failKb > 0 && worstPeak > failKb { + return fmt.Errorf("controller %s peaked at %d MiB during %s, over the %d MiB limit; heap profiles, if any were captured, are at %s", + worst.Id, mib(worstPeak), phase, mib(failKb), ctrlHeapPattern(worst)) + } + return nil +} + +// reportCtrlMemoryUpgrade reports each controller's memory across an upgrade against its own steady +// state immediately before it, and fails when a node exceeds either ctrlMemory.failAtRatio or the +// absolute ctrlMemory.failAtMb ceiling. The baseline window is [baselineStart, upgradeStart) and the +// upgrade window runs from upgradeStart to now. +// +// The ratio is the check that bites at this model's scale. The absolute ceiling is sized for the +// failure #4219 reports, which needs a far larger data set to reach, whereas a node using several +// times its own steady state is visible whatever the data set is. +func reportCtrlMemoryUpgrade(run model.Run, baselineStart, upgradeStart time.Time, phase string) error { + m := run.GetModel() + log := tui.ValidationLogger() + failKb := ctrlMemThresholdKb(m, "ctrlMemory.failAtMb") + failRatio := ctrlMemRatio(m, "ctrlMemory.failAtRatio") + + var violations []string + for _, c := range ctrlComponents(m) { + series, err := readCtrlMemSeries(c) + if err != nil { + log.WithError(err).Warnf("no memory samples for %s", c.Id) + continue + } + base := series.statsIn(baselineStart, upgradeStart) + upgrade := series.statsIn(upgradeStart, time.Time{}) + if upgrade.samples == 0 { + log.Warnf("controller memory [%s] %s: no samples", phase, c.Id) + continue + } + + growth := "n/a" + if base.peakKb > 0 { + growth = fmt.Sprintf("%.1fx", float64(upgrade.peakKb)/float64(base.peakKb)) + } + log.Infof("controller memory [%s] %s: baseline peak %d MiB, upgrade peak %d MiB (%s), current %d MiB", + phase, c.Id, mib(base.peakKb), mib(upgrade.peakKb), growth, mib(upgrade.lastKb)) + + before := len(violations) + if failRatio > 0 && base.peakKb > 0 && float64(upgrade.peakKb) > failRatio*float64(base.peakKb) { + violations = append(violations, fmt.Sprintf( + "%s grew to %s of its %d MiB baseline (%d MiB), over the %.1fx limit", + c.Id, growth, mib(base.peakKb), mib(upgrade.peakKb), failRatio)) + } + if failKb > 0 && upgrade.peakKb > failKb { + violations = append(violations, fmt.Sprintf("%s peaked at %d MiB, over the %d MiB limit", + c.Id, mib(upgrade.peakKb), mib(failKb))) + } + if len(violations) > before { + log.Infof("heap profiles for %s, if any were captured, are at %s", c.Id, ctrlHeapPattern(c)) + } + } + + if len(violations) > 0 { + return fmt.Errorf("controller memory grew unexpectedly during %s: %s", phase, strings.Join(violations, "; ")) + } + return nil +} + +// summarizeCtrlMemory logs one row per controller per sampler run, which is one row per iteration, so +// a finished multi-iteration run can be read back with a single command instead of scrolled for. +func summarizeCtrlMemory(run model.Run) error { + log := tui.ValidationLogger() + + var worstPeak int64 + worst := "" + for _, c := range ctrlComponents(run.GetModel()) { + series, err := readCtrlMemSeries(c) + if err != nil { + log.WithError(err).Warnf("no memory samples for %s", c.Id) + continue + } + if len(series.samples) == 0 { + log.Warnf("controller memory summary: %s has no samples", c.Id) + continue + } + + // a series with no marker predates them, so it is reported as the single window it is + starts := series.starts + if len(starts) == 0 { + starts = []int64{series.samples[0].ts} + } + for i, start := range starts { + from := time.Unix(start, 0) + to := time.Time{} + if i+1 < len(starts) { + to = time.Unix(starts[i+1], 0) + } + stats := series.statsIn(from, to) + log.Infof("controller memory summary: %s iteration %d (%s): peak %d MiB, final %d MiB, %d samples", + c.Id, i+1, from.Format(time.RFC3339), mib(stats.peakKb), mib(stats.lastKb), stats.samples) + if stats.peakKb > worstPeak { + worstPeak = stats.peakKb + worst = fmt.Sprintf("%s in iteration %d", c.Id, i+1) + } + } + } + + if worst != "" { + log.Infof("controller memory summary: highest peak was %d MiB, %s", mib(worstPeak), worst) + } + return nil +} diff --git a/zititest/models/upgrade-test/ctrlmem_test.go b/zititest/models/upgrade-test/ctrlmem_test.go new file mode 100644 index 000000000..ac49535fa --- /dev/null +++ b/zititest/models/upgrade-test/ctrlmem_test.go @@ -0,0 +1,104 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// sampleFile is two iterations of sampler output: a controller that idles around 100 MiB, restarts +// (the zeros), and comes back heavier in the second iteration. +const sampleFile = `# start,1000 +1001,102400 +1002,104448 +1003,0 +1004,106496 +# start,2000 +2001,102400 +2002,0 +2003,512000 +2004,460800 +` + +// TestParseCtrlMemSeries pins that the start markers are read as markers and not as samples. They +// share the file with the samples, so a marker parsed as one would read a timestamp as a resident set +// size and report a peak of about 1.6 TiB. +func TestParseCtrlMemSeries(t *testing.T) { + samples, starts := parseCtrlMemSeries(sampleFile) + + require.Equal(t, []int64{1000, 2000}, starts) + require.Len(t, samples, 8) + require.Equal(t, memSample{ts: 1001, rss: 102400}, samples[0]) + require.Equal(t, memSample{ts: 2004, rss: 460800}, samples[7]) +} + +// TestParseMemSampleRejectsNonSamples pins what does not count as a sample. A torn final line is +// normal, since the file is appended to by a shell loop that can be killed mid-write. +func TestParseMemSampleRejectsNonSamples(t *testing.T) { + for _, line := range []string{"", "# start,1700000000", "garbage", "1700000000", "abc,123", "123,abc"} { + _, ok := parseMemSample(line) + require.False(t, ok, "must not parse [%s] as a sample", line) + } +} + +// TestStatsInDefaultsToTheCurrentIteration pins the window a report defaults to. Samples accumulate +// across iterations in one file, so starting from the first marker rather than the last would report a +// peak from an earlier iteration as if it belonged to this one. +func TestStatsInDefaultsToTheCurrentIteration(t *testing.T) { + series := testSeries(t, sampleFile) + + stats := series.statsIn(time.Time{}, time.Time{}) + require.Equal(t, 4, stats.samples, "only the samples after the last marker") + require.Equal(t, int64(512000), stats.peakKb) + require.Equal(t, int64(460800), stats.lastKb, "last is the newest sample, not the largest") +} + +// TestStatsInBoundedWindow pins that a window excludes its end, so the baseline and upgrade windows an +// upgrade report uses cannot both claim the same sample. +func TestStatsInBoundedWindow(t *testing.T) { + series := testSeries(t, sampleFile) + + base := series.statsIn(time.Unix(1000, 0), time.Unix(2000, 0)) + require.Equal(t, 4, base.samples) + require.Equal(t, int64(106496), base.peakKb) + + upgrade := series.statsIn(time.Unix(2000, 0), time.Time{}) + require.Equal(t, 4, upgrade.samples) + require.Equal(t, int64(512000), upgrade.peakKb) + + // the ratio the upgrade check works from: 500 MiB against a 104 MiB baseline + require.InDelta(t, 4.8, float64(upgrade.peakKb)/float64(base.peakKb), 0.1) +} + +// TestStatsInWithoutMarkers pins that samples written before markers existed still report, rather than +// reporting nothing because every line fell outside the window. +func TestStatsInWithoutMarkers(t *testing.T) { + series := testSeries(t, "1001,102400\n1002,204800\n") + + stats := series.statsIn(time.Time{}, time.Time{}) + require.Equal(t, 2, stats.samples) + require.Equal(t, int64(204800), stats.peakKb) +} + +func testSeries(t *testing.T, out string) *ctrlMemSeries { + t.Helper() + samples, starts := parseCtrlMemSeries(out) + return &ctrlMemSeries{samples: samples, starts: starts} +} diff --git a/zititest/models/upgrade-test/ha.go b/zititest/models/upgrade-test/ha.go new file mode 100644 index 000000000..172ee1a24 --- /dev/null +++ b/zititest/models/upgrade-test/ha.go @@ -0,0 +1,195 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "net" + "path/filepath" + "time" + + "github.com/openziti/fablab/kernel/lib/actions/component" + "github.com/openziti/fablab/kernel/lib/tui" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/foundation/v2/netz" + "github.com/openziti/ziti/zititest/zitilab" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/chaos" +) + +// pushControllerConfig re-renders a controller's config and copies just that file to its host. It +// deliberately avoids the whole-kit rsync (RsyncStaged), which does a --delete sync and would wipe +// runtime state on the host, including the standalone bbolt db we migrate from and the raft data dir. +func pushControllerConfig(run model.Run, c *model.Component, ctrlType *zitilab.ControllerType) error { + if err := ctrlType.StageFiles(run, c); err != nil { + return err + } + configName := ctrlType.ConfigName + if configName == "" { + configName = c.Id + ".yml" + } + local := filepath.Join(run.GetConfigDir(), configName) + remote := fmt.Sprintf("/home/%s/fablab/cfg/%s", c.GetHost().GetSshUser(), configName) + return c.GetHost().SendFile(local, remote) +} + +// waitForPortInactive blocks until a TCP dial to address fails (the port stops accepting) or the +// timeout elapses. It is the inverse of netz.WaitForPortActive, used to detect a controller exiting +// after a snapshot restore so we know when to restart it. +func waitForPortInactive(address string, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + for { + conn, err := net.DialTimeout("tcp", address, 2*time.Second) + if err != nil { + return nil + } + _ = conn.Close() + if time.Now().After(deadline) { + return fmt.Errorf("port %s still active after %s", address, timeout) + } + time.Sleep(time.Second) + } +} + +// upgradeControllerToHa converts the standalone ctrl1 into a single-node HA cluster in place: it flips +// ctrl1 to cluster mode, re-renders its config (with the cluster directive plus the migrateDb source +// so raft is seeded from the existing standalone bbolt), pushes it, then restarts. The controller +// snapshots its bbolt into raft and comes up as a bootstrapped one-node cluster with the existing +// identities, services, and terminators intact. +// +// Applying that snapshot needs a second restart, and which side performs it depends on the version +// under test. Only a controller whose Network.RestoreSnapshot honors restartSelfOnSnapshot re-execs; +// the released 2.0.x this normally upgrades to exits instead, and nothing here supervises the process. +// An exited controller is picked back up by the steady-state gate that follows, whose management +// client restarts a controller it cannot reach. So the availability check below can pass against a +// process that is seconds from exiting, and recovery then depends on that gate rather than on +// anything in this function. That is tolerated rather than handled: making it deterministic means +// waiting for the port to drop and starting the controller explicitly, the way addClusterNodes does. +func upgradeControllerToHa(run model.Run) error { + m := run.GetModel() + log := tui.ValidationLogger() + + c := m.MustSelectComponent("#ctrl1") + ctrlType, ok := c.Type.(*zitilab.ControllerType) + if !ok { + return fmt.Errorf("component ctrl1 is not a controller") + } + + log.Infof("converting ctrl1 to a single-node HA cluster") + // cluster mode makes ctrl.yml.tmpl emit the cluster (raft) directive; migrateDb also emits the + // source db line so the controller seeds raft from the existing standalone bbolt on first start. + c.PutVariable("cluster", true) + c.PutVariable("migrateDb", true) + + if err := pushControllerConfig(run, c, ctrlType); err != nil { + return err + } + + // RestartSelected waits for the old process to exit before starting the new one; a plain stop/start + // would race, since Start no-ops while the old process is still shutting down + if err := chaos.RestartSelected(run, 1, c); err != nil { + return err + } + // generous timeout: this covers the snapshot apply and whatever restart follows it + if err := edge.ControllerAvailable("#ctrl1", 120*time.Second).Execute(run); err != nil { + return err + } + return edge.Login("#ctrl1").Execute(run) +} + +// addClusterNodes brings ctrl2 and ctrl3 online and joins them to ctrl1's cluster. Each is staged on +// the upgraded (toVersion) binary in cluster mode and started, then the leader runs "agent cluster add" +// for each node's control address, growing the single-node cluster to three. +func addClusterNodes(run model.Run) error { + m := run.GetModel() + toVersion := m.MustStringVariable("toVersion") + log := tui.ValidationLogger() + + nodes := m.SelectComponents(".cluster-node") + if len(nodes) == 0 { + return fmt.Errorf("no cluster-node controllers found to add") + } + + // a joining node must run the same (upgraded) binary as the leader and be in cluster mode; it does + // not migrate a db (no migrateDb), it replicates state from the leader after being added. + for _, c := range nodes { + ctrlType, ok := c.Type.(*zitilab.ControllerType) + if !ok { + return fmt.Errorf("component %s is not a controller", c.Id) + } + c.PutVariable("cluster", true) + ctrlType.SetVersion(toVersion) + if err := pushControllerConfig(run, c, ctrlType); err != nil { + return err + } + } + + if err := component.StartInParallel(".cluster-node", 5).Execute(run); err != nil { + return err + } + + // add each new node from the leader (ctrl1); the controller advertises cluster membership on :6262 + leader := m.MustSelectComponent("#ctrl1") + leaderType, ok := leader.Type.(*zitilab.ControllerType) + if !ok { + return fmt.Errorf("component ctrl1 is not a controller") + } + leaderBinary := leaderType.GetBinaryPath(leader) + for _, c := range nodes { + hostPort := c.GetHost().PublicIp + ":6262" + + // StartInParallel returns once the process is launched, not once it is listening, so wait for + // the node's cluster port to be open before adding it rather than retrying a membership change. + log.Infof("waiting for %s cluster port %s", c.Id, hostPort) + if err := netz.WaitForPortActive(hostPort, 90*time.Second); err != nil { + return fmt.Errorf("cluster port for %s (%s) never became available: %w", c.Id, hostPort, err) + } + + log.Infof("adding %s to the cluster at tls:%s", c.Id, hostPort) + cmd := fmt.Sprintf("%s agent cluster add tls:%s", leaderBinary, hostPort) + if err := leader.GetHost().ExecLogOnlyOnError(cmd); err != nil { + return fmt.Errorf("failed to add %s to the cluster at tls:%s: %w", c.Id, hostPort, err) + } + + // The leader streams its db to the joining node, which restores the snapshot and then replaces + // itself: the config sets restartSelfOnSnapshot, so the node re-execs and the old process exits. + // The cluster port drops while the replacement sleeps out its start delay, which is the signal + // the restore happened. + // + // Do not wait for the node's process to disappear. The replacement runs the same command line, + // so it matches the component's process filter and the node never looks stopped. + log.Infof("waiting for %s to restore the cluster snapshot and restart itself", c.Id) + if err := waitForPortInactive(hostPort, 90*time.Second); err != nil { + return fmt.Errorf("%s did not restart after joining (snapshot restore expected): %w", c.Id, err) + } + // A no-op while the node is restarting itself, since Start returns early when a matching process + // is already running. Kept as the safety net for a node that exited instead of re-execing. + if err := component.Start("#" + c.Id).Execute(run); err != nil { + return err + } + if err := netz.WaitForPortActive(hostPort, 90*time.Second); err != nil { + return fmt.Errorf("%s cluster port did not come back after restart: %w", c.Id, err) + } + // let the membership change commit and replicate before adding the next node + time.Sleep(10 * time.Second) + } + + // give raft time to settle the 3-node cluster before the stability gate runs + log.Infof("waiting for the 3-node cluster to settle") + time.Sleep(30 * time.Second) + return nil +} diff --git a/zititest/models/upgrade-test/main.go b/zititest/models/upgrade-test/main.go new file mode 100644 index 000000000..931e391a2 --- /dev/null +++ b/zititest/models/upgrade-test/main.go @@ -0,0 +1,640 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "embed" + "os" + "path" + "strings" + "time" + + "github.com/michaelquigley/pfxlog" + "github.com/openziti/fablab" + "github.com/openziti/fablab/kernel/lib/actions/component" + "github.com/openziti/fablab/kernel/lib/binding" + "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key" + semaphore "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore" + terraformInit "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform" + distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution" + "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync" + awsSshKeyDispose "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key" + "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/fablab/resources" + "github.com/openziti/ziti/zititest/models/test_resources" + "github.com/openziti/ziti/zititest/zitilab" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/models" + zitiLibOps "github.com/openziti/ziti/zititest/zitilab/runlevel/5_operation" +) + +//go:embed configs +var configResource embed.FS + +// livenessWorkload is a short loop4 workload used purely to prove a service's data path is +// currently healthy. A scenario run executes one of these per hosting flavor. +var livenessWorkload = "" + + `concurrency: 1 + iterations: 1 + dialer: + txRequests: 20 + rxTimeout: 5s + payloadMinBytes: 512 + payloadMaxBytes: 512 + listener: + rxTimeout: 5s +` + +func getUniqueId() string { + if runId := os.Getenv("GITHUB_RUN_ID"); runId != "" { + return "-" + runId + "." + os.Getenv("GITHUB_RUN_ATTEMPT") + } + return "-" + os.Getenv("USER") +} + +// rotatingLogConfig is the log configuration applied to every ziti component in this model. The +// rotate strategy keeps each component's log size-bounded (via "ziti ops log-pipe") so a long +// multi-iteration run cannot fill the host disk. Combined with routers no longer running at debug, +// this keeps the per-host log footprint small across all iterations. +// +// The pipe binary is pinned to the locally-built ziti (the empty version) because the staged +// fromVersion/toVersion binaries predate "ops log-pipe". Without the pin, the pipe would run a binary +// that lacks the command and take the component's stdout down with it on most of the versions this +// run cycles through. +func rotatingLogConfig() zitilab.LogConfig { + localBuild := "" + return zitilab.LogConfig{ + Strategy: zitilab.LogStrategyRotate, + PipeBinaryVersion: &localBuild, + } +} + +var m = &model.Model{ + Id: "upgrade-test", + Scope: model.Scope{ + Defaults: model.Variables{ + // version knobs. The system starts on fromVersion and is upgraded to toVersion + // in place. zetVersion is fixed for the whole run (ZET stays up, not upgraded). + // toVersion is the label the built binary is staged under and, for ref builds, the version + // stamped into it, so it must be a plain numeric semver (the SDK version parser rejects + // pre-release/build-metadata suffixes). toVersionRef selects the actual source. + "fromVersion": "v1.6.20", + "toVersion": "v2.0.3", + // zetVersion tracks the ziti-edge-tunnel bundled in the latest stable Ziti Desktop Edge for + // Windows (WDE 2.11.2.5 ships v1.18.0). Pin to v1.11.4 (WDE 2.9.7.1) to reproduce the older + // client's post-2.0-migration session-recovery failure. + "zetVersion": "v1.18.0", + // ziti-tunnel runs the locally-built ziti by default (so it exercises the current SDK, + // e.g. to validate SDK fixes). Set to a release version to test a specific released build. + "zitiTunnelVersion": "", + // toVersionSource optionally points at a locally-built ziti binary to stage under the + // toVersion name, so the upgrade runs a patched controller/router instead of the released + // toVersion. Empty falls through to toVersionRef. The ZITI_TOVERSION_PATH env var overrides + // this, and a source binary wins over a ref build wherever both are set. + "toVersionSource": "", + // toVersionRef optionally names a git ref (branch, tag, or SHA) on openziti/ziti to build + // from source and stage under the toVersion name, so the upgrade can run a fix no release + // carries yet. Empty downloads the released toVersion instead, which is the normal case. + // The ZITI_TOVERSION_REF env var overrides this; toVersionSource wins over both. + // Changing the ref alone is enough to get a new binary: the staging op is keyed on the ref as + // well as the version, and acquire caches on the commit the ref resolves to, so a branch that + // has moved rebuilds rather than serving the previous build. + "toVersionRef": "", + + // nextVersion is the optional second upgrade hop: once the cluster is up on toVersion it is + // upgraded again, cluster and all, to nextVersion. Leave it empty to end the iteration at + // toVersion and skip the phase entirely. nextVersionSource/nextVersionRef work exactly like + // their toVersion counterparts, with ZITI_NEXTVERSION_PATH and ZITI_NEXTVERSION_REF as the + // environment overrides. + // + // 2.1 has no release yet, so the default builds it from main. That is a moving ref: what a + // run tested is whatever main pointed at that day. Set nextVersionRef to a tag once 2.1 + // releases, at which point nextVersion alone selects a downloaded release. + "nextVersion": "v2.1.0", + "nextVersionSource": "", + "nextVersionRef": "main", + // clusterUpgradeMode picks how the cluster moves to nextVersion. "rolling" restarts one node + // at a time, which leaves the cluster in mixed-version read-only mode until the last node is + // done; "all-at-once" restarts every node together, so no node sees a version mismatch but + // every node cold-starts simultaneously. + "clusterUpgradeMode": clusterUpgradeRolling, + + // controller memory sampling. Every controller host records its controller's RSS once a + // second for the whole iteration, so an upgrade's memory cost can be compared against the + // node's own steady state and against its peers. + "ctrlMemory": model.Variables{ + // heapDumpAtMb captures one heap profile per controller the first time RSS crosses this, + // which is the artifact a memory regression is diagnosed from and the one thing that + // cannot be recovered after the fact. Zero disables the capture. + "heapDumpAtMb": 400, + // failAtMb fails a phase whose peak exceeds it. Zero disables the check. + "failAtMb": 1024, + // failAtRatio fails the cluster upgrade when a controller's peak over the upgrade exceeds + // this multiple of its own peak over the steady state just before it. This is the check + // that bites at this model's scale, where the absolute ceiling is far out of reach; the + // value is a first guess and wants calibrating against a run that is known good. Anything + // at or below one disables it. + "failAtRatio": 3, + }, + + // auth/cluster flags. Phase 1 runs a single standalone controller on legacy auth, + // so all three default to false. OIDC becomes available via the 2.0 controller's + // auto-binding, not by setting these. + "oidc": false, + "ha": false, + "cluster": false, + + // steady-state gate tuning + "steadyState": model.Variables{ + "requiredCleanRuns": 3, + "recoveryTimeout": "1m", + "stabilityWindow": "2m", + }, + + "livenessWorkload": livenessWorkload, + + "environment": "upgrade-test" + getUniqueId(), + "credentials": model.Variables{ + "aws": model.Variables{ + "managed_key": true, + }, + "ssh": model.Variables{ + "username": "ubuntu", + }, + "edge": model.Variables{ + "username": "admin", + "password": "admin", + }, + }, + }, + }, + + StructureFactories: []model.Factory{ + // apply the bulk from/zet versions and the shared log config to every component + model.FactoryFunc(func(m *model.Model) error { + fromVersion := m.MustStringVariable("fromVersion") + zetVersion := m.MustStringVariable("zetVersion") + zitiTunnelVersion := m.MustStringVariable("zitiTunnelVersion") + return m.ForEachComponent("*", 1, func(c *model.Component) error { + switch t := c.Type.(type) { + case *zitilab.ControllerType: + t.Version = fromVersion + t.LogConfig = rotatingLogConfig() + case *zitilab.RouterType: + t.Version = fromVersion + t.LogConfig = rotatingLogConfig() + case *zitilab.ZitiTunnelType: + t.Version = zitiTunnelVersion + t.LogConfig = rotatingLogConfig() + case *zitilab.ZitiEdgeTunnelType: + t.Version = zetVersion + t.ZitiVersion = fromVersion + t.LogConfig = rotatingLogConfig() + t.InitType(c) + } + return nil + }) + }), + // per-component version overrides, e.g. -l router_west_version=v1.6.10 + model.FactoryFunc(func(m *model.Model) error { + return m.ForEachComponent("*", 1, func(c *model.Component) error { + versioned, ok := c.Type.(interface{ SetVersion(string) }) + if !ok { + return nil + } + varName := strings.ReplaceAll(c.Id, "-", "_") + "_version" + if version, found := m.GetStringVariable(varName); found { + versioned.SetVersion(version) + } + return nil + }) + }), + // instance sizing + model.FactoryFunc(func(m *model.Model) error { + return m.ForEachHost("*", 1, func(host *model.Host) error { + if strings.HasPrefix(host.Id, "ctrl") { + host.InstanceType = "t3.medium" + } else { + host.InstanceType = "c5.large" + } + return nil + }) + }), + }, + + Factories: []model.Factory{ + model.FactoryFunc(func(m *model.Model) error { + pfxlog.Logger().Infof("environment [%s]", m.MustStringVariable("environment")) + return nil + }), + // sim harness: metrics collection and the steady-state validation gate + model.FactoryFunc(func(m *model.Model) error { + simServices := zitiLibOps.NewSimServices(func(s string) string { + return "component#" + s + }) + + m.AddActivationStageF(simServices.SetupSimControllerIdentity) + m.AddOperatingStage(simServices.CollectSimMetricStage("metrics")) + + m.AddActionF("startSimMetrics", func(run model.Run) error { + return simServices.CollectSimMetrics(run, "metrics") + }) + + gate := newSteadyStateGate(simServices) + m.AddActionF("validateSteadyState", gate.validate) + m.AddActionF("validateSteadyStateAfterDisruption", gate.validateAfterDisruption) + + // resetToBaseline returns the system to a fresh standalone fromVersion baseline so a new + // testIteration can run from a known state. It undoes the in-process mutations a pass makes + // (controller/router binary versions, HA/migration flags) and drops the in-harness + // sim-controller client's cached context/enrollment (stale once the controller is wiped), + // then re-runs the standard configuration -> distribution -> activation pipeline. Re-rendering + // the configs from the reset state is what brings the controller back up standalone rather + // than reusing the prior pass's HA config; Activate also re-bootstraps and re-enrolls the + // sim-controller identity (an activation stage). + m.AddActionF("resetToBaseline", func(run model.Run) error { + fromVersion := m.MustStringVariable("fromVersion") + if err := m.ForEachComponent("*", 1, func(c *model.Component) error { + switch t := c.Type.(type) { + case *zitilab.ControllerType: + t.SetVersion(fromVersion) + c.PutVariable("cluster", false) + c.PutVariable("migrateDb", false) + case *zitilab.RouterType: + t.SetVersion(fromVersion) + } + return nil + }); err != nil { + return err + } + + simServices.Reset() + + if err := m.Build(run); err != nil { + return err + } + if err := m.Sync(run); err != nil { + return err + } + return m.Activate(run) + }) + + return nil + }), + }, + + Resources: model.Resources{ + resources.Configs: resources.SubFolder(configResource, "configs"), + resources.Binaries: os.DirFS(path.Join(os.Getenv("GOPATH"), "bin")), + resources.Terraform: test_resources.TerraformResources(), + }, + + Regions: model.Regions{ + "us-east-1": { + Region: "us-east-1", + Site: "us-east-1a", + Hosts: model.Hosts{ + "ctrl1": { + Components: model.Components{ + "ctrl1": { + Scope: model.Scope{Tags: model.Tags{"ctrl", "bootstrap-ctrl"}}, + Type: &zitilab.ControllerType{}, + }, + }, + }, + // provisioned up front, started only during the add-nodes phase + "ctrl2": { + Components: model.Components{ + "ctrl2": { + Scope: model.Scope{Tags: model.Tags{"ctrl", "ha", "cluster-node"}}, + Type: &zitilab.ControllerType{}, + }, + }, + }, + "router-east-1": { + Scope: model.Scope{Tags: model.Tags{"ert-client"}}, + Components: model.Components{ + "router-east-1": { + // ert-proxy-client makes this ERT run its tunnel binding in proxy mode (local + // loop-ert listener on :15390) so the co-located loop4 dialer below can push + // traffic through the ERT client path. + Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client", "ert-proxy-client"}}, + Type: &zitilab.RouterType{}, + }, + "loop4-client-stable": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "stable"}}, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-client.yml.tmpl", + Mode: zitilab.Loop4RemoteControlled, + }, + }, + "loop4-client-restart": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "restart"}}, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-client.yml.tmpl", + Mode: zitilab.Loop4RemoteControlled, + }, + }, + // drives traffic through the ERT client's local proxy listener (router-east-1 is a + // single router, not a stable/restart pair, so one dialer) + "loop-ert-dialer": { + Scope: model.Scope{ + Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client"}, + Defaults: model.Variables{"loopTunnelAddress": "tcp:127.0.0.1:15390"}, + }, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-transport-dialer.yml.tmpl", + Mode: zitilab.Loop4RemoteControlled, + }, + }, + }, + }, + "router-east-2": { + Components: model.Components{ + "router-east-2": { + Scope: model.Scope{Tags: model.Tags{"edge-router", "initiator"}}, + Type: &zitilab.RouterType{}, + }, + }, + }, + // Tunneler clients are grouped by stable/restart rather than by flavor. Each host runs one + // Go ziti-tunnel (proxy mode, no :53) and one ZET (tproxy, needs :53). Splitting the ZET + // pair across the two hosts keeps them off each other's :53, and the Go client's proxy mode + // means it never contends for :53 with the co-located ZET. Each tunneler client has a + // co-located loop4 dialer pushing traffic through its client-side data path. + "tunnel-clients-stable": { + Components: model.Components{ + "ziti-tunnel-client-stable": { + Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client", "ziti-tunnel-client", "stable"}}, + Type: &zitilab.ZitiTunnelType{ + Mode: zitilab.ZitiTunnelModeProxy, + ProxyServices: []string{"loop-ziti-tunnel:15387"}, + }, + }, + "loop-zt-dialer-stable": { + Scope: model.Scope{ + Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "stable"}, + Defaults: model.Variables{"loopTunnelAddress": "tcp:127.0.0.1:15387"}, + }, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-transport-dialer.yml.tmpl", + Mode: zitilab.Loop4RemoteControlled, + }, + }, + "ziti-edge-tunnel-client-stable": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "zet", "zet-client", "stable"}}, + Type: &zitilab.ZitiEdgeTunnelType{VerbosityLevel: 3}, + }, + "loop-zet-dialer-stable": { + Scope: model.Scope{ + Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "stable"}, + Defaults: model.Variables{"loopTunnelAddress": "tcp:loop-zet.ziti:15391"}, + }, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-transport-dialer.yml.tmpl", + Mode: zitilab.Loop4RemoteControlled, + }, + }, + }, + }, + "tunnel-clients-restart": { + Components: model.Components{ + "ziti-tunnel-client-restart": { + Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client", "ziti-tunnel-client", "restart"}}, + Type: &zitilab.ZitiTunnelType{ + Mode: zitilab.ZitiTunnelModeProxy, + ProxyServices: []string{"loop-ziti-tunnel:15388"}, + }, + }, + "loop-zt-dialer-restart": { + Scope: model.Scope{ + Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "restart"}, + Defaults: model.Variables{"loopTunnelAddress": "tcp:127.0.0.1:15388"}, + }, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-transport-dialer.yml.tmpl", + Mode: zitilab.Loop4RemoteControlled, + }, + }, + "ziti-edge-tunnel-client-restart": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "zet", "zet-client", "restart"}}, + Type: &zitilab.ZitiEdgeTunnelType{VerbosityLevel: 3}, + }, + "loop-zet-dialer-restart": { + Scope: model.Scope{ + Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "restart"}, + Defaults: model.Variables{"loopTunnelAddress": "tcp:loop-zet.ziti:15391"}, + }, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-transport-dialer.yml.tmpl", + Mode: zitilab.Loop4RemoteControlled, + }, + }, + }, + }, + }, + }, + "us-west-2": { + Region: "us-west-2", + Site: "us-west-2b", + Hosts: model.Hosts{ + // provisioned up front, started only during the add-nodes phase + "ctrl3": { + Components: model.Components{ + "ctrl3": { + Scope: model.Scope{Tags: model.Tags{"ctrl", "ha", "cluster-node"}}, + Type: &zitilab.ControllerType{}, + }, + }, + }, + "router-west": { + Components: model.Components{ + "router-west": { + Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host"}}, + Type: &zitilab.RouterType{}, + }, + // paired Go SDK listeners that bind the loop-sdk service (two terminators) + "loop4-sdk-host-stable": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "loop-sdk-host", "stable"}}, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-sdk-host.yml.tmpl", + Mode: zitilab.Loop4Listener, + }, + }, + "loop4-sdk-host-restart": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "loop-sdk-host", "restart"}}, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-sdk-host.yml.tmpl", + Mode: zitilab.Loop4Listener, + }, + }, + // plain-TCP backend the ERT router forwards loop-ert to (localhost:3456) + "loop4-ert-backend": { + Scope: model.Scope{Tags: model.Tags{"loop-backend"}}, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-backend-host.yml.tmpl", + Mode: zitilab.Loop4Listener, + }, + }, + }, + }, + // The ziti-tunnel and ZET hosts share a box: both run in host mode (no :53 intercept), and + // both forward their loop service to a single local loop4 backend on 127.0.0.1:3456. + "tunnel-hosts": { + Components: model.Components{ + "ziti-tunnel-host-stable": { + Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host", "stable"}}, + Type: &zitilab.ZitiTunnelType{ + Mode: zitilab.ZitiTunnelModeHost, + Verbose: true, + }, + }, + "ziti-tunnel-host-restart": { + Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host", "restart"}}, + Type: &zitilab.ZitiTunnelType{ + Mode: zitilab.ZitiTunnelModeHost, + Verbose: true, + }, + }, + // host mode (run-host) is required here: the default intercept mode would have both + // instances contend for the shared box's tproxy/DNS state, including port 53 + "ziti-edge-tunnel-host-stable": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host", "zet", "stable"}}, + Type: &zitilab.ZitiEdgeTunnelType{ + Mode: zitilab.ZitiEdgeTunnelModeHost, + VerbosityLevel: 3, + }, + }, + "ziti-edge-tunnel-host-restart": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host", "zet", "restart"}}, + Type: &zitilab.ZitiEdgeTunnelType{ + Mode: zitilab.ZitiEdgeTunnelModeHost, + VerbosityLevel: 3, + }, + }, + // single plain-TCP loop4 backend both the ziti-tunnel and ZET hosts forward to + "loop4-tunnel-backend": { + Scope: model.Scope{Tags: model.Tags{"loop-backend"}}, + Type: &zitilab.Loop4SimType{ + ConfigSource: "loop4-backend-host.yml.tmpl", + Mode: zitilab.Loop4Listener, + }, + }, + }, + }, + }, + }, + }, + + Actions: model.ActionBinders{ + "bootstrap": newBootstrapAction(), + "start": newStartAction(), + "stop": model.Bind(component.StopInParallel("*", 15)), + "login": model.Bind(edge.Login("#ctrl1")), + // individual upgrade steps, runnable on their own + "upgradeController": model.BindF(upgradeController), + "upgradeRouters": model.BindF(upgradeRouters), + "restartAndVerifyOidc": model.BindF(restartAndVerifyOidc), + "restartZetWorkaround": model.BindF(restartZetWorkaround), + "upgradeControllerToHa": model.BindF(upgradeControllerToHa), + "addClusterNodes": model.BindF(addClusterNodes), + // the optional second upgrade hop, plus its pieces + "upgradeToNextVersion": model.BindF(upgradeToNextVersion), + "upgradeClusterControllers": model.BindF(upgradeClusterControllers), + "upgradeClusterRouters": model.BindF(upgradeClusterRouters), + // controller memory sampling, runnable on its own against a live instance + "startCtrlMemory": model.BindF(startCtrlMemorySamplers), + "stopCtrlMemory": model.BindF(stopCtrlMemorySamplers), + "reportCtrlMemory": model.BindF(func(run model.Run) error { + return reportCtrlMemory(run, time.Time{}, "iteration") + }), + "ctrlMemorySummary": model.BindF(summarizeCtrlMemory), + // testIteration is the exec-loop entry point (fablab exec-loop testIteration). It runs + // one full pass of the upgrade sequence. It begins by resetting to a fresh standalone fromVersion + // baseline, so each pass is self-contained and repeated iterations (or a re-run after a failed + // pass) start from a known state rather than re-upgrading an already upgraded controller. + "testIteration": model.BindF(func(run model.Run) error { + return run.GetModel().Exec(run, + "resetToBaseline", // fresh standalone fromVersion baseline (undo any prior pass) + "startSimMetrics", + "startCtrlMemory", // per-second controller RSS sampling for the whole pass + "validateSteadyState", // baseline: expected terminators present, clients healthy on legacy + "upgradeController", // ctrl fromVersion -> toVersion (OIDC auto-binds) + "restartZetWorkaround", // old ziti-edge-tunnel can't recover sessions post-migration; restart to re-auth + "validateSteadyStateAfterDisruption", // terminators re-establish, traffic recovers, clients still up + "restartAndVerifyOidc", // restart -restart instances, confirm OIDC on re-auth + "validateSteadyStateAfterDisruption", // everything back and clean + "upgradeRouters", // routers fromVersion -> toVersion, rolling one at a time + "validateSteadyStateAfterDisruption", // routers back on toVersion, terminators + traffic recovered + "upgradeControllerToHa", // ctrl1 standalone -> single-node HA cluster (bbolt -> raft) + "validateSteadyStateAfterDisruption", // cluster up, terminators + traffic recovered + "addClusterNodes", // start ctrl2/ctrl3 and join them to the cluster + "validateSteadyStateAfterDisruption", // 3-node cluster stable, terminators + traffic recovered + "upgradeToNextVersion", // optional: whole system toVersion -> nextVersion + "reportCtrlMemory", // controller memory over the whole pass + ) + }), + }, + + Infrastructure: model.Stages{ + aws_ssh_key.Express(), + &terraformInit.Terraform{ + Retries: 3, + ReadyCheck: &semaphore.ReadyStage{ + MaxWait: 90 * time.Second, + }, + }, + }, + + Distribution: model.Stages{ + // Stop all components before pushing files. A leftover component from a prior run (e.g. a broken + // controller spamming logs after a failed HA phase) can hold a large deleted log file open, + // keeping the host disk full; the ssh-key and rsync steps below then fail on the full disk and + // abort the whole refresh before it can reset anything. Stopping first releases those handles. + model.RunAction("stop"), + distribution.DistributeSshKey("*"), + // pre-stage the upgrade binaries so in-place upgrades need only a version swap + restart + model.StageActionF(toTarget.stage), + model.StageActionF(nextTarget.stage), + rsync.RsyncStaged(), + }, + + Activation: model.Stages{ + model.RunAction("stop"), + model.RunAction("bootstrap"), + model.RunAction("start"), + }, + + Operation: model.Stages{ + model.RunAction("login"), + edge.SyncModelRouterIds(models.EdgeRouterTag), + }, + + Disposal: model.Stages{ + terraform.Dispose(), + awsSshKeyDispose.Dispose(), + }, +} + +func main() { + model.AddBootstrapExtension(binding.AwsCredentialsLoader) + model.AddBootstrapExtension(aws_ssh_key.KeyManager) + + fablab.InitModel(m) + fablab.Run() +} diff --git a/zititest/models/upgrade-test/oidc.go b/zititest/models/upgrade-test/oidc.go new file mode 100644 index 000000000..2ee933524 --- /dev/null +++ b/zititest/models/upgrade-test/oidc.go @@ -0,0 +1,219 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" + "time" + + "github.com/openziti/fablab/kernel/lib/tui" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/chaos" + "github.com/openziti/ziti/zititest/zitilab/cli" +) + +// apiSessionEvent mirrors the fields we care about from the controller's apiSession event +// (controller/event/api_session.go). Type is "legacy" or "jwt"; jwt is an OIDC/token session. +type apiSessionEvent struct { + Namespace string `json:"namespace"` + EventType string `json:"event_type"` + Type string `json:"type"` + IdentityId string `json:"identity_id"` +} + +// restartAndVerifyOidc restarts the -restart instance of each client/host pair (leaving the -stable +// instance up) and reads the controller's apiSession event log to determine the session type each +// restarted client authenticated with. Go SDK clients/hosts are expected to switch to OIDC (jwt) once +// the controller advertises it; ZET (C SDK) behavior is recorded informationally. +func restartAndVerifyOidc(run model.Run) error { + m := run.GetModel() + // all intentional test output routes to the validation pane; channel/library noise falls to actions + log := tui.ValidationLogger() + + sshUser := m.MustStringVariable("credentials.ssh.username") + logPath := fmt.Sprintf("/home/%s/logs/api-sessions.log", sshUser) + ctrl := m.MustSelectHost("component.bootstrap-ctrl") + + restartComps := m.SelectComponents(".restart") + if len(restartComps) == 0 { + return fmt.Errorf("no components tagged 'restart' found") + } + + // refresh the CLI session before querying identities: the login from the controller upgrade can go + // stale across the preceding validation window, and a stale session would make the identity lookup + // fail confusingly rather than cleanly. + if err := edge.Login("#ctrl1").Execute(run); err != nil { + return err + } + + nameToId, err := restartIdentityIds(m) + if err != nil { + return err + } + + // snapshot the log so we only consider events produced by the restart + preCount, err := logLineCount(ctrl, logPath) + if err != nil { + return err + } + + log.Infof("restarting %d -restart instance(s); -stable siblings stay up", len(restartComps)) + if err = chaos.RestartSelected(run, 10, restartComps...); err != nil { + return err + } + + // poll for new api-session created events until every restarted identity is seen or we time out + idToType := map[string]string{} + deadline := time.Now().Add(90 * time.Second) + for { + events, err := newApiSessionEvents(ctrl, logPath, preCount) + if err != nil { + return err + } + for _, e := range events { + if e.Namespace == "apiSession" && e.EventType == "created" && e.IdentityId != "" { + idToType[e.IdentityId] = e.Type + } + } + if allSeen(restartComps, nameToId, idToType) || time.Now().After(deadline) { + break + } + time.Sleep(3 * time.Second) + } + + var problems []string + for _, c := range restartComps { + id, ok := nameToId[c.Id] + if !ok { + problems = append(problems, c.Id+": identity id not found") + continue + } + sessType, seen := idToType[id] + if !seen { + problems = append(problems, c.Id+": no api-session created after restart") + continue + } + if c.HasTag("zet") { + // ZET (C SDK) OIDC support is version-dependent; record but do not fail on it. + log.Infof("OIDC restart check [ZET, informational]: %s -> api-session type=%s", c.Id, sessType) + continue + } + if sessType != apiSessionTypeJwt { + problems = append(problems, fmt.Sprintf("%s: expected jwt (OIDC) after restart, got %s", c.Id, sessType)) + } else { + log.Infof("OIDC restart check: %s switched to OIDC (jwt)", c.Id) + } + } + + if len(problems) > 0 { + return fmt.Errorf("OIDC restart verification failed: %s", strings.Join(problems, "; ")) + } + return nil +} + +const apiSessionTypeJwt = "jwt" + +// allSeen reports whether every restart component's identity has a recorded session type yet. +func allSeen(comps []*model.Component, nameToId, idToType map[string]string) bool { + for _, c := range comps { + id, ok := nameToId[c.Id] + if !ok { + return false + } + if _, seen := idToType[id]; !seen { + return false + } + } + return true +} + +// restartIdentityIds returns a map of component id (== identity name) to identity id for every +// identity whose name ends in "-restart". +func restartIdentityIds(m *model.Model) (map[string]string, error) { + out, err := cli.Exec(m, "edge", "list", "identities", "limit none", "-j") + if err != nil { + return nil, err + } + + var resp struct { + Data []struct { + Id string `json:"id"` + Name string `json:"name"` + } `json:"data"` + } + if err = json.Unmarshal([]byte(out), &resp); err != nil { + return nil, fmt.Errorf("parsing identities list: %w", err) + } + + // the default admin always exists, so an empty list is never legitimate; treat it as a controller + // or session failure rather than letting downstream lookups report "identity id not found". + if len(resp.Data) == 0 { + return nil, fmt.Errorf("edge list identities returned no identities; the default admin should always be present, so this indicates a controller or session problem") + } + + result := map[string]string{} + for _, d := range resp.Data { + if strings.HasSuffix(d.Name, "-restart") { + result[d.Name] = d.Id + } + } + return result, nil +} + +// logLineCount returns the current line count of the given remote log file, or 0 if it is absent. +func logLineCount(host *model.Host, path string) (int, error) { + out, err := host.ExecLogged(fmt.Sprintf("wc -l < %s 2>/dev/null || echo 0", path)) + if err != nil { + return 0, err + } + return strconv.Atoi(strings.TrimSpace(lastToken(out))) +} + +// newApiSessionEvents reads and parses the api-session log lines appended after afterLine. +func newApiSessionEvents(host *model.Host, path string, afterLine int) ([]apiSessionEvent, error) { + out, err := host.ExecLogged(fmt.Sprintf("tail -n +%d %s 2>/dev/null || true", afterLine+1, path)) + if err != nil { + return nil, err + } + var events []apiSessionEvent + for _, line := range strings.Split(out, "\n") { + line = strings.TrimSpace(line) + if !strings.HasPrefix(line, "{") { + continue + } + var e apiSessionEvent + if err = json.Unmarshal([]byte(line), &e); err != nil { + continue + } + events = append(events, e) + } + return events, nil +} + +// lastToken returns the last whitespace-separated token of s, which lets us tolerate any command +// echo or log prefix that precedes a numeric result. +func lastToken(s string) string { + fields := strings.Fields(s) + if len(fields) == 0 { + return "0" + } + return fields[len(fields)-1] +} diff --git a/zititest/models/upgrade-test/start.go b/zititest/models/upgrade-test/start.go new file mode 100644 index 000000000..641bf6b2b --- /dev/null +++ b/zititest/models/upgrade-test/start.go @@ -0,0 +1,59 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "time" + + "github.com/openziti/fablab/kernel/lib/actions" + "github.com/openziti/fablab/kernel/lib/actions/component" + "github.com/openziti/fablab/kernel/lib/actions/semaphore" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/models" +) + +type startAction struct{} + +// newStartAction starts the phase-1 system: the single initial controller, the routers, the +// loop4 hosts/backends, and finally the clients (including the remote-controlled sim client). +func newStartAction() model.ActionBinder { + action := &startAction{} + return action.bind +} + +func (a *startAction) bind(m *model.Model) model.Action { + workflow := actions.Workflow() + + workflow.AddAction(component.Start("#ctrl1")) + workflow.AddAction(edge.ControllerAvailable("#ctrl1", 30*time.Second)) + workflow.AddAction(edge.Login("#ctrl1")) + + workflow.AddAction(component.StartInParallel(models.EdgeRouterTag, 10)) + workflow.AddAction(semaphore.Sleep(2 * time.Second)) + + // hosts: plain-TCP loop4 backends, the SDK loop4 host, and the ZET/ziti-tunnel hosts + workflow.AddAction(component.StartInParallel(".loop-backend", 10)) + workflow.AddAction(component.StartInParallel(".sdk-app.host", 10)) + workflow.AddAction(semaphore.Sleep(2 * time.Second)) + + // clients: ZET client, ziti-tunnel client, and the remote-controlled sim client + workflow.AddAction(component.StartInParallel(".sdk-app.client", 10)) + workflow.AddAction(semaphore.Sleep(2 * time.Second)) + + return workflow +} diff --git a/zititest/models/upgrade-test/steadystate.go b/zititest/models/upgrade-test/steadystate.go new file mode 100644 index 000000000..f8dd6582f --- /dev/null +++ b/zititest/models/upgrade-test/steadystate.go @@ -0,0 +1,389 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + "github.com/openziti/channel/v5" + "github.com/openziti/fablab/kernel/lib/tui" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/v2/controller/rest_client/terminator" + "github.com/openziti/ziti/v2/zitirest" + "github.com/openziti/ziti/zititest/ziti-traffic-test/loop4" + "github.com/openziti/ziti/zititest/zitilab" + "github.com/openziti/ziti/zititest/zitilab/chaos" + zitiLibOps "github.com/openziti/ziti/zititest/zitilab/runlevel/5_operation" + "github.com/openziti/ziti/zititest/zitilab/validations" +) + +// noopControllerCallback satisfies loop4.ControllerCallback without requesting any diagnostics. +type noopControllerCallback struct{} + +func (noopControllerCallback) DiagnosticRequested(*channel.Message, channel.Channel) { + tui.ValidationLogger().Debug("sim diagnostic requested; ignoring") +} + +// expectedTerminator is a service and the number of terminators that must be present for it once the +// system is healthy. +type expectedTerminator struct { + service string + count int64 +} + +// expectedTerminators is the full set of terminators the topology should have when healthy: one loop +// service per hosting flavor plus the two sim control-plane services. It is verified both on initial +// setup and after every disruptive step (each host must re-establish its terminator). +var expectedTerminators = []expectedTerminator{ + {service: "loop-sdk", count: 2}, // loop4-sdk-host-stable/-restart + {service: "loop-ziti-tunnel", count: 2}, // ziti-tunnel-host-stable/-restart + {service: "loop-zet", count: 2}, // ziti-edge-tunnel-host-stable/-restart + {service: "loop-ert", count: 1}, // router-west edge-router tunneler + {service: "sim-control", count: 1}, // sim harness + {service: "metrics", count: 1}, // sim harness +} + +// totalExpectedTerminators returns the sum of all expected terminator counts. +func totalExpectedTerminators() int64 { + var total int64 + for _, e := range expectedTerminators { + total += e.count + } + return total +} + +// steadyStateGate validates that traffic is healthy by driving discrete loop4 scenario runs. +// A run is "clean" when every remote-controlled sim client reports success across every +// hosting flavor. The gate first waits out any churn (recovery), then requires sustained +// clean runs (stability) before it returns. +type steadyStateGate struct { + sim *zitiLibOps.SimServices + + // requiredCleanRuns is how many consecutive clean runs declare the system recovered. + requiredCleanRuns int + // recoveryTimeout bounds how long we tolerate failures before giving up on recovery. + recoveryTimeout time.Duration + // stabilityWindow is how long runs must stay clean once recovered. + stabilityWindow time.Duration + + // postDisruptionDelay is how long to wait after a disruptive step before checking terminators, + // letting the initial churn begin to settle. + postDisruptionDelay time.Duration + // terminatorSettleTimeout bounds how long to wait for all expected terminators to (re-)establish. + terminatorSettleTimeout time.Duration + + connectTimeout time.Duration + scenarioTimeout time.Duration + interRunDelay time.Duration +} + +// newSteadyStateGate returns a gate with default tuning; tuning is refined from model +// variables under the steadyState.* namespace at validation time. +func newSteadyStateGate(sim *zitiLibOps.SimServices) *steadyStateGate { + return &steadyStateGate{ + sim: sim, + requiredCleanRuns: 3, + recoveryTimeout: time.Minute, + stabilityWindow: 2 * time.Minute, + postDisruptionDelay: 30 * time.Second, + terminatorSettleTimeout: time.Minute, + connectTimeout: 60 * time.Second, + scenarioTimeout: 2 * time.Minute, + interRunDelay: time.Second, + } +} + +func (g *steadyStateGate) loadTuning(m *model.Model) { + if v := m.GetStringVariableOr("steadyState.requiredCleanRuns", ""); v != "" { + if n, err := strconv.Atoi(v); err == nil && n > 0 { + g.requiredCleanRuns = n + } + } + if v := m.GetStringVariableOr("steadyState.recoveryTimeout", ""); v != "" { + if d, err := time.ParseDuration(v); err == nil { + g.recoveryTimeout = d + } + } + if v := m.GetStringVariableOr("steadyState.stabilityWindow", ""); v != "" { + if d, err := time.ParseDuration(v); err == nil { + g.stabilityWindow = d + } + } + if v := m.GetStringVariableOr("steadyState.postDisruptionDelay", ""); v != "" { + if d, err := time.ParseDuration(v); err == nil { + g.postDisruptionDelay = d + } + } + if v := m.GetStringVariableOr("steadyState.terminatorSettleTimeout", ""); v != "" { + if d, err := time.ParseDuration(v); err == nil { + g.terminatorSettleTimeout = d + } + } +} + +func (g *steadyStateGate) validate(run model.Run) error { + m := run.GetModel() + g.loadTuning(m) + + // gate output (scenario runs, recovery/stability progress) routes to the TUI validation pane + log := tui.ValidationLogger() + + // ensure the harness is hosting sim-control before we verify terminators; the first call binds it, + // and its sim-control/metrics terminators are part of the expected set checked below. Post-disruption + // this reuses the existing controller, so the terminator must re-establish on its own (SDK recovery). + simControl, err := g.sim.GetSimController(run, "sim-control", noopControllerCallback{}) + if err != nil { + return err + } + + // every expected terminator must be present and valid before we drive traffic; on initial setup + // this confirms the topology, after a disruption it confirms every host has re-established. + if err = g.verifyTerminators(run); err != nil { + return err + } + + sims := m.FilterComponents(".loop-client", func(c *model.Component) bool { + t, ok := c.Type.(*zitilab.Loop4SimType) + return ok && t.Mode == zitilab.Loop4RemoteControlled + }) + if len(sims) == 0 { + return fmt.Errorf("no remote-controlled loop clients found") + } + + log.Infof("steady-state gate: waiting up to %s for %d sim client(s) to connect", g.connectTimeout, len(sims)) + if err = simControl.WaitForAllConnected(g.connectTimeout, sims); err != nil { + return err + } + + // recovery: keep running until we string together enough clean runs, or time out + deadline := time.Now().Add(g.recoveryTimeout) + consecutive := 0 + for consecutive < g.requiredCleanRuns { + if time.Now().After(deadline) { + return fmt.Errorf("did not reach %d consecutive clean runs within %s", g.requiredCleanRuns, g.recoveryTimeout) + } + if err = g.runScenario(simControl, sims); err != nil { + log.WithError(err).Warnf("scenario run failed during recovery; resetting clean-run count from %d", consecutive) + consecutive = 0 + time.Sleep(g.interRunDelay) + continue + } + consecutive++ + log.Infof("steady-state gate recovery: %d/%d consecutive clean runs", consecutive, g.requiredCleanRuns) + } + + // stability: any failure in the window fails the gate + log.Infof("steady-state gate: recovered; requiring clean runs for %s", g.stabilityWindow) + stabilityEnd := time.Now().Add(g.stabilityWindow) + for time.Now().Before(stabilityEnd) { + if err = g.runScenario(simControl, sims); err != nil { + return fmt.Errorf("stability window failed: %w", err) + } + time.Sleep(g.interRunDelay) + } + + log.Info("steady-state gate: passed") + return nil +} + +// runScenario fires a single scenario across all sim clients and returns an error unless +// every client reports success. A disconnected client is treated as a failure. +func (g *steadyStateGate) runScenario(simControl *loop4.RemoteController, sims []*model.Component) error { + if missing := simControl.MissingComponents(sims); len(missing) > 0 { + return fmt.Errorf("sim clients not connected: %v", missing) + } + results, err := simControl.StartSimScenarios() + if err != nil { + return err + } + return results.GetResults(g.scenarioTimeout) +} + +// validateAfterDisruption is the gate variant used after a disruptive step (controller/router +// restart). It waits for the churn to begin settling before running the normal gate, which then +// waits for every expected terminator to re-establish. +func (g *steadyStateGate) validateAfterDisruption(run model.Run) error { + m := run.GetModel() + g.loadTuning(m) + log := tui.ValidationLogger() + log.Infof("post-disruption settle: waiting %s before verifying terminators", g.postDisruptionDelay) + time.Sleep(g.postDisruptionDelay) + + // Pre-2.0 routers have a bug where, when the controller upgrade purges their legacy sessions, they + // drop the affected hosted terminators locally without telling the controller, leaving stale + // terminators the router no longer hosts. Proactively reconcile them (delete the ones routers no + // longer host) so the strict validation reflects reality. Gate on the routers' *current* version, + // not fromVersion: once every router is upgraded to 2.0+ this must stop running so genuine 2.0+ + // terminator bugs are caught rather than masked. + if anyPreV2Router(m) { + if err := g.reconcileStaleTerminators(run); err != nil { + return err + } + } + + return g.validate(run) +} + +// reconcileStaleTerminators deletes edge terminators the routers no longer host (the pre-2.0 upgrade +// leaves these behind). It is scoped to edge terminators: an unscoped fix would inspect ERT terminators +// on the routers, which panics pre-2.0 routers that host none. +func (g *steadyStateGate) reconcileStaleTerminators(run model.Run) error { + log := tui.ValidationLogger() + ctrl := run.GetModel().MustSelectComponent("#ctrl1") + clients, err := chaos.EnsureLoggedIntoCtrl(run, ctrl, time.Minute) + if err != nil { + return fmt.Errorf("unable to log into #ctrl1 to reconcile stale terminators: %w", err) + } + fixed, err := validations.FixInvalidTerminators(clients, `binding="edge" limit none`, g.terminatorSettleTimeout) + if err != nil { + return fmt.Errorf("failed to reconcile stale terminators: %w", err) + } + log.Infof("reconciled stale terminators (pre-2.0 upgrade workaround): %d fixed", fixed) + return nil +} + +// isPreV2 reports whether version is a pre-2.0 release (e.g. v1.x). Empty or unparsable versions are +// treated as not pre-2.0, so the stale-terminator workaround stays off unless we know it is needed. +func isPreV2(version string) bool { + major, _, _ := strings.Cut(strings.TrimPrefix(strings.TrimSpace(version), "v"), ".") + n, err := strconv.Atoi(major) + if err != nil { + return false + } + return n < 2 +} + +// anyPreV2Router reports whether any router component is still on a pre-2.0 version. The +// stale-terminator reconciliation only applies to pre-2.0 router behavior, so it must stop once +// every router has been upgraded, otherwise it masks genuine 2.0+ terminator bugs. +func anyPreV2Router(m *model.Model) bool { + result := false + _ = m.ForEachComponent("*", 1, func(c *model.Component) error { + if rt, ok := c.Type.(*zitilab.RouterType); ok && isPreV2(rt.Version) { + result = true + } + return nil + }) + return result +} + +// verifyTerminators waits until every expected terminator is present for its service, then confirms +// the router SDK and ERT terminators are valid all the way up the stack. It bounds the wait by +// terminatorSettleTimeout so a host that never re-establishes fails the gate. +func (g *steadyStateGate) verifyTerminators(run model.Run) error { + log := tui.ValidationLogger() + ctrl := run.GetModel().MustSelectComponent("#ctrl1") + + log.Infof("verifying %d expected terminators are present (timeout %s)", totalExpectedTerminators(), g.terminatorSettleTimeout) + deadline := time.Now().Add(g.terminatorSettleTimeout) + var clients *zitirest.Clients + var lastLog time.Time + for { + if clients == nil { + var err error + if clients, err = chaos.EnsureLoggedIntoCtrl(run, ctrl, time.Minute); err != nil { + if time.Now().After(deadline) { + return fmt.Errorf("unable to log into #ctrl1 to verify terminators: %w", err) + } + time.Sleep(5 * time.Second) + continue + } + } + + missing, err := g.missingTerminators(clients) + if err != nil { + clients = nil + if time.Now().After(deadline) { + return fmt.Errorf("unable to list terminators: %w", err) + } + time.Sleep(5 * time.Second) + continue + } + if len(missing) == 0 { + break + } + if time.Now().After(deadline) { + return fmt.Errorf("expected terminators not present within %s: %v", g.terminatorSettleTimeout, missing) + } + if time.Since(lastLog) > 15*time.Second { + log.Infof("waiting for terminators: %v", missing) + lastLog = time.Now() + } + time.Sleep(5 * time.Second) + } + + // terminators are all present; confirm they're valid up the stack (router <-> controller agree). + // ERT validation is scoped to edge-router tunnelers: inspecting the ERT registry on a non-hosting + // router (e.g. an initiator) panics older (1.6.x) routers, and only ERT hosts have ERT terminators. + log.Info("all expected terminators present; validating them up the stack") + validationDeadline := time.Now().Add(g.terminatorSettleTimeout) + return validations.ValidateTerminatorsForCtrlWithFilters(run, ctrl, validationDeadline, + validations.MinCount(totalExpectedTerminators()), + validations.ValidateSdkTerminators|validations.ValidateErtTerminators, + "limit none", ertRouterFilter(run.GetModel())) +} + +// ertRouterFilter builds a router filter selecting the edge-router tunnelers (the ".ert-host" routers), +// so ERT terminator validation only inspects routers that actually host ERT terminators. If none are +// found it returns a filter that matches no routers. +func ertRouterFilter(m *model.Model) string { + routers := m.SelectComponents(".ert-host") + if len(routers) == 0 { + return `name = "" limit none` + } + names := make([]string, 0, len(routers)) + for _, r := range routers { + names = append(names, fmt.Sprintf("%q", r.Id)) + } + return fmt.Sprintf("name in [%s] limit none", strings.Join(names, ",")) +} + +// missingTerminators returns, for each expected service that is short, a human-readable "service: +// have/want" entry. An empty result means every expected terminator is present. +func (g *steadyStateGate) missingTerminators(clients *zitirest.Clients) ([]string, error) { + var missing []string + for _, e := range expectedTerminators { + count, err := terminatorCountForService(clients, e.service) + if err != nil { + return nil, err + } + if count < e.count { + missing = append(missing, fmt.Sprintf("%s: %d/%d", e.service, count, e.count)) + } + } + return missing, nil +} + +// terminatorCountForService returns the number of terminators the controller has for the named service. +func terminatorCountForService(clients *zitirest.Clients, service string) (int64, error) { + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + + filter := fmt.Sprintf(`service.name="%s" limit 1`, service) + result, err := clients.Fabric.Terminator.ListTerminators(&terminator.ListTerminatorsParams{ + Filter: &filter, + Context: ctx, + }, nil) + if err != nil { + return 0, err + } + return *result.Payload.Meta.Pagination.TotalCount, nil +} diff --git a/zititest/models/upgrade-test/upgrade.go b/zititest/models/upgrade-test/upgrade.go new file mode 100644 index 000000000..fdc4e974c --- /dev/null +++ b/zititest/models/upgrade-test/upgrade.go @@ -0,0 +1,217 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/openziti/fablab/kernel/lib/tui" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/foundation/v2/versions" + "github.com/openziti/ziti/v2/ziti/util" + "github.com/openziti/ziti/zititest/zitilab" + "github.com/openziti/ziti/zititest/zitilab/actions/edge" + "github.com/openziti/ziti/zititest/zitilab/chaos" + "github.com/openziti/ziti/zititest/zitilab/models" + "github.com/openziti/ziti/zititest/zitilab/stageziti" +) + +// envOrModelVar returns the value of envVar when it is set in the environment, otherwise the model +// variable name. The environment wins so a single run can override a value the model hardcodes, and +// an env var set to the empty string is honored as such: that is the only way to switch off a +// non-empty model default without editing the model. +func envOrModelVar(m *model.Model, name, envVar string) string { + if v, found := os.LookupEnv(envVar); found { + return v + } + return m.GetStringVariableOr(name, "") +} + +// upgradeTarget names the model variables and environment overrides that select the ziti binary for +// one of the model's upgrade targets. +type upgradeTarget struct { + // version holds the version label the binary is staged under and, for ref and source builds, the + // version stamped into it. An empty value means the target is not in play. + version string + source string + ref string + sourceEnv string + refEnv string +} + +var ( + // toTarget is the 1.6 -> 2.0 upgrade the model always runs. + toTarget = upgradeTarget{ + version: "toVersion", + source: "toVersionSource", + ref: "toVersionRef", + sourceEnv: "ZITI_TOVERSION_PATH", + refEnv: "ZITI_TOVERSION_REF", + } + // nextTarget is the optional 2.0 -> 2.1 cluster upgrade that follows it. + nextTarget = upgradeTarget{ + version: "nextVersion", + source: "nextVersionSource", + ref: "nextVersionRef", + sourceEnv: "ZITI_NEXTVERSION_PATH", + refEnv: "ZITI_NEXTVERSION_REF", + } +) + +// stage pre-stages the target's ziti binary into the kit so it is rsynced to every host alongside the +// fromVersion binary. In-place upgrades then only require a version swap and a restart, with no +// mid-run binary transfer. A target whose version is unset stages nothing. +// +// The binary is chosen in precedence order, with each environment variable overriding its model +// variable: +// - the source variable: a locally-built ziti binary, copied under the target's version name. Use +// this to run a patched controller/router from a binary you already built. +// - the ref variable: a git ref (branch, tag, or SHA) on openziti/ziti, built from source via +// acquire and staged under the target's version name. Use this to run a fix, or a release that +// does not exist yet, without hand-building. The build targets the hosts' platform, so it works +// from a mac or arm machine too, but it must be a git ref: a release version belongs in the +// version variable. +// - otherwise the released version is downloaded. +func (self upgradeTarget) stage(run model.Run) error { + m := run.GetModel() + version := m.GetStringVariableOr(self.version, "") + if version == "" { + return nil + } + log := tui.ValidationLogger() + target := filepath.Join(run.GetBinDir(), "ziti-"+version) + + c := m.MustSelectComponent("#ctrl1") + + source := envOrModelVar(m, self.source, self.sourceEnv) + if source != "" { + log.Infof("staging local ziti [%s] as the %s binary [%s]", source, self.version, target) + return util.CopyFile(source, target) + } + + ref := envOrModelVar(m, self.ref, self.refEnv) + if ref != "" { + log.Infof("pre-staging %s ziti built from ref %s as [%s]", self.version, ref, target) + return stageziti.StageZitiFromRefOnce(run, c, ref, version, "") + } + + log.Infof("pre-staging %s ziti binary [%s]", self.version, version) + return stageziti.StageZitiOnce(run, c, version, "") +} + +// upgradeController upgrades ctrl1 from fromVersion to toVersion in place: it swaps the controller +// binary and restarts. The controller config is unchanged, so the 2.0 binary auto-binds the edge-oidc +// API and OIDC becomes available with no config change. +func upgradeController(run model.Run) error { + m := run.GetModel() + toVersion := m.MustStringVariable("toVersion") + log := tui.ValidationLogger() + + c := m.MustSelectComponent("#ctrl1") + ctrlType, ok := c.Type.(*zitilab.ControllerType) + if !ok { + return fmt.Errorf("component ctrl1 is not a controller") + } + + log.Infof("upgrading ctrl1 to %s", toVersion) + ctrlType.SetVersion(toVersion) + + // RestartSelected waits for the old process to exit before starting the new one; a plain stop/start + // would race, since Start no-ops while the old (pre-upgrade) process is still shutting down + if err := chaos.RestartSelected(run, 1, c); err != nil { + return err + } + if err := edge.ControllerAvailable("#ctrl1", 60*time.Second).Execute(run); err != nil { + return err + } + return edge.Login("#ctrl1").Execute(run) +} + +// upgradeRouters upgrades the edge routers from fromVersion to toVersion in place. +func upgradeRouters(run model.Run) error { + return upgradeRoutersTo(run, run.GetModel().MustStringVariable("toVersion")) +} + +// upgradeRoutersTo upgrades the edge routers to version in place, one at a time so traffic keeps +// flowing through the others. Each router restart briefly disrupts circuits on that router, which is +// expected; the others carry traffic in the meantime. +func upgradeRoutersTo(run model.Run, version string) error { + m := run.GetModel() + log := tui.ValidationLogger() + + routers := m.SelectComponents(models.EdgeRouterTag) + for _, c := range routers { + rt, ok := c.Type.(*zitilab.RouterType) + if !ok { + continue + } + log.Infof("upgrading router %s to %s", c.Id, version) + rt.SetVersion(version) + if err := chaos.RestartSelected(run, 1, c); err != nil { + return err + } + // give the router time to reconnect and re-establish terminators before moving to the next + time.Sleep(15 * time.Second) + } + return nil +} + +// zetRestartWorkaroundMaxVersion is the newest ziti-edge-tunnel version known to need the +// post-controller-upgrade restart workaround. At or below this version, ziti-edge-tunnel does not +// rebuild its edge sessions after the 1.x->2.x controller upgrade invalidates them (the required JWT +// session migration): it keeps failing dials with "session closed" until its api-session refresh +// (~20 minutes), far outside the steady-state gate window. Restarting it forces a fresh authentication +// and valid sessions. Newer versions get no workaround, so if the gap is not actually fixed there the +// gate still catches it. Only raise this once a newer ziti-edge-tunnel is confirmed to self-recover. +const zetRestartWorkaroundMaxVersion = "v1.18.0" + +// zetNeedsRestartWorkaround reports whether zetVersion is old enough to need the post-upgrade ZET +// restart workaround. An empty or unparsable version is treated as needing it, so an unknown build +// is not silently left in the broken state. +func zetNeedsRestartWorkaround(zetVersion string) bool { + v, err := versions.ParseSemVer(zetVersion) + if err != nil { + return true + } + return v.CompareTo(versions.MustParseSemVer(zetRestartWorkaroundMaxVersion)) <= 0 +} + +// restartZetWorkaround restarts every ziti-edge-tunnel instance after the controller upgrade, but only +// when zetVersion is old enough to need it (see zetRestartWorkaroundMaxVersion). Those versions cannot +// recover their edge sessions on their own after the 1.x->2.x session migration, so the restart forces +// a re-auth and the ZET data path recovers within the gate window instead of after the api-session +// refresh. This deliberately breaks ZET traffic continuity across the controller upgrade, which is the +// known limitation being worked around. +func restartZetWorkaround(run model.Run) error { + m := run.GetModel() + zetVersion := m.GetStringVariableOr("zetVersion", "") + log := tui.ValidationLogger() + + if !zetNeedsRestartWorkaround(zetVersion) { + log.Infof("ziti-edge-tunnel %s is newer than %s; skipping post-upgrade ZET restart workaround", + zetVersion, zetRestartWorkaroundMaxVersion) + return nil + } + + zets := m.SelectComponents(".zet") + log.Infof("ziti-edge-tunnel %s needs the post-2.0-migration restart workaround; restarting %d ZET instance(s)", + zetVersion, len(zets)) + return chaos.RestartSelected(run, 10, zets...) +} diff --git a/zititest/ziti-traffic-test/loop4/dialer.go b/zititest/ziti-traffic-test/loop4/dialer.go index eac631f1e..0e7d78b4f 100644 --- a/zititest/ziti-traffic-test/loop4/dialer.go +++ b/zititest/ziti-traffic-test/loop4/dialer.go @@ -154,6 +154,8 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res workloadStart := time.Now() var result *Result + var lastDialErr error + succeeded := false for i := int64(0); i < workload.Iterations || workload.Iterations == -1; i++ { log = log.WithField("iteration", i+1) if conn != nil { @@ -170,6 +172,7 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res if err != nil { log.WithError(err).Error("failed to dial") connectFailures.Mark(1) + lastDialErr = err continue } @@ -233,6 +236,7 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res active.Add(-1) completed.Mark(1) + succeeded = true log.Debug("completed iteration") if scenario.ConnectionDelay > 0 { time.Sleep(time.Duration(sim.scenario.ConnectionDelay) * time.Millisecond) @@ -248,6 +252,17 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res sim.metricsReporter.Flush() } + // A workload that completed no successful iteration is a failure, even when every failure was a + // dial failure (e.g. connection refused). Dial failures are otherwise swallowed by the retry + // loop above, so without this a service that can't be reached at all would report success. + if !succeeded { + if lastDialErr == nil { + lastDialErr = fmt.Errorf("workload %s completed no successful iterations", workload.Name) + } + sim.reportErr(resultCh, lastDialErr, "unknown") + return + } + resultCh <- &Result{ Success: true, } diff --git a/zititest/ziti-traffic-test/loop4/remoteControlled.go b/zititest/ziti-traffic-test/loop4/remoteControlled.go index 5a5d7c751..0aa332bed 100644 --- a/zititest/ziti-traffic-test/loop4/remoteControlled.go +++ b/zititest/ziti-traffic-test/loop4/remoteControlled.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" "net" + "sync" "time" "github.com/michaelquigley/pfxlog" @@ -41,15 +42,122 @@ func init() { loop4Cmd.AddCommand(newRemoteControlledCmd()) } +// scenarioResultSendTimeout bounds how long a finished scenario keeps trying to report its result +// while the sim is reconnecting. Past it the controller's own scenario timeout takes over. +const scenarioResultSendTimeout = 60 * time.Second + +// completedScenarioCacheSize is how many finished scenarios are remembered for replay. The controller +// runs one scenario at a time, so a resend more than a handful of scenarios stale cannot occur. +const completedScenarioCacheSize = 16 + type remoteControlledCmd struct { *Sim notifyClose chan struct{} + + // controlCh is the live channel to the sim controller. A reconnect replaces it, so anything + // reporting to the controller must read it at send time rather than capture it: a scenario often + // outlives the channel its run request arrived on. + controlCh concurrenz.AtomicValue[channel.Channel] + + scenarioLock sync.Mutex + // runningScenarioId is the scenario currently being run, empty when idle. + runningScenarioId string + // completed holds recent scenario outcomes for replay, oldest first in completedIds. + completed map[string]scenarioOutcome + completedIds []string +} + +// scenarioOutcome is a scenario run's reportable result. +type scenarioOutcome struct { + success bool + message string +} + +// scenarioAction is what to do with an incoming run request. +type scenarioAction int + +const ( + // actionRun means the sim is claimed for the scenario and the caller should run it. + actionRun scenarioAction = iota + // actionIgnore means this scenario is already running and will report on its own. + actionIgnore + // actionReplay means this scenario already finished; answer from its recorded outcome. + actionReplay + // actionReject means a different scenario holds the sim. + actionReject +) + +// scenarioDecision resolves one run request. runningId is set for actionReject, outcome for +// actionReplay. +type scenarioDecision struct { + action scenarioAction + runningId string + outcome scenarioOutcome +} + +// decideScenario resolves a run request for scenarioId, claiming the sim when it returns actionRun. +// +// Only one scenario runs at a time: two would reset and record the same sim-wide metrics and drive the +// same workloads, so their results would interfere. +func (cmd *remoteControlledCmd) decideScenario(scenarioId string) scenarioDecision { + cmd.scenarioLock.Lock() + defer cmd.scenarioLock.Unlock() + + if cmd.runningScenarioId == scenarioId { + return scenarioDecision{action: actionIgnore} + } + // Ahead of the busy check on purpose: a resend for a scenario that already finished has to be + // answered from the cache even when a later scenario now holds the sim, or it reruns work the + // controller has already accepted and then fails the scenario that follows it. + if outcome, found := cmd.completed[scenarioId]; found { + return scenarioDecision{action: actionReplay, outcome: outcome} + } + if cmd.runningScenarioId != "" { + // Recorded here, under the lock that saw the conflict, so the rejection is terminal. Left + // unrecorded, a resend of this scenario would find the sim idle and start it, and the run would + // still be going when the controller moved on, rejecting the scenario after it in turn. + outcome := scenarioOutcome{ + success: false, + message: fmt.Sprintf("sim is busy running scenario %s", cmd.runningScenarioId), + } + cmd.recordOutcome(scenarioId, outcome) + return scenarioDecision{action: actionReject, runningId: cmd.runningScenarioId, outcome: outcome} + } + cmd.runningScenarioId = scenarioId + return scenarioDecision{action: actionRun} +} + +// recordOutcome caches scenarioId's terminal outcome, evicting the oldest entry when full. The first +// outcome recorded for a scenario wins. Callers must hold scenarioLock. +func (cmd *remoteControlledCmd) recordOutcome(scenarioId string, outcome scenarioOutcome) { + if _, found := cmd.completed[scenarioId]; found { + return + } + cmd.completed[scenarioId] = outcome + cmd.completedIds = append(cmd.completedIds, scenarioId) + if len(cmd.completedIds) > completedScenarioCacheSize { + delete(cmd.completed, cmd.completedIds[0]) + cmd.completedIds = cmd.completedIds[1:] + } +} + +// finishScenario releases the sim and records scenarioId's outcome, so a resend that crosses with the +// result is replayed rather than run again. +func (cmd *remoteControlledCmd) finishScenario(scenarioId string, outcome scenarioOutcome) { + cmd.scenarioLock.Lock() + defer cmd.scenarioLock.Unlock() + + if cmd.runningScenarioId == scenarioId { + cmd.runningScenarioId = "" + } + cmd.recordOutcome(scenarioId, outcome) } func newRemoteControlledCmd() *cobra.Command { dialer := &remoteControlledCmd{ Sim: NewSim(), notifyClose: make(chan struct{}, 1), + completed: map[string]scenarioOutcome{}, } cmd := &cobra.Command{ @@ -105,6 +213,10 @@ func (cmd *remoteControlledCmd) runRemoteControlled(_ *cobra.Command, args []str if err = cmd.handleRemoteControlConn(sdkClient, conn); err != nil { log.WithError(err).Error("unable to channelize remote controller connection") + // Close the dialed conn before retrying. Otherwise a failed channelize (e.g. a hello + // that times out during controller churn) leaves the edge conn and its fabric circuit + // open with no reader, leaking a circuit on every retry. + _ = conn.Close() time.Sleep(1 * time.Second) attempt++ continue @@ -139,6 +251,9 @@ func (cmd *remoteControlledCmd) handleRemoteControlConn(sdk ziti.Context, conn n } func (cmd *remoteControlledCmd) BindChannel(binding channel.Binding) error { + // Publish the channel before any handler can fire, so a request arriving immediately still has a + // channel to answer on. + cmd.controlCh.Store(binding.GetChannel()) binding.AddReceiveHandlerF(int32(loop4Pb.ContentType_RunScenarioRequestType), cmd.HandleRunScenario) binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { select { @@ -149,35 +264,102 @@ func (cmd *remoteControlledCmd) BindChannel(binding channel.Binding) error { return nil } -func (cmd *remoteControlledCmd) HandleRunScenario(msg *channel.Message, ch channel.Channel) { +// HandleRunScenario starts a scenario on request. The channel the request arrived on is deliberately +// unused: results go to whatever channel is live when the run finishes, which a reconnect may change. +// +// Nothing here sends a result inline. This runs on the channel's receive loop, and sendScenarioResult +// retries for up to a minute, which would stall every read on that channel, heartbeats included, and +// so risk killing the connection the result still has to go out on. The replay and reject paths are +// reached during reconnect and resend handling, which is exactly when that channel is least able to +// absorb it. +func (cmd *remoteControlledCmd) HandleRunScenario(msg *channel.Message, _ channel.Channel) { scenarioId, _ := msg.GetStringHeader(int32(loop4Pb.HeaderType_ScenarioId)) - go cmd.runRemoteScenario(scenarioId, cmd.scenario, ch) + if scenarioId == "" { + pfxlog.Logger().Error("run scenario request missing scenario id, ignoring") + return + } + log := pfxlog.Logger().WithField("scenarioId", scenarioId) + + switch decision := cmd.decideScenario(scenarioId); decision.action { + case actionIgnore: + // A re-send after this sim reconnected. The run already in progress reports for it, over + // whatever channel is live when it finishes. + log.Info("scenario already running, ignoring duplicate run request") + return + case actionReplay: + log.Info("scenario already finished, replaying its result") + go cmd.sendScenarioResult(scenarioId, decision.outcome) + return + case actionReject: + // Answer instead of running. Staying silent would leave the controller waiting out its + // scenario timeout for a result that is never coming. + log.WithField("runningScenarioId", decision.runningId). + Info("another scenario is running, rejecting run request") + go cmd.sendScenarioResult(scenarioId, decision.outcome) + return + } + + go func() { + cmd.finishScenario(scenarioId, cmd.runRemoteScenario(scenarioId, cmd.scenario)) + }() } -func (cmd *remoteControlledCmd) sendScenarioResult(ch channel.Channel, id string, success bool, result string) { +// sendScenarioResult reports a scenario's outcome, retrying over whatever control channel is current +// while the sim reconnects. +// +// A scenario routinely outlives the channel its run request arrived on: the controller replaces that +// channel when the sim reconnects and closes the old one. Reporting to the original channel would put +// the result nowhere, leaving the controller to wait out its scenario timeout for a run that had in +// fact finished. +func (cmd *remoteControlledCmd) sendScenarioResult(id string, outcome scenarioOutcome) { log := pfxlog.Logger().WithField("scenarioId", id) - msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioResultType), []byte(result)) - msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), id) - msg.PutBoolHeader(int32(loop4Pb.HeaderType_ScenarioSuccess), success) - if err := msg.WithTimeout(10 * time.Second).Send(ch); err != nil { - log.WithError(err).Error("unable to send scenario run result message") - } else { - log.Info("scenario result successfully reported") + deadline := time.Now().Add(scenarioResultSendTimeout) + for { + msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioResultType), []byte(outcome.message)) + msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), id) + msg.PutBoolHeader(int32(loop4Pb.HeaderType_ScenarioSuccess), outcome.success) + + err := cmd.sendToController(msg) + if err == nil { + log.Info("scenario result successfully reported") + return + } + if time.Now().After(deadline) { + log.WithError(err).Errorf("giving up reporting scenario result after %s", scenarioResultSendTimeout) + return + } + log.WithError(err).Info("unable to report scenario result, retrying on the current channel") + time.Sleep(time.Second) } } -func (cmd *remoteControlledCmd) sendDiagnosticRequest(ch channel.Channel, requestId string) { +func (cmd *remoteControlledCmd) sendDiagnosticRequest(requestId string) { log := pfxlog.Logger().WithField("requestId", requestId) msg := channel.NewMessage(int32(loop4Pb.ContentType_RequestDiagnostic), nil) msg.PutStringHeader(int32(loop4Pb.HeaderType_RequestIdHeader), requestId) - if err := msg.WithTimeout(10 * time.Second).Send(ch); err != nil { + if err := cmd.sendToController(msg); err != nil { log.WithError(err).Error("unable to send diagnostic request message") } else { log.Info("diagnostic successfully requested") } } +// sendToController sends msg over the control channel that is live now, rather than one captured +// earlier, which a reconnect may since have replaced. +func (cmd *remoteControlledCmd) sendToController(msg *channel.Message) error { + ch := cmd.controlCh.Load() + if ch == nil { + return errors.New("no control channel established") + } + if ch.IsClosed() { + return errors.New("control channel is closed") + } + // Wait for the wire, not just the send queue: a queued result on a channel that then dies would be + // reported as delivered, and the scenario retired as reported. + return msg.WithTimeout(10 * time.Second).SendAndWaitForWire(ch) +} + var triggerInspectAtomic concurrenz.AtomicValue[func(circuitId string)] func triggerInspect(circuitId string) { @@ -189,11 +371,11 @@ func triggerInspect(circuitId string) { cb(circuitId) } -func (cmd *remoteControlledCmd) runRemoteScenario(scenarioId string, scenario *Scenario, ch channel.Channel) { +func (cmd *remoteControlledCmd) runRemoteScenario(scenarioId string, scenario *Scenario) scenarioOutcome { log := pfxlog.Logger() triggerInspectAtomic.Store(func(circuitId string) { - cmd.sendDiagnosticRequest(ch, circuitId) + cmd.sendDiagnosticRequest(circuitId) }) // reset metrics @@ -209,17 +391,16 @@ func (cmd *remoteControlledCmd) runRemoteScenario(scenarioId string, scenario *S err := cmd.runScenario(scenario) - runSucceeded := true - resultMsg := "success" + outcome := scenarioOutcome{success: true, message: "success"} if err != nil { - runSucceeded = false - resultMsg = err.Error() + outcome = scenarioOutcome{success: false, message: err.Error()} log.WithError(err).Errorf("scenario run unsuccessful") } else { log.Info("scenario run successful") } - cmd.sendScenarioResult(ch, scenarioId, runSucceeded, resultMsg) + cmd.sendScenarioResult(scenarioId, outcome) + return outcome } func GetSdkIdentity(sdk ziti.Context) (*identity.TokenId, error) { diff --git a/zititest/ziti-traffic-test/loop4/remoteController.go b/zititest/ziti-traffic-test/loop4/remoteController.go index 55442d3fa..57ea2850a 100644 --- a/zititest/ziti-traffic-test/loop4/remoteController.go +++ b/zititest/ziti-traffic-test/loop4/remoteController.go @@ -21,6 +21,7 @@ import ( "fmt" "net" "strings" + "sync" "sync/atomic" "time" @@ -54,6 +55,11 @@ type RemoteController struct { closed atomic.Bool resultsTracker cmap.ConcurrentMap[string, *ScenarioResults] + + // dispatchLock serializes dispatching a new scenario against re-sending in-flight ones to a + // reconnecting sim, so a scenario is never re-sent while its dispatch is still deciding whether it + // will be abandoned. + dispatchLock sync.Mutex } func (self *RemoteController) AcceptConnections(service string) error { @@ -97,8 +103,11 @@ func (self *RemoteController) Close() error { } func (self *RemoteController) handleConnection(conn net.Conn) error { + // handleConnection owns conn: on success it is owned by the channel wrapping it (whose Close closes + // it), so every error path must close it here or the accepted conn and its circuit leak. tokenId, err := GetSdkIdentity(self.client) if err != nil { + _ = conn.Close() return err } listener := channel.NewExistingConnListener(tokenId, conn, nil) @@ -107,17 +116,67 @@ func (self *RemoteController) handleConnection(conn net.Conn) error { var ch channel.Channel ch, err = channel.NewSingleChannel("control", listener, channel.BindHandlerF(self.BindChannel), options) if err != nil { + _ = conn.Close() return fmt.Errorf("unable to establish connection from sim (%w)", err) } clientId := string(ch.Headers()[HeaderClientId]) - self.clients.Set(clientId, ch) + // Install the new channel and capture any channel it supersedes. A reconnecting sim opens a new + // channel while its old one is still registered; without closing the old one it (and its underlying + // edge conn and fabric circuit) leaks, since nothing else ever closes it. A leaked circuit stays + // valid to the fabric forever, so the peer never receives a close and can block on it indefinitely. + // The old channel must be closed after Upsert returns, not inside the callback: Upsert holds the + // shard lock across the callback, and Close runs the close handler synchronously, which calls + // RemoveCb and would deadlock trying to re-acquire that same lock. + var superseded channel.Channel + self.clients.Upsert(clientId, ch, func(exists bool, oldCh channel.Channel, newCh channel.Channel) channel.Channel { + if exists && oldCh != nil && oldCh != newCh { + superseded = oldCh + } + return newCh + }) + if superseded != nil { + _ = superseded.Close() + } pfxlog.Logger().WithField("id", clientId).Info("new sim connection established") + self.resendPendingScenarios(clientId, ch) + return nil } +// resendPendingScenarios re-sends the run request for every in-flight scenario that still expects a +// result from this client. A sim that reconnects mid-scenario gets a fresh channel that never received +// the original request; without re-sending, its result would never arrive and the scenario would block +// until it timed out. +// +// It takes dispatchLock so it observes a scenario only once dispatch has settled. Reading the state +// mid-dispatch would let a reconnect re-send a scenario that the still-running dispatch is about to +// abandon, leaving the sim running a scenario nobody awaits while the caller starts its replacement. +func (self *RemoteController) resendPendingScenarios(clientId string, ch channel.Channel) { + self.dispatchLock.Lock() + defer self.dispatchLock.Unlock() + + for _, results := range self.resultsTracker.Items() { + if results.done.Load() || results.completed.Load() { + continue + } + if _, expected := results.expected[clientId]; !expected { + continue + } + if results.results.Has(clientId) { + continue + } + log := pfxlog.Logger().WithField("scenarioId", results.id).WithField("clientId", clientId) + if err := self.sendScenarioRequest(results.id, ch); err != nil { + log.WithError(err).Error("failed to re-send scenario request to reconnected sim") + } else { + log.Info("re-sent scenario request to reconnected sim") + } + } +} + func (self *RemoteController) BindChannel(binding channel.Binding) error { binding.AddReceiveHandlerF(int32(loop4Pb.ContentType_RunScenarioResultType), self.handleScenarioResult) if self.cb != nil { @@ -125,8 +184,17 @@ func (self *RemoteController) BindChannel(binding channel.Binding) error { } binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { clientId := string(ch.Headers()[HeaderClientId]) - pfxlog.Logger().WithField("id", clientId).Info("sim client channel closed, removing from clients map") - self.clients.Remove(clientId) + // Only remove the entry if it still points at this channel. A reconnecting sim registers + // its new channel before the old channel's close handler runs; removing unconditionally + // would evict the live replacement and make a healthy sim look disconnected. + removed := self.clients.RemoveCb(clientId, func(_ string, v channel.Channel, _ bool) bool { + return v == ch + }) + if removed { + pfxlog.Logger().WithField("id", clientId).Info("sim client channel closed, removed from clients map") + } else { + pfxlog.Logger().WithField("id", clientId).Info("stale sim channel closed; superseded by newer connection, not removing") + } })) return nil } @@ -138,12 +206,23 @@ func (self *RemoteController) handleScenarioResult(msg *channel.Message, ch chan } else { results, _ := self.resultsTracker.Get(id) if results == nil { - pfxlog.Logger().Errorf("scenario result message for scenario id [%s] received, but no results tracker found", id) + // Expected for a result that arrives after its scenario was retired, which a client that + // reported late or reconnected after the fact will produce. + pfxlog.Logger().WithField("scenarioId", id). + Info("ignoring scenario result for a scenario no longer being tracked") return } clientId := string(ch.Headers()[HeaderClientId]) + // Ignore results from clients that were not part of this scenario's dispatch set (e.g. a sim that + // connected after the scenario started). Counting them would corrupt the completion check. + if _, expected := results.expected[clientId]; !expected { + pfxlog.Logger().WithField("scenarioId", id).WithField("clientId", clientId). + Info("ignoring scenario result from client not in scenario's expected set") + return + } + success, _ := msg.GetBoolHeader(int32(loop4Pb.HeaderType_ScenarioSuccess)) pfxlog.Logger(). @@ -157,7 +236,7 @@ func (self *RemoteController) handleScenarioResult(msg *channel.Message, ch chan message: string(msg.Body), } results.results.Set(clientId, *result) - if results.results.Count() == results.expectedResults { + if results.results.Count() == len(results.expected) { if results.completed.CompareAndSwap(false, true) { close(results.complete) } @@ -193,45 +272,88 @@ func (self *RemoteController) MissingComponents(components []*model.Component) [ return result } +// StartSimScenarios dispatches a new scenario run request to every currently connected sim and returns +// the tracker to await its results on. Dispatch holds dispatchLock so a sim reconnecting partway +// through cannot observe (and re-send) a scenario whose dispatch is still in progress. func (self *RemoteController) StartSimScenarios() (*ScenarioResults, error) { + self.dispatchLock.Lock() + defer self.dispatchLock.Unlock() + scenarioId := uuid.NewString() log := pfxlog.Logger().WithField("scenarioId", scenarioId) + clients := self.clients.Items() + expected := make(map[string]struct{}, len(clients)) + for clientId := range clients { + expected[clientId] = struct{}{} + } + results := &ScenarioResults{ - id: scenarioId, - results: cmap.New[ScenarioResult](), - complete: make(chan struct{}), - expectedResults: self.clients.Count(), + controller: self, + id: scenarioId, + results: cmap.New[ScenarioResult](), + complete: make(chan struct{}), + expected: expected, } self.resultsTracker.Set(scenarioId, results) - for _, client := range self.clients.Items() { - msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioRequestType), nil) - msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), scenarioId) - if err := msg.WithTimeout(10 * time.Second).SendAndWaitForWire(client); err != nil { - return nil, err + for clientId, client := range clients { + if err := self.sendScenarioRequest(scenarioId, client); err != nil { + // The tracker is already published, but the caller gets no handle to retire it, so do it here. + // Left live, a sim reconnecting later would have this scenario re-sent to it and run it + // alongside whatever scenario the caller starts in place of this failed one. + results.retire() + return nil, fmt.Errorf("failed to send scenario request to %s: %w", clientId, err) } - log.WithField("clientId", client.Id()).Info("scenario run request sent") + log.WithField("clientId", clientId).Info("scenario run request sent") } return results, nil } +// sendScenarioRequest sends one run-scenario request for the given scenario to a single sim channel. +func (self *RemoteController) sendScenarioRequest(scenarioId string, client channel.Channel) error { + msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioRequestType), nil) + msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), scenarioId) + return msg.WithTimeout(10 * time.Second).SendAndWaitForWire(client) +} + type ScenarioResult struct { success bool message string } type ScenarioResults struct { - id string - results cmap.ConcurrentMap[string, ScenarioResult] - complete chan struct{} - completed atomic.Bool - expectedResults int + // controller owns the tracker this scenario is registered in, so retire can drop it. + controller *RemoteController + + id string + results cmap.ConcurrentMap[string, ScenarioResult] + complete chan struct{} + completed atomic.Bool + // expected holds the client ids the scenario was dispatched to. It is populated before the results + // tracker is published and never mutated afterward, so it is safe for concurrent reads. + expected map[string]struct{} + // done marks the scenario as no longer awaited. retire clears the tracker entry as well, but a + // resend scan already walking a snapshot of the tracker can still see the entry, and this flag is + // what stops it re-sending a scenario nobody is waiting on. + done atomic.Bool +} + +// retire marks the scenario as no longer awaited and drops it from the tracker. Every scenario ends +// here, whether its results arrived, its wait timed out, or its dispatch failed part way. Without it +// the tracker only ever grows: each entry holds a sharded result map and its expected-client set for +// the life of the process, and every sim reconnect re-walks all of them. +// +// The flag is set before the removal so a resend scan holding an older snapshot still skips it. +func (self *ScenarioResults) retire() { + self.done.Store(true) + self.controller.resultsTracker.Remove(self.id) } func (self *ScenarioResults) GetResults(timeout time.Duration) error { + defer self.retire() start := time.Now() var err error select { diff --git a/zititest/zitilab/component_controller.go b/zititest/zitilab/component_controller.go index 257ec7d21..aef1dbdd4 100644 --- a/zititest/zitilab/component_controller.go +++ b/zititest/zitilab/component_controller.go @@ -44,9 +44,12 @@ type ControllerType struct { ConfigSource string ConfigName string Version string - LocalPath string - DNSNames []string - Debug bool + // SourceRef, when set, is a git ref (branch, tag, or SHA) on openziti/ziti built from source and + // stamped as Version, instead of downloading the Version release. Use it to run an unreleased build. + SourceRef string + LocalPath string + DNSNames []string + Debug bool LogConfig } @@ -115,7 +118,7 @@ func (self *ControllerType) StageFiles(r model.Run, c *model.Component) error { return err } - if err := stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath); err != nil { + if err := stageziti.StageZitiForComponentOnce(r, c, self.Version, self.SourceRef, self.LocalPath); err != nil { return err } diff --git a/zititest/zitilab/component_loop4_sim.go b/zititest/zitilab/component_loop4_sim.go index fe17be134..ee9f00b57 100644 --- a/zititest/zitilab/component_loop4_sim.go +++ b/zititest/zitilab/component_loop4_sim.go @@ -96,9 +96,14 @@ func (self *Loop4SimType) GetConfigName(c *model.Component) string { return configName } -func (self *Loop4SimType) getProcessFilter() func(string) bool { +func (self *Loop4SimType) getProcessFilter(c *model.Component) func(string) bool { + // Match on the component's config file (which contains the component id) in addition to the + // binary name, so multiple loop4 instances can run on the same host and still be identified + // individually. Single-instance hosts continue to match, since the running process always + // includes its own config path. + configFile := fmt.Sprintf("%s.yml", c.Id) return func(s string) bool { - return strings.Contains(s, "ziti-traffic-test") + return strings.Contains(s, "ziti-traffic-test") && strings.Contains(s, configFile) } } @@ -107,7 +112,7 @@ func (self *Loop4SimType) GetConfigPath(c *model.Component) string { } func (self *Loop4SimType) IsRunning(_ model.Run, c *model.Component) (bool, error) { - pids, err := c.GetHost().FindProcesses(self.getProcessFilter()) + pids, err := c.GetHost().FindProcesses(self.getProcessFilter(c)) if err != nil { return false, err } @@ -139,5 +144,5 @@ func (self *Loop4SimType) Start(_ model.Run, c *model.Component) error { } func (self *Loop4SimType) Stop(_ model.Run, c *model.Component) error { - return c.GetHost().KillProcesses("-TERM", self.getProcessFilter()) + return c.GetHost().KillProcesses("-TERM", self.getProcessFilter(c)) } diff --git a/zititest/zitilab/component_router.go b/zititest/zitilab/component_router.go index 6ed55bdd4..6f7577ff4 100644 --- a/zititest/zitilab/component_router.go +++ b/zititest/zitilab/component_router.go @@ -44,8 +44,11 @@ const ( ) type RouterType struct { - Configs map[string]Config - Version string + Configs map[string]Config + Version string + // SourceRef, when set, is a git ref (branch, tag, or SHA) on openziti/ziti built from source and + // stamped as Version, instead of downloading the Version release. Use it to run an unreleased build. + SourceRef string LocalPath string Debug bool LogConfig @@ -105,7 +108,7 @@ func (self *RouterType) StageFiles(r model.Run, c *model.Component) error { } } - if err := stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath); err != nil { + if err := stageziti.StageZitiForComponentOnce(r, c, self.Version, self.SourceRef, self.LocalPath); err != nil { return err } diff --git a/zititest/zitilab/component_ziti_edge_tunnel.go b/zititest/zitilab/component_ziti_edge_tunnel.go index da30b9b08..0af815b6f 100644 --- a/zititest/zitilab/component_ziti_edge_tunnel.go +++ b/zititest/zitilab/component_ziti_edge_tunnel.go @@ -105,7 +105,11 @@ func (self *ZitiEdgeTunnelType) StageFiles(r model.Run, c *model.Component) erro func (self *ZitiEdgeTunnelType) getProcessFilter(c *model.Component) func(string) bool { return func(s string) bool { - return strings.Contains(s, self.getBinaryName()) && + // Match on the version-agnostic base binary name, not getBinaryName() (which includes the + // version): the component's config file uniquely identifies its process, and matching the + // versioned name would fail to find a running instance after a version change, leaving the + // old-version process alive on stop/restart. + return strings.Contains(s, "ziti-edge-tunnel") && strings.Contains(s, fmt.Sprintf("%s.json", c.Id)) && !strings.Contains(s, "sudo ") } diff --git a/zititest/zitilab/component_ziti_tunnel.go b/zititest/zitilab/component_ziti_tunnel.go index 98e7dc9d5..e7cf05a86 100644 --- a/zititest/zitilab/component_ziti_tunnel.go +++ b/zititest/zitilab/component_ziti_tunnel.go @@ -52,8 +52,11 @@ func (self ZitiTunnelMode) String() string { } type ZitiTunnelType struct { - Mode ZitiTunnelMode - Version string + Mode ZitiTunnelMode + Version string + // SourceRef, when set, is a git ref (branch, tag, or SHA) on openziti/ziti built from source and + // stamped as Version, instead of downloading the Version release. Use it to run an unreleased build. + SourceRef string LocalPath string ConfigPathF func(c *model.Component) string Count uint8 @@ -61,6 +64,9 @@ type ZitiTunnelType struct { ControlRouterConnections uint8 EnableSdkFlowControl bool Verbose bool + // ProxyServices are "service:port" pairs passed to 'ziti tunnel proxy' so the tunneler opens a + // local TCP listener per service. Only used in proxy mode. + ProxyServices []string LogConfig } @@ -111,7 +117,7 @@ func (self *ZitiTunnelType) Dump() any { } func (self *ZitiTunnelType) StageFiles(r model.Run, c *model.Component) error { - if err := stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath); err != nil { + if err := stageziti.StageZitiForComponentOnce(r, c, self.Version, self.SourceRef, self.LocalPath); err != nil { return err } // The rotate log strategy runs "ops log-pipe" from the component's ziti binary (Version); @@ -213,8 +219,13 @@ func (self *ZitiTunnelType) StartIndividual(c *model.Component, idx int) error { redirect := logRedirect(c, logsPath, binaryPath) - serviceCmd := fmt.Sprintf("%s %s tunnel %s %s %s --cli-agent-alias %s --log-formatter json -i %s %s &", - useSudo, binaryPath, mode.String(), connectCfg, verbose, c.Id, configPath, redirect) + proxyServices := "" + if mode == ZitiTunnelModeProxy && len(self.ProxyServices) > 0 { + proxyServices = strings.Join(self.ProxyServices, " ") + } + + serviceCmd := fmt.Sprintf("%s %s tunnel %s %s %s %s --cli-agent-alias %s --log-formatter json -i %s %s &", + useSudo, binaryPath, mode.String(), proxyServices, connectCfg, verbose, c.Id, configPath, redirect) // Only clear prior history for truncate; append and rotate must preserve it. var cmds []string diff --git a/zititest/zitilab/runlevel/5_operation/client_metrics.go b/zititest/zitilab/runlevel/5_operation/client_metrics.go index ebabf6547..f7ece612b 100644 --- a/zititest/zitilab/runlevel/5_operation/client_metrics.go +++ b/zititest/zitilab/runlevel/5_operation/client_metrics.go @@ -18,6 +18,7 @@ package zitilib_runlevel_5_operation import ( "encoding/binary" + "fmt" "io" "net" "strings" @@ -43,6 +44,7 @@ const ( func NewSimServices(hostSelectorF func(string) string) *SimServices { return &SimServices{ idToSelectorMapper: hostSelectorF, + connectTimeout: 60 * time.Second, } } @@ -53,6 +55,9 @@ type SimServices struct { lock sync.Mutex zitiContext ziti.Context metricsStarted atomic.Bool + // connectTimeout bounds establishing the sim-controller ziti context, so an unreachable + // controller fails fast instead of hanging on the SDK's unbounded version-info retry. + connectTimeout time.Duration remoteController *loop4.RemoteController } @@ -90,12 +95,36 @@ func (self *SimServices) GetZitiContext(run model.Run) (ziti.Context, error) { if err != nil { return nil, err } + + // Authenticate up front with a deadline so an unreachable controller fails fast. The SDK's + // version-info caching retries forever with no timeout (openziti/sdk-golang#976), so the first + // Listen/Dial would otherwise block indefinitely and hang the caller (e.g. the steady-state gate). + if err = authenticateWithTimeout(context, self.connectTimeout); err != nil { + context.Close() + return nil, err + } + self.zitiContext = context } return self.zitiContext, nil } +// authenticateWithTimeout runs context.Authenticate() with a deadline so an unreachable controller +// surfaces an error instead of blocking indefinitely on the SDK's unbounded version-info retry. If +// Authenticate is wedged in that retry, its goroutine can outlive the timeout; that is acceptable +// here since the run is failing anyway. +func authenticateWithTimeout(context ziti.Context, timeout time.Duration) error { + done := make(chan error, 1) + go func() { done <- context.Authenticate() }() + select { + case err := <-done: + return err + case <-time.After(timeout): + return fmt.Errorf("timed out after %s establishing sim controller connection; controller unreachable?", timeout) + } +} + func (self *SimServices) CollectSimMetrics(run model.Run, service string) error { if !self.metricsStarted.CompareAndSwap(false, true) { return nil @@ -232,3 +261,26 @@ func (self *SimServices) GetSimController(run model.Run, service string, callbac return self.remoteController, nil } + +// Reset tears down the cached sim-controller context, remote controller, and metrics listener so a +// subsequent GetZitiContext/GetSimController/CollectSimMetrics rebuilds them. Use it after the +// controller has been re-bootstrapped (e.g. resetting a test between iterations), where the cached +// api session and sim-controller enrollment are stale and would otherwise be reused. +func (self *SimServices) Reset() { + self.lock.Lock() + defer self.lock.Unlock() + + if self.remoteController != nil { + _ = self.remoteController.Close() + self.remoteController = nil + } + if self.listener != nil { + _ = self.listener.Close() + self.listener = nil + } + if self.zitiContext != nil { + self.zitiContext.Close() + self.zitiContext = nil + } + self.metricsStarted.Store(false) +} diff --git a/zititest/zitilab/stageziti/stageziti.go b/zititest/zitilab/stageziti/stageziti.go index 56e3442ca..afdc63ae8 100644 --- a/zititest/zitilab/stageziti/stageziti.go +++ b/zititest/zitilab/stageziti/stageziti.go @@ -1,6 +1,7 @@ package stageziti import ( + "context" "fmt" "os" "os/exec" @@ -8,12 +9,20 @@ import ( "strings" "github.com/openziti/fablab/kernel/model" + "github.com/openziti/sdk-golang/acquire" "github.com/openziti/ziti/v2/common/getziti" "github.com/openziti/ziti/v2/ziti/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +// hostGoos and hostGoarch are the platform of the hosts fablab provisions, which is what every +// binary staged into the kit has to be built for, regardless of what the orchestrator runs on. +const ( + hostGoos = "linux" + hostGoarch = "amd64" +) + func StageZitiOnce(run model.Run, component *model.Component, version string, source string) error { op := "install.ziti-" if version == "" { @@ -68,19 +77,81 @@ func StageZitiEdgeTunnelOnce(run model.Run, component *model.Component, version func StageZiti(run model.Run, component *model.Component, version string, source string) error { return StageExecutable(run, "ziti", component, version, source, func() error { - return getziti.InstallZiti(version, "linux", "amd64", run.GetBinDir(), false) + return getziti.InstallZiti(version, hostGoos, hostGoarch, run.GetBinDir(), false) }) } +// StageZitiForComponentOnce stages the ziti binary for a component once per run. When sourceRef is set +// it builds that git ref (branch, tag, or SHA) on openziti/ziti and stamps it as version; otherwise it +// stages localPath or the released version, exactly as StageZitiOnce does. This lets any component +// (controller, router, ziti-tunnel) run an unreleased build while reporting a chosen version. +func StageZitiForComponentOnce(run model.Run, component *model.Component, version, sourceRef, localPath string) error { + if sourceRef != "" { + return StageZitiFromRefOnce(run, component, sourceRef, version, localPath) + } + return StageZitiOnce(run, component, version, localPath) +} + +// StageZitiFromRefOnce is StageZitiFromRef guarded by run.DoOnce, keyed on the ref and stamped version +// so distinct (ref, version) builds each run once while a repeated one is reused. +func StageZitiFromRefOnce(run model.Run, component *model.Component, sourceRef, stampVersion, localPath string) error { + op := fmt.Sprintf("install.ziti-ref-%s-as-%s", sourceRef, stampVersion) + return run.DoOnce(op, func() error { + return StageZitiFromRef(run, component, sourceRef, stampVersion, localPath) + }) +} + +// StageZitiFromRef builds the ziti binary from sourceRef (a git ref on openziti/ziti), stamps it to +// report stampVersion, and stages it under the ziti- name so an in-place version swap can +// pick it up. An empty stampVersion stages it as plain ziti and leaves the build unstamped, which is +// how every other staging path names an unversioned binary and what the component start paths resolve. +// It always delegates to acquire, which resolves the ref to a commit and caches by that immutable id, +// so an unchanged ref reuses the cached build while a changed ref (or a branch moved to a new commit) +// rebuilds. It deliberately does not skip on an existing ziti-: that would key +// reuse on the version name and silently serve a stale binary when the ref changed under the same +// stampVersion. If localPath is set it is staged directly instead of building, so a prebuilt binary +// still wins. GITHUB_TOKEN, when set, raises the ref-resolution rate limit. +// +// The build targets hostGoos/hostGoarch even when the orchestrator runs on something else, so a run +// driven from a mac or an arm box still stages a binary the hosts can execute. +func StageZitiFromRef(run model.Run, component *model.Component, sourceRef, stampVersion, localPath string) error { + fileName := "ziti" + if stampVersion != "" { + fileName += "-" + stampVersion + } + target := filepath.Join(run.GetBinDir(), fileName) + + if localPath != "" { + logrus.Infof("[%s] => [%s]", localPath, target) + return util.CopyFile(localPath, target) + } + + cacheDir, err := acquire.DefaultCacheDir() + if err != nil { + return err + } + cfg := acquire.Versions{Source: acquire.Source{Org: "openziti", Repo: "ziti"}} + src := acquire.NewGitHubReleaseSource(cfg.Source.Org, cfg.Source.Repo, os.Getenv("GITHUB_TOKEN")) + + logrus.Infof("building ziti from ref %s (stamped %s) -> %s", sourceRef, stampVersion, target) + built, id, err := acquire.Ziti(context.Background(), sourceRef, cfg, src, cacheDir, + acquire.WithVersion(stampVersion), acquire.WithPlatform(hostGoos, hostGoarch)) + if err != nil { + return fmt.Errorf("building ziti from ref %q: %w", sourceRef, err) + } + logrus.Infof("built ziti from ref %s (commit %s) stamped %s", sourceRef, id.Tag, stampVersion) + return util.CopyFile(built, target) +} + func StageZrok(run model.Run, component *model.Component, version string, source string) error { return StageExecutable(run, "zrok", component, version, source, func() error { - return getziti.InstallZrok(version, "linux", "amd64", run.GetBinDir(), false) + return getziti.InstallZrok(version, hostGoos, hostGoarch, run.GetBinDir(), false) }) } func StageCaddy(run model.Run, component *model.Component, version string, source string) error { return StageExecutable(run, "caddy", component, version, source, func() error { - return getziti.InstallCaddy(version, "linux", "amd64", run.GetBinDir(), false) + return getziti.InstallCaddy(version, hostGoos, hostGoarch, run.GetBinDir(), false) }) } @@ -175,5 +246,5 @@ func StageZitiEdgeTunnel(run model.Run, component *model.Component, version stri } logrus.Infof("%s not present, attempting to fetch", target) - return getziti.InstallZitiEdgeTunnel(version, "linux", "amd64", run.GetBinDir(), false) + return getziti.InstallZitiEdgeTunnel(version, hostGoos, hostGoarch, run.GetBinDir(), false) } diff --git a/zititest/zitilab/validations/terminators.go b/zititest/zitilab/validations/terminators.go index 9143b9082..3198bfc8f 100644 --- a/zititest/zitilab/validations/terminators.go +++ b/zititest/zitilab/validations/terminators.go @@ -59,6 +59,14 @@ func ValidateTerminators(run model.Run, timeout time.Duration, countOk func(int6 } func ValidateTerminatorsForCtrl(run model.Run, c *model.Component, deadline time.Time, countOk func(int64) bool, validationType TerminatorValidationType) error { + return ValidateTerminatorsForCtrlWithFilters(run, c, deadline, countOk, validationType, "limit none", "limit none") +} + +// ValidateTerminatorsForCtrlWithFilters is like ValidateTerminatorsForCtrl but restricts the SDK and +// ERT terminator validations to the routers matching sdkFilter and ertFilter respectively. Use it to +// skip routers that do not host the terminator type being validated, for example inspecting the ERT +// terminators only on edge-router tunnelers. +func ValidateTerminatorsForCtrlWithFilters(run model.Run, c *model.Component, deadline time.Time, countOk func(int64) bool, validationType TerminatorValidationType, sdkFilter, ertFilter string) error { logger := tui.ValidationLogger().WithField("ctrl", c.Id) var clients *zitirest.Clients @@ -101,10 +109,14 @@ func ValidateTerminatorsForCtrl(run model.Run, c *model.Component, deadline time var validators []validatorEntry if validationType&ValidateSdkTerminators != 0 { - validators = append(validators, validatorEntry{name: "sdk", validate: ValidateRouterSdkTerminators}) + validators = append(validators, validatorEntry{name: "sdk", validate: func(id string, cl *zitirest.Clients) (int, error) { + return ValidateRouterSdkTerminatorsWithFilter(id, cl, sdkFilter) + }}) } if validationType&ValidateErtTerminators != 0 { - validators = append(validators, validatorEntry{name: "ert", validate: ValidateRouterErtTerminators}) + validators = append(validators, validatorEntry{name: "ert", validate: func(id string, cl *zitirest.Clients) (int, error) { + return ValidateRouterErtTerminatorsWithFilter(id, cl, ertFilter) + }}) } for _, v := range validators { @@ -158,7 +170,85 @@ func GetTerminatorCount(clients *zitirest.Clients) (int64, error) { return count, nil } +// FixInvalidTerminators asks the controller to validate the terminators matching filter against their +// routers and delete the ones the routers no longer host, returning the number fixed (deleted). Scope +// the filter (e.g. binding="edge") to the terminator types whose routers can be inspected safely. +// timeout bounds how long to wait for the per-terminator results after validation starts. +func FixInvalidTerminators(clients *zitirest.Clients, filter string, timeout time.Duration) (int, error) { + logger := tui.ValidationLogger() + + closeNotify := make(chan struct{}) + eventNotify := make(chan *mgmt_pb.TerminatorDetail, 16) + + bindHandler := func(binding channel.Binding) error { + binding.AddReceiveHandlerF(int32(mgmt_pb.ContentType_ValidateTerminatorResultType), func(msg *channel.Message, _ channel.Channel) { + detail := &mgmt_pb.TerminatorDetail{} + if err := proto.Unmarshal(msg.Body, detail); err != nil { + pfxlog.Logger().WithError(err).Error("unable to unmarshal terminator detail") + return + } + eventNotify <- detail + }) + binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) { + close(closeNotify) + })) + return nil + } + + ch, err := clients.NewWsMgmtChannel(channel.BindHandlerF(bindHandler)) + if err != nil { + return 0, err + } + defer func() { + _ = ch.Close() + }() + + request := &mgmt_pb.ValidateTerminatorsRequest{ + TerminatorsFilter: filter, + FixInvalid: true, + } + responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(30 * time.Second).SendForReply(ch) + + response := &mgmt_pb.ValidateTerminatorsResponse{} + if err = protobufs.TypedResponse(response).Unmarshall(responseMsg, err); err != nil { + return 0, err + } + + if !response.Success { + return 0, fmt.Errorf("failed to start terminator validation: %s", response.Message) + } + + // A router that stops answering mid-validation lets the controller-side request time out without + // emitting a detail or closing this channel, while the response count still includes its + // terminators, so bound the wait instead of blocking forever on details that will never arrive. + expiredC := time.After(timeout) + + fixed := 0 + for expected := response.TerminatorCount; expected > 0; expected-- { + select { + case <-closeNotify: + return fixed, errors.New("unexpected close of mgmt channel during terminator fix") + case <-expiredC: + return fixed, fmt.Errorf("timed out after %s waiting for terminator details, %d of %d not received", + timeout, expected, response.TerminatorCount) + case detail := <-eventNotify: + if detail.Fixed { + fixed++ + logger.Infof("fixed invalid terminator %s (service=%s router=%s state=%s)", + detail.TerminatorId, detail.ServiceName, detail.RouterName, detail.State) + } + } + } + return fixed, nil +} + func ValidateRouterSdkTerminators(id string, clients *zitirest.Clients) (int, error) { + return ValidateRouterSdkTerminatorsWithFilter(id, clients, "limit none") +} + +// ValidateRouterSdkTerminatorsWithFilter validates the SDK terminators on the routers matching filter, +// returning the number found invalid. +func ValidateRouterSdkTerminatorsWithFilter(id string, clients *zitirest.Clients, filter string) (int, error) { logger := tui.ValidationLogger().WithField("ctrl", id) closeNotify := make(chan struct{}) @@ -191,7 +281,7 @@ func ValidateRouterSdkTerminators(id string, clients *zitirest.Clients) (int, er }() request := &mgmt_pb.ValidateRouterSdkTerminatorsRequest{ - Filter: "limit none", + Filter: filter, } responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(10 * time.Second).SendForReply(ch) @@ -235,6 +325,13 @@ func ValidateRouterSdkTerminators(id string, clients *zitirest.Clients) (int, er } func ValidateRouterErtTerminators(id string, clients *zitirest.Clients) (int, error) { + return ValidateRouterErtTerminatorsWithFilter(id, clients, "limit none") +} + +// ValidateRouterErtTerminatorsWithFilter validates the ERT terminators on the routers matching filter, +// returning the number found invalid. Restricting the filter to edge-router tunnelers avoids inspecting +// routers that host no ERT terminators. +func ValidateRouterErtTerminatorsWithFilter(id string, clients *zitirest.Clients, filter string) (int, error) { logger := tui.ValidationLogger().WithField("ctrl", id) closeNotify := make(chan struct{}) @@ -267,7 +364,7 @@ func ValidateRouterErtTerminators(id string, clients *zitirest.Clients) (int, er }() request := &mgmt_pb.ValidateRouterErtTerminatorsRequest{ - Filter: "limit none", + Filter: filter, } responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(10 * time.Second).SendForReply(ch)