mirror of
https://github.com/openziti/ziti.git
synced 2026-09-09 16:25:41 +00:00
Add in-place upgrade smoke test. Fixes #4199
- adds a fablab model (zititest/models/upgrade-test) that stands up a multi-region HA controller cluster, routers, and loop4 sim traffic clients (SDK, ERT, ZET, tunneler) and drives repeatable in-place upgrade/downgrade iterations, re-bootstrapping in place each iteration - exercises HA cluster disruption and recovery (snapshot restore, node rejoin) between iterations and validates steady-state traffic and expected terminators after each disruption - honors the raft restartSelf setting on migration-snapshot restore, so a restored controller restarts itself instead of exiting, and promotes RestartController to a package-level function - re-sends loop4 run-scenario requests to sims that reconnect mid-scenario, tracking each scenario's expected client set explicitly and ignoring results from clients outside it, so post-disruption client reconnection no longer stalls validation - closes leaked loop4 sim-control connections on error and on supersession - rotates component logs and pins the log-pipe binary to the local build, and drops router debug logging, to keep long iterating runs disk-bounded - excludes the doc/unsettled/ scratch area for in-progress design docs from version control
This commit is contained in:
Paul Lorenz
@@ -0,0 +1,151 @@
|
||||
name: Upgrade Test
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
zetVersion:
|
||||
description: 'ziti-edge-tunnel version for every ZET client and host (blank uses the model default)'
|
||||
required: false
|
||||
default: ''
|
||||
fromVersion:
|
||||
description: 'version the system starts on and is upgraded from (blank uses the model default)'
|
||||
required: false
|
||||
default: ''
|
||||
toVersion:
|
||||
description: 'version to upgrade to, and the version a ref build is stamped with (blank uses the model default)'
|
||||
required: false
|
||||
default: ''
|
||||
toVersionRef:
|
||||
description: 'git ref on openziti/ziti to build toVersion from; blank uses the model default'
|
||||
required: false
|
||||
default: ''
|
||||
nextVersion:
|
||||
description: 'second upgrade hop, applied to the whole cluster after toVersion; blank uses the model default, "none" skips the phase'
|
||||
required: false
|
||||
default: ''
|
||||
nextVersionRef:
|
||||
description: 'git ref on openziti/ziti to build nextVersion from; blank uses the model default'
|
||||
required: false
|
||||
default: ''
|
||||
clusterUpgradeMode:
|
||||
description: 'how the cluster moves to nextVersion'
|
||||
required: false
|
||||
type: choice
|
||||
options:
|
||||
- ''
|
||||
- rolling
|
||||
- all-at-once
|
||||
default: ''
|
||||
|
||||
# one upgrade test at a time: each run provisions a multi-region AWS topology
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}
|
||||
cancel-in-progress: false
|
||||
|
||||
env:
|
||||
GOFLAGS: "-trimpath"
|
||||
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
|
||||
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
|
||||
AWS_DEFAULT_REGION: "us-east-2"
|
||||
gh_ci_key: ${{ secrets.GH_CI_KEY }}
|
||||
BUILD_NUMBER: ${{ format('{0}-{1}-{2}', github.run_id, github.run_number, github.run_attempt) }}
|
||||
|
||||
jobs:
|
||||
upgrade-test:
|
||||
name: Fablab Upgrade Test
|
||||
if: github.repository_owner == 'openziti'
|
||||
runs-on: ubuntu-24.04
|
||||
timeout-minutes: 180
|
||||
|
||||
steps:
|
||||
- name: Git Checkout
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Install Go
|
||||
uses: ./.github/actions/setup-go
|
||||
|
||||
- name: Install Ziti CI
|
||||
uses: openziti/ziti-ci@v1
|
||||
|
||||
- name: Install Terraform CLI
|
||||
uses: hashicorp/setup-terraform@v4
|
||||
with:
|
||||
terraform_version: ~1.5
|
||||
|
||||
- name: Build
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
ziti_ci_gpg_key: ${{ secrets.ZITI_CI_GPG_KEY }}
|
||||
ziti_ci_gpg_key_id: ${{ secrets.ZITI_CI_GPG_KEY_ID }}
|
||||
shell: bash
|
||||
run: |
|
||||
$(go env GOPATH)/bin/ziti-ci configure-git
|
||||
$(go env GOPATH)/bin/ziti-ci generate-build-info common/version/info_generated.go version
|
||||
pushd zititest && go mod tidy && go install -tags all ./... && popd
|
||||
go install -tags=all,tests ./...
|
||||
|
||||
- name: Create Test Environment
|
||||
env:
|
||||
# inputs reach the script through the environment, never interpolated into it
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
ZET_VERSION: ${{ inputs.zetVersion }}
|
||||
FROM_VERSION: ${{ inputs.fromVersion }}
|
||||
TO_VERSION: ${{ inputs.toVersion }}
|
||||
TO_VERSION_REF: ${{ inputs.toVersionRef }}
|
||||
NEXT_VERSION: ${{ inputs.nextVersion }}
|
||||
NEXT_VERSION_REF: ${{ inputs.nextVersionRef }}
|
||||
CLUSTER_UPGRADE_MODE: ${{ inputs.clusterUpgradeMode }}
|
||||
shell: bash
|
||||
run: |
|
||||
echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
|
||||
|
||||
# pass only what was supplied, so the model's own defaults stay authoritative
|
||||
LABELS="environment=gh-upgrade-test"
|
||||
if [ -n "$ZET_VERSION" ]; then LABELS="$LABELS,zetVersion=$ZET_VERSION"; fi
|
||||
if [ -n "$FROM_VERSION" ]; then LABELS="$LABELS,fromVersion=$FROM_VERSION"; fi
|
||||
if [ -n "$TO_VERSION" ]; then LABELS="$LABELS,toVersion=$TO_VERSION"; fi
|
||||
if [ -n "$TO_VERSION_REF" ]; then LABELS="$LABELS,toVersionRef=$TO_VERSION_REF"; fi
|
||||
# "none" is how a run switches the optional second hop off, since a blank input means
|
||||
# "use the model default" and the model defaults it on
|
||||
if [ "$NEXT_VERSION" = "none" ]; then
|
||||
LABELS="$LABELS,nextVersion="
|
||||
elif [ -n "$NEXT_VERSION" ]; then
|
||||
LABELS="$LABELS,nextVersion=$NEXT_VERSION"
|
||||
fi
|
||||
if [ -n "$NEXT_VERSION_REF" ]; then LABELS="$LABELS,nextVersionRef=$NEXT_VERSION_REF"; fi
|
||||
if [ -n "$CLUSTER_UPGRADE_MODE" ]; then LABELS="$LABELS,clusterUpgradeMode=$CLUSTER_UPGRADE_MODE"; fi
|
||||
echo "using labels: $LABELS"
|
||||
|
||||
INSTANCE="upgrade-test-${GITHUB_RUN_NUMBER}"
|
||||
echo "INSTANCE=${INSTANCE}" >> "$GITHUB_ENV"
|
||||
$(go env GOPATH)/bin/upgrade-test create -d "${INSTANCE}" -n "${INSTANCE}" -l "$LABELS"
|
||||
$(go env GOPATH)/bin/upgrade-test up
|
||||
|
||||
- name: Run Upgrade Iteration
|
||||
shell: bash
|
||||
run: |
|
||||
$(go env GOPATH)/bin/upgrade-test exec testIteration
|
||||
|
||||
- name: Create Logs Archive
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
$(go env GOPATH)/bin/upgrade-test get files '*' "./logs/{{ .Id }}/" ./logs
|
||||
|
||||
- name: Upload artifacts
|
||||
uses: actions/upload-artifact@v7
|
||||
if: always()
|
||||
with:
|
||||
name: logs-upgrade-test-${{ github.run_id }}
|
||||
path: logs/
|
||||
compression-level: 7
|
||||
retention-days: 5
|
||||
|
||||
- name: Tear down Test Environment
|
||||
timeout-minutes: 30
|
||||
if: always()
|
||||
shell: bash
|
||||
run: |
|
||||
$(go env GOPATH)/bin/upgrade-test dispose
|
||||
@@ -44,6 +44,7 @@ go.work.sum
|
||||
.github/workflows/localtest.yml
|
||||
doc/ha/pki
|
||||
doc/ha/data
|
||||
doc/unsettled/
|
||||
simple-transfer-*/
|
||||
etc/endpoints
|
||||
etc/endpoints.yml
|
||||
|
||||
@@ -301,6 +301,7 @@ func NewController(cfg *config.Config, versionProvider versions.VersionProvider)
|
||||
|
||||
if n, err := network.NewNetwork(c, appEnv); err == nil {
|
||||
c.network = n
|
||||
n.SetRestartSelfOnSnapshot(c.config.Raft != nil && c.config.Raft.RestartSelf)
|
||||
} else {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ import (
|
||||
"github.com/openziti/ziti/v2/controller/event"
|
||||
"github.com/openziti/ziti/v2/controller/idgen"
|
||||
"github.com/openziti/ziti/v2/controller/model"
|
||||
"github.com/openziti/ziti/v2/controller/raft"
|
||||
"github.com/openziti/ziti/v2/controller/storage/boltz"
|
||||
"github.com/openziti/ziti/v2/controller/storage/objectz"
|
||||
"github.com/openziti/ziti/v2/controller/xt"
|
||||
@@ -114,6 +115,10 @@ type Network struct {
|
||||
|
||||
config Config
|
||||
|
||||
// restartSelfOnSnapshot, when true, restarts this controller in place after a snapshot restore
|
||||
// (e.g. a node joining the cluster) instead of exiting and relying on an external process manager.
|
||||
restartSelfOnSnapshot bool
|
||||
|
||||
Inspections *InspectionsManager
|
||||
RouterMessaging *RouterMessaging
|
||||
inspectionTargets concurrenz.CopyOnWriteSlice[InspectTarget]
|
||||
@@ -1494,7 +1499,15 @@ func (network *Network) RestoreSnapshot(cmd *command.SyncSnapshotCommand, index
|
||||
}
|
||||
|
||||
time.AfterFunc(5*time.Second, func() {
|
||||
log.Info("database restore requires controller restart. exiting...")
|
||||
if network.restartSelfOnSnapshot {
|
||||
log.Info("database restore requires controller restart, restarting...")
|
||||
// RestartController returns only when the restart failed; on success it replaces this
|
||||
// process and never comes back.
|
||||
err := raft.RestartController()
|
||||
log.WithError(err).Error("failed to restart controller after snapshot restore, exiting...")
|
||||
} else {
|
||||
log.Info("database restore requires controller restart. exiting...")
|
||||
}
|
||||
os.Exit(0)
|
||||
})
|
||||
|
||||
@@ -1524,6 +1537,13 @@ func (network *Network) ensureRaftIndex(index uint64) error {
|
||||
})
|
||||
}
|
||||
|
||||
// SetRestartSelfOnSnapshot controls whether the controller restarts itself after a snapshot restore
|
||||
// (true) or exits expecting an external restart (false). It mirrors the raft restartSelfOnSnapshot
|
||||
// setting and is applied by the owning controller after construction.
|
||||
func (network *Network) SetRestartSelfOnSnapshot(v bool) {
|
||||
network.restartSelfOnSnapshot = v
|
||||
}
|
||||
|
||||
func (network *Network) AddInspectTarget(target InspectTarget) {
|
||||
network.inspectionTargets.Append(target)
|
||||
}
|
||||
|
||||
@@ -443,7 +443,7 @@ func (self *BoltDbFsm) Restore(snapshot io.ReadCloser) error {
|
||||
time.AfterFunc(5*time.Second, func() {
|
||||
if self.restartSelf {
|
||||
log.Info("restored snapshot to initialized system, restart required, restarting now")
|
||||
if err = self.RestartController(); err != nil {
|
||||
if err = RestartController(); err != nil {
|
||||
log.WithError(err).Error("failed to restart controller, exiting now")
|
||||
os.Exit(0)
|
||||
}
|
||||
@@ -464,7 +464,7 @@ func (self *BoltDbFsm) Restore(snapshot io.ReadCloser) error {
|
||||
|
||||
// RestartController starts a new controller process with the same parameters and exits the current process.
|
||||
// This is useful when the controller needs to be restarted after applying a snapshot or other configuration changes.
|
||||
func (self *BoltDbFsm) RestartController() error {
|
||||
func RestartController() error {
|
||||
log := pfxlog.Logger()
|
||||
|
||||
// Get the current executable path
|
||||
|
||||
@@ -23,6 +23,7 @@ require (
|
||||
github.com/openziti/foundation/v2 v2.0.100
|
||||
github.com/openziti/identity v1.0.140
|
||||
github.com/openziti/metrics v1.4.5
|
||||
github.com/openziti/sdk-golang/acquire v0.3.0
|
||||
github.com/openziti/sdk-golang/v2 v2.0.0-pre4
|
||||
github.com/openziti/transport/v2 v2.0.220
|
||||
github.com/openziti/ziti/v2 v2.0.3
|
||||
@@ -236,6 +237,7 @@ require (
|
||||
go4.org v0.0.0-20260112195520-a5071408f32f // indirect
|
||||
golang.org/x/crypto v0.55.0 // indirect
|
||||
golang.org/x/exp v0.0.0-20260820142414-ca536658362e // indirect
|
||||
golang.org/x/mod v0.39.0 // indirect
|
||||
golang.org/x/oauth2 v0.36.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
|
||||
@@ -619,6 +619,8 @@ github.com/openziti/metrics v1.4.5 h1:p51HYSQyqaDafizPWluhYjU/rVfY52snER8/BHpfPr
|
||||
github.com/openziti/metrics v1.4.5/go.mod h1:MOLcoTxhPNla6+NWUCMVTnl1PNqTU40qrbKVa/lVVgg=
|
||||
github.com/openziti/runzmd v1.0.92 h1:dnv+luFxYrv3DY1FPfLsFO9/d21PijhLiAHehJHGPyA=
|
||||
github.com/openziti/runzmd v1.0.92/go.mod h1:818zJed1POFnZPR2DndES9/7VS3RMByKHH2/MEw2KfA=
|
||||
github.com/openziti/sdk-golang/acquire v0.3.0 h1:wNi3JaXUoSGxissurb9SUObLOtkanNQNJiZXro6Wup0=
|
||||
github.com/openziti/sdk-golang/acquire v0.3.0/go.mod h1:fM5a2tGvDdeUxiq3NT2vU4qCSmWniIItSEQV4O6vJuo=
|
||||
github.com/openziti/sdk-golang/v2 v2.0.0-pre4 h1:/3a0/mGYczTRFH+h4NYn2d+xWJl8xVCHU1g2XJcAMng=
|
||||
github.com/openziti/sdk-golang/v2 v2.0.0-pre4/go.mod h1:WYPHgevyOIHFrRLpeftzBAYOvd9YAJOxSNW3g+ORzIg=
|
||||
github.com/openziti/secretstream v0.1.52 h1:UL8bshHGpXRLt7wjIeyvySXsmoBNoRsumyTsAh+O8ns=
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
# upgrade-test model
|
||||
|
||||
## Goal
|
||||
|
||||
Stand up a full smoke-style OpenZiti topology on a parameterized "from" version using legacy
|
||||
authentication, confirm traffic is flowing, then walk the system through a sequence of in-place
|
||||
upgrades while asserting that traffic keeps flowing throughout:
|
||||
|
||||
1. Start on the "from" version (default: latest 1.6.x) with a single standalone controller, legacy auth.
|
||||
2. Upgrade the controller to the "to" version (default: v2.0.3). This auto-enables OIDC.
|
||||
3. Upgrade the routers to the "to" version.
|
||||
4. Convert the controller to HA (single-node cluster).
|
||||
5. Add two more controller nodes to form a 3-node cluster.
|
||||
6. Optionally upgrade the whole cluster again, to the "next" version (default: 2.1 built from `main`).
|
||||
|
||||
The ZET (ziti-edge-tunnel), ERT (edge-router-tunneler), and ziti tunnel clients and hosts stay up
|
||||
across the entire sequence. The whole run is repeatable in a loop so we can flush out timing errors
|
||||
and race conditions.
|
||||
|
||||
## Topology
|
||||
|
||||
All AWS hosts are provisioned up front. Adding AWS instances mid-run is avoided; the two extra
|
||||
controllers simply sit idle until the "add nodes" phase.
|
||||
|
||||
Controllers:
|
||||
- `ctrl1` - runs from the start.
|
||||
- `ctrl2`, `ctrl3` - provisioned but not started until the add-nodes phase.
|
||||
|
||||
Routers (same as smoke):
|
||||
- `router-east-1` - ERT client.
|
||||
- `router-east-2` - initiator.
|
||||
- `router-west` - ERT host.
|
||||
|
||||
Clients and hosts that must stay up across the whole sequence, each present as a **pair** so that
|
||||
one instance can be restarted while the other keeps serving (see OIDC verification). Each pair is named
|
||||
`<name>-stable` and `<name>-restart`, and carries a `stable` or `restart` tag so the restart and
|
||||
PID-persistence actions can select the whole set by role:
|
||||
- `loop4-client-stable` / `-restart` - Go SDK sim dialer.
|
||||
- `loop4-sdk-host-stable` / `-restart` - Go SDK sim listener (binds the `loop-sdk` service).
|
||||
- `ziti-edge-tunnel-client-stable` / `-restart` and `ziti-edge-tunnel-host-stable` / `-restart` - ZET, the C SDK tunneler.
|
||||
- `ziti-tunnel-client-stable` / `-restart` and `ziti-tunnel-host-stable` / `-restart` - Go SDK tunneler.
|
||||
|
||||
The two instances of each pair co-locate on the same host as each other, so pairing adds components
|
||||
but no AWS instances. Process filters keep co-located instances individually controllable: ZET by its
|
||||
config id, ziti-tunnel by `--cli-agent-alias`, and loop4 once its process filter includes the config id
|
||||
(a small, backward-compatible change to `Loop4SimType`). The paired hosts binding the same service
|
||||
simply create two terminators, so restarting one leaves the other serving.
|
||||
|
||||
Support components (single, not paired): plain-TCP loop4 backends that the tunneler hosts forward their
|
||||
loop service to (see Liveness). `loop4-ert-backend` sits on the ERT host; the ZET and ziti-tunnel hosts
|
||||
share a box and a single `loop4-tunnel-backend`. These have no ziti identity, so they are not part of the
|
||||
OIDC story.
|
||||
|
||||
## Version parameterization
|
||||
|
||||
Both ends are fully parameterized (version X -> version Y), defaulting to latest 1.6.x -> v2.0.3.
|
||||
|
||||
- `fromVersion` / `toVersion` model variables drive the controller and router binaries.
|
||||
- `zetVersion` for ZET, covering both its client and host instances. ZET is not upgraded mid-run; it
|
||||
stays up the whole time. Parameterizing it lets us sweep ZET versions across separate runs. Default is
|
||||
the version shipped with the current stable Windows desktop edge client: `v1.18.0` (bundled in Ziti
|
||||
Desktop Edge for Windows release 2.11.2.5, the latest stable release; note 2.11.2.7 bundles v1.18.2
|
||||
but is a pre-release).
|
||||
- `nextVersion` is an optional second hop: once the 3-node cluster is up on `toVersion`, the whole
|
||||
cluster is upgraded again. `nextVersionSource` / `nextVersionRef` work exactly like their `toVersion`
|
||||
counterparts. Setting `nextVersion` to the empty string skips the phase and its two gates entirely.
|
||||
- The existing per-component `<componentId>_version` label knob still works for one-off overrides.
|
||||
|
||||
Both the "from" and "to" binaries are pre-staged on every host during distribution. Because binaries
|
||||
are versioned and coexist on-host (`ziti-<version>`, `ziti-edge-tunnel-<version>`), every upgrade is
|
||||
just `SetVersion(newVersion)` followed by a restart, and a downgrade (for reset) is the same in reverse.
|
||||
No mid-run binary rsync is needed. `ControllerType.Start` resolves the binary from the component's
|
||||
current `Version` at start time, so a version swap plus restart is sufficient.
|
||||
|
||||
The Go SDK clients (`ziti-tunnel-*`, `loop4-*`) start on `fromVersion` and stay there. That is
|
||||
deliberately the interesting compatibility case for the OIDC question below.
|
||||
|
||||
## Config strategy
|
||||
|
||||
One controller config works for both binaries. It has no explicit OIDC section, so the 2.0 binary
|
||||
auto-binds the `edge-oidc` API on its first restart (`ensureOidcOnClientApiServer` in
|
||||
`controller/controller.go`). That auto-bind is the "upgrade forces OIDC to become available" event, and
|
||||
it happens with no config change.
|
||||
|
||||
For the HA conversion, the controller config is re-rendered to add the `cluster:` directive and rsynced
|
||||
to the host before restart. The standalone binary then auto-migrates the bbolt DB into raft and comes up
|
||||
as a single-node HA cluster.
|
||||
|
||||
Router controller-endpoint handling after the HA conversion (whether routers need all three controller
|
||||
addresses in config or auto-learn them from the cluster) is verified during implementation.
|
||||
|
||||
## Liveness
|
||||
|
||||
We adopt the `circuit-test` harness: `SimServices` plus remote-controlled loop4 clients with
|
||||
`iterations: -1` (run forever, auto-reconnect after restarts), streaming success/failure metrics every
|
||||
five seconds.
|
||||
|
||||
To drive traffic through the ZET and ziti-tunnel hosting paths (not just the pure-SDK path), we define
|
||||
services whose terminators are the ZET host and the ziti-tunnel host, each backed by a plain-TCP loop4
|
||||
listener behind the tunnel:
|
||||
|
||||
```
|
||||
loop4 dialer (SDK) -> ziti service -> ZET / ziti-tunnel host -> TCP -> loop4 listener
|
||||
```
|
||||
|
||||
The circuit-test model already uses loop4 this way, so this is a proven pattern. We also keep a
|
||||
pure-SDK-hosted service and an ERT-hosted service, so the sim continuously exercises all four hosting
|
||||
flavors.
|
||||
|
||||
Quiescent-vs-churn validation (no failures when idle, tolerated during restarts). Rather than correlate
|
||||
raw 5-second metric samples with churn windows, validation is expressed in terms of the sim's discrete
|
||||
scenario runs (the remote-controlled pattern), which is far less flake-prone. After each disruptive step:
|
||||
|
||||
- Recovery gate: require N consecutive clean scenario runs (zero errors) achieved within a recovery
|
||||
timeout (default ~1 minute). This tolerates the transient failures expected while a controller or
|
||||
router restarts.
|
||||
- Stability gate: once recovered, require scenario runs to continue clean for a stability window
|
||||
(default ~2 minutes) before advancing to the next phase.
|
||||
- N, the recovery timeout, and the stability window are all tunable knobs.
|
||||
|
||||
The baseline (phase 1) and the final validation (phase 6) use the same clean-runs gate with no preceding
|
||||
disruption.
|
||||
|
||||
## Phase sequence
|
||||
|
||||
The main orchestration action runs these phases in order.
|
||||
|
||||
0. Bootstrap on `fromVersion`: standalone `ctrl1`, routers, all clients and hosts enrolled; start; wait
|
||||
ready.
|
||||
1. Baseline: start the continuous sim; assert clean steady-state. Traffic is flowing on legacy auth.
|
||||
2. Upgrade controller to `toVersion`: stop `ctrl1`, `SetVersion`, restart (same standalone config, so
|
||||
OIDC auto-binds). Assert recovery and clean steady-state, then run the restart-one-of-each OIDC
|
||||
verification (see below).
|
||||
3. Rolling router upgrade to `toVersion`: one router at a time (stop, `SetVersion`, start, wait) so
|
||||
traffic keeps flowing through the others. Assert recovery after each, clean steady-state after all.
|
||||
4. Convert controller to HA (single node): stop `ctrl1`, swap in the cluster-directive config, restart
|
||||
so it auto-migrates to raft. Assert recovery and clean steady-state, then re-run the restart-one-of-each
|
||||
verification.
|
||||
5. Add two nodes: start `ctrl2` and `ctrl3`, run `ziti agent cluster add` for each, wait for a stable
|
||||
three-voter quorum, update router (and possibly SDK client) controller lists if needed. Assert
|
||||
recovery and clean steady-state, then re-run the restart-one-of-each verification.
|
||||
6. Final validation: full clean steady-state.
|
||||
7. Optional cluster upgrade to `nextVersion`: upgrade all three controllers, assert clean steady-state,
|
||||
roll the routers, assert clean steady-state again. Skipped when `nextVersion` is empty.
|
||||
|
||||
Not implemented: asserting that the *un-restarted* instance of each pair kept its original PID across
|
||||
phases 1-5, which would prove the long-lived clients truly stayed up rather than having been quietly
|
||||
restarted. The steady-state gate checks they are healthy, not that they are the same processes.
|
||||
|
||||
## OIDC verification via client restart
|
||||
|
||||
We verify OIDC by observing real clients rather than a synthetic probe. This is the only way to cover
|
||||
ZET (the C SDK), and it exercises the exact auth path production clients use.
|
||||
|
||||
The mechanism relies on two facts:
|
||||
|
||||
- A long-lived Go SDK context caches the controller's advertised capabilities via `sync.Once` and never
|
||||
re-reads them on reconnect (`edge-apis/client_edge_client.go`). So a client that stays up across the
|
||||
1.6 -> 2.0 upgrade keeps using its legacy session; only a freshly started client re-reads capabilities
|
||||
and can switch to OIDC.
|
||||
- The controller emits an `apiSession` event (`controller/event/api_session.go`) whose `type` field is
|
||||
`legacy` or `jwt` (jwt is the token/OIDC session), keyed by `identity_id` and `ip_address`. We enable
|
||||
this event with a file handler in `ctrl.yml.tmpl`, exactly like the entityChange/circuit/link/router
|
||||
events already there. (The `authentication` event's `type` is the credential kind, `cert`/`updb`/`ext-jwt`,
|
||||
not the session mechanism, so it does not distinguish legacy from OIDC; we use `apiSession`.)
|
||||
|
||||
Because every client and host is a pair, after the controller upgrade we restart the `-restart` instance
|
||||
of each pair, leaving the `-stable` instance up:
|
||||
|
||||
- The `-stable` instance proves the client survives the upgrade on its existing legacy session, with no
|
||||
re-auth and no traffic gap (its PID is checked for continuity in the final phase).
|
||||
- The `-restart` instance is a fresh process, so it re-authenticates. We read the new `apiSession`
|
||||
`created` event for that instance (matched by identity or source IP) and check its `type`.
|
||||
|
||||
Expected results: freshly started Go SDK clients (loop4, ziti-tunnel) default to dynamic OIDC detection,
|
||||
so they should come back as `jwt`. ZET (C SDK, `v1.18.0`) auto-switch behavior is genuinely uncertain; if
|
||||
a restarted ZET comes back `legacy`, that is a real finding, and the `apiSession` event makes it visible
|
||||
either way. So ZET's expected value is TBD by design, the test documents actual behavior.
|
||||
|
||||
This replaces the earlier synthetic edge_apis probe and the standalone capability assertion: a restarted
|
||||
client returning `jwt` already implies the controller advertises OIDC.
|
||||
|
||||
Phases 4-5 (HA and multi-controller) are the predicted breaking point for the long-lived legacy siblings.
|
||||
"They keep working" is the pass criterion. If a later phase breaks them, that triggers an in-scope
|
||||
sdk-golang fix to make long-lived contexts re-detect OIDC / handle the cluster on reconnect. The test is
|
||||
designed to surface exactly that.
|
||||
|
||||
## Cluster upgrade and controller memory
|
||||
|
||||
The optional second hop upgrades an established 3-node cluster rather than building one, which is the
|
||||
scenario production operators actually run and the one [#4219](https://github.com/openziti/ziti/issues/4219)
|
||||
reports a controller OOM in. `clusterUpgradeMode` picks how it happens:
|
||||
|
||||
- `rolling` (default) restarts one node at a time, followers first and the leader last. Peers hold the
|
||||
cluster in mixed-version read-only mode until the last node is done, which is the state #4219 blames.
|
||||
- `all-at-once` restarts every node together, so no node ever sees a version mismatch. This is the
|
||||
workaround the issue asks about, and it is also the control case: every node still cold-starts, so if
|
||||
memory climbs here too, the mismatch is not the mechanism.
|
||||
|
||||
No steady-state gate runs between nodes in rolling mode. The cluster is read-only while versions differ,
|
||||
so anything needing a write would fail for reasons the phase is not testing. The gate runs once the whole
|
||||
cluster is on `nextVersion`.
|
||||
|
||||
Every controller host samples its controller's RSS once a second for the whole iteration, writing
|
||||
`~/logs/<ctrl>-mem.csv`. The controller is located by its agent alias, the same discriminator fablab's
|
||||
process filter uses, so sampling follows the process across an upgrade and records a zero while it is
|
||||
down: a node in a crash loop shows up as a sawtooth rather than a gap. Peaks are reported per node after
|
||||
the cluster upgrade and again at the end of the iteration.
|
||||
|
||||
`ctrlMemory.heapDumpAtMb` captures a heap profile (`~/logs/<ctrl>-mem-<epoch>.pprof`) the first time RSS
|
||||
crosses it. That profile is the thing #4219 is missing and the one artifact that cannot be recovered after
|
||||
the fact. The samples and profiles are under `~/logs`, so CI collects them with the rest of the logs.
|
||||
|
||||
Two checks can fail the cluster upgrade, both disabled by setting them to zero:
|
||||
|
||||
- `ctrlMemory.failAtRatio` (default 3) compares each controller's peak over the upgrade against its own
|
||||
peak over the two minutes of settled traffic immediately before it. This is the one that bites at this
|
||||
scale, and it is the shape #4219 reports: roughly 5x a 150-190 MiB baseline. The default is a first
|
||||
guess, since a cold start legitimately overshoots a warm steady state; calibrate it against a run known
|
||||
to be good.
|
||||
- `ctrlMemory.failAtMb` (default 1024) is the absolute ceiling, sized for the reported failure. It is a
|
||||
backstop, and is unlikely to be reached without a much larger data set.
|
||||
|
||||
`fablab exec ctrlMemorySummary` prints one row per controller per iteration for the whole run, plus the
|
||||
highest peak seen and where, so a finished run can be read back with one command rather than scrolled for.
|
||||
`fablab exec reportCtrlMemory` is the narrower form, covering only the current iteration.
|
||||
|
||||
Nothing is discarded between iterations. Samples accumulate in one file behind a `# start,<epoch>` marker
|
||||
that scopes each report to the current iteration, and the profile path carries the sampler's start time,
|
||||
so iteration 4 cannot delete the profile iteration 3 captured. At a line a second the samples cost roughly
|
||||
1.7 MB per controller per day.
|
||||
|
||||
Scale caveat: this model runs a handful of routers, identities and services, so cold-start cost here is a
|
||||
fraction of a production controller's. Expect the *shape* to be informative (does the upgraded node
|
||||
diverge from its warm peers, and does it diverge in `all-at-once` too) rather than the magnitude. Chasing
|
||||
the reported 1 GiB number would mean seeding a much larger data set first.
|
||||
|
||||
## Iteration and reset
|
||||
|
||||
A `testIteration` action runs `{reset -> full upgrade sequence}` once, and `fablab exec-loop` repeats it
|
||||
for as long as asked. Rather than hand-rolling a surgical state wipe, reset leans on the existing fablab
|
||||
machinery: wipe everything (controller DB and raft state, PKI/enrollment, cached tokens) and re-run
|
||||
distribution and bootstrap from the `fromVersion` standalone configuration. This is more trustworthy
|
||||
than trying to selectively clear state, and an incomplete wipe would produce cross-iteration
|
||||
contamination that looks exactly like a race.
|
||||
|
||||
Each iteration is fully independent, which is what flushes timing and race issues. SDK clients restart
|
||||
per iteration because the wipe invalidates identities, but they stay up across all phases within an
|
||||
iteration, which is what matters for catching mid-upgrade problems. A full downgrade-in-place without a
|
||||
wipe is not possible, since the 2.0 DB schema cannot be read by the 1.6 binary.
|
||||
|
||||
A bare `fablab exec testIteration` runs a single bounded pass; `fablab exec-loop <until> testIteration`
|
||||
is the opt-in form for long local soak runs and a time-boxed slot in the validation suite.
|
||||
|
||||
## Longer-term direction
|
||||
|
||||
The version pair is a parameter, but the model is currently shaped around one specific transition:
|
||||
1.6.x -> 2.0.x. The goal is to run LTS -> current instead, so each release is exercised against the
|
||||
version most deployments are actually upgrading from, and the pair moves as LTS moves.
|
||||
|
||||
Two things stand in the way, both of which encode "this is the 1.6 -> 2.0 upgrade" rather than "this is
|
||||
an upgrade":
|
||||
|
||||
- The transition-specific workarounds. `restartZetWorkaround` restarts ziti-edge-tunnel because versions
|
||||
at or below `zetRestartWorkaroundMaxVersion` cannot rebuild their edge sessions after the 2.0 JWT
|
||||
session migration, and `reconcileStaleTerminators` runs only while `anyPreV2Router` holds, cleaning up
|
||||
terminators that pre-2.0 routers drop without telling the controller. Both are gated on version so
|
||||
they switch themselves off, but a different pair needs its own set, and there is no structure yet for
|
||||
saying which workarounds belong to which transition.
|
||||
- The phase sequence itself. The standalone -> HA conversion and the cluster-node join are steps in the
|
||||
1.6 -> 2.0 story specifically. An LTS -> current run where both ends are already HA would want a
|
||||
different sequence, so the phases need to become selectable rather than a fixed list.
|
||||
|
||||
Neither is large on its own, but together they are why the version defaults are not simply pointed at a
|
||||
moving LTS target today.
|
||||
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/openziti/fablab/kernel/lib/actions"
|
||||
"github.com/openziti/fablab/kernel/lib/actions/component"
|
||||
"github.com/openziti/fablab/kernel/lib/actions/host"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/ziti/zititest/zitilab"
|
||||
zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
|
||||
"github.com/openziti/ziti/zititest/zitilab/actions/edge"
|
||||
"github.com/openziti/ziti/zititest/zitilab/models"
|
||||
)
|
||||
|
||||
type bootstrapAction struct{}
|
||||
|
||||
// newBootstrapAction builds the phase-1 bootstrap: a single standalone controller on the
|
||||
// from-version, enrolled routers and identities, and the loop services that the sim dials
|
||||
// across all four hosting flavors (SDK, ERT, ZET, ziti-tunnel).
|
||||
func newBootstrapAction() model.ActionBinder {
|
||||
action := &bootstrapAction{}
|
||||
return action.bind
|
||||
}
|
||||
|
||||
func (a *bootstrapAction) bind(m *model.Model) model.Action {
|
||||
workflow := actions.Workflow()
|
||||
|
||||
// Stop every component first so a re-bootstrap starts from a clean slate. Otherwise sim clients
|
||||
// and tunnelers left running from a prior iteration keep stale sessions (and an old-version
|
||||
// controller/router process can survive), which fails the next baseline.
|
||||
workflow.AddAction(component.StopInParallel("*", 25))
|
||||
|
||||
// Only ctrl1 runs in phase 1. ctrl2/ctrl3 are provisioned but idle until the add-nodes phase.
|
||||
workflow.AddAction(component.StopInParallel(".ctrl", 15))
|
||||
workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*"))
|
||||
workflow.AddAction(host.GroupExec("component.ctrl", 5, "rm -rf ./fablab/ctrldata ./fablab/ctrl.db"))
|
||||
|
||||
workflow.AddAction(component.Exec("#ctrl1", zitilab.ControllerActionInitStandalone))
|
||||
workflow.AddAction(component.Start("#ctrl1"))
|
||||
workflow.AddAction(edge.ControllerAvailable("#ctrl1", 30*time.Second))
|
||||
workflow.AddAction(edge.Login("#ctrl1"))
|
||||
|
||||
workflow.AddAction(component.StopInParallel(models.EdgeRouterTag, 25))
|
||||
workflow.AddAction(edge.InitEdgeRouters(models.EdgeRouterTag, 2))
|
||||
workflow.AddAction(edge.InitIdentities(models.SdkAppTag, 2))
|
||||
|
||||
// Shared host.v1 config: every tunneler-hosted loop service forwards to a local loop4
|
||||
// listener on tcp:127.0.0.1:3456.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "config", "loop-backend", "host.v1", `
|
||||
{
|
||||
"address" : "localhost",
|
||||
"port" : 3456,
|
||||
"protocol" : "tcp"
|
||||
}`))
|
||||
|
||||
// intercept.v1 for loop-zet: ZET is tproxy-only (no proxy-port mode), so it needs an intercept
|
||||
// config to know the address/port to intercept. The Go ziti-tunnel and ERT clients use proxy mode
|
||||
// with the port on the command line, so they need no intercept config. Use a hostname (not a bare
|
||||
// IP): ZET's tproxy assigns an IP from its own DNS range for the hostname and intercepts that; a
|
||||
// static IP isn't intercepted. The co-located loop4 dialer targets this hostname, which resolves
|
||||
// through ZET's resolver into the intercept.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "config", "loop-zet-intercept", "intercept.v1", `
|
||||
{
|
||||
"addresses": ["loop-zet.ziti"],
|
||||
"portRanges": [{ "low": 15391, "high": 15391 }],
|
||||
"protocols": ["tcp"]
|
||||
}`))
|
||||
|
||||
// One loop service per hosting flavor.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-sdk", "-a", "loop-svc,loop-sdk-host-svc"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-ert", "-c", "loop-backend", "-a", "loop-svc,loop-ert-svc"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-zet", "-c", "loop-backend,loop-zet-intercept", "-a", "loop-svc,loop-zet-svc"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service", "loop-ziti-tunnel", "-c", "loop-backend", "-a", "loop-svc,loop-zt-svc"))
|
||||
|
||||
// Bind policies: each flavor's service is hosted by the matching host identity.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-sdk-hosts", "Bind", "--service-roles", "#loop-sdk-host-svc", "--identity-roles", "#loop-sdk-host"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ert-hosts", "Bind", "--service-roles", "#loop-ert-svc", "--identity-roles", "#ert-host"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-zet-hosts", "Bind", "--service-roles", "#loop-zet-svc", "--identity-roles", "#zet-host"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ziti-tunnel-hosts", "Bind", "--service-roles", "#loop-zt-svc", "--identity-roles", "#ziti-tunnel-host"))
|
||||
|
||||
// Dial policy: the sim client dials every loop service.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-clients", "Dial", "--service-roles", "#loop-svc", "--identity-roles", "#loop-client"))
|
||||
|
||||
// Dial policy: the ziti-tunnel proxy clients dial loop-ziti-tunnel, so a co-located loop4 dialer can
|
||||
// push traffic through the tunneler's local proxy listener (exercises the ziti-tunnel client path).
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ziti-tunnel-clients", "Dial", "--service-roles", "#loop-zt-svc", "--identity-roles", "#ziti-tunnel-client"))
|
||||
|
||||
// Dial policy: the ERT proxy client (router-east-1) dials loop-ert, so its proxy listener + a
|
||||
// co-located loop4 dialer exercise the ERT client path.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-ert-clients", "Dial", "--service-roles", "#loop-ert-svc", "--identity-roles", "#ert-proxy-client"))
|
||||
|
||||
// Dial policy: the ZET clients dial loop-zet so their tproxy intercept + a co-located loop4 dialer
|
||||
// exercise the ZET client path.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "loop-zet-clients", "Dial", "--service-roles", "#loop-zet-svc", "--identity-roles", "#zet-client"))
|
||||
|
||||
// Sim control-plane services (metrics reporting + scenario control), hosted by the
|
||||
// sim-controller identity created during activation.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service", "metrics", "-a", "sim-services"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service", "sim-control", "-a", "sim-services"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "sim-service-hosts", "Bind", "--service-roles", "#sim-services", "--identity-roles", "#sim-services-host"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "sim-service-clients", "Dial", "--service-roles", "#sim-services", "--identity-roles", "#sim-services-client"))
|
||||
|
||||
// Broad routing: every identity may use every edge router, and every service may be
|
||||
// routed over every edge router. Simple and sufficient for the upgrade test.
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "edge-router-policy", "all-endpoints", "--edge-router-roles", "#all", "--identity-roles", "#all"))
|
||||
workflow.AddAction(zitilib_actions.Edge("create", "service-edge-router-policy", "all-services", "--service-roles", "#all", "--edge-router-roles", "#all"))
|
||||
|
||||
return workflow
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/openziti/fablab/kernel/lib/tui"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/foundation/v2/netz"
|
||||
"github.com/openziti/ziti/zititest/zitilab"
|
||||
"github.com/openziti/ziti/zititest/zitilab/actions/edge"
|
||||
"github.com/openziti/ziti/zititest/zitilab/chaos"
|
||||
)
|
||||
|
||||
// cluster upgrade modes, selected by the clusterUpgradeMode variable.
|
||||
const (
|
||||
// clusterUpgradeRolling restarts one node at a time, so the cluster runs mixed versions until
|
||||
// the last node is done.
|
||||
clusterUpgradeRolling = "rolling"
|
||||
// clusterUpgradeAllAtOnce restarts every node together, so no node sees a version mismatch but
|
||||
// every node pays its cold-start cost simultaneously.
|
||||
clusterUpgradeAllAtOnce = "all-at-once"
|
||||
)
|
||||
|
||||
const (
|
||||
// clusterPortUpTimeout bounds how long a restarted node may take to listen again. A node that
|
||||
// keeps dying and restarting can flap the port open, so this is a floor on detection, not a
|
||||
// health check; clusterSettleTimeout is what actually decides health.
|
||||
clusterPortUpTimeout = 2 * time.Minute
|
||||
// clusterSettleTimeout bounds how long the cluster may take to report every peer connected with
|
||||
// a leader elected. A node stuck in a crash loop never gets there, which is how a node that
|
||||
// cannot stay up surfaces as a failure rather than a hang.
|
||||
clusterSettleTimeout = 3 * time.Minute
|
||||
)
|
||||
|
||||
// ansiEscape matches the terminal color codes the inspect command emits, which are not stripped when
|
||||
// its output is captured over ssh on some platforms and would otherwise be taken for JSON.
|
||||
var ansiEscape = regexp.MustCompile(`\x1b\[[0-9;]*[a-zA-Z]`)
|
||||
|
||||
// ctrlPeer is the part of a controller's connected-peers inspection this model uses.
|
||||
type ctrlPeer struct {
|
||||
Id string `json:"id"`
|
||||
IsLeader bool `json:"isLeader"`
|
||||
IsConnected bool `json:"isConnected"`
|
||||
}
|
||||
|
||||
// inspectPeers returns c's own view of the cluster. The inspect request is made with c's own binary,
|
||||
// so the agent client always matches the controller it is talking to even while a rolling upgrade has
|
||||
// different nodes on different versions.
|
||||
func inspectPeers(c *model.Component) ([]ctrlPeer, error) {
|
||||
ctrlType, ok := c.Type.(*zitilab.ControllerType)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("component %s is not a controller", c.Id)
|
||||
}
|
||||
cmd := fmt.Sprintf("%s agent inspect -a %s connected-peers", ctrlType.GetBinaryPath(c), c.Id)
|
||||
out, err := c.GetHost().ExecLogged(cmd)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to inspect peers on %s: %w", c.Id, err)
|
||||
}
|
||||
out = ansiEscape.ReplaceAllString(out, "")
|
||||
start := strings.Index(out, "[")
|
||||
end := strings.LastIndex(out, "]")
|
||||
if start < 0 || end < start {
|
||||
return nil, fmt.Errorf("no peer list in the inspect output from %s: %s", c.Id, strings.TrimSpace(out))
|
||||
}
|
||||
var peers []ctrlPeer
|
||||
if err := json.Unmarshal([]byte(out[start:end+1]), &peers); err != nil {
|
||||
return nil, fmt.Errorf("unparsable peer list from %s: %w", c.Id, err)
|
||||
}
|
||||
return peers, nil
|
||||
}
|
||||
|
||||
// clusterUpgradeOrder returns ctrls with the leader last, the ordering a rolling upgrade wants:
|
||||
// restarting a follower costs nothing, while restarting the leader forces an election. When no node
|
||||
// reports a leader, or reports one under an id that is not a component id, the bootstrap node stands
|
||||
// in: it seeded the cluster and the others joined it, so it is the node most likely to be leading.
|
||||
func clusterUpgradeOrder(ctrls []*model.Component) []*model.Component {
|
||||
leaderId := ""
|
||||
for _, c := range ctrls {
|
||||
peers, err := inspectPeers(c)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, p := range peers {
|
||||
if p.IsLeader {
|
||||
leaderId = p.Id
|
||||
}
|
||||
}
|
||||
if leaderId != "" {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
var followers, leader []*model.Component
|
||||
for _, c := range ctrls {
|
||||
if c.Id == leaderId {
|
||||
leader = append(leader, c)
|
||||
} else {
|
||||
followers = append(followers, c)
|
||||
}
|
||||
}
|
||||
if len(leader) == 0 {
|
||||
for i, c := range followers {
|
||||
if c.HasTag("bootstrap-ctrl") {
|
||||
followers = append(followers[:i], followers[i+1:]...)
|
||||
leader = append(leader, c)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return append(followers, leader...)
|
||||
}
|
||||
|
||||
// checkClusterConnected reports whether every controller sees the full peer set connected with a
|
||||
// leader elected.
|
||||
func checkClusterConnected(ctrls []*model.Component) error {
|
||||
for _, c := range ctrls {
|
||||
peers, err := inspectPeers(c)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(peers) != len(ctrls) {
|
||||
return fmt.Errorf("%s sees %d peers, expected %d", c.Id, len(peers), len(ctrls))
|
||||
}
|
||||
hasLeader := false
|
||||
for _, p := range peers {
|
||||
if !p.IsConnected {
|
||||
return fmt.Errorf("%s reports peer %s as not connected", c.Id, p.Id)
|
||||
}
|
||||
hasLeader = hasLeader || p.IsLeader
|
||||
}
|
||||
if !hasLeader {
|
||||
return fmt.Errorf("%s reports no leader", c.Id)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// waitForClusterConnected blocks until the cluster reports itself healthy or timeout elapses.
|
||||
func waitForClusterConnected(ctrls []*model.Component, timeout time.Duration) error {
|
||||
log := tui.ValidationLogger()
|
||||
deadline := time.Now().Add(timeout)
|
||||
var lastErr error
|
||||
for {
|
||||
lastErr = checkClusterConnected(ctrls)
|
||||
if lastErr == nil {
|
||||
log.Infof("cluster reports %d peers connected with a leader", len(ctrls))
|
||||
return nil
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("cluster did not settle within %s: %w", timeout, lastErr)
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// restartCtrlOnVersion swaps c's binary to version and restarts it, waiting for its cluster port to
|
||||
// come back. The config is untouched: a cluster node's config does not depend on the ziti version.
|
||||
func restartCtrlOnVersion(run model.Run, c *model.Component, version string) error {
|
||||
ctrlType, ok := c.Type.(*zitilab.ControllerType)
|
||||
if !ok {
|
||||
return fmt.Errorf("component %s is not a controller", c.Id)
|
||||
}
|
||||
ctrlType.SetVersion(version)
|
||||
|
||||
// RestartSelected waits for the old process to exit before starting the new one; a plain
|
||||
// stop/start would race, since Start no-ops while the old process is still shutting down
|
||||
if err := chaos.RestartSelected(run, 1, c); err != nil {
|
||||
return err
|
||||
}
|
||||
hostPort := c.GetHost().PublicIp + ":6262"
|
||||
if err := netz.WaitForPortActive(hostPort, clusterPortUpTimeout); err != nil {
|
||||
return fmt.Errorf("%s cluster port did not come back after upgrading to %s: %w", c.Id, version, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollingClusterUpgrade upgrades one node at a time, waiting for the cluster to settle before moving
|
||||
// on, so a node that cannot survive the upgrade fails the phase at the node that broke.
|
||||
func rollingClusterUpgrade(run model.Run, ctrls []*model.Component, version string) error {
|
||||
log := tui.ValidationLogger()
|
||||
for _, c := range clusterUpgradeOrder(ctrls) {
|
||||
log.Infof("upgrading cluster node %s to %s", c.Id, version)
|
||||
if err := restartCtrlOnVersion(run, c, version); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := waitForClusterConnected(ctrls, clusterSettleTimeout); err != nil {
|
||||
return fmt.Errorf("cluster unhealthy after upgrading %s to %s: %w", c.Id, version, err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// allAtOnceClusterUpgrade restarts every node on the new version together, so the cluster never runs
|
||||
// mixed versions.
|
||||
func allAtOnceClusterUpgrade(run model.Run, ctrls []*model.Component, version string) error {
|
||||
log := tui.ValidationLogger()
|
||||
for _, c := range ctrls {
|
||||
ctrlType, ok := c.Type.(*zitilab.ControllerType)
|
||||
if !ok {
|
||||
return fmt.Errorf("component %s is not a controller", c.Id)
|
||||
}
|
||||
ctrlType.SetVersion(version)
|
||||
}
|
||||
|
||||
log.Infof("restarting all %d cluster nodes on %s at once", len(ctrls), version)
|
||||
if err := chaos.RestartSelected(run, len(ctrls), ctrls...); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, c := range ctrls {
|
||||
hostPort := c.GetHost().PublicIp + ":6262"
|
||||
if err := netz.WaitForPortActive(hostPort, clusterPortUpTimeout); err != nil {
|
||||
return fmt.Errorf("%s cluster port did not come back after upgrading to %s: %w", c.Id, version, err)
|
||||
}
|
||||
}
|
||||
return waitForClusterConnected(ctrls, clusterSettleTimeout)
|
||||
}
|
||||
|
||||
// upgradeClusterControllers upgrades the whole controller cluster from toVersion to nextVersion,
|
||||
// reporting each node's peak memory over the upgrade. clusterUpgradeMode selects between a rolling
|
||||
// upgrade and an all-at-once one.
|
||||
//
|
||||
// No steady-state gate runs between nodes in rolling mode. Peers hold the cluster read-only while a
|
||||
// version mismatch exists, so anything needing a write would fail for reasons this phase is not
|
||||
// testing; the gate runs once the whole cluster is on nextVersion.
|
||||
func upgradeClusterControllers(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
nextVersion := m.GetStringVariableOr("nextVersion", "")
|
||||
if nextVersion == "" {
|
||||
log.Infof("nextVersion is not set, skipping the cluster upgrade")
|
||||
return nil
|
||||
}
|
||||
|
||||
ctrls := ctrlComponents(m)
|
||||
if len(ctrls) == 0 {
|
||||
return fmt.Errorf("no controllers found to upgrade")
|
||||
}
|
||||
|
||||
mode := m.GetStringVariableOr("clusterUpgradeMode", clusterUpgradeRolling)
|
||||
// the steady-state gate just ran, so the window immediately behind us is settled traffic on
|
||||
// toVersion, which is the baseline each node's upgrade is measured against
|
||||
upgradeStart := time.Now()
|
||||
baselineStart := upgradeStart.Add(-ctrlMemBaselineWindow)
|
||||
|
||||
var err error
|
||||
switch mode {
|
||||
case clusterUpgradeRolling:
|
||||
err = rollingClusterUpgrade(run, ctrls, nextVersion)
|
||||
case clusterUpgradeAllAtOnce:
|
||||
err = allAtOnceClusterUpgrade(run, ctrls, nextVersion)
|
||||
default:
|
||||
return fmt.Errorf("unknown clusterUpgradeMode [%s], expected %s or %s",
|
||||
mode, clusterUpgradeRolling, clusterUpgradeAllAtOnce)
|
||||
}
|
||||
|
||||
// report either way: a node that died on the way up is the case the samples exist to explain
|
||||
if reportErr := reportCtrlMemoryUpgrade(run, baselineStart, upgradeStart, "cluster upgrade to "+nextVersion); reportErr != nil {
|
||||
if err == nil {
|
||||
return reportErr
|
||||
}
|
||||
log.WithError(reportErr).Warn("controller memory report after a failed cluster upgrade")
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return edge.Login("#ctrl1").Execute(run)
|
||||
}
|
||||
|
||||
// upgradeClusterRouters rolls the routers onto nextVersion, completing the upgrade the controllers
|
||||
// started.
|
||||
func upgradeClusterRouters(run model.Run) error {
|
||||
nextVersion := run.GetModel().GetStringVariableOr("nextVersion", "")
|
||||
if nextVersion == "" {
|
||||
tui.ValidationLogger().Infof("nextVersion is not set, skipping the router upgrade")
|
||||
return nil
|
||||
}
|
||||
return upgradeRoutersTo(run, nextVersion)
|
||||
}
|
||||
|
||||
// upgradeToNextVersion runs the optional cluster upgrade phase: the controllers, then the routers,
|
||||
// with a steady-state gate after each. The whole phase is skipped when nextVersion is unset, so an
|
||||
// iteration that ends at toVersion does not pay for two extra gates.
|
||||
func upgradeToNextVersion(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
if m.GetStringVariableOr("nextVersion", "") == "" {
|
||||
tui.ValidationLogger().Infof("nextVersion is not set, skipping the cluster upgrade phase")
|
||||
return nil
|
||||
}
|
||||
return m.Exec(run,
|
||||
"upgradeClusterControllers", // 3-node cluster toVersion -> nextVersion
|
||||
"validateSteadyStateAfterDisruption", // cluster back, terminators + traffic recovered
|
||||
"upgradeClusterRouters", // routers toVersion -> nextVersion, rolling one at a time
|
||||
"validateSteadyStateAfterDisruption", // whole system on nextVersion and clean
|
||||
)
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
v: 3
|
||||
|
||||
{{if .Component.GetFlag "cluster"}}
|
||||
cluster:
|
||||
dataDir: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrldata
|
||||
# the standalone->HA migration seeds raft from the source db via a snapshot, which requires a
|
||||
# controller restart to apply; restart ourselves so no external restart is needed.
|
||||
restartSelfOnSnapshot: true
|
||||
{{if .Component.GetFlag "migrateDb"}}
|
||||
# migrate the existing standalone bbolt db into raft on first cluster start (bootstrap node only)
|
||||
db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrl.db
|
||||
{{end}}
|
||||
{{else}}
|
||||
db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrl.db
|
||||
routerDataModel:
|
||||
enabled: true
|
||||
{{end}}
|
||||
|
||||
identity:
|
||||
cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem
|
||||
key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key
|
||||
ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.chain.pem
|
||||
|
||||
trustDomain: upgrade-test
|
||||
|
||||
# the endpoint that routers will connect to the controller over.
|
||||
ctrl:
|
||||
listener: tls:0.0.0.0:6262
|
||||
options:
|
||||
advertiseAddress: tls:{{ .Host.PublicIp }}:6262
|
||||
|
||||
events:
|
||||
entityChangeEventsLogger:
|
||||
subscriptions:
|
||||
- type: entityChange
|
||||
propagateAlways: true
|
||||
handler:
|
||||
type: file
|
||||
format: json
|
||||
path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/entity-change.log
|
||||
|
||||
circuitEventsLogger:
|
||||
subscriptions:
|
||||
- type: circuit
|
||||
handler:
|
||||
type: file
|
||||
format: json
|
||||
path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/circuits.log
|
||||
|
||||
linkEventsLogger:
|
||||
subscriptions:
|
||||
- type: link
|
||||
handler:
|
||||
type: file
|
||||
format: json
|
||||
path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/links.log
|
||||
|
||||
routerEventsLogger:
|
||||
subscriptions:
|
||||
- type: router
|
||||
handler:
|
||||
type: file
|
||||
format: json
|
||||
path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/router.log
|
||||
|
||||
# api-session events carry the session type (legacy vs jwt), keyed by identity and source IP.
|
||||
# The upgrade test reads this log to confirm a restarted client re-authenticated via OIDC (jwt).
|
||||
apiSessionEventsLogger:
|
||||
subscriptions:
|
||||
- type: apiSession
|
||||
handler:
|
||||
type: file
|
||||
format: json
|
||||
path: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/logs/api-sessions.log
|
||||
|
||||
healthChecks:
|
||||
boltCheck:
|
||||
interval: 30s
|
||||
timeout: 15s
|
||||
initialDelay: 15s
|
||||
|
||||
edge:
|
||||
api:
|
||||
sessionTimeout: 30m
|
||||
address: {{ .Host.PublicIp }}:1280
|
||||
enrollment:
|
||||
signingCert:
|
||||
cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert
|
||||
key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key
|
||||
edgeIdentity:
|
||||
duration: 5m
|
||||
edgeRouter:
|
||||
duration: 5m
|
||||
|
||||
web:
|
||||
- name: all-apis-localhost
|
||||
bindPoints:
|
||||
- interface: 0.0.0.0:1280
|
||||
address: {{ .Host.PublicIp }}:1280
|
||||
options:
|
||||
idleTimeout: 5000ms
|
||||
readTimeout: 5000ms
|
||||
writeTimeout: 100000ms
|
||||
minTLSVersion: TLS1.2
|
||||
maxTLSVersion: TLS1.3
|
||||
apis:
|
||||
- binding: health-checks
|
||||
- binding: fabric
|
||||
- binding: edge-management
|
||||
- binding: edge-client
|
||||
{{if .Component.GetFlag "oidc"}}
|
||||
- binding: edge-oidc
|
||||
{{end}}
|
||||
@@ -0,0 +1,11 @@
|
||||
# Plain-TCP loop4 listener that sits behind a ZET or ziti-tunnel host. The tunneler binds
|
||||
# the ziti service (loop-zet / loop-ziti-tunnel) and forwards it to localhost:3456, where
|
||||
# this listener speaks the loop4 protocol back to the remote dialer.
|
||||
connectors:
|
||||
backend:
|
||||
transport:
|
||||
address: tcp:127.0.0.1:3456
|
||||
|
||||
workloads:
|
||||
- name: loop-backend
|
||||
connector: backend
|
||||
@@ -0,0 +1,42 @@
|
||||
connectors:
|
||||
default:
|
||||
sdk:
|
||||
identity_file: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/cfg/{{ .Component.Id }}.json
|
||||
report_sdk_metrics: true
|
||||
ha: {{ .Component.GetFlag "ha" }}
|
||||
max_control_underlays: 1
|
||||
test_service: loop-sdk
|
||||
|
||||
remoteControlled:
|
||||
connector: default
|
||||
service: sim-control
|
||||
|
||||
metrics:
|
||||
connector: default
|
||||
service: metrics
|
||||
interval: 5s
|
||||
clientId: {{ .Component.Id }}
|
||||
|
||||
# One workload per hosting flavor. Each dials the ziti service that is hosted by the
|
||||
# corresponding host type (SDK, ERT, ZET, ziti-tunnel). A scenario run executes every
|
||||
# workload once, so a clean run means every flavor's data path is currently healthy.
|
||||
workloads:
|
||||
- name: loop-sdk
|
||||
service_name: loop-sdk
|
||||
connector: default
|
||||
{{ $.Model.MustVariable "livenessWorkload" }}
|
||||
|
||||
- name: loop-ert
|
||||
service_name: loop-ert
|
||||
connector: default
|
||||
{{ $.Model.MustVariable "livenessWorkload" }}
|
||||
|
||||
- name: loop-zet
|
||||
service_name: loop-zet
|
||||
connector: default
|
||||
{{ $.Model.MustVariable "livenessWorkload" }}
|
||||
|
||||
- name: loop-ziti-tunnel
|
||||
service_name: loop-ziti-tunnel
|
||||
connector: default
|
||||
{{ $.Model.MustVariable "livenessWorkload" }}
|
||||
@@ -0,0 +1,13 @@
|
||||
# loop4 SDK listener: binds the loop-sdk ziti service directly via the Go SDK. Deployed as a pair
|
||||
# so one instance can be restarted (to observe OIDC re-auth) while the other keeps a terminator up.
|
||||
connectors:
|
||||
default:
|
||||
sdk:
|
||||
identity_file: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/cfg/{{ .Component.Id }}.json
|
||||
ha: {{ .Component.GetFlag "ha" }}
|
||||
max_control_underlays: 1
|
||||
|
||||
workloads:
|
||||
- name: loop-sdk
|
||||
connector: default
|
||||
service_name: loop-sdk
|
||||
@@ -0,0 +1,30 @@
|
||||
# A loop4 dialer that drives traffic through a co-located tunneler's local listener instead of
|
||||
# dialing a ziti service directly. The 'default' SDK connector is only used for sim control-plane
|
||||
# (scenario control + metrics); the workload runs over the 'viaTunnel' transport connector, which
|
||||
# connects to the tunneler's local TCP port. That port forwards through the tunneler client into the
|
||||
# ziti service and on to the loop4 backend, exercising the tunneler's client-side data path.
|
||||
connectors:
|
||||
default:
|
||||
sdk:
|
||||
identity_file: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/cfg/{{ .Component.Id }}.json
|
||||
report_sdk_metrics: true
|
||||
ha: {{ .Component.GetFlag "ha" }}
|
||||
max_control_underlays: 1
|
||||
viaTunnel:
|
||||
transport:
|
||||
address: {{ .Component.MustStringVariable "loopTunnelAddress" }}
|
||||
|
||||
remoteControlled:
|
||||
connector: default
|
||||
service: sim-control
|
||||
|
||||
metrics:
|
||||
connector: default
|
||||
service: metrics
|
||||
interval: 5s
|
||||
clientId: {{ .Component.Id }}
|
||||
|
||||
workloads:
|
||||
- name: via-tunnel
|
||||
connector: viaTunnel
|
||||
{{ $.Model.MustVariable "livenessWorkload" }}
|
||||
@@ -0,0 +1,78 @@
|
||||
{{$ssh_username := .Model.MustVariable "credentials.ssh.username"}}
|
||||
{{$identity := .Component.Id}}
|
||||
{{$router_ip := .Host.PublicIp}}
|
||||
|
||||
v: 3
|
||||
|
||||
enableDebugOps: true
|
||||
|
||||
{{if .Component.GetFlag "ha"}}
|
||||
ha:
|
||||
enabled: true
|
||||
{{end}}
|
||||
|
||||
identity:
|
||||
cert: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-client.cert
|
||||
server_cert: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.cert
|
||||
key: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}.key
|
||||
ca: /home/{{$ssh_username}}/fablab/cfg/{{$identity}}-server.chain.pem
|
||||
|
||||
tls:
|
||||
handshakeTimeout: 30s
|
||||
|
||||
# Only the initial controller is listed here. Additional cluster members (added during the HA
|
||||
# phase) are learned dynamically from the controller once it is running in cluster mode.
|
||||
ctrl:
|
||||
endpoints: {{ range $host := .Model.MustSelectHosts "component.bootstrap-ctrl" 1 }}
|
||||
- tls:{{ $host.PublicIp }}:6262{{end}}
|
||||
startupTimeout: 5m
|
||||
|
||||
healthChecks:
|
||||
ctrlPingCheck:
|
||||
interval: 30s
|
||||
timeout: 15s
|
||||
initialDelay: 15s
|
||||
|
||||
metrics:
|
||||
reportInterval: 15s
|
||||
messageQueueSize: 10
|
||||
|
||||
link:
|
||||
listeners:
|
||||
- binding: transport
|
||||
bind: tls:0.0.0.0:6000
|
||||
advertise: tls:{{$router_ip}}:6000
|
||||
dialers:
|
||||
- binding: transport
|
||||
options:
|
||||
connectTimeout: 30s
|
||||
|
||||
listeners:
|
||||
{{if .Component.HasTag "tunneler"}}
|
||||
- binding: tunnel
|
||||
options:
|
||||
{{- if .Component.HasTag "ert-proxy-client"}}
|
||||
# proxy mode: open a local TCP listener for loop-ert so a co-located loop4 dialer can push
|
||||
# traffic through this ERT client (no tproxy/:53, so it can share a host).
|
||||
mode: proxy
|
||||
services:
|
||||
- loop-ert:15390
|
||||
{{- else}}
|
||||
mode: tproxy
|
||||
{{- end}}
|
||||
{{end}}
|
||||
- binding: edge
|
||||
address: tls:0.0.0.0:6262
|
||||
options:
|
||||
advertise: {{ .Host.PublicIp }}:6262
|
||||
|
||||
edge:
|
||||
csr:
|
||||
country: US
|
||||
province: NC
|
||||
locality: Charlotte
|
||||
organization: NetFoundry
|
||||
organizationalUnit: Ziti
|
||||
sans:
|
||||
ip:
|
||||
- {{ .Host.PublicIp }}
|
||||
@@ -0,0 +1,435 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/openziti/fablab/kernel/lib/tui"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/ziti/zititest/zitilab"
|
||||
)
|
||||
|
||||
// memSamplerScript records one "<epoch>,<rss_kb>" line per second for a controller, and captures a
|
||||
// heap profile the first time RSS crosses the soft threshold. The heap path is unique per sampler
|
||||
// start, so the guard means one profile per iteration rather than one per host for the whole run.
|
||||
//
|
||||
// The controller is located by the agent alias fablab passes at start, which is the same
|
||||
// discriminator its process filter uses and is independent of the binary version, so the sampler
|
||||
// keeps following the process across an upgrade. A stopped or restarting controller records a zero
|
||||
// rather than a gap, which is what makes a restart loop visible in the samples.
|
||||
//
|
||||
// The alias only ever appears inside this file, never on the sampler's own command line, so pgrep
|
||||
// cannot match the sampler instead of the controller. The pid is recorded so the sampler can be
|
||||
// stopped without a pattern match, for the same reason.
|
||||
const memSamplerScript = `#!/bin/sh
|
||||
samples=%s
|
||||
heap=%s
|
||||
ziti=%s
|
||||
alias=%s
|
||||
soft_kb=%d
|
||||
echo $$ > %s
|
||||
while true; do
|
||||
pids=$(pgrep -f "cli-agent-alias $alias " 2>/dev/null | tr '\n' ',' | sed 's/,$//')
|
||||
rss=0
|
||||
if [ -n "$pids" ]; then
|
||||
rss=$(ps -o rss= -p "$pids" 2>/dev/null | awk '{s+=$1} END {print s+0}')
|
||||
fi
|
||||
echo "$(date +%%s),$rss" >> "$samples"
|
||||
if [ "$soft_kb" -gt 0 ] && [ "$rss" -gt "$soft_kb" ] && [ ! -f "$heap" ]; then
|
||||
"$ziti" agent pprof-heap -a "$alias" -o "$heap" >/dev/null 2>&1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
`
|
||||
|
||||
// samplerStartMarker prefixes the line written when a sampler starts. It divides the accumulated
|
||||
// samples into one window per iteration, so a report can scope itself without being told when the
|
||||
// iteration began.
|
||||
const samplerStartMarker = "# start,"
|
||||
|
||||
// ctrlMemBaselineWindow is how far back an upgrade report looks for the steady state to compare
|
||||
// against. The steady-state gate runs immediately before the cluster upgrade, so this lands on
|
||||
// settled traffic rather than on the tail of an earlier disruption.
|
||||
const ctrlMemBaselineWindow = 2 * time.Minute
|
||||
|
||||
// ctrlMemPath builds a per-controller path under the host's log directory, which is outside the kit
|
||||
// and so survives the --delete rsync that resetToBaseline runs.
|
||||
func ctrlMemPath(c *model.Component, suffix string) string {
|
||||
return fmt.Sprintf("/home/%s/logs/%s-%s", c.GetHost().GetSshUser(), c.Id, suffix)
|
||||
}
|
||||
|
||||
// ctrlHeapPattern is where c's heap profiles land, one per sampler start.
|
||||
func ctrlHeapPattern(c *model.Component) string {
|
||||
return ctrlMemPath(c, "mem-*.pprof")
|
||||
}
|
||||
|
||||
// ctrlComponents returns the model's controllers sorted by id, so every phase that walks them uses
|
||||
// the same order.
|
||||
func ctrlComponents(m *model.Model) []*model.Component {
|
||||
ctrls := m.SelectComponents(".ctrl")
|
||||
sort.Slice(ctrls, func(i, j int) bool { return ctrls[i].Id < ctrls[j].Id })
|
||||
return ctrls
|
||||
}
|
||||
|
||||
// ctrlMemThresholdKb reads a MiB-valued model variable as KiB, the unit ps reports RSS in. A missing
|
||||
// or unparsable value disables the threshold.
|
||||
func ctrlMemThresholdKb(m *model.Model, name string) int64 {
|
||||
mb, err := strconv.ParseInt(m.GetStringVariableOr(name, ""), 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return mb * 1024
|
||||
}
|
||||
|
||||
// ctrlMemRatio reads a multiplier-valued model variable. A missing or unparsable value, or anything
|
||||
// at or below one, disables the check.
|
||||
func ctrlMemRatio(m *model.Model, name string) float64 {
|
||||
ratio, err := strconv.ParseFloat(m.GetStringVariableOr(name, ""), 64)
|
||||
if err != nil || ratio <= 1 {
|
||||
return 0
|
||||
}
|
||||
return ratio
|
||||
}
|
||||
|
||||
// startCtrlMemorySamplers installs and starts the RSS sampler on every controller host, replacing any
|
||||
// sampler left over from a previous pass. Sampling covers the whole iteration, not just the cluster
|
||||
// upgrade, so a node's upgrade behavior can be compared against its own earlier steady state as well
|
||||
// as against its peers.
|
||||
//
|
||||
// Nothing already recorded is discarded. The samples accumulate across iterations behind a start
|
||||
// marker that scopes reports to the current one, and each start gets its own heap profile path, so an
|
||||
// iteration cannot destroy the evidence an earlier one captured.
|
||||
func startCtrlMemorySamplers(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
softKb := ctrlMemThresholdKb(m, "ctrlMemory.heapDumpAtMb")
|
||||
startedAt := time.Now().Unix()
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
for _, c := range ctrlComponents(m) {
|
||||
samples := ctrlMemPath(c, "mem.csv")
|
||||
heap := ctrlMemPath(c, fmt.Sprintf("mem-%d.pprof", startedAt))
|
||||
scriptPath := ctrlMemSamplerScriptPath(c)
|
||||
script := fmt.Sprintf(memSamplerScript, samples, heap, zitilab.GetZitiBinaryPath(c, ""), c.Id, softKb,
|
||||
ctrlMemSamplerPidPath(c))
|
||||
|
||||
if err := stopCtrlMemorySampler(c); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := c.GetHost().SendData([]byte(script), scriptPath); err != nil {
|
||||
return fmt.Errorf("failed to install the memory sampler on %s: %w", c.Id, err)
|
||||
}
|
||||
// Two commands, not one backgrounded list. Backgrounding a list makes bash fork a subshell
|
||||
// that holds the ssh session's stdout while it waits for the sampler, and the sampler never
|
||||
// exits, so the session never closes. A simple command with its own redirects does not.
|
||||
// The marker is written first, so no sample of this iteration precedes it.
|
||||
marker := fmt.Sprintf("echo '%s%d' >> %s", samplerStartMarker, startedAt, samples)
|
||||
start := fmt.Sprintf("nohup sh %s >/dev/null 2>&1 &", scriptPath)
|
||||
if err := c.GetHost().ExecLogOnlyOnError(marker, start); err != nil {
|
||||
return fmt.Errorf("failed to start the memory sampler on %s: %w", c.Id, err)
|
||||
}
|
||||
log.Infof("controller memory sampler running on %s, samples in %s", c.Id, samples)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ctrlMemSamplerScriptPath is where c's sampler script is installed.
|
||||
func ctrlMemSamplerScriptPath(c *model.Component) string {
|
||||
return ctrlMemPath(c, "mem-sampler.sh")
|
||||
}
|
||||
|
||||
// ctrlMemSamplerPidPath is where c's sampler records its pid so it can be stopped again.
|
||||
func ctrlMemSamplerPidPath(c *model.Component) string {
|
||||
return ctrlMemPath(c, "mem-sampler.pid")
|
||||
}
|
||||
|
||||
// stopCtrlMemorySampler kills c's sampler if one is running, and is a no-op when none is.
|
||||
//
|
||||
// The pid comes from the file the sampler writes, never from a pattern match. ssh runs a command
|
||||
// through a shell whose own command line is the whole command string, so a pkill pattern naming the
|
||||
// sampler matches that shell too and the sampler stop kills its own session. The recorded pid is
|
||||
// checked against the running process before signalling, since a pid left behind by a host reboot
|
||||
// could by then belong to something else.
|
||||
func stopCtrlMemorySampler(c *model.Component) error {
|
||||
pidFile := ctrlMemSamplerPidPath(c)
|
||||
cmd := fmt.Sprintf(
|
||||
`pid=$(cat %s 2>/dev/null); if [ -n "$pid" ] && grep -qasF %s /proc/$pid/cmdline; then kill "$pid"; fi; rm -f %s; true`,
|
||||
pidFile, ctrlMemSamplerScriptPath(c), pidFile)
|
||||
return c.GetHost().ExecLogOnlyOnError(cmd)
|
||||
}
|
||||
|
||||
// stopCtrlMemorySamplers stops sampling on every controller host. Nothing calls this during a run;
|
||||
// it is here so sampling can be paused on a live instance.
|
||||
func stopCtrlMemorySamplers(run model.Run) error {
|
||||
for _, c := range ctrlComponents(run.GetModel()) {
|
||||
if err := stopCtrlMemorySampler(c); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// memSample is one sampler reading.
|
||||
type memSample struct {
|
||||
ts int64
|
||||
rss int64
|
||||
}
|
||||
|
||||
// ctrlMemSeries is everything one controller's sampler has recorded, with the start markers that
|
||||
// divide it into one window per iteration.
|
||||
type ctrlMemSeries struct {
|
||||
c *model.Component
|
||||
samples []memSample
|
||||
starts []int64
|
||||
}
|
||||
|
||||
// ctrlMemStats summarizes a controller's samples over some window.
|
||||
type ctrlMemStats struct {
|
||||
peakKb int64
|
||||
lastKb int64
|
||||
samples int
|
||||
}
|
||||
|
||||
// readCtrlMemSeries fetches and parses everything c's sampler has recorded. The whole file is read
|
||||
// once so a caller can summarize several windows of it without going back over the network.
|
||||
func readCtrlMemSeries(c *model.Component) (*ctrlMemSeries, error) {
|
||||
out, err := c.GetHost().ExecLogged("cat " + ctrlMemPath(c, "mem.csv"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read memory samples for %s: %w", c.Id, err)
|
||||
}
|
||||
samples, starts := parseCtrlMemSeries(out)
|
||||
return &ctrlMemSeries{c: c, samples: samples, starts: starts}, nil
|
||||
}
|
||||
|
||||
// parseCtrlMemSeries splits sampler output into samples and the timestamps of the start markers
|
||||
// between them. Unparsable lines are skipped: the file is appended to by a shell loop that can be
|
||||
// killed mid-write, so a torn final line is normal rather than a fault.
|
||||
func parseCtrlMemSeries(out string) ([]memSample, []int64) {
|
||||
var samples []memSample
|
||||
var starts []int64
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if value, isMarker := strings.CutPrefix(line, samplerStartMarker); isMarker {
|
||||
if ts, err := strconv.ParseInt(value, 10, 64); err == nil {
|
||||
starts = append(starts, ts)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if sample, ok := parseMemSample(line); ok {
|
||||
samples = append(samples, sample)
|
||||
}
|
||||
}
|
||||
return samples, starts
|
||||
}
|
||||
|
||||
// parseMemSample parses one "<epoch>,<rss_kb>" sampler line, reporting false for anything else.
|
||||
func parseMemSample(line string) (memSample, bool) {
|
||||
tsField, rssField, found := strings.Cut(strings.TrimSpace(line), ",")
|
||||
if !found {
|
||||
return memSample{}, false
|
||||
}
|
||||
ts, err := strconv.ParseInt(tsField, 10, 64)
|
||||
if err != nil {
|
||||
return memSample{}, false
|
||||
}
|
||||
rss, err := strconv.ParseInt(rssField, 10, 64)
|
||||
if err != nil {
|
||||
return memSample{}, false
|
||||
}
|
||||
return memSample{ts: ts, rss: rss}, true
|
||||
}
|
||||
|
||||
// statsIn summarizes the samples in [from, to). A zero from starts at the most recent sampler start,
|
||||
// which scopes a report to the current iteration; a zero to runs to the newest sample.
|
||||
func (self *ctrlMemSeries) statsIn(from, to time.Time) ctrlMemStats {
|
||||
start := self.currentWindowStart()
|
||||
if !from.IsZero() {
|
||||
start = from.Unix()
|
||||
}
|
||||
end := int64(0)
|
||||
if !to.IsZero() {
|
||||
end = to.Unix()
|
||||
}
|
||||
|
||||
var stats ctrlMemStats
|
||||
for _, sample := range self.samples {
|
||||
if sample.ts < start || (end != 0 && sample.ts >= end) {
|
||||
continue
|
||||
}
|
||||
stats.samples++
|
||||
stats.lastKb = sample.rss
|
||||
if sample.rss > stats.peakKb {
|
||||
stats.peakKb = sample.rss
|
||||
}
|
||||
}
|
||||
return stats
|
||||
}
|
||||
|
||||
// currentWindowStart returns the timestamp of the most recent sampler start. A series with no marker
|
||||
// yields a start that admits every sample, which is what a sampler predating the marker leaves behind.
|
||||
func (self *ctrlMemSeries) currentWindowStart() int64 {
|
||||
if len(self.starts) == 0 {
|
||||
return 0
|
||||
}
|
||||
return self.starts[len(self.starts)-1]
|
||||
}
|
||||
|
||||
// mib renders a KiB reading the way the reports talk about memory.
|
||||
func mib(kb int64) int64 {
|
||||
return kb / 1024
|
||||
}
|
||||
|
||||
// reportCtrlMemory logs peak and current RSS for every controller over the window opening at since (a
|
||||
// zero since means the current iteration), and fails when a controller's peak exceeds
|
||||
// ctrlMemory.failAtMb.
|
||||
func reportCtrlMemory(run model.Run, since time.Time, phase string) error {
|
||||
m := run.GetModel()
|
||||
log := tui.ValidationLogger()
|
||||
failKb := ctrlMemThresholdKb(m, "ctrlMemory.failAtMb")
|
||||
|
||||
var worstPeak int64
|
||||
var worst *model.Component
|
||||
|
||||
for _, c := range ctrlComponents(m) {
|
||||
series, err := readCtrlMemSeries(c)
|
||||
if err != nil {
|
||||
log.WithError(err).Warnf("no memory samples for %s", c.Id)
|
||||
continue
|
||||
}
|
||||
stats := series.statsIn(since, time.Time{})
|
||||
if stats.samples == 0 {
|
||||
log.Warnf("controller memory [%s] %s: no samples", phase, c.Id)
|
||||
continue
|
||||
}
|
||||
log.Infof("controller memory [%s] %s: peak %d MiB, current %d MiB, %d samples",
|
||||
phase, c.Id, mib(stats.peakKb), mib(stats.lastKb), stats.samples)
|
||||
if stats.peakKb > worstPeak {
|
||||
worstPeak, worst = stats.peakKb, c
|
||||
}
|
||||
}
|
||||
|
||||
if failKb > 0 && worstPeak > failKb {
|
||||
return fmt.Errorf("controller %s peaked at %d MiB during %s, over the %d MiB limit; heap profiles, if any were captured, are at %s",
|
||||
worst.Id, mib(worstPeak), phase, mib(failKb), ctrlHeapPattern(worst))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// reportCtrlMemoryUpgrade reports each controller's memory across an upgrade against its own steady
|
||||
// state immediately before it, and fails when a node exceeds either ctrlMemory.failAtRatio or the
|
||||
// absolute ctrlMemory.failAtMb ceiling. The baseline window is [baselineStart, upgradeStart) and the
|
||||
// upgrade window runs from upgradeStart to now.
|
||||
//
|
||||
// The ratio is the check that bites at this model's scale. The absolute ceiling is sized for the
|
||||
// failure #4219 reports, which needs a far larger data set to reach, whereas a node using several
|
||||
// times its own steady state is visible whatever the data set is.
|
||||
func reportCtrlMemoryUpgrade(run model.Run, baselineStart, upgradeStart time.Time, phase string) error {
|
||||
m := run.GetModel()
|
||||
log := tui.ValidationLogger()
|
||||
failKb := ctrlMemThresholdKb(m, "ctrlMemory.failAtMb")
|
||||
failRatio := ctrlMemRatio(m, "ctrlMemory.failAtRatio")
|
||||
|
||||
var violations []string
|
||||
for _, c := range ctrlComponents(m) {
|
||||
series, err := readCtrlMemSeries(c)
|
||||
if err != nil {
|
||||
log.WithError(err).Warnf("no memory samples for %s", c.Id)
|
||||
continue
|
||||
}
|
||||
base := series.statsIn(baselineStart, upgradeStart)
|
||||
upgrade := series.statsIn(upgradeStart, time.Time{})
|
||||
if upgrade.samples == 0 {
|
||||
log.Warnf("controller memory [%s] %s: no samples", phase, c.Id)
|
||||
continue
|
||||
}
|
||||
|
||||
growth := "n/a"
|
||||
if base.peakKb > 0 {
|
||||
growth = fmt.Sprintf("%.1fx", float64(upgrade.peakKb)/float64(base.peakKb))
|
||||
}
|
||||
log.Infof("controller memory [%s] %s: baseline peak %d MiB, upgrade peak %d MiB (%s), current %d MiB",
|
||||
phase, c.Id, mib(base.peakKb), mib(upgrade.peakKb), growth, mib(upgrade.lastKb))
|
||||
|
||||
before := len(violations)
|
||||
if failRatio > 0 && base.peakKb > 0 && float64(upgrade.peakKb) > failRatio*float64(base.peakKb) {
|
||||
violations = append(violations, fmt.Sprintf(
|
||||
"%s grew to %s of its %d MiB baseline (%d MiB), over the %.1fx limit",
|
||||
c.Id, growth, mib(base.peakKb), mib(upgrade.peakKb), failRatio))
|
||||
}
|
||||
if failKb > 0 && upgrade.peakKb > failKb {
|
||||
violations = append(violations, fmt.Sprintf("%s peaked at %d MiB, over the %d MiB limit",
|
||||
c.Id, mib(upgrade.peakKb), mib(failKb)))
|
||||
}
|
||||
if len(violations) > before {
|
||||
log.Infof("heap profiles for %s, if any were captured, are at %s", c.Id, ctrlHeapPattern(c))
|
||||
}
|
||||
}
|
||||
|
||||
if len(violations) > 0 {
|
||||
return fmt.Errorf("controller memory grew unexpectedly during %s: %s", phase, strings.Join(violations, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// summarizeCtrlMemory logs one row per controller per sampler run, which is one row per iteration, so
|
||||
// a finished multi-iteration run can be read back with a single command instead of scrolled for.
|
||||
func summarizeCtrlMemory(run model.Run) error {
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
var worstPeak int64
|
||||
worst := ""
|
||||
for _, c := range ctrlComponents(run.GetModel()) {
|
||||
series, err := readCtrlMemSeries(c)
|
||||
if err != nil {
|
||||
log.WithError(err).Warnf("no memory samples for %s", c.Id)
|
||||
continue
|
||||
}
|
||||
if len(series.samples) == 0 {
|
||||
log.Warnf("controller memory summary: %s has no samples", c.Id)
|
||||
continue
|
||||
}
|
||||
|
||||
// a series with no marker predates them, so it is reported as the single window it is
|
||||
starts := series.starts
|
||||
if len(starts) == 0 {
|
||||
starts = []int64{series.samples[0].ts}
|
||||
}
|
||||
for i, start := range starts {
|
||||
from := time.Unix(start, 0)
|
||||
to := time.Time{}
|
||||
if i+1 < len(starts) {
|
||||
to = time.Unix(starts[i+1], 0)
|
||||
}
|
||||
stats := series.statsIn(from, to)
|
||||
log.Infof("controller memory summary: %s iteration %d (%s): peak %d MiB, final %d MiB, %d samples",
|
||||
c.Id, i+1, from.Format(time.RFC3339), mib(stats.peakKb), mib(stats.lastKb), stats.samples)
|
||||
if stats.peakKb > worstPeak {
|
||||
worstPeak = stats.peakKb
|
||||
worst = fmt.Sprintf("%s in iteration %d", c.Id, i+1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if worst != "" {
|
||||
log.Infof("controller memory summary: highest peak was %d MiB, %s", mib(worstPeak), worst)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// sampleFile is two iterations of sampler output: a controller that idles around 100 MiB, restarts
|
||||
// (the zeros), and comes back heavier in the second iteration.
|
||||
const sampleFile = `# start,1000
|
||||
1001,102400
|
||||
1002,104448
|
||||
1003,0
|
||||
1004,106496
|
||||
# start,2000
|
||||
2001,102400
|
||||
2002,0
|
||||
2003,512000
|
||||
2004,460800
|
||||
`
|
||||
|
||||
// TestParseCtrlMemSeries pins that the start markers are read as markers and not as samples. They
|
||||
// share the file with the samples, so a marker parsed as one would read a timestamp as a resident set
|
||||
// size and report a peak of about 1.6 TiB.
|
||||
func TestParseCtrlMemSeries(t *testing.T) {
|
||||
samples, starts := parseCtrlMemSeries(sampleFile)
|
||||
|
||||
require.Equal(t, []int64{1000, 2000}, starts)
|
||||
require.Len(t, samples, 8)
|
||||
require.Equal(t, memSample{ts: 1001, rss: 102400}, samples[0])
|
||||
require.Equal(t, memSample{ts: 2004, rss: 460800}, samples[7])
|
||||
}
|
||||
|
||||
// TestParseMemSampleRejectsNonSamples pins what does not count as a sample. A torn final line is
|
||||
// normal, since the file is appended to by a shell loop that can be killed mid-write.
|
||||
func TestParseMemSampleRejectsNonSamples(t *testing.T) {
|
||||
for _, line := range []string{"", "# start,1700000000", "garbage", "1700000000", "abc,123", "123,abc"} {
|
||||
_, ok := parseMemSample(line)
|
||||
require.False(t, ok, "must not parse [%s] as a sample", line)
|
||||
}
|
||||
}
|
||||
|
||||
// TestStatsInDefaultsToTheCurrentIteration pins the window a report defaults to. Samples accumulate
|
||||
// across iterations in one file, so starting from the first marker rather than the last would report a
|
||||
// peak from an earlier iteration as if it belonged to this one.
|
||||
func TestStatsInDefaultsToTheCurrentIteration(t *testing.T) {
|
||||
series := testSeries(t, sampleFile)
|
||||
|
||||
stats := series.statsIn(time.Time{}, time.Time{})
|
||||
require.Equal(t, 4, stats.samples, "only the samples after the last marker")
|
||||
require.Equal(t, int64(512000), stats.peakKb)
|
||||
require.Equal(t, int64(460800), stats.lastKb, "last is the newest sample, not the largest")
|
||||
}
|
||||
|
||||
// TestStatsInBoundedWindow pins that a window excludes its end, so the baseline and upgrade windows an
|
||||
// upgrade report uses cannot both claim the same sample.
|
||||
func TestStatsInBoundedWindow(t *testing.T) {
|
||||
series := testSeries(t, sampleFile)
|
||||
|
||||
base := series.statsIn(time.Unix(1000, 0), time.Unix(2000, 0))
|
||||
require.Equal(t, 4, base.samples)
|
||||
require.Equal(t, int64(106496), base.peakKb)
|
||||
|
||||
upgrade := series.statsIn(time.Unix(2000, 0), time.Time{})
|
||||
require.Equal(t, 4, upgrade.samples)
|
||||
require.Equal(t, int64(512000), upgrade.peakKb)
|
||||
|
||||
// the ratio the upgrade check works from: 500 MiB against a 104 MiB baseline
|
||||
require.InDelta(t, 4.8, float64(upgrade.peakKb)/float64(base.peakKb), 0.1)
|
||||
}
|
||||
|
||||
// TestStatsInWithoutMarkers pins that samples written before markers existed still report, rather than
|
||||
// reporting nothing because every line fell outside the window.
|
||||
func TestStatsInWithoutMarkers(t *testing.T) {
|
||||
series := testSeries(t, "1001,102400\n1002,204800\n")
|
||||
|
||||
stats := series.statsIn(time.Time{}, time.Time{})
|
||||
require.Equal(t, 2, stats.samples)
|
||||
require.Equal(t, int64(204800), stats.peakKb)
|
||||
}
|
||||
|
||||
func testSeries(t *testing.T, out string) *ctrlMemSeries {
|
||||
t.Helper()
|
||||
samples, starts := parseCtrlMemSeries(out)
|
||||
return &ctrlMemSeries{samples: samples, starts: starts}
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/openziti/fablab/kernel/lib/actions/component"
|
||||
"github.com/openziti/fablab/kernel/lib/tui"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/foundation/v2/netz"
|
||||
"github.com/openziti/ziti/zititest/zitilab"
|
||||
"github.com/openziti/ziti/zititest/zitilab/actions/edge"
|
||||
"github.com/openziti/ziti/zititest/zitilab/chaos"
|
||||
)
|
||||
|
||||
// pushControllerConfig re-renders a controller's config and copies just that file to its host. It
|
||||
// deliberately avoids the whole-kit rsync (RsyncStaged), which does a --delete sync and would wipe
|
||||
// runtime state on the host, including the standalone bbolt db we migrate from and the raft data dir.
|
||||
func pushControllerConfig(run model.Run, c *model.Component, ctrlType *zitilab.ControllerType) error {
|
||||
if err := ctrlType.StageFiles(run, c); err != nil {
|
||||
return err
|
||||
}
|
||||
configName := ctrlType.ConfigName
|
||||
if configName == "" {
|
||||
configName = c.Id + ".yml"
|
||||
}
|
||||
local := filepath.Join(run.GetConfigDir(), configName)
|
||||
remote := fmt.Sprintf("/home/%s/fablab/cfg/%s", c.GetHost().GetSshUser(), configName)
|
||||
return c.GetHost().SendFile(local, remote)
|
||||
}
|
||||
|
||||
// waitForPortInactive blocks until a TCP dial to address fails (the port stops accepting) or the
|
||||
// timeout elapses. It is the inverse of netz.WaitForPortActive, used to detect a controller exiting
|
||||
// after a snapshot restore so we know when to restart it.
|
||||
func waitForPortInactive(address string, timeout time.Duration) error {
|
||||
deadline := time.Now().Add(timeout)
|
||||
for {
|
||||
conn, err := net.DialTimeout("tcp", address, 2*time.Second)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
_ = conn.Close()
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("port %s still active after %s", address, timeout)
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
// upgradeControllerToHa converts the standalone ctrl1 into a single-node HA cluster in place: it flips
|
||||
// ctrl1 to cluster mode, re-renders its config (with the cluster directive plus the migrateDb source
|
||||
// so raft is seeded from the existing standalone bbolt), pushes it, then restarts. The controller
|
||||
// snapshots its bbolt into raft and comes up as a bootstrapped one-node cluster with the existing
|
||||
// identities, services, and terminators intact.
|
||||
//
|
||||
// Applying that snapshot needs a second restart, and which side performs it depends on the version
|
||||
// under test. Only a controller whose Network.RestoreSnapshot honors restartSelfOnSnapshot re-execs;
|
||||
// the released 2.0.x this normally upgrades to exits instead, and nothing here supervises the process.
|
||||
// An exited controller is picked back up by the steady-state gate that follows, whose management
|
||||
// client restarts a controller it cannot reach. So the availability check below can pass against a
|
||||
// process that is seconds from exiting, and recovery then depends on that gate rather than on
|
||||
// anything in this function. That is tolerated rather than handled: making it deterministic means
|
||||
// waiting for the port to drop and starting the controller explicitly, the way addClusterNodes does.
|
||||
func upgradeControllerToHa(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
c := m.MustSelectComponent("#ctrl1")
|
||||
ctrlType, ok := c.Type.(*zitilab.ControllerType)
|
||||
if !ok {
|
||||
return fmt.Errorf("component ctrl1 is not a controller")
|
||||
}
|
||||
|
||||
log.Infof("converting ctrl1 to a single-node HA cluster")
|
||||
// cluster mode makes ctrl.yml.tmpl emit the cluster (raft) directive; migrateDb also emits the
|
||||
// source db line so the controller seeds raft from the existing standalone bbolt on first start.
|
||||
c.PutVariable("cluster", true)
|
||||
c.PutVariable("migrateDb", true)
|
||||
|
||||
if err := pushControllerConfig(run, c, ctrlType); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// RestartSelected waits for the old process to exit before starting the new one; a plain stop/start
|
||||
// would race, since Start no-ops while the old process is still shutting down
|
||||
if err := chaos.RestartSelected(run, 1, c); err != nil {
|
||||
return err
|
||||
}
|
||||
// generous timeout: this covers the snapshot apply and whatever restart follows it
|
||||
if err := edge.ControllerAvailable("#ctrl1", 120*time.Second).Execute(run); err != nil {
|
||||
return err
|
||||
}
|
||||
return edge.Login("#ctrl1").Execute(run)
|
||||
}
|
||||
|
||||
// addClusterNodes brings ctrl2 and ctrl3 online and joins them to ctrl1's cluster. Each is staged on
|
||||
// the upgraded (toVersion) binary in cluster mode and started, then the leader runs "agent cluster add"
|
||||
// for each node's control address, growing the single-node cluster to three.
|
||||
func addClusterNodes(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
toVersion := m.MustStringVariable("toVersion")
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
nodes := m.SelectComponents(".cluster-node")
|
||||
if len(nodes) == 0 {
|
||||
return fmt.Errorf("no cluster-node controllers found to add")
|
||||
}
|
||||
|
||||
// a joining node must run the same (upgraded) binary as the leader and be in cluster mode; it does
|
||||
// not migrate a db (no migrateDb), it replicates state from the leader after being added.
|
||||
for _, c := range nodes {
|
||||
ctrlType, ok := c.Type.(*zitilab.ControllerType)
|
||||
if !ok {
|
||||
return fmt.Errorf("component %s is not a controller", c.Id)
|
||||
}
|
||||
c.PutVariable("cluster", true)
|
||||
ctrlType.SetVersion(toVersion)
|
||||
if err := pushControllerConfig(run, c, ctrlType); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err := component.StartInParallel(".cluster-node", 5).Execute(run); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// add each new node from the leader (ctrl1); the controller advertises cluster membership on :6262
|
||||
leader := m.MustSelectComponent("#ctrl1")
|
||||
leaderType, ok := leader.Type.(*zitilab.ControllerType)
|
||||
if !ok {
|
||||
return fmt.Errorf("component ctrl1 is not a controller")
|
||||
}
|
||||
leaderBinary := leaderType.GetBinaryPath(leader)
|
||||
for _, c := range nodes {
|
||||
hostPort := c.GetHost().PublicIp + ":6262"
|
||||
|
||||
// StartInParallel returns once the process is launched, not once it is listening, so wait for
|
||||
// the node's cluster port to be open before adding it rather than retrying a membership change.
|
||||
log.Infof("waiting for %s cluster port %s", c.Id, hostPort)
|
||||
if err := netz.WaitForPortActive(hostPort, 90*time.Second); err != nil {
|
||||
return fmt.Errorf("cluster port for %s (%s) never became available: %w", c.Id, hostPort, err)
|
||||
}
|
||||
|
||||
log.Infof("adding %s to the cluster at tls:%s", c.Id, hostPort)
|
||||
cmd := fmt.Sprintf("%s agent cluster add tls:%s", leaderBinary, hostPort)
|
||||
if err := leader.GetHost().ExecLogOnlyOnError(cmd); err != nil {
|
||||
return fmt.Errorf("failed to add %s to the cluster at tls:%s: %w", c.Id, hostPort, err)
|
||||
}
|
||||
|
||||
// The leader streams its db to the joining node, which restores the snapshot and then replaces
|
||||
// itself: the config sets restartSelfOnSnapshot, so the node re-execs and the old process exits.
|
||||
// The cluster port drops while the replacement sleeps out its start delay, which is the signal
|
||||
// the restore happened.
|
||||
//
|
||||
// Do not wait for the node's process to disappear. The replacement runs the same command line,
|
||||
// so it matches the component's process filter and the node never looks stopped.
|
||||
log.Infof("waiting for %s to restore the cluster snapshot and restart itself", c.Id)
|
||||
if err := waitForPortInactive(hostPort, 90*time.Second); err != nil {
|
||||
return fmt.Errorf("%s did not restart after joining (snapshot restore expected): %w", c.Id, err)
|
||||
}
|
||||
// A no-op while the node is restarting itself, since Start returns early when a matching process
|
||||
// is already running. Kept as the safety net for a node that exited instead of re-execing.
|
||||
if err := component.Start("#" + c.Id).Execute(run); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := netz.WaitForPortActive(hostPort, 90*time.Second); err != nil {
|
||||
return fmt.Errorf("%s cluster port did not come back after restart: %w", c.Id, err)
|
||||
}
|
||||
// let the membership change commit and replicate before adding the next node
|
||||
time.Sleep(10 * time.Second)
|
||||
}
|
||||
|
||||
// give raft time to settle the 3-node cluster before the stability gate runs
|
||||
log.Infof("waiting for the 3-node cluster to settle")
|
||||
time.Sleep(30 * time.Second)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,640 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"embed"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/michaelquigley/pfxlog"
|
||||
"github.com/openziti/fablab"
|
||||
"github.com/openziti/fablab/kernel/lib/actions/component"
|
||||
"github.com/openziti/fablab/kernel/lib/binding"
|
||||
"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
|
||||
semaphore "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
|
||||
terraformInit "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
|
||||
distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution"
|
||||
"github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync"
|
||||
awsSshKeyDispose "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
|
||||
"github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/fablab/resources"
|
||||
"github.com/openziti/ziti/zititest/models/test_resources"
|
||||
"github.com/openziti/ziti/zititest/zitilab"
|
||||
"github.com/openziti/ziti/zititest/zitilab/actions/edge"
|
||||
"github.com/openziti/ziti/zititest/zitilab/models"
|
||||
zitiLibOps "github.com/openziti/ziti/zititest/zitilab/runlevel/5_operation"
|
||||
)
|
||||
|
||||
//go:embed configs
|
||||
var configResource embed.FS
|
||||
|
||||
// livenessWorkload is a short loop4 workload used purely to prove a service's data path is
|
||||
// currently healthy. A scenario run executes one of these per hosting flavor.
|
||||
var livenessWorkload = "" +
|
||||
`concurrency: 1
|
||||
iterations: 1
|
||||
dialer:
|
||||
txRequests: 20
|
||||
rxTimeout: 5s
|
||||
payloadMinBytes: 512
|
||||
payloadMaxBytes: 512
|
||||
listener:
|
||||
rxTimeout: 5s
|
||||
`
|
||||
|
||||
func getUniqueId() string {
|
||||
if runId := os.Getenv("GITHUB_RUN_ID"); runId != "" {
|
||||
return "-" + runId + "." + os.Getenv("GITHUB_RUN_ATTEMPT")
|
||||
}
|
||||
return "-" + os.Getenv("USER")
|
||||
}
|
||||
|
||||
// rotatingLogConfig is the log configuration applied to every ziti component in this model. The
|
||||
// rotate strategy keeps each component's log size-bounded (via "ziti ops log-pipe") so a long
|
||||
// multi-iteration run cannot fill the host disk. Combined with routers no longer running at debug,
|
||||
// this keeps the per-host log footprint small across all iterations.
|
||||
//
|
||||
// The pipe binary is pinned to the locally-built ziti (the empty version) because the staged
|
||||
// fromVersion/toVersion binaries predate "ops log-pipe". Without the pin, the pipe would run a binary
|
||||
// that lacks the command and take the component's stdout down with it on most of the versions this
|
||||
// run cycles through.
|
||||
func rotatingLogConfig() zitilab.LogConfig {
|
||||
localBuild := ""
|
||||
return zitilab.LogConfig{
|
||||
Strategy: zitilab.LogStrategyRotate,
|
||||
PipeBinaryVersion: &localBuild,
|
||||
}
|
||||
}
|
||||
|
||||
var m = &model.Model{
|
||||
Id: "upgrade-test",
|
||||
Scope: model.Scope{
|
||||
Defaults: model.Variables{
|
||||
// version knobs. The system starts on fromVersion and is upgraded to toVersion
|
||||
// in place. zetVersion is fixed for the whole run (ZET stays up, not upgraded).
|
||||
// toVersion is the label the built binary is staged under and, for ref builds, the version
|
||||
// stamped into it, so it must be a plain numeric semver (the SDK version parser rejects
|
||||
// pre-release/build-metadata suffixes). toVersionRef selects the actual source.
|
||||
"fromVersion": "v1.6.20",
|
||||
"toVersion": "v2.0.3",
|
||||
// zetVersion tracks the ziti-edge-tunnel bundled in the latest stable Ziti Desktop Edge for
|
||||
// Windows (WDE 2.11.2.5 ships v1.18.0). Pin to v1.11.4 (WDE 2.9.7.1) to reproduce the older
|
||||
// client's post-2.0-migration session-recovery failure.
|
||||
"zetVersion": "v1.18.0",
|
||||
// ziti-tunnel runs the locally-built ziti by default (so it exercises the current SDK,
|
||||
// e.g. to validate SDK fixes). Set to a release version to test a specific released build.
|
||||
"zitiTunnelVersion": "",
|
||||
// toVersionSource optionally points at a locally-built ziti binary to stage under the
|
||||
// toVersion name, so the upgrade runs a patched controller/router instead of the released
|
||||
// toVersion. Empty falls through to toVersionRef. The ZITI_TOVERSION_PATH env var overrides
|
||||
// this, and a source binary wins over a ref build wherever both are set.
|
||||
"toVersionSource": "",
|
||||
// toVersionRef optionally names a git ref (branch, tag, or SHA) on openziti/ziti to build
|
||||
// from source and stage under the toVersion name, so the upgrade can run a fix no release
|
||||
// carries yet. Empty downloads the released toVersion instead, which is the normal case.
|
||||
// The ZITI_TOVERSION_REF env var overrides this; toVersionSource wins over both.
|
||||
// Changing the ref alone is enough to get a new binary: the staging op is keyed on the ref as
|
||||
// well as the version, and acquire caches on the commit the ref resolves to, so a branch that
|
||||
// has moved rebuilds rather than serving the previous build.
|
||||
"toVersionRef": "",
|
||||
|
||||
// nextVersion is the optional second upgrade hop: once the cluster is up on toVersion it is
|
||||
// upgraded again, cluster and all, to nextVersion. Leave it empty to end the iteration at
|
||||
// toVersion and skip the phase entirely. nextVersionSource/nextVersionRef work exactly like
|
||||
// their toVersion counterparts, with ZITI_NEXTVERSION_PATH and ZITI_NEXTVERSION_REF as the
|
||||
// environment overrides.
|
||||
//
|
||||
// 2.1 has no release yet, so the default builds it from main. That is a moving ref: what a
|
||||
// run tested is whatever main pointed at that day. Set nextVersionRef to a tag once 2.1
|
||||
// releases, at which point nextVersion alone selects a downloaded release.
|
||||
"nextVersion": "v2.1.0",
|
||||
"nextVersionSource": "",
|
||||
"nextVersionRef": "main",
|
||||
// clusterUpgradeMode picks how the cluster moves to nextVersion. "rolling" restarts one node
|
||||
// at a time, which leaves the cluster in mixed-version read-only mode until the last node is
|
||||
// done; "all-at-once" restarts every node together, so no node sees a version mismatch but
|
||||
// every node cold-starts simultaneously.
|
||||
"clusterUpgradeMode": clusterUpgradeRolling,
|
||||
|
||||
// controller memory sampling. Every controller host records its controller's RSS once a
|
||||
// second for the whole iteration, so an upgrade's memory cost can be compared against the
|
||||
// node's own steady state and against its peers.
|
||||
"ctrlMemory": model.Variables{
|
||||
// heapDumpAtMb captures one heap profile per controller the first time RSS crosses this,
|
||||
// which is the artifact a memory regression is diagnosed from and the one thing that
|
||||
// cannot be recovered after the fact. Zero disables the capture.
|
||||
"heapDumpAtMb": 400,
|
||||
// failAtMb fails a phase whose peak exceeds it. Zero disables the check.
|
||||
"failAtMb": 1024,
|
||||
// failAtRatio fails the cluster upgrade when a controller's peak over the upgrade exceeds
|
||||
// this multiple of its own peak over the steady state just before it. This is the check
|
||||
// that bites at this model's scale, where the absolute ceiling is far out of reach; the
|
||||
// value is a first guess and wants calibrating against a run that is known good. Anything
|
||||
// at or below one disables it.
|
||||
"failAtRatio": 3,
|
||||
},
|
||||
|
||||
// auth/cluster flags. Phase 1 runs a single standalone controller on legacy auth,
|
||||
// so all three default to false. OIDC becomes available via the 2.0 controller's
|
||||
// auto-binding, not by setting these.
|
||||
"oidc": false,
|
||||
"ha": false,
|
||||
"cluster": false,
|
||||
|
||||
// steady-state gate tuning
|
||||
"steadyState": model.Variables{
|
||||
"requiredCleanRuns": 3,
|
||||
"recoveryTimeout": "1m",
|
||||
"stabilityWindow": "2m",
|
||||
},
|
||||
|
||||
"livenessWorkload": livenessWorkload,
|
||||
|
||||
"environment": "upgrade-test" + getUniqueId(),
|
||||
"credentials": model.Variables{
|
||||
"aws": model.Variables{
|
||||
"managed_key": true,
|
||||
},
|
||||
"ssh": model.Variables{
|
||||
"username": "ubuntu",
|
||||
},
|
||||
"edge": model.Variables{
|
||||
"username": "admin",
|
||||
"password": "admin",
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
StructureFactories: []model.Factory{
|
||||
// apply the bulk from/zet versions and the shared log config to every component
|
||||
model.FactoryFunc(func(m *model.Model) error {
|
||||
fromVersion := m.MustStringVariable("fromVersion")
|
||||
zetVersion := m.MustStringVariable("zetVersion")
|
||||
zitiTunnelVersion := m.MustStringVariable("zitiTunnelVersion")
|
||||
return m.ForEachComponent("*", 1, func(c *model.Component) error {
|
||||
switch t := c.Type.(type) {
|
||||
case *zitilab.ControllerType:
|
||||
t.Version = fromVersion
|
||||
t.LogConfig = rotatingLogConfig()
|
||||
case *zitilab.RouterType:
|
||||
t.Version = fromVersion
|
||||
t.LogConfig = rotatingLogConfig()
|
||||
case *zitilab.ZitiTunnelType:
|
||||
t.Version = zitiTunnelVersion
|
||||
t.LogConfig = rotatingLogConfig()
|
||||
case *zitilab.ZitiEdgeTunnelType:
|
||||
t.Version = zetVersion
|
||||
t.ZitiVersion = fromVersion
|
||||
t.LogConfig = rotatingLogConfig()
|
||||
t.InitType(c)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}),
|
||||
// per-component version overrides, e.g. -l router_west_version=v1.6.10
|
||||
model.FactoryFunc(func(m *model.Model) error {
|
||||
return m.ForEachComponent("*", 1, func(c *model.Component) error {
|
||||
versioned, ok := c.Type.(interface{ SetVersion(string) })
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
varName := strings.ReplaceAll(c.Id, "-", "_") + "_version"
|
||||
if version, found := m.GetStringVariable(varName); found {
|
||||
versioned.SetVersion(version)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}),
|
||||
// instance sizing
|
||||
model.FactoryFunc(func(m *model.Model) error {
|
||||
return m.ForEachHost("*", 1, func(host *model.Host) error {
|
||||
if strings.HasPrefix(host.Id, "ctrl") {
|
||||
host.InstanceType = "t3.medium"
|
||||
} else {
|
||||
host.InstanceType = "c5.large"
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}),
|
||||
},
|
||||
|
||||
Factories: []model.Factory{
|
||||
model.FactoryFunc(func(m *model.Model) error {
|
||||
pfxlog.Logger().Infof("environment [%s]", m.MustStringVariable("environment"))
|
||||
return nil
|
||||
}),
|
||||
// sim harness: metrics collection and the steady-state validation gate
|
||||
model.FactoryFunc(func(m *model.Model) error {
|
||||
simServices := zitiLibOps.NewSimServices(func(s string) string {
|
||||
return "component#" + s
|
||||
})
|
||||
|
||||
m.AddActivationStageF(simServices.SetupSimControllerIdentity)
|
||||
m.AddOperatingStage(simServices.CollectSimMetricStage("metrics"))
|
||||
|
||||
m.AddActionF("startSimMetrics", func(run model.Run) error {
|
||||
return simServices.CollectSimMetrics(run, "metrics")
|
||||
})
|
||||
|
||||
gate := newSteadyStateGate(simServices)
|
||||
m.AddActionF("validateSteadyState", gate.validate)
|
||||
m.AddActionF("validateSteadyStateAfterDisruption", gate.validateAfterDisruption)
|
||||
|
||||
// resetToBaseline returns the system to a fresh standalone fromVersion baseline so a new
|
||||
// testIteration can run from a known state. It undoes the in-process mutations a pass makes
|
||||
// (controller/router binary versions, HA/migration flags) and drops the in-harness
|
||||
// sim-controller client's cached context/enrollment (stale once the controller is wiped),
|
||||
// then re-runs the standard configuration -> distribution -> activation pipeline. Re-rendering
|
||||
// the configs from the reset state is what brings the controller back up standalone rather
|
||||
// than reusing the prior pass's HA config; Activate also re-bootstraps and re-enrolls the
|
||||
// sim-controller identity (an activation stage).
|
||||
m.AddActionF("resetToBaseline", func(run model.Run) error {
|
||||
fromVersion := m.MustStringVariable("fromVersion")
|
||||
if err := m.ForEachComponent("*", 1, func(c *model.Component) error {
|
||||
switch t := c.Type.(type) {
|
||||
case *zitilab.ControllerType:
|
||||
t.SetVersion(fromVersion)
|
||||
c.PutVariable("cluster", false)
|
||||
c.PutVariable("migrateDb", false)
|
||||
case *zitilab.RouterType:
|
||||
t.SetVersion(fromVersion)
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
simServices.Reset()
|
||||
|
||||
if err := m.Build(run); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := m.Sync(run); err != nil {
|
||||
return err
|
||||
}
|
||||
return m.Activate(run)
|
||||
})
|
||||
|
||||
return nil
|
||||
}),
|
||||
},
|
||||
|
||||
Resources: model.Resources{
|
||||
resources.Configs: resources.SubFolder(configResource, "configs"),
|
||||
resources.Binaries: os.DirFS(path.Join(os.Getenv("GOPATH"), "bin")),
|
||||
resources.Terraform: test_resources.TerraformResources(),
|
||||
},
|
||||
|
||||
Regions: model.Regions{
|
||||
"us-east-1": {
|
||||
Region: "us-east-1",
|
||||
Site: "us-east-1a",
|
||||
Hosts: model.Hosts{
|
||||
"ctrl1": {
|
||||
Components: model.Components{
|
||||
"ctrl1": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ctrl", "bootstrap-ctrl"}},
|
||||
Type: &zitilab.ControllerType{},
|
||||
},
|
||||
},
|
||||
},
|
||||
// provisioned up front, started only during the add-nodes phase
|
||||
"ctrl2": {
|
||||
Components: model.Components{
|
||||
"ctrl2": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ctrl", "ha", "cluster-node"}},
|
||||
Type: &zitilab.ControllerType{},
|
||||
},
|
||||
},
|
||||
},
|
||||
"router-east-1": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ert-client"}},
|
||||
Components: model.Components{
|
||||
"router-east-1": {
|
||||
// ert-proxy-client makes this ERT run its tunnel binding in proxy mode (local
|
||||
// loop-ert listener on :15390) so the co-located loop4 dialer below can push
|
||||
// traffic through the ERT client path.
|
||||
Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client", "ert-proxy-client"}},
|
||||
Type: &zitilab.RouterType{},
|
||||
},
|
||||
"loop4-client-stable": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "stable"}},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-client.yml.tmpl",
|
||||
Mode: zitilab.Loop4RemoteControlled,
|
||||
},
|
||||
},
|
||||
"loop4-client-restart": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "restart"}},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-client.yml.tmpl",
|
||||
Mode: zitilab.Loop4RemoteControlled,
|
||||
},
|
||||
},
|
||||
// drives traffic through the ERT client's local proxy listener (router-east-1 is a
|
||||
// single router, not a stable/restart pair, so one dialer)
|
||||
"loop-ert-dialer": {
|
||||
Scope: model.Scope{
|
||||
Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client"},
|
||||
Defaults: model.Variables{"loopTunnelAddress": "tcp:127.0.0.1:15390"},
|
||||
},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-transport-dialer.yml.tmpl",
|
||||
Mode: zitilab.Loop4RemoteControlled,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"router-east-2": {
|
||||
Components: model.Components{
|
||||
"router-east-2": {
|
||||
Scope: model.Scope{Tags: model.Tags{"edge-router", "initiator"}},
|
||||
Type: &zitilab.RouterType{},
|
||||
},
|
||||
},
|
||||
},
|
||||
// Tunneler clients are grouped by stable/restart rather than by flavor. Each host runs one
|
||||
// Go ziti-tunnel (proxy mode, no :53) and one ZET (tproxy, needs :53). Splitting the ZET
|
||||
// pair across the two hosts keeps them off each other's :53, and the Go client's proxy mode
|
||||
// means it never contends for :53 with the co-located ZET. Each tunneler client has a
|
||||
// co-located loop4 dialer pushing traffic through its client-side data path.
|
||||
"tunnel-clients-stable": {
|
||||
Components: model.Components{
|
||||
"ziti-tunnel-client-stable": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client", "ziti-tunnel-client", "stable"}},
|
||||
Type: &zitilab.ZitiTunnelType{
|
||||
Mode: zitilab.ZitiTunnelModeProxy,
|
||||
ProxyServices: []string{"loop-ziti-tunnel:15387"},
|
||||
},
|
||||
},
|
||||
"loop-zt-dialer-stable": {
|
||||
Scope: model.Scope{
|
||||
Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "stable"},
|
||||
Defaults: model.Variables{"loopTunnelAddress": "tcp:127.0.0.1:15387"},
|
||||
},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-transport-dialer.yml.tmpl",
|
||||
Mode: zitilab.Loop4RemoteControlled,
|
||||
},
|
||||
},
|
||||
"ziti-edge-tunnel-client-stable": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "zet", "zet-client", "stable"}},
|
||||
Type: &zitilab.ZitiEdgeTunnelType{VerbosityLevel: 3},
|
||||
},
|
||||
"loop-zet-dialer-stable": {
|
||||
Scope: model.Scope{
|
||||
Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "stable"},
|
||||
Defaults: model.Variables{"loopTunnelAddress": "tcp:loop-zet.ziti:15391"},
|
||||
},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-transport-dialer.yml.tmpl",
|
||||
Mode: zitilab.Loop4RemoteControlled,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"tunnel-clients-restart": {
|
||||
Components: model.Components{
|
||||
"ziti-tunnel-client-restart": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client", "ziti-tunnel-client", "restart"}},
|
||||
Type: &zitilab.ZitiTunnelType{
|
||||
Mode: zitilab.ZitiTunnelModeProxy,
|
||||
ProxyServices: []string{"loop-ziti-tunnel:15388"},
|
||||
},
|
||||
},
|
||||
"loop-zt-dialer-restart": {
|
||||
Scope: model.Scope{
|
||||
Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "restart"},
|
||||
Defaults: model.Variables{"loopTunnelAddress": "tcp:127.0.0.1:15388"},
|
||||
},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-transport-dialer.yml.tmpl",
|
||||
Mode: zitilab.Loop4RemoteControlled,
|
||||
},
|
||||
},
|
||||
"ziti-edge-tunnel-client-restart": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "client", "zet", "zet-client", "restart"}},
|
||||
Type: &zitilab.ZitiEdgeTunnelType{VerbosityLevel: 3},
|
||||
},
|
||||
"loop-zet-dialer-restart": {
|
||||
Scope: model.Scope{
|
||||
Tags: model.Tags{"sdk-app", "client", "loop-client", "sim-services-client", "restart"},
|
||||
Defaults: model.Variables{"loopTunnelAddress": "tcp:loop-zet.ziti:15391"},
|
||||
},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-transport-dialer.yml.tmpl",
|
||||
Mode: zitilab.Loop4RemoteControlled,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
"us-west-2": {
|
||||
Region: "us-west-2",
|
||||
Site: "us-west-2b",
|
||||
Hosts: model.Hosts{
|
||||
// provisioned up front, started only during the add-nodes phase
|
||||
"ctrl3": {
|
||||
Components: model.Components{
|
||||
"ctrl3": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ctrl", "ha", "cluster-node"}},
|
||||
Type: &zitilab.ControllerType{},
|
||||
},
|
||||
},
|
||||
},
|
||||
"router-west": {
|
||||
Components: model.Components{
|
||||
"router-west": {
|
||||
Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host"}},
|
||||
Type: &zitilab.RouterType{},
|
||||
},
|
||||
// paired Go SDK listeners that bind the loop-sdk service (two terminators)
|
||||
"loop4-sdk-host-stable": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "loop-sdk-host", "stable"}},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-sdk-host.yml.tmpl",
|
||||
Mode: zitilab.Loop4Listener,
|
||||
},
|
||||
},
|
||||
"loop4-sdk-host-restart": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "loop-sdk-host", "restart"}},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-sdk-host.yml.tmpl",
|
||||
Mode: zitilab.Loop4Listener,
|
||||
},
|
||||
},
|
||||
// plain-TCP backend the ERT router forwards loop-ert to (localhost:3456)
|
||||
"loop4-ert-backend": {
|
||||
Scope: model.Scope{Tags: model.Tags{"loop-backend"}},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-backend-host.yml.tmpl",
|
||||
Mode: zitilab.Loop4Listener,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
// The ziti-tunnel and ZET hosts share a box: both run in host mode (no :53 intercept), and
|
||||
// both forward their loop service to a single local loop4 backend on 127.0.0.1:3456.
|
||||
"tunnel-hosts": {
|
||||
Components: model.Components{
|
||||
"ziti-tunnel-host-stable": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host", "stable"}},
|
||||
Type: &zitilab.ZitiTunnelType{
|
||||
Mode: zitilab.ZitiTunnelModeHost,
|
||||
Verbose: true,
|
||||
},
|
||||
},
|
||||
"ziti-tunnel-host-restart": {
|
||||
Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host", "restart"}},
|
||||
Type: &zitilab.ZitiTunnelType{
|
||||
Mode: zitilab.ZitiTunnelModeHost,
|
||||
Verbose: true,
|
||||
},
|
||||
},
|
||||
// host mode (run-host) is required here: the default intercept mode would have both
|
||||
// instances contend for the shared box's tproxy/DNS state, including port 53
|
||||
"ziti-edge-tunnel-host-stable": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host", "zet", "stable"}},
|
||||
Type: &zitilab.ZitiEdgeTunnelType{
|
||||
Mode: zitilab.ZitiEdgeTunnelModeHost,
|
||||
VerbosityLevel: 3,
|
||||
},
|
||||
},
|
||||
"ziti-edge-tunnel-host-restart": {
|
||||
Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host", "zet", "restart"}},
|
||||
Type: &zitilab.ZitiEdgeTunnelType{
|
||||
Mode: zitilab.ZitiEdgeTunnelModeHost,
|
||||
VerbosityLevel: 3,
|
||||
},
|
||||
},
|
||||
// single plain-TCP loop4 backend both the ziti-tunnel and ZET hosts forward to
|
||||
"loop4-tunnel-backend": {
|
||||
Scope: model.Scope{Tags: model.Tags{"loop-backend"}},
|
||||
Type: &zitilab.Loop4SimType{
|
||||
ConfigSource: "loop4-backend-host.yml.tmpl",
|
||||
Mode: zitilab.Loop4Listener,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
Actions: model.ActionBinders{
|
||||
"bootstrap": newBootstrapAction(),
|
||||
"start": newStartAction(),
|
||||
"stop": model.Bind(component.StopInParallel("*", 15)),
|
||||
"login": model.Bind(edge.Login("#ctrl1")),
|
||||
// individual upgrade steps, runnable on their own
|
||||
"upgradeController": model.BindF(upgradeController),
|
||||
"upgradeRouters": model.BindF(upgradeRouters),
|
||||
"restartAndVerifyOidc": model.BindF(restartAndVerifyOidc),
|
||||
"restartZetWorkaround": model.BindF(restartZetWorkaround),
|
||||
"upgradeControllerToHa": model.BindF(upgradeControllerToHa),
|
||||
"addClusterNodes": model.BindF(addClusterNodes),
|
||||
// the optional second upgrade hop, plus its pieces
|
||||
"upgradeToNextVersion": model.BindF(upgradeToNextVersion),
|
||||
"upgradeClusterControllers": model.BindF(upgradeClusterControllers),
|
||||
"upgradeClusterRouters": model.BindF(upgradeClusterRouters),
|
||||
// controller memory sampling, runnable on its own against a live instance
|
||||
"startCtrlMemory": model.BindF(startCtrlMemorySamplers),
|
||||
"stopCtrlMemory": model.BindF(stopCtrlMemorySamplers),
|
||||
"reportCtrlMemory": model.BindF(func(run model.Run) error {
|
||||
return reportCtrlMemory(run, time.Time{}, "iteration")
|
||||
}),
|
||||
"ctrlMemorySummary": model.BindF(summarizeCtrlMemory),
|
||||
// testIteration is the exec-loop entry point (fablab exec-loop <until> testIteration). It runs
|
||||
// one full pass of the upgrade sequence. It begins by resetting to a fresh standalone fromVersion
|
||||
// baseline, so each pass is self-contained and repeated iterations (or a re-run after a failed
|
||||
// pass) start from a known state rather than re-upgrading an already upgraded controller.
|
||||
"testIteration": model.BindF(func(run model.Run) error {
|
||||
return run.GetModel().Exec(run,
|
||||
"resetToBaseline", // fresh standalone fromVersion baseline (undo any prior pass)
|
||||
"startSimMetrics",
|
||||
"startCtrlMemory", // per-second controller RSS sampling for the whole pass
|
||||
"validateSteadyState", // baseline: expected terminators present, clients healthy on legacy
|
||||
"upgradeController", // ctrl fromVersion -> toVersion (OIDC auto-binds)
|
||||
"restartZetWorkaround", // old ziti-edge-tunnel can't recover sessions post-migration; restart to re-auth
|
||||
"validateSteadyStateAfterDisruption", // terminators re-establish, traffic recovers, clients still up
|
||||
"restartAndVerifyOidc", // restart -restart instances, confirm OIDC on re-auth
|
||||
"validateSteadyStateAfterDisruption", // everything back and clean
|
||||
"upgradeRouters", // routers fromVersion -> toVersion, rolling one at a time
|
||||
"validateSteadyStateAfterDisruption", // routers back on toVersion, terminators + traffic recovered
|
||||
"upgradeControllerToHa", // ctrl1 standalone -> single-node HA cluster (bbolt -> raft)
|
||||
"validateSteadyStateAfterDisruption", // cluster up, terminators + traffic recovered
|
||||
"addClusterNodes", // start ctrl2/ctrl3 and join them to the cluster
|
||||
"validateSteadyStateAfterDisruption", // 3-node cluster stable, terminators + traffic recovered
|
||||
"upgradeToNextVersion", // optional: whole system toVersion -> nextVersion
|
||||
"reportCtrlMemory", // controller memory over the whole pass
|
||||
)
|
||||
}),
|
||||
},
|
||||
|
||||
Infrastructure: model.Stages{
|
||||
aws_ssh_key.Express(),
|
||||
&terraformInit.Terraform{
|
||||
Retries: 3,
|
||||
ReadyCheck: &semaphore.ReadyStage{
|
||||
MaxWait: 90 * time.Second,
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
Distribution: model.Stages{
|
||||
// Stop all components before pushing files. A leftover component from a prior run (e.g. a broken
|
||||
// controller spamming logs after a failed HA phase) can hold a large deleted log file open,
|
||||
// keeping the host disk full; the ssh-key and rsync steps below then fail on the full disk and
|
||||
// abort the whole refresh before it can reset anything. Stopping first releases those handles.
|
||||
model.RunAction("stop"),
|
||||
distribution.DistributeSshKey("*"),
|
||||
// pre-stage the upgrade binaries so in-place upgrades need only a version swap + restart
|
||||
model.StageActionF(toTarget.stage),
|
||||
model.StageActionF(nextTarget.stage),
|
||||
rsync.RsyncStaged(),
|
||||
},
|
||||
|
||||
Activation: model.Stages{
|
||||
model.RunAction("stop"),
|
||||
model.RunAction("bootstrap"),
|
||||
model.RunAction("start"),
|
||||
},
|
||||
|
||||
Operation: model.Stages{
|
||||
model.RunAction("login"),
|
||||
edge.SyncModelRouterIds(models.EdgeRouterTag),
|
||||
},
|
||||
|
||||
Disposal: model.Stages{
|
||||
terraform.Dispose(),
|
||||
awsSshKeyDispose.Dispose(),
|
||||
},
|
||||
}
|
||||
|
||||
func main() {
|
||||
model.AddBootstrapExtension(binding.AwsCredentialsLoader)
|
||||
model.AddBootstrapExtension(aws_ssh_key.KeyManager)
|
||||
|
||||
fablab.InitModel(m)
|
||||
fablab.Run()
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/openziti/fablab/kernel/lib/tui"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/ziti/zititest/zitilab/actions/edge"
|
||||
"github.com/openziti/ziti/zititest/zitilab/chaos"
|
||||
"github.com/openziti/ziti/zititest/zitilab/cli"
|
||||
)
|
||||
|
||||
// apiSessionEvent mirrors the fields we care about from the controller's apiSession event
|
||||
// (controller/event/api_session.go). Type is "legacy" or "jwt"; jwt is an OIDC/token session.
|
||||
type apiSessionEvent struct {
|
||||
Namespace string `json:"namespace"`
|
||||
EventType string `json:"event_type"`
|
||||
Type string `json:"type"`
|
||||
IdentityId string `json:"identity_id"`
|
||||
}
|
||||
|
||||
// restartAndVerifyOidc restarts the -restart instance of each client/host pair (leaving the -stable
|
||||
// instance up) and reads the controller's apiSession event log to determine the session type each
|
||||
// restarted client authenticated with. Go SDK clients/hosts are expected to switch to OIDC (jwt) once
|
||||
// the controller advertises it; ZET (C SDK) behavior is recorded informationally.
|
||||
func restartAndVerifyOidc(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
// all intentional test output routes to the validation pane; channel/library noise falls to actions
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
sshUser := m.MustStringVariable("credentials.ssh.username")
|
||||
logPath := fmt.Sprintf("/home/%s/logs/api-sessions.log", sshUser)
|
||||
ctrl := m.MustSelectHost("component.bootstrap-ctrl")
|
||||
|
||||
restartComps := m.SelectComponents(".restart")
|
||||
if len(restartComps) == 0 {
|
||||
return fmt.Errorf("no components tagged 'restart' found")
|
||||
}
|
||||
|
||||
// refresh the CLI session before querying identities: the login from the controller upgrade can go
|
||||
// stale across the preceding validation window, and a stale session would make the identity lookup
|
||||
// fail confusingly rather than cleanly.
|
||||
if err := edge.Login("#ctrl1").Execute(run); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
nameToId, err := restartIdentityIds(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// snapshot the log so we only consider events produced by the restart
|
||||
preCount, err := logLineCount(ctrl, logPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Infof("restarting %d -restart instance(s); -stable siblings stay up", len(restartComps))
|
||||
if err = chaos.RestartSelected(run, 10, restartComps...); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// poll for new api-session created events until every restarted identity is seen or we time out
|
||||
idToType := map[string]string{}
|
||||
deadline := time.Now().Add(90 * time.Second)
|
||||
for {
|
||||
events, err := newApiSessionEvents(ctrl, logPath, preCount)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, e := range events {
|
||||
if e.Namespace == "apiSession" && e.EventType == "created" && e.IdentityId != "" {
|
||||
idToType[e.IdentityId] = e.Type
|
||||
}
|
||||
}
|
||||
if allSeen(restartComps, nameToId, idToType) || time.Now().After(deadline) {
|
||||
break
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
}
|
||||
|
||||
var problems []string
|
||||
for _, c := range restartComps {
|
||||
id, ok := nameToId[c.Id]
|
||||
if !ok {
|
||||
problems = append(problems, c.Id+": identity id not found")
|
||||
continue
|
||||
}
|
||||
sessType, seen := idToType[id]
|
||||
if !seen {
|
||||
problems = append(problems, c.Id+": no api-session created after restart")
|
||||
continue
|
||||
}
|
||||
if c.HasTag("zet") {
|
||||
// ZET (C SDK) OIDC support is version-dependent; record but do not fail on it.
|
||||
log.Infof("OIDC restart check [ZET, informational]: %s -> api-session type=%s", c.Id, sessType)
|
||||
continue
|
||||
}
|
||||
if sessType != apiSessionTypeJwt {
|
||||
problems = append(problems, fmt.Sprintf("%s: expected jwt (OIDC) after restart, got %s", c.Id, sessType))
|
||||
} else {
|
||||
log.Infof("OIDC restart check: %s switched to OIDC (jwt)", c.Id)
|
||||
}
|
||||
}
|
||||
|
||||
if len(problems) > 0 {
|
||||
return fmt.Errorf("OIDC restart verification failed: %s", strings.Join(problems, "; "))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
const apiSessionTypeJwt = "jwt"
|
||||
|
||||
// allSeen reports whether every restart component's identity has a recorded session type yet.
|
||||
func allSeen(comps []*model.Component, nameToId, idToType map[string]string) bool {
|
||||
for _, c := range comps {
|
||||
id, ok := nameToId[c.Id]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if _, seen := idToType[id]; !seen {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// restartIdentityIds returns a map of component id (== identity name) to identity id for every
|
||||
// identity whose name ends in "-restart".
|
||||
func restartIdentityIds(m *model.Model) (map[string]string, error) {
|
||||
out, err := cli.Exec(m, "edge", "list", "identities", "limit none", "-j")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var resp struct {
|
||||
Data []struct {
|
||||
Id string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
} `json:"data"`
|
||||
}
|
||||
if err = json.Unmarshal([]byte(out), &resp); err != nil {
|
||||
return nil, fmt.Errorf("parsing identities list: %w", err)
|
||||
}
|
||||
|
||||
// the default admin always exists, so an empty list is never legitimate; treat it as a controller
|
||||
// or session failure rather than letting downstream lookups report "identity id not found".
|
||||
if len(resp.Data) == 0 {
|
||||
return nil, fmt.Errorf("edge list identities returned no identities; the default admin should always be present, so this indicates a controller or session problem")
|
||||
}
|
||||
|
||||
result := map[string]string{}
|
||||
for _, d := range resp.Data {
|
||||
if strings.HasSuffix(d.Name, "-restart") {
|
||||
result[d.Name] = d.Id
|
||||
}
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// logLineCount returns the current line count of the given remote log file, or 0 if it is absent.
|
||||
func logLineCount(host *model.Host, path string) (int, error) {
|
||||
out, err := host.ExecLogged(fmt.Sprintf("wc -l < %s 2>/dev/null || echo 0", path))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return strconv.Atoi(strings.TrimSpace(lastToken(out)))
|
||||
}
|
||||
|
||||
// newApiSessionEvents reads and parses the api-session log lines appended after afterLine.
|
||||
func newApiSessionEvents(host *model.Host, path string, afterLine int) ([]apiSessionEvent, error) {
|
||||
out, err := host.ExecLogged(fmt.Sprintf("tail -n +%d %s 2>/dev/null || true", afterLine+1, path))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var events []apiSessionEvent
|
||||
for _, line := range strings.Split(out, "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if !strings.HasPrefix(line, "{") {
|
||||
continue
|
||||
}
|
||||
var e apiSessionEvent
|
||||
if err = json.Unmarshal([]byte(line), &e); err != nil {
|
||||
continue
|
||||
}
|
||||
events = append(events, e)
|
||||
}
|
||||
return events, nil
|
||||
}
|
||||
|
||||
// lastToken returns the last whitespace-separated token of s, which lets us tolerate any command
|
||||
// echo or log prefix that precedes a numeric result.
|
||||
func lastToken(s string) string {
|
||||
fields := strings.Fields(s)
|
||||
if len(fields) == 0 {
|
||||
return "0"
|
||||
}
|
||||
return fields[len(fields)-1]
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/openziti/fablab/kernel/lib/actions"
|
||||
"github.com/openziti/fablab/kernel/lib/actions/component"
|
||||
"github.com/openziti/fablab/kernel/lib/actions/semaphore"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/ziti/zititest/zitilab/actions/edge"
|
||||
"github.com/openziti/ziti/zititest/zitilab/models"
|
||||
)
|
||||
|
||||
type startAction struct{}
|
||||
|
||||
// newStartAction starts the phase-1 system: the single initial controller, the routers, the
|
||||
// loop4 hosts/backends, and finally the clients (including the remote-controlled sim client).
|
||||
func newStartAction() model.ActionBinder {
|
||||
action := &startAction{}
|
||||
return action.bind
|
||||
}
|
||||
|
||||
func (a *startAction) bind(m *model.Model) model.Action {
|
||||
workflow := actions.Workflow()
|
||||
|
||||
workflow.AddAction(component.Start("#ctrl1"))
|
||||
workflow.AddAction(edge.ControllerAvailable("#ctrl1", 30*time.Second))
|
||||
workflow.AddAction(edge.Login("#ctrl1"))
|
||||
|
||||
workflow.AddAction(component.StartInParallel(models.EdgeRouterTag, 10))
|
||||
workflow.AddAction(semaphore.Sleep(2 * time.Second))
|
||||
|
||||
// hosts: plain-TCP loop4 backends, the SDK loop4 host, and the ZET/ziti-tunnel hosts
|
||||
workflow.AddAction(component.StartInParallel(".loop-backend", 10))
|
||||
workflow.AddAction(component.StartInParallel(".sdk-app.host", 10))
|
||||
workflow.AddAction(semaphore.Sleep(2 * time.Second))
|
||||
|
||||
// clients: ZET client, ziti-tunnel client, and the remote-controlled sim client
|
||||
workflow.AddAction(component.StartInParallel(".sdk-app.client", 10))
|
||||
workflow.AddAction(semaphore.Sleep(2 * time.Second))
|
||||
|
||||
return workflow
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/openziti/channel/v5"
|
||||
"github.com/openziti/fablab/kernel/lib/tui"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/ziti/v2/controller/rest_client/terminator"
|
||||
"github.com/openziti/ziti/v2/zitirest"
|
||||
"github.com/openziti/ziti/zititest/ziti-traffic-test/loop4"
|
||||
"github.com/openziti/ziti/zititest/zitilab"
|
||||
"github.com/openziti/ziti/zititest/zitilab/chaos"
|
||||
zitiLibOps "github.com/openziti/ziti/zititest/zitilab/runlevel/5_operation"
|
||||
"github.com/openziti/ziti/zititest/zitilab/validations"
|
||||
)
|
||||
|
||||
// noopControllerCallback satisfies loop4.ControllerCallback without requesting any diagnostics.
|
||||
type noopControllerCallback struct{}
|
||||
|
||||
func (noopControllerCallback) DiagnosticRequested(*channel.Message, channel.Channel) {
|
||||
tui.ValidationLogger().Debug("sim diagnostic requested; ignoring")
|
||||
}
|
||||
|
||||
// expectedTerminator is a service and the number of terminators that must be present for it once the
|
||||
// system is healthy.
|
||||
type expectedTerminator struct {
|
||||
service string
|
||||
count int64
|
||||
}
|
||||
|
||||
// expectedTerminators is the full set of terminators the topology should have when healthy: one loop
|
||||
// service per hosting flavor plus the two sim control-plane services. It is verified both on initial
|
||||
// setup and after every disruptive step (each host must re-establish its terminator).
|
||||
var expectedTerminators = []expectedTerminator{
|
||||
{service: "loop-sdk", count: 2}, // loop4-sdk-host-stable/-restart
|
||||
{service: "loop-ziti-tunnel", count: 2}, // ziti-tunnel-host-stable/-restart
|
||||
{service: "loop-zet", count: 2}, // ziti-edge-tunnel-host-stable/-restart
|
||||
{service: "loop-ert", count: 1}, // router-west edge-router tunneler
|
||||
{service: "sim-control", count: 1}, // sim harness
|
||||
{service: "metrics", count: 1}, // sim harness
|
||||
}
|
||||
|
||||
// totalExpectedTerminators returns the sum of all expected terminator counts.
|
||||
func totalExpectedTerminators() int64 {
|
||||
var total int64
|
||||
for _, e := range expectedTerminators {
|
||||
total += e.count
|
||||
}
|
||||
return total
|
||||
}
|
||||
|
||||
// steadyStateGate validates that traffic is healthy by driving discrete loop4 scenario runs.
|
||||
// A run is "clean" when every remote-controlled sim client reports success across every
|
||||
// hosting flavor. The gate first waits out any churn (recovery), then requires sustained
|
||||
// clean runs (stability) before it returns.
|
||||
type steadyStateGate struct {
|
||||
sim *zitiLibOps.SimServices
|
||||
|
||||
// requiredCleanRuns is how many consecutive clean runs declare the system recovered.
|
||||
requiredCleanRuns int
|
||||
// recoveryTimeout bounds how long we tolerate failures before giving up on recovery.
|
||||
recoveryTimeout time.Duration
|
||||
// stabilityWindow is how long runs must stay clean once recovered.
|
||||
stabilityWindow time.Duration
|
||||
|
||||
// postDisruptionDelay is how long to wait after a disruptive step before checking terminators,
|
||||
// letting the initial churn begin to settle.
|
||||
postDisruptionDelay time.Duration
|
||||
// terminatorSettleTimeout bounds how long to wait for all expected terminators to (re-)establish.
|
||||
terminatorSettleTimeout time.Duration
|
||||
|
||||
connectTimeout time.Duration
|
||||
scenarioTimeout time.Duration
|
||||
interRunDelay time.Duration
|
||||
}
|
||||
|
||||
// newSteadyStateGate returns a gate with default tuning; tuning is refined from model
|
||||
// variables under the steadyState.* namespace at validation time.
|
||||
func newSteadyStateGate(sim *zitiLibOps.SimServices) *steadyStateGate {
|
||||
return &steadyStateGate{
|
||||
sim: sim,
|
||||
requiredCleanRuns: 3,
|
||||
recoveryTimeout: time.Minute,
|
||||
stabilityWindow: 2 * time.Minute,
|
||||
postDisruptionDelay: 30 * time.Second,
|
||||
terminatorSettleTimeout: time.Minute,
|
||||
connectTimeout: 60 * time.Second,
|
||||
scenarioTimeout: 2 * time.Minute,
|
||||
interRunDelay: time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *steadyStateGate) loadTuning(m *model.Model) {
|
||||
if v := m.GetStringVariableOr("steadyState.requiredCleanRuns", ""); v != "" {
|
||||
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||
g.requiredCleanRuns = n
|
||||
}
|
||||
}
|
||||
if v := m.GetStringVariableOr("steadyState.recoveryTimeout", ""); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
g.recoveryTimeout = d
|
||||
}
|
||||
}
|
||||
if v := m.GetStringVariableOr("steadyState.stabilityWindow", ""); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
g.stabilityWindow = d
|
||||
}
|
||||
}
|
||||
if v := m.GetStringVariableOr("steadyState.postDisruptionDelay", ""); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
g.postDisruptionDelay = d
|
||||
}
|
||||
}
|
||||
if v := m.GetStringVariableOr("steadyState.terminatorSettleTimeout", ""); v != "" {
|
||||
if d, err := time.ParseDuration(v); err == nil {
|
||||
g.terminatorSettleTimeout = d
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (g *steadyStateGate) validate(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
g.loadTuning(m)
|
||||
|
||||
// gate output (scenario runs, recovery/stability progress) routes to the TUI validation pane
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
// ensure the harness is hosting sim-control before we verify terminators; the first call binds it,
|
||||
// and its sim-control/metrics terminators are part of the expected set checked below. Post-disruption
|
||||
// this reuses the existing controller, so the terminator must re-establish on its own (SDK recovery).
|
||||
simControl, err := g.sim.GetSimController(run, "sim-control", noopControllerCallback{})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// every expected terminator must be present and valid before we drive traffic; on initial setup
|
||||
// this confirms the topology, after a disruption it confirms every host has re-established.
|
||||
if err = g.verifyTerminators(run); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
sims := m.FilterComponents(".loop-client", func(c *model.Component) bool {
|
||||
t, ok := c.Type.(*zitilab.Loop4SimType)
|
||||
return ok && t.Mode == zitilab.Loop4RemoteControlled
|
||||
})
|
||||
if len(sims) == 0 {
|
||||
return fmt.Errorf("no remote-controlled loop clients found")
|
||||
}
|
||||
|
||||
log.Infof("steady-state gate: waiting up to %s for %d sim client(s) to connect", g.connectTimeout, len(sims))
|
||||
if err = simControl.WaitForAllConnected(g.connectTimeout, sims); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// recovery: keep running until we string together enough clean runs, or time out
|
||||
deadline := time.Now().Add(g.recoveryTimeout)
|
||||
consecutive := 0
|
||||
for consecutive < g.requiredCleanRuns {
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("did not reach %d consecutive clean runs within %s", g.requiredCleanRuns, g.recoveryTimeout)
|
||||
}
|
||||
if err = g.runScenario(simControl, sims); err != nil {
|
||||
log.WithError(err).Warnf("scenario run failed during recovery; resetting clean-run count from %d", consecutive)
|
||||
consecutive = 0
|
||||
time.Sleep(g.interRunDelay)
|
||||
continue
|
||||
}
|
||||
consecutive++
|
||||
log.Infof("steady-state gate recovery: %d/%d consecutive clean runs", consecutive, g.requiredCleanRuns)
|
||||
}
|
||||
|
||||
// stability: any failure in the window fails the gate
|
||||
log.Infof("steady-state gate: recovered; requiring clean runs for %s", g.stabilityWindow)
|
||||
stabilityEnd := time.Now().Add(g.stabilityWindow)
|
||||
for time.Now().Before(stabilityEnd) {
|
||||
if err = g.runScenario(simControl, sims); err != nil {
|
||||
return fmt.Errorf("stability window failed: %w", err)
|
||||
}
|
||||
time.Sleep(g.interRunDelay)
|
||||
}
|
||||
|
||||
log.Info("steady-state gate: passed")
|
||||
return nil
|
||||
}
|
||||
|
||||
// runScenario fires a single scenario across all sim clients and returns an error unless
|
||||
// every client reports success. A disconnected client is treated as a failure.
|
||||
func (g *steadyStateGate) runScenario(simControl *loop4.RemoteController, sims []*model.Component) error {
|
||||
if missing := simControl.MissingComponents(sims); len(missing) > 0 {
|
||||
return fmt.Errorf("sim clients not connected: %v", missing)
|
||||
}
|
||||
results, err := simControl.StartSimScenarios()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return results.GetResults(g.scenarioTimeout)
|
||||
}
|
||||
|
||||
// validateAfterDisruption is the gate variant used after a disruptive step (controller/router
|
||||
// restart). It waits for the churn to begin settling before running the normal gate, which then
|
||||
// waits for every expected terminator to re-establish.
|
||||
func (g *steadyStateGate) validateAfterDisruption(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
g.loadTuning(m)
|
||||
log := tui.ValidationLogger()
|
||||
log.Infof("post-disruption settle: waiting %s before verifying terminators", g.postDisruptionDelay)
|
||||
time.Sleep(g.postDisruptionDelay)
|
||||
|
||||
// Pre-2.0 routers have a bug where, when the controller upgrade purges their legacy sessions, they
|
||||
// drop the affected hosted terminators locally without telling the controller, leaving stale
|
||||
// terminators the router no longer hosts. Proactively reconcile them (delete the ones routers no
|
||||
// longer host) so the strict validation reflects reality. Gate on the routers' *current* version,
|
||||
// not fromVersion: once every router is upgraded to 2.0+ this must stop running so genuine 2.0+
|
||||
// terminator bugs are caught rather than masked.
|
||||
if anyPreV2Router(m) {
|
||||
if err := g.reconcileStaleTerminators(run); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return g.validate(run)
|
||||
}
|
||||
|
||||
// reconcileStaleTerminators deletes edge terminators the routers no longer host (the pre-2.0 upgrade
|
||||
// leaves these behind). It is scoped to edge terminators: an unscoped fix would inspect ERT terminators
|
||||
// on the routers, which panics pre-2.0 routers that host none.
|
||||
func (g *steadyStateGate) reconcileStaleTerminators(run model.Run) error {
|
||||
log := tui.ValidationLogger()
|
||||
ctrl := run.GetModel().MustSelectComponent("#ctrl1")
|
||||
clients, err := chaos.EnsureLoggedIntoCtrl(run, ctrl, time.Minute)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to log into #ctrl1 to reconcile stale terminators: %w", err)
|
||||
}
|
||||
fixed, err := validations.FixInvalidTerminators(clients, `binding="edge" limit none`, g.terminatorSettleTimeout)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to reconcile stale terminators: %w", err)
|
||||
}
|
||||
log.Infof("reconciled stale terminators (pre-2.0 upgrade workaround): %d fixed", fixed)
|
||||
return nil
|
||||
}
|
||||
|
||||
// isPreV2 reports whether version is a pre-2.0 release (e.g. v1.x). Empty or unparsable versions are
|
||||
// treated as not pre-2.0, so the stale-terminator workaround stays off unless we know it is needed.
|
||||
func isPreV2(version string) bool {
|
||||
major, _, _ := strings.Cut(strings.TrimPrefix(strings.TrimSpace(version), "v"), ".")
|
||||
n, err := strconv.Atoi(major)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
return n < 2
|
||||
}
|
||||
|
||||
// anyPreV2Router reports whether any router component is still on a pre-2.0 version. The
|
||||
// stale-terminator reconciliation only applies to pre-2.0 router behavior, so it must stop once
|
||||
// every router has been upgraded, otherwise it masks genuine 2.0+ terminator bugs.
|
||||
func anyPreV2Router(m *model.Model) bool {
|
||||
result := false
|
||||
_ = m.ForEachComponent("*", 1, func(c *model.Component) error {
|
||||
if rt, ok := c.Type.(*zitilab.RouterType); ok && isPreV2(rt.Version) {
|
||||
result = true
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return result
|
||||
}
|
||||
|
||||
// verifyTerminators waits until every expected terminator is present for its service, then confirms
|
||||
// the router SDK and ERT terminators are valid all the way up the stack. It bounds the wait by
|
||||
// terminatorSettleTimeout so a host that never re-establishes fails the gate.
|
||||
func (g *steadyStateGate) verifyTerminators(run model.Run) error {
|
||||
log := tui.ValidationLogger()
|
||||
ctrl := run.GetModel().MustSelectComponent("#ctrl1")
|
||||
|
||||
log.Infof("verifying %d expected terminators are present (timeout %s)", totalExpectedTerminators(), g.terminatorSettleTimeout)
|
||||
deadline := time.Now().Add(g.terminatorSettleTimeout)
|
||||
var clients *zitirest.Clients
|
||||
var lastLog time.Time
|
||||
for {
|
||||
if clients == nil {
|
||||
var err error
|
||||
if clients, err = chaos.EnsureLoggedIntoCtrl(run, ctrl, time.Minute); err != nil {
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("unable to log into #ctrl1 to verify terminators: %w", err)
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
missing, err := g.missingTerminators(clients)
|
||||
if err != nil {
|
||||
clients = nil
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("unable to list terminators: %w", err)
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
if len(missing) == 0 {
|
||||
break
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
return fmt.Errorf("expected terminators not present within %s: %v", g.terminatorSettleTimeout, missing)
|
||||
}
|
||||
if time.Since(lastLog) > 15*time.Second {
|
||||
log.Infof("waiting for terminators: %v", missing)
|
||||
lastLog = time.Now()
|
||||
}
|
||||
time.Sleep(5 * time.Second)
|
||||
}
|
||||
|
||||
// terminators are all present; confirm they're valid up the stack (router <-> controller agree).
|
||||
// ERT validation is scoped to edge-router tunnelers: inspecting the ERT registry on a non-hosting
|
||||
// router (e.g. an initiator) panics older (1.6.x) routers, and only ERT hosts have ERT terminators.
|
||||
log.Info("all expected terminators present; validating them up the stack")
|
||||
validationDeadline := time.Now().Add(g.terminatorSettleTimeout)
|
||||
return validations.ValidateTerminatorsForCtrlWithFilters(run, ctrl, validationDeadline,
|
||||
validations.MinCount(totalExpectedTerminators()),
|
||||
validations.ValidateSdkTerminators|validations.ValidateErtTerminators,
|
||||
"limit none", ertRouterFilter(run.GetModel()))
|
||||
}
|
||||
|
||||
// ertRouterFilter builds a router filter selecting the edge-router tunnelers (the ".ert-host" routers),
|
||||
// so ERT terminator validation only inspects routers that actually host ERT terminators. If none are
|
||||
// found it returns a filter that matches no routers.
|
||||
func ertRouterFilter(m *model.Model) string {
|
||||
routers := m.SelectComponents(".ert-host")
|
||||
if len(routers) == 0 {
|
||||
return `name = "" limit none`
|
||||
}
|
||||
names := make([]string, 0, len(routers))
|
||||
for _, r := range routers {
|
||||
names = append(names, fmt.Sprintf("%q", r.Id))
|
||||
}
|
||||
return fmt.Sprintf("name in [%s] limit none", strings.Join(names, ","))
|
||||
}
|
||||
|
||||
// missingTerminators returns, for each expected service that is short, a human-readable "service:
|
||||
// have/want" entry. An empty result means every expected terminator is present.
|
||||
func (g *steadyStateGate) missingTerminators(clients *zitirest.Clients) ([]string, error) {
|
||||
var missing []string
|
||||
for _, e := range expectedTerminators {
|
||||
count, err := terminatorCountForService(clients, e.service)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if count < e.count {
|
||||
missing = append(missing, fmt.Sprintf("%s: %d/%d", e.service, count, e.count))
|
||||
}
|
||||
}
|
||||
return missing, nil
|
||||
}
|
||||
|
||||
// terminatorCountForService returns the number of terminators the controller has for the named service.
|
||||
func terminatorCountForService(clients *zitirest.Clients, service string) (int64, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
|
||||
defer cancel()
|
||||
|
||||
filter := fmt.Sprintf(`service.name="%s" limit 1`, service)
|
||||
result, err := clients.Fabric.Terminator.ListTerminators(&terminator.ListTerminatorsParams{
|
||||
Filter: &filter,
|
||||
Context: ctx,
|
||||
}, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return *result.Payload.Meta.Pagination.TotalCount, nil
|
||||
}
|
||||
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
(c) Copyright NetFoundry Inc.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
https://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
"github.com/openziti/fablab/kernel/lib/tui"
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/foundation/v2/versions"
|
||||
"github.com/openziti/ziti/v2/ziti/util"
|
||||
"github.com/openziti/ziti/zititest/zitilab"
|
||||
"github.com/openziti/ziti/zititest/zitilab/actions/edge"
|
||||
"github.com/openziti/ziti/zititest/zitilab/chaos"
|
||||
"github.com/openziti/ziti/zititest/zitilab/models"
|
||||
"github.com/openziti/ziti/zititest/zitilab/stageziti"
|
||||
)
|
||||
|
||||
// envOrModelVar returns the value of envVar when it is set in the environment, otherwise the model
|
||||
// variable name. The environment wins so a single run can override a value the model hardcodes, and
|
||||
// an env var set to the empty string is honored as such: that is the only way to switch off a
|
||||
// non-empty model default without editing the model.
|
||||
func envOrModelVar(m *model.Model, name, envVar string) string {
|
||||
if v, found := os.LookupEnv(envVar); found {
|
||||
return v
|
||||
}
|
||||
return m.GetStringVariableOr(name, "")
|
||||
}
|
||||
|
||||
// upgradeTarget names the model variables and environment overrides that select the ziti binary for
|
||||
// one of the model's upgrade targets.
|
||||
type upgradeTarget struct {
|
||||
// version holds the version label the binary is staged under and, for ref and source builds, the
|
||||
// version stamped into it. An empty value means the target is not in play.
|
||||
version string
|
||||
source string
|
||||
ref string
|
||||
sourceEnv string
|
||||
refEnv string
|
||||
}
|
||||
|
||||
var (
|
||||
// toTarget is the 1.6 -> 2.0 upgrade the model always runs.
|
||||
toTarget = upgradeTarget{
|
||||
version: "toVersion",
|
||||
source: "toVersionSource",
|
||||
ref: "toVersionRef",
|
||||
sourceEnv: "ZITI_TOVERSION_PATH",
|
||||
refEnv: "ZITI_TOVERSION_REF",
|
||||
}
|
||||
// nextTarget is the optional 2.0 -> 2.1 cluster upgrade that follows it.
|
||||
nextTarget = upgradeTarget{
|
||||
version: "nextVersion",
|
||||
source: "nextVersionSource",
|
||||
ref: "nextVersionRef",
|
||||
sourceEnv: "ZITI_NEXTVERSION_PATH",
|
||||
refEnv: "ZITI_NEXTVERSION_REF",
|
||||
}
|
||||
)
|
||||
|
||||
// stage pre-stages the target's ziti binary into the kit so it is rsynced to every host alongside the
|
||||
// fromVersion binary. In-place upgrades then only require a version swap and a restart, with no
|
||||
// mid-run binary transfer. A target whose version is unset stages nothing.
|
||||
//
|
||||
// The binary is chosen in precedence order, with each environment variable overriding its model
|
||||
// variable:
|
||||
// - the source variable: a locally-built ziti binary, copied under the target's version name. Use
|
||||
// this to run a patched controller/router from a binary you already built.
|
||||
// - the ref variable: a git ref (branch, tag, or SHA) on openziti/ziti, built from source via
|
||||
// acquire and staged under the target's version name. Use this to run a fix, or a release that
|
||||
// does not exist yet, without hand-building. The build targets the hosts' platform, so it works
|
||||
// from a mac or arm machine too, but it must be a git ref: a release version belongs in the
|
||||
// version variable.
|
||||
// - otherwise the released version is downloaded.
|
||||
func (self upgradeTarget) stage(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
version := m.GetStringVariableOr(self.version, "")
|
||||
if version == "" {
|
||||
return nil
|
||||
}
|
||||
log := tui.ValidationLogger()
|
||||
target := filepath.Join(run.GetBinDir(), "ziti-"+version)
|
||||
|
||||
c := m.MustSelectComponent("#ctrl1")
|
||||
|
||||
source := envOrModelVar(m, self.source, self.sourceEnv)
|
||||
if source != "" {
|
||||
log.Infof("staging local ziti [%s] as the %s binary [%s]", source, self.version, target)
|
||||
return util.CopyFile(source, target)
|
||||
}
|
||||
|
||||
ref := envOrModelVar(m, self.ref, self.refEnv)
|
||||
if ref != "" {
|
||||
log.Infof("pre-staging %s ziti built from ref %s as [%s]", self.version, ref, target)
|
||||
return stageziti.StageZitiFromRefOnce(run, c, ref, version, "")
|
||||
}
|
||||
|
||||
log.Infof("pre-staging %s ziti binary [%s]", self.version, version)
|
||||
return stageziti.StageZitiOnce(run, c, version, "")
|
||||
}
|
||||
|
||||
// upgradeController upgrades ctrl1 from fromVersion to toVersion in place: it swaps the controller
|
||||
// binary and restarts. The controller config is unchanged, so the 2.0 binary auto-binds the edge-oidc
|
||||
// API and OIDC becomes available with no config change.
|
||||
func upgradeController(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
toVersion := m.MustStringVariable("toVersion")
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
c := m.MustSelectComponent("#ctrl1")
|
||||
ctrlType, ok := c.Type.(*zitilab.ControllerType)
|
||||
if !ok {
|
||||
return fmt.Errorf("component ctrl1 is not a controller")
|
||||
}
|
||||
|
||||
log.Infof("upgrading ctrl1 to %s", toVersion)
|
||||
ctrlType.SetVersion(toVersion)
|
||||
|
||||
// RestartSelected waits for the old process to exit before starting the new one; a plain stop/start
|
||||
// would race, since Start no-ops while the old (pre-upgrade) process is still shutting down
|
||||
if err := chaos.RestartSelected(run, 1, c); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := edge.ControllerAvailable("#ctrl1", 60*time.Second).Execute(run); err != nil {
|
||||
return err
|
||||
}
|
||||
return edge.Login("#ctrl1").Execute(run)
|
||||
}
|
||||
|
||||
// upgradeRouters upgrades the edge routers from fromVersion to toVersion in place.
|
||||
func upgradeRouters(run model.Run) error {
|
||||
return upgradeRoutersTo(run, run.GetModel().MustStringVariable("toVersion"))
|
||||
}
|
||||
|
||||
// upgradeRoutersTo upgrades the edge routers to version in place, one at a time so traffic keeps
|
||||
// flowing through the others. Each router restart briefly disrupts circuits on that router, which is
|
||||
// expected; the others carry traffic in the meantime.
|
||||
func upgradeRoutersTo(run model.Run, version string) error {
|
||||
m := run.GetModel()
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
routers := m.SelectComponents(models.EdgeRouterTag)
|
||||
for _, c := range routers {
|
||||
rt, ok := c.Type.(*zitilab.RouterType)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
log.Infof("upgrading router %s to %s", c.Id, version)
|
||||
rt.SetVersion(version)
|
||||
if err := chaos.RestartSelected(run, 1, c); err != nil {
|
||||
return err
|
||||
}
|
||||
// give the router time to reconnect and re-establish terminators before moving to the next
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// zetRestartWorkaroundMaxVersion is the newest ziti-edge-tunnel version known to need the
|
||||
// post-controller-upgrade restart workaround. At or below this version, ziti-edge-tunnel does not
|
||||
// rebuild its edge sessions after the 1.x->2.x controller upgrade invalidates them (the required JWT
|
||||
// session migration): it keeps failing dials with "session closed" until its api-session refresh
|
||||
// (~20 minutes), far outside the steady-state gate window. Restarting it forces a fresh authentication
|
||||
// and valid sessions. Newer versions get no workaround, so if the gap is not actually fixed there the
|
||||
// gate still catches it. Only raise this once a newer ziti-edge-tunnel is confirmed to self-recover.
|
||||
const zetRestartWorkaroundMaxVersion = "v1.18.0"
|
||||
|
||||
// zetNeedsRestartWorkaround reports whether zetVersion is old enough to need the post-upgrade ZET
|
||||
// restart workaround. An empty or unparsable version is treated as needing it, so an unknown build
|
||||
// is not silently left in the broken state.
|
||||
func zetNeedsRestartWorkaround(zetVersion string) bool {
|
||||
v, err := versions.ParseSemVer(zetVersion)
|
||||
if err != nil {
|
||||
return true
|
||||
}
|
||||
return v.CompareTo(versions.MustParseSemVer(zetRestartWorkaroundMaxVersion)) <= 0
|
||||
}
|
||||
|
||||
// restartZetWorkaround restarts every ziti-edge-tunnel instance after the controller upgrade, but only
|
||||
// when zetVersion is old enough to need it (see zetRestartWorkaroundMaxVersion). Those versions cannot
|
||||
// recover their edge sessions on their own after the 1.x->2.x session migration, so the restart forces
|
||||
// a re-auth and the ZET data path recovers within the gate window instead of after the api-session
|
||||
// refresh. This deliberately breaks ZET traffic continuity across the controller upgrade, which is the
|
||||
// known limitation being worked around.
|
||||
func restartZetWorkaround(run model.Run) error {
|
||||
m := run.GetModel()
|
||||
zetVersion := m.GetStringVariableOr("zetVersion", "")
|
||||
log := tui.ValidationLogger()
|
||||
|
||||
if !zetNeedsRestartWorkaround(zetVersion) {
|
||||
log.Infof("ziti-edge-tunnel %s is newer than %s; skipping post-upgrade ZET restart workaround",
|
||||
zetVersion, zetRestartWorkaroundMaxVersion)
|
||||
return nil
|
||||
}
|
||||
|
||||
zets := m.SelectComponents(".zet")
|
||||
log.Infof("ziti-edge-tunnel %s needs the post-2.0-migration restart workaround; restarting %d ZET instance(s)",
|
||||
zetVersion, len(zets))
|
||||
return chaos.RestartSelected(run, 10, zets...)
|
||||
}
|
||||
@@ -154,6 +154,8 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res
|
||||
workloadStart := time.Now()
|
||||
|
||||
var result *Result
|
||||
var lastDialErr error
|
||||
succeeded := false
|
||||
for i := int64(0); i < workload.Iterations || workload.Iterations == -1; i++ {
|
||||
log = log.WithField("iteration", i+1)
|
||||
if conn != nil {
|
||||
@@ -170,6 +172,7 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res
|
||||
if err != nil {
|
||||
log.WithError(err).Error("failed to dial")
|
||||
connectFailures.Mark(1)
|
||||
lastDialErr = err
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -233,6 +236,7 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res
|
||||
|
||||
active.Add(-1)
|
||||
completed.Mark(1)
|
||||
succeeded = true
|
||||
log.Debug("completed iteration")
|
||||
if scenario.ConnectionDelay > 0 {
|
||||
time.Sleep(time.Duration(sim.scenario.ConnectionDelay) * time.Millisecond)
|
||||
@@ -248,6 +252,17 @@ func (sim *Sim) RunWorkload(scenario *Scenario, workload *Workload, idx int, res
|
||||
sim.metricsReporter.Flush()
|
||||
}
|
||||
|
||||
// A workload that completed no successful iteration is a failure, even when every failure was a
|
||||
// dial failure (e.g. connection refused). Dial failures are otherwise swallowed by the retry
|
||||
// loop above, so without this a service that can't be reached at all would report success.
|
||||
if !succeeded {
|
||||
if lastDialErr == nil {
|
||||
lastDialErr = fmt.Errorf("workload %s completed no successful iterations", workload.Name)
|
||||
}
|
||||
sim.reportErr(resultCh, lastDialErr, "unknown")
|
||||
return
|
||||
}
|
||||
|
||||
resultCh <- &Result{
|
||||
Success: true,
|
||||
}
|
||||
|
||||
@@ -20,6 +20,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/michaelquigley/pfxlog"
|
||||
@@ -41,15 +42,122 @@ func init() {
|
||||
loop4Cmd.AddCommand(newRemoteControlledCmd())
|
||||
}
|
||||
|
||||
// scenarioResultSendTimeout bounds how long a finished scenario keeps trying to report its result
|
||||
// while the sim is reconnecting. Past it the controller's own scenario timeout takes over.
|
||||
const scenarioResultSendTimeout = 60 * time.Second
|
||||
|
||||
// completedScenarioCacheSize is how many finished scenarios are remembered for replay. The controller
|
||||
// runs one scenario at a time, so a resend more than a handful of scenarios stale cannot occur.
|
||||
const completedScenarioCacheSize = 16
|
||||
|
||||
type remoteControlledCmd struct {
|
||||
*Sim
|
||||
notifyClose chan struct{}
|
||||
|
||||
// controlCh is the live channel to the sim controller. A reconnect replaces it, so anything
|
||||
// reporting to the controller must read it at send time rather than capture it: a scenario often
|
||||
// outlives the channel its run request arrived on.
|
||||
controlCh concurrenz.AtomicValue[channel.Channel]
|
||||
|
||||
scenarioLock sync.Mutex
|
||||
// runningScenarioId is the scenario currently being run, empty when idle.
|
||||
runningScenarioId string
|
||||
// completed holds recent scenario outcomes for replay, oldest first in completedIds.
|
||||
completed map[string]scenarioOutcome
|
||||
completedIds []string
|
||||
}
|
||||
|
||||
// scenarioOutcome is a scenario run's reportable result.
|
||||
type scenarioOutcome struct {
|
||||
success bool
|
||||
message string
|
||||
}
|
||||
|
||||
// scenarioAction is what to do with an incoming run request.
|
||||
type scenarioAction int
|
||||
|
||||
const (
|
||||
// actionRun means the sim is claimed for the scenario and the caller should run it.
|
||||
actionRun scenarioAction = iota
|
||||
// actionIgnore means this scenario is already running and will report on its own.
|
||||
actionIgnore
|
||||
// actionReplay means this scenario already finished; answer from its recorded outcome.
|
||||
actionReplay
|
||||
// actionReject means a different scenario holds the sim.
|
||||
actionReject
|
||||
)
|
||||
|
||||
// scenarioDecision resolves one run request. runningId is set for actionReject, outcome for
|
||||
// actionReplay.
|
||||
type scenarioDecision struct {
|
||||
action scenarioAction
|
||||
runningId string
|
||||
outcome scenarioOutcome
|
||||
}
|
||||
|
||||
// decideScenario resolves a run request for scenarioId, claiming the sim when it returns actionRun.
|
||||
//
|
||||
// Only one scenario runs at a time: two would reset and record the same sim-wide metrics and drive the
|
||||
// same workloads, so their results would interfere.
|
||||
func (cmd *remoteControlledCmd) decideScenario(scenarioId string) scenarioDecision {
|
||||
cmd.scenarioLock.Lock()
|
||||
defer cmd.scenarioLock.Unlock()
|
||||
|
||||
if cmd.runningScenarioId == scenarioId {
|
||||
return scenarioDecision{action: actionIgnore}
|
||||
}
|
||||
// Ahead of the busy check on purpose: a resend for a scenario that already finished has to be
|
||||
// answered from the cache even when a later scenario now holds the sim, or it reruns work the
|
||||
// controller has already accepted and then fails the scenario that follows it.
|
||||
if outcome, found := cmd.completed[scenarioId]; found {
|
||||
return scenarioDecision{action: actionReplay, outcome: outcome}
|
||||
}
|
||||
if cmd.runningScenarioId != "" {
|
||||
// Recorded here, under the lock that saw the conflict, so the rejection is terminal. Left
|
||||
// unrecorded, a resend of this scenario would find the sim idle and start it, and the run would
|
||||
// still be going when the controller moved on, rejecting the scenario after it in turn.
|
||||
outcome := scenarioOutcome{
|
||||
success: false,
|
||||
message: fmt.Sprintf("sim is busy running scenario %s", cmd.runningScenarioId),
|
||||
}
|
||||
cmd.recordOutcome(scenarioId, outcome)
|
||||
return scenarioDecision{action: actionReject, runningId: cmd.runningScenarioId, outcome: outcome}
|
||||
}
|
||||
cmd.runningScenarioId = scenarioId
|
||||
return scenarioDecision{action: actionRun}
|
||||
}
|
||||
|
||||
// recordOutcome caches scenarioId's terminal outcome, evicting the oldest entry when full. The first
|
||||
// outcome recorded for a scenario wins. Callers must hold scenarioLock.
|
||||
func (cmd *remoteControlledCmd) recordOutcome(scenarioId string, outcome scenarioOutcome) {
|
||||
if _, found := cmd.completed[scenarioId]; found {
|
||||
return
|
||||
}
|
||||
cmd.completed[scenarioId] = outcome
|
||||
cmd.completedIds = append(cmd.completedIds, scenarioId)
|
||||
if len(cmd.completedIds) > completedScenarioCacheSize {
|
||||
delete(cmd.completed, cmd.completedIds[0])
|
||||
cmd.completedIds = cmd.completedIds[1:]
|
||||
}
|
||||
}
|
||||
|
||||
// finishScenario releases the sim and records scenarioId's outcome, so a resend that crosses with the
|
||||
// result is replayed rather than run again.
|
||||
func (cmd *remoteControlledCmd) finishScenario(scenarioId string, outcome scenarioOutcome) {
|
||||
cmd.scenarioLock.Lock()
|
||||
defer cmd.scenarioLock.Unlock()
|
||||
|
||||
if cmd.runningScenarioId == scenarioId {
|
||||
cmd.runningScenarioId = ""
|
||||
}
|
||||
cmd.recordOutcome(scenarioId, outcome)
|
||||
}
|
||||
|
||||
func newRemoteControlledCmd() *cobra.Command {
|
||||
dialer := &remoteControlledCmd{
|
||||
Sim: NewSim(),
|
||||
notifyClose: make(chan struct{}, 1),
|
||||
completed: map[string]scenarioOutcome{},
|
||||
}
|
||||
|
||||
cmd := &cobra.Command{
|
||||
@@ -105,6 +213,10 @@ func (cmd *remoteControlledCmd) runRemoteControlled(_ *cobra.Command, args []str
|
||||
|
||||
if err = cmd.handleRemoteControlConn(sdkClient, conn); err != nil {
|
||||
log.WithError(err).Error("unable to channelize remote controller connection")
|
||||
// Close the dialed conn before retrying. Otherwise a failed channelize (e.g. a hello
|
||||
// that times out during controller churn) leaves the edge conn and its fabric circuit
|
||||
// open with no reader, leaking a circuit on every retry.
|
||||
_ = conn.Close()
|
||||
time.Sleep(1 * time.Second)
|
||||
attempt++
|
||||
continue
|
||||
@@ -139,6 +251,9 @@ func (cmd *remoteControlledCmd) handleRemoteControlConn(sdk ziti.Context, conn n
|
||||
}
|
||||
|
||||
func (cmd *remoteControlledCmd) BindChannel(binding channel.Binding) error {
|
||||
// Publish the channel before any handler can fire, so a request arriving immediately still has a
|
||||
// channel to answer on.
|
||||
cmd.controlCh.Store(binding.GetChannel())
|
||||
binding.AddReceiveHandlerF(int32(loop4Pb.ContentType_RunScenarioRequestType), cmd.HandleRunScenario)
|
||||
binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) {
|
||||
select {
|
||||
@@ -149,35 +264,102 @@ func (cmd *remoteControlledCmd) BindChannel(binding channel.Binding) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (cmd *remoteControlledCmd) HandleRunScenario(msg *channel.Message, ch channel.Channel) {
|
||||
// HandleRunScenario starts a scenario on request. The channel the request arrived on is deliberately
|
||||
// unused: results go to whatever channel is live when the run finishes, which a reconnect may change.
|
||||
//
|
||||
// Nothing here sends a result inline. This runs on the channel's receive loop, and sendScenarioResult
|
||||
// retries for up to a minute, which would stall every read on that channel, heartbeats included, and
|
||||
// so risk killing the connection the result still has to go out on. The replay and reject paths are
|
||||
// reached during reconnect and resend handling, which is exactly when that channel is least able to
|
||||
// absorb it.
|
||||
func (cmd *remoteControlledCmd) HandleRunScenario(msg *channel.Message, _ channel.Channel) {
|
||||
scenarioId, _ := msg.GetStringHeader(int32(loop4Pb.HeaderType_ScenarioId))
|
||||
go cmd.runRemoteScenario(scenarioId, cmd.scenario, ch)
|
||||
if scenarioId == "" {
|
||||
pfxlog.Logger().Error("run scenario request missing scenario id, ignoring")
|
||||
return
|
||||
}
|
||||
log := pfxlog.Logger().WithField("scenarioId", scenarioId)
|
||||
|
||||
switch decision := cmd.decideScenario(scenarioId); decision.action {
|
||||
case actionIgnore:
|
||||
// A re-send after this sim reconnected. The run already in progress reports for it, over
|
||||
// whatever channel is live when it finishes.
|
||||
log.Info("scenario already running, ignoring duplicate run request")
|
||||
return
|
||||
case actionReplay:
|
||||
log.Info("scenario already finished, replaying its result")
|
||||
go cmd.sendScenarioResult(scenarioId, decision.outcome)
|
||||
return
|
||||
case actionReject:
|
||||
// Answer instead of running. Staying silent would leave the controller waiting out its
|
||||
// scenario timeout for a result that is never coming.
|
||||
log.WithField("runningScenarioId", decision.runningId).
|
||||
Info("another scenario is running, rejecting run request")
|
||||
go cmd.sendScenarioResult(scenarioId, decision.outcome)
|
||||
return
|
||||
}
|
||||
|
||||
go func() {
|
||||
cmd.finishScenario(scenarioId, cmd.runRemoteScenario(scenarioId, cmd.scenario))
|
||||
}()
|
||||
}
|
||||
|
||||
func (cmd *remoteControlledCmd) sendScenarioResult(ch channel.Channel, id string, success bool, result string) {
|
||||
// sendScenarioResult reports a scenario's outcome, retrying over whatever control channel is current
|
||||
// while the sim reconnects.
|
||||
//
|
||||
// A scenario routinely outlives the channel its run request arrived on: the controller replaces that
|
||||
// channel when the sim reconnects and closes the old one. Reporting to the original channel would put
|
||||
// the result nowhere, leaving the controller to wait out its scenario timeout for a run that had in
|
||||
// fact finished.
|
||||
func (cmd *remoteControlledCmd) sendScenarioResult(id string, outcome scenarioOutcome) {
|
||||
log := pfxlog.Logger().WithField("scenarioId", id)
|
||||
|
||||
msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioResultType), []byte(result))
|
||||
msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), id)
|
||||
msg.PutBoolHeader(int32(loop4Pb.HeaderType_ScenarioSuccess), success)
|
||||
if err := msg.WithTimeout(10 * time.Second).Send(ch); err != nil {
|
||||
log.WithError(err).Error("unable to send scenario run result message")
|
||||
} else {
|
||||
log.Info("scenario result successfully reported")
|
||||
deadline := time.Now().Add(scenarioResultSendTimeout)
|
||||
for {
|
||||
msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioResultType), []byte(outcome.message))
|
||||
msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), id)
|
||||
msg.PutBoolHeader(int32(loop4Pb.HeaderType_ScenarioSuccess), outcome.success)
|
||||
|
||||
err := cmd.sendToController(msg)
|
||||
if err == nil {
|
||||
log.Info("scenario result successfully reported")
|
||||
return
|
||||
}
|
||||
if time.Now().After(deadline) {
|
||||
log.WithError(err).Errorf("giving up reporting scenario result after %s", scenarioResultSendTimeout)
|
||||
return
|
||||
}
|
||||
log.WithError(err).Info("unable to report scenario result, retrying on the current channel")
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func (cmd *remoteControlledCmd) sendDiagnosticRequest(ch channel.Channel, requestId string) {
|
||||
func (cmd *remoteControlledCmd) sendDiagnosticRequest(requestId string) {
|
||||
log := pfxlog.Logger().WithField("requestId", requestId)
|
||||
msg := channel.NewMessage(int32(loop4Pb.ContentType_RequestDiagnostic), nil)
|
||||
msg.PutStringHeader(int32(loop4Pb.HeaderType_RequestIdHeader), requestId)
|
||||
if err := msg.WithTimeout(10 * time.Second).Send(ch); err != nil {
|
||||
if err := cmd.sendToController(msg); err != nil {
|
||||
log.WithError(err).Error("unable to send diagnostic request message")
|
||||
} else {
|
||||
log.Info("diagnostic successfully requested")
|
||||
}
|
||||
}
|
||||
|
||||
// sendToController sends msg over the control channel that is live now, rather than one captured
|
||||
// earlier, which a reconnect may since have replaced.
|
||||
func (cmd *remoteControlledCmd) sendToController(msg *channel.Message) error {
|
||||
ch := cmd.controlCh.Load()
|
||||
if ch == nil {
|
||||
return errors.New("no control channel established")
|
||||
}
|
||||
if ch.IsClosed() {
|
||||
return errors.New("control channel is closed")
|
||||
}
|
||||
// Wait for the wire, not just the send queue: a queued result on a channel that then dies would be
|
||||
// reported as delivered, and the scenario retired as reported.
|
||||
return msg.WithTimeout(10 * time.Second).SendAndWaitForWire(ch)
|
||||
}
|
||||
|
||||
var triggerInspectAtomic concurrenz.AtomicValue[func(circuitId string)]
|
||||
|
||||
func triggerInspect(circuitId string) {
|
||||
@@ -189,11 +371,11 @@ func triggerInspect(circuitId string) {
|
||||
cb(circuitId)
|
||||
}
|
||||
|
||||
func (cmd *remoteControlledCmd) runRemoteScenario(scenarioId string, scenario *Scenario, ch channel.Channel) {
|
||||
func (cmd *remoteControlledCmd) runRemoteScenario(scenarioId string, scenario *Scenario) scenarioOutcome {
|
||||
log := pfxlog.Logger()
|
||||
|
||||
triggerInspectAtomic.Store(func(circuitId string) {
|
||||
cmd.sendDiagnosticRequest(ch, circuitId)
|
||||
cmd.sendDiagnosticRequest(circuitId)
|
||||
})
|
||||
|
||||
// reset metrics
|
||||
@@ -209,17 +391,16 @@ func (cmd *remoteControlledCmd) runRemoteScenario(scenarioId string, scenario *S
|
||||
|
||||
err := cmd.runScenario(scenario)
|
||||
|
||||
runSucceeded := true
|
||||
resultMsg := "success"
|
||||
outcome := scenarioOutcome{success: true, message: "success"}
|
||||
if err != nil {
|
||||
runSucceeded = false
|
||||
resultMsg = err.Error()
|
||||
outcome = scenarioOutcome{success: false, message: err.Error()}
|
||||
log.WithError(err).Errorf("scenario run unsuccessful")
|
||||
} else {
|
||||
log.Info("scenario run successful")
|
||||
}
|
||||
|
||||
cmd.sendScenarioResult(ch, scenarioId, runSucceeded, resultMsg)
|
||||
cmd.sendScenarioResult(scenarioId, outcome)
|
||||
return outcome
|
||||
}
|
||||
|
||||
func GetSdkIdentity(sdk ziti.Context) (*identity.TokenId, error) {
|
||||
|
||||
@@ -21,6 +21,7 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -54,6 +55,11 @@ type RemoteController struct {
|
||||
closed atomic.Bool
|
||||
|
||||
resultsTracker cmap.ConcurrentMap[string, *ScenarioResults]
|
||||
|
||||
// dispatchLock serializes dispatching a new scenario against re-sending in-flight ones to a
|
||||
// reconnecting sim, so a scenario is never re-sent while its dispatch is still deciding whether it
|
||||
// will be abandoned.
|
||||
dispatchLock sync.Mutex
|
||||
}
|
||||
|
||||
func (self *RemoteController) AcceptConnections(service string) error {
|
||||
@@ -97,8 +103,11 @@ func (self *RemoteController) Close() error {
|
||||
}
|
||||
|
||||
func (self *RemoteController) handleConnection(conn net.Conn) error {
|
||||
// handleConnection owns conn: on success it is owned by the channel wrapping it (whose Close closes
|
||||
// it), so every error path must close it here or the accepted conn and its circuit leak.
|
||||
tokenId, err := GetSdkIdentity(self.client)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return err
|
||||
}
|
||||
listener := channel.NewExistingConnListener(tokenId, conn, nil)
|
||||
@@ -107,17 +116,67 @@ func (self *RemoteController) handleConnection(conn net.Conn) error {
|
||||
var ch channel.Channel
|
||||
ch, err = channel.NewSingleChannel("control", listener, channel.BindHandlerF(self.BindChannel), options)
|
||||
if err != nil {
|
||||
_ = conn.Close()
|
||||
return fmt.Errorf("unable to establish connection from sim (%w)", err)
|
||||
}
|
||||
|
||||
clientId := string(ch.Headers()[HeaderClientId])
|
||||
self.clients.Set(clientId, ch)
|
||||
// Install the new channel and capture any channel it supersedes. A reconnecting sim opens a new
|
||||
// channel while its old one is still registered; without closing the old one it (and its underlying
|
||||
// edge conn and fabric circuit) leaks, since nothing else ever closes it. A leaked circuit stays
|
||||
// valid to the fabric forever, so the peer never receives a close and can block on it indefinitely.
|
||||
// The old channel must be closed after Upsert returns, not inside the callback: Upsert holds the
|
||||
// shard lock across the callback, and Close runs the close handler synchronously, which calls
|
||||
// RemoveCb and would deadlock trying to re-acquire that same lock.
|
||||
var superseded channel.Channel
|
||||
self.clients.Upsert(clientId, ch, func(exists bool, oldCh channel.Channel, newCh channel.Channel) channel.Channel {
|
||||
if exists && oldCh != nil && oldCh != newCh {
|
||||
superseded = oldCh
|
||||
}
|
||||
return newCh
|
||||
})
|
||||
if superseded != nil {
|
||||
_ = superseded.Close()
|
||||
}
|
||||
|
||||
pfxlog.Logger().WithField("id", clientId).Info("new sim connection established")
|
||||
|
||||
self.resendPendingScenarios(clientId, ch)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// resendPendingScenarios re-sends the run request for every in-flight scenario that still expects a
|
||||
// result from this client. A sim that reconnects mid-scenario gets a fresh channel that never received
|
||||
// the original request; without re-sending, its result would never arrive and the scenario would block
|
||||
// until it timed out.
|
||||
//
|
||||
// It takes dispatchLock so it observes a scenario only once dispatch has settled. Reading the state
|
||||
// mid-dispatch would let a reconnect re-send a scenario that the still-running dispatch is about to
|
||||
// abandon, leaving the sim running a scenario nobody awaits while the caller starts its replacement.
|
||||
func (self *RemoteController) resendPendingScenarios(clientId string, ch channel.Channel) {
|
||||
self.dispatchLock.Lock()
|
||||
defer self.dispatchLock.Unlock()
|
||||
|
||||
for _, results := range self.resultsTracker.Items() {
|
||||
if results.done.Load() || results.completed.Load() {
|
||||
continue
|
||||
}
|
||||
if _, expected := results.expected[clientId]; !expected {
|
||||
continue
|
||||
}
|
||||
if results.results.Has(clientId) {
|
||||
continue
|
||||
}
|
||||
log := pfxlog.Logger().WithField("scenarioId", results.id).WithField("clientId", clientId)
|
||||
if err := self.sendScenarioRequest(results.id, ch); err != nil {
|
||||
log.WithError(err).Error("failed to re-send scenario request to reconnected sim")
|
||||
} else {
|
||||
log.Info("re-sent scenario request to reconnected sim")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (self *RemoteController) BindChannel(binding channel.Binding) error {
|
||||
binding.AddReceiveHandlerF(int32(loop4Pb.ContentType_RunScenarioResultType), self.handleScenarioResult)
|
||||
if self.cb != nil {
|
||||
@@ -125,8 +184,17 @@ func (self *RemoteController) BindChannel(binding channel.Binding) error {
|
||||
}
|
||||
binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) {
|
||||
clientId := string(ch.Headers()[HeaderClientId])
|
||||
pfxlog.Logger().WithField("id", clientId).Info("sim client channel closed, removing from clients map")
|
||||
self.clients.Remove(clientId)
|
||||
// Only remove the entry if it still points at this channel. A reconnecting sim registers
|
||||
// its new channel before the old channel's close handler runs; removing unconditionally
|
||||
// would evict the live replacement and make a healthy sim look disconnected.
|
||||
removed := self.clients.RemoveCb(clientId, func(_ string, v channel.Channel, _ bool) bool {
|
||||
return v == ch
|
||||
})
|
||||
if removed {
|
||||
pfxlog.Logger().WithField("id", clientId).Info("sim client channel closed, removed from clients map")
|
||||
} else {
|
||||
pfxlog.Logger().WithField("id", clientId).Info("stale sim channel closed; superseded by newer connection, not removing")
|
||||
}
|
||||
}))
|
||||
return nil
|
||||
}
|
||||
@@ -138,12 +206,23 @@ func (self *RemoteController) handleScenarioResult(msg *channel.Message, ch chan
|
||||
} else {
|
||||
results, _ := self.resultsTracker.Get(id)
|
||||
if results == nil {
|
||||
pfxlog.Logger().Errorf("scenario result message for scenario id [%s] received, but no results tracker found", id)
|
||||
// Expected for a result that arrives after its scenario was retired, which a client that
|
||||
// reported late or reconnected after the fact will produce.
|
||||
pfxlog.Logger().WithField("scenarioId", id).
|
||||
Info("ignoring scenario result for a scenario no longer being tracked")
|
||||
return
|
||||
}
|
||||
|
||||
clientId := string(ch.Headers()[HeaderClientId])
|
||||
|
||||
// Ignore results from clients that were not part of this scenario's dispatch set (e.g. a sim that
|
||||
// connected after the scenario started). Counting them would corrupt the completion check.
|
||||
if _, expected := results.expected[clientId]; !expected {
|
||||
pfxlog.Logger().WithField("scenarioId", id).WithField("clientId", clientId).
|
||||
Info("ignoring scenario result from client not in scenario's expected set")
|
||||
return
|
||||
}
|
||||
|
||||
success, _ := msg.GetBoolHeader(int32(loop4Pb.HeaderType_ScenarioSuccess))
|
||||
|
||||
pfxlog.Logger().
|
||||
@@ -157,7 +236,7 @@ func (self *RemoteController) handleScenarioResult(msg *channel.Message, ch chan
|
||||
message: string(msg.Body),
|
||||
}
|
||||
results.results.Set(clientId, *result)
|
||||
if results.results.Count() == results.expectedResults {
|
||||
if results.results.Count() == len(results.expected) {
|
||||
if results.completed.CompareAndSwap(false, true) {
|
||||
close(results.complete)
|
||||
}
|
||||
@@ -193,45 +272,88 @@ func (self *RemoteController) MissingComponents(components []*model.Component) [
|
||||
return result
|
||||
}
|
||||
|
||||
// StartSimScenarios dispatches a new scenario run request to every currently connected sim and returns
|
||||
// the tracker to await its results on. Dispatch holds dispatchLock so a sim reconnecting partway
|
||||
// through cannot observe (and re-send) a scenario whose dispatch is still in progress.
|
||||
func (self *RemoteController) StartSimScenarios() (*ScenarioResults, error) {
|
||||
self.dispatchLock.Lock()
|
||||
defer self.dispatchLock.Unlock()
|
||||
|
||||
scenarioId := uuid.NewString()
|
||||
log := pfxlog.Logger().WithField("scenarioId", scenarioId)
|
||||
|
||||
clients := self.clients.Items()
|
||||
expected := make(map[string]struct{}, len(clients))
|
||||
for clientId := range clients {
|
||||
expected[clientId] = struct{}{}
|
||||
}
|
||||
|
||||
results := &ScenarioResults{
|
||||
id: scenarioId,
|
||||
results: cmap.New[ScenarioResult](),
|
||||
complete: make(chan struct{}),
|
||||
expectedResults: self.clients.Count(),
|
||||
controller: self,
|
||||
id: scenarioId,
|
||||
results: cmap.New[ScenarioResult](),
|
||||
complete: make(chan struct{}),
|
||||
expected: expected,
|
||||
}
|
||||
|
||||
self.resultsTracker.Set(scenarioId, results)
|
||||
|
||||
for _, client := range self.clients.Items() {
|
||||
msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioRequestType), nil)
|
||||
msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), scenarioId)
|
||||
if err := msg.WithTimeout(10 * time.Second).SendAndWaitForWire(client); err != nil {
|
||||
return nil, err
|
||||
for clientId, client := range clients {
|
||||
if err := self.sendScenarioRequest(scenarioId, client); err != nil {
|
||||
// The tracker is already published, but the caller gets no handle to retire it, so do it here.
|
||||
// Left live, a sim reconnecting later would have this scenario re-sent to it and run it
|
||||
// alongside whatever scenario the caller starts in place of this failed one.
|
||||
results.retire()
|
||||
return nil, fmt.Errorf("failed to send scenario request to %s: %w", clientId, err)
|
||||
}
|
||||
log.WithField("clientId", client.Id()).Info("scenario run request sent")
|
||||
log.WithField("clientId", clientId).Info("scenario run request sent")
|
||||
}
|
||||
|
||||
return results, nil
|
||||
}
|
||||
|
||||
// sendScenarioRequest sends one run-scenario request for the given scenario to a single sim channel.
|
||||
func (self *RemoteController) sendScenarioRequest(scenarioId string, client channel.Channel) error {
|
||||
msg := channel.NewMessage(int32(loop4Pb.ContentType_RunScenarioRequestType), nil)
|
||||
msg.PutStringHeader(int32(loop4Pb.HeaderType_ScenarioId), scenarioId)
|
||||
return msg.WithTimeout(10 * time.Second).SendAndWaitForWire(client)
|
||||
}
|
||||
|
||||
type ScenarioResult struct {
|
||||
success bool
|
||||
message string
|
||||
}
|
||||
|
||||
type ScenarioResults struct {
|
||||
id string
|
||||
results cmap.ConcurrentMap[string, ScenarioResult]
|
||||
complete chan struct{}
|
||||
completed atomic.Bool
|
||||
expectedResults int
|
||||
// controller owns the tracker this scenario is registered in, so retire can drop it.
|
||||
controller *RemoteController
|
||||
|
||||
id string
|
||||
results cmap.ConcurrentMap[string, ScenarioResult]
|
||||
complete chan struct{}
|
||||
completed atomic.Bool
|
||||
// expected holds the client ids the scenario was dispatched to. It is populated before the results
|
||||
// tracker is published and never mutated afterward, so it is safe for concurrent reads.
|
||||
expected map[string]struct{}
|
||||
// done marks the scenario as no longer awaited. retire clears the tracker entry as well, but a
|
||||
// resend scan already walking a snapshot of the tracker can still see the entry, and this flag is
|
||||
// what stops it re-sending a scenario nobody is waiting on.
|
||||
done atomic.Bool
|
||||
}
|
||||
|
||||
// retire marks the scenario as no longer awaited and drops it from the tracker. Every scenario ends
|
||||
// here, whether its results arrived, its wait timed out, or its dispatch failed part way. Without it
|
||||
// the tracker only ever grows: each entry holds a sharded result map and its expected-client set for
|
||||
// the life of the process, and every sim reconnect re-walks all of them.
|
||||
//
|
||||
// The flag is set before the removal so a resend scan holding an older snapshot still skips it.
|
||||
func (self *ScenarioResults) retire() {
|
||||
self.done.Store(true)
|
||||
self.controller.resultsTracker.Remove(self.id)
|
||||
}
|
||||
|
||||
func (self *ScenarioResults) GetResults(timeout time.Duration) error {
|
||||
defer self.retire()
|
||||
start := time.Now()
|
||||
var err error
|
||||
select {
|
||||
|
||||
@@ -44,9 +44,12 @@ type ControllerType struct {
|
||||
ConfigSource string
|
||||
ConfigName string
|
||||
Version string
|
||||
LocalPath string
|
||||
DNSNames []string
|
||||
Debug bool
|
||||
// SourceRef, when set, is a git ref (branch, tag, or SHA) on openziti/ziti built from source and
|
||||
// stamped as Version, instead of downloading the Version release. Use it to run an unreleased build.
|
||||
SourceRef string
|
||||
LocalPath string
|
||||
DNSNames []string
|
||||
Debug bool
|
||||
LogConfig
|
||||
}
|
||||
|
||||
@@ -115,7 +118,7 @@ func (self *ControllerType) StageFiles(r model.Run, c *model.Component) error {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath); err != nil {
|
||||
if err := stageziti.StageZitiForComponentOnce(r, c, self.Version, self.SourceRef, self.LocalPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -96,9 +96,14 @@ func (self *Loop4SimType) GetConfigName(c *model.Component) string {
|
||||
return configName
|
||||
}
|
||||
|
||||
func (self *Loop4SimType) getProcessFilter() func(string) bool {
|
||||
func (self *Loop4SimType) getProcessFilter(c *model.Component) func(string) bool {
|
||||
// Match on the component's config file (which contains the component id) in addition to the
|
||||
// binary name, so multiple loop4 instances can run on the same host and still be identified
|
||||
// individually. Single-instance hosts continue to match, since the running process always
|
||||
// includes its own config path.
|
||||
configFile := fmt.Sprintf("%s.yml", c.Id)
|
||||
return func(s string) bool {
|
||||
return strings.Contains(s, "ziti-traffic-test")
|
||||
return strings.Contains(s, "ziti-traffic-test") && strings.Contains(s, configFile)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -107,7 +112,7 @@ func (self *Loop4SimType) GetConfigPath(c *model.Component) string {
|
||||
}
|
||||
|
||||
func (self *Loop4SimType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
|
||||
pids, err := c.GetHost().FindProcesses(self.getProcessFilter())
|
||||
pids, err := c.GetHost().FindProcesses(self.getProcessFilter(c))
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
@@ -139,5 +144,5 @@ func (self *Loop4SimType) Start(_ model.Run, c *model.Component) error {
|
||||
}
|
||||
|
||||
func (self *Loop4SimType) Stop(_ model.Run, c *model.Component) error {
|
||||
return c.GetHost().KillProcesses("-TERM", self.getProcessFilter())
|
||||
return c.GetHost().KillProcesses("-TERM", self.getProcessFilter(c))
|
||||
}
|
||||
|
||||
@@ -44,8 +44,11 @@ const (
|
||||
)
|
||||
|
||||
type RouterType struct {
|
||||
Configs map[string]Config
|
||||
Version string
|
||||
Configs map[string]Config
|
||||
Version string
|
||||
// SourceRef, when set, is a git ref (branch, tag, or SHA) on openziti/ziti built from source and
|
||||
// stamped as Version, instead of downloading the Version release. Use it to run an unreleased build.
|
||||
SourceRef string
|
||||
LocalPath string
|
||||
Debug bool
|
||||
LogConfig
|
||||
@@ -105,7 +108,7 @@ func (self *RouterType) StageFiles(r model.Run, c *model.Component) error {
|
||||
}
|
||||
}
|
||||
|
||||
if err := stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath); err != nil {
|
||||
if err := stageziti.StageZitiForComponentOnce(r, c, self.Version, self.SourceRef, self.LocalPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@@ -105,7 +105,11 @@ func (self *ZitiEdgeTunnelType) StageFiles(r model.Run, c *model.Component) erro
|
||||
|
||||
func (self *ZitiEdgeTunnelType) getProcessFilter(c *model.Component) func(string) bool {
|
||||
return func(s string) bool {
|
||||
return strings.Contains(s, self.getBinaryName()) &&
|
||||
// Match on the version-agnostic base binary name, not getBinaryName() (which includes the
|
||||
// version): the component's config file uniquely identifies its process, and matching the
|
||||
// versioned name would fail to find a running instance after a version change, leaving the
|
||||
// old-version process alive on stop/restart.
|
||||
return strings.Contains(s, "ziti-edge-tunnel") &&
|
||||
strings.Contains(s, fmt.Sprintf("%s.json", c.Id)) &&
|
||||
!strings.Contains(s, "sudo ")
|
||||
}
|
||||
|
||||
@@ -52,8 +52,11 @@ func (self ZitiTunnelMode) String() string {
|
||||
}
|
||||
|
||||
type ZitiTunnelType struct {
|
||||
Mode ZitiTunnelMode
|
||||
Version string
|
||||
Mode ZitiTunnelMode
|
||||
Version string
|
||||
// SourceRef, when set, is a git ref (branch, tag, or SHA) on openziti/ziti built from source and
|
||||
// stamped as Version, instead of downloading the Version release. Use it to run an unreleased build.
|
||||
SourceRef string
|
||||
LocalPath string
|
||||
ConfigPathF func(c *model.Component) string
|
||||
Count uint8
|
||||
@@ -61,6 +64,9 @@ type ZitiTunnelType struct {
|
||||
ControlRouterConnections uint8
|
||||
EnableSdkFlowControl bool
|
||||
Verbose bool
|
||||
// ProxyServices are "service:port" pairs passed to 'ziti tunnel proxy' so the tunneler opens a
|
||||
// local TCP listener per service. Only used in proxy mode.
|
||||
ProxyServices []string
|
||||
LogConfig
|
||||
}
|
||||
|
||||
@@ -111,7 +117,7 @@ func (self *ZitiTunnelType) Dump() any {
|
||||
}
|
||||
|
||||
func (self *ZitiTunnelType) StageFiles(r model.Run, c *model.Component) error {
|
||||
if err := stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath); err != nil {
|
||||
if err := stageziti.StageZitiForComponentOnce(r, c, self.Version, self.SourceRef, self.LocalPath); err != nil {
|
||||
return err
|
||||
}
|
||||
// The rotate log strategy runs "ops log-pipe" from the component's ziti binary (Version);
|
||||
@@ -213,8 +219,13 @@ func (self *ZitiTunnelType) StartIndividual(c *model.Component, idx int) error {
|
||||
|
||||
redirect := logRedirect(c, logsPath, binaryPath)
|
||||
|
||||
serviceCmd := fmt.Sprintf("%s %s tunnel %s %s %s --cli-agent-alias %s --log-formatter json -i %s %s &",
|
||||
useSudo, binaryPath, mode.String(), connectCfg, verbose, c.Id, configPath, redirect)
|
||||
proxyServices := ""
|
||||
if mode == ZitiTunnelModeProxy && len(self.ProxyServices) > 0 {
|
||||
proxyServices = strings.Join(self.ProxyServices, " ")
|
||||
}
|
||||
|
||||
serviceCmd := fmt.Sprintf("%s %s tunnel %s %s %s %s --cli-agent-alias %s --log-formatter json -i %s %s &",
|
||||
useSudo, binaryPath, mode.String(), proxyServices, connectCfg, verbose, c.Id, configPath, redirect)
|
||||
|
||||
// Only clear prior history for truncate; append and rotate must preserve it.
|
||||
var cmds []string
|
||||
|
||||
@@ -18,6 +18,7 @@ package zitilib_runlevel_5_operation
|
||||
|
||||
import (
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"strings"
|
||||
@@ -43,6 +44,7 @@ const (
|
||||
func NewSimServices(hostSelectorF func(string) string) *SimServices {
|
||||
return &SimServices{
|
||||
idToSelectorMapper: hostSelectorF,
|
||||
connectTimeout: 60 * time.Second,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -53,6 +55,9 @@ type SimServices struct {
|
||||
lock sync.Mutex
|
||||
zitiContext ziti.Context
|
||||
metricsStarted atomic.Bool
|
||||
// connectTimeout bounds establishing the sim-controller ziti context, so an unreachable
|
||||
// controller fails fast instead of hanging on the SDK's unbounded version-info retry.
|
||||
connectTimeout time.Duration
|
||||
|
||||
remoteController *loop4.RemoteController
|
||||
}
|
||||
@@ -90,12 +95,36 @@ func (self *SimServices) GetZitiContext(run model.Run) (ziti.Context, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Authenticate up front with a deadline so an unreachable controller fails fast. The SDK's
|
||||
// version-info caching retries forever with no timeout (openziti/sdk-golang#976), so the first
|
||||
// Listen/Dial would otherwise block indefinitely and hang the caller (e.g. the steady-state gate).
|
||||
if err = authenticateWithTimeout(context, self.connectTimeout); err != nil {
|
||||
context.Close()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
self.zitiContext = context
|
||||
}
|
||||
|
||||
return self.zitiContext, nil
|
||||
}
|
||||
|
||||
// authenticateWithTimeout runs context.Authenticate() with a deadline so an unreachable controller
|
||||
// surfaces an error instead of blocking indefinitely on the SDK's unbounded version-info retry. If
|
||||
// Authenticate is wedged in that retry, its goroutine can outlive the timeout; that is acceptable
|
||||
// here since the run is failing anyway.
|
||||
func authenticateWithTimeout(context ziti.Context, timeout time.Duration) error {
|
||||
done := make(chan error, 1)
|
||||
go func() { done <- context.Authenticate() }()
|
||||
select {
|
||||
case err := <-done:
|
||||
return err
|
||||
case <-time.After(timeout):
|
||||
return fmt.Errorf("timed out after %s establishing sim controller connection; controller unreachable?", timeout)
|
||||
}
|
||||
}
|
||||
|
||||
func (self *SimServices) CollectSimMetrics(run model.Run, service string) error {
|
||||
if !self.metricsStarted.CompareAndSwap(false, true) {
|
||||
return nil
|
||||
@@ -232,3 +261,26 @@ func (self *SimServices) GetSimController(run model.Run, service string, callbac
|
||||
|
||||
return self.remoteController, nil
|
||||
}
|
||||
|
||||
// Reset tears down the cached sim-controller context, remote controller, and metrics listener so a
|
||||
// subsequent GetZitiContext/GetSimController/CollectSimMetrics rebuilds them. Use it after the
|
||||
// controller has been re-bootstrapped (e.g. resetting a test between iterations), where the cached
|
||||
// api session and sim-controller enrollment are stale and would otherwise be reused.
|
||||
func (self *SimServices) Reset() {
|
||||
self.lock.Lock()
|
||||
defer self.lock.Unlock()
|
||||
|
||||
if self.remoteController != nil {
|
||||
_ = self.remoteController.Close()
|
||||
self.remoteController = nil
|
||||
}
|
||||
if self.listener != nil {
|
||||
_ = self.listener.Close()
|
||||
self.listener = nil
|
||||
}
|
||||
if self.zitiContext != nil {
|
||||
self.zitiContext.Close()
|
||||
self.zitiContext = nil
|
||||
}
|
||||
self.metricsStarted.Store(false)
|
||||
}
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
package stageziti
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
@@ -8,12 +9,20 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/openziti/fablab/kernel/model"
|
||||
"github.com/openziti/sdk-golang/acquire"
|
||||
"github.com/openziti/ziti/v2/common/getziti"
|
||||
"github.com/openziti/ziti/v2/ziti/util"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// hostGoos and hostGoarch are the platform of the hosts fablab provisions, which is what every
|
||||
// binary staged into the kit has to be built for, regardless of what the orchestrator runs on.
|
||||
const (
|
||||
hostGoos = "linux"
|
||||
hostGoarch = "amd64"
|
||||
)
|
||||
|
||||
func StageZitiOnce(run model.Run, component *model.Component, version string, source string) error {
|
||||
op := "install.ziti-"
|
||||
if version == "" {
|
||||
@@ -68,19 +77,81 @@ func StageZitiEdgeTunnelOnce(run model.Run, component *model.Component, version
|
||||
|
||||
func StageZiti(run model.Run, component *model.Component, version string, source string) error {
|
||||
return StageExecutable(run, "ziti", component, version, source, func() error {
|
||||
return getziti.InstallZiti(version, "linux", "amd64", run.GetBinDir(), false)
|
||||
return getziti.InstallZiti(version, hostGoos, hostGoarch, run.GetBinDir(), false)
|
||||
})
|
||||
}
|
||||
|
||||
// StageZitiForComponentOnce stages the ziti binary for a component once per run. When sourceRef is set
|
||||
// it builds that git ref (branch, tag, or SHA) on openziti/ziti and stamps it as version; otherwise it
|
||||
// stages localPath or the released version, exactly as StageZitiOnce does. This lets any component
|
||||
// (controller, router, ziti-tunnel) run an unreleased build while reporting a chosen version.
|
||||
func StageZitiForComponentOnce(run model.Run, component *model.Component, version, sourceRef, localPath string) error {
|
||||
if sourceRef != "" {
|
||||
return StageZitiFromRefOnce(run, component, sourceRef, version, localPath)
|
||||
}
|
||||
return StageZitiOnce(run, component, version, localPath)
|
||||
}
|
||||
|
||||
// StageZitiFromRefOnce is StageZitiFromRef guarded by run.DoOnce, keyed on the ref and stamped version
|
||||
// so distinct (ref, version) builds each run once while a repeated one is reused.
|
||||
func StageZitiFromRefOnce(run model.Run, component *model.Component, sourceRef, stampVersion, localPath string) error {
|
||||
op := fmt.Sprintf("install.ziti-ref-%s-as-%s", sourceRef, stampVersion)
|
||||
return run.DoOnce(op, func() error {
|
||||
return StageZitiFromRef(run, component, sourceRef, stampVersion, localPath)
|
||||
})
|
||||
}
|
||||
|
||||
// StageZitiFromRef builds the ziti binary from sourceRef (a git ref on openziti/ziti), stamps it to
|
||||
// report stampVersion, and stages it under the ziti-<stampVersion> name so an in-place version swap can
|
||||
// pick it up. An empty stampVersion stages it as plain ziti and leaves the build unstamped, which is
|
||||
// how every other staging path names an unversioned binary and what the component start paths resolve.
|
||||
// It always delegates to acquire, which resolves the ref to a commit and caches by that immutable id,
|
||||
// so an unchanged ref reuses the cached build while a changed ref (or a branch moved to a new commit)
|
||||
// rebuilds. It deliberately does not skip on an existing ziti-<stampVersion>: that would key
|
||||
// reuse on the version name and silently serve a stale binary when the ref changed under the same
|
||||
// stampVersion. If localPath is set it is staged directly instead of building, so a prebuilt binary
|
||||
// still wins. GITHUB_TOKEN, when set, raises the ref-resolution rate limit.
|
||||
//
|
||||
// The build targets hostGoos/hostGoarch even when the orchestrator runs on something else, so a run
|
||||
// driven from a mac or an arm box still stages a binary the hosts can execute.
|
||||
func StageZitiFromRef(run model.Run, component *model.Component, sourceRef, stampVersion, localPath string) error {
|
||||
fileName := "ziti"
|
||||
if stampVersion != "" {
|
||||
fileName += "-" + stampVersion
|
||||
}
|
||||
target := filepath.Join(run.GetBinDir(), fileName)
|
||||
|
||||
if localPath != "" {
|
||||
logrus.Infof("[%s] => [%s]", localPath, target)
|
||||
return util.CopyFile(localPath, target)
|
||||
}
|
||||
|
||||
cacheDir, err := acquire.DefaultCacheDir()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg := acquire.Versions{Source: acquire.Source{Org: "openziti", Repo: "ziti"}}
|
||||
src := acquire.NewGitHubReleaseSource(cfg.Source.Org, cfg.Source.Repo, os.Getenv("GITHUB_TOKEN"))
|
||||
|
||||
logrus.Infof("building ziti from ref %s (stamped %s) -> %s", sourceRef, stampVersion, target)
|
||||
built, id, err := acquire.Ziti(context.Background(), sourceRef, cfg, src, cacheDir,
|
||||
acquire.WithVersion(stampVersion), acquire.WithPlatform(hostGoos, hostGoarch))
|
||||
if err != nil {
|
||||
return fmt.Errorf("building ziti from ref %q: %w", sourceRef, err)
|
||||
}
|
||||
logrus.Infof("built ziti from ref %s (commit %s) stamped %s", sourceRef, id.Tag, stampVersion)
|
||||
return util.CopyFile(built, target)
|
||||
}
|
||||
|
||||
func StageZrok(run model.Run, component *model.Component, version string, source string) error {
|
||||
return StageExecutable(run, "zrok", component, version, source, func() error {
|
||||
return getziti.InstallZrok(version, "linux", "amd64", run.GetBinDir(), false)
|
||||
return getziti.InstallZrok(version, hostGoos, hostGoarch, run.GetBinDir(), false)
|
||||
})
|
||||
}
|
||||
|
||||
func StageCaddy(run model.Run, component *model.Component, version string, source string) error {
|
||||
return StageExecutable(run, "caddy", component, version, source, func() error {
|
||||
return getziti.InstallCaddy(version, "linux", "amd64", run.GetBinDir(), false)
|
||||
return getziti.InstallCaddy(version, hostGoos, hostGoarch, run.GetBinDir(), false)
|
||||
})
|
||||
}
|
||||
|
||||
@@ -175,5 +246,5 @@ func StageZitiEdgeTunnel(run model.Run, component *model.Component, version stri
|
||||
}
|
||||
logrus.Infof("%s not present, attempting to fetch", target)
|
||||
|
||||
return getziti.InstallZitiEdgeTunnel(version, "linux", "amd64", run.GetBinDir(), false)
|
||||
return getziti.InstallZitiEdgeTunnel(version, hostGoos, hostGoarch, run.GetBinDir(), false)
|
||||
}
|
||||
|
||||
@@ -59,6 +59,14 @@ func ValidateTerminators(run model.Run, timeout time.Duration, countOk func(int6
|
||||
}
|
||||
|
||||
func ValidateTerminatorsForCtrl(run model.Run, c *model.Component, deadline time.Time, countOk func(int64) bool, validationType TerminatorValidationType) error {
|
||||
return ValidateTerminatorsForCtrlWithFilters(run, c, deadline, countOk, validationType, "limit none", "limit none")
|
||||
}
|
||||
|
||||
// ValidateTerminatorsForCtrlWithFilters is like ValidateTerminatorsForCtrl but restricts the SDK and
|
||||
// ERT terminator validations to the routers matching sdkFilter and ertFilter respectively. Use it to
|
||||
// skip routers that do not host the terminator type being validated, for example inspecting the ERT
|
||||
// terminators only on edge-router tunnelers.
|
||||
func ValidateTerminatorsForCtrlWithFilters(run model.Run, c *model.Component, deadline time.Time, countOk func(int64) bool, validationType TerminatorValidationType, sdkFilter, ertFilter string) error {
|
||||
logger := tui.ValidationLogger().WithField("ctrl", c.Id)
|
||||
|
||||
var clients *zitirest.Clients
|
||||
@@ -101,10 +109,14 @@ func ValidateTerminatorsForCtrl(run model.Run, c *model.Component, deadline time
|
||||
|
||||
var validators []validatorEntry
|
||||
if validationType&ValidateSdkTerminators != 0 {
|
||||
validators = append(validators, validatorEntry{name: "sdk", validate: ValidateRouterSdkTerminators})
|
||||
validators = append(validators, validatorEntry{name: "sdk", validate: func(id string, cl *zitirest.Clients) (int, error) {
|
||||
return ValidateRouterSdkTerminatorsWithFilter(id, cl, sdkFilter)
|
||||
}})
|
||||
}
|
||||
if validationType&ValidateErtTerminators != 0 {
|
||||
validators = append(validators, validatorEntry{name: "ert", validate: ValidateRouterErtTerminators})
|
||||
validators = append(validators, validatorEntry{name: "ert", validate: func(id string, cl *zitirest.Clients) (int, error) {
|
||||
return ValidateRouterErtTerminatorsWithFilter(id, cl, ertFilter)
|
||||
}})
|
||||
}
|
||||
|
||||
for _, v := range validators {
|
||||
@@ -158,7 +170,85 @@ func GetTerminatorCount(clients *zitirest.Clients) (int64, error) {
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// FixInvalidTerminators asks the controller to validate the terminators matching filter against their
|
||||
// routers and delete the ones the routers no longer host, returning the number fixed (deleted). Scope
|
||||
// the filter (e.g. binding="edge") to the terminator types whose routers can be inspected safely.
|
||||
// timeout bounds how long to wait for the per-terminator results after validation starts.
|
||||
func FixInvalidTerminators(clients *zitirest.Clients, filter string, timeout time.Duration) (int, error) {
|
||||
logger := tui.ValidationLogger()
|
||||
|
||||
closeNotify := make(chan struct{})
|
||||
eventNotify := make(chan *mgmt_pb.TerminatorDetail, 16)
|
||||
|
||||
bindHandler := func(binding channel.Binding) error {
|
||||
binding.AddReceiveHandlerF(int32(mgmt_pb.ContentType_ValidateTerminatorResultType), func(msg *channel.Message, _ channel.Channel) {
|
||||
detail := &mgmt_pb.TerminatorDetail{}
|
||||
if err := proto.Unmarshal(msg.Body, detail); err != nil {
|
||||
pfxlog.Logger().WithError(err).Error("unable to unmarshal terminator detail")
|
||||
return
|
||||
}
|
||||
eventNotify <- detail
|
||||
})
|
||||
binding.AddCloseHandler(channel.CloseHandlerF(func(ch channel.Channel) {
|
||||
close(closeNotify)
|
||||
}))
|
||||
return nil
|
||||
}
|
||||
|
||||
ch, err := clients.NewWsMgmtChannel(channel.BindHandlerF(bindHandler))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer func() {
|
||||
_ = ch.Close()
|
||||
}()
|
||||
|
||||
request := &mgmt_pb.ValidateTerminatorsRequest{
|
||||
TerminatorsFilter: filter,
|
||||
FixInvalid: true,
|
||||
}
|
||||
responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(30 * time.Second).SendForReply(ch)
|
||||
|
||||
response := &mgmt_pb.ValidateTerminatorsResponse{}
|
||||
if err = protobufs.TypedResponse(response).Unmarshall(responseMsg, err); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if !response.Success {
|
||||
return 0, fmt.Errorf("failed to start terminator validation: %s", response.Message)
|
||||
}
|
||||
|
||||
// A router that stops answering mid-validation lets the controller-side request time out without
|
||||
// emitting a detail or closing this channel, while the response count still includes its
|
||||
// terminators, so bound the wait instead of blocking forever on details that will never arrive.
|
||||
expiredC := time.After(timeout)
|
||||
|
||||
fixed := 0
|
||||
for expected := response.TerminatorCount; expected > 0; expected-- {
|
||||
select {
|
||||
case <-closeNotify:
|
||||
return fixed, errors.New("unexpected close of mgmt channel during terminator fix")
|
||||
case <-expiredC:
|
||||
return fixed, fmt.Errorf("timed out after %s waiting for terminator details, %d of %d not received",
|
||||
timeout, expected, response.TerminatorCount)
|
||||
case detail := <-eventNotify:
|
||||
if detail.Fixed {
|
||||
fixed++
|
||||
logger.Infof("fixed invalid terminator %s (service=%s router=%s state=%s)",
|
||||
detail.TerminatorId, detail.ServiceName, detail.RouterName, detail.State)
|
||||
}
|
||||
}
|
||||
}
|
||||
return fixed, nil
|
||||
}
|
||||
|
||||
func ValidateRouterSdkTerminators(id string, clients *zitirest.Clients) (int, error) {
|
||||
return ValidateRouterSdkTerminatorsWithFilter(id, clients, "limit none")
|
||||
}
|
||||
|
||||
// ValidateRouterSdkTerminatorsWithFilter validates the SDK terminators on the routers matching filter,
|
||||
// returning the number found invalid.
|
||||
func ValidateRouterSdkTerminatorsWithFilter(id string, clients *zitirest.Clients, filter string) (int, error) {
|
||||
logger := tui.ValidationLogger().WithField("ctrl", id)
|
||||
|
||||
closeNotify := make(chan struct{})
|
||||
@@ -191,7 +281,7 @@ func ValidateRouterSdkTerminators(id string, clients *zitirest.Clients) (int, er
|
||||
}()
|
||||
|
||||
request := &mgmt_pb.ValidateRouterSdkTerminatorsRequest{
|
||||
Filter: "limit none",
|
||||
Filter: filter,
|
||||
}
|
||||
responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(10 * time.Second).SendForReply(ch)
|
||||
|
||||
@@ -235,6 +325,13 @@ func ValidateRouterSdkTerminators(id string, clients *zitirest.Clients) (int, er
|
||||
}
|
||||
|
||||
func ValidateRouterErtTerminators(id string, clients *zitirest.Clients) (int, error) {
|
||||
return ValidateRouterErtTerminatorsWithFilter(id, clients, "limit none")
|
||||
}
|
||||
|
||||
// ValidateRouterErtTerminatorsWithFilter validates the ERT terminators on the routers matching filter,
|
||||
// returning the number found invalid. Restricting the filter to edge-router tunnelers avoids inspecting
|
||||
// routers that host no ERT terminators.
|
||||
func ValidateRouterErtTerminatorsWithFilter(id string, clients *zitirest.Clients, filter string) (int, error) {
|
||||
logger := tui.ValidationLogger().WithField("ctrl", id)
|
||||
|
||||
closeNotify := make(chan struct{})
|
||||
@@ -267,7 +364,7 @@ func ValidateRouterErtTerminators(id string, clients *zitirest.Clients) (int, er
|
||||
}()
|
||||
|
||||
request := &mgmt_pb.ValidateRouterErtTerminatorsRequest{
|
||||
Filter: "limit none",
|
||||
Filter: filter,
|
||||
}
|
||||
responseMsg, err := protobufs.MarshalTyped(request).WithTimeout(10 * time.Second).SendForReply(ch)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user