From 18b5ae2f5220f4c8da7b997ba313514282f5bff8 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Mon, 26 Jun 2023 10:42:02 -0400 Subject: [PATCH 01/14] document cross-build procedure --- .github/workflows/main.yml | 14 ++++++++++++++ BUILD.md | 2 +- docker-images/cross-build/README.md | 18 +++++++++--------- 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4b5098230..75d56b336 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -381,3 +381,17 @@ jobs: secrets: inherit with: ziti-version: ${{ needs.publish.outputs.ZITI_VERSION }} + + call-publish-linux-install-packages: + # always() re-enables evaluating conditionals in forks even if Windows or + # macOS builds were skipped + if: | + always() + && needs.publish.result == 'success' + && github.ref == 'refs/heads/main' + name: Publish Release Linux Install Packages + needs: publish + uses: ./.github/workflows/publish-linux-install-packages.yml + secrets: inherit + with: + ziti-version: ${{ needs.publish.outputs.ZITI_VERSION }} diff --git a/BUILD.md b/BUILD.md index caa3d8ee8..d1c355bcc 100644 --- a/BUILD.md +++ b/BUILD.md @@ -5,4 +5,4 @@ Please refer to [the local development README](./doc/002-local-dev.md) for build ## Crossbuilds -When you push to your repo fork then GitHub Actions will automatically crossbuild for several OSs and CPU architectures. You'll then be able to download the built artifacts from the GitHub UI. The easiest way to crossbuild the Linux exectuables locally is to build and run the crossbuild container. Please refer to [the crossbuild container README](../Dockerfile.linux-build.README) for those steps. For hints on crossbuilding for MacOS and Windows see [the main GitHub Actions workflow](../.github/workflows/main.yml) which defines the steps that are run when you push to GitHub. +When you push to your repo fork then GitHub Actions will automatically crossbuild for several OSs and CPU architectures. You'll then be able to download the built artifacts from the GitHub UI. The easiest way to crossbuild the Linux exectuables locally is to build and run the crossbuild container. Please refer to [the crossbuild container README](./docker-images/cross-build/README.md) for those steps. For hints on crossbuilding for MacOS and Windows see [the main GitHub Actions workflow](../.github/workflows/main.yml) which defines the steps that are run when you push to GitHub. diff --git a/docker-images/cross-build/README.md b/docker-images/cross-build/README.md index 8cb86dacf..4f789cd77 100644 --- a/docker-images/cross-build/README.md +++ b/docker-images/cross-build/README.md @@ -12,9 +12,9 @@ This article supports local development by providing a local containerized metho You only need to build the container image once unless you change the Dockerfile or `./linux-build.sh` (the container's entrypoint). ```bash -# build a container image named "zitibuilder" with the same version of Go that's declared in go.mod +# build a container image named "ziti-go-builder" with the same version of Go that's declared in go.mod docker buildx build \ - --tag=zitibuilder \ + --tag=ziti-go-builder \ --build-arg uid=$UID \ --build-arg gid=$GID \ --build-arg golang_version=$(grep -Po '^go\s+\K\d+\.\d+(\.\d+)?$' go.mod) \ @@ -34,16 +34,16 @@ Executing the following `docker run` command will: # build for all three architectures: amd64 arm arm64 docker run \ --rm \ - --name=zitibuilder \ + --name=ziti-go-builder \ --volume=$PWD:/mnt \ - zitibuilder + ziti-go-builder # build only amd64 docker run \ --rm \ - --name=zitibuilder \ + --name=ziti-go-builder \ --volume=$PWD:/mnt \ - zitibuilder \ + ziti-go-builder \ amd64 ``` @@ -53,7 +53,7 @@ You will find the built artifacts in `./release`. ```bash ❯ docker buildx build \ - --tag=zitibuilder \ + --tag=ziti-go-builder \ --build-arg uid=$UID \ --build-arg gid=$GID \ --build-arg golang_version="$(/bin/grep -Po '^go\s+\K\d+\.\d+(\.\d+)?$' go.mod)" \ @@ -89,9 +89,9 @@ You will find the built artifacts in `./release`. ❯ docker run \ --rm \ - --name=zitibuilder \ + --name=ziti-go-builder \ --volume=$PWD:/mnt \ - zitibuilder + ziti-go-builder Number of parallel builds: 4 --> linux/arm: github.com/openziti/ziti/ziti From 244dae4f8661c77dbd0ae18a38a0536498f97e20 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Thu, 10 Aug 2023 18:03:02 -0400 Subject: [PATCH 02/14] build and publish linux package ziti-cli --- .github/workflows/main.yml | 10 +- .github/workflows/publish-linux-packages.yml | 126 +++++++++++++++++++ build/dist-packages/README.md | 6 + build/dist-packages/linux/nfpm-ziti-cli.yaml | 30 +++++ 4 files changed, 168 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/publish-linux-packages.yml create mode 100644 build/dist-packages/README.md create mode 100644 build/dist-packages/linux/nfpm-ziti-cli.yaml diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 75d56b336..66fe69274 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -382,16 +382,18 @@ jobs: with: ziti-version: ${{ needs.publish.outputs.ZITI_VERSION }} - call-publish-linux-install-packages: + # call on release-next and main branches to publish linux packages to + # "testing" and "release" package repos in Artifactory + call-publish-linux-packages: # always() re-enables evaluating conditionals in forks even if Windows or # macOS builds were skipped if: | always() && needs.publish.result == 'success' - && github.ref == 'refs/heads/main' - name: Publish Release Linux Install Packages + && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release-next') + name: Publish Release Linux Packages needs: publish - uses: ./.github/workflows/publish-linux-install-packages.yml + uses: ./.github/workflows/publish-linux-packages.yml secrets: inherit with: ziti-version: ${{ needs.publish.outputs.ZITI_VERSION }} diff --git a/.github/workflows/publish-linux-packages.yml b/.github/workflows/publish-linux-packages.yml new file mode 100644 index 000000000..c68f6f285 --- /dev/null +++ b/.github/workflows/publish-linux-packages.yml @@ -0,0 +1,126 @@ +name: Publish Linux Packages + +on: + workflow_call: + inputs: + ziti-version: + description: generated by 'ziti-ci get-current-version' + type: string + required: true + +# cancel older, redundant runs of same workflow on same branch +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + publish-linux-packages: + runs-on: ubuntu-latest + strategy: + matrix: + package_name: + - ziti-cli + arch: + - goreleaser: amd64 + gox: amd64 + deb: amd64 + rpm: x86_64 + - goreleaser: arm64 + gox: arm64 + deb: arm64 + rpm: aarch64 + - goreleaser: armv7 + gox: arm + deb: armv7 + rpm: armv7 + nfpm_packager: + - rpm + - deb + # - archlinux # (pacman) + # - apk + env: + ZITI_VERSION: ${{ inputs.ziti-version || github.event.inputs.ziti-version }} + ZITI_MAINTAINER: "OpenZiti Maintainers " + ZITI_HOMEPAGE: "https://openziti.io" + ZITI_VENDOR: "NetFoundry" + GOARCH: ${{ matrix.arch.goreleaser }} + steps: + - name: Checkout Workspace + uses: actions/checkout@v3 + + - name: Download Linux Release Artifacts + uses: actions/download-artifact@v3 + with: + name: linux-release-${{ github.run_id }} + path: release/ + + - name: Move Release Artifact for Architecture to Predictable Location for nfpm + run: | + mv -v ./release/${{ matrix.arch.gox }}/linux/ziti \ + ./release/ziti + + - name: Build Package + id: nfpm + uses: burningalchemist/action-gh-nfpm@v1 + # uses: netfoundry/action-gh-nfpm@main + with: + nfpm_version: "2.32.0" + packager: ${{ matrix.nfpm_packager }} + config: build/dist-packages/linux/nfpm-${{ matrix.package_name }}.yaml + target: release/ + + - name: get the package name from the output + run: echo ${{ steps.nfpm.outputs.package }} + shell: bash + + - run: ls -lh release/ + + - name: upload package artifact to build summary page + uses: actions/upload-artifact@v3 + with: + name: ${{ matrix.package_name }}-${{ matrix.arch.goreleaser }}-${{ matrix.nfpm_packager }} + path: ./release/${{ matrix.package_name }}*.${{ matrix.nfpm_packager }} + if-no-files-found: error + + - name: Configure jFrog CLI + uses: jfrog/setup-jfrog-cli@v3 + env: + JF_ENV_1: ${{ secrets.ZITI_ARTIFACTORY_CLI_CONFIG_PACKAGE_UPLOAD }} + + - name: Upload RPM to Artifactory testing repo + if: ${{ !github.event.release.published && matrix.nfpm_packager == 'rpm' }} + run: > + jf rt upload + ./release/${{ matrix.package_name }}*.${{ matrix.nfpm_packager }} + /zitipax-openziti-rpm-stable/testing/${{ matrix.arch.rpm }}/ + --recursive=false + --flat=true + + - name: Upload RPM to Artifactory release repo + if: ${{ github.event.release.published && matrix.nfpm_packager == 'rpm' }} + run: > + jf rt upload + ./release/${{ matrix.package_name }}*.${{ matrix.nfpm_packager }} + /zitipax-openziti-rpm-stable/release/${{ matrix.arch.rpm }}/ + --recursive=false + --flat=true + + - name: Upload DEB to Artifactory testing repo + if: ${{ !github.event.release.published && matrix.nfpm_packager == 'deb' }} + run: > + jf rt upload + ./release/${{ matrix.package_name }}*.${{ matrix.nfpm_packager }} + /zitipax-openziti-deb-stable/pool/${{ matrix.package_name }}/testing/${{ matrix.arch.deb }}/ + --deb=testing/main/${{ matrix.arch.deb }} + --recursive=false + --flat=true + + - name: Upload DEB to Artifactory release repo + if: ${{ github.event.release.published && matrix.nfpm_packager == 'deb' }} + run: > + jf rt upload + ./release/${{ matrix.package_name }}*.${{ matrix.nfpm_packager }} + /zitipax-openziti-deb-stable/pool/${{ matrix.package_name }}/release/${{ matrix.arch.deb }}/ + --deb=release/main/${{ matrix.arch.deb }} + --recursive=false + --flat=true diff --git a/build/dist-packages/README.md b/build/dist-packages/README.md new file mode 100644 index 000000000..d5ed583e4 --- /dev/null +++ b/build/dist-packages/README.md @@ -0,0 +1,6 @@ + +# nfpm-configs + +These are `nfpm` configuration files. `nfpm` generates Linux packages, e.g., +RPM. These files references environment variables set in GitHub Actions workflow +`.github/workflows/publish-linux-install-packages.yml`. diff --git a/build/dist-packages/linux/nfpm-ziti-cli.yaml b/build/dist-packages/linux/nfpm-ziti-cli.yaml new file mode 100644 index 000000000..c47117e1f --- /dev/null +++ b/build/dist-packages/linux/nfpm-ziti-cli.yaml @@ -0,0 +1,30 @@ +# nfpm configuration file +# +# check https://nfpm.goreleaser.com/configuration for detailed usage +# +name: ziti-cli +arch: ${GOARCH} +platform: linux +version: ${ZITI_VERSION} +maintainer: ${ZITI_MAINTAINER} +description: > + The ziti-cli package provides the ziti executable binary as a command line + interface for Ziti. +vendor: ${ZITI_VENDOR} +homepage: ${ZITI_HOMEPAGE} +license: Apache-2.0 +contents: + - src: ./release/ziti + dst: /opt/openziti/bin/ziti + file_info: + mode: 0755 + - src: /opt/openziti/bin/ziti + dst: /usr/bin/ziti + type: symlink + +# packager-neutral scripts may be overriden by packager-specific scripts +# scripts: + # preinstall: ./scripts/preinstall.sh + # postinstall: ./scripts/postinstall.sh + # preremove: ./scripts/preremove.sh + # postremove: ./scripts/postremove.sh From 77f2c860dbe8c4220dde663a3a27b6f54a77689b Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 23 Aug 2023 16:12:37 -0400 Subject: [PATCH 03/14] use openziti/hello-world container image --- quickstart/docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/quickstart/docker/docker-compose.yml b/quickstart/docker/docker-compose.yml index 200fc3457..b74cd71d1 100644 --- a/quickstart/docker/docker-compose.yml +++ b/quickstart/docker/docker-compose.yml @@ -173,7 +173,7 @@ services: command: "/var/openziti/scripts/run-router.sh private" web-test-blue: - image: crccheck/hello-world + image: openziti/hello-world ports: - 80:8000 networks: From f95c44b47a97302b97f80ec9915195bb2a36d737 Mon Sep 17 00:00:00 2001 From: Ken Bingham Date: Thu, 31 Aug 2023 13:31:20 -0400 Subject: [PATCH 04/14] Update .github/workflows/main.yml --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 66fe69274..7e8a8883c 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -391,7 +391,7 @@ jobs: always() && needs.publish.result == 'success' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/release-next') - name: Publish Release Linux Packages + name: Publish Linux Packages needs: publish uses: ./.github/workflows/publish-linux-packages.yml secrets: inherit From 442eebd0ab22b8a96dc25770b28589006ddaa36f Mon Sep 17 00:00:00 2001 From: Ken Bingham Date: Thu, 31 Aug 2023 13:31:49 -0400 Subject: [PATCH 05/14] Update build/dist-packages/README.md --- build/dist-packages/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/dist-packages/README.md b/build/dist-packages/README.md index d5ed583e4..e450aa624 100644 --- a/build/dist-packages/README.md +++ b/build/dist-packages/README.md @@ -2,5 +2,5 @@ # nfpm-configs These are `nfpm` configuration files. `nfpm` generates Linux packages, e.g., -RPM. These files references environment variables set in GitHub Actions workflow +RPM. These files reference environment variables set in GitHub Actions workflow `.github/workflows/publish-linux-install-packages.yml`. From b1eca77869d43a2dfc98638330a278671be45433 Mon Sep 17 00:00:00 2001 From: Ken Bingham Date: Thu, 31 Aug 2023 13:33:57 -0400 Subject: [PATCH 06/14] stop using Actions concurrency on this callable workflow --- .github/workflows/publish-linux-packages.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/publish-linux-packages.yml b/.github/workflows/publish-linux-packages.yml index c68f6f285..23f660bdd 100644 --- a/.github/workflows/publish-linux-packages.yml +++ b/.github/workflows/publish-linux-packages.yml @@ -8,11 +8,6 @@ on: type: string required: true -# cancel older, redundant runs of same workflow on same branch -concurrency: - group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} - cancel-in-progress: true - jobs: publish-linux-packages: runs-on: ubuntu-latest From 2910014ee9aaa3754a2f5eb8a401867c80bc424c Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Fri, 25 Aug 2023 18:44:24 -0400 Subject: [PATCH 07/14] test the current branch by building ziti and the quickstart container image and running in compose and running quickstart_test.go --- .github/workflows/test-quickstart.yml | 31 ++++++ quickstart/test/compose-test.zsh | 154 ++++++++++++++++++++++++++ 2 files changed, 185 insertions(+) create mode 100644 .github/workflows/test-quickstart.yml create mode 100755 quickstart/test/compose-test.zsh diff --git a/.github/workflows/test-quickstart.yml b/.github/workflows/test-quickstart.yml new file mode 100644 index 000000000..caff5fddc --- /dev/null +++ b/.github/workflows/test-quickstart.yml @@ -0,0 +1,31 @@ +name: Test Quickstart +on: + workflow_dispatch: + # test quickstart changes after merge + push: + branches: + - release-next + - main + paths: + - 'quickstart/**' + # test quickstart changes before merge + pull_request: + paths: + - 'quickstart/**' + +# cancel older, redundant runs of same workflow on same branch +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref_name }} + cancel-in-progress: true + +jobs: + compose-test: + name: Test Compose Quickstart + runs-on: ubuntu-latest + steps: + - name: Shallow checkout + uses: actions/checkout@v3 + - name: Install zsh + run: sudo apt-get update && sudo apt-get install --yes zsh + - name: Build and run a quickstart container image + run: ./quickstart/test/compose-test.zsh diff --git a/quickstart/test/compose-test.zsh b/quickstart/test/compose-test.zsh new file mode 100755 index 000000000..4fb55ff0d --- /dev/null +++ b/quickstart/test/compose-test.zsh @@ -0,0 +1,154 @@ +#!/usr/bin/env zsh +# +# this script tests the quickstart's ziti-cli-functions.sh, container image creation process, and Compose project by +# gathering files from a particular GitHub repo ref or a filesystem path and running the quickstart's Go test suite +# against the running Compose project +# + +set -euo pipefail + +function down_project() { + # don't destroy volumes or temp dir so we can inspect when running locally + docker compose kill + # rm -rf "${TESTDIR}" + echo "INFO: Stopped Compose project: ${TESTDIR}" +} + +DATESTAMP=$(date +%Y%m%d%H%M%S) +# generate a random password for the controller's admin user to ensure we're testing the expected instance +ZITI_PWD="$(set +o pipefail; LC_ALL=C tr -dc -- -A-Z-a-z-0-9 < /dev/urandom 2>/dev/null | head -c5)" +BASENAME="$(basename "$0")" +DIRNAME="$(dirname "$0")" +if [[ -n "${ZITI_QUICK_DIR:-}" ]]; then + if [[ -d "${ZITI_QUICK_DIR}" ]]; then + ZITI_QUICK_DIR="$(realpath "${ZITI_QUICK_DIR}")" + else + if [[ -d "${DIRNAME}/${ZITI_QUICK_DIR}" ]]; then + ZITI_QUICK_DIR="$(realpath "${DIRNAME}/${ZITI_QUICK_DIR}")" + else + echo "ERROR: ZITI_QUICK_DIR is set but is not a directory: ${ZITI_QUICK_DIR}" >&2 + exit 1 + fi + fi +fi +# avoid re-using directories from previous runs to keep this one-shot (non-idempotent) script simple because we needn't +# consider the state of the test dir +TESTDIR="$(mktemp -d -t "${BASENAME%.*}.${DATESTAMP}.XXX")" + +# if unset, set ZITI_QUICK_DIR to this script's parent dir which is always the quickstart root in the git repo +if [[ -z "${ZITI_QUICK_DIR:-}" ]]; then + ZITI_QUICK_DIR="$(realpath "${DIRNAME}/..")" +fi +# if unset, set ZITI_QUICK_IMAGE_TAG to this run's dirname +if [[ -z "${ZITI_QUICK_IMAGE_TAG:-}" ]]; then + ZITI_QUICK_IMAGE_TAG=$(basename "${TESTDIR}") +fi + +# case "${1:-}" in +# shift +# ;; +# --help|-h) +# echo "Usage: $BASENAME [--local|--help]" +# exit 0 +# ;; +# esac + +cd "${TESTDIR}" +echo "INFO: Testing Compose project $PWD" + +declare -a QUICK_FILES=( + ../go.{mod,sum} + test/{quickstart_test.go,compose.override.yml} + docker/{simplified-docker-compose.yml,.env} +) + # TODO: re-add cert checks files after https://github.com/openziti/ziti/pull/1278 + # test/{quickstart_test.go,compose.override.yml,check-cert-chains.zsh} +# download the quickstart Go test suite files from GitHub unless a local dir is specified +if [[ -n "${ZITI_QUICK_DIR:-}" ]]; then + for FILE in "${QUICK_FILES[@]}"; do + cp "${ZITI_QUICK_DIR}/${FILE}" . + done + if [[ -n "${ZITI_QUICK_IMAGE_TAG:-}" ]]; then + if [[ -x "${ZITI_QUICK_DIR:-}/docker/createLocalImage.sh" ]]; then + ( + cd "${ZITI_QUICK_DIR}/docker" + unset ZITI_VERSION ZITI_OVERRIDE_VERSION # always build the local source + ./createLocalImage.sh --build "${ZITI_QUICK_IMAGE_TAG}" + ) + else + echo "ERROR: ZITI_QUICK_IMAGE_TAG is set but ZITI_QUICK_DIR/docker/createLocalImage.sh is not executable" >&2 + exit 1 + fi + fi +elif [[ -n "${ZITI_QUICK_IMAGE_TAG:-}" ]]; then + echo "ERROR: ZITI_QUICK_IMAGE_TAG is set but ZITI_QUICK_DIR is not set" >&2 + exit 1 +else + echo "ERROR: ZITI_QUICK_IMAGE_TAG is not set, try running with --local" >&2 + exit 1 +fi + +# rename the simplified Compose file to the default Compose project file name +mv ./simplified-docker-compose.yml ./compose.yml + +# learn the expected Go version from the Go mod file so we can pull the correct container image +ZITI_GO_VERSION="$(grep -Po '^go\s+\K\d+\.\d+(\.\d+)?$' ./go.mod)" +# make this var available in the Compose project +sed -Ei "s/^(#\s+)?(ZITI_GO_VERSION)=.*/\2=${ZITI_GO_VERSION}/" ./.env +sed -Ei "s/^(#\s+)?(ZITI_PWD)=.*/\2=${ZITI_PWD}/" ./.env +sed -Ei "s/^(#\s+)?(ZITI_INTERFACE)=.*/\2=${ZITI_INTERFACE:-127.0.0.1}/" ./.env + +# pull images preemptively that we never build locally because pull=never when using a local quickstart image +for IMAGE in \ + "golang:${ZITI_GO_VERSION}-alpine" \ + "openziti/zac:latest" +do + docker pull --quiet "${IMAGE}" &>/dev/null +done + +# any halt after this point should cause the Compose project to be torn down +trap down_project SIGTERM SIGINT EXIT + +# if ZITI_QUICK_IMAGE_TAG is set then run the locally-built image +if [[ -n "${ZITI_QUICK_IMAGE_TAG:-}" ]]; then + sed -Ei "s/^(#\s+)?(ZITI_VERSION)=.*/\2=${ZITI_QUICK_IMAGE_TAG}/" ./.env + docker compose up --detach --pull=never &>/dev/null # no pull because local quickstart image +else + echo "ERROR: ZITI_QUICK_IMAGE_TAG is not set" >&2 + exit 1 +fi + +# copy files that are not present in older quickstart container images to the persistent volume; this allows us to run +# the test suite against them and investigate if the test fails and the container is destroyed +for FILE in \ + "" + # check-cert-chains.zsh + # TODO: re-add cert checks to cp list after https://github.com/openziti/ziti/pull/1278 +do + docker compose cp \ + "./${FILE}" \ + "ziti-controller:/persistent/${FILE}" &>/dev/null +done +# TODO: build these executables into the container image? + +# wait for the controller and router to be ready and run the certificate check script; NOUNSET option is enabled after +# sourcing quickstart functions and env because there are some unset variables in those +docker compose exec ziti-controller \ + bash -eo pipefail -c ' + source "${ZITI_SCRIPTS}/ziti-cli-functions.sh" >/dev/null; + echo "INFO: waiting for controller"; + sleep 3; + source /persistent/ziti.env >/dev/null; + set -u; + _wait_for_controller >/dev/null; + echo "INFO: waiting for public router"; + sleep 3; + source /persistent/ziti.env >/dev/null; + _wait_for_public_router >/dev/null; + ' + # TODO: re-add cert checks to above test suite after https://github.com/openziti/ziti/pull/1278 + # zsh /persistent/check-cert-chains.zsh; +docker compose run quickstart-test + +echo -e "\nINFO: Test completed successfully." + From 9078bf9aec29fbb3cf8e040e228fc3ea56a3baee Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 5 Sep 2023 13:06:48 -0400 Subject: [PATCH 08/14] stop enforcing concurrency for reusable workflows --- .github/workflows/publish-docker-images.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index bc7746f57..fc9fce4c2 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -8,11 +8,6 @@ on: type: string required: true -# cancel older, redundant runs of same workflow on same branch -concurrency: - group: ${{ github.workflow }}-${{github.event_name}}-${{ github.head_ref || github.ref_name }} - cancel-in-progress: true - jobs: publish-docker-images: runs-on: ubuntu-latest From 5a0688d1abe3bc98d525c33429f1056db571fe15 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Fri, 1 Sep 2023 12:45:50 -0400 Subject: [PATCH 09/14] don't sleep if smoketest cancelled --- .github/workflows/main.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ac4fbe551..71134638b 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -206,13 +206,15 @@ jobs: runs-on: ubuntu-20.04 needs: [ fablab-smoketest ] steps: + # allow time for investigation unless the workflow is cancelled or the smoketest succeeded or was skipped - name: Sleep If Failed - if: needs.fablab-smoketest.result != 'success' && needs.fablab-smoketest.result != 'skipped' + if: ${{ needs.fablab-smoketest.result != 'success' && needs.fablab-smoketest.result != 'skipped' && needs.fablab-smoketest.result != 'cancelled' }} run: | sleep 30m + # release cloud resources if the smoketest succeeded, failed, or was cancelled; unnecessary if skipped - name: Teardown Test Environment - if: always() && needs.fablab-smoketest.result != 'skipped' + if: needs.fablab-smoketest.result != 'skipped' env: FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }} run: | From 037f36a79c5c8c1d4625e3390891adf043522062 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 5 Sep 2023 14:22:49 -0400 Subject: [PATCH 10/14] tolerate non-existent fablab simple transfer bundle --- .github/workflows/main.yml | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 71134638b..7e66fb118 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -208,7 +208,7 @@ jobs: steps: # allow time for investigation unless the workflow is cancelled or the smoketest succeeded or was skipped - name: Sleep If Failed - if: ${{ needs.fablab-smoketest.result != 'success' && needs.fablab-smoketest.result != 'skipped' && needs.fablab-smoketest.result != 'cancelled' }} + if: needs.fablab-smoketest.result != 'success' && needs.fablab-smoketest.result != 'skipped' && needs.fablab-smoketest.result != 'cancelled' run: | sleep 30m @@ -218,12 +218,19 @@ jobs: env: FABLAB_PASSPHRASE: ${{ secrets.FABLAB_PASSPHRASE }} run: | - aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg . - gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg - tar -xzf simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz - ./simple-transfer-${GITHUB_RUN_NUMBER}/simple-transfer import simple-transfer-${GITHUB_RUN_NUMBER} - ./simple-transfer-${GITHUB_RUN_NUMBER}/simple-transfer dispose - aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg + if aws s3api head-object \ + --bucket ziti-smoketest-fablab-instances \ + --key simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg + then + aws s3 cp s3://ziti-smoketest-fablab-instances/simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg . + gpg --quiet --batch --yes --decrypt --passphrase=${FABLAB_PASSPHRASE} --output simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg + tar -xzf simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz + ./simple-transfer-${GITHUB_RUN_NUMBER}/simple-transfer import simple-transfer-${GITHUB_RUN_NUMBER} + ./simple-transfer-${GITHUB_RUN_NUMBER}/simple-transfer dispose + aws s3 rm s3://ziti-smoketest-fablab-instances/simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg + else + echo "WARN: No instance archive found for simple-transfer-${GITHUB_RUN_NUMBER}.tar.gz.gpg" >&2 + fi publish: name: Publish Binaries From 76bb9861273bd06c2fef8e0fbc7534a66ca13d0c Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Tue, 5 Sep 2023 15:41:06 -0400 Subject: [PATCH 11/14] organize files used to distribute ziti --- .github/workflows/publish-docker-images.yml | 10 +++++----- .github/workflows/publish-linux-packages.yml | 2 +- {build => dist}/dist-packages/README.md | 0 {build => dist}/dist-packages/linux/nfpm-ziti-cli.yaml | 0 .../docker-images}/cross-build/Dockerfile | 0 .../docker-images}/cross-build/README.md | 0 .../docker-images}/cross-build/linux-build.sh | 0 .../docker-images}/ziti-cli/Dockerfile | 0 {docker-images => dist/docker-images}/ziti-cli/bashrc | 0 .../docker-images}/ziti-cli/entrypoint.sh | 0 .../docker-images}/ziti-controller/Dockerfile | 0 .../docker-images}/ziti-controller/entrypoint.sh | 0 .../docker-images}/ziti-router/Dockerfile | 0 .../docker-images}/ziti-router/entrypoint.sh | 0 .../docker-images}/ziti-tunnel/Dockerfile | 0 .../docker-images}/ziti-tunnel/entrypoint.sh | 0 16 files changed, 6 insertions(+), 6 deletions(-) rename {build => dist}/dist-packages/README.md (100%) rename {build => dist}/dist-packages/linux/nfpm-ziti-cli.yaml (100%) rename {docker-images => dist/docker-images}/cross-build/Dockerfile (100%) rename {docker-images => dist/docker-images}/cross-build/README.md (100%) rename {docker-images => dist/docker-images}/cross-build/linux-build.sh (100%) rename {docker-images => dist/docker-images}/ziti-cli/Dockerfile (100%) rename {docker-images => dist/docker-images}/ziti-cli/bashrc (100%) rename {docker-images => dist/docker-images}/ziti-cli/entrypoint.sh (100%) rename {docker-images => dist/docker-images}/ziti-controller/Dockerfile (100%) rename {docker-images => dist/docker-images}/ziti-controller/entrypoint.sh (100%) rename {docker-images => dist/docker-images}/ziti-router/Dockerfile (100%) rename {docker-images => dist/docker-images}/ziti-router/entrypoint.sh (100%) rename {docker-images => dist/docker-images}/ziti-tunnel/Dockerfile (100%) rename {docker-images => dist/docker-images}/ziti-tunnel/entrypoint.sh (100%) diff --git a/.github/workflows/publish-docker-images.yml b/.github/workflows/publish-docker-images.yml index fc9fce4c2..5dfbd5ae2 100644 --- a/.github/workflows/publish-docker-images.yml +++ b/.github/workflows/publish-docker-images.yml @@ -65,11 +65,11 @@ jobs: with: builder: ${{ steps.buildx.outputs.name }} context: ${{ github.workspace }}/ - file: ${{ github.workspace }}/docker-images/ziti-cli/Dockerfile + file: ${{ github.workspace }}/dist/docker-images/ziti-cli/Dockerfile platforms: linux/amd64,linux/arm64 tags: ${{ steps.tagprep_cli.outputs.DOCKER_TAGS }} build-args: | - DOCKER_BUILD_DIR=./docker-images/ziti-cli + DOCKER_BUILD_DIR=./dist/docker-images/ziti-cli push: true - name: Set Up Container Image Tags for Controller Container @@ -92,7 +92,7 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - context: ${{ github.workspace }}/docker-images/ziti-controller/ + context: ${{ github.workspace }}/dist/docker-images/ziti-controller/ platforms: linux/amd64,linux/arm64 tags: ${{ steps.tagprep_ctrl.outputs.DOCKER_TAGS }} build-args: | @@ -117,7 +117,7 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - context: ${{ github.workspace }}/docker-images/ziti-router/ + context: ${{ github.workspace }}/dist/docker-images/ziti-router/ platforms: linux/amd64,linux/arm64 tags: ${{ steps.tagprep_router.outputs.DOCKER_TAGS }} build-args: | @@ -146,7 +146,7 @@ jobs: uses: docker/build-push-action@v3 with: builder: ${{ steps.buildx.outputs.name }} - context: ${{ github.workspace }}/docker-images/ziti-tunnel/ + context: ${{ github.workspace }}/dist/docker-images/ziti-tunnel/ platforms: linux/amd64,linux/arm64 tags: ${{ steps.tagprep_tun.outputs.DOCKER_TAGS }} build-args: | diff --git a/.github/workflows/publish-linux-packages.yml b/.github/workflows/publish-linux-packages.yml index 23f660bdd..45556d1da 100644 --- a/.github/workflows/publish-linux-packages.yml +++ b/.github/workflows/publish-linux-packages.yml @@ -61,7 +61,7 @@ jobs: with: nfpm_version: "2.32.0" packager: ${{ matrix.nfpm_packager }} - config: build/dist-packages/linux/nfpm-${{ matrix.package_name }}.yaml + config: dist/dist-packages/linux/nfpm-${{ matrix.package_name }}.yaml target: release/ - name: get the package name from the output diff --git a/build/dist-packages/README.md b/dist/dist-packages/README.md similarity index 100% rename from build/dist-packages/README.md rename to dist/dist-packages/README.md diff --git a/build/dist-packages/linux/nfpm-ziti-cli.yaml b/dist/dist-packages/linux/nfpm-ziti-cli.yaml similarity index 100% rename from build/dist-packages/linux/nfpm-ziti-cli.yaml rename to dist/dist-packages/linux/nfpm-ziti-cli.yaml diff --git a/docker-images/cross-build/Dockerfile b/dist/docker-images/cross-build/Dockerfile similarity index 100% rename from docker-images/cross-build/Dockerfile rename to dist/docker-images/cross-build/Dockerfile diff --git a/docker-images/cross-build/README.md b/dist/docker-images/cross-build/README.md similarity index 100% rename from docker-images/cross-build/README.md rename to dist/docker-images/cross-build/README.md diff --git a/docker-images/cross-build/linux-build.sh b/dist/docker-images/cross-build/linux-build.sh similarity index 100% rename from docker-images/cross-build/linux-build.sh rename to dist/docker-images/cross-build/linux-build.sh diff --git a/docker-images/ziti-cli/Dockerfile b/dist/docker-images/ziti-cli/Dockerfile similarity index 100% rename from docker-images/ziti-cli/Dockerfile rename to dist/docker-images/ziti-cli/Dockerfile diff --git a/docker-images/ziti-cli/bashrc b/dist/docker-images/ziti-cli/bashrc similarity index 100% rename from docker-images/ziti-cli/bashrc rename to dist/docker-images/ziti-cli/bashrc diff --git a/docker-images/ziti-cli/entrypoint.sh b/dist/docker-images/ziti-cli/entrypoint.sh similarity index 100% rename from docker-images/ziti-cli/entrypoint.sh rename to dist/docker-images/ziti-cli/entrypoint.sh diff --git a/docker-images/ziti-controller/Dockerfile b/dist/docker-images/ziti-controller/Dockerfile similarity index 100% rename from docker-images/ziti-controller/Dockerfile rename to dist/docker-images/ziti-controller/Dockerfile diff --git a/docker-images/ziti-controller/entrypoint.sh b/dist/docker-images/ziti-controller/entrypoint.sh similarity index 100% rename from docker-images/ziti-controller/entrypoint.sh rename to dist/docker-images/ziti-controller/entrypoint.sh diff --git a/docker-images/ziti-router/Dockerfile b/dist/docker-images/ziti-router/Dockerfile similarity index 100% rename from docker-images/ziti-router/Dockerfile rename to dist/docker-images/ziti-router/Dockerfile diff --git a/docker-images/ziti-router/entrypoint.sh b/dist/docker-images/ziti-router/entrypoint.sh similarity index 100% rename from docker-images/ziti-router/entrypoint.sh rename to dist/docker-images/ziti-router/entrypoint.sh diff --git a/docker-images/ziti-tunnel/Dockerfile b/dist/docker-images/ziti-tunnel/Dockerfile similarity index 100% rename from docker-images/ziti-tunnel/Dockerfile rename to dist/docker-images/ziti-tunnel/Dockerfile diff --git a/docker-images/ziti-tunnel/entrypoint.sh b/dist/docker-images/ziti-tunnel/entrypoint.sh similarity index 100% rename from docker-images/ziti-tunnel/entrypoint.sh rename to dist/docker-images/ziti-tunnel/entrypoint.sh From 947f1f33093cc7bcbec3d1036b88746c4993d762 Mon Sep 17 00:00:00 2001 From: gberl002 Date: Wed, 6 Sep 2023 09:45:50 -0400 Subject: [PATCH 12/14] Minor changes to get this script working on Mac Signed-off-by: gberl002 --- quickstart/docker/createLocalImage.sh | 2 +- quickstart/test/compose-test.zsh | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/quickstart/docker/createLocalImage.sh b/quickstart/docker/createLocalImage.sh index 21f155ada..e349c60b3 100755 --- a/quickstart/docker/createLocalImage.sh +++ b/quickstart/docker/createLocalImage.sh @@ -7,7 +7,7 @@ ZITI_BIN="${SCRIPT_DIR}/image/ziti-bin" case "${1:-}" in --build) mkdir -p "${ZITI_BIN}" - go build -o "${ZITI_BIN}" "${SCRIPT_DIR}/../../..." + GOOS="linux" go build -o "${ZITI_BIN}" "${SCRIPT_DIR}/../../..." shift ;; esac diff --git a/quickstart/test/compose-test.zsh b/quickstart/test/compose-test.zsh index 4fb55ff0d..1f7d109a6 100755 --- a/quickstart/test/compose-test.zsh +++ b/quickstart/test/compose-test.zsh @@ -92,11 +92,11 @@ fi mv ./simplified-docker-compose.yml ./compose.yml # learn the expected Go version from the Go mod file so we can pull the correct container image -ZITI_GO_VERSION="$(grep -Po '^go\s+\K\d+\.\d+(\.\d+)?$' ./go.mod)" +ZITI_GO_VERSION="1.20" # make this var available in the Compose project -sed -Ei "s/^(#\s+)?(ZITI_GO_VERSION)=.*/\2=${ZITI_GO_VERSION}/" ./.env -sed -Ei "s/^(#\s+)?(ZITI_PWD)=.*/\2=${ZITI_PWD}/" ./.env -sed -Ei "s/^(#\s+)?(ZITI_INTERFACE)=.*/\2=${ZITI_INTERFACE:-127.0.0.1}/" ./.env +sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_GO_VERSION)=.*/\2=${ZITI_GO_VERSION}/" ./.env +sed -Ei '' "s/^(#\s+)?(ZITI_PWD)=.*/\2=${ZITI_PWD}/" ./.env +sed -Ei '' "s/^(#\s+)?(ZITI_INTERFACE)=.*/\2=${ZITI_INTERFACE:-127.0.0.1}/" ./.env # pull images preemptively that we never build locally because pull=never when using a local quickstart image for IMAGE in \ @@ -111,7 +111,7 @@ trap down_project SIGTERM SIGINT EXIT # if ZITI_QUICK_IMAGE_TAG is set then run the locally-built image if [[ -n "${ZITI_QUICK_IMAGE_TAG:-}" ]]; then - sed -Ei "s/^(#\s+)?(ZITI_VERSION)=.*/\2=${ZITI_QUICK_IMAGE_TAG}/" ./.env + sed -Ei '' "s/^(#\s+)?(ZITI_VERSION)=.*/\2=${ZITI_QUICK_IMAGE_TAG}/" ./.env docker compose up --detach --pull=never &>/dev/null # no pull because local quickstart image else echo "ERROR: ZITI_QUICK_IMAGE_TAG is not set" >&2 From e8f8da3c5e41f666d1e5ac8c0c7b71fb170e9bd8 Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 6 Sep 2023 09:50:47 -0400 Subject: [PATCH 13/14] patch for macOS --- quickstart/test/compose-test.zsh | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/quickstart/test/compose-test.zsh b/quickstart/test/compose-test.zsh index 1f7d109a6..dc323e5d4 100755 --- a/quickstart/test/compose-test.zsh +++ b/quickstart/test/compose-test.zsh @@ -92,11 +92,11 @@ fi mv ./simplified-docker-compose.yml ./compose.yml # learn the expected Go version from the Go mod file so we can pull the correct container image -ZITI_GO_VERSION="1.20" +ZITI_GO_VERSION="$(awk '/^go\s+/ {print $2}' ./go.mod)" # make this var available in the Compose project sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_GO_VERSION)=.*/\2=${ZITI_GO_VERSION}/" ./.env -sed -Ei '' "s/^(#\s+)?(ZITI_PWD)=.*/\2=${ZITI_PWD}/" ./.env -sed -Ei '' "s/^(#\s+)?(ZITI_INTERFACE)=.*/\2=${ZITI_INTERFACE:-127.0.0.1}/" ./.env +sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_PWD)=.*/\2=${ZITI_PWD}/" ./.env +sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_INTERFACE)=.*/\2=${ZITI_INTERFACE:-127.0.0.1}/" ./.env # pull images preemptively that we never build locally because pull=never when using a local quickstart image for IMAGE in \ @@ -111,7 +111,7 @@ trap down_project SIGTERM SIGINT EXIT # if ZITI_QUICK_IMAGE_TAG is set then run the locally-built image if [[ -n "${ZITI_QUICK_IMAGE_TAG:-}" ]]; then - sed -Ei '' "s/^(#\s+)?(ZITI_VERSION)=.*/\2=${ZITI_QUICK_IMAGE_TAG}/" ./.env + sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_VERSION)=.*/\2=${ZITI_QUICK_IMAGE_TAG}/" ./.env docker compose up --detach --pull=never &>/dev/null # no pull because local quickstart image else echo "ERROR: ZITI_QUICK_IMAGE_TAG is not set" >&2 @@ -120,15 +120,15 @@ fi # copy files that are not present in older quickstart container images to the persistent volume; this allows us to run # the test suite against them and investigate if the test fails and the container is destroyed -for FILE in \ - "" +# for FILE in \ + # "" # check-cert-chains.zsh # TODO: re-add cert checks to cp list after https://github.com/openziti/ziti/pull/1278 -do - docker compose cp \ - "./${FILE}" \ - "ziti-controller:/persistent/${FILE}" &>/dev/null -done +# do + # docker compose cp \ + # "./${FILE}" \ + # "ziti-controller:/persistent/${FILE}" &>/dev/null +# done # TODO: build these executables into the container image? # wait for the controller and router to be ready and run the certificate check script; NOUNSET option is enabled after @@ -137,12 +137,10 @@ docker compose exec ziti-controller \ bash -eo pipefail -c ' source "${ZITI_SCRIPTS}/ziti-cli-functions.sh" >/dev/null; echo "INFO: waiting for controller"; - sleep 3; source /persistent/ziti.env >/dev/null; set -u; _wait_for_controller >/dev/null; echo "INFO: waiting for public router"; - sleep 3; source /persistent/ziti.env >/dev/null; _wait_for_public_router >/dev/null; ' From a2d1103c94af4fce3e4d44a8cc19a3e0cb25b81d Mon Sep 17 00:00:00 2001 From: Kenneth Bingham Date: Wed, 6 Sep 2023 10:11:51 -0400 Subject: [PATCH 14/14] move atomically instead of munge in place because syntax diff bsd vs gnu sed --- quickstart/test/compose-test.zsh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/quickstart/test/compose-test.zsh b/quickstart/test/compose-test.zsh index dc323e5d4..fd42f1a8e 100755 --- a/quickstart/test/compose-test.zsh +++ b/quickstart/test/compose-test.zsh @@ -92,11 +92,12 @@ fi mv ./simplified-docker-compose.yml ./compose.yml # learn the expected Go version from the Go mod file so we can pull the correct container image -ZITI_GO_VERSION="$(awk '/^go\s+/ {print $2}' ./go.mod)" +ZITI_GO_VERSION="$(awk '/^go[[:space:]]+/ {print $2}' ./go.mod)" # make this var available in the Compose project -sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_GO_VERSION)=.*/\2=${ZITI_GO_VERSION}/" ./.env -sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_PWD)=.*/\2=${ZITI_PWD}/" ./.env -sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_INTERFACE)=.*/\2=${ZITI_INTERFACE:-127.0.0.1}/" ./.env +sed -Ee "s/^(#[[:space:]]+)?(ZITI_GO_VERSION)=.*/\2=${ZITI_GO_VERSION}/" \ + -e "s/^(#[[:space:]]+)?(ZITI_PWD)=.*/\2=${ZITI_PWD}/" \ + -e "s/^(#[[:space:]]+)?(ZITI_INTERFACE)=.*/\2=${ZITI_INTERFACE:-127.0.0.1}/" ./.env > ./.env.tmp +mv ./.env.tmp ./.env # pull images preemptively that we never build locally because pull=never when using a local quickstart image for IMAGE in \ @@ -111,7 +112,8 @@ trap down_project SIGTERM SIGINT EXIT # if ZITI_QUICK_IMAGE_TAG is set then run the locally-built image if [[ -n "${ZITI_QUICK_IMAGE_TAG:-}" ]]; then - sed -Ei '' "s/^(#[[:space:]]+)?(ZITI_VERSION)=.*/\2=${ZITI_QUICK_IMAGE_TAG}/" ./.env + sed -Ee "s/^(#[[:space:]]+)?(ZITI_VERSION)=.*/\2=${ZITI_QUICK_IMAGE_TAG}/" ./.env > ./.env.tmp + mv ./.env.tmp ./.env docker compose up --detach --pull=never &>/dev/null # no pull because local quickstart image else echo "ERROR: ZITI_QUICK_IMAGE_TAG is not set" >&2 @@ -138,7 +140,6 @@ docker compose exec ziti-controller \ source "${ZITI_SCRIPTS}/ziti-cli-functions.sh" >/dev/null; echo "INFO: waiting for controller"; source /persistent/ziti.env >/dev/null; - set -u; _wait_for_controller >/dev/null; echo "INFO: waiting for public router"; source /persistent/ziti.env >/dev/null;