Merge pull request #1146 from openziti/ziti-components

Make ziti fablab component strategies

.github/workflows/main.yml

@@ -147,8 +147,7 @@ jobs:
       - name: Test Ziti Command
         run: |
           echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV"
-          $(go env GOPATH)/bin/simple-transfer run
-          # pushd zititest && go test -v ./tests/... && popd
+          pushd zititest && go test -timeout 30m -v ./tests/... && popd
 
       - name: Test Ziti Command Teardown
         if: always()

common/getziti/extract.go

@@ -0,0 +1,131 @@
+package getziti
+
+import (
+	"archive/tar"
+	"archive/zip"
+	"compress/gzip"
+	"fmt"
+	"github.com/michaelquigley/pfxlog"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+)
+
+func Unzip(src, dest string, filter func(path string) (string, bool)) error {
+	r, err := zip.OpenReader(src)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := r.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	// Closure to address file descriptors issue with all the deferred .Close() methods
+	extractAndWriteFile := func(f *zip.File) error {
+		rc, err := f.Open()
+		if err != nil {
+			return err
+		}
+		defer func() {
+			if err := rc.Close(); err != nil {
+				panic(err)
+			}
+		}()
+
+		path := filepath.Join(dest, f.Name)
+
+		// Check for ZipSlip (Directory traversal)
+		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
+			return fmt.Errorf("illegal file path: %s", path)
+		}
+
+		if !f.FileInfo().IsDir() {
+			fileDest, accept := filter(f.Name)
+			if !accept {
+				return nil
+			}
+			fullPath := filepath.Join(dest, fileDest)
+
+			return CopyReaderToFile(rc, fullPath, f.Mode())
+		}
+		return nil
+	}
+
+	for _, f := range r.File {
+		if err = extractAndWriteFile(f); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func UnTarGz(src, dest string, f func(path string) (string, bool)) error {
+	fr, err := os.Open(src)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err = fr.Close(); err != nil {
+			panic(err)
+		}
+	}()
+
+	gzr, err := gzip.NewReader(fr)
+	if err != nil {
+		return err
+	}
+
+	r := tar.NewReader(gzr)
+
+	for {
+		header, err := r.Next()
+		if err == io.EOF {
+			return nil
+		}
+
+		if err != nil {
+			return err
+		}
+
+		if header == nil {
+			continue
+		}
+
+		if header.Typeflag == tar.TypeReg {
+			fileDest, accept := f(header.Name)
+			if !accept {
+				continue
+			}
+			fullPath := filepath.Join(dest, fileDest)
+
+			if err = CopyReaderToFile(r, fullPath, os.FileMode(header.Mode)); err != nil {
+				return err
+			}
+		}
+	}
+}
+
+func CopyReaderToFile(in io.Reader, dst string, mode os.FileMode) error {
+	out, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR, mode)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := out.Close(); e != nil {
+			pfxlog.Logger().WithError(err).Errorf("failed to copy dest file (%s)", dst)
+		}
+	}()
+
+	_, err = io.Copy(out, in)
+	if err != nil {
+		return err
+	}
+
+	return out.Sync()
+}

common/getziti/github.go

@@ -1,12 +1,15 @@
 package getziti
 
 import (
+	"context"
 	"fmt"
 	"github.com/blang/semver"
 	"github.com/go-resty/resty/v2"
 	"github.com/michaelquigley/pfxlog"
+	c "github.com/openziti/ziti/ziti/constants"
 	"github.com/pkg/errors"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"sort"
@@ -115,7 +118,7 @@ func GetLatestGitHubReleaseAsset(appName string, appGitHub string, version strin
 			version = strings.TrimPrefix(version, "v")
 		}
 
-		if appName == "ziti-edge-tunnel" {
+		if appName == "ziti" || appName == "ziti-edge-tunnel" {
 			if !strings.HasPrefix(version, "v") {
 				version = "v" + version
 			}
@@ -126,7 +129,10 @@ func GetLatestGitHubReleaseAsset(appName string, appGitHub string, version strin
 		version = "tags/" + version
 	}
 
+	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
+
 	resp, err := getRequest(verbose).
+		SetContext(ctx).
 		SetQueryParams(map[string]string{}).
 		SetHeader("Accept", "application/vnd.github.v3+json").
 		SetResult(&GitHubReleasesData{}).
@@ -149,7 +155,12 @@ func GetLatestGitHubReleaseAsset(appName string, appGitHub string, version strin
 
 // DownloadGitHubReleaseAsset will download a file from the given GitHUb release area
 func DownloadGitHubReleaseAsset(fullUrl string, filepath string) (err error) {
-	resp, err := getRequest(false).
+	resp, err := resty.
+		New().
+		SetTimeout(time.Minute).
+		SetRetryCount(2).
+		SetRedirectPolicy(resty.FlexibleRedirectPolicy(15)).
+		R().
 		SetOutput(filepath).
 		Get(fullUrl)
 
@@ -177,37 +188,86 @@ func FindVersionAndInstallGitHubRelease(zitiApp string, zitiAppGitHub string, ta
 	if err != nil {
 		return err
 	}
-	return InstallGitHubRelease(zitiApp, release, binDir, targetOS, targetArch)
+	return InstallGitHubRelease(zitiApp, release, binDir, targetOS, targetArch, version)
 }
 
-func InstallGitHubRelease(zitiApp string, release *GitHubReleasesData, binDir string, targetOS, targetArch string) error {
-	fileName := zitiApp
-	if targetOS == "windows" {
-		fileName += ".exe"
-	}
-
-	fullPath := filepath.Join(binDir, fileName)
-	ext := ".zip"
-	zipFile := fullPath + ext
-
+func InstallGitHubRelease(zitiApp string, release *GitHubReleasesData, binDir string, targetOS, targetArch, version string) error {
 	releaseUrl, err := release.GetDownloadUrl(zitiApp, targetOS, targetArch)
 	if err != nil {
 		return err
 	}
 
-	err = DownloadGitHubReleaseAsset(releaseUrl, zipFile)
+	parsedUrl, err := url.Parse(releaseUrl)
 	if err != nil {
 		return err
 	}
 
-	err = Unzip(zipFile, binDir)
+	fileName := filepath.Base(parsedUrl.Path)
+
+	fullPath := filepath.Join(binDir, fileName)
+	err = DownloadGitHubReleaseAsset(releaseUrl, fullPath)
 	if err != nil {
 		return err
 	}
-	err = os.Remove(zipFile)
-	if err != nil {
-		return err
+
+	defer func() {
+		err = os.Remove(fullPath)
+		if err != nil {
+			pfxlog.Logger().WithError(err).Errorf("failed to removed downloaded release archive %s", fullPath)
+		}
+	}()
+
+	if strings.HasSuffix(fileName, ".zip") {
+		if zitiApp == c.ZITI_EDGE_TUNNEL {
+			count := 0
+			zitiFileName := "ziti-edge-tunnel-" + version
+			err = Unzip(fullPath, binDir, func(path string) (string, bool) {
+				if path == "ziti-edge-tunnel" {
+					count++
+					return zitiFileName, true
+				}
+				return "", false
+			})
+
+			if err != nil {
+				return err
+			}
+
+			if count != 1 {
+				return errors.Errorf("didn't find ziti-edge-tunnel executable in release archive. count: %v", count)
+			}
+
+			pfxlog.Logger().Infof("Successfully installed '%s' version '%s' to %s", zitiApp, release.Version, filepath.Join(binDir, zitiFileName))
+			return nil
+		} else {
+			return errors.Errorf("unsupported application '%s'", zitiApp)
+		}
+	} else if strings.HasSuffix(fileName, ".tar.gz") {
+		if zitiApp == c.ZITI {
+			count := 0
+			zitiFileName := "ziti-" + version
+			err = UnTarGz(fullPath, binDir, func(path string) (string, bool) {
+				if path == "ziti/ziti" {
+					count++
+					return zitiFileName, true
+				}
+				return "", false
+			})
+
+			if err != nil {
+				return err
+			}
+
+			if count != 1 {
+				return errors.Errorf("didn't find ziti executable in release archive. count: %v", count)
+			}
+
+			pfxlog.Logger().Infof("Successfully installed '%s' version '%s' to %s", zitiApp, release.Version, filepath.Join(binDir, zitiFileName))
+			return nil
+		} else {
+			return errors.Errorf("unsupported application '%s'", zitiApp)
+		}
+	} else {
+		return errors.Errorf("unsupported release file type '%s'", fileName)
 	}
-	pfxlog.Logger().Infof("Successfully installed '%s' version '%s'", zitiApp, release.Version)
-	return os.Chmod(fullPath, 0755)
 }

common/getziti/install_ziti.go

@@ -0,0 +1,26 @@
+package getziti
+
+import (
+	"fmt"
+	"github.com/blang/semver"
+	c "github.com/openziti/ziti/ziti/constants"
+	"strings"
+)
+
+func InstallZiti(targetVersion, targetOS, targetArch, binDir string, verbose bool) error {
+	var newVersion semver.Version
+
+	if targetVersion != "" {
+		newVersion = semver.MustParse(strings.TrimPrefix(targetVersion, "v"))
+	} else {
+		v, err := GetLatestGitHubReleaseVersion(c.ZITI, verbose)
+		if err != nil {
+			return err
+		}
+		newVersion = v
+	}
+
+	fmt.Println("Attempting to install '" + c.ZITI + "' version: v" + newVersion.String())
+	return FindVersionAndInstallGitHubRelease(
+		c.ZITI, c.ZITI, targetOS, targetArch, binDir, "v"+newVersion.String(), verbose)
+}

common/getziti/unzip.go

@@ -1,74 +0,0 @@
-package getziti
-
-import (
-	"archive/zip"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-)
-
-func Unzip(src, dest string) error {
-	r, err := zip.OpenReader(src)
-	if err != nil {
-		return err
-	}
-	defer func() {
-		if err := r.Close(); err != nil {
-			panic(err)
-		}
-	}()
-
-	os.MkdirAll(dest, 0755)
-
-	// Closure to address file descriptors issue with all the deferred .Close() methods
-	extractAndWriteFile := func(f *zip.File) error {
-		rc, err := f.Open()
-		if err != nil {
-			return err
-		}
-		defer func() {
-			if err := rc.Close(); err != nil {
-				panic(err)
-			}
-		}()
-
-		path := filepath.Join(dest, f.Name)
-
-		// Check for ZipSlip (Directory traversal)
-		if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) {
-			return fmt.Errorf("illegal file path: %s", path)
-		}
-
-		if f.FileInfo().IsDir() {
-			os.MkdirAll(path, f.Mode())
-		} else {
-			os.MkdirAll(filepath.Dir(path), f.Mode())
-			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
-			if err != nil {
-				return err
-			}
-			defer func() {
-				if err := f.Close(); err != nil {
-					panic(err)
-				}
-			}()
-
-			_, err = io.Copy(f, rc)
-			if err != nil {
-				return err
-			}
-		}
-		return nil
-	}
-
-	for _, f := range r.File {
-		err := extractAndWriteFile(f)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

ziti/cmd/demo/echo_server.go

@@ -25,6 +25,7 @@ import (
 	"github.com/openziti/identity"
 	"github.com/openziti/sdk-golang/ziti"
 	"github.com/openziti/sdk-golang/ziti/edge"
+	"github.com/openziti/ziti/common/version"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
@@ -52,6 +53,10 @@ type echoServer struct {
 	bindWithIdentity bool
 	zitiIdentity     identity.Identity
 	zitiListener     edge.Listener
+
+	cliAgentEnabled bool
+	cliAgentAddr    string
+	cliAgentAlias   string
 }
 
 func newEchoServerCmd() *cobra.Command {
@@ -73,7 +78,9 @@ func newEchoServerCmd() *cobra.Command {
 	cmd.Flags().StringVarP(&server.configFile, "identity", "i", "", "Specify the Ziti identity to use. If not specified the Ziti listener won't be started")
 	cmd.Flags().StringVarP(&server.service, "service", "s", "echo", "Ziti service to bind. Defaults to 'echo'")
 	cmd.Flags().BoolVar(&server.bindWithIdentity, "addressable", false, "Specify if this application should be addressable by the edge identity")
-	cmd.Flags().SetInterspersed(true)
+	cmd.Flags().BoolVar(&server.cliAgentEnabled, "cli-agent", true, "Enable/disable CLI Agent (enabled by default)")
+	cmd.Flags().StringVar(&server.cliAgentAddr, "cli-agent-addr", "", "Specify where CLI Agent should list (ex: unix:/tmp/myfile.sock or tcp:127.0.0.1:10001)")
+	cmd.Flags().StringVar(&server.cliAgentAlias, "cli-agent-alias", "", "Alias which can be used by ziti agent commands to find this instance")
 
 	return cmd
 }
@@ -108,6 +115,19 @@ func (self *echoServer) run(*cobra.Command, []string) {
 		log.Fatalf("No port specified and no ziti identity specified. Not starting either listener, so nothing to do. exiting.")
 	}
 
+	if self.cliAgentEnabled {
+		options := agent.Options{
+			Addr:       self.cliAgentAddr,
+			AppType:    "echo-server",
+			AppVersion: version.GetVersion(),
+			AppAlias:   self.cliAgentAlias,
+		}
+
+		if err := agent.Listen(options); err != nil {
+			pfxlog.Logger().WithError(err).Error("unable to start CLI agent")
+		}
+	}
+
 	if self.healthCheckAddr != "" {
 		go func() {
 			err := http.ListenAndServe(self.healthCheckAddr, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {

ziti/cmd/demo/zcat.go

@@ -20,6 +20,7 @@ import (
 	"github.com/michaelquigley/pfxlog"
 	"github.com/openziti/foundation/v2/info"
 	"github.com/openziti/sdk-golang/ziti"
+	"github.com/openziti/sdk-golang/ziti/edge"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"io"
@@ -138,4 +139,9 @@ func (self *zcatAction) copy(writer io.Writer, reader io.Reader) {
 	if err != nil {
 		pfxlog.Logger().WithError(err).Error("error while copying bytes")
 	}
+	if zconn, ok := writer.(edge.Conn); ok {
+		if err = zconn.CloseWrite(); err != nil {
+			pfxlog.Logger().WithError(err).Error("error closing write side of ziti connection")
+		}
+	}
 }

ziti/cmd/pki.go

@@ -60,6 +60,7 @@ type PKIFlags struct {
 	Email                 []string
 	PKI                   *pki.ZitiPKI
 	SpiffeID              string
+	AllowOverwrite        bool
 }
 
 var (

ziti/cmd/pki_create_server.go

@@ -77,6 +77,7 @@ func (o *PKICreateServerOptions) addPKICreateServerFlags(cmd *cobra.Command) {
 	cmd.Flags().IntVarP(&o.Flags.CAMaxpath, "max-path-len", "", -1, "Intermediate maximum path length")
 	cmd.Flags().IntVarP(&o.Flags.CAPrivateKeySize, "private-key-size", "", 4096, "Size of the private key")
 	cmd.Flags().StringVar(&o.Flags.SpiffeID, "spiffe-id", "", "Optionally provide the path portion of a SPIFFE id. The trust domain will be taken from the signing certificate.")
+	cmd.Flags().BoolVar(&o.Flags.AllowOverwrite, "allow-overwrite", false, "Allow overwrite existing certs")
 }
 
 // Run implements this command
@@ -156,6 +157,7 @@ func (o *PKICreateServerOptions) Run() error {
 		Template:            template,
 		IsClientCertificate: false,
 		PrivateKeySize:      o.Flags.CAPrivateKeySize,
+		AllowOverwrite:      o.Flags.AllowOverwrite,
 	}
 
 	if err := o.Flags.PKI.Sign(signer, req); err != nil {

ziti/pki/pki/pki.go

@@ -50,7 +50,9 @@ type Request struct {
 	IsClientCertificate bool
 	PrivateKeySize      int
 	Template            *x509.Certificate
+	AllowOverwrite      bool
 }
+
 type CSRRequest struct {
 	Name                string
 	IsClientCertificate bool
@@ -146,7 +148,7 @@ func (e *ZitiPKI) Sign(signer *certificate.Bundle, req *Request) error {
 		return fmt.Errorf("failed creating and signing certificate: %v", err)
 	}
 
-	if err := e.Store.Add(signer.Name, req.Name, req.Template.IsCA, x509.MarshalPKCS1PrivateKey(privateKey), rawCert); err != nil {
+	if err := e.Store.Add(signer.Name, req.Name, req.Template.IsCA, x509.MarshalPKCS1PrivateKey(privateKey), rawCert, req.AllowOverwrite); err != nil {
 		return fmt.Errorf("failed saving generated bundle: %v", err)
 	}
 	return nil

ziti/pki/store/local.go

@@ -120,8 +120,8 @@ func readPEM(path string) ([]byte, error) {
 }
 
 // Add adds the given bundle to the local filesystem.
-func (l *Local) Add(caName, name string, isCa bool, key, cert []byte) error {
-	if l.Exists(caName, name) {
+func (l *Local) Add(caName, name string, isCa bool, key, cert []byte, allowOverwrite bool) error {
+	if !allowOverwrite && l.Exists(caName, name) {
 		return fmt.Errorf("a bundle already exists for the name %v within CA %v", name, caName)
 	}
 	if err := l.writeBundle(caName, name, isCa, key, cert); err != nil {

ziti/pki/store/store.go

@@ -33,9 +33,10 @@ type Store interface {
 	//  Is the bundle to add an intermediate CA.
 	//  The raw private key.
 	//  The raw certificate.
+	//  Flag indicating if existing certs can be overwritten
 	//
 	// Returns an error if it failed to store the bundle.
-	Add(string, string, bool, []byte, []byte) error
+	Add(string, string, bool, []byte, []byte, bool) error
 
 	// Chain concats a signing cert and a newly signed certificate bundle and adds the chained cert to the store.
 	//

ziti/tunnel/root.go

@@ -68,6 +68,7 @@ func NewTunnelCmd(standalone bool) *cobra.Command {
 	root.PersistentFlags().StringP(dnsSvcIpRangeFlag, "d", "100.64.0.1/10", "cidr to use when assigning IPs to unresolvable intercept hostnames")
 	root.PersistentFlags().BoolVar(&cliAgentEnabled, "cli-agent", true, "Enable/disable CLI Agent (enabled by default)")
 	root.PersistentFlags().StringVar(&cliAgentAddr, "cli-agent-addr", "", "Specify where CLI Agent should list (ex: unix:/tmp/myfile.sock or tcp:127.0.0.1:10001)")
+	root.PersistentFlags().StringVar(&cliAgentAlias, "cli-agent-alias", "", "Alias which can be used by ziti agent commands to find this instance")
 
 	p := common.NewOptionsProvider(os.Stdout, os.Stderr)
 	root.AddCommand(enrollment.NewEnrollCommand(p))
@@ -83,6 +84,7 @@ var interceptor intercept.Interceptor
 var logFormatter string
 var cliAgentEnabled bool
 var cliAgentAddr string
+var cliAgentAlias string
 
 func Execute() {
 	if err := NewTunnelCmd(true).Execute(); err != nil {
@@ -120,7 +122,13 @@ func rootPostRun(cmd *cobra.Command, _ []string) {
 		// don't use the agent's shutdown handler. it calls os.Exit on SIGINT
 		// which interferes with the servicePoller shutdown
 		cleanup := false
-		if err := agent.Listen(agent.Options{Addr: cliAgentAddr, ShutdownCleanup: &cleanup}); err != nil {
+		err := agent.Listen(agent.Options{
+			Addr:            cliAgentAddr,
+			ShutdownCleanup: &cleanup,
+			AppAlias:        cliAgentAlias,
+		})
+
+		if err != nil {
 			pfxlog.Logger().WithError(err).Error("unable to start CLI agent")
 		}
 	}

zititest/go.mod

@@ -7,25 +7,24 @@ replace github.com/openziti/ziti => ../
 require (
 	github.com/Jeffail/gabs v1.4.0
 	github.com/Jeffail/gabs/v2 v2.7.0
+	github.com/google/uuid v1.3.0
 	github.com/michaelquigley/pfxlog v0.6.10
 	github.com/openziti/channel/v2 v2.0.81
 	github.com/openziti/edge v0.24.348
-	github.com/openziti/fablab v0.4.52
+	github.com/openziti/fablab v0.5.0
 	github.com/openziti/fabric v0.23.39
 	github.com/openziti/foundation/v2 v2.0.26
 	github.com/openziti/identity v1.0.57
 	github.com/openziti/sdk-golang v0.20.67
 	github.com/openziti/storage v0.2.8
 	github.com/openziti/transport/v2 v2.0.91
-	github.com/openziti/ziti v0.28.0
+	github.com/openziti/ziti v0.28.3
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/spf13/cobra v1.7.0
 	github.com/stretchr/testify v1.8.4
 	go.etcd.io/bbolt v1.3.7
 	golang.org/x/net v0.11.0
 	google.golang.org/protobuf v1.30.0
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -78,7 +77,6 @@ require (
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
-	github.com/google/uuid v1.3.0 // indirect
 	github.com/gorilla/handlers v1.5.1 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
@@ -152,6 +150,7 @@ require (
 	github.com/speps/go-hashids v2.0.0+incompatible // indirect
 	github.com/spf13/afero v1.6.0 // indirect
 	github.com/spf13/cast v1.4.1 // indirect
+	github.com/spf13/cobra v1.7.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.10.0 // indirect
@@ -165,7 +164,7 @@ require (
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
 	github.com/xeipuuv/gojsonschema v1.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
-	go.mongodb.org/mongo-driver v1.11.7 // indirect
+	go.mongodb.org/mongo-driver v1.12.0 // indirect
 	go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect
 	go.opentelemetry.io/otel v1.16.0 // indirect
 	go.opentelemetry.io/otel/metric v1.16.0 // indirect
@@ -184,6 +183,7 @@ require (
 	gopkg.in/resty.v1 v1.12.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	nhooyr.io/websocket v1.8.7 // indirect
 	rsc.io/goversion v1.2.0 // indirect
 )

zititest/go.sum

@@ -647,8 +647,8 @@ github.com/openziti/edge v0.24.348 h1:m0zh1n5OBj7Tjemu9XfxdYhbKvo6HdGQMHfC/Zledb
 github.com/openziti/edge v0.24.348/go.mod h1:wcoBVKqW99/CBg7LjzbHT3dJ8JMlRZelPhbjNcjwMr0=
 github.com/openziti/edge-api v0.25.29 h1:DbZ0GKpch9ik+XnC0qoP1dOg1AmgUEeWH4VIyOIehrQ=
 github.com/openziti/edge-api v0.25.29/go.mod h1:mqYquh1RYvHjV/M9MGO4Ng0Xq9ixAH8o8cT5kw5XCWY=
-github.com/openziti/fablab v0.4.52 h1:X5+NaRnfPxbzhxUh9b4IQmVNVgYWFf8/Ck2sKNz77ho=
-github.com/openziti/fablab v0.4.52/go.mod h1:aF8Fu3C8hnnaOJyAZz3AXGY4cMxQyBLhoIlSRJ/VVpc=
+github.com/openziti/fablab v0.5.0 h1:+CMVE08hS62UR3tIfEvMb3XcWwzD1np//cpHHTboVW8=
+github.com/openziti/fablab v0.5.0/go.mod h1:VLvs0AAuKAFqy8jhGvs+FgszYEbB456duPOBfRj4peU=
 github.com/openziti/fabric v0.23.39 h1:t6npkfBsPr+8GvF9X/rTLHxGsce5FTa4NoA80MInxYc=
 github.com/openziti/fabric v0.23.39/go.mod h1:px2zUzQrgObmv6sdXgGJ028Wr45aGcgpFvfIP/qgDFo=
 github.com/openziti/foundation/v2 v2.0.26 h1:RISGM97aIFnnEkxrcVsO6fnGg9W4ByydSnqWAVkJAqY=
@@ -819,7 +819,6 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w=
 github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0=
-github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
 github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM=
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
@@ -842,8 +841,10 @@ github.com/vultr/govultr/v2 v2.0.0/go.mod h1:2PsEeg+gs3p/Fo5Pw8F9mv+DUBEOlrNZ8Gm
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
 github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
@@ -867,8 +868,8 @@ go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsX
 go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg=
 go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng=
 go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8=
-go.mongodb.org/mongo-driver v1.11.7 h1:LIwYxASDLGUg/8wOhgOOZhX8tQa/9tgZPgzZoVqJvcs=
-go.mongodb.org/mongo-driver v1.11.7/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY=
+go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE=
+go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0=
 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak=
 go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
 go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk=
@@ -1143,6 +1144,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=

zititest/models/ha/actions/start.go

@@ -7,7 +7,6 @@ import (
 	"github.com/openziti/fablab/kernel/lib/actions/component"
 	"github.com/openziti/fablab/kernel/lib/actions/semaphore"
 	"github.com/openziti/fablab/kernel/model"
-	util_actions "github.com/openziti/ziti/zititest/utils/actions"
 	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
 	"github.com/openziti/ziti/zititest/zitilab/models"
 )
@@ -30,7 +29,7 @@ func (a *startAction) bind(m *model.Model) model.Action {
 	workflow.AddAction(zitilib_actions.StartMetricbeat("*", a.Metricbeat.ConfigPath, a.Metricbeat.DataPath, a.Metricbeat.LogPath))
 	workflow.AddAction(zitilib_actions.StartConsul("*", a.Consul.ServerAddr, a.Consul.ConfigDir, a.Consul.DataPath, a.Consul.LogPath))
 	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-	workflow.AddAction(util_actions.StartEchoServers("#echo-server"))
+	workflow.AddAction(component.Start("#echo-server"))
 	workflow.AddAction(semaphore.Sleep(2 * time.Second))
 
 	return workflow

zititest/models/ha/configs/ctrl.yml.tmpl

@@ -5,15 +5,15 @@ raft:
   dataDir: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrldata
 
 identity:
-  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/certs/{{ .Component.PublicIdentity }}-server.cert
-  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/keys/{{ .Component.PublicIdentity }}-server.key
-  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/certs/{{ .Component.PublicIdentity }}-server.chain.pem
+  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.cert
+  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key
+  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem
 
 # the endpoint that routers will connect to the controller over.
 ctrl:
   listener: tls:0.0.0.0:6262
   options:
-    advertiseAddress: tls:{{.Host.PublicIp}}:6262
+    advertiseAddress: tls:{{ .Host.PublicIp }}:6262
     # (optional) settings
     # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000)
     #maxQueuedConnects:      50
@@ -82,7 +82,7 @@ edge:
     # address - required
     # The default address (host:port) to use for enrollment for the Client API. This value must match one of the addresses
     # defined in a bind point's address field for the `edge-client` API in the web section.
-    address: {{.Host.PublicIp}}:1280
+    address: {{ .Host.PublicIp }}:1280
   # enrollment - required
   # A section containing settings pertaining to enrollment.
   enrollment:
@@ -91,8 +91,8 @@ edge:
     # a signing certificate from the PKI that the Ziti environment is using to sign certificates. The signingCert.cert
     # will be added to the /.well-known CA store that is used to bootstrap trust with the Ziti Controller.
     signingCert:
-      cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/certs/{{ .Component.PublicIdentity }}.cert
-      key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/keys/{{ .Component.PublicIdentity }}.key
+      cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert
+      key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key
 
     # edgeIdentity - optional
     # A section for identity enrollment specific settings
@@ -130,7 +130,7 @@ web:
         # address - required
         # The public address that external incoming requests will be able to resolve. Used in request processing and
         # response content that requires full host:port/path addresses.
-        address: {{.Host.PublicIp}}:1280
+        address: {{ .Host.PublicIp }}:1280
 
         # newAddress - optional
         # A host:port string which will be sent out as an HTTP header "ziti-new-address" if specified. If the header

zititest/models/ha/main.go

@@ -25,8 +25,6 @@ import (
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
 	semaphore0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
 	terraform_0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
-	"github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config"
-	"github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit"
 	distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution"
 	"github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync"
 	aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
@@ -37,7 +35,6 @@ import (
 	"github.com/openziti/ziti/zititest/models/test_resources"
 	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
-	zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration"
 	"github.com/sirupsen/logrus"
 	"os"
 	"time"
@@ -85,14 +82,8 @@ var m = &model.Model{
 					InstanceType: "t3.micro",
 					Components: model.Components{
 						"ctrl1": {
-							Scope:          model.Scope{Tags: model.Tags{"ctrl", "spiffe:controller"}},
-							BinaryName:     "ziti controller",
-							ConfigSrc:      "ctrl.yml.tmpl",
-							ConfigName:     "ctrl1.yml",
-							PublicIdentity: "ctrl1",
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type:  &zitilab.ControllerType{},
 						},
 					},
 				},
@@ -100,14 +91,8 @@ var m = &model.Model{
 					InstanceType: "t3.micro",
 					Components: model.Components{
 						"ctrl2": {
-							Scope:          model.Scope{Tags: model.Tags{"ctrl", "spiffe:controller"}},
-							BinaryName:     "ziti controller",
-							ConfigSrc:      "ctrl.yml.tmpl",
-							ConfigName:     "ctrl2.yml",
-							PublicIdentity: "ctrl2",
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type:  &zitilab.ControllerType{},
 						},
 					},
 				},
@@ -116,19 +101,12 @@ var m = &model.Model{
 					InstanceType: "t2.micro",
 					Components: model.Components{
 						"router-east": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "terminator"}},
-							BinaryName:     "ziti router",
-							ConfigSrc:      "router.yml.tmpl",
-							ConfigName:     "router-east.yml",
-							PublicIdentity: "router-east",
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}},
+							Type:  &zitilab.RouterType{},
 						},
 						"echo-server": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "service"}},
-							BinaryName:     "echo-server",
-							PublicIdentity: "echo-server",
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}},
+							Type:  &zitilab.EchoServerType{},
 						},
 					},
 				},
@@ -142,14 +120,8 @@ var m = &model.Model{
 					InstanceType: "t3.micro",
 					Components: model.Components{
 						"ctrl3": {
-							Scope:          model.Scope{Tags: model.Tags{"ctrl", "spiffe:controller"}},
-							BinaryName:     "ziti controller",
-							ConfigSrc:      "ctrl.yml.tmpl",
-							ConfigName:     "ctrl3.yml",
-							PublicIdentity: "ctrl3",
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type:  &zitilab.ControllerType{},
 						},
 					},
 				},
@@ -159,19 +131,12 @@ var m = &model.Model{
 					InstanceType: "t2.micro",
 					Components: model.Components{
 						"router-west": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "terminator"}},
-							BinaryName:     "ziti router",
-							ConfigSrc:      "router.yml.tmpl",
-							ConfigName:     "router-west.yml",
-							PublicIdentity: "router-west",
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}},
+							Type:  &zitilab.RouterType{},
 						},
-						"echo-client": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "client"}},
-							BinaryName:     "echo-client",
-							PublicIdentity: "echo-client",
-						},
-						"consul": {
-							BinaryName: "consul",
+						"zcat": {
+							Scope: model.Scope{Tags: model.Tags{"sdk-app"}},
+							Type:  &zitilab.ZCatType{},
 						},
 					},
 				},
@@ -196,21 +161,13 @@ var m = &model.Model{
 		"login": model.Bind(edge.Login("#ctrl1")),
 	},
 
-	Infrastructure: model.InfrastructureStages{
+	Infrastructure: model.Stages{
 		aws_ssh_key.Express(),
 		terraform_0.Express(),
 		semaphore0.Ready(time.Minute),
 	},
 
-	Configuration: model.ConfigurationStages{
-		zitilib_runlevel_1_configuration.IfPkiNeedsRefresh(
-			zitilib_runlevel_1_configuration.Fabric("simple-transfer.test", ".ctrl"),
-		),
-		config.Component(),
-		devkit.DevKitF(zitilab.ZitiRoot, []string{"ziti", "ziti-echo"}),
-	},
-
-	Distribution: model.DistributionStages{
+	Distribution: model.Stages{
 		distribution.DistributeSshKey("*"),
 		distribution.Locations("*", "logs"),
 		distribution.DistributeDataWithReplaceCallbacks(
@@ -277,7 +234,7 @@ var m = &model.Model{
 		rsync.RsyncStaged(),
 	},
 
-	Disposal: model.DisposalStages{
+	Disposal: model.Stages{
 		terraform.Dispose(),
 		aws_ssh_key2.Dispose(),
 	},

zititest/models/router-test/main.go

@@ -14,8 +14,6 @@ import (
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
-	"github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config"
-	"github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit"
 	distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution"
 	"github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync"
 	aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
@@ -26,7 +24,6 @@ import (
 	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
 	"github.com/openziti/ziti/zititest/zitilab/models"
-	zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration"
 	"os"
 	"path"
 	"strings"
@@ -73,10 +70,8 @@ func (d dbStrategy) GetSite(router *persistence.EdgeRouter) (string, bool) {
 }
 
 func (d dbStrategy) PostProcess(router *persistence.EdgeRouter, c *model.Component) {
-	c.PublicIdentity = router.Id
 	if router.IsTunnelerEnabled {
 		c.Scope.Tags = append(c.Scope.Tags, "tunneler")
-		c.RunWithSudo = true
 	}
 	c.Scope.Tags = append(c.Scope.Tags, "edge-router")
 	c.Scope.Tags = append(c.Scope.Tags, "pre-created")
@@ -123,11 +118,7 @@ var m = &model.Model{
 					InstanceType: "c5.large",
 					Components: model.Components{
 						"ctrl": {
-							Scope:          model.Scope{Tags: model.Tags{"ctrl"}},
-							BinaryName:     "ziti controller",
-							ConfigSrc:      "ctrl.yml",
-							ConfigName:     "ctrl.yml",
-							PublicIdentity: "ctrl",
+							Type: &zitilab.RouterType{},
 						},
 					},
 				},
@@ -158,28 +149,20 @@ var m = &model.Model{
 		"login": model.Bind(edge.Login("#ctrl")),
 	},
 
-	Infrastructure: model.InfrastructureStages{
+	Infrastructure: model.Stages{
 		aws_ssh_key.Express(),
 		terraform_0.Express(),
 		semaphore_0.Ready(90 * time.Second),
 	},
 
-	Configuration: model.ConfigurationStages{
-		zitilib_runlevel_1_configuration.IfPkiNeedsRefresh(
-			zitilib_runlevel_1_configuration.Fabric("router.test", ".ctrl"),
-		),
-		config.Component(),
-		devkit.DevKitF(zitilab.ZitiRoot, []string{"ziti"}),
-	},
-
-	Distribution: model.DistributionStages{
+	Distribution: model.Stages{
 		distribution.DistributeSshKey("*"),
 		distribution.Locations("*", "logs"),
 		rsync.RsyncStaged(),
 		rsync.NewRsyncHost("#ctrl", getDbFile(), "/home/ubuntu/fablab/ctrl.db"),
 	},
 
-	Disposal: model.DisposalStages{
+	Disposal: model.Stages{
 		terraform.Dispose(),
 		aws_ssh_key2.Dispose(),
 	},

zititest/models/simple/actions/bootstrap.go

@@ -17,6 +17,10 @@
 package actions
 
 import (
+	"fmt"
+	"github.com/openziti/ziti/zititest/zitilab"
+	"time"
+
 	"github.com/openziti/fablab/kernel/lib/actions"
 	"github.com/openziti/fablab/kernel/lib/actions/component"
 	"github.com/openziti/fablab/kernel/lib/actions/host"
@@ -24,7 +28,6 @@ import (
 	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
 	"github.com/openziti/ziti/zititest/zitilab/models"
-	"time"
 )
 
 type bootstrapAction struct{}
@@ -39,7 +42,7 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action {
 
 	workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*"))
 	workflow.AddAction(component.Stop("#ctrl"))
-	workflow.AddAction(edge.InitController("#ctrl"))
+	workflow.AddAction(component.Exec("#ctrl", zitilab.ControllerActionInitStandalone))
 	workflow.AddAction(component.Start("#ctrl"))
 	workflow.AddAction(edge.ControllerAvailable("#ctrl", 30*time.Second))
 
@@ -66,56 +69,52 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action {
 			"protocol" : "tcp"
 		}`))
 
-	workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ert-unencrypted", "intercept.v1", `
+	workflow.AddAction(zitilib_actions.Edge("create", "config", "iperf-host", "host.v1", `
 		{
-			"addresses": ["ziti-files-ert-unencrypted.s3-us-west-1.amazonaws.ziti"],
-			"portRanges" : [ { "low": 443, "high": 443 } ],
-			"protocols": ["tcp"]
+			"address" : "localhost", 
+			"port" : 5201, 
+			"protocol" : "tcp"
 		}`))
 
-	workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ert", "intercept.v1", `
-		{
-			"addresses": ["ziti-files-ert.s3-us-west-1.amazonaws.ziti"],
-			"portRanges" : [ { "low": 443, "high": 443 } ],
-			"protocols": ["tcp"]
-		}`))
+	for _, encrypted := range []bool{false, true} {
+		for _, hostType := range []string{"ert", "zet", "ziti-tunnel"} {
+			suffix := ""
+			encryptionFlag := "ON"
 
-	workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-zet-unencrypted", "intercept.v1", `
-		{
-			"addresses": ["ziti-files-zet-unencrypted.s3-us-west-1.amazonaws.ziti"],
-			"portRanges" : [ { "low": 443, "high": 443 } ],
-			"protocols": ["tcp"]
-		}`))
+			if !encrypted {
+				suffix = "-unencrypted"
+				encryptionFlag = "OFF"
+			}
 
-	workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-zet", "intercept.v1", `
-		{
-			"addresses": ["ziti-files-zet.s3-us-west-1.amazonaws.ziti"],
-			"portRanges" : [ { "low": 443, "high": 443 } ],
-			"protocols": ["tcp"]
-		}`))
+			filesConfigName := fmt.Sprintf("files-intercept-%s%s", hostType, suffix)
+			filesConfigDef := fmt.Sprintf(`
+				{
+					"addresses": ["files-%s%s.s3-us-west-1.amazonaws.ziti"],
+					"portRanges" : [ { "low": 443, "high": 443 } ],
+					"protocols": ["tcp"]
+				}`, hostType, suffix)
 
-	workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ziti-tunnel-unencrypted", "intercept.v1", `
-		{
-			"addresses": ["ziti-files-ziti-tunnel-unencrypted.s3-us-west-1.amazonaws.ziti"],
-			"portRanges" : [ { "low": 443, "high": 443 } ],
-			"protocols": ["tcp"]
-		}`))
+			workflow.AddAction(zitilib_actions.Edge("create", "config", filesConfigName, "intercept.v1", filesConfigDef))
 
-	workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ziti-tunnel", "intercept.v1", `
-		{
-			"addresses": ["ziti-files-ziti-tunnel.s3-us-west-1.amazonaws.ziti"],
-			"portRanges" : [ { "low": 443, "high": 443 } ],
-			"protocols": ["tcp"]
-		}`))
+			iperfConfigName := fmt.Sprintf("iperf-intercept-%s%s", hostType, suffix)
+			iperfConfigDef := fmt.Sprintf(`
+				{
+					"addresses": ["iperf-%s%s.ziti"],
+					"portRanges" : [ { "low": 5201, "high": 5201 } ],
+					"protocols": ["tcp"]
+				}`, hostType, suffix)
 
-	workflow.AddAction(zitilib_actions.Edge("create", "service", "ert-files-unencrypted", "-c", "files-host,files-intercept-ert-unencrypted", "-e", "OFF", "-a", "ert"))
-	workflow.AddAction(zitilib_actions.Edge("create", "service", "ert-files", "-c", "files-host,files-intercept-ert", "-a", "ert"))
+			workflow.AddAction(zitilib_actions.Edge("create", "config", iperfConfigName, "intercept.v1", iperfConfigDef))
 
-	workflow.AddAction(zitilib_actions.Edge("create", "service", "zet-files-unencrypted", "-c", "files-host,files-intercept-zet-unencrypted", "-e", "OFF", "-a", "zet"))
-	workflow.AddAction(zitilib_actions.Edge("create", "service", "zet-files", "-c", "files-host,files-intercept-zet", "-a", "zet"))
+			filesServiceName := fmt.Sprintf("%s-files%s", hostType, suffix)
+			filesConfigs := fmt.Sprintf("files-host,%s", filesConfigName)
+			workflow.AddAction(zitilib_actions.Edge("create", "service", filesServiceName, "-c", filesConfigs, "-e", encryptionFlag, "-a", hostType))
 
-	workflow.AddAction(zitilib_actions.Edge("create", "service", "ziti-tunnel-files-unencrypted", "-c", "files-host,files-intercept-ziti-tunnel-unencrypted", "-e", "OFF", "-a", "ziti-tunnel"))
-	workflow.AddAction(zitilib_actions.Edge("create", "service", "ziti-tunnel-files", "-c", "files-host,files-intercept-ziti-tunnel", "-a", "ziti-tunnel"))
+			iperfServiceName := fmt.Sprintf("%s-iperf%s", hostType, suffix)
+			iperfConfigs := fmt.Sprintf("iperf-host,%s", iperfConfigName)
+			workflow.AddAction(zitilib_actions.Edge("create", "service", iperfServiceName, "-c", iperfConfigs, "-e", encryptionFlag, "-a", hostType))
+		}
+	}
 
 	workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "ert-hosts", "Bind", "--service-roles", "#ert", "--identity-roles", "#ert-host"))
 	workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "zet-hosts", "Bind", "--service-roles", "#zet", "--identity-roles", "#zet-host"))
@@ -127,16 +126,5 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action {
 
 	workflow.AddAction(component.Stop(models.ControllerTag))
 
-	workflow.AddAction(model.ActionFunc(func(m *model.Model) error {
-		cmds := []string{
-			"sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf",
-			"sudo systemctl restart systemd-resolved",
-		}
-
-		return m.ForEachComponent(".ziti-tunnel", 2, func(c *model.Component) error {
-			return host.Exec(c.GetHost(), cmds...).Execute(c.GetModel())
-		})
-	}))
-
 	return workflow
 }

zititest/models/simple/actions/start.go

@@ -17,8 +17,6 @@
 package actions
 
 import (
-	"fmt"
-	"github.com/openziti/fablab/kernel/lib"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
 	"time"
 
@@ -26,7 +24,6 @@ import (
 	"github.com/openziti/fablab/kernel/lib/actions/component"
 	"github.com/openziti/fablab/kernel/lib/actions/semaphore"
 	"github.com/openziti/fablab/kernel/model"
-	util_actions "github.com/openziti/ziti/zititest/utils/actions"
 	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
 	"github.com/openziti/ziti/zititest/zitilab/models"
 )
@@ -44,25 +41,17 @@ func (a *startAction) bind(m *model.Model) model.Action {
 	workflow.AddAction(component.Start("#ctrl"))
 	workflow.AddAction(edge.ControllerAvailable("#ctrl", 30*time.Second))
 	workflow.AddAction(component.StartInParallel(models.EdgeRouterTag, 25))
+	workflow.AddAction(component.StartInParallel(".iperf", 5))
 
 	workflow.AddAction(semaphore.Sleep(2 * time.Second))
 	workflow.AddAction(zitilib_actions.StartMetricbeat("*", a.Metricbeat.ConfigPath, a.Metricbeat.DataPath, a.Metricbeat.LogPath))
 	workflow.AddAction(zitilib_actions.StartConsul("*", a.Consul.ServerAddr, a.Consul.ConfigDir, a.Consul.DataPath, a.Consul.LogPath))
 	workflow.AddAction(semaphore.Sleep(2 * time.Second))
-	workflow.AddAction(util_actions.StartEchoServers("#echo-server"))
-	workflow.AddAction(semaphore.Sleep(2 * time.Second))
+	workflow.AddAction(component.StartInParallel(".sdk-app", 5))
 
-	workflow.AddAction(model.ActionFunc(func(m *model.Model) error {
-		return m.ForEachComponent(".sdk-app", 5, func(c *model.Component) error {
-			factory := lib.NewSshConfigFactory(c.GetHost())
-
-			serviceCmd := fmt.Sprintf("nohup sudo /home/%s/fablab/bin/%s run -i /home/%s/fablab/cfg/%s > logs/%s.log 2>&1 &",
-				factory.User(), c.BinaryName, factory.User(), c.PublicIdentity+".json", c.PublicIdentity)
-
-			_, err := lib.RemoteExec(factory, serviceCmd)
-			return err
-		})
-	}))
+	workflow.AddAction(semaphore.Sleep(5 * time.Second))
+	workflow.AddAction(zitilib_actions.Edge("list", "edge-routers", "limit none"))
+	workflow.AddAction(zitilib_actions.Edge("list", "terminators", "limit none"))
 
 	return workflow
 }

zititest/models/simple/configs/ctrl.yml.tmpl

@@ -3,9 +3,9 @@ v: 3
 db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrl.db
 
 identity:
-  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.PublicIdentity }}-server.cert
-  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/keys/{{ .Component.PublicIdentity }}-server.key
-  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.PublicIdentity }}-server.chain.pem
+  cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.Id }}-server.cert
+  key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/keys/{{ .Component.Id }}-server.key
+  ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.Id }}-server.chain.pem
 
 # the endpoint that routers will connect to the controller over.
 ctrl:

zititest/models/simple/configs/router.yml.tmpl

@@ -1,5 +1,5 @@
 {{$ssh_username := .Model.MustVariable "credentials.ssh.username"}}
-{{$identity := .Component.PublicIdentity}}
+{{$identity := .Component.Id}}
 {{$ctrl_ip := publicIp "component#ctrl"}}
 {{$router_ip := .Host.PublicIp}}
   

zititest/models/simple/simple.go

@@ -25,8 +25,6 @@ import (
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
 	semaphore0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
 	terraform_0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
-	"github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config"
-	"github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit"
 	distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution"
 	"github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync"
 	aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
@@ -34,12 +32,9 @@ import (
 	"github.com/openziti/fablab/kernel/model"
 	"github.com/openziti/fablab/resources"
 	actions2 "github.com/openziti/ziti/zititest/models/simple/actions"
-	"github.com/openziti/ziti/zititest/models/simple/stages/5_operation"
 	"github.com/openziti/ziti/zititest/models/test_resources"
 	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
-	zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration"
-	"github.com/openziti/ziti/zititest/zitilab/stageziti"
 	"github.com/sirupsen/logrus"
 	"os"
 	"time"
@@ -84,10 +79,14 @@ var Model = &model.Model{
 		model.FactoryFunc(func(m *model.Model) error {
 			pfxlog.Logger().Infof("environment [%s]", m.MustStringVariable("environment"))
 			m.AddActivationActions("stop", "bootstrap", "start")
-			m.AddOperatingStage(runlevel_5_operation.AssertEcho("#echo-client"))
-
 			return nil
 		}),
+		model.FactoryFunc(func(m *model.Model) error {
+			return m.ForEachHost("*", 1, func(host *model.Host) error {
+				host.InstanceType = "t2.micro"
+				return nil
+			})
+		}),
 	},
 
 	Resources: model.Resources{
@@ -101,84 +100,51 @@ var Model = &model.Model{
 			Site:   "us-east-1a",
 			Hosts: model.Hosts{
 				"ctrl": {
-					InstanceType: "t2.micro",
 					Components: model.Components{
 						"ctrl": {
-							Scope:          model.Scope{Tags: model.Tags{"ctrl"}},
-							BinaryName:     "ziti controller",
-							ConfigSrc:      "ctrl.yml",
-							ConfigName:     "ctrl.yml",
-							PublicIdentity: "ctrl",
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type:  &zitilab.ControllerType{},
 						},
 					},
 				},
 				"router-east-1": {
-					Scope:        model.Scope{Tags: model.Tags{"ert-client"}},
-					InstanceType: "t2.micro",
+					Scope: model.Scope{Tags: model.Tags{"ert-client"}},
 					Components: model.Components{
 						"router-east-1": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client"}},
-							BinaryName:     "ziti router",
-							ConfigSrc:      "router.yml",
-							ConfigName:     "router-east-1.yml",
-							PublicIdentity: "router-east-1",
-							RunWithSudo:    true,
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client"}},
+							Type:  &zitilab.RouterType{},
 						},
-						"echo-client": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "client"}},
-							BinaryName:     "echo-client",
-							PublicIdentity: "echo-client",
-						},
-						"consul": {
-							BinaryName: "consul",
+						"zcat": {
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}},
+							Type:  &zitilab.ZCatType{},
 						},
 					},
 				},
 				"router-east-2": {
-					InstanceType: "t2.micro",
 					Components: model.Components{
 						"router-east-2": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "initiator"}},
-							BinaryName:     "ziti router",
-							ConfigSrc:      "router.yml",
-							ConfigName:     "router-east-2.yml",
-							PublicIdentity: "router-east-2",
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "initiator"}},
+							Type:  &zitilab.RouterType{},
 						},
 					},
 				},
 				"ziti-edge-tunnel-client": {
-					Scope:        model.Scope{Tags: model.Tags{"zet-client"}},
-					InstanceType: "t2.micro",
+					Scope: model.Scope{Tags: model.Tags{"zet-client"}},
 					Components: model.Components{
 						"ziti-edge-tunnel-client": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "client"}},
-							BinaryName:     "ziti-edge-tunnel",
-							PublicIdentity: "ziti-edge-tunnel-client",
-							RunWithSudo:    true,
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}},
+							Type: &zitilab.ZitiEdgeTunnelType{
+								Version: "v0.21.4",
+							},
 						},
 					},
 				},
 				"ziti-tunnel-client": {
-					Scope:        model.Scope{Tags: model.Tags{"ziti-tunnel-client"}},
-					InstanceType: "t2.micro",
+					Scope: model.Scope{Tags: model.Tags{"ziti-tunnel-client"}},
 					Components: model.Components{
 						"ziti-tunnel-client": {
-							Scope:          model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client"}},
-							BinaryName:     "ziti tunnel",
-							PublicIdentity: "ziti-tunnel-client",
-							RunWithSudo:    true,
-						},
-						"consul": {
-							BinaryName: "consul",
+							Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client"}},
+							Type:  &zitilab.ZitiTunnelType{},
 						},
 					},
 				},
@@ -189,52 +155,46 @@ var Model = &model.Model{
 			Site:   "us-west-2b",
 			Hosts: model.Hosts{
 				"router-west": {
-					Scope:        model.Scope{Tags: model.Tags{}},
-					InstanceType: "t2.micro",
 					Components: model.Components{
 						"router-west": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host"}},
-							BinaryName:     "ziti router",
-							ConfigSrc:      "router.yml",
-							ConfigName:     "router-west.yml",
-							PublicIdentity: "router-west",
-							RunWithSudo:    true,
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host"}},
+							Type:  &zitilab.RouterType{},
 						},
 						"echo-server": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "service"}},
-							BinaryName:     "echo-server",
-							PublicIdentity: "echo-server",
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}},
+							Type:  &zitilab.EchoServerType{},
 						},
-						"consul": {
-							BinaryName: "consul",
+						"iperf-server-ert": {
+							Scope: model.Scope{Tags: model.Tags{"iperf", "service"}},
+							Type:  &zitilab.IPerfServerType{},
 						},
 					},
 				},
 				"ziti-edge-tunnel-host": {
-					InstanceType: "t2.micro",
 					Components: model.Components{
 						"ziti-edge-tunnel-host": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host"}},
-							BinaryName:     "ziti-edge-tunnel",
-							PublicIdentity: "ziti-edge-tunnel-host",
-							RunWithSudo:    true,
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host"}},
+							Type: &zitilab.ZitiEdgeTunnelType{
+								Version: "v0.21.4",
+							},
 						},
-						"consul": {
-							BinaryName: "consul",
+						"iperf-server-zet": {
+							Scope: model.Scope{Tags: model.Tags{"iperf", "service"}},
+							Type:  &zitilab.IPerfServerType{},
 						},
 					},
 				},
 				"ziti-tunnel-host": {
-					InstanceType: "t2.micro",
 					Components: model.Components{
 						"ziti-tunnel-host": {
-							Scope:          model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host"}},
-							BinaryName:     "ziti tunnel",
-							PublicIdentity: "ziti-tunnel-host",
-							RunWithSudo:    true,
+							Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host"}},
+							Type: &zitilab.ZitiTunnelType{
+								Mode: zitilab.ZitiTunnelModeHost,
+							},
 						},
-						"consul": {
-							BinaryName: "consul",
+						"iperf-server-zt": {
+							Scope: model.Scope{Tags: model.Tags{"iperf", "service"}},
+							Type:  &zitilab.IPerfServerType{},
 						},
 					},
 				},
@@ -259,22 +219,17 @@ var Model = &model.Model{
 		"login": model.Bind(edge.Login("#ctrl")),
 	},
 
-	Infrastructure: model.InfrastructureStages{
+	Infrastructure: model.Stages{
 		aws_ssh_key.Express(),
-		terraform_0.Express(),
-		semaphore0.Ready(time.Minute),
+		&terraform_0.Terraform{
+			Retries: 3,
+			ReadyCheck: &semaphore0.ReadyStage{
+				MaxWait: 90 * time.Second,
+			},
+		},
 	},
 
-	Configuration: model.ConfigurationStages{
-		zitilib_runlevel_1_configuration.IfPkiNeedsRefresh(
-			zitilib_runlevel_1_configuration.Fabric("simple-transfer.test", "#ctrl"),
-		),
-		config.Component(),
-		devkit.DevKitF(zitilab.ZitiRoot, []string{"ziti", "ziti-echo"}),
-		stageziti.FetchZitiEdgeTunnel("v0.21.4"),
-	},
-
-	Distribution: model.DistributionStages{
+	Distribution: model.Stages{
 		distribution.DistributeSshKey("*"),
 		distribution.Locations("*", "logs"),
 		distribution.DistributeDataWithReplaceCallbacks(
@@ -341,7 +296,7 @@ var Model = &model.Model{
 		rsync.RsyncStaged(),
 	},
 
-	Disposal: model.DisposalStages{
+	Disposal: model.Stages{
 		terraform.Dispose(),
 		aws_ssh_key2.Dispose(),
 	},

zititest/models/simple/stages/5_operation/echo_client.go

@@ -1,49 +0,0 @@
-package runlevel_5_operation
-
-import (
-	"crypto/rand"
-	"fmt"
-	"net/url"
-	"strings"
-
-	"github.com/openziti/fablab/kernel/lib"
-	"github.com/openziti/fablab/kernel/model"
-	"github.com/sirupsen/logrus"
-)
-
-type echoClient struct {
-	componentSpec string
-	message       string
-}
-
-func AssertEcho(componentSpec string) model.OperatingStage {
-	data := make([]byte, 10000)
-	_, _ = rand.Read(data)
-
-	return &echoClient{
-		componentSpec: componentSpec,
-		message:       url.QueryEscape(string(data)),
-	}
-}
-
-func (ec *echoClient) Operate(run model.Run) error {
-	return run.GetModel().ForEachComponent(ec.componentSpec, 1, func(c *model.Component) error {
-		ssh := lib.NewSshConfigFactory(c.GetHost())
-		remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".json"
-
-		echoClientCmd := fmt.Sprintf("/home/%s/fablab/bin/ziti-echo client --identity %s %s 2>&1",
-			ssh.User(), remoteConfigFile, ec.message)
-
-		if output, err := lib.RemoteExec(ssh, echoClientCmd); err != nil {
-			logrus.Errorf("error starting echo client [%s] (%v)", output, err)
-			return err
-		} else {
-			//trim the newline ssh added
-			output = strings.TrimRight(output, "\n")
-			if output != ec.message {
-				return fmt.Errorf("got message [%s] expected [%s]", output, ec.message)
-			}
-		}
-		return nil
-	})
-}

zititest/models/simple/stages/5_operation/echo_server.go

@@ -1,41 +0,0 @@
-package runlevel_5_operation
-
-import (
-	"fmt"
-
-	"github.com/openziti/fablab/kernel/lib"
-	"github.com/openziti/fablab/kernel/model"
-	"github.com/sirupsen/logrus"
-)
-
-type echoServer struct {
-	joiner chan struct{}
-	host   *model.Host
-}
-
-func EchoServer(host *model.Host, joiner chan struct{}) model.OperatingStage {
-	return &echoServer{
-		joiner: joiner,
-		host:   host,
-	}
-}
-
-func (es *echoServer) Operate(run model.Run) error {
-	go func() {
-		defer func() {
-			if es.joiner != nil {
-				close(es.joiner)
-				logrus.Debug("closed joiner")
-			}
-		}()
-
-		ssh := lib.NewSshConfigFactory(es.host)
-
-		echoServerCmd := fmt.Sprintf("ziti edge tutorial ziti-echo-server")
-
-		if output, err := lib.RemoteExec(ssh, echoServerCmd); err != nil {
-			logrus.Errorf("error starting echo server [%s] (%v)", output, err)
-		}
-	}()
-	return nil
-}

zititest/models/smart/actions/bootstrap.go

@@ -44,7 +44,7 @@ func (self *bootstrapAction) bind(m *model.Model) model.Action {
 	workflow.AddAction(semaphore.Sleep(2 * time.Second))
 
 	for _, router := range m.SelectComponents(models.RouterTag) {
-		cert := fmt.Sprintf("/intermediate/certs/%s-client.cert", router.PublicIdentity)
+		cert := fmt.Sprintf("/intermediate/certs/%s-client.cert", router.Id)
 		workflow.AddAction(zitilib_actions.Fabric("create", "router", filepath.Join(model.PkiBuild(), cert)))
 	}
 
@@ -82,7 +82,7 @@ func (_ *bootstrapAction) createServiceActions(m *model.Model) ([]model.Action,
 
 	for _, h := range hosts {
 		serviceActions = append(serviceActions, zitilib_actions.Fabric("create", "service", h.GetId()))
-		serviceActions = append(serviceActions, zitilib_actions.Fabric("create", "terminator", h.GetId(), router.PublicIdentity, "tcp:"+h.PrivateIp+":8171"))
+		serviceActions = append(serviceActions, zitilib_actions.Fabric("create", "terminator", h.GetId(), router.Id, "tcp:"+h.PrivateIp+":8171"))
 	}
 
 	return serviceActions, nil

zititest/models/smart/main.go

@@ -68,11 +68,8 @@ var smartrouting = &model.Model{
 				"ctrl": {
 					Components: model.Components{
 						"ctrl": {
-							Scope:          model.Scope{Tags: model.Tags{"ctrl"}},
-							BinaryName:     "ziti-controller",
-							ConfigSrc:      "ctrl.yml",
-							ConfigName:     "ctrl.yml",
-							PublicIdentity: "ctrl",
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type:  &zitilab.ControllerType{},
 						},
 					},
 				},
@@ -80,11 +77,10 @@ var smartrouting = &model.Model{
 					Scope: model.Scope{Tags: model.Tags{"iperf_server"}},
 					Components: model.Components{
 						"001": {
-							Scope:          model.Scope{Tags: model.Tags{"initiator", "router"}},
-							BinaryName:     "ziti-router",
-							ConfigSrc:      "ingress_router.yml",
-							ConfigName:     "001.yml",
-							PublicIdentity: "001",
+							Scope: model.Scope{Tags: model.Tags{"initiator", "router"}},
+							Type: &zitilab.RouterType{
+								ConfigSource: "ingress_router.yml.tmpl",
+							},
 						},
 					},
 				},
@@ -110,10 +106,9 @@ var smartrouting = &model.Model{
 					Scope: model.Scope{Tags: model.Tags{"router"}},
 					Components: model.Components{
 						"002": {
-							BinaryName:     "ziti-router",
-							ConfigSrc:      "transit_router.yml",
-							ConfigName:     "002.yml",
-							PublicIdentity: "002",
+							Type: &zitilab.RouterType{
+								ConfigSource: "transit_router.yml.tmpl",
+							},
 						},
 					},
 				},
@@ -126,11 +121,10 @@ var smartrouting = &model.Model{
 				"004": {
 					Components: model.Components{
 						"004": {
-							Scope:          model.Scope{Tags: model.Tags{"router"}},
-							BinaryName:     "ziti-router",
-							ConfigSrc:      "transit_router.yml",
-							ConfigName:     "004.yml",
-							PublicIdentity: "004",
+							Scope: model.Scope{Tags: model.Tags{"router"}},
+							Type: &zitilab.RouterType{
+								ConfigSource: "transit_router.yml.tmpl",
+							},
 						},
 					},
 				},
@@ -143,11 +137,10 @@ var smartrouting = &model.Model{
 				"003": {
 					Components: model.Components{
 						"003": {
-							Scope:          model.Scope{Tags: model.Tags{"router", "terminator"}},
-							BinaryName:     "ziti-router",
-							ConfigSrc:      "egress_router.yml",
-							ConfigName:     "003.yml",
-							PublicIdentity: "003",
+							Scope: model.Scope{Tags: model.Tags{"router", "terminator"}},
+							Type: &zitilab.RouterType{
+								ConfigSource: "egress_router.yml.tmpl",
+							},
 						},
 					},
 				},

zititest/models/smart/stages.go

@@ -28,9 +28,7 @@ import (
 	aws_ssh_keys6 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
 	terraform6 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform"
 	"github.com/openziti/fablab/kernel/model"
-	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/openziti/ziti/zititest/zitilab/models"
-	zitilab_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration"
 	zitilab_5_operation "github.com/openziti/ziti/zititest/zitilab/runlevel/5_operation"
 	"time"
 )
@@ -43,7 +41,7 @@ func newStageFactory() model.Factory {
 // represent the various phases of the model.
 func (self *stageFactory) Build(m *model.Model) error {
 	// set up ssh keys (if configured), create the cloud infrastructure and restart all instances after they've been updated
-	m.Infrastructure = model.InfrastructureStages{
+	m.Infrastructure = model.Stages{
 		aws_ssh_keys0.Express(),
 		terraform0.Express(),
 		semaphore0.Restart(90 * time.Second),
@@ -55,20 +53,17 @@ func (self *stageFactory) Build(m *model.Model) error {
 	// 2. component configurations, which includes controller and router configurations
 	// 3. loop tester configurations
 	// 4. The actual binaries, including the controller, router, ziti-fabric CLI and the ziti-fabric-test tool
-	m.Configuration = model.ConfigurationStages{
-		zitilab_runlevel_1_configuration.IfNoPki(zitilab_runlevel_1_configuration.Fabric("smart.test")),
-		config.Component(),
+	m.Configuration = model.Stages{
 		config.Static([]config.StaticConfig{
 			{Src: "10-ambient.loop2.yml", Name: "10-ambient.loop2.yml"},
 			{Src: "4k-chatter.loop2.yml", Name: "4k-chatter.loop2.yml"},
 			{Src: "remote_identities.yml", Name: "remote_identities.yml"},
 		}),
-		zitilab.DefaultZitiBinaries(),
 	}
 
 	// Create log directories on hosts that need them
 	// Push the artifacts gather in the configuration stage up to the hosts
-	m.Distribution = model.DistributionStages{
+	m.Distribution = model.Stages{
 		distribution.Locations(models.HasControllerComponent, "logs"),
 		distribution.Locations(models.HasRouterComponent, "logs"),
 		distribution.Locations(models.LoopListenerTag, "logs"),
@@ -94,7 +89,7 @@ func (self *stageFactory) Build(m *model.Model) error {
 	}
 
 	// finally, dispose of the instances and remove any added keys
-	m.Disposal = model.DisposalStages{
+	m.Disposal = model.Stages{
 		terraform6.Dispose(),
 		aws_ssh_keys6.Dispose(),
 	}

zititest/models/stall/main.go

@@ -11,7 +11,6 @@ import (
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key"
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore"
 	"github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform"
-	"github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config"
 	distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution"
 	"github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync"
 	aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key"
@@ -20,7 +19,6 @@ import (
 	"github.com/openziti/fablab/resources"
 	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/openziti/ziti/zititest/zitilab/actions/edge"
-	zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration"
 	"os"
 	"time"
 )
@@ -83,11 +81,8 @@ var m = &model.Model{
 					InstanceType: "c5.large",
 					Components: model.Components{
 						"ctrl": {
-							Scope:          model.Scope{Tags: model.Tags{"ctrl"}},
-							BinaryName:     "ziti-controller",
-							ConfigSrc:      "ctrl.yml",
-							ConfigName:     "ctrl.yml",
-							PublicIdentity: "ctrl",
+							Scope: model.Scope{Tags: model.Tags{"ctrl"}},
+							Type:  &zitilab.ControllerType{},
 						},
 					},
 				},
@@ -95,11 +90,8 @@ var m = &model.Model{
 					InstanceType: "c5.large",
 					Components: model.Components{
 						"metrics-router": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "no-traversal"}},
-							BinaryName:     "ziti-router",
-							ConfigSrc:      "router.yml",
-							ConfigName:     "metrics-router.yml",
-							PublicIdentity: "metrics-router",
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "no-traversal"}},
+							Type:  &zitilab.RouterType{},
 						},
 					},
 				},
@@ -114,17 +106,13 @@ var m = &model.Model{
 					InstanceType: "c5.large",
 					Components: model.Components{
 						"router-west-{{ .Host.ScaleIndex }}": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "tunneler", "terminator"}},
-							BinaryName:     "ziti-router",
-							ConfigSrc:      "router.yml",
-							ConfigName:     "router-west-{{ .Host.ScaleIndex }}.yml",
-							PublicIdentity: "router-west-{{ .Host.ScaleIndex }}",
-							RunWithSudo:    true,
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "terminator"}},
+							Type:  &zitilab.RouterType{},
 						},
 						"loop-listener-{{ .Host.ScaleIndex }}": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "service"}},
-							BinaryName:     "ziti-fabric-test",
-							PublicIdentity: "loop-listener-{{ .Host.ScaleIndex }}",
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}},
+							// BinaryName:     "ziti-fabric-test",
+							//PublicIdentity: "loop-listener-{{ .Host.ScaleIndex }}",
 						},
 					},
 				},
@@ -139,19 +127,15 @@ var m = &model.Model{
 					InstanceType: "c5.large",
 					Components: model.Components{
 						"router-ap-{{ .Host.ScaleIndex }}": {
-							Scope:          model.Scope{Tags: model.Tags{"edge-router", "tunneler", "initiator"}},
-							BinaryName:     "ziti-router",
-							ConfigSrc:      "router.yml",
-							ConfigName:     "router-ap-{{ .Host.ScaleIndex }}.yml",
-							PublicIdentity: "router-ap-{{ .Host.ScaleIndex }}",
-							RunWithSudo:    true,
+							Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "initiator"}},
+							Type:  &zitilab.RouterType{},
 						},
 						"loop-client-{{ .Host.ScaleIndex }}": {
-							Scope:          model.Scope{Tags: model.Tags{"sdk-app", "client"}},
-							BinaryName:     "ziti-fabric-test",
-							ConfigSrc:      "test.loop3.yml",
-							ConfigName:     "test.loop3.yml",
-							PublicIdentity: "loop-client-{{ .Host.ScaleIndex }}",
+							Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}},
+							//BinaryName:     "ziti-fabric-test",
+							//ConfigSrc:      "test.loop3.yml",
+							//ConfigName:     "test.loop3.yml",
+							//PublicIdentity: "loop-client-{{ .Host.ScaleIndex }}",
 						},
 					},
 				},
@@ -172,27 +156,19 @@ var m = &model.Model{
 		"login": model.Bind(edge.Login("#ctrl")),
 	},
 
-	Infrastructure: model.InfrastructureStages{
+	Infrastructure: model.Stages{
 		aws_ssh_key.Express(),
 		terraform_0.Express(),
 		semaphore_0.Restart(90 * time.Second),
 	},
 
-	Configuration: model.ConfigurationStages{
-		zitilib_runlevel_1_configuration.IfPkiNeedsRefresh(
-			zitilib_runlevel_1_configuration.Fabric("stall.test"),
-		),
-		config.Component(),
-		zitilab.DefaultZitiBinaries(),
-	},
-
-	Distribution: model.DistributionStages{
+	Distribution: model.Stages{
 		distribution.DistributeSshKey("*"),
 		distribution.Locations("*", "logs"),
 		rsync.RsyncStaged(),
 	},
 
-	Disposal: model.DisposalStages{
+	Disposal: model.Stages{
 		terraform.Dispose(),
 		aws_ssh_key2.Dispose(),
 	},

zititest/models/stall/stages.go

@@ -73,7 +73,7 @@ func (_ *stageFactory) listeners(m *model.Model) error {
 	// only start 1 listener
 	c := components[0]
 
-	remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.PublicIdentity)
+	remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.Id)
 	stage := zitilib_5_operation.Loop3Listener(c.GetHost(), nil, "tcp:0.0.0.0:8171",
 		"--config-file", remoteConfigFile, "--health-check-addr 127.0.0.1:8172")
 	m.AddOperatingStage(stage)
@@ -91,8 +91,8 @@ func (_ *stageFactory) dialers(m *model.Model, phase fablib_5_operation.Phase) e
 	// only start 1 dialer
 	c := components[0]
 
-	remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.PublicIdentity)
-	stage := zitilib_5_operation.Loop3Dialer(c.GetHost(), c.ConfigName, "tcp:test.service:8171", phase.AddJoiner(), "--config-file", remoteConfigFile, "-d")
+	remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.Id)
+	stage := zitilib_5_operation.Loop3Dialer(c.GetHost(), c.Id, "tcp:test.service:8171", phase.AddJoiner(), "--config-file", remoteConfigFile, "-d")
 	m.AddOperatingStage(stage)
 
 	return nil

zititest/tests/echo_test.go

@@ -0,0 +1,52 @@
+/*
+	(c) Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package tests
+
+import (
+	"fmt"
+	"github.com/google/uuid"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/stretchr/testify/require"
+	"strings"
+	"testing"
+)
+
+func TestSdkEcho(t *testing.T) {
+	components := run.GetModel().SelectComponents("#zcat")
+
+	req := require.New(t)
+	req.True(len(components) > 0)
+
+	data := ""
+	for i := 0; i < 1000; i++ {
+		data += uuid.NewString()
+	}
+
+	for _, c := range components {
+		ssh := lib.NewSshConfigFactory(c.GetHost())
+		remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.Id + ".json"
+
+		echoClientCmd := fmt.Sprintf(`echo "%s" | /home/%s/fablab/bin/ziti learn demo zcat --identity %s ziti:echo 2>&1`,
+			string(data), ssh.User(), remoteConfigFile)
+
+		output, err := lib.RemoteExec(ssh, echoClientCmd)
+		req.NoError(err)
+		//trim the newline ssh added
+		output = strings.TrimRight(output, "\n")
+		req.Equal(string(data), output)
+	}
+}

zititest/tests/files_test.go

@@ -32,8 +32,8 @@ var hashes = map[string]string{
 }
 
 var timeouts = map[string]time.Duration{
-	"1KB":   5 * time.Second,
-	"100KB": 5 * time.Second,
+	"1KB":   10 * time.Second,
+	"100KB": 10 * time.Second,
 	"20MB":  40 * time.Second,
 }
 
@@ -44,38 +44,69 @@ const (
 	ClientWget httpClient = "wget"
 )
 
-func TestCurlFiles(t *testing.T) {
+func TestDownloadFiles(t *testing.T) {
 	allZetHostedFailed := true
-	for _, hostType := range []string{"ert", "zet", "ziti-tunnel"} {
-		for _, clientType := range []string{"ert", "zet", "ziti-tunnel"} {
-			for _, encrypted := range []bool{true, false} {
-				success := testFileDownload(t, clientType, ClientCurl, hostType, encrypted, "1KB")
-				if hostType == "zet" && success {
-					allZetHostedFailed = false
-				}
-			}
-		}
-	}
+	allZetClientsFailed := true
 
-	for _, size := range []string{"100KB", "20MB"} {
-		for _, clientType := range []string{"ert", "zet"} {
-			for _, hostType := range []string{"ert", "zet"} {
-				for _, client := range []httpClient{ClientCurl, ClientWget} {
-					for _, encrypted := range []bool{true, false} {
-						success := testFileDownload(t, clientType, client, hostType, encrypted, size)
-						if hostType == "zet" && success {
-							allZetHostedFailed = false
+	t.Run("download-tests", func(t *testing.T) {
+		t.Run("test-ert-downloads", func(t *testing.T) {
+			t.Parallel()
+
+			for _, size := range []string{"1KB", "100KB", "20MB"} {
+				for _, hostType := range []string{"ert", "zet", "ziti-tunnel"} {
+					for _, client := range []httpClient{ClientCurl, ClientWget} {
+						for _, encrypted := range []bool{true, false} {
+							success := testFileDownload(t, "ert", client, hostType, encrypted, size)
+							if hostType == "zet" && success {
+								allZetHostedFailed = false
+							}
 						}
 					}
 				}
 			}
-		}
-	}
+		})
 
-	testFileDownload(t, "ziti-tunnel", ClientCurl, "ziti-tunnel", true, "20MB")
+		t.Run("test-zet-downloads", func(t *testing.T) {
+			t.Parallel()
+
+			for _, size := range []string{"1KB", "100KB", "20MB"} {
+				for _, hostType := range []string{"zet", "ziti-tunnel", "ert"} {
+					for _, client := range []httpClient{ClientCurl, ClientWget} {
+						for _, encrypted := range []bool{true, false} {
+							success := testFileDownload(t, "zet", client, hostType, encrypted, size)
+							if hostType == "zet" && success {
+								allZetHostedFailed = false
+							}
+							if success {
+								allZetClientsFailed = false
+							}
+						}
+					}
+				}
+			}
+		})
+
+		t.Run("test-ziti-tunnel-downloads", func(t *testing.T) {
+			t.Parallel()
+
+			for _, size := range []string{"1KB", "100KB", "20MB"} {
+				for _, hostType := range []string{"ziti-tunnel", "ert", "zet"} {
+					for _, client := range []httpClient{ClientCurl, ClientWget} {
+						for _, encrypted := range []bool{true, false} {
+							success := testFileDownload(t, "ziti-tunnel", client, hostType, encrypted, size)
+							if hostType == "zet" && success {
+								allZetHostedFailed = false
+							}
+						}
+					}
+				}
+			}
+		})
+	})
 
 	req := require.New(t)
 	req.False(allZetHostedFailed, "all zet hosted file transfer should not failed, indicates bigger issue")
+	req.False(allZetClientsFailed, "all zet client file transfers should not failed, indicates bigger issue")
 }
 
 func testFileDownload(t *testing.T, hostSelector string, client httpClient, hostType string, encrypted bool, fileSize string) bool {
@@ -96,7 +127,7 @@ func testFileDownload(t *testing.T, hostSelector string, client httpClient, host
 			urlExtra = "-unencrypted"
 		}
 
-		url := fmt.Sprintf("https://ziti-files-%s%s.s3-us-west-1.amazonaws.ziti/%s.zip", hostType, urlExtra, fileSize)
+		url := fmt.Sprintf("https://files-%s%s.s3-us-west-1.amazonaws.ziti/%s.zip", hostType, urlExtra, fileSize)
 		sshConfigFactory := lib.NewSshConfigFactory(host)
 
 		var cmd string
@@ -112,6 +143,13 @@ func testFileDownload(t *testing.T, hostSelector string, client httpClient, host
 			t.Skipf("zet hosted file transfer failed [%v]", err.Error())
 			return
 		}
+
+		if hostSelector == "zet" && err != nil {
+			t.Skipf("zet client file transfer failed [%v]", err.Error())
+			return
+		}
+
+		t.Log(o)
 		req.NoError(err)
 		req.Equal(hashes[fileSize], o[0:32])
 		success = true

zititest/tests/iperf_test.go

@@ -0,0 +1,127 @@
+/*
+	(c) Copyright NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package tests
+
+import (
+	"fmt"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/stretchr/testify/require"
+	"testing"
+	"time"
+)
+
+func TestIPerf(t *testing.T) {
+	t.Run("iperf-tests", func(t *testing.T) {
+		t.Run("ert-hosted", func(t *testing.T) {
+			t.Parallel()
+
+			for _, clientType := range []string{"ert", "zet", "ziti-tunnel"} {
+				for _, encrypted := range []bool{true, false} {
+					for _, reversed := range []bool{true, false} {
+						testIPerf(t, clientType, "ert", encrypted, reversed)
+					}
+				}
+			}
+		})
+
+		t.Run("zet-hosted", func(t *testing.T) {
+			t.Parallel()
+
+			for _, clientType := range []string{"zet", "ziti-tunnel", "ert"} {
+				for _, encrypted := range []bool{true, false} {
+					for _, reversed := range []bool{true, false} {
+						testIPerf(t, clientType, "zet", encrypted, reversed)
+					}
+				}
+			}
+		})
+
+		t.Run("ziti-tunnel-hosted", func(t *testing.T) {
+			t.Parallel()
+
+			for _, clientType := range []string{"ziti-tunnel", "ert", "zet"} {
+				for _, encrypted := range []bool{true, false} {
+					for _, reversed := range []bool{true, false} {
+						testIPerf(t, clientType, "ziti-tunnel", encrypted, reversed)
+					}
+				}
+			}
+		})
+	})
+}
+
+func testIPerf(t *testing.T, hostSelector string, hostType string, encrypted bool, reversed bool) bool {
+	encDesk := "encrypted"
+	if !encrypted {
+		encDesk = "unencrypted"
+	}
+
+	direction := "->"
+	if reversed {
+		direction = "<-"
+	}
+
+	success := false
+
+	t.Run(fmt.Sprintf("(%s%s%s)-%v", hostSelector, direction, hostType, encDesk), func(t *testing.T) {
+		host, err := model.GetModel().SelectHost("." + hostSelector + "-client")
+		req := require.New(t)
+		req.NoError(err)
+
+		urlExtra := ""
+		if !encrypted {
+			urlExtra = "-unencrypted"
+		}
+
+		addr := fmt.Sprintf("iperf-%s%s.ziti", hostType, urlExtra)
+
+		extraOptions := ""
+		if reversed {
+			extraOptions += " -R"
+		}
+
+		cmd := fmt.Sprintf(`set -o pipefail; iperf3 -c %s -P 1 -t 10 %s`, addr, extraOptions)
+
+		sshConfigFactory := lib.NewSshConfigFactory(host)
+		o, err := lib.RemoteExecAllWithTimeout(sshConfigFactory, 20*time.Second, cmd)
+		if hostType == "zet" && err != nil {
+			t.Skipf("zet hosted iperf test failed [%v]", err.Error())
+			return
+		}
+
+		if hostType == "ziti-tunnel" && err != nil {
+			t.Skipf("ziti-tunnel hosted iperf test failed [%v]", err.Error())
+			return
+		}
+
+		if hostSelector == "zet" && err != nil {
+			t.Skipf("zet client iperf test failed [%v]", err.Error())
+			return
+		}
+
+		if hostSelector == "ziti-tunnel" && err != nil {
+			t.Skipf("ziti-tunnel client iperf test failed [%v]", err.Error())
+			return
+		}
+
+		t.Log(o)
+		req.NoError(err)
+		success = true
+	})
+	return success
+}

zititest/tests/model_test.go

@@ -44,5 +44,9 @@ func init() {
 		logrus.Fatalf("unable to bootstrap (%s)", err)
 	}
 
-	run = model.NewRun()
+	var err error
+	run, err = model.NewRun()
+	if err != nil {
+		logrus.WithError(err).Fatal("error initializing run")
+	}
 }

zititest/utils/actions/echoServer.go

@@ -1,36 +0,0 @@
-package actions
-
-import (
-	"fmt"
-
-	"github.com/openziti/fablab/kernel/lib"
-	"github.com/openziti/fablab/kernel/model"
-	"github.com/sirupsen/logrus"
-)
-
-type echoServerStart struct {
-	componentSpec string
-}
-
-func StartEchoServers(componentSpec string) model.Action {
-	return &echoServerStart{
-		componentSpec: componentSpec,
-	}
-}
-
-func (esi *echoServerStart) Execute(m *model.Model) error {
-	return m.ForEachComponent(esi.componentSpec, 1, func(c *model.Component) error {
-		ssh := lib.NewSshConfigFactory(c.GetHost())
-		remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".json"
-
-		echoServerCmd := fmt.Sprintf("nohup /home/%s/fablab/bin/ziti-echo server --identity %s > /home/ubuntu/logs/ziti-echo.log 2>&1 &",
-			ssh.User(), remoteConfigFile)
-
-		if output, err := lib.RemoteExec(ssh, echoServerCmd); err != nil {
-			logrus.Errorf("error starting echo server [%s] (%v)", output, err)
-			return err
-		}
-		logrus.Info("echo server started")
-		return nil
-	})
-}

zititest/utils/ziti-echo/cmd/client.go

@@ -1,73 +0,0 @@
-package cmd
-
-import (
-	"context"
-	"fmt"
-	"io"
-	"log"
-	"net"
-	"net/http"
-	"net/url"
-	"strings"
-
-	"github.com/openziti/sdk-golang/ziti"
-	"github.com/spf13/cobra"
-)
-
-var clientCmd = &cobra.Command{
-	Use:   "client {string-to-echo}",
-	Short: "Simple ziti enabled echo client",
-	Args:  cobra.MinimumNArgs(1),
-	Run:   echoClient,
-}
-
-func init() {
-	clientCmd.Flags().SetInterspersed(true)
-	rootCmd.AddCommand(clientCmd)
-}
-
-func echoClient(cmd *cobra.Command, args []string) {
-	zitiContext, err := ziti.NewContextFromFile(identityFile)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	dial := func(_ context.Context, _, addr string) (net.Conn, error) {
-		service := strings.Split(addr, ":")[0]
-		return zitiContext.Dial(service)
-	}
-
-	zitiTransport := http.DefaultTransport.(*http.Transport).Clone()
-	zitiTransport.DialContext = dial
-
-	zec := &zitiEchoClient{
-		client: &http.Client{Transport: zitiTransport},
-	}
-
-	resp, err := zec.Echo(strings.Join(args, " "))
-	if resp != "" {
-		fmt.Print(resp)
-	}
-	if err != nil {
-		log.Fatal(err)
-	}
-}
-
-type zitiEchoClient struct {
-	client *http.Client
-}
-
-func (zec *zitiEchoClient) Echo(input string) (string, error) {
-	u := fmt.Sprintf("http://echo?input=%v", url.QueryEscape(input))
-	resp, err := zec.client.Get(u)
-	if err != nil {
-		return "", err
-	}
-	defer resp.Body.Close()
-
-	b, err := io.ReadAll(resp.Body)
-	if err != nil {
-		return "", err
-	}
-	return string(b), err
-}

zititest/utils/ziti-echo/cmd/root.go

@@ -1,26 +0,0 @@
-package cmd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/spf13/cobra"
-)
-
-var identityFile string
-
-var rootCmd = &cobra.Command{
-	Use:   "ziti-echo",
-	Short: "A simple Echo Service",
-}
-
-func init() {
-	rootCmd.PersistentFlags().StringVar(&identityFile, "identity", "i", "ziti identity file")
-}
-
-func Execute() {
-	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}

zititest/utils/ziti-echo/cmd/server.go

@@ -1,82 +0,0 @@
-package cmd
-
-import (
-	"log"
-	"net"
-	"net/http"
-	"os"
-	"os/signal"
-	"syscall"
-	"time"
-
-	"github.com/openziti/sdk-golang/ziti"
-	"github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-var serverCmd = &cobra.Command{
-	Use:   "server",
-	Short: "Simple Ziti enabled echo serer",
-	Run:   runServer,
-}
-
-func init() {
-	rootCmd.AddCommand(serverCmd)
-}
-
-func runServer(cmd *cobra.Command, args []string) {
-	sigs := make(chan os.Signal, 1)
-	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
-
-	echoServer := &zitiEchoServer{
-		identityJson: identityFile,
-	}
-
-	if err := echoServer.run(); err != nil {
-		log.Fatal(err)
-	}
-
-	for {
-		select {
-		case <-time.After(10 * time.Second):
-			logrus.Debug("Beating to show server still running")
-		case <-sigs:
-			logrus.Info("Server shutting down...")
-			return
-		}
-	}
-}
-
-type zitiEchoServer struct {
-	identityJson string
-	listener     net.Listener
-}
-
-func (s *zitiEchoServer) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
-	input := r.URL.Query().Get("input")
-	result := input
-	logrus.Infof("\nziti-http-echo-server: ")
-	logrus.Infof("received input '%v'\n", input)
-	if _, err := rw.Write([]byte(result)); err != nil {
-		panic(err)
-	}
-}
-
-func (s *zitiEchoServer) run() (err error) {
-	zitiContext, err := ziti.NewContextFromFile(s.identityJson)
-	if err != nil {
-		return err
-	}
-	if s.listener, err = zitiContext.Listen("echo"); err != nil {
-		return err
-	}
-
-	logrus.Info("\nziti-http-echo-server: ")
-	logrus.Info("listening for connections from echo server")
-	go func() { _ = http.Serve(s.listener, http.HandlerFunc(s.ServeHTTP)) }()
-	return nil
-}
-
-func (s *zitiEchoServer) stop() error {
-	return s.listener.Close()
-}

zititest/utils/ziti-echo/main.go

@@ -1,9 +0,0 @@
-package main
-
-import (
-	"github.com/openziti/ziti/zititest/utils/ziti-echo/cmd"
-)
-
-func main() {
-	cmd.Execute()
-}

zititest/zitilab/actions/consul.go

@@ -26,8 +26,8 @@ func StartConsul(hostSpec, consulServer, configDir, dataPath, logFile string) mo
 	}
 }
 
-func (cs *consulStart) Execute(m *model.Model) error {
-	return m.ForEachHost(cs.hostSpec, 24, func(c *model.Host) error {
+func (cs *consulStart) Execute(run model.Run) error {
+	return run.GetModel().ForEachHost(cs.hostSpec, 24, func(c *model.Host) error {
 		ssh := lib.NewSshConfigFactory(c)
 
 		cmd := fmt.Sprintf("screen -d -m nohup consul agent -join %s -config-dir %s -data-dir %s -log-file %s 2>&1 &", cs.consulServer, cs.configDir, cs.dataPath, cs.logFile)

zititest/zitilab/actions/edge/ctrl_available.go

@@ -14,8 +14,8 @@ func ControllerAvailable(componentSpec string, timeout time.Duration) model.Acti
 	}
 }
 
-func (self *edgeAvailable) Execute(m *model.Model) error {
-	for _, c := range m.SelectComponents(self.componentSpec) {
+func (self *edgeAvailable) Execute(run model.Run) error {
+	for _, c := range run.GetModel().SelectComponents(self.componentSpec) {
 		if err := netz.WaitForPortActive(c.Host.PublicIp+":1280", self.timeout); err != nil {
 			return errors.Wrap(err, "controller didn't start in time")
 		}

zititest/zitilab/actions/edge/ctrl_init.go

@@ -3,8 +3,10 @@ package edge
 import (
 	"fmt"
 	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/lib/actions/component"
 	"github.com/openziti/fablab/kernel/lib/actions/host"
 	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/pkg/errors"
 )
 
@@ -14,28 +16,8 @@ func InitController(componentSpec string) model.Action {
 	}
 }
 
-func (init *edgeInit) Execute(m *model.Model) error {
-	username := m.MustStringVariable("credentials.edge.username")
-	password := m.MustStringVariable("credentials.edge.password")
-
-	if username == "" {
-		return errors.New("variable credentials/edge/username must be a string")
-	}
-
-	if password == "" {
-		return errors.New("variable credentials/edge/password must be a string")
-	}
-
-	for _, c := range m.SelectComponents(init.componentSpec) {
-		sshConfigFactory := lib.NewSshConfigFactory(c.GetHost())
-
-		tmpl := "rm -f /home/%v/fablab/ctrl.db && set -o pipefail; /home/%s/fablab/bin/%s --log-formatter pfxlog edge init /home/%s/fablab/cfg/%s -u %s -p %s 2>&1 | tee logs/%s.edge.init.log"
-		if err := host.Exec(c.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), sshConfigFactory.User(), c.BinaryName, sshConfigFactory.User(), c.ConfigName, username, password, c.BinaryName)).Execute(m); err != nil {
-			return err
-		}
-	}
-
-	return nil
+func (init *edgeInit) Execute(run model.Run) error {
+	return component.Exec(init.componentSpec, zitilab.ControllerActionInitStandalone).Execute(run)
 }
 
 type edgeInit struct {
@@ -48,7 +30,8 @@ func InitRaftController(componentSpec string) model.Action {
 	}
 }
 
-func (init *raftInit) Execute(m *model.Model) error {
+func (init *raftInit) Execute(run model.Run) error {
+	m := run.GetModel()
 	username := m.MustStringVariable("credentials.edge.username")
 	password := m.MustStringVariable("credentials.edge.password")
 
@@ -64,7 +47,7 @@ func (init *raftInit) Execute(m *model.Model) error {
 		sshConfigFactory := lib.NewSshConfigFactory(c.GetHost())
 
 		tmpl := "set -o pipefail; /home/%s/fablab/bin/ziti agent controller init %s %s default.admin 2>&1 | tee logs/controller.edge.init.log"
-		if err := host.Exec(c.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), username, password)).Execute(m); err != nil {
+		if err := host.Exec(c.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), username, password)).Execute(run); err != nil {
 			return err
 		}
 	}

zititest/zitilab/actions/edge/init_identities.go

@@ -16,22 +16,22 @@ func InitIdentities(componentSpec string, concurrency int) model.Action {
 	}
 }
 
-func (action *initIdentitiesAction) Execute(m *model.Model) error {
-	return m.ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error {
-		if err := zitilib_actions.EdgeExec(m, "delete", "identity", c.PublicIdentity); err != nil {
+func (action *initIdentitiesAction) Execute(run model.Run) error {
+	return run.GetModel().ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error {
+		if err := zitilib_actions.EdgeExec(run.GetModel(), "delete", "identity", c.Id); err != nil {
 			return err
 		}
 
-		return action.createAndEnrollIdentity(c)
+		return action.createAndEnrollIdentity(run, c)
 	})
 }
 
-func (action *initIdentitiesAction) createAndEnrollIdentity(c *model.Component) error {
+func (action *initIdentitiesAction) createAndEnrollIdentity(run model.Run, c *model.Component) error {
 	ssh := lib.NewSshConfigFactory(c.GetHost())
 
-	jwtFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".jwt")
+	jwtFileName := filepath.Join(run.GetTmpDir(), c.Id+".jwt")
 
-	err := zitilib_actions.EdgeExec(c.GetModel(), "create", "identity", "service", c.PublicIdentity,
+	err := zitilib_actions.EdgeExec(c.GetModel(), "create", "identity", "service", c.Id,
 		"--jwt-output-file", jwtFileName,
 		"-a", strings.Join(c.Tags, ","))
 
@@ -39,7 +39,7 @@ func (action *initIdentitiesAction) createAndEnrollIdentity(c *model.Component)
 		return err
 	}
 
-	configFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".json")
+	configFileName := filepath.Join(run.GetTmpDir(), c.Id+".json")
 
 	_, err = cli.Exec(c.GetModel(), "edge", "enroll", "--jwt", jwtFileName, "--out", configFileName)
 
@@ -47,7 +47,7 @@ func (action *initIdentitiesAction) createAndEnrollIdentity(c *model.Component)
 		return err
 	}
 
-	remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".json"
+	remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.Id + ".json"
 	return lib.SendFile(ssh, configFileName, remoteConfigFile)
 }
 

zititest/zitilab/actions/edge/init_routers.go

@@ -1,14 +1,9 @@
 package edge
 
 import (
-	"fmt"
-	"github.com/michaelquigley/pfxlog"
-	"github.com/openziti/fablab/kernel/lib"
-	"github.com/openziti/fablab/kernel/lib/actions/host"
+	"github.com/openziti/fablab/kernel/lib/actions/component"
 	"github.com/openziti/fablab/kernel/model"
-	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
-	"path/filepath"
-	"strings"
+	"github.com/openziti/ziti/zititest/zitilab"
 )
 
 func InitEdgeRouters(componentSpec string, concurrency int) model.Action {
@@ -18,65 +13,8 @@ func InitEdgeRouters(componentSpec string, concurrency int) model.Action {
 	}
 }
 
-func (action *initEdgeRoutersAction) Execute(m *model.Model) error {
-	return m.ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error {
-		if err := zitilib_actions.EdgeExec(m, "delete", "edge-router", c.PublicIdentity); err != nil {
-			pfxlog.Logger().
-				WithError(err).
-				WithField("router", c.PublicIdentity).
-				Warn("unable to delete router (may not be present")
-		}
-
-		return action.createAndEnrollRouter(c)
-	})
-}
-
-func (action *initEdgeRoutersAction) createAndEnrollRouter(c *model.Component) error {
-	ssh := lib.NewSshConfigFactory(c.GetHost())
-
-	jwtFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".jwt")
-
-	attributes := strings.Join(c.Tags, ",")
-
-	args := []string{"create", "edge-router", c.PublicIdentity, "-j", "--jwt-output-file", jwtFileName, "-a", attributes}
-
-	isTunneler := c.HasLocalOrAncestralTag("tunneler")
-	if isTunneler {
-		args = append(args, "--tunneler-enabled")
-	}
-
-	if c.HasLocalOrAncestralTag("no-traversal") {
-		args = append(args, "--no-traversal")
-	}
-
-	if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil {
-		return err
-	}
-
-	if isTunneler {
-		if err := zitilib_actions.EdgeExec(c.GetModel(), "update", "identity", c.PublicIdentity, "-a", attributes); err != nil {
-			return err
-		}
-	}
-
-	remoteJwt := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".jwt"
-	if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil {
-		return err
-	}
-
-	tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%s enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log "
-	cmds := []string{
-		"mkdir -p /home/ubuntu/logs",
-		fmt.Sprintf(tmpl, c.BinaryName, c.ConfigName, remoteJwt, c.ConfigName),
-	}
-
-	if isTunneler {
-		cmds = append(cmds,
-			"sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf",
-			"sudo systemctl restart systemd-resolved")
-	}
-
-	return host.Exec(c.GetHost(), cmds...).Execute(c.GetModel())
+func (action *initEdgeRoutersAction) Execute(run model.Run) error {
+	return component.ExecInParallel(action.componentSpec, action.concurrency, zitilab.RouterActionsCreateAndEnroll).Execute(run)
 }
 
 type initEdgeRoutersAction struct {

zititest/zitilab/actions/edge/login.go

@@ -14,7 +14,8 @@ func Login(componentSelector string) model.Action {
 	}
 }
 
-func (l *login) Execute(m *model.Model) error {
+func (l *login) Execute(run model.Run) error {
+	m := run.GetModel()
 	ctrl, err := m.SelectComponent(l.componentSelector)
 	if err != nil {
 		return err
@@ -24,7 +25,7 @@ func (l *login) Execute(m *model.Model) error {
 	password := m.MustStringVariable("credentials.edge.password")
 	edgeApiBaseUrl := ctrl.Host.PublicIp + ":1280"
 
-	caChain := filepath.Join(model.PkiBuild(), ctrl.PublicIdentity, "certs", ctrl.PublicIdentity+".cert")
+	caChain := filepath.Join(model.KitBuild(), model.BuildPkiDir, ctrl.Id, "certs", ctrl.Id+".cert")
 
 	if username == "" {
 		return errors.New("variable credentials/edge/username must be a string")

zititest/zitilab/actions/edge/raft_join.go

@@ -18,8 +18,8 @@ type raftJoin struct {
 	componentSpec string
 }
 
-func (self *raftJoin) Execute(m *model.Model) error {
-	ctrls := m.SelectComponents(self.componentSpec)
+func (self *raftJoin) Execute(run model.Run) error {
+	ctrls := run.GetModel().SelectComponents(self.componentSpec)
 	if len(ctrls) < 1 {
 		return errors.Errorf("no controllers found with spec '%v'", self.componentSpec)
 	}
@@ -27,7 +27,7 @@ func (self *raftJoin) Execute(m *model.Model) error {
 	sshConfigFactory := lib.NewSshConfigFactory(primary.GetHost())
 	for _, c := range ctrls[1:] {
 		tmpl := "/home/%s/fablab/bin/ziti agent cluster add %v --id %v"
-		if err := host.Exec(primary.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), "tls:"+c.Host.PublicIp+":6262", c.PublicIdentity)).Execute(m); err != nil {
+		if err := host.Exec(primary.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), "tls:"+c.Host.PublicIp+":6262", c.Id)).Execute(run); err != nil {
 			return err
 		}
 	}

zititest/zitilab/actions/edge/reenroll_routers.go

@@ -1,12 +1,9 @@
 package edge
 
 import (
-	"fmt"
-	"github.com/openziti/fablab/kernel/lib"
-	"github.com/openziti/fablab/kernel/lib/actions/host"
+	"github.com/openziti/fablab/kernel/lib/actions/component"
 	"github.com/openziti/fablab/kernel/model"
-	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
-	"path/filepath"
+	"github.com/openziti/ziti/zititest/zitilab"
 )
 
 func ReEnrollEdgeRouters(componentSpec string, concurrency int) model.Action {
@@ -16,41 +13,8 @@ func ReEnrollEdgeRouters(componentSpec string, concurrency int) model.Action {
 	}
 }
 
-func (action *reEnrollEdgeRoutersAction) Execute(m *model.Model) error {
-	return m.ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error {
-		return action.reEnrollRouter(c)
-	})
-}
-
-func (action *reEnrollEdgeRoutersAction) reEnrollRouter(c *model.Component) error {
-	ssh := lib.NewSshConfigFactory(c.GetHost())
-
-	jwtFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".jwt")
-
-	args := []string{"re-enroll", "edge-router", c.PublicIdentity, "-j", "--jwt-output-file", jwtFileName}
-
-	if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil {
-		return err
-	}
-
-	remoteJwt := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".jwt"
-	if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil {
-		return err
-	}
-
-	tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%s enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log "
-	cmds := []string{
-		"mkdir -p /home/ubuntu/logs",
-		fmt.Sprintf(tmpl, c.BinaryName, c.ConfigName, remoteJwt, c.ConfigName),
-	}
-
-	if c.HasLocalOrAncestralTag("tunneler") {
-		cmds = append(cmds,
-			"sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf",
-			"sudo systemctl restart systemd-resolved")
-	}
-
-	return host.Exec(c.GetHost(), cmds...).Execute(c.GetModel())
+func (action *reEnrollEdgeRoutersAction) Execute(run model.Run) error {
+	return component.ExecInParallel(action.componentSpec, action.concurrency, zitilab.RouterActionsReEnroll).Execute(run)
 }
 
 type reEnrollEdgeRoutersAction struct {

zititest/zitilab/actions/edge/sync_model_edge_state.go

@@ -14,13 +14,13 @@ func SyncModelEdgeState(componentSpec string) model.Action {
 	}
 }
 
-func (action *syncModelEdgeStateAction) Execute(m *model.Model) error {
-	routerComponents := m.SelectComponents(action.componentSpec)
+func (action *syncModelEdgeStateAction) Execute(run model.Run) error {
+	routerComponents := run.GetModel().SelectComponents(action.componentSpec)
 	if len(routerComponents) == 0 {
 		return errors.Errorf("no router components found for selector '%v'", action.componentSpec)
 	}
 
-	output, err := zitilib_actions.EdgeExecWithOutput(m, "list", "edge-routers", "--output-json", "true limit none")
+	output, err := zitilib_actions.EdgeExecWithOutput(run.GetModel(), "list", "edge-routers", "--output-json", "true limit none")
 	if err != nil {
 		return err
 	}
@@ -45,7 +45,7 @@ func (action *syncModelEdgeStateAction) Execute(m *model.Model) error {
 		routerName := router.S("name").Data().(string)
 
 		for _, c := range routerComponents {
-			if c.PublicIdentity == routerName {
+			if c.Id == routerName {
 				routerId = strings.ReplaceAll(routerId, ".", ":")
 				c.Tags = append(c.Tags, "edgeId:"+routerId)
 			}

zititest/zitilab/actions/edge_cli.go

@@ -27,8 +27,8 @@ func Edge(args ...string) model.Action {
 	}
 }
 
-func (a *edge) Execute(m *model.Model) error {
-	return EdgeExec(m, a.args...)
+func (a *edge) Execute(run model.Run) error {
+	return EdgeExec(run.GetModel(), a.args...)
 }
 
 type edge struct {

zititest/zitilab/actions/fabric_cli.go

@@ -27,8 +27,8 @@ func Fabric(args ...string) model.Action {
 	}
 }
 
-func (a *fabric) Execute(m *model.Model) error {
-	_, err := cli.Exec(m, append([]string{"fabric", "-i", model.ActiveInstanceId()}, a.args...)...)
+func (a *fabric) Execute(run model.Run) error {
+	_, err := cli.Exec(run.GetModel(), append([]string{"fabric", "-i", model.ActiveInstanceId()}, a.args...)...)
 	return err
 }
 

zititest/zitilab/actions/logs.go

@@ -31,13 +31,13 @@ func Logs() model.Action {
 	return &logs{}
 }
 
-func (self *logs) Execute(m *model.Model) error {
-	if !m.IsBound() {
+func (self *logs) Execute(run model.Run) error {
+	if !run.GetModel().IsBound() {
 		return fmt.Errorf("model not bound")
 	}
 
 	snapshot := fmt.Sprintf("%d", info.NowInMilliseconds())
-	for rn, r := range m.Regions {
+	for rn, r := range run.GetModel().Regions {
 		for hn, h := range r.Hosts {
 			ssh := lib.NewSshConfigFactory(h)
 			if err := self.forHost(snapshot, rn, hn, ssh); err != nil {

zititest/zitilab/actions/metricbeat.go

@@ -24,8 +24,8 @@ func StartMetricbeat(hostSpec, configPath, dataPath, logPath string) model.Actio
 	}
 }
 
-func (mbs *metricbeatStart) Execute(m *model.Model) error {
-	return m.ForEachHost(mbs.hostSpec, 24, func(c *model.Host) error {
+func (mbs *metricbeatStart) Execute(run model.Run) error {
+	return run.GetModel().ForEachHost(mbs.hostSpec, 24, func(c *model.Host) error {
 		ssh := lib.NewSshConfigFactory(c)
 
 		cmd := fmt.Sprintf("screen -d -m nohup metricbeat --path.config %s --path.data %s --path.logs %s 2>&1 &", mbs.configPath, mbs.dataPath, mbs.logPath)

zititest/zitilab/component_common.go

@@ -0,0 +1,70 @@
+/*
+	Copyright 2019 NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package zitilab
+
+import (
+	"fmt"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/sirupsen/logrus"
+	"strings"
+)
+
+func getZitiProcessFilter(c *model.Component, zitiType string) func(string) bool {
+	return func(s string) bool {
+		return strings.Contains(s, "ziti") &&
+			strings.Contains(s, zitiType) &&
+			strings.Contains(s, fmt.Sprintf("--cli-agent-alias %s", c.Id)) &&
+			!strings.Contains(s, "sudo ")
+	}
+}
+
+func startZitiComponent(c *model.Component, zitiType string, version string, configName string) error {
+	binaryName := "ziti"
+	if version != "" {
+		binaryName += "-" + version
+	}
+
+	factory := lib.NewSshConfigFactory(c.GetHost())
+
+	binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), binaryName)
+	configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s", factory.User(), configName)
+	logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id)
+
+	if zitiType == "tunnel" {
+
+	}
+
+	useSudo := ""
+	if zitiType == "tunnel" || c.HasTag("tunneler") {
+		useSudo = "sudo"
+	}
+
+	serviceCmd := fmt.Sprintf("nohup %s %s %s run --log-formatter pfxlog %s --cli-agent-alias %s > %s 2>&1 &",
+		useSudo, binaryPath, zitiType, configPath, c.Id, logsPath)
+
+	value, err := lib.RemoteExec(factory, serviceCmd)
+	if err != nil {
+		return err
+	}
+
+	if len(value) > 0 {
+		logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n"))
+	}
+
+	return nil
+}

zititest/zitilab/component_controller.go

@@ -0,0 +1,136 @@
+/*
+	Copyright 2019 NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package zitilab
+
+import (
+	"fmt"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/lib/actions/host"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/zititest/zitilab/pki"
+	"github.com/openziti/ziti/zititest/zitilab/stageziti"
+	"github.com/pkg/errors"
+	"io/fs"
+)
+
+var _ model.ComponentType = (*ControllerType)(nil)
+var _ model.ServerComponent = (*ControllerType)(nil)
+var _ model.FileStagingComponent = (*ControllerType)(nil)
+var _ model.ActionsComponent = (*ControllerType)(nil)
+
+const (
+	ControllerActionInitStandalone = "initStandalone"
+)
+
+type ControllerType struct {
+	ConfigSourceFS fs.FS
+	ConfigSource   string
+	ConfigName     string
+	Version        string
+	LocalPath      string
+}
+
+func (self *ControllerType) GetActions() map[string]model.ComponentAction {
+	return map[string]model.ComponentAction{
+		ControllerActionInitStandalone: model.ComponentActionF(self.InitStandalone),
+	}
+}
+
+func (self *ControllerType) Dump() any {
+	return map[string]string{
+		"type_id":       "controller",
+		"config_source": self.ConfigSource,
+		"config_name":   self.ConfigName,
+		"version":       self.Version,
+		"local_path":    self.LocalPath,
+	}
+}
+
+func (self *ControllerType) StageFiles(r model.Run, c *model.Component) error {
+	configSource := self.ConfigSource
+	if configSource == "" {
+		configSource = "ctrl.yml.tmpl"
+	}
+
+	configName := self.getConfigName(c)
+
+	if err := lib.GenerateConfigForComponent(c, self.ConfigSourceFS, configSource, configName, r); err != nil {
+		return err
+	}
+
+	if err := pki.CreateControllerCerts(r, c, c.Id); err != nil {
+		return err
+	}
+
+	return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath)
+}
+
+func (self *ControllerType) getConfigName(c *model.Component) string {
+	configName := self.ConfigName
+	if configName == "" {
+		configName = c.Id + ".yml"
+	}
+	return configName
+}
+
+func (self *ControllerType) getProcessFilter(c *model.Component) func(string) bool {
+	return getZitiProcessFilter(c, "controller")
+}
+
+func (self *ControllerType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	pids, err := lib.FindProcesses(factory, self.getProcessFilter(c))
+	if err != nil {
+		return false, err
+	}
+	return len(pids) > 0, nil
+}
+
+func (self *ControllerType) Start(_ model.Run, c *model.Component) error {
+	return startZitiComponent(c, "controller", self.Version, self.getConfigName(c))
+}
+
+func (self *ControllerType) Stop(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	return lib.RemoteKillFilterF(factory, self.getProcessFilter(c))
+}
+
+func (self *ControllerType) InitStandalone(run model.Run, c *model.Component) error {
+	username := c.MustStringVariable("credentials.edge.username")
+	password := c.MustStringVariable("credentials.edge.password")
+
+	if username == "" {
+		return errors.New("variable credentials/edge/username must be a string")
+	}
+
+	if password == "" {
+		return errors.New("variable credentials/edge/password must be a string")
+	}
+
+	factory := lib.NewSshConfigFactory(c.GetHost())
+
+	binaryName := "ziti"
+	if self.Version != "" {
+		binaryName += "-" + self.Version
+	}
+
+	configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s", factory.User(), self.getConfigName(c))
+
+	tmpl := "rm -f /home/%v/fablab/ctrl.db && set -o pipefail; /home/%s/fablab/bin/%s controller --log-formatter pfxlog edge init %s -u %s -p %s 2>&1 | tee logs/controller.edge.init.log"
+	cmd := fmt.Sprintf(tmpl, factory.User(), factory.User(), binaryName, configPath, username, password)
+	return host.Exec(c.GetHost(), cmd).Execute(run)
+}

zititest/zitilab/component_echo_server.go

@@ -0,0 +1,76 @@
+package zitilab
+
+import (
+	"fmt"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/zititest/zitilab/stageziti"
+	"github.com/sirupsen/logrus"
+	"strings"
+)
+
+var _ model.ComponentType = (*EchoServerType)(nil)
+
+type EchoServerMode int
+
+type EchoServerType struct {
+	Version   string
+	LocalPath string
+}
+
+func (self *EchoServerType) Dump() any {
+	return map[string]string{
+		"type_id":    "echo-server",
+		"version":    self.Version,
+		"local_path": self.LocalPath,
+	}
+}
+
+func (self *EchoServerType) StageFiles(r model.Run, c *model.Component) error {
+	return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath)
+}
+
+func (self *EchoServerType) getProcessFilter(c *model.Component) func(string) bool {
+	return getZitiProcessFilter(c, "echo-server")
+}
+
+func (self *EchoServerType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	pids, err := lib.FindProcesses(factory, self.getProcessFilter(c))
+	if err != nil {
+		return false, err
+	}
+	return len(pids) > 0, nil
+}
+
+func (self *EchoServerType) Start(_ model.Run, c *model.Component) error {
+	binaryName := "ziti"
+	if self.Version != "" {
+		binaryName += "-" + self.Version
+	}
+
+	factory := lib.NewSshConfigFactory(c.GetHost())
+
+	binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), binaryName)
+	configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s.json", factory.User(), c.Id)
+	logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id)
+
+	serviceCmd := fmt.Sprintf("nohup %s learn demo echo-server -i %s --cli-agent-alias %s > %s 2>&1 &",
+		binaryPath, configPath, c.Id, logsPath)
+
+	value, err := lib.RemoteExec(factory, serviceCmd)
+	if err != nil {
+		return err
+	}
+
+	if len(value) > 0 {
+		logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n"))
+	}
+
+	return nil
+}
+
+func (self *EchoServerType) Stop(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	return lib.RemoteKillFilterF(factory, self.getProcessFilter(c))
+}

zititest/zitilab/component_iperf.go

@@ -0,0 +1,83 @@
+/*
+	Copyright 2019 NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package zitilab
+
+import (
+	"fmt"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/sirupsen/logrus"
+	"strings"
+)
+
+var _ model.ComponentType = (*IPerfServerType)(nil)
+
+type IPerfServerType struct {
+	Port uint16
+}
+
+func (self *IPerfServerType) Dump() any {
+	return map[string]string{
+		"type_id": "iperf-server",
+		"port":    fmt.Sprintf("%v", self.GetPort()),
+	}
+}
+
+func (self *IPerfServerType) GetPort() uint16 {
+	if self.Port == 0 {
+		return 5201
+	}
+	return self.Port
+}
+
+func (self *IPerfServerType) getProcessFilter(*model.Component) func(string) bool {
+	return func(s string) bool {
+		return strings.Contains(s, fmt.Sprintf("iperf3 -s -p %v", self.GetPort()))
+	}
+}
+
+func (self *IPerfServerType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	pids, err := lib.FindProcesses(factory, self.getProcessFilter(c))
+	if err != nil {
+		return false, err
+	}
+	return len(pids) > 0, nil
+}
+
+func (self *IPerfServerType) Start(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+
+	logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id)
+	serviceCmd := fmt.Sprintf("nohup iperf3 -s -p %v > %s 2>&1 &", self.GetPort(), logsPath)
+
+	value, err := lib.RemoteExec(factory, serviceCmd)
+	if err != nil {
+		return err
+	}
+
+	if len(value) > 0 {
+		logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n"))
+	}
+
+	return nil
+}
+
+func (self *IPerfServerType) Stop(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	return lib.RemoteKillFilterF(factory, self.getProcessFilter(c))
+}

zititest/zitilab/component_router.go

@@ -0,0 +1,200 @@
+/*
+	Copyright 2019 NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package zitilab
+
+import (
+	"fmt"
+	"github.com/michaelquigley/pfxlog"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/lib/actions/host"
+	"github.com/openziti/fablab/kernel/model"
+	zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions"
+	"github.com/openziti/ziti/zititest/zitilab/stageziti"
+	"io/fs"
+	"path/filepath"
+	"strings"
+)
+
+var _ model.ComponentType = (*RouterType)(nil)
+
+const (
+	RouterActionsCreateAndEnroll = "createAndEnroll"
+	RouterActionsReEnroll        = "reEnroll"
+)
+
+type RouterType struct {
+	ConfigSourceFS fs.FS
+	ConfigSource   string
+	ConfigName     string
+	Version        string
+	LocalPath      string
+}
+
+func (self *RouterType) GetActions() map[string]model.ComponentAction {
+	return map[string]model.ComponentAction{
+		RouterActionsCreateAndEnroll: model.ComponentActionF(self.CreateAndEnroll),
+		RouterActionsReEnroll:        model.ComponentActionF(self.ReEnroll),
+	}
+}
+
+func (self *RouterType) Dump() any {
+	return map[string]string{
+		"type_id":       "router",
+		"config_source": self.ConfigSource,
+		"config_name":   self.ConfigName,
+		"version":       self.Version,
+		"local_path":    self.LocalPath,
+	}
+}
+
+func (self *RouterType) InitializeHost(run model.Run, c *model.Component) error {
+	if self.isTunneler(c) {
+		cmds := []string{
+			"sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf",
+			"sudo systemctl restart systemd-resolved",
+			"mkdir -p /home/ubuntu/logs",
+		}
+		return host.Exec(c.GetHost(), cmds...).Execute(run)
+	}
+	return nil
+}
+
+func (self *RouterType) StageFiles(r model.Run, c *model.Component) error {
+	configSource := self.ConfigSource
+	if configSource == "" {
+		configSource = "router.yml.tmpl"
+	}
+
+	configName := self.getConfigName(c)
+
+	if err := lib.GenerateConfigForComponent(c, self.ConfigSourceFS, configSource, configName, r); err != nil {
+		return err
+	}
+
+	return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath)
+}
+
+func (self *RouterType) isTunneler(c *model.Component) bool {
+	return c.HasLocalOrAncestralTag("tunneler")
+}
+
+func (self *RouterType) getConfigName(c *model.Component) string {
+	configName := self.ConfigName
+	if configName == "" {
+		configName = c.Id + ".yml"
+	}
+	return configName
+}
+
+func (self *RouterType) getBinaryName() string {
+	binaryName := "ziti"
+	version := self.Version
+	if version != "" {
+		binaryName += "-" + version
+	}
+	return binaryName
+}
+
+func (self *RouterType) getProcessFilter(c *model.Component) func(string) bool {
+	return getZitiProcessFilter(c, "router")
+}
+
+func (self *RouterType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	pids, err := lib.FindProcesses(factory, self.getProcessFilter(c))
+	if err != nil {
+		return false, err
+	}
+	return len(pids) > 0, nil
+}
+
+func (self *RouterType) Start(_ model.Run, c *model.Component) error {
+	return startZitiComponent(c, "router", self.Version, self.getConfigName(c))
+}
+
+func (self *RouterType) Stop(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	return lib.RemoteKillFilterF(factory, self.getProcessFilter(c))
+}
+
+func (self *RouterType) CreateAndEnroll(run model.Run, c *model.Component) error {
+	if err := zitilib_actions.EdgeExec(c.GetModel(), "delete", "edge-router", c.Id); err != nil {
+		pfxlog.Logger().
+			WithError(err).
+			WithField("router", c.Id).
+			Warn("unable to delete router (may not be present")
+	}
+
+	ssh := lib.NewSshConfigFactory(c.GetHost())
+
+	jwtFileName := filepath.Join(run.GetTmpDir(), c.Id+".jwt")
+
+	attributes := strings.Join(c.Tags, ",")
+
+	args := []string{"create", "edge-router", c.Id, "-j", "--jwt-output-file", jwtFileName, "-a", attributes}
+
+	isTunneler := c.HasLocalOrAncestralTag("tunneler")
+	if isTunneler {
+		args = append(args, "--tunneler-enabled")
+	}
+
+	if c.HasLocalOrAncestralTag("no-traversal") {
+		args = append(args, "--no-traversal")
+	}
+
+	if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil {
+		return err
+	}
+
+	if isTunneler {
+		if err := zitilib_actions.EdgeExec(c.GetModel(), "update", "identity", c.Id, "-a", attributes); err != nil {
+			return err
+		}
+	}
+
+	remoteJwt := "/home/ubuntu/fablab/cfg/" + c.Id + ".jwt"
+	if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil {
+		return err
+	}
+
+	tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%v router enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log "
+	cmd := fmt.Sprintf(tmpl, self.getBinaryName(), self.getConfigName(c), remoteJwt, c.Id)
+
+	return host.Exec(c.GetHost(), cmd).Execute(run)
+}
+
+func (self *RouterType) ReEnroll(run model.Run, c *model.Component) error {
+	ssh := lib.NewSshConfigFactory(c.GetHost())
+
+	jwtFileName := filepath.Join(model.ConfigBuild(), c.Id+".jwt")
+
+	args := []string{"re-enroll", "edge-router", c.Id, "-j", "--jwt-output-file", jwtFileName}
+
+	if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil {
+		return err
+	}
+
+	remoteJwt := "/home/ubuntu/fablab/cfg/" + c.Id + ".jwt"
+	if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil {
+		return err
+	}
+
+	tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%s enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log "
+	cmd := fmt.Sprintf(tmpl, self.getBinaryName(), self.getConfigName(c), remoteJwt, c.Id)
+
+	return host.Exec(c.GetHost(), cmd).Execute(run)
+}

zititest/zitilab/component_zcat.go

@@ -0,0 +1,49 @@
+package zitilab
+
+import (
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/zititest/zitilab/stageziti"
+	"strings"
+)
+
+var _ model.ComponentType = (*ZCatType)(nil)
+
+type ZCatMode int
+
+type ZCatType struct {
+	Version   string
+	LocalPath string
+}
+
+func (self *ZCatType) Dump() any {
+	return map[string]string{
+		"type_id":    "zcat",
+		"version":    self.Version,
+		"local_path": self.LocalPath,
+	}
+}
+
+func (self *ZCatType) StageFiles(r model.Run, c *model.Component) error {
+	return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath)
+}
+
+func (self *ZCatType) getProcessFilter(c *model.Component) func(string) bool {
+	return func(s string) bool {
+		return strings.Contains(s, "ziti") && strings.Contains(s, "zcat ")
+	}
+}
+
+func (self *ZCatType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	pids, err := lib.FindProcesses(factory, self.getProcessFilter(c))
+	if err != nil {
+		return false, err
+	}
+	return len(pids) > 0, nil
+}
+
+func (self *ZCatType) Stop(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	return lib.RemoteKillFilterF(factory, self.getProcessFilter(c))
+}

zititest/zitilab/component_ziti_edge_tunnel.go

@@ -0,0 +1,104 @@
+/*
+	Copyright 2019 NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package zitilab
+
+import (
+	"fmt"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/zititest/zitilab/stageziti"
+	"github.com/sirupsen/logrus"
+	"strings"
+)
+
+var _ model.ComponentType = (*ZitiEdgeTunnelType)(nil)
+
+type ZitiEdgeTunnelType struct {
+	Version   string
+	LocalPath string
+}
+
+func (self *ZitiEdgeTunnelType) Dump() any {
+	return map[string]string{
+		"type_id":    "ziti-edge-tunnel",
+		"version":    self.Version,
+		"local_path": self.LocalPath,
+	}
+}
+
+func (self *ZitiEdgeTunnelType) getVersion() string {
+	if strings.HasPrefix(self.Version, "v") {
+		return self.Version[1:]
+	}
+	return self.Version
+}
+
+func (self *ZitiEdgeTunnelType) getBinaryName() string {
+	binaryName := "ziti-edge-tunnel"
+	version := self.getVersion()
+	if version != "" {
+		binaryName += "-" + version
+	}
+	return binaryName
+}
+
+func (self *ZitiEdgeTunnelType) StageFiles(r model.Run, c *model.Component) error {
+	return stageziti.StageZitiEdgeTunnelOnce(r, c, self.getVersion(), self.LocalPath)
+}
+
+func (self *ZitiEdgeTunnelType) getProcessFilter(c *model.Component) func(string) bool {
+	return func(s string) bool {
+		return strings.Contains(s, self.getBinaryName()) &&
+			strings.Contains(s, fmt.Sprintf("%s.json", c.Id)) &&
+			!strings.Contains(s, "sudo ")
+	}
+}
+
+func (self *ZitiEdgeTunnelType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	pids, err := lib.FindProcesses(factory, self.getProcessFilter(c))
+	if err != nil {
+		return false, err
+	}
+	return len(pids) > 0, nil
+}
+
+func (self *ZitiEdgeTunnelType) Start(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+
+	binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), self.getBinaryName())
+	configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s.json", factory.User(), c.Id)
+	logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id)
+
+	serviceCmd := fmt.Sprintf("nohup sudo %s run -i %s > %s 2>&1 &", binaryPath, configPath, logsPath)
+
+	value, err := lib.RemoteExec(factory, serviceCmd)
+	if err != nil {
+		return err
+	}
+
+	if len(value) > 0 {
+		logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n"))
+	}
+
+	return nil
+}
+
+func (self *ZitiEdgeTunnelType) Stop(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	return lib.RemoteKillFilterF(factory, self.getProcessFilter(c))
+}

zititest/zitilab/component_ziti_tunnel.go

@@ -0,0 +1,131 @@
+/*
+	Copyright 2019 NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package zitilab
+
+import (
+	"fmt"
+	"github.com/openziti/fablab/kernel/lib"
+	"github.com/openziti/fablab/kernel/lib/actions/host"
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/zititest/zitilab/stageziti"
+	"github.com/sirupsen/logrus"
+	"strings"
+)
+
+var _ model.ComponentType = (*ZitiTunnelType)(nil)
+
+type ZitiTunnelMode int
+
+const (
+	ZitiTunnelModeTproxy ZitiTunnelMode = 0
+	ZitiTunnelModeProxy  ZitiTunnelMode = 1
+	ZitiTunnelModeHost   ZitiTunnelMode = 2
+)
+
+func (self ZitiTunnelMode) String() string {
+	if self == ZitiTunnelModeTproxy {
+		return "tproxy"
+	}
+	if self == ZitiTunnelModeProxy {
+		return "proxy"
+	}
+	if self == ZitiTunnelModeHost {
+		return "host"
+	}
+	return "invalid"
+}
+
+type ZitiTunnelType struct {
+	Mode      ZitiTunnelMode
+	Version   string
+	LocalPath string
+}
+
+func (self *ZitiTunnelType) Dump() any {
+	return map[string]string{
+		"type_id":    "ziti-tunnel",
+		"version":    self.Version,
+		"local_path": self.LocalPath,
+	}
+}
+
+func (self *ZitiTunnelType) StageFiles(r model.Run, c *model.Component) error {
+	return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath)
+}
+
+func (self *ZitiTunnelType) InitializeHost(run model.Run, c *model.Component) error {
+	cmds := []string{"mkdir -p /home/ubuntu/logs"}
+	if self.Mode == ZitiTunnelModeTproxy {
+		cmds = append(cmds,
+			"sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf",
+			"sudo systemctl restart systemd-resolved",
+		)
+	}
+	return host.Exec(c.GetHost(), cmds...).Execute(run)
+}
+
+func (self *ZitiTunnelType) getProcessFilter(c *model.Component) func(string) bool {
+	return getZitiProcessFilter(c, "tunnel")
+}
+
+func (self *ZitiTunnelType) IsRunning(_ model.Run, c *model.Component) (bool, error) {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	pids, err := lib.FindProcesses(factory, self.getProcessFilter(c))
+	if err != nil {
+		return false, err
+	}
+	return len(pids) > 0, nil
+}
+
+func (self *ZitiTunnelType) Start(_ model.Run, c *model.Component) error {
+	binaryName := "ziti"
+	if self.Version != "" {
+		binaryName += "-" + self.Version
+	}
+
+	mode := self.Mode
+
+	factory := lib.NewSshConfigFactory(c.GetHost())
+
+	binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), binaryName)
+	configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s.json", factory.User(), c.Id)
+	logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id)
+
+	useSudo := ""
+	if mode == ZitiTunnelModeTproxy {
+		useSudo = "sudo"
+	}
+
+	serviceCmd := fmt.Sprintf("nohup %s %s tunnel %s --log-formatter pfxlog -i %s --cli-agent-alias %s > %s 2>&1 &",
+		useSudo, binaryPath, mode.String(), configPath, c.Id, logsPath)
+
+	value, err := lib.RemoteExec(factory, serviceCmd)
+	if err != nil {
+		return err
+	}
+
+	if len(value) > 0 {
+		logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n"))
+	}
+
+	return nil
+}
+
+func (self *ZitiTunnelType) Stop(_ model.Run, c *model.Component) error {
+	factory := lib.NewSshConfigFactory(c.GetHost())
+	return lib.RemoteKillSignalFilterF(factory, "-KILL", self.getProcessFilter(c))
+}

zititest/zitilab/console/action.go

@@ -30,7 +30,7 @@ func Console() model.Action {
 	return &console{}
 }
 
-func (consoleAction *console) Execute(m *model.Model) error {
+func (consoleAction *console) Execute(model.Run) error {
 	server := NewServer()
 	go server.Listen()
 

zititest/zitilab/models/db_builder.go

@@ -7,6 +7,7 @@ import (
 	"github.com/openziti/fabric/controller/db"
 	"github.com/openziti/fabric/controller/network"
 	"github.com/openziti/storage/boltz"
+	"github.com/openziti/ziti/zititest/zitilab"
 	"github.com/pkg/errors"
 	"go.etcd.io/bbolt"
 	"strings"
@@ -112,11 +113,9 @@ func (self *ZitiDbBuilder) Build(m *model.Model) error {
 				region.Hosts["router_"+id] = host
 
 				component := &model.Component{
-					Scope:      model.Scope{Tags: model.Tags{}},
-					ConfigSrc:  "router.yml",
-					ConfigName: fmt.Sprintf("router-%v.yml", er.Id),
-					BinaryName: "ziti router",
-					Host:       host,
+					Scope: model.Scope{Tags: model.Tags{}},
+					Type:  &zitilab.RouterType{},
+					Host:  host,
 				}
 
 				host.Components[er.Id] = component

zititest/zitilab/paths.go

@@ -25,7 +25,7 @@ func ZitiRoot() string {
 	return zitiRoot
 }
 
-func DefaultZitiBinaries() model.ConfigurationStage {
+func DefaultZitiBinaries() model.Stage {
 	zitiBinaries := []string{
 		"ziti",
 		"ziti-controller",

zititest/zitilab/pki/cert.go

@@ -0,0 +1,153 @@
+/*
+	Copyright 2019 NetFoundry Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+
+	https://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
+*/
+
+package pki
+
+import (
+	"bytes"
+	"fmt"
+	"github.com/pkg/errors"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/openziti/fablab/kernel/model"
+	"github.com/openziti/ziti/ziti/cmd"
+	"github.com/sirupsen/logrus"
+)
+
+func HasExistingCA(run model.Run, name string) (bool, error) {
+	return run.DirExists(filepath.Join(model.BuildKitDir, model.BuildPkiDir, name))
+}
+
+func EnsureCaExists(run model.Run, trustDomain string, name string) error {
+	if caExists, err := HasExistingCA(run, name); caExists || err != nil {
+		return err
+	}
+
+	return GenerateCA(run, trustDomain, name)
+}
+
+func GenerateCA(run model.Run, trustDomain string, name string) error {
+	var pkiOut bytes.Buffer
+	var pkiErr bytes.Buffer
+	ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr)
+	args := []string{"pki", "create", "ca", "--pki-root", run.GetPkiDir(), "--ca-name", name, "--ca-file", name, "--trust-domain", trustDomain, "--"}
+	ziticli.SetArgs(args)
+	logrus.Infof("%v", args)
+	if err := ziticli.Execute(); err != nil {
+		logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n"))
+		return errors.Wrapf(err, "error generating ca '%s'", name)
+	}
+
+	return nil
+}
+
+func EnsureIntermediateCaExists(run model.Run, caName, name string) error {
+	logrus.Infof("generating signing certificate [%s]", name)
+
+	if caExists, err := HasExistingCA(run, name); caExists || err != nil {
+		return err
+	}
+
+	var pkiOut bytes.Buffer
+	var pkiErr bytes.Buffer
+
+	ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr)
+
+	args := []string{"pki", "create", "intermediate", "--pki-root", run.GetPkiDir(), "--ca-name", caName,
+		"--intermediate-file", name, "--intermediate-name", name + " signing cert"}
+	ziticli.SetArgs(args)
+	logrus.Infof("%v", args)
+	if err := ziticli.Execute(); err != nil {
+		logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n"))
+		return fmt.Errorf("error generating key (%s)", err)
+	}
+	return nil
+}
+
+func loadInfoFile(file string) string {
+	data, err := os.ReadFile(file)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return ""
+		}
+		logrus.WithError(err).Info("unable to read cert info file '%s'", file)
+	}
+	return string(data)
+}
+
+func EnsureServerCertExists(run model.Run, name string, ip string, spiffeId string) error {
+	logrus.Infof("generating server certificate [%s:%s]", name, ip)
+
+	certFile := name + "-server"
+	certFilePath := filepath.Join(model.BuildKitDir, model.BuildPkiDir, name, "certs", certFile)
+
+	certExists, err := run.FileExists(certFilePath + ".cert")
+	if err != nil {
+		return err
+	}
+
+	certInfo := ip + "\n" + spiffeId + "\n"
+	infoFile := filepath.Join(run.GetWorkingDir(), certFilePath+".info")
+
+	if certExists {
+		cachedInfo := loadInfoFile(infoFile)
+		if certInfo == cachedInfo {
+			logrus.Infof("server cert %v already up to date", name)
+			return nil
+		}
+	}
+
+	var pkiOut bytes.Buffer
+	var pkiErr bytes.Buffer
+	ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr)
+
+	args := []string{"pki", "create", "server",
+		"--server-name", name,
+		"--pki-root", run.GetPkiDir(),
+		"--ca-name", name,
+		"--server-file", certFile,
+		"--allow-overwrite",
+		"--ip", ip}
+
+	if spiffeId != "" {
+		args = append(args, "--spiffe-id", spiffeId)
+	}
+
+	logrus.Infof("%v", args)
+	ziticli.SetArgs(args)
+	if err := ziticli.Execute(); err != nil {
+		logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n"))
+		return fmt.Errorf("error generating server certificate (%s)", err)
+	}
+
+	return os.WriteFile(infoFile, []byte(certInfo), 0600)
+}
+
+func CreateControllerCerts(run model.Run, component *model.Component, name string) error {
+	trustDomain := component.GetStringVariableOr("ca.trustDomain", "ziti.test")
+	rootCaName := component.GetStringVariableOr("ca.rootName", "root")
+	if err := EnsureCaExists(run, trustDomain, rootCaName); err != nil {
+		return fmt.Errorf("error generating ca (%s)", err)
+	}
+
+	if err := EnsureIntermediateCaExists(run, rootCaName, name); err != nil {
+		return errors.Wrapf(err, "error generating public identity for component [%s]", component.Id)
+	}
+
+	return EnsureServerCertExists(run, name, component.Host.PublicIp, "controller/"+name)
+}

zititest/zitilab/runlevel/0_infrastructure/consul.go

@@ -12,13 +12,13 @@ type installConsul struct {
 	hostSpec string
 }
 
-func InstallConsul(hostSpec string) model.InfrastructureStage {
+func InstallConsul(hostSpec string) model.Stage {
 	return &installConsul{
 		hostSpec: hostSpec,
 	}
 }
 
-func (imb *installConsul) Express(run model.Run) error {
+func (imb *installConsul) Execute(run model.Run) error {
 	return run.GetModel().ForEachHost(imb.hostSpec, 25, func(host *model.Host) error {
 		ssh := lib.NewSshConfigFactory(host)
 

zititest/zitilab/runlevel/0_infrastructure/metricbeat.go

@@ -15,14 +15,14 @@ type installMetricbeat struct {
 	version  string
 }
 
-func InstallMetricbeat(hostSpec, version string) model.InfrastructureStage {
+func InstallMetricbeat(hostSpec, version string) model.Stage {
 	return &installMetricbeat{
 		hostSpec: hostSpec,
 		version:  version,
 	}
 }
 
-func (imb *installMetricbeat) Express(run model.Run) error {
+func (imb *installMetricbeat) Execute(run model.Run) error {
 	return run.GetModel().ForEachHost(imb.hostSpec, 25, func(host *model.Host) error {
 		ssh := lib.NewSshConfigFactory(host)
 

zititest/zitilab/runlevel/1_configuration/cert.go

@@ -1,93 +0,0 @@
-/*
-	Copyright 2019 NetFoundry Inc.
-
-	Licensed under the Apache License, Version 2.0 (the "License");
-	you may not use this file except in compliance with the License.
-	You may obtain a copy of the License at
-
-	https://www.apache.org/licenses/LICENSE-2.0
-
-	Unless required by applicable law or agreed to in writing, software
-	distributed under the License is distributed on an "AS IS" BASIS,
-	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-	See the License for the specific language governing permissions and
-	limitations under the License.
-*/
-
-package zitilib_runlevel_1_configuration
-
-import (
-	"bytes"
-	"fmt"
-	"strings"
-
-	"github.com/openziti/fablab/kernel/model"
-	"github.com/openziti/ziti/ziti/cmd"
-	"github.com/sirupsen/logrus"
-)
-
-func generateCa(trustDomain string) error {
-	if caExists, err := hasExistingCA("root"); caExists || err != nil {
-		return err
-	}
-
-	var pkiOut bytes.Buffer
-	var pkiErr bytes.Buffer
-	ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr)
-	args := []string{"pki", "create", "ca", "--pki-root", model.PkiBuild(), "--ca-name", "root", "--ca-file", "root", "--trust-domain", trustDomain, "--"}
-	ziticli.SetArgs(args)
-	logrus.Infof("%v", args)
-	if err := ziticli.Execute(); err != nil {
-		logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n"))
-		return fmt.Errorf("error generating key (%s)", err)
-	}
-
-	return nil
-}
-
-func generateSigningCert(name string) error {
-	logrus.Infof("generating signing certificate [%s]", name)
-
-	var pkiOut bytes.Buffer
-	var pkiErr bytes.Buffer
-
-	ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr)
-
-	args := []string{"pki", "create", "intermediate", "--pki-root", model.PkiBuild(), "--ca-name", "root",
-		"--intermediate-file", name, "--intermediate-name", name + " signing cert"}
-	ziticli.SetArgs(args)
-	logrus.Infof("%v", args)
-	if err := ziticli.Execute(); err != nil {
-		logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n"))
-		return fmt.Errorf("error generating key (%s)", err)
-	}
-	return nil
-}
-
-func generateCert(name, ip, spiffeId string) error {
-	logrus.Infof("generating server certificate [%s:%s]", name, ip)
-
-	var pkiOut bytes.Buffer
-	var pkiErr bytes.Buffer
-	ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr)
-
-	args := []string{"pki", "create", "server",
-		"--server-name", name,
-		"--pki-root", model.PkiBuild(),
-		"--ca-name", name,
-		"--server-file", fmt.Sprintf("%s-server", name),
-		"--ip", ip}
-
-	if spiffeId != "" {
-		args = append(args, "--spiffe-id", spiffeId)
-	}
-
-	logrus.Infof("%v", args)
-	ziticli.SetArgs(args)
-	if err := ziticli.Execute(); err != nil {
-		logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n"))
-		return fmt.Errorf("error generating server certificate (%s)", err)
-	}
-
-	return nil
-}

zititest/zitilab/runlevel/1_configuration/fabric.go

@@ -1,176 +0,0 @@
-/*
-	Copyright 2019 NetFoundry Inc.
-
-	Licensed under the Apache License, Version 2.0 (the "License");
-	you may not use this file except in compliance with the License.
-	You may obtain a copy of the License at
-
-	https://www.apache.org/licenses/LICENSE-2.0
-
-	Unless required by applicable law or agreed to in writing, software
-	distributed under the License is distributed on an "AS IS" BASIS,
-	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-	See the License for the specific language governing permissions and
-	limitations under the License.
-*/
-
-package zitilib_runlevel_1_configuration
-
-import (
-	"fmt"
-	"github.com/openziti/fablab/kernel/model"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v3"
-	"os"
-	"path"
-	"strings"
-)
-
-func Fabric(trustDomain string, componentSpecs ...string) model.ConfigurationStage {
-	if len(componentSpecs) == 0 {
-		componentSpecs = []string{"*"}
-	}
-	return &fabric{
-		trustDomain:    trustDomain,
-		componentSpecs: componentSpecs,
-	}
-}
-
-type fabric struct {
-	trustDomain    string
-	componentSpecs []string
-}
-
-func (f *fabric) getSpiffeId(c *model.Component, id string) string {
-	for _, tag := range c.Tags {
-		if strings.HasPrefix(tag, "spiffe:") {
-			prefix := tag[len("spiffe:"):]
-			return prefix + "/" + id
-		}
-	}
-	return id
-}
-
-func (f *fabric) Configure(run model.Run) error {
-	m := run.GetModel()
-	if err := generateCa(f.trustDomain); err != nil {
-		return fmt.Errorf("error generating ca (%s)", err)
-	}
-
-	ips := map[string]string{}
-
-	processedComponents := map[*model.Component]struct{}{}
-
-	for _, spec := range f.componentSpecs {
-		for _, component := range m.SelectComponents(spec) {
-			if _, found := processedComponents[component]; found {
-				continue
-			}
-			processedComponents[component] = struct{}{}
-
-			if component.PublicIdentity != "" {
-				logrus.Infof("generating public ip identity [%s/%s] on [%s/%s]", component.Id, component.PublicIdentity, component.Region().Id, component.Host.Id)
-				if err := generateSigningCert(component.PublicIdentity); err != nil {
-					return errors.Wrapf(err, "error generating public identity [%s/%s]", component.Id, component.PublicIdentity)
-				}
-
-				if err := generateCert(component.PublicIdentity, component.Host.PublicIp, f.getSpiffeId(component, component.PublicIdentity)); err != nil {
-					return fmt.Errorf("error generating public identity [%s/%s]", component.Id, component.PublicIdentity)
-				}
-
-				ips[component.GetPathId()+".public"] = component.Host.PublicIp
-			}
-
-			if component.PrivateIdentity != "" {
-				logrus.Infof("generating private ip identity [%s/%s] on [%s/%s]", component.Id, component.PrivateIdentity, component.Region().Id, component.Host.Id)
-				if err := generateSigningCert(component.PrivateIdentity); err != nil {
-					return errors.Wrapf(err, "error generating private identity [%s/%s]", component.Id, component.PublicIdentity)
-				}
-
-				if err := generateCert(component.PrivateIdentity, component.Host.PrivateIp, f.getSpiffeId(component, component.PrivateIdentity)); err != nil {
-					return fmt.Errorf("error generating private identity [%s/%s]", component.Id, component.PrivateIdentity)
-				}
-				ips[component.GetPathId()+".private"] = component.Host.PrivateIp
-			}
-		}
-	}
-
-	return storeIps(ips)
-}
-
-func haveIpsChanged(m *model.Model) (bool, error) {
-	ipFile := path.Join(model.PkiBuild(), "ips")
-	ips := map[string]string{}
-	if _, err := os.Stat(ipFile); err == nil {
-		ipData, err := os.ReadFile(ipFile)
-		if err != nil {
-			return false, err
-		}
-		if err = yaml.Unmarshal(ipData, &ips); err != nil {
-			return false, err
-		}
-	}
-
-	for _, c := range m.SelectComponents("*") {
-		pubIdKey := c.GetPathId() + ".public"
-		if c.PublicIdentity != "" {
-			if ip, found := ips[pubIdKey]; found {
-				if ip == c.Host.PublicIp {
-					delete(ips, pubIdKey)
-				} else {
-					logrus.Infof("public ip for %v/%v has changed from %v -> %v. rebuilding pki", c.Id, c.PublicIdentity, c.Host.PublicIp, ip)
-					return true, nil
-				}
-			} else {
-				logrus.Infof("missing cert public identity of %v/%v. rebuilding pki", c.Id, c.PublicIdentity)
-				return true, nil
-			}
-		}
-
-		privIdKey := c.GetPathId() + ".private"
-		if c.PrivateIdentity != "" {
-			if ip, found := ips[privIdKey]; found {
-				if ip == c.Host.PrivateIp {
-					delete(ips, privIdKey)
-				} else {
-					logrus.Infof("private ip for %v/%v has changed from %v -> %v. rebuilding pki", c.Id, c.PrivateIdentity, c.Host.PrivateIp, ip)
-					return true, nil
-				}
-			} else {
-				logrus.Infof("missing cert for private identity of %v/%v. rebuilding pki", c.Id, c.PrivateIdentity)
-			}
-		}
-	}
-
-	return false, nil
-}
-
-func storeIps(ips map[string]string) error {
-	ipFile := path.Join(model.PkiBuild(), "ips")
-	data, err := yaml.Marshal(ips)
-	if err != nil {
-		return err
-	}
-	return os.WriteFile(ipFile, data, 0600)
-}
-
-func hasExisitingPki() (bool, error) {
-	if _, err := os.Stat(model.PkiBuild()); err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return true, err
-	}
-	return true, nil
-}
-
-func hasExistingCA(name string) (bool, error) {
-	if _, err := os.Stat(path.Join(model.PkiBuild(), name)); err != nil {
-		if os.IsNotExist(err) {
-			return false, nil
-		}
-		return true, err
-	}
-	return true, nil
-}

zititest/zitilab/runlevel/1_configuration/if_no_pki.go

@@ -1,50 +0,0 @@
-/*
-	Copyright 2019 NetFoundry Inc.
-
-	Licensed under the Apache License, Version 2.0 (the "License");
-	you may not use this file except in compliance with the License.
-	You may obtain a copy of the License at
-
-	https://www.apache.org/licenses/LICENSE-2.0
-
-	Unless required by applicable law or agreed to in writing, software
-	distributed under the License is distributed on an "AS IS" BASIS,
-	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-	See the License for the specific language governing permissions and
-	limitations under the License.
-*/
-
-package zitilib_runlevel_1_configuration
-
-import (
-	"fmt"
-	"github.com/openziti/fablab/kernel/model"
-	"github.com/sirupsen/logrus"
-)
-
-func IfNoPki(stages ...model.ConfigurationStage) model.ConfigurationStage {
-	return &ifNoPki{stages: stages}
-}
-
-func (self *ifNoPki) Configure(run model.Run) error {
-	if existing, err := hasExisitingPki(); err == nil {
-		if existing {
-			logrus.Infof("skipping configuration. existing pki system at [%s]", model.PkiBuild())
-			return nil
-		}
-	} else {
-		return fmt.Errorf("error checking pki existence at [%s] (%s)", model.PkiBuild(), err)
-	}
-
-	for _, stage := range self.stages {
-		if err := stage.Configure(run); err != nil {
-			return fmt.Errorf("error running configuration stage (%w)", err)
-		}
-	}
-
-	return nil
-}
-
-type ifNoPki struct {
-	stages []model.ConfigurationStage
-}

zititest/zitilab/runlevel/1_configuration/if_pki_needs_refresh.go

@@ -1,51 +0,0 @@
-/*
-	Copyright 2019 NetFoundry Inc.
-
-	Licensed under the Apache License, Version 2.0 (the "License");
-	you may not use this file except in compliance with the License.
-	You may obtain a copy of the License at
-
-	https://www.apache.org/licenses/LICENSE-2.0
-
-	Unless required by applicable law or agreed to in writing, software
-	distributed under the License is distributed on an "AS IS" BASIS,
-	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-	See the License for the specific language governing permissions and
-	limitations under the License.
-*/
-
-package zitilib_runlevel_1_configuration
-
-import (
-	"fmt"
-	"github.com/openziti/fablab/kernel/model"
-	"os"
-)
-
-func IfPkiNeedsRefresh(stages ...model.ConfigurationStage) model.ConfigurationStage {
-	return &ifPkiNeedsRefresh{stages: stages}
-}
-
-func (self *ifPkiNeedsRefresh) Configure(run model.Run) error {
-	if changed, err := haveIpsChanged(run.GetModel()); err != nil || !changed {
-		return err
-	}
-
-	if existing, err := hasExisitingPki(); err == nil && existing {
-		if err = os.RemoveAll(model.PkiBuild()); err != nil {
-			return err
-		}
-	}
-
-	for _, stage := range self.stages {
-		if err := stage.Configure(run); err != nil {
-			return fmt.Errorf("error running configuration stage (%w)", err)
-		}
-	}
-
-	return nil
-}
-
-type ifPkiNeedsRefresh struct {
-	stages []model.ConfigurationStage
-}

zititest/zitilab/runlevel/5_operation/circuit_metrics.go

@@ -14,7 +14,7 @@ import (
 	"time"
 )
 
-func CircuitMetrics(pollFreq time.Duration, closer <-chan struct{}, f func(string) string) model.OperatingStage {
+func CircuitMetrics(pollFreq time.Duration, closer <-chan struct{}, f func(string) string) model.Stage {
 	return &circuitMetrics{
 		closer:             closer,
 		circuits:           map[string]struct{}{},
@@ -34,7 +34,7 @@ type circuitMetrics struct {
 	idToSelectorMapper func(string) string
 }
 
-func (self *circuitMetrics) Operate(run model.Run) error {
+func (self *circuitMetrics) Execute(run model.Run) error {
 	self.model = run.GetModel()
 	bindHandler := func(binding channel.Binding) error {
 		binding.AddReceiveHandler(int32(mgmt_pb.ContentType_StreamEventsEventType), channel.ReceiveHandlerF(self.receiveCircuitEvents))

zititest/zitilab/runlevel/5_operation/client_metrics.go

@@ -53,7 +53,7 @@ type ClientMetrics struct {
 	idToSelectorMapper func(string) string
 }
 
-func (metrics *ClientMetrics) Activate(run model.Run) error {
+func (metrics *ClientMetrics) Execute(run model.Run) error {
 	if err := zitilib_actions.EdgeExec(run.GetModel(), "delete", "identity", "metrics-host"); err != nil {
 		return err
 	}

zititest/zitilab/runlevel/5_operation/loop_dialer.go

@@ -24,7 +24,7 @@ import (
 	"strings"
 )
 
-func Loop3Dialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.OperatingStage {
+func Loop3Dialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.Stage {
 	return &loopDialer{
 		host:      host,
 		scenario:  scenario,
@@ -35,7 +35,7 @@ func Loop3Dialer(host *model.Host, scenario, endpoint string, joiner chan struct
 	}
 }
 
-func LoopDialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.OperatingStage {
+func LoopDialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.Stage {
 	return &loopDialer{
 		host:      host,
 		scenario:  scenario,
@@ -46,7 +46,7 @@ func LoopDialer(host *model.Host, scenario, endpoint string, joiner chan struct{
 	}
 }
 
-func (self *loopDialer) Operate(run model.Run) error {
+func (self *loopDialer) Execute(run model.Run) error {
 	ssh := lib.NewSshConfigFactory(self.host)
 	if err := lib.RemoteKill(ssh, fmt.Sprintf("ziti-fabric-test %v dialer", self.subcmd)); err != nil {
 		return fmt.Errorf("error killing %v listeners (%w)", self.subcmd, err)

zititest/zitilab/runlevel/5_operation/loop_listener.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 )
 
-func Loop3Listener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.OperatingStage {
+func Loop3Listener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.Stage {
 	return &loopListener{
 		host:        host,
 		joiner:      joiner,
@@ -18,7 +18,7 @@ func Loop3Listener(host *model.Host, joiner chan struct{}, bindAddress string, e
 	}
 }
 
-func LoopListener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.OperatingStage {
+func LoopListener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.Stage {
 	return &loopListener{
 		host:        host,
 		joiner:      joiner,
@@ -28,7 +28,7 @@ func LoopListener(host *model.Host, joiner chan struct{}, bindAddress string, ex
 	}
 }
 
-func (self *loopListener) Operate(run model.Run) error {
+func (self *loopListener) Execute(run model.Run) error {
 	ssh := lib.NewSshConfigFactory(self.host)
 	if err := lib.RemoteKill(ssh, fmt.Sprintf("ziti-fabric-test %v listener", self.subcmd)); err != nil {
 		return fmt.Errorf("error killing %v listeners (%w)", self.subcmd, err)

zititest/zitilab/runlevel/5_operation/mesh.go

@@ -26,11 +26,11 @@ import (
 	"time"
 )
 
-func Mesh(closer <-chan struct{}) model.OperatingStage {
+func Mesh(closer <-chan struct{}) model.Stage {
 	return &mesh{closer: closer}
 }
 
-func (mesh *mesh) Operate(run model.Run) error {
+func (mesh *mesh) Execute(run model.Run) error {
 	if endpoint, id, err := dotziti.LoadIdentity(model.ActiveInstanceId()); err == nil {
 		if address, err := transport.ParseAddress(endpoint); err == nil {
 			dialer := channel.NewClassicDialer(id, address, nil)

zititest/zitilab/runlevel/5_operation/model_metrics.go

@@ -27,13 +27,13 @@ import (
 	"time"
 )
 
-func ModelMetrics(closer <-chan struct{}) model.OperatingStage {
+func ModelMetrics(closer <-chan struct{}) model.Stage {
 	return ModelMetricsWithIdMapper(closer, func(id string) string {
 		return "#" + id
 	})
 }
 
-func ModelMetricsWithIdMapper(closer <-chan struct{}, f func(string) string) model.OperatingStage {
+func ModelMetricsWithIdMapper(closer <-chan struct{}, f func(string) string) model.Stage {
 	return &modelMetrics{
 		closer:             closer,
 		idToSelectorMapper: f,
@@ -47,7 +47,7 @@ type modelMetrics struct {
 	idToSelectorMapper func(string) string
 }
 
-func (self *modelMetrics) Operate(run model.Run) error {
+func (self *modelMetrics) Execute(run model.Run) error {
 	self.m = run.GetModel()
 
 	bindHandler := func(binding channel.Binding) error {

zititest/zitilab/stageziti/stageziti.go

@@ -1,31 +1,120 @@
 package stageziti
 
 import (
-	"github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit"
 	"github.com/openziti/fablab/kernel/model"
 	"github.com/openziti/ziti/common/getziti"
+	"github.com/openziti/ziti/ziti/util"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"os"
+	"os/exec"
+	"path/filepath"
 )
 
-func FetchZitiEdgeTunnel(version string) model.ConfigurationStage {
-	return &ZitiEdgeTunnelFetchStage{
-		Version: version,
+func StageZitiOnce(run model.Run, component *model.Component, version string, source string) error {
+	op := "install.ziti-"
+	if version == "" {
+		op += "local"
+	} else {
+		op += version
 	}
+
+	return run.DoOnce(op, func() error {
+		return StageZiti(run, component, version, source)
+	})
 }
 
-type ZitiEdgeTunnelFetchStage struct {
-	Version string
-	OS      string
-	Arch    string
+func StageZitiEdgeTunnelOnce(run model.Run, component *model.Component, version string, source string) error {
+	op := "install.ziti-edge-tunnel-"
+	if version == "" {
+		op += "local"
+	} else {
+		op += version
+	}
+
+	return run.DoOnce(op, func() error {
+		return StageZitiEdgeTunnel(run, component, version, source)
+	})
 }
 
-func (self *ZitiEdgeTunnelFetchStage) Configure(run model.Run) error {
-	return devkit.FetchBinaries(func(binDir string) error {
-		if self.OS == "" {
-			self.OS = "linux"
+func StageZiti(run model.Run, component *model.Component, version string, source string) error {
+	fileName := "ziti"
+	if version != "" {
+		fileName += "-" + version
+	}
+
+	target := filepath.Join(run.GetBinDir(), fileName)
+	if version == "" || version == "latest" {
+		_ = os.Remove(target)
+	}
+
+	if version == "" {
+		if source != "" {
+			logrus.Infof("[%s] => [%s]", source, target)
+			return util.CopyFile(source, target)
 		}
-		if self.Arch == "" {
-			self.Arch = "amd64"
+		if envSource, found := component.GetStringVariable("ziti.path"); found {
+			logrus.Infof("[%s] => [%s]", envSource, target)
+			return util.CopyFile(envSource, target)
 		}
-		return getziti.InstallZitiEdgeTunnel(self.Version, self.OS, self.Arch, binDir, false)
-	}).Configure(run)
+		if zitiPath, err := exec.LookPath("ziti"); err == nil {
+			logrus.Infof("[%s] => [%s]", zitiPath, target)
+			return util.CopyFile(zitiPath, target)
+		}
+		return errors.New("ziti binary not found in path, no path provided and no ziti.path env variable set")
+	}
+
+	found, err := run.FileExists(filepath.Join(model.BuildKitDir, model.BuildBinDir, fileName))
+	if err != nil {
+		return err
+	}
+
+	if found {
+		logrus.Infof("%s already present, not downloading again", target)
+		return nil
+	}
+	logrus.Infof("%s not present, attempting to fetch", target)
+
+	return getziti.InstallZiti(version, "linux", "amd64", run.GetBinDir(), false)
+}
+
+func StageZitiEdgeTunnel(run model.Run, component *model.Component, version string, source string) error {
+	fileName := "ziti-edge-tunnel"
+	if version != "" {
+		fileName += "-" + version
+	}
+
+	target := filepath.Join(run.GetBinDir(), fileName)
+	if version == "" || version == "latest" {
+		_ = os.Remove(target)
+	}
+
+	if version == "" {
+		if source != "" {
+			logrus.Infof("[%s] => [%s]", source, target)
+			return util.CopyFile(source, target)
+		}
+		if envSource, found := component.GetStringVariable("ziti-edge-tunnel.path"); found {
+			logrus.Infof("[%s] => [%s]", envSource, target)
+			return util.CopyFile(envSource, target)
+		}
+		if zitiPath, err := exec.LookPath("ziti-edge-tunnel"); err == nil {
+			logrus.Infof("[%s] => [%s]", zitiPath, target)
+			return util.CopyFile(zitiPath, target)
+		}
+		return errors.New("ziti-edge-tunnel binary not found in path, no path provided and no ziti-edge-tunnel.path env variable set")
+	}
+
+	found, err := run.FileExists(filepath.Join(model.BuildKitDir, model.BuildBinDir, fileName))
+	if err != nil {
+		return err
+	}
+
+	if found {
+		logrus.Infof("%s already present, not downloading again", target)
+		return nil
+	}
+	logrus.Infof("%s not present, attempting to fetch", target)
+
+	return getziti.InstallZitiEdgeTunnel(version, "linux", "amd64", run.GetBinDir(), false)
 }