diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6d2c3fd6f..1922103b6 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -147,8 +147,7 @@ jobs: - name: Test Ziti Command run: | echo "ZITI_ROOT=$(go env GOPATH)/bin" >> "$GITHUB_ENV" - $(go env GOPATH)/bin/simple-transfer run - # pushd zititest && go test -v ./tests/... && popd + pushd zititest && go test -timeout 30m -v ./tests/... && popd - name: Test Ziti Command Teardown if: always() diff --git a/common/getziti/extract.go b/common/getziti/extract.go new file mode 100644 index 000000000..f06d1c150 --- /dev/null +++ b/common/getziti/extract.go @@ -0,0 +1,131 @@ +package getziti + +import ( + "archive/tar" + "archive/zip" + "compress/gzip" + "fmt" + "github.com/michaelquigley/pfxlog" + "io" + "os" + "path/filepath" + "strings" +) + +func Unzip(src, dest string, filter func(path string) (string, bool)) error { + r, err := zip.OpenReader(src) + if err != nil { + return err + } + defer func() { + if err := r.Close(); err != nil { + panic(err) + } + }() + + // Closure to address file descriptors issue with all the deferred .Close() methods + extractAndWriteFile := func(f *zip.File) error { + rc, err := f.Open() + if err != nil { + return err + } + defer func() { + if err := rc.Close(); err != nil { + panic(err) + } + }() + + path := filepath.Join(dest, f.Name) + + // Check for ZipSlip (Directory traversal) + if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) { + return fmt.Errorf("illegal file path: %s", path) + } + + if !f.FileInfo().IsDir() { + fileDest, accept := filter(f.Name) + if !accept { + return nil + } + fullPath := filepath.Join(dest, fileDest) + + return CopyReaderToFile(rc, fullPath, f.Mode()) + } + return nil + } + + for _, f := range r.File { + if err = extractAndWriteFile(f); err != nil { + return err + } + } + + return nil +} + +func UnTarGz(src, dest string, f func(path string) (string, bool)) error { + fr, err := os.Open(src) + if err != nil { + return err + } + + defer func() { + if err = fr.Close(); err != nil { + panic(err) + } + }() + + gzr, err := gzip.NewReader(fr) + if err != nil { + return err + } + + r := tar.NewReader(gzr) + + for { + header, err := r.Next() + if err == io.EOF { + return nil + } + + if err != nil { + return err + } + + if header == nil { + continue + } + + if header.Typeflag == tar.TypeReg { + fileDest, accept := f(header.Name) + if !accept { + continue + } + fullPath := filepath.Join(dest, fileDest) + + if err = CopyReaderToFile(r, fullPath, os.FileMode(header.Mode)); err != nil { + return err + } + } + } +} + +func CopyReaderToFile(in io.Reader, dst string, mode os.FileMode) error { + out, err := os.OpenFile(dst, os.O_CREATE|os.O_RDWR, mode) + if err != nil { + return err + } + + defer func() { + if e := out.Close(); e != nil { + pfxlog.Logger().WithError(err).Errorf("failed to copy dest file (%s)", dst) + } + }() + + _, err = io.Copy(out, in) + if err != nil { + return err + } + + return out.Sync() +} diff --git a/common/getziti/github.go b/common/getziti/github.go index aa56d9186..0e62f9687 100644 --- a/common/getziti/github.go +++ b/common/getziti/github.go @@ -1,12 +1,15 @@ package getziti import ( + "context" "fmt" "github.com/blang/semver" "github.com/go-resty/resty/v2" "github.com/michaelquigley/pfxlog" + c "github.com/openziti/ziti/ziti/constants" "github.com/pkg/errors" "net/http" + "net/url" "os" "path/filepath" "sort" @@ -115,7 +118,7 @@ func GetLatestGitHubReleaseAsset(appName string, appGitHub string, version strin version = strings.TrimPrefix(version, "v") } - if appName == "ziti-edge-tunnel" { + if appName == "ziti" || appName == "ziti-edge-tunnel" { if !strings.HasPrefix(version, "v") { version = "v" + version } @@ -126,7 +129,10 @@ func GetLatestGitHubReleaseAsset(appName string, appGitHub string, version strin version = "tags/" + version } + ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) + resp, err := getRequest(verbose). + SetContext(ctx). SetQueryParams(map[string]string{}). SetHeader("Accept", "application/vnd.github.v3+json"). SetResult(&GitHubReleasesData{}). @@ -149,7 +155,12 @@ func GetLatestGitHubReleaseAsset(appName string, appGitHub string, version strin // DownloadGitHubReleaseAsset will download a file from the given GitHUb release area func DownloadGitHubReleaseAsset(fullUrl string, filepath string) (err error) { - resp, err := getRequest(false). + resp, err := resty. + New(). + SetTimeout(time.Minute). + SetRetryCount(2). + SetRedirectPolicy(resty.FlexibleRedirectPolicy(15)). + R(). SetOutput(filepath). Get(fullUrl) @@ -177,37 +188,86 @@ func FindVersionAndInstallGitHubRelease(zitiApp string, zitiAppGitHub string, ta if err != nil { return err } - return InstallGitHubRelease(zitiApp, release, binDir, targetOS, targetArch) + return InstallGitHubRelease(zitiApp, release, binDir, targetOS, targetArch, version) } -func InstallGitHubRelease(zitiApp string, release *GitHubReleasesData, binDir string, targetOS, targetArch string) error { - fileName := zitiApp - if targetOS == "windows" { - fileName += ".exe" - } - - fullPath := filepath.Join(binDir, fileName) - ext := ".zip" - zipFile := fullPath + ext - +func InstallGitHubRelease(zitiApp string, release *GitHubReleasesData, binDir string, targetOS, targetArch, version string) error { releaseUrl, err := release.GetDownloadUrl(zitiApp, targetOS, targetArch) if err != nil { return err } - err = DownloadGitHubReleaseAsset(releaseUrl, zipFile) + parsedUrl, err := url.Parse(releaseUrl) if err != nil { return err } - err = Unzip(zipFile, binDir) + fileName := filepath.Base(parsedUrl.Path) + + fullPath := filepath.Join(binDir, fileName) + err = DownloadGitHubReleaseAsset(releaseUrl, fullPath) if err != nil { return err } - err = os.Remove(zipFile) - if err != nil { - return err + + defer func() { + err = os.Remove(fullPath) + if err != nil { + pfxlog.Logger().WithError(err).Errorf("failed to removed downloaded release archive %s", fullPath) + } + }() + + if strings.HasSuffix(fileName, ".zip") { + if zitiApp == c.ZITI_EDGE_TUNNEL { + count := 0 + zitiFileName := "ziti-edge-tunnel-" + version + err = Unzip(fullPath, binDir, func(path string) (string, bool) { + if path == "ziti-edge-tunnel" { + count++ + return zitiFileName, true + } + return "", false + }) + + if err != nil { + return err + } + + if count != 1 { + return errors.Errorf("didn't find ziti-edge-tunnel executable in release archive. count: %v", count) + } + + pfxlog.Logger().Infof("Successfully installed '%s' version '%s' to %s", zitiApp, release.Version, filepath.Join(binDir, zitiFileName)) + return nil + } else { + return errors.Errorf("unsupported application '%s'", zitiApp) + } + } else if strings.HasSuffix(fileName, ".tar.gz") { + if zitiApp == c.ZITI { + count := 0 + zitiFileName := "ziti-" + version + err = UnTarGz(fullPath, binDir, func(path string) (string, bool) { + if path == "ziti/ziti" { + count++ + return zitiFileName, true + } + return "", false + }) + + if err != nil { + return err + } + + if count != 1 { + return errors.Errorf("didn't find ziti executable in release archive. count: %v", count) + } + + pfxlog.Logger().Infof("Successfully installed '%s' version '%s' to %s", zitiApp, release.Version, filepath.Join(binDir, zitiFileName)) + return nil + } else { + return errors.Errorf("unsupported application '%s'", zitiApp) + } + } else { + return errors.Errorf("unsupported release file type '%s'", fileName) } - pfxlog.Logger().Infof("Successfully installed '%s' version '%s'", zitiApp, release.Version) - return os.Chmod(fullPath, 0755) } diff --git a/common/getziti/install_ziti.go b/common/getziti/install_ziti.go new file mode 100644 index 000000000..ecb285c2b --- /dev/null +++ b/common/getziti/install_ziti.go @@ -0,0 +1,26 @@ +package getziti + +import ( + "fmt" + "github.com/blang/semver" + c "github.com/openziti/ziti/ziti/constants" + "strings" +) + +func InstallZiti(targetVersion, targetOS, targetArch, binDir string, verbose bool) error { + var newVersion semver.Version + + if targetVersion != "" { + newVersion = semver.MustParse(strings.TrimPrefix(targetVersion, "v")) + } else { + v, err := GetLatestGitHubReleaseVersion(c.ZITI, verbose) + if err != nil { + return err + } + newVersion = v + } + + fmt.Println("Attempting to install '" + c.ZITI + "' version: v" + newVersion.String()) + return FindVersionAndInstallGitHubRelease( + c.ZITI, c.ZITI, targetOS, targetArch, binDir, "v"+newVersion.String(), verbose) +} diff --git a/common/getziti/unzip.go b/common/getziti/unzip.go deleted file mode 100644 index 7770442da..000000000 --- a/common/getziti/unzip.go +++ /dev/null @@ -1,74 +0,0 @@ -package getziti - -import ( - "archive/zip" - "fmt" - "io" - "os" - "path/filepath" - "strings" -) - -func Unzip(src, dest string) error { - r, err := zip.OpenReader(src) - if err != nil { - return err - } - defer func() { - if err := r.Close(); err != nil { - panic(err) - } - }() - - os.MkdirAll(dest, 0755) - - // Closure to address file descriptors issue with all the deferred .Close() methods - extractAndWriteFile := func(f *zip.File) error { - rc, err := f.Open() - if err != nil { - return err - } - defer func() { - if err := rc.Close(); err != nil { - panic(err) - } - }() - - path := filepath.Join(dest, f.Name) - - // Check for ZipSlip (Directory traversal) - if !strings.HasPrefix(path, filepath.Clean(dest)+string(os.PathSeparator)) { - return fmt.Errorf("illegal file path: %s", path) - } - - if f.FileInfo().IsDir() { - os.MkdirAll(path, f.Mode()) - } else { - os.MkdirAll(filepath.Dir(path), f.Mode()) - f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) - if err != nil { - return err - } - defer func() { - if err := f.Close(); err != nil { - panic(err) - } - }() - - _, err = io.Copy(f, rc) - if err != nil { - return err - } - } - return nil - } - - for _, f := range r.File { - err := extractAndWriteFile(f) - if err != nil { - return err - } - } - - return nil -} diff --git a/ziti/cmd/demo/echo_server.go b/ziti/cmd/demo/echo_server.go index 768b05606..80878ba4e 100644 --- a/ziti/cmd/demo/echo_server.go +++ b/ziti/cmd/demo/echo_server.go @@ -25,6 +25,7 @@ import ( "github.com/openziti/identity" "github.com/openziti/sdk-golang/ziti" "github.com/openziti/sdk-golang/ziti/edge" + "github.com/openziti/ziti/common/version" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -52,6 +53,10 @@ type echoServer struct { bindWithIdentity bool zitiIdentity identity.Identity zitiListener edge.Listener + + cliAgentEnabled bool + cliAgentAddr string + cliAgentAlias string } func newEchoServerCmd() *cobra.Command { @@ -73,7 +78,9 @@ func newEchoServerCmd() *cobra.Command { cmd.Flags().StringVarP(&server.configFile, "identity", "i", "", "Specify the Ziti identity to use. If not specified the Ziti listener won't be started") cmd.Flags().StringVarP(&server.service, "service", "s", "echo", "Ziti service to bind. Defaults to 'echo'") cmd.Flags().BoolVar(&server.bindWithIdentity, "addressable", false, "Specify if this application should be addressable by the edge identity") - cmd.Flags().SetInterspersed(true) + cmd.Flags().BoolVar(&server.cliAgentEnabled, "cli-agent", true, "Enable/disable CLI Agent (enabled by default)") + cmd.Flags().StringVar(&server.cliAgentAddr, "cli-agent-addr", "", "Specify where CLI Agent should list (ex: unix:/tmp/myfile.sock or tcp:127.0.0.1:10001)") + cmd.Flags().StringVar(&server.cliAgentAlias, "cli-agent-alias", "", "Alias which can be used by ziti agent commands to find this instance") return cmd } @@ -108,6 +115,19 @@ func (self *echoServer) run(*cobra.Command, []string) { log.Fatalf("No port specified and no ziti identity specified. Not starting either listener, so nothing to do. exiting.") } + if self.cliAgentEnabled { + options := agent.Options{ + Addr: self.cliAgentAddr, + AppType: "echo-server", + AppVersion: version.GetVersion(), + AppAlias: self.cliAgentAlias, + } + + if err := agent.Listen(options); err != nil { + pfxlog.Logger().WithError(err).Error("unable to start CLI agent") + } + } + if self.healthCheckAddr != "" { go func() { err := http.ListenAndServe(self.healthCheckAddr, http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { diff --git a/ziti/cmd/demo/zcat.go b/ziti/cmd/demo/zcat.go index 3635cbc66..45b0d0c15 100644 --- a/ziti/cmd/demo/zcat.go +++ b/ziti/cmd/demo/zcat.go @@ -20,6 +20,7 @@ import ( "github.com/michaelquigley/pfxlog" "github.com/openziti/foundation/v2/info" "github.com/openziti/sdk-golang/ziti" + "github.com/openziti/sdk-golang/ziti/edge" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "io" @@ -138,4 +139,9 @@ func (self *zcatAction) copy(writer io.Writer, reader io.Reader) { if err != nil { pfxlog.Logger().WithError(err).Error("error while copying bytes") } + if zconn, ok := writer.(edge.Conn); ok { + if err = zconn.CloseWrite(); err != nil { + pfxlog.Logger().WithError(err).Error("error closing write side of ziti connection") + } + } } diff --git a/ziti/cmd/pki.go b/ziti/cmd/pki.go index 6eec79dfd..e1991f270 100644 --- a/ziti/cmd/pki.go +++ b/ziti/cmd/pki.go @@ -60,6 +60,7 @@ type PKIFlags struct { Email []string PKI *pki.ZitiPKI SpiffeID string + AllowOverwrite bool } var ( diff --git a/ziti/cmd/pki_create_server.go b/ziti/cmd/pki_create_server.go index 7a3e5995e..6cc578e3d 100644 --- a/ziti/cmd/pki_create_server.go +++ b/ziti/cmd/pki_create_server.go @@ -77,6 +77,7 @@ func (o *PKICreateServerOptions) addPKICreateServerFlags(cmd *cobra.Command) { cmd.Flags().IntVarP(&o.Flags.CAMaxpath, "max-path-len", "", -1, "Intermediate maximum path length") cmd.Flags().IntVarP(&o.Flags.CAPrivateKeySize, "private-key-size", "", 4096, "Size of the private key") cmd.Flags().StringVar(&o.Flags.SpiffeID, "spiffe-id", "", "Optionally provide the path portion of a SPIFFE id. The trust domain will be taken from the signing certificate.") + cmd.Flags().BoolVar(&o.Flags.AllowOverwrite, "allow-overwrite", false, "Allow overwrite existing certs") } // Run implements this command @@ -156,6 +157,7 @@ func (o *PKICreateServerOptions) Run() error { Template: template, IsClientCertificate: false, PrivateKeySize: o.Flags.CAPrivateKeySize, + AllowOverwrite: o.Flags.AllowOverwrite, } if err := o.Flags.PKI.Sign(signer, req); err != nil { diff --git a/ziti/pki/pki/pki.go b/ziti/pki/pki/pki.go index f9207a8da..e33224f76 100644 --- a/ziti/pki/pki/pki.go +++ b/ziti/pki/pki/pki.go @@ -50,7 +50,9 @@ type Request struct { IsClientCertificate bool PrivateKeySize int Template *x509.Certificate + AllowOverwrite bool } + type CSRRequest struct { Name string IsClientCertificate bool @@ -146,7 +148,7 @@ func (e *ZitiPKI) Sign(signer *certificate.Bundle, req *Request) error { return fmt.Errorf("failed creating and signing certificate: %v", err) } - if err := e.Store.Add(signer.Name, req.Name, req.Template.IsCA, x509.MarshalPKCS1PrivateKey(privateKey), rawCert); err != nil { + if err := e.Store.Add(signer.Name, req.Name, req.Template.IsCA, x509.MarshalPKCS1PrivateKey(privateKey), rawCert, req.AllowOverwrite); err != nil { return fmt.Errorf("failed saving generated bundle: %v", err) } return nil diff --git a/ziti/pki/store/local.go b/ziti/pki/store/local.go index f6a1eaf90..2cdea6ec4 100644 --- a/ziti/pki/store/local.go +++ b/ziti/pki/store/local.go @@ -120,8 +120,8 @@ func readPEM(path string) ([]byte, error) { } // Add adds the given bundle to the local filesystem. -func (l *Local) Add(caName, name string, isCa bool, key, cert []byte) error { - if l.Exists(caName, name) { +func (l *Local) Add(caName, name string, isCa bool, key, cert []byte, allowOverwrite bool) error { + if !allowOverwrite && l.Exists(caName, name) { return fmt.Errorf("a bundle already exists for the name %v within CA %v", name, caName) } if err := l.writeBundle(caName, name, isCa, key, cert); err != nil { diff --git a/ziti/pki/store/store.go b/ziti/pki/store/store.go index 0ac99c53f..1060bb6b0 100644 --- a/ziti/pki/store/store.go +++ b/ziti/pki/store/store.go @@ -33,9 +33,10 @@ type Store interface { // Is the bundle to add an intermediate CA. // The raw private key. // The raw certificate. + // Flag indicating if existing certs can be overwritten // // Returns an error if it failed to store the bundle. - Add(string, string, bool, []byte, []byte) error + Add(string, string, bool, []byte, []byte, bool) error // Chain concats a signing cert and a newly signed certificate bundle and adds the chained cert to the store. // diff --git a/ziti/tunnel/root.go b/ziti/tunnel/root.go index 4f7d51b3b..fe02ff061 100644 --- a/ziti/tunnel/root.go +++ b/ziti/tunnel/root.go @@ -68,6 +68,7 @@ func NewTunnelCmd(standalone bool) *cobra.Command { root.PersistentFlags().StringP(dnsSvcIpRangeFlag, "d", "100.64.0.1/10", "cidr to use when assigning IPs to unresolvable intercept hostnames") root.PersistentFlags().BoolVar(&cliAgentEnabled, "cli-agent", true, "Enable/disable CLI Agent (enabled by default)") root.PersistentFlags().StringVar(&cliAgentAddr, "cli-agent-addr", "", "Specify where CLI Agent should list (ex: unix:/tmp/myfile.sock or tcp:127.0.0.1:10001)") + root.PersistentFlags().StringVar(&cliAgentAlias, "cli-agent-alias", "", "Alias which can be used by ziti agent commands to find this instance") p := common.NewOptionsProvider(os.Stdout, os.Stderr) root.AddCommand(enrollment.NewEnrollCommand(p)) @@ -83,6 +84,7 @@ var interceptor intercept.Interceptor var logFormatter string var cliAgentEnabled bool var cliAgentAddr string +var cliAgentAlias string func Execute() { if err := NewTunnelCmd(true).Execute(); err != nil { @@ -120,7 +122,13 @@ func rootPostRun(cmd *cobra.Command, _ []string) { // don't use the agent's shutdown handler. it calls os.Exit on SIGINT // which interferes with the servicePoller shutdown cleanup := false - if err := agent.Listen(agent.Options{Addr: cliAgentAddr, ShutdownCleanup: &cleanup}); err != nil { + err := agent.Listen(agent.Options{ + Addr: cliAgentAddr, + ShutdownCleanup: &cleanup, + AppAlias: cliAgentAlias, + }) + + if err != nil { pfxlog.Logger().WithError(err).Error("unable to start CLI agent") } } diff --git a/zititest/go.mod b/zititest/go.mod index 204fba56f..593c1a6ab 100644 --- a/zititest/go.mod +++ b/zititest/go.mod @@ -7,25 +7,24 @@ replace github.com/openziti/ziti => ../ require ( github.com/Jeffail/gabs v1.4.0 github.com/Jeffail/gabs/v2 v2.7.0 + github.com/google/uuid v1.3.0 github.com/michaelquigley/pfxlog v0.6.10 github.com/openziti/channel/v2 v2.0.81 github.com/openziti/edge v0.24.348 - github.com/openziti/fablab v0.4.52 + github.com/openziti/fablab v0.5.0 github.com/openziti/fabric v0.23.39 github.com/openziti/foundation/v2 v2.0.26 github.com/openziti/identity v1.0.57 github.com/openziti/sdk-golang v0.20.67 github.com/openziti/storage v0.2.8 github.com/openziti/transport/v2 v2.0.91 - github.com/openziti/ziti v0.28.0 + github.com/openziti/ziti v0.28.3 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 - github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 go.etcd.io/bbolt v1.3.7 golang.org/x/net v0.11.0 google.golang.org/protobuf v1.30.0 - gopkg.in/yaml.v3 v3.0.1 ) require ( @@ -78,7 +77,6 @@ require ( github.com/golang/protobuf v1.5.2 // indirect github.com/gomarkdown/markdown v0.0.0-20230322041520-c84983bdbf2a // indirect github.com/google/go-cmp v0.5.9 // indirect - github.com/google/uuid v1.3.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/mux v1.8.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -152,6 +150,7 @@ require ( github.com/speps/go-hashids v2.0.0+incompatible // indirect github.com/spf13/afero v1.6.0 // indirect github.com/spf13/cast v1.4.1 // indirect + github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.10.0 // indirect @@ -165,7 +164,7 @@ require ( github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect github.com/xeipuuv/gojsonschema v1.2.0 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect - go.mongodb.org/mongo-driver v1.11.7 // indirect + go.mongodb.org/mongo-driver v1.12.0 // indirect go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 // indirect go.opentelemetry.io/otel v1.16.0 // indirect go.opentelemetry.io/otel/metric v1.16.0 // indirect @@ -184,6 +183,7 @@ require ( gopkg.in/resty.v1 v1.12.0 // indirect gopkg.in/square/go-jose.v2 v2.5.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect nhooyr.io/websocket v1.8.7 // indirect rsc.io/goversion v1.2.0 // indirect ) diff --git a/zititest/go.sum b/zititest/go.sum index 02fef3ee6..6add6d731 100644 --- a/zititest/go.sum +++ b/zititest/go.sum @@ -647,8 +647,8 @@ github.com/openziti/edge v0.24.348 h1:m0zh1n5OBj7Tjemu9XfxdYhbKvo6HdGQMHfC/Zledb github.com/openziti/edge v0.24.348/go.mod h1:wcoBVKqW99/CBg7LjzbHT3dJ8JMlRZelPhbjNcjwMr0= github.com/openziti/edge-api v0.25.29 h1:DbZ0GKpch9ik+XnC0qoP1dOg1AmgUEeWH4VIyOIehrQ= github.com/openziti/edge-api v0.25.29/go.mod h1:mqYquh1RYvHjV/M9MGO4Ng0Xq9ixAH8o8cT5kw5XCWY= -github.com/openziti/fablab v0.4.52 h1:X5+NaRnfPxbzhxUh9b4IQmVNVgYWFf8/Ck2sKNz77ho= -github.com/openziti/fablab v0.4.52/go.mod h1:aF8Fu3C8hnnaOJyAZz3AXGY4cMxQyBLhoIlSRJ/VVpc= +github.com/openziti/fablab v0.5.0 h1:+CMVE08hS62UR3tIfEvMb3XcWwzD1np//cpHHTboVW8= +github.com/openziti/fablab v0.5.0/go.mod h1:VLvs0AAuKAFqy8jhGvs+FgszYEbB456duPOBfRj4peU= github.com/openziti/fabric v0.23.39 h1:t6npkfBsPr+8GvF9X/rTLHxGsce5FTa4NoA80MInxYc= github.com/openziti/fabric v0.23.39/go.mod h1:px2zUzQrgObmv6sdXgGJ028Wr45aGcgpFvfIP/qgDFo= github.com/openziti/foundation/v2 v2.0.26 h1:RISGM97aIFnnEkxrcVsO6fnGg9W4ByydSnqWAVkJAqY= @@ -819,7 +819,6 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125 h1:3SNcvBmEPE1YlB1JpVZouslJpI3GBNoiqW7+wb0Rz7w= github.com/teris-io/shortid v0.0.0-20201117134242-e59966efd125/go.mod h1:M8agBzgqHIhgj7wEn9/0hJUZcrvt9VY+Ln+S1I5Mha0= -github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= @@ -842,8 +841,10 @@ github.com/vultr/govultr/v2 v2.0.0/go.mod h1:2PsEeg+gs3p/Fo5Pw8F9mv+DUBEOlrNZ8Gm github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= @@ -867,8 +868,8 @@ go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsX go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.11.7 h1:LIwYxASDLGUg/8wOhgOOZhX8tQa/9tgZPgzZoVqJvcs= -go.mongodb.org/mongo-driver v1.11.7/go.mod h1:G9TgswdsWjX4tmDA5zfs2+6AEPpYJwqblyjsfuh8oXY= +go.mongodb.org/mongo-driver v1.12.0 h1:aPx33jmn/rQuJXPQLZQ8NtfPQG8CaqgLThFtqRb0PiE= +go.mongodb.org/mongo-driver v1.12.0/go.mod h1:AZkxhPnFJUoH7kZlFkVKucV20K387miPfm7oimrSmK0= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -1143,6 +1144,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= diff --git a/zititest/models/ha/actions/start.go b/zititest/models/ha/actions/start.go index 0b615c754..4b4b232cc 100644 --- a/zititest/models/ha/actions/start.go +++ b/zititest/models/ha/actions/start.go @@ -7,7 +7,6 @@ import ( "github.com/openziti/fablab/kernel/lib/actions/component" "github.com/openziti/fablab/kernel/lib/actions/semaphore" "github.com/openziti/fablab/kernel/model" - util_actions "github.com/openziti/ziti/zititest/utils/actions" zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" "github.com/openziti/ziti/zititest/zitilab/models" ) @@ -30,7 +29,7 @@ func (a *startAction) bind(m *model.Model) model.Action { workflow.AddAction(zitilib_actions.StartMetricbeat("*", a.Metricbeat.ConfigPath, a.Metricbeat.DataPath, a.Metricbeat.LogPath)) workflow.AddAction(zitilib_actions.StartConsul("*", a.Consul.ServerAddr, a.Consul.ConfigDir, a.Consul.DataPath, a.Consul.LogPath)) workflow.AddAction(semaphore.Sleep(2 * time.Second)) - workflow.AddAction(util_actions.StartEchoServers("#echo-server")) + workflow.AddAction(component.Start("#echo-server")) workflow.AddAction(semaphore.Sleep(2 * time.Second)) return workflow diff --git a/zititest/models/ha/configs/ctrl.yml.tmpl b/zititest/models/ha/configs/ctrl.yml.tmpl index b5add84e4..94f690531 100644 --- a/zititest/models/ha/configs/ctrl.yml.tmpl +++ b/zititest/models/ha/configs/ctrl.yml.tmpl @@ -5,15 +5,15 @@ raft: dataDir: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrldata identity: - cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/certs/{{ .Component.PublicIdentity }}-server.cert - key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/keys/{{ .Component.PublicIdentity }}-server.key - ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/certs/{{ .Component.PublicIdentity }}-server.chain.pem + cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.cert + key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}-server.key + ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}-server.chain.pem # the endpoint that routers will connect to the controller over. ctrl: listener: tls:0.0.0.0:6262 options: - advertiseAddress: tls:{{.Host.PublicIp}}:6262 + advertiseAddress: tls:{{ .Host.PublicIp }}:6262 # (optional) settings # set the maximum number of connect requests that are buffered and waiting to be acknowledged (1 to 5000, default 1000) #maxQueuedConnects: 50 @@ -82,7 +82,7 @@ edge: # address - required # The default address (host:port) to use for enrollment for the Client API. This value must match one of the addresses # defined in a bind point's address field for the `edge-client` API in the web section. - address: {{.Host.PublicIp}}:1280 + address: {{ .Host.PublicIp }}:1280 # enrollment - required # A section containing settings pertaining to enrollment. enrollment: @@ -91,8 +91,8 @@ edge: # a signing certificate from the PKI that the Ziti environment is using to sign certificates. The signingCert.cert # will be added to the /.well-known CA store that is used to bootstrap trust with the Ziti Controller. signingCert: - cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/certs/{{ .Component.PublicIdentity }}.cert - key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.PublicIdentity }}/keys/{{ .Component.PublicIdentity }}.key + cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/certs/{{ .Component.Id }}.cert + key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/{{ .Component.Id }}/keys/{{ .Component.Id }}.key # edgeIdentity - optional # A section for identity enrollment specific settings @@ -130,7 +130,7 @@ web: # address - required # The public address that external incoming requests will be able to resolve. Used in request processing and # response content that requires full host:port/path addresses. - address: {{.Host.PublicIp}}:1280 + address: {{ .Host.PublicIp }}:1280 # newAddress - optional # A host:port string which will be sent out as an HTTP header "ziti-new-address" if specified. If the header diff --git a/zititest/models/ha/main.go b/zititest/models/ha/main.go index 3212bc922..dc049a4c7 100644 --- a/zititest/models/ha/main.go +++ b/zititest/models/ha/main.go @@ -25,8 +25,6 @@ import ( "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key" semaphore0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore" terraform_0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform" - "github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config" - "github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit" distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution" "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync" aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key" @@ -37,7 +35,6 @@ import ( "github.com/openziti/ziti/zititest/models/test_resources" "github.com/openziti/ziti/zititest/zitilab" "github.com/openziti/ziti/zititest/zitilab/actions/edge" - zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration" "github.com/sirupsen/logrus" "os" "time" @@ -85,14 +82,8 @@ var m = &model.Model{ InstanceType: "t3.micro", Components: model.Components{ "ctrl1": { - Scope: model.Scope{Tags: model.Tags{"ctrl", "spiffe:controller"}}, - BinaryName: "ziti controller", - ConfigSrc: "ctrl.yml.tmpl", - ConfigName: "ctrl1.yml", - PublicIdentity: "ctrl1", - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"ctrl"}}, + Type: &zitilab.ControllerType{}, }, }, }, @@ -100,14 +91,8 @@ var m = &model.Model{ InstanceType: "t3.micro", Components: model.Components{ "ctrl2": { - Scope: model.Scope{Tags: model.Tags{"ctrl", "spiffe:controller"}}, - BinaryName: "ziti controller", - ConfigSrc: "ctrl.yml.tmpl", - ConfigName: "ctrl2.yml", - PublicIdentity: "ctrl2", - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"ctrl"}}, + Type: &zitilab.ControllerType{}, }, }, }, @@ -116,19 +101,12 @@ var m = &model.Model{ InstanceType: "t2.micro", Components: model.Components{ "router-east": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}}, - BinaryName: "ziti router", - ConfigSrc: "router.yml.tmpl", - ConfigName: "router-east.yml", - PublicIdentity: "router-east", + Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}}, + Type: &zitilab.RouterType{}, }, "echo-server": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}}, - BinaryName: "echo-server", - PublicIdentity: "echo-server", - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}}, + Type: &zitilab.EchoServerType{}, }, }, }, @@ -142,14 +120,8 @@ var m = &model.Model{ InstanceType: "t3.micro", Components: model.Components{ "ctrl3": { - Scope: model.Scope{Tags: model.Tags{"ctrl", "spiffe:controller"}}, - BinaryName: "ziti controller", - ConfigSrc: "ctrl.yml.tmpl", - ConfigName: "ctrl3.yml", - PublicIdentity: "ctrl3", - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"ctrl"}}, + Type: &zitilab.ControllerType{}, }, }, }, @@ -159,19 +131,12 @@ var m = &model.Model{ InstanceType: "t2.micro", Components: model.Components{ "router-west": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}}, - BinaryName: "ziti router", - ConfigSrc: "router.yml.tmpl", - ConfigName: "router-west.yml", - PublicIdentity: "router-west", + Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator"}}, + Type: &zitilab.RouterType{}, }, - "echo-client": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}}, - BinaryName: "echo-client", - PublicIdentity: "echo-client", - }, - "consul": { - BinaryName: "consul", + "zcat": { + Scope: model.Scope{Tags: model.Tags{"sdk-app"}}, + Type: &zitilab.ZCatType{}, }, }, }, @@ -196,21 +161,13 @@ var m = &model.Model{ "login": model.Bind(edge.Login("#ctrl1")), }, - Infrastructure: model.InfrastructureStages{ + Infrastructure: model.Stages{ aws_ssh_key.Express(), terraform_0.Express(), semaphore0.Ready(time.Minute), }, - Configuration: model.ConfigurationStages{ - zitilib_runlevel_1_configuration.IfPkiNeedsRefresh( - zitilib_runlevel_1_configuration.Fabric("simple-transfer.test", ".ctrl"), - ), - config.Component(), - devkit.DevKitF(zitilab.ZitiRoot, []string{"ziti", "ziti-echo"}), - }, - - Distribution: model.DistributionStages{ + Distribution: model.Stages{ distribution.DistributeSshKey("*"), distribution.Locations("*", "logs"), distribution.DistributeDataWithReplaceCallbacks( @@ -277,7 +234,7 @@ var m = &model.Model{ rsync.RsyncStaged(), }, - Disposal: model.DisposalStages{ + Disposal: model.Stages{ terraform.Dispose(), aws_ssh_key2.Dispose(), }, diff --git a/zititest/models/router-test/main.go b/zititest/models/router-test/main.go index 33bba3da6..ef8aa404e 100644 --- a/zititest/models/router-test/main.go +++ b/zititest/models/router-test/main.go @@ -14,8 +14,6 @@ import ( "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key" "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore" "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform" - "github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config" - "github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit" distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution" "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync" aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key" @@ -26,7 +24,6 @@ import ( "github.com/openziti/ziti/zititest/zitilab" "github.com/openziti/ziti/zititest/zitilab/actions/edge" "github.com/openziti/ziti/zititest/zitilab/models" - zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration" "os" "path" "strings" @@ -73,10 +70,8 @@ func (d dbStrategy) GetSite(router *persistence.EdgeRouter) (string, bool) { } func (d dbStrategy) PostProcess(router *persistence.EdgeRouter, c *model.Component) { - c.PublicIdentity = router.Id if router.IsTunnelerEnabled { c.Scope.Tags = append(c.Scope.Tags, "tunneler") - c.RunWithSudo = true } c.Scope.Tags = append(c.Scope.Tags, "edge-router") c.Scope.Tags = append(c.Scope.Tags, "pre-created") @@ -123,11 +118,7 @@ var m = &model.Model{ InstanceType: "c5.large", Components: model.Components{ "ctrl": { - Scope: model.Scope{Tags: model.Tags{"ctrl"}}, - BinaryName: "ziti controller", - ConfigSrc: "ctrl.yml", - ConfigName: "ctrl.yml", - PublicIdentity: "ctrl", + Type: &zitilab.RouterType{}, }, }, }, @@ -158,28 +149,20 @@ var m = &model.Model{ "login": model.Bind(edge.Login("#ctrl")), }, - Infrastructure: model.InfrastructureStages{ + Infrastructure: model.Stages{ aws_ssh_key.Express(), terraform_0.Express(), semaphore_0.Ready(90 * time.Second), }, - Configuration: model.ConfigurationStages{ - zitilib_runlevel_1_configuration.IfPkiNeedsRefresh( - zitilib_runlevel_1_configuration.Fabric("router.test", ".ctrl"), - ), - config.Component(), - devkit.DevKitF(zitilab.ZitiRoot, []string{"ziti"}), - }, - - Distribution: model.DistributionStages{ + Distribution: model.Stages{ distribution.DistributeSshKey("*"), distribution.Locations("*", "logs"), rsync.RsyncStaged(), rsync.NewRsyncHost("#ctrl", getDbFile(), "/home/ubuntu/fablab/ctrl.db"), }, - Disposal: model.DisposalStages{ + Disposal: model.Stages{ terraform.Dispose(), aws_ssh_key2.Dispose(), }, diff --git a/zititest/models/simple/actions/bootstrap.go b/zititest/models/simple/actions/bootstrap.go index 65feab4c0..f9c7f4391 100644 --- a/zititest/models/simple/actions/bootstrap.go +++ b/zititest/models/simple/actions/bootstrap.go @@ -17,6 +17,10 @@ package actions import ( + "fmt" + "github.com/openziti/ziti/zititest/zitilab" + "time" + "github.com/openziti/fablab/kernel/lib/actions" "github.com/openziti/fablab/kernel/lib/actions/component" "github.com/openziti/fablab/kernel/lib/actions/host" @@ -24,7 +28,6 @@ import ( zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" "github.com/openziti/ziti/zititest/zitilab/actions/edge" "github.com/openziti/ziti/zititest/zitilab/models" - "time" ) type bootstrapAction struct{} @@ -39,7 +42,7 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action { workflow.AddAction(host.GroupExec("*", 25, "rm -f logs/*")) workflow.AddAction(component.Stop("#ctrl")) - workflow.AddAction(edge.InitController("#ctrl")) + workflow.AddAction(component.Exec("#ctrl", zitilab.ControllerActionInitStandalone)) workflow.AddAction(component.Start("#ctrl")) workflow.AddAction(edge.ControllerAvailable("#ctrl", 30*time.Second)) @@ -66,56 +69,52 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action { "protocol" : "tcp" }`)) - workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ert-unencrypted", "intercept.v1", ` + workflow.AddAction(zitilib_actions.Edge("create", "config", "iperf-host", "host.v1", ` { - "addresses": ["ziti-files-ert-unencrypted.s3-us-west-1.amazonaws.ziti"], - "portRanges" : [ { "low": 443, "high": 443 } ], - "protocols": ["tcp"] + "address" : "localhost", + "port" : 5201, + "protocol" : "tcp" }`)) - workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ert", "intercept.v1", ` - { - "addresses": ["ziti-files-ert.s3-us-west-1.amazonaws.ziti"], - "portRanges" : [ { "low": 443, "high": 443 } ], - "protocols": ["tcp"] - }`)) + for _, encrypted := range []bool{false, true} { + for _, hostType := range []string{"ert", "zet", "ziti-tunnel"} { + suffix := "" + encryptionFlag := "ON" - workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-zet-unencrypted", "intercept.v1", ` - { - "addresses": ["ziti-files-zet-unencrypted.s3-us-west-1.amazonaws.ziti"], - "portRanges" : [ { "low": 443, "high": 443 } ], - "protocols": ["tcp"] - }`)) + if !encrypted { + suffix = "-unencrypted" + encryptionFlag = "OFF" + } - workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-zet", "intercept.v1", ` - { - "addresses": ["ziti-files-zet.s3-us-west-1.amazonaws.ziti"], - "portRanges" : [ { "low": 443, "high": 443 } ], - "protocols": ["tcp"] - }`)) + filesConfigName := fmt.Sprintf("files-intercept-%s%s", hostType, suffix) + filesConfigDef := fmt.Sprintf(` + { + "addresses": ["files-%s%s.s3-us-west-1.amazonaws.ziti"], + "portRanges" : [ { "low": 443, "high": 443 } ], + "protocols": ["tcp"] + }`, hostType, suffix) - workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ziti-tunnel-unencrypted", "intercept.v1", ` - { - "addresses": ["ziti-files-ziti-tunnel-unencrypted.s3-us-west-1.amazonaws.ziti"], - "portRanges" : [ { "low": 443, "high": 443 } ], - "protocols": ["tcp"] - }`)) + workflow.AddAction(zitilib_actions.Edge("create", "config", filesConfigName, "intercept.v1", filesConfigDef)) - workflow.AddAction(zitilib_actions.Edge("create", "config", "files-intercept-ziti-tunnel", "intercept.v1", ` - { - "addresses": ["ziti-files-ziti-tunnel.s3-us-west-1.amazonaws.ziti"], - "portRanges" : [ { "low": 443, "high": 443 } ], - "protocols": ["tcp"] - }`)) + iperfConfigName := fmt.Sprintf("iperf-intercept-%s%s", hostType, suffix) + iperfConfigDef := fmt.Sprintf(` + { + "addresses": ["iperf-%s%s.ziti"], + "portRanges" : [ { "low": 5201, "high": 5201 } ], + "protocols": ["tcp"] + }`, hostType, suffix) - workflow.AddAction(zitilib_actions.Edge("create", "service", "ert-files-unencrypted", "-c", "files-host,files-intercept-ert-unencrypted", "-e", "OFF", "-a", "ert")) - workflow.AddAction(zitilib_actions.Edge("create", "service", "ert-files", "-c", "files-host,files-intercept-ert", "-a", "ert")) + workflow.AddAction(zitilib_actions.Edge("create", "config", iperfConfigName, "intercept.v1", iperfConfigDef)) - workflow.AddAction(zitilib_actions.Edge("create", "service", "zet-files-unencrypted", "-c", "files-host,files-intercept-zet-unencrypted", "-e", "OFF", "-a", "zet")) - workflow.AddAction(zitilib_actions.Edge("create", "service", "zet-files", "-c", "files-host,files-intercept-zet", "-a", "zet")) + filesServiceName := fmt.Sprintf("%s-files%s", hostType, suffix) + filesConfigs := fmt.Sprintf("files-host,%s", filesConfigName) + workflow.AddAction(zitilib_actions.Edge("create", "service", filesServiceName, "-c", filesConfigs, "-e", encryptionFlag, "-a", hostType)) - workflow.AddAction(zitilib_actions.Edge("create", "service", "ziti-tunnel-files-unencrypted", "-c", "files-host,files-intercept-ziti-tunnel-unencrypted", "-e", "OFF", "-a", "ziti-tunnel")) - workflow.AddAction(zitilib_actions.Edge("create", "service", "ziti-tunnel-files", "-c", "files-host,files-intercept-ziti-tunnel", "-a", "ziti-tunnel")) + iperfServiceName := fmt.Sprintf("%s-iperf%s", hostType, suffix) + iperfConfigs := fmt.Sprintf("iperf-host,%s", iperfConfigName) + workflow.AddAction(zitilib_actions.Edge("create", "service", iperfServiceName, "-c", iperfConfigs, "-e", encryptionFlag, "-a", hostType)) + } + } workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "ert-hosts", "Bind", "--service-roles", "#ert", "--identity-roles", "#ert-host")) workflow.AddAction(zitilib_actions.Edge("create", "service-policy", "zet-hosts", "Bind", "--service-roles", "#zet", "--identity-roles", "#zet-host")) @@ -127,16 +126,5 @@ func (a *bootstrapAction) bind(m *model.Model) model.Action { workflow.AddAction(component.Stop(models.ControllerTag)) - workflow.AddAction(model.ActionFunc(func(m *model.Model) error { - cmds := []string{ - "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", - "sudo systemctl restart systemd-resolved", - } - - return m.ForEachComponent(".ziti-tunnel", 2, func(c *model.Component) error { - return host.Exec(c.GetHost(), cmds...).Execute(c.GetModel()) - }) - })) - return workflow } diff --git a/zititest/models/simple/actions/start.go b/zititest/models/simple/actions/start.go index a5deccf4b..576267706 100644 --- a/zititest/models/simple/actions/start.go +++ b/zititest/models/simple/actions/start.go @@ -17,8 +17,6 @@ package actions import ( - "fmt" - "github.com/openziti/fablab/kernel/lib" "github.com/openziti/ziti/zititest/zitilab/actions/edge" "time" @@ -26,7 +24,6 @@ import ( "github.com/openziti/fablab/kernel/lib/actions/component" "github.com/openziti/fablab/kernel/lib/actions/semaphore" "github.com/openziti/fablab/kernel/model" - util_actions "github.com/openziti/ziti/zititest/utils/actions" zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" "github.com/openziti/ziti/zititest/zitilab/models" ) @@ -44,25 +41,17 @@ func (a *startAction) bind(m *model.Model) model.Action { workflow.AddAction(component.Start("#ctrl")) workflow.AddAction(edge.ControllerAvailable("#ctrl", 30*time.Second)) workflow.AddAction(component.StartInParallel(models.EdgeRouterTag, 25)) + workflow.AddAction(component.StartInParallel(".iperf", 5)) workflow.AddAction(semaphore.Sleep(2 * time.Second)) workflow.AddAction(zitilib_actions.StartMetricbeat("*", a.Metricbeat.ConfigPath, a.Metricbeat.DataPath, a.Metricbeat.LogPath)) workflow.AddAction(zitilib_actions.StartConsul("*", a.Consul.ServerAddr, a.Consul.ConfigDir, a.Consul.DataPath, a.Consul.LogPath)) workflow.AddAction(semaphore.Sleep(2 * time.Second)) - workflow.AddAction(util_actions.StartEchoServers("#echo-server")) - workflow.AddAction(semaphore.Sleep(2 * time.Second)) + workflow.AddAction(component.StartInParallel(".sdk-app", 5)) - workflow.AddAction(model.ActionFunc(func(m *model.Model) error { - return m.ForEachComponent(".sdk-app", 5, func(c *model.Component) error { - factory := lib.NewSshConfigFactory(c.GetHost()) - - serviceCmd := fmt.Sprintf("nohup sudo /home/%s/fablab/bin/%s run -i /home/%s/fablab/cfg/%s > logs/%s.log 2>&1 &", - factory.User(), c.BinaryName, factory.User(), c.PublicIdentity+".json", c.PublicIdentity) - - _, err := lib.RemoteExec(factory, serviceCmd) - return err - }) - })) + workflow.AddAction(semaphore.Sleep(5 * time.Second)) + workflow.AddAction(zitilib_actions.Edge("list", "edge-routers", "limit none")) + workflow.AddAction(zitilib_actions.Edge("list", "terminators", "limit none")) return workflow } diff --git a/zititest/models/simple/configs/ctrl.yml b/zititest/models/simple/configs/ctrl.yml.tmpl similarity index 97% rename from zititest/models/simple/configs/ctrl.yml rename to zititest/models/simple/configs/ctrl.yml.tmpl index c57660cad..8e2f46c7b 100644 --- a/zititest/models/simple/configs/ctrl.yml +++ b/zititest/models/simple/configs/ctrl.yml.tmpl @@ -3,9 +3,9 @@ v: 3 db: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/ctrl.db identity: - cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.PublicIdentity }}-server.cert - key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/keys/{{ .Component.PublicIdentity }}-server.key - ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.PublicIdentity }}-server.chain.pem + cert: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.Id }}-server.cert + key: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/keys/{{ .Component.Id }}-server.key + ca: /home/{{ .Model.MustVariable "credentials.ssh.username" }}/fablab/pki/ctrl/certs/{{ .Component.Id }}-server.chain.pem # the endpoint that routers will connect to the controller over. ctrl: diff --git a/zititest/models/simple/configs/router.yml b/zititest/models/simple/configs/router.yml.tmpl similarity index 98% rename from zititest/models/simple/configs/router.yml rename to zititest/models/simple/configs/router.yml.tmpl index 056cd6d53..788799f78 100644 --- a/zititest/models/simple/configs/router.yml +++ b/zititest/models/simple/configs/router.yml.tmpl @@ -1,5 +1,5 @@ {{$ssh_username := .Model.MustVariable "credentials.ssh.username"}} -{{$identity := .Component.PublicIdentity}} +{{$identity := .Component.Id}} {{$ctrl_ip := publicIp "component#ctrl"}} {{$router_ip := .Host.PublicIp}} diff --git a/zititest/models/simple/simple.go b/zititest/models/simple/simple.go index a0c6a268a..a9436c4d0 100644 --- a/zititest/models/simple/simple.go +++ b/zititest/models/simple/simple.go @@ -25,8 +25,6 @@ import ( "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key" semaphore0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore" terraform_0 "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform" - "github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config" - "github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit" distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution" "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync" aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key" @@ -34,12 +32,9 @@ import ( "github.com/openziti/fablab/kernel/model" "github.com/openziti/fablab/resources" actions2 "github.com/openziti/ziti/zititest/models/simple/actions" - "github.com/openziti/ziti/zititest/models/simple/stages/5_operation" "github.com/openziti/ziti/zititest/models/test_resources" "github.com/openziti/ziti/zititest/zitilab" "github.com/openziti/ziti/zititest/zitilab/actions/edge" - zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration" - "github.com/openziti/ziti/zititest/zitilab/stageziti" "github.com/sirupsen/logrus" "os" "time" @@ -84,10 +79,14 @@ var Model = &model.Model{ model.FactoryFunc(func(m *model.Model) error { pfxlog.Logger().Infof("environment [%s]", m.MustStringVariable("environment")) m.AddActivationActions("stop", "bootstrap", "start") - m.AddOperatingStage(runlevel_5_operation.AssertEcho("#echo-client")) - return nil }), + model.FactoryFunc(func(m *model.Model) error { + return m.ForEachHost("*", 1, func(host *model.Host) error { + host.InstanceType = "t2.micro" + return nil + }) + }), }, Resources: model.Resources{ @@ -101,84 +100,51 @@ var Model = &model.Model{ Site: "us-east-1a", Hosts: model.Hosts{ "ctrl": { - InstanceType: "t2.micro", Components: model.Components{ "ctrl": { - Scope: model.Scope{Tags: model.Tags{"ctrl"}}, - BinaryName: "ziti controller", - ConfigSrc: "ctrl.yml", - ConfigName: "ctrl.yml", - PublicIdentity: "ctrl", - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"ctrl"}}, + Type: &zitilab.ControllerType{}, }, }, }, "router-east-1": { - Scope: model.Scope{Tags: model.Tags{"ert-client"}}, - InstanceType: "t2.micro", + Scope: model.Scope{Tags: model.Tags{"ert-client"}}, Components: model.Components{ "router-east-1": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client"}}, - BinaryName: "ziti router", - ConfigSrc: "router.yml", - ConfigName: "router-east-1.yml", - PublicIdentity: "router-east-1", - RunWithSudo: true, + Scope: model.Scope{Tags: model.Tags{"edge-router", "terminator", "tunneler", "client"}}, + Type: &zitilab.RouterType{}, }, - "echo-client": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}}, - BinaryName: "echo-client", - PublicIdentity: "echo-client", - }, - "consul": { - BinaryName: "consul", + "zcat": { + Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}}, + Type: &zitilab.ZCatType{}, }, }, }, "router-east-2": { - InstanceType: "t2.micro", Components: model.Components{ "router-east-2": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "initiator"}}, - BinaryName: "ziti router", - ConfigSrc: "router.yml", - ConfigName: "router-east-2.yml", - PublicIdentity: "router-east-2", - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"edge-router", "initiator"}}, + Type: &zitilab.RouterType{}, }, }, }, "ziti-edge-tunnel-client": { - Scope: model.Scope{Tags: model.Tags{"zet-client"}}, - InstanceType: "t2.micro", + Scope: model.Scope{Tags: model.Tags{"zet-client"}}, Components: model.Components{ "ziti-edge-tunnel-client": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}}, - BinaryName: "ziti-edge-tunnel", - PublicIdentity: "ziti-edge-tunnel-client", - RunWithSudo: true, - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}}, + Type: &zitilab.ZitiEdgeTunnelType{ + Version: "v0.21.4", + }, }, }, }, "ziti-tunnel-client": { - Scope: model.Scope{Tags: model.Tags{"ziti-tunnel-client"}}, - InstanceType: "t2.micro", + Scope: model.Scope{Tags: model.Tags{"ziti-tunnel-client"}}, Components: model.Components{ "ziti-tunnel-client": { - Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client"}}, - BinaryName: "ziti tunnel", - PublicIdentity: "ziti-tunnel-client", - RunWithSudo: true, - }, - "consul": { - BinaryName: "consul", + Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "client"}}, + Type: &zitilab.ZitiTunnelType{}, }, }, }, @@ -189,52 +155,46 @@ var Model = &model.Model{ Site: "us-west-2b", Hosts: model.Hosts{ "router-west": { - Scope: model.Scope{Tags: model.Tags{}}, - InstanceType: "t2.micro", Components: model.Components{ "router-west": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host"}}, - BinaryName: "ziti router", - ConfigSrc: "router.yml", - ConfigName: "router-west.yml", - PublicIdentity: "router-west", - RunWithSudo: true, + Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "host", "ert-host"}}, + Type: &zitilab.RouterType{}, }, "echo-server": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}}, - BinaryName: "echo-server", - PublicIdentity: "echo-server", + Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}}, + Type: &zitilab.EchoServerType{}, }, - "consul": { - BinaryName: "consul", + "iperf-server-ert": { + Scope: model.Scope{Tags: model.Tags{"iperf", "service"}}, + Type: &zitilab.IPerfServerType{}, }, }, }, "ziti-edge-tunnel-host": { - InstanceType: "t2.micro", Components: model.Components{ "ziti-edge-tunnel-host": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host"}}, - BinaryName: "ziti-edge-tunnel", - PublicIdentity: "ziti-edge-tunnel-host", - RunWithSudo: true, + Scope: model.Scope{Tags: model.Tags{"sdk-app", "host", "zet-host"}}, + Type: &zitilab.ZitiEdgeTunnelType{ + Version: "v0.21.4", + }, }, - "consul": { - BinaryName: "consul", + "iperf-server-zet": { + Scope: model.Scope{Tags: model.Tags{"iperf", "service"}}, + Type: &zitilab.IPerfServerType{}, }, }, }, "ziti-tunnel-host": { - InstanceType: "t2.micro", Components: model.Components{ "ziti-tunnel-host": { - Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host"}}, - BinaryName: "ziti tunnel", - PublicIdentity: "ziti-tunnel-host", - RunWithSudo: true, + Scope: model.Scope{Tags: model.Tags{"ziti-tunnel", "sdk-app", "host", "ziti-tunnel-host"}}, + Type: &zitilab.ZitiTunnelType{ + Mode: zitilab.ZitiTunnelModeHost, + }, }, - "consul": { - BinaryName: "consul", + "iperf-server-zt": { + Scope: model.Scope{Tags: model.Tags{"iperf", "service"}}, + Type: &zitilab.IPerfServerType{}, }, }, }, @@ -259,22 +219,17 @@ var Model = &model.Model{ "login": model.Bind(edge.Login("#ctrl")), }, - Infrastructure: model.InfrastructureStages{ + Infrastructure: model.Stages{ aws_ssh_key.Express(), - terraform_0.Express(), - semaphore0.Ready(time.Minute), + &terraform_0.Terraform{ + Retries: 3, + ReadyCheck: &semaphore0.ReadyStage{ + MaxWait: 90 * time.Second, + }, + }, }, - Configuration: model.ConfigurationStages{ - zitilib_runlevel_1_configuration.IfPkiNeedsRefresh( - zitilib_runlevel_1_configuration.Fabric("simple-transfer.test", "#ctrl"), - ), - config.Component(), - devkit.DevKitF(zitilab.ZitiRoot, []string{"ziti", "ziti-echo"}), - stageziti.FetchZitiEdgeTunnel("v0.21.4"), - }, - - Distribution: model.DistributionStages{ + Distribution: model.Stages{ distribution.DistributeSshKey("*"), distribution.Locations("*", "logs"), distribution.DistributeDataWithReplaceCallbacks( @@ -341,7 +296,7 @@ var Model = &model.Model{ rsync.RsyncStaged(), }, - Disposal: model.DisposalStages{ + Disposal: model.Stages{ terraform.Dispose(), aws_ssh_key2.Dispose(), }, diff --git a/zititest/models/simple/stages/5_operation/echo_client.go b/zititest/models/simple/stages/5_operation/echo_client.go deleted file mode 100644 index 4ae6f01aa..000000000 --- a/zititest/models/simple/stages/5_operation/echo_client.go +++ /dev/null @@ -1,49 +0,0 @@ -package runlevel_5_operation - -import ( - "crypto/rand" - "fmt" - "net/url" - "strings" - - "github.com/openziti/fablab/kernel/lib" - "github.com/openziti/fablab/kernel/model" - "github.com/sirupsen/logrus" -) - -type echoClient struct { - componentSpec string - message string -} - -func AssertEcho(componentSpec string) model.OperatingStage { - data := make([]byte, 10000) - _, _ = rand.Read(data) - - return &echoClient{ - componentSpec: componentSpec, - message: url.QueryEscape(string(data)), - } -} - -func (ec *echoClient) Operate(run model.Run) error { - return run.GetModel().ForEachComponent(ec.componentSpec, 1, func(c *model.Component) error { - ssh := lib.NewSshConfigFactory(c.GetHost()) - remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".json" - - echoClientCmd := fmt.Sprintf("/home/%s/fablab/bin/ziti-echo client --identity %s %s 2>&1", - ssh.User(), remoteConfigFile, ec.message) - - if output, err := lib.RemoteExec(ssh, echoClientCmd); err != nil { - logrus.Errorf("error starting echo client [%s] (%v)", output, err) - return err - } else { - //trim the newline ssh added - output = strings.TrimRight(output, "\n") - if output != ec.message { - return fmt.Errorf("got message [%s] expected [%s]", output, ec.message) - } - } - return nil - }) -} diff --git a/zititest/models/simple/stages/5_operation/echo_server.go b/zititest/models/simple/stages/5_operation/echo_server.go deleted file mode 100644 index bc69b453b..000000000 --- a/zititest/models/simple/stages/5_operation/echo_server.go +++ /dev/null @@ -1,41 +0,0 @@ -package runlevel_5_operation - -import ( - "fmt" - - "github.com/openziti/fablab/kernel/lib" - "github.com/openziti/fablab/kernel/model" - "github.com/sirupsen/logrus" -) - -type echoServer struct { - joiner chan struct{} - host *model.Host -} - -func EchoServer(host *model.Host, joiner chan struct{}) model.OperatingStage { - return &echoServer{ - joiner: joiner, - host: host, - } -} - -func (es *echoServer) Operate(run model.Run) error { - go func() { - defer func() { - if es.joiner != nil { - close(es.joiner) - logrus.Debug("closed joiner") - } - }() - - ssh := lib.NewSshConfigFactory(es.host) - - echoServerCmd := fmt.Sprintf("ziti edge tutorial ziti-echo-server") - - if output, err := lib.RemoteExec(ssh, echoServerCmd); err != nil { - logrus.Errorf("error starting echo server [%s] (%v)", output, err) - } - }() - return nil -} diff --git a/zititest/models/smart/actions/bootstrap.go b/zititest/models/smart/actions/bootstrap.go index e4f99f3d2..02eb9e8c5 100644 --- a/zititest/models/smart/actions/bootstrap.go +++ b/zititest/models/smart/actions/bootstrap.go @@ -44,7 +44,7 @@ func (self *bootstrapAction) bind(m *model.Model) model.Action { workflow.AddAction(semaphore.Sleep(2 * time.Second)) for _, router := range m.SelectComponents(models.RouterTag) { - cert := fmt.Sprintf("/intermediate/certs/%s-client.cert", router.PublicIdentity) + cert := fmt.Sprintf("/intermediate/certs/%s-client.cert", router.Id) workflow.AddAction(zitilib_actions.Fabric("create", "router", filepath.Join(model.PkiBuild(), cert))) } @@ -82,7 +82,7 @@ func (_ *bootstrapAction) createServiceActions(m *model.Model) ([]model.Action, for _, h := range hosts { serviceActions = append(serviceActions, zitilib_actions.Fabric("create", "service", h.GetId())) - serviceActions = append(serviceActions, zitilib_actions.Fabric("create", "terminator", h.GetId(), router.PublicIdentity, "tcp:"+h.PrivateIp+":8171")) + serviceActions = append(serviceActions, zitilib_actions.Fabric("create", "terminator", h.GetId(), router.Id, "tcp:"+h.PrivateIp+":8171")) } return serviceActions, nil diff --git a/zititest/models/smart/config/ctrl.yml b/zititest/models/smart/config/ctrl.yml.tmpl similarity index 100% rename from zititest/models/smart/config/ctrl.yml rename to zititest/models/smart/config/ctrl.yml.tmpl diff --git a/zititest/models/smart/main.go b/zititest/models/smart/main.go index 4025e87f9..59a873252 100644 --- a/zititest/models/smart/main.go +++ b/zititest/models/smart/main.go @@ -68,11 +68,8 @@ var smartrouting = &model.Model{ "ctrl": { Components: model.Components{ "ctrl": { - Scope: model.Scope{Tags: model.Tags{"ctrl"}}, - BinaryName: "ziti-controller", - ConfigSrc: "ctrl.yml", - ConfigName: "ctrl.yml", - PublicIdentity: "ctrl", + Scope: model.Scope{Tags: model.Tags{"ctrl"}}, + Type: &zitilab.ControllerType{}, }, }, }, @@ -80,11 +77,10 @@ var smartrouting = &model.Model{ Scope: model.Scope{Tags: model.Tags{"iperf_server"}}, Components: model.Components{ "001": { - Scope: model.Scope{Tags: model.Tags{"initiator", "router"}}, - BinaryName: "ziti-router", - ConfigSrc: "ingress_router.yml", - ConfigName: "001.yml", - PublicIdentity: "001", + Scope: model.Scope{Tags: model.Tags{"initiator", "router"}}, + Type: &zitilab.RouterType{ + ConfigSource: "ingress_router.yml.tmpl", + }, }, }, }, @@ -110,10 +106,9 @@ var smartrouting = &model.Model{ Scope: model.Scope{Tags: model.Tags{"router"}}, Components: model.Components{ "002": { - BinaryName: "ziti-router", - ConfigSrc: "transit_router.yml", - ConfigName: "002.yml", - PublicIdentity: "002", + Type: &zitilab.RouterType{ + ConfigSource: "transit_router.yml.tmpl", + }, }, }, }, @@ -126,11 +121,10 @@ var smartrouting = &model.Model{ "004": { Components: model.Components{ "004": { - Scope: model.Scope{Tags: model.Tags{"router"}}, - BinaryName: "ziti-router", - ConfigSrc: "transit_router.yml", - ConfigName: "004.yml", - PublicIdentity: "004", + Scope: model.Scope{Tags: model.Tags{"router"}}, + Type: &zitilab.RouterType{ + ConfigSource: "transit_router.yml.tmpl", + }, }, }, }, @@ -143,11 +137,10 @@ var smartrouting = &model.Model{ "003": { Components: model.Components{ "003": { - Scope: model.Scope{Tags: model.Tags{"router", "terminator"}}, - BinaryName: "ziti-router", - ConfigSrc: "egress_router.yml", - ConfigName: "003.yml", - PublicIdentity: "003", + Scope: model.Scope{Tags: model.Tags{"router", "terminator"}}, + Type: &zitilab.RouterType{ + ConfigSource: "egress_router.yml.tmpl", + }, }, }, }, diff --git a/zititest/models/smart/stages.go b/zititest/models/smart/stages.go index 795aee737..f0772641d 100644 --- a/zititest/models/smart/stages.go +++ b/zititest/models/smart/stages.go @@ -28,9 +28,7 @@ import ( aws_ssh_keys6 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key" terraform6 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/terraform" "github.com/openziti/fablab/kernel/model" - "github.com/openziti/ziti/zititest/zitilab" "github.com/openziti/ziti/zititest/zitilab/models" - zitilab_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration" zitilab_5_operation "github.com/openziti/ziti/zititest/zitilab/runlevel/5_operation" "time" ) @@ -43,7 +41,7 @@ func newStageFactory() model.Factory { // represent the various phases of the model. func (self *stageFactory) Build(m *model.Model) error { // set up ssh keys (if configured), create the cloud infrastructure and restart all instances after they've been updated - m.Infrastructure = model.InfrastructureStages{ + m.Infrastructure = model.Stages{ aws_ssh_keys0.Express(), terraform0.Express(), semaphore0.Restart(90 * time.Second), @@ -55,20 +53,17 @@ func (self *stageFactory) Build(m *model.Model) error { // 2. component configurations, which includes controller and router configurations // 3. loop tester configurations // 4. The actual binaries, including the controller, router, ziti-fabric CLI and the ziti-fabric-test tool - m.Configuration = model.ConfigurationStages{ - zitilab_runlevel_1_configuration.IfNoPki(zitilab_runlevel_1_configuration.Fabric("smart.test")), - config.Component(), + m.Configuration = model.Stages{ config.Static([]config.StaticConfig{ {Src: "10-ambient.loop2.yml", Name: "10-ambient.loop2.yml"}, {Src: "4k-chatter.loop2.yml", Name: "4k-chatter.loop2.yml"}, {Src: "remote_identities.yml", Name: "remote_identities.yml"}, }), - zitilab.DefaultZitiBinaries(), } // Create log directories on hosts that need them // Push the artifacts gather in the configuration stage up to the hosts - m.Distribution = model.DistributionStages{ + m.Distribution = model.Stages{ distribution.Locations(models.HasControllerComponent, "logs"), distribution.Locations(models.HasRouterComponent, "logs"), distribution.Locations(models.LoopListenerTag, "logs"), @@ -94,7 +89,7 @@ func (self *stageFactory) Build(m *model.Model) error { } // finally, dispose of the instances and remove any added keys - m.Disposal = model.DisposalStages{ + m.Disposal = model.Stages{ terraform6.Dispose(), aws_ssh_keys6.Dispose(), } diff --git a/zititest/models/stall/main.go b/zititest/models/stall/main.go index 28e94f39e..c48df8ebc 100644 --- a/zititest/models/stall/main.go +++ b/zititest/models/stall/main.go @@ -11,7 +11,6 @@ import ( "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/aws_ssh_key" "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/semaphore" "github.com/openziti/fablab/kernel/lib/runlevel/0_infrastructure/terraform" - "github.com/openziti/fablab/kernel/lib/runlevel/1_configuration/config" distribution "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution" "github.com/openziti/fablab/kernel/lib/runlevel/3_distribution/rsync" aws_ssh_key2 "github.com/openziti/fablab/kernel/lib/runlevel/6_disposal/aws_ssh_key" @@ -20,7 +19,6 @@ import ( "github.com/openziti/fablab/resources" "github.com/openziti/ziti/zititest/zitilab" "github.com/openziti/ziti/zititest/zitilab/actions/edge" - zitilib_runlevel_1_configuration "github.com/openziti/ziti/zititest/zitilab/runlevel/1_configuration" "os" "time" ) @@ -83,11 +81,8 @@ var m = &model.Model{ InstanceType: "c5.large", Components: model.Components{ "ctrl": { - Scope: model.Scope{Tags: model.Tags{"ctrl"}}, - BinaryName: "ziti-controller", - ConfigSrc: "ctrl.yml", - ConfigName: "ctrl.yml", - PublicIdentity: "ctrl", + Scope: model.Scope{Tags: model.Tags{"ctrl"}}, + Type: &zitilab.ControllerType{}, }, }, }, @@ -95,11 +90,8 @@ var m = &model.Model{ InstanceType: "c5.large", Components: model.Components{ "metrics-router": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "no-traversal"}}, - BinaryName: "ziti-router", - ConfigSrc: "router.yml", - ConfigName: "metrics-router.yml", - PublicIdentity: "metrics-router", + Scope: model.Scope{Tags: model.Tags{"edge-router", "no-traversal"}}, + Type: &zitilab.RouterType{}, }, }, }, @@ -114,17 +106,13 @@ var m = &model.Model{ InstanceType: "c5.large", Components: model.Components{ "router-west-{{ .Host.ScaleIndex }}": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "terminator"}}, - BinaryName: "ziti-router", - ConfigSrc: "router.yml", - ConfigName: "router-west-{{ .Host.ScaleIndex }}.yml", - PublicIdentity: "router-west-{{ .Host.ScaleIndex }}", - RunWithSudo: true, + Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "terminator"}}, + Type: &zitilab.RouterType{}, }, "loop-listener-{{ .Host.ScaleIndex }}": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}}, - BinaryName: "ziti-fabric-test", - PublicIdentity: "loop-listener-{{ .Host.ScaleIndex }}", + Scope: model.Scope{Tags: model.Tags{"sdk-app", "service"}}, + // BinaryName: "ziti-fabric-test", + //PublicIdentity: "loop-listener-{{ .Host.ScaleIndex }}", }, }, }, @@ -139,19 +127,15 @@ var m = &model.Model{ InstanceType: "c5.large", Components: model.Components{ "router-ap-{{ .Host.ScaleIndex }}": { - Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "initiator"}}, - BinaryName: "ziti-router", - ConfigSrc: "router.yml", - ConfigName: "router-ap-{{ .Host.ScaleIndex }}.yml", - PublicIdentity: "router-ap-{{ .Host.ScaleIndex }}", - RunWithSudo: true, + Scope: model.Scope{Tags: model.Tags{"edge-router", "tunneler", "initiator"}}, + Type: &zitilab.RouterType{}, }, "loop-client-{{ .Host.ScaleIndex }}": { - Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}}, - BinaryName: "ziti-fabric-test", - ConfigSrc: "test.loop3.yml", - ConfigName: "test.loop3.yml", - PublicIdentity: "loop-client-{{ .Host.ScaleIndex }}", + Scope: model.Scope{Tags: model.Tags{"sdk-app", "client"}}, + //BinaryName: "ziti-fabric-test", + //ConfigSrc: "test.loop3.yml", + //ConfigName: "test.loop3.yml", + //PublicIdentity: "loop-client-{{ .Host.ScaleIndex }}", }, }, }, @@ -172,27 +156,19 @@ var m = &model.Model{ "login": model.Bind(edge.Login("#ctrl")), }, - Infrastructure: model.InfrastructureStages{ + Infrastructure: model.Stages{ aws_ssh_key.Express(), terraform_0.Express(), semaphore_0.Restart(90 * time.Second), }, - Configuration: model.ConfigurationStages{ - zitilib_runlevel_1_configuration.IfPkiNeedsRefresh( - zitilib_runlevel_1_configuration.Fabric("stall.test"), - ), - config.Component(), - zitilab.DefaultZitiBinaries(), - }, - - Distribution: model.DistributionStages{ + Distribution: model.Stages{ distribution.DistributeSshKey("*"), distribution.Locations("*", "logs"), rsync.RsyncStaged(), }, - Disposal: model.DisposalStages{ + Disposal: model.Stages{ terraform.Dispose(), aws_ssh_key2.Dispose(), }, diff --git a/zititest/models/stall/stages.go b/zititest/models/stall/stages.go index 9b3ca2db6..0ac5e21dc 100644 --- a/zititest/models/stall/stages.go +++ b/zititest/models/stall/stages.go @@ -73,7 +73,7 @@ func (_ *stageFactory) listeners(m *model.Model) error { // only start 1 listener c := components[0] - remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.PublicIdentity) + remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.Id) stage := zitilib_5_operation.Loop3Listener(c.GetHost(), nil, "tcp:0.0.0.0:8171", "--config-file", remoteConfigFile, "--health-check-addr 127.0.0.1:8172") m.AddOperatingStage(stage) @@ -91,8 +91,8 @@ func (_ *stageFactory) dialers(m *model.Model, phase fablib_5_operation.Phase) e // only start 1 dialer c := components[0] - remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.PublicIdentity) - stage := zitilib_5_operation.Loop3Dialer(c.GetHost(), c.ConfigName, "tcp:test.service:8171", phase.AddJoiner(), "--config-file", remoteConfigFile, "-d") + remoteConfigFile := fmt.Sprintf("/home/%v/fablab/cfg/%v.json", m.MustVariable("credentials.ssh.username"), c.Id) + stage := zitilib_5_operation.Loop3Dialer(c.GetHost(), c.Id, "tcp:test.service:8171", phase.AddJoiner(), "--config-file", remoteConfigFile, "-d") m.AddOperatingStage(stage) return nil diff --git a/zititest/tests/echo_test.go b/zititest/tests/echo_test.go new file mode 100644 index 000000000..c18a8742a --- /dev/null +++ b/zititest/tests/echo_test.go @@ -0,0 +1,52 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tests + +import ( + "fmt" + "github.com/google/uuid" + "github.com/openziti/fablab/kernel/lib" + "github.com/stretchr/testify/require" + "strings" + "testing" +) + +func TestSdkEcho(t *testing.T) { + components := run.GetModel().SelectComponents("#zcat") + + req := require.New(t) + req.True(len(components) > 0) + + data := "" + for i := 0; i < 1000; i++ { + data += uuid.NewString() + } + + for _, c := range components { + ssh := lib.NewSshConfigFactory(c.GetHost()) + remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.Id + ".json" + + echoClientCmd := fmt.Sprintf(`echo "%s" | /home/%s/fablab/bin/ziti learn demo zcat --identity %s ziti:echo 2>&1`, + string(data), ssh.User(), remoteConfigFile) + + output, err := lib.RemoteExec(ssh, echoClientCmd) + req.NoError(err) + //trim the newline ssh added + output = strings.TrimRight(output, "\n") + req.Equal(string(data), output) + } +} diff --git a/zititest/tests/files_test.go b/zititest/tests/files_test.go index 5e4d5de14..562025bd4 100644 --- a/zititest/tests/files_test.go +++ b/zititest/tests/files_test.go @@ -32,8 +32,8 @@ var hashes = map[string]string{ } var timeouts = map[string]time.Duration{ - "1KB": 5 * time.Second, - "100KB": 5 * time.Second, + "1KB": 10 * time.Second, + "100KB": 10 * time.Second, "20MB": 40 * time.Second, } @@ -44,38 +44,69 @@ const ( ClientWget httpClient = "wget" ) -func TestCurlFiles(t *testing.T) { +func TestDownloadFiles(t *testing.T) { allZetHostedFailed := true - for _, hostType := range []string{"ert", "zet", "ziti-tunnel"} { - for _, clientType := range []string{"ert", "zet", "ziti-tunnel"} { - for _, encrypted := range []bool{true, false} { - success := testFileDownload(t, clientType, ClientCurl, hostType, encrypted, "1KB") - if hostType == "zet" && success { - allZetHostedFailed = false - } - } - } - } + allZetClientsFailed := true - for _, size := range []string{"100KB", "20MB"} { - for _, clientType := range []string{"ert", "zet"} { - for _, hostType := range []string{"ert", "zet"} { - for _, client := range []httpClient{ClientCurl, ClientWget} { - for _, encrypted := range []bool{true, false} { - success := testFileDownload(t, clientType, client, hostType, encrypted, size) - if hostType == "zet" && success { - allZetHostedFailed = false + t.Run("download-tests", func(t *testing.T) { + t.Run("test-ert-downloads", func(t *testing.T) { + t.Parallel() + + for _, size := range []string{"1KB", "100KB", "20MB"} { + for _, hostType := range []string{"ert", "zet", "ziti-tunnel"} { + for _, client := range []httpClient{ClientCurl, ClientWget} { + for _, encrypted := range []bool{true, false} { + success := testFileDownload(t, "ert", client, hostType, encrypted, size) + if hostType == "zet" && success { + allZetHostedFailed = false + } } } } } - } - } + }) - testFileDownload(t, "ziti-tunnel", ClientCurl, "ziti-tunnel", true, "20MB") + t.Run("test-zet-downloads", func(t *testing.T) { + t.Parallel() + + for _, size := range []string{"1KB", "100KB", "20MB"} { + for _, hostType := range []string{"zet", "ziti-tunnel", "ert"} { + for _, client := range []httpClient{ClientCurl, ClientWget} { + for _, encrypted := range []bool{true, false} { + success := testFileDownload(t, "zet", client, hostType, encrypted, size) + if hostType == "zet" && success { + allZetHostedFailed = false + } + if success { + allZetClientsFailed = false + } + } + } + } + } + }) + + t.Run("test-ziti-tunnel-downloads", func(t *testing.T) { + t.Parallel() + + for _, size := range []string{"1KB", "100KB", "20MB"} { + for _, hostType := range []string{"ziti-tunnel", "ert", "zet"} { + for _, client := range []httpClient{ClientCurl, ClientWget} { + for _, encrypted := range []bool{true, false} { + success := testFileDownload(t, "ziti-tunnel", client, hostType, encrypted, size) + if hostType == "zet" && success { + allZetHostedFailed = false + } + } + } + } + } + }) + }) req := require.New(t) req.False(allZetHostedFailed, "all zet hosted file transfer should not failed, indicates bigger issue") + req.False(allZetClientsFailed, "all zet client file transfers should not failed, indicates bigger issue") } func testFileDownload(t *testing.T, hostSelector string, client httpClient, hostType string, encrypted bool, fileSize string) bool { @@ -96,7 +127,7 @@ func testFileDownload(t *testing.T, hostSelector string, client httpClient, host urlExtra = "-unencrypted" } - url := fmt.Sprintf("https://ziti-files-%s%s.s3-us-west-1.amazonaws.ziti/%s.zip", hostType, urlExtra, fileSize) + url := fmt.Sprintf("https://files-%s%s.s3-us-west-1.amazonaws.ziti/%s.zip", hostType, urlExtra, fileSize) sshConfigFactory := lib.NewSshConfigFactory(host) var cmd string @@ -112,6 +143,13 @@ func testFileDownload(t *testing.T, hostSelector string, client httpClient, host t.Skipf("zet hosted file transfer failed [%v]", err.Error()) return } + + if hostSelector == "zet" && err != nil { + t.Skipf("zet client file transfer failed [%v]", err.Error()) + return + } + + t.Log(o) req.NoError(err) req.Equal(hashes[fileSize], o[0:32]) success = true diff --git a/zititest/tests/iperf_test.go b/zititest/tests/iperf_test.go new file mode 100644 index 000000000..1d339b861 --- /dev/null +++ b/zititest/tests/iperf_test.go @@ -0,0 +1,127 @@ +/* + (c) Copyright NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package tests + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/model" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func TestIPerf(t *testing.T) { + t.Run("iperf-tests", func(t *testing.T) { + t.Run("ert-hosted", func(t *testing.T) { + t.Parallel() + + for _, clientType := range []string{"ert", "zet", "ziti-tunnel"} { + for _, encrypted := range []bool{true, false} { + for _, reversed := range []bool{true, false} { + testIPerf(t, clientType, "ert", encrypted, reversed) + } + } + } + }) + + t.Run("zet-hosted", func(t *testing.T) { + t.Parallel() + + for _, clientType := range []string{"zet", "ziti-tunnel", "ert"} { + for _, encrypted := range []bool{true, false} { + for _, reversed := range []bool{true, false} { + testIPerf(t, clientType, "zet", encrypted, reversed) + } + } + } + }) + + t.Run("ziti-tunnel-hosted", func(t *testing.T) { + t.Parallel() + + for _, clientType := range []string{"ziti-tunnel", "ert", "zet"} { + for _, encrypted := range []bool{true, false} { + for _, reversed := range []bool{true, false} { + testIPerf(t, clientType, "ziti-tunnel", encrypted, reversed) + } + } + } + }) + }) +} + +func testIPerf(t *testing.T, hostSelector string, hostType string, encrypted bool, reversed bool) bool { + encDesk := "encrypted" + if !encrypted { + encDesk = "unencrypted" + } + + direction := "->" + if reversed { + direction = "<-" + } + + success := false + + t.Run(fmt.Sprintf("(%s%s%s)-%v", hostSelector, direction, hostType, encDesk), func(t *testing.T) { + host, err := model.GetModel().SelectHost("." + hostSelector + "-client") + req := require.New(t) + req.NoError(err) + + urlExtra := "" + if !encrypted { + urlExtra = "-unencrypted" + } + + addr := fmt.Sprintf("iperf-%s%s.ziti", hostType, urlExtra) + + extraOptions := "" + if reversed { + extraOptions += " -R" + } + + cmd := fmt.Sprintf(`set -o pipefail; iperf3 -c %s -P 1 -t 10 %s`, addr, extraOptions) + + sshConfigFactory := lib.NewSshConfigFactory(host) + o, err := lib.RemoteExecAllWithTimeout(sshConfigFactory, 20*time.Second, cmd) + if hostType == "zet" && err != nil { + t.Skipf("zet hosted iperf test failed [%v]", err.Error()) + return + } + + if hostType == "ziti-tunnel" && err != nil { + t.Skipf("ziti-tunnel hosted iperf test failed [%v]", err.Error()) + return + } + + if hostSelector == "zet" && err != nil { + t.Skipf("zet client iperf test failed [%v]", err.Error()) + return + } + + if hostSelector == "ziti-tunnel" && err != nil { + t.Skipf("ziti-tunnel client iperf test failed [%v]", err.Error()) + return + } + + t.Log(o) + req.NoError(err) + success = true + }) + return success +} diff --git a/zititest/tests/model_test.go b/zititest/tests/model_test.go index 1b990b707..0d22ea134 100644 --- a/zititest/tests/model_test.go +++ b/zititest/tests/model_test.go @@ -44,5 +44,9 @@ func init() { logrus.Fatalf("unable to bootstrap (%s)", err) } - run = model.NewRun() + var err error + run, err = model.NewRun() + if err != nil { + logrus.WithError(err).Fatal("error initializing run") + } } diff --git a/zititest/utils/actions/echoServer.go b/zititest/utils/actions/echoServer.go deleted file mode 100644 index 001ce9280..000000000 --- a/zititest/utils/actions/echoServer.go +++ /dev/null @@ -1,36 +0,0 @@ -package actions - -import ( - "fmt" - - "github.com/openziti/fablab/kernel/lib" - "github.com/openziti/fablab/kernel/model" - "github.com/sirupsen/logrus" -) - -type echoServerStart struct { - componentSpec string -} - -func StartEchoServers(componentSpec string) model.Action { - return &echoServerStart{ - componentSpec: componentSpec, - } -} - -func (esi *echoServerStart) Execute(m *model.Model) error { - return m.ForEachComponent(esi.componentSpec, 1, func(c *model.Component) error { - ssh := lib.NewSshConfigFactory(c.GetHost()) - remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".json" - - echoServerCmd := fmt.Sprintf("nohup /home/%s/fablab/bin/ziti-echo server --identity %s > /home/ubuntu/logs/ziti-echo.log 2>&1 &", - ssh.User(), remoteConfigFile) - - if output, err := lib.RemoteExec(ssh, echoServerCmd); err != nil { - logrus.Errorf("error starting echo server [%s] (%v)", output, err) - return err - } - logrus.Info("echo server started") - return nil - }) -} diff --git a/zititest/utils/ziti-echo/cmd/client.go b/zititest/utils/ziti-echo/cmd/client.go deleted file mode 100644 index 91d72ee36..000000000 --- a/zititest/utils/ziti-echo/cmd/client.go +++ /dev/null @@ -1,73 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "io" - "log" - "net" - "net/http" - "net/url" - "strings" - - "github.com/openziti/sdk-golang/ziti" - "github.com/spf13/cobra" -) - -var clientCmd = &cobra.Command{ - Use: "client {string-to-echo}", - Short: "Simple ziti enabled echo client", - Args: cobra.MinimumNArgs(1), - Run: echoClient, -} - -func init() { - clientCmd.Flags().SetInterspersed(true) - rootCmd.AddCommand(clientCmd) -} - -func echoClient(cmd *cobra.Command, args []string) { - zitiContext, err := ziti.NewContextFromFile(identityFile) - if err != nil { - log.Fatal(err) - } - - dial := func(_ context.Context, _, addr string) (net.Conn, error) { - service := strings.Split(addr, ":")[0] - return zitiContext.Dial(service) - } - - zitiTransport := http.DefaultTransport.(*http.Transport).Clone() - zitiTransport.DialContext = dial - - zec := &zitiEchoClient{ - client: &http.Client{Transport: zitiTransport}, - } - - resp, err := zec.Echo(strings.Join(args, " ")) - if resp != "" { - fmt.Print(resp) - } - if err != nil { - log.Fatal(err) - } -} - -type zitiEchoClient struct { - client *http.Client -} - -func (zec *zitiEchoClient) Echo(input string) (string, error) { - u := fmt.Sprintf("http://echo?input=%v", url.QueryEscape(input)) - resp, err := zec.client.Get(u) - if err != nil { - return "", err - } - defer resp.Body.Close() - - b, err := io.ReadAll(resp.Body) - if err != nil { - return "", err - } - return string(b), err -} diff --git a/zititest/utils/ziti-echo/cmd/root.go b/zititest/utils/ziti-echo/cmd/root.go deleted file mode 100644 index deade56ba..000000000 --- a/zititest/utils/ziti-echo/cmd/root.go +++ /dev/null @@ -1,26 +0,0 @@ -package cmd - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" -) - -var identityFile string - -var rootCmd = &cobra.Command{ - Use: "ziti-echo", - Short: "A simple Echo Service", -} - -func init() { - rootCmd.PersistentFlags().StringVar(&identityFile, "identity", "i", "ziti identity file") -} - -func Execute() { - if err := rootCmd.Execute(); err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/zititest/utils/ziti-echo/cmd/server.go b/zititest/utils/ziti-echo/cmd/server.go deleted file mode 100644 index ee7a45fee..000000000 --- a/zititest/utils/ziti-echo/cmd/server.go +++ /dev/null @@ -1,82 +0,0 @@ -package cmd - -import ( - "log" - "net" - "net/http" - "os" - "os/signal" - "syscall" - "time" - - "github.com/openziti/sdk-golang/ziti" - "github.com/sirupsen/logrus" - "github.com/spf13/cobra" -) - -var serverCmd = &cobra.Command{ - Use: "server", - Short: "Simple Ziti enabled echo serer", - Run: runServer, -} - -func init() { - rootCmd.AddCommand(serverCmd) -} - -func runServer(cmd *cobra.Command, args []string) { - sigs := make(chan os.Signal, 1) - signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM) - - echoServer := &zitiEchoServer{ - identityJson: identityFile, - } - - if err := echoServer.run(); err != nil { - log.Fatal(err) - } - - for { - select { - case <-time.After(10 * time.Second): - logrus.Debug("Beating to show server still running") - case <-sigs: - logrus.Info("Server shutting down...") - return - } - } -} - -type zitiEchoServer struct { - identityJson string - listener net.Listener -} - -func (s *zitiEchoServer) ServeHTTP(rw http.ResponseWriter, r *http.Request) { - input := r.URL.Query().Get("input") - result := input - logrus.Infof("\nziti-http-echo-server: ") - logrus.Infof("received input '%v'\n", input) - if _, err := rw.Write([]byte(result)); err != nil { - panic(err) - } -} - -func (s *zitiEchoServer) run() (err error) { - zitiContext, err := ziti.NewContextFromFile(s.identityJson) - if err != nil { - return err - } - if s.listener, err = zitiContext.Listen("echo"); err != nil { - return err - } - - logrus.Info("\nziti-http-echo-server: ") - logrus.Info("listening for connections from echo server") - go func() { _ = http.Serve(s.listener, http.HandlerFunc(s.ServeHTTP)) }() - return nil -} - -func (s *zitiEchoServer) stop() error { - return s.listener.Close() -} diff --git a/zititest/utils/ziti-echo/main.go b/zititest/utils/ziti-echo/main.go deleted file mode 100644 index f81ee18e2..000000000 --- a/zititest/utils/ziti-echo/main.go +++ /dev/null @@ -1,9 +0,0 @@ -package main - -import ( - "github.com/openziti/ziti/zititest/utils/ziti-echo/cmd" -) - -func main() { - cmd.Execute() -} diff --git a/zititest/zitilab/actions/consul.go b/zititest/zitilab/actions/consul.go index 406e1539e..eef623a9b 100644 --- a/zititest/zitilab/actions/consul.go +++ b/zititest/zitilab/actions/consul.go @@ -26,8 +26,8 @@ func StartConsul(hostSpec, consulServer, configDir, dataPath, logFile string) mo } } -func (cs *consulStart) Execute(m *model.Model) error { - return m.ForEachHost(cs.hostSpec, 24, func(c *model.Host) error { +func (cs *consulStart) Execute(run model.Run) error { + return run.GetModel().ForEachHost(cs.hostSpec, 24, func(c *model.Host) error { ssh := lib.NewSshConfigFactory(c) cmd := fmt.Sprintf("screen -d -m nohup consul agent -join %s -config-dir %s -data-dir %s -log-file %s 2>&1 &", cs.consulServer, cs.configDir, cs.dataPath, cs.logFile) diff --git a/zititest/zitilab/actions/edge/ctrl_available.go b/zititest/zitilab/actions/edge/ctrl_available.go index 9ce4be04c..819a09536 100644 --- a/zititest/zitilab/actions/edge/ctrl_available.go +++ b/zititest/zitilab/actions/edge/ctrl_available.go @@ -14,8 +14,8 @@ func ControllerAvailable(componentSpec string, timeout time.Duration) model.Acti } } -func (self *edgeAvailable) Execute(m *model.Model) error { - for _, c := range m.SelectComponents(self.componentSpec) { +func (self *edgeAvailable) Execute(run model.Run) error { + for _, c := range run.GetModel().SelectComponents(self.componentSpec) { if err := netz.WaitForPortActive(c.Host.PublicIp+":1280", self.timeout); err != nil { return errors.Wrap(err, "controller didn't start in time") } diff --git a/zititest/zitilab/actions/edge/ctrl_init.go b/zititest/zitilab/actions/edge/ctrl_init.go index 46cf721c9..823798de5 100644 --- a/zititest/zitilab/actions/edge/ctrl_init.go +++ b/zititest/zitilab/actions/edge/ctrl_init.go @@ -3,8 +3,10 @@ package edge import ( "fmt" "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/lib/actions/component" "github.com/openziti/fablab/kernel/lib/actions/host" "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab" "github.com/pkg/errors" ) @@ -14,28 +16,8 @@ func InitController(componentSpec string) model.Action { } } -func (init *edgeInit) Execute(m *model.Model) error { - username := m.MustStringVariable("credentials.edge.username") - password := m.MustStringVariable("credentials.edge.password") - - if username == "" { - return errors.New("variable credentials/edge/username must be a string") - } - - if password == "" { - return errors.New("variable credentials/edge/password must be a string") - } - - for _, c := range m.SelectComponents(init.componentSpec) { - sshConfigFactory := lib.NewSshConfigFactory(c.GetHost()) - - tmpl := "rm -f /home/%v/fablab/ctrl.db && set -o pipefail; /home/%s/fablab/bin/%s --log-formatter pfxlog edge init /home/%s/fablab/cfg/%s -u %s -p %s 2>&1 | tee logs/%s.edge.init.log" - if err := host.Exec(c.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), sshConfigFactory.User(), c.BinaryName, sshConfigFactory.User(), c.ConfigName, username, password, c.BinaryName)).Execute(m); err != nil { - return err - } - } - - return nil +func (init *edgeInit) Execute(run model.Run) error { + return component.Exec(init.componentSpec, zitilab.ControllerActionInitStandalone).Execute(run) } type edgeInit struct { @@ -48,7 +30,8 @@ func InitRaftController(componentSpec string) model.Action { } } -func (init *raftInit) Execute(m *model.Model) error { +func (init *raftInit) Execute(run model.Run) error { + m := run.GetModel() username := m.MustStringVariable("credentials.edge.username") password := m.MustStringVariable("credentials.edge.password") @@ -64,7 +47,7 @@ func (init *raftInit) Execute(m *model.Model) error { sshConfigFactory := lib.NewSshConfigFactory(c.GetHost()) tmpl := "set -o pipefail; /home/%s/fablab/bin/ziti agent controller init %s %s default.admin 2>&1 | tee logs/controller.edge.init.log" - if err := host.Exec(c.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), username, password)).Execute(m); err != nil { + if err := host.Exec(c.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), username, password)).Execute(run); err != nil { return err } } diff --git a/zititest/zitilab/actions/edge/init_identities.go b/zititest/zitilab/actions/edge/init_identities.go index c05bf5275..8c8192fc5 100644 --- a/zititest/zitilab/actions/edge/init_identities.go +++ b/zititest/zitilab/actions/edge/init_identities.go @@ -16,22 +16,22 @@ func InitIdentities(componentSpec string, concurrency int) model.Action { } } -func (action *initIdentitiesAction) Execute(m *model.Model) error { - return m.ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error { - if err := zitilib_actions.EdgeExec(m, "delete", "identity", c.PublicIdentity); err != nil { +func (action *initIdentitiesAction) Execute(run model.Run) error { + return run.GetModel().ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error { + if err := zitilib_actions.EdgeExec(run.GetModel(), "delete", "identity", c.Id); err != nil { return err } - return action.createAndEnrollIdentity(c) + return action.createAndEnrollIdentity(run, c) }) } -func (action *initIdentitiesAction) createAndEnrollIdentity(c *model.Component) error { +func (action *initIdentitiesAction) createAndEnrollIdentity(run model.Run, c *model.Component) error { ssh := lib.NewSshConfigFactory(c.GetHost()) - jwtFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".jwt") + jwtFileName := filepath.Join(run.GetTmpDir(), c.Id+".jwt") - err := zitilib_actions.EdgeExec(c.GetModel(), "create", "identity", "service", c.PublicIdentity, + err := zitilib_actions.EdgeExec(c.GetModel(), "create", "identity", "service", c.Id, "--jwt-output-file", jwtFileName, "-a", strings.Join(c.Tags, ",")) @@ -39,7 +39,7 @@ func (action *initIdentitiesAction) createAndEnrollIdentity(c *model.Component) return err } - configFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".json") + configFileName := filepath.Join(run.GetTmpDir(), c.Id+".json") _, err = cli.Exec(c.GetModel(), "edge", "enroll", "--jwt", jwtFileName, "--out", configFileName) @@ -47,7 +47,7 @@ func (action *initIdentitiesAction) createAndEnrollIdentity(c *model.Component) return err } - remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".json" + remoteConfigFile := "/home/ubuntu/fablab/cfg/" + c.Id + ".json" return lib.SendFile(ssh, configFileName, remoteConfigFile) } diff --git a/zititest/zitilab/actions/edge/init_routers.go b/zititest/zitilab/actions/edge/init_routers.go index 38385f37b..69281e386 100644 --- a/zititest/zitilab/actions/edge/init_routers.go +++ b/zititest/zitilab/actions/edge/init_routers.go @@ -1,14 +1,9 @@ package edge import ( - "fmt" - "github.com/michaelquigley/pfxlog" - "github.com/openziti/fablab/kernel/lib" - "github.com/openziti/fablab/kernel/lib/actions/host" + "github.com/openziti/fablab/kernel/lib/actions/component" "github.com/openziti/fablab/kernel/model" - zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" - "path/filepath" - "strings" + "github.com/openziti/ziti/zititest/zitilab" ) func InitEdgeRouters(componentSpec string, concurrency int) model.Action { @@ -18,65 +13,8 @@ func InitEdgeRouters(componentSpec string, concurrency int) model.Action { } } -func (action *initEdgeRoutersAction) Execute(m *model.Model) error { - return m.ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error { - if err := zitilib_actions.EdgeExec(m, "delete", "edge-router", c.PublicIdentity); err != nil { - pfxlog.Logger(). - WithError(err). - WithField("router", c.PublicIdentity). - Warn("unable to delete router (may not be present") - } - - return action.createAndEnrollRouter(c) - }) -} - -func (action *initEdgeRoutersAction) createAndEnrollRouter(c *model.Component) error { - ssh := lib.NewSshConfigFactory(c.GetHost()) - - jwtFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".jwt") - - attributes := strings.Join(c.Tags, ",") - - args := []string{"create", "edge-router", c.PublicIdentity, "-j", "--jwt-output-file", jwtFileName, "-a", attributes} - - isTunneler := c.HasLocalOrAncestralTag("tunneler") - if isTunneler { - args = append(args, "--tunneler-enabled") - } - - if c.HasLocalOrAncestralTag("no-traversal") { - args = append(args, "--no-traversal") - } - - if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil { - return err - } - - if isTunneler { - if err := zitilib_actions.EdgeExec(c.GetModel(), "update", "identity", c.PublicIdentity, "-a", attributes); err != nil { - return err - } - } - - remoteJwt := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".jwt" - if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil { - return err - } - - tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%s enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log " - cmds := []string{ - "mkdir -p /home/ubuntu/logs", - fmt.Sprintf(tmpl, c.BinaryName, c.ConfigName, remoteJwt, c.ConfigName), - } - - if isTunneler { - cmds = append(cmds, - "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", - "sudo systemctl restart systemd-resolved") - } - - return host.Exec(c.GetHost(), cmds...).Execute(c.GetModel()) +func (action *initEdgeRoutersAction) Execute(run model.Run) error { + return component.ExecInParallel(action.componentSpec, action.concurrency, zitilab.RouterActionsCreateAndEnroll).Execute(run) } type initEdgeRoutersAction struct { diff --git a/zititest/zitilab/actions/edge/login.go b/zititest/zitilab/actions/edge/login.go index 359ac1c8c..f5be5d81a 100644 --- a/zititest/zitilab/actions/edge/login.go +++ b/zititest/zitilab/actions/edge/login.go @@ -14,7 +14,8 @@ func Login(componentSelector string) model.Action { } } -func (l *login) Execute(m *model.Model) error { +func (l *login) Execute(run model.Run) error { + m := run.GetModel() ctrl, err := m.SelectComponent(l.componentSelector) if err != nil { return err @@ -24,7 +25,7 @@ func (l *login) Execute(m *model.Model) error { password := m.MustStringVariable("credentials.edge.password") edgeApiBaseUrl := ctrl.Host.PublicIp + ":1280" - caChain := filepath.Join(model.PkiBuild(), ctrl.PublicIdentity, "certs", ctrl.PublicIdentity+".cert") + caChain := filepath.Join(model.KitBuild(), model.BuildPkiDir, ctrl.Id, "certs", ctrl.Id+".cert") if username == "" { return errors.New("variable credentials/edge/username must be a string") diff --git a/zititest/zitilab/actions/edge/raft_join.go b/zititest/zitilab/actions/edge/raft_join.go index c975557ee..f3c3676f4 100644 --- a/zititest/zitilab/actions/edge/raft_join.go +++ b/zititest/zitilab/actions/edge/raft_join.go @@ -18,8 +18,8 @@ type raftJoin struct { componentSpec string } -func (self *raftJoin) Execute(m *model.Model) error { - ctrls := m.SelectComponents(self.componentSpec) +func (self *raftJoin) Execute(run model.Run) error { + ctrls := run.GetModel().SelectComponents(self.componentSpec) if len(ctrls) < 1 { return errors.Errorf("no controllers found with spec '%v'", self.componentSpec) } @@ -27,7 +27,7 @@ func (self *raftJoin) Execute(m *model.Model) error { sshConfigFactory := lib.NewSshConfigFactory(primary.GetHost()) for _, c := range ctrls[1:] { tmpl := "/home/%s/fablab/bin/ziti agent cluster add %v --id %v" - if err := host.Exec(primary.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), "tls:"+c.Host.PublicIp+":6262", c.PublicIdentity)).Execute(m); err != nil { + if err := host.Exec(primary.GetHost(), fmt.Sprintf(tmpl, sshConfigFactory.User(), "tls:"+c.Host.PublicIp+":6262", c.Id)).Execute(run); err != nil { return err } } diff --git a/zititest/zitilab/actions/edge/reenroll_routers.go b/zititest/zitilab/actions/edge/reenroll_routers.go index 78a3fe336..0b7890211 100644 --- a/zititest/zitilab/actions/edge/reenroll_routers.go +++ b/zititest/zitilab/actions/edge/reenroll_routers.go @@ -1,12 +1,9 @@ package edge import ( - "fmt" - "github.com/openziti/fablab/kernel/lib" - "github.com/openziti/fablab/kernel/lib/actions/host" + "github.com/openziti/fablab/kernel/lib/actions/component" "github.com/openziti/fablab/kernel/model" - zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" - "path/filepath" + "github.com/openziti/ziti/zititest/zitilab" ) func ReEnrollEdgeRouters(componentSpec string, concurrency int) model.Action { @@ -16,41 +13,8 @@ func ReEnrollEdgeRouters(componentSpec string, concurrency int) model.Action { } } -func (action *reEnrollEdgeRoutersAction) Execute(m *model.Model) error { - return m.ForEachComponent(action.componentSpec, action.concurrency, func(c *model.Component) error { - return action.reEnrollRouter(c) - }) -} - -func (action *reEnrollEdgeRoutersAction) reEnrollRouter(c *model.Component) error { - ssh := lib.NewSshConfigFactory(c.GetHost()) - - jwtFileName := filepath.Join(model.ConfigBuild(), c.PublicIdentity+".jwt") - - args := []string{"re-enroll", "edge-router", c.PublicIdentity, "-j", "--jwt-output-file", jwtFileName} - - if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil { - return err - } - - remoteJwt := "/home/ubuntu/fablab/cfg/" + c.PublicIdentity + ".jwt" - if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil { - return err - } - - tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%s enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log " - cmds := []string{ - "mkdir -p /home/ubuntu/logs", - fmt.Sprintf(tmpl, c.BinaryName, c.ConfigName, remoteJwt, c.ConfigName), - } - - if c.HasLocalOrAncestralTag("tunneler") { - cmds = append(cmds, - "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", - "sudo systemctl restart systemd-resolved") - } - - return host.Exec(c.GetHost(), cmds...).Execute(c.GetModel()) +func (action *reEnrollEdgeRoutersAction) Execute(run model.Run) error { + return component.ExecInParallel(action.componentSpec, action.concurrency, zitilab.RouterActionsReEnroll).Execute(run) } type reEnrollEdgeRoutersAction struct { diff --git a/zititest/zitilab/actions/edge/sync_model_edge_state.go b/zititest/zitilab/actions/edge/sync_model_edge_state.go index 1b85bb815..9201923fa 100644 --- a/zititest/zitilab/actions/edge/sync_model_edge_state.go +++ b/zititest/zitilab/actions/edge/sync_model_edge_state.go @@ -14,13 +14,13 @@ func SyncModelEdgeState(componentSpec string) model.Action { } } -func (action *syncModelEdgeStateAction) Execute(m *model.Model) error { - routerComponents := m.SelectComponents(action.componentSpec) +func (action *syncModelEdgeStateAction) Execute(run model.Run) error { + routerComponents := run.GetModel().SelectComponents(action.componentSpec) if len(routerComponents) == 0 { return errors.Errorf("no router components found for selector '%v'", action.componentSpec) } - output, err := zitilib_actions.EdgeExecWithOutput(m, "list", "edge-routers", "--output-json", "true limit none") + output, err := zitilib_actions.EdgeExecWithOutput(run.GetModel(), "list", "edge-routers", "--output-json", "true limit none") if err != nil { return err } @@ -45,7 +45,7 @@ func (action *syncModelEdgeStateAction) Execute(m *model.Model) error { routerName := router.S("name").Data().(string) for _, c := range routerComponents { - if c.PublicIdentity == routerName { + if c.Id == routerName { routerId = strings.ReplaceAll(routerId, ".", ":") c.Tags = append(c.Tags, "edgeId:"+routerId) } diff --git a/zititest/zitilab/actions/edge_cli.go b/zititest/zitilab/actions/edge_cli.go index d67c04ab4..f80c85534 100644 --- a/zititest/zitilab/actions/edge_cli.go +++ b/zititest/zitilab/actions/edge_cli.go @@ -27,8 +27,8 @@ func Edge(args ...string) model.Action { } } -func (a *edge) Execute(m *model.Model) error { - return EdgeExec(m, a.args...) +func (a *edge) Execute(run model.Run) error { + return EdgeExec(run.GetModel(), a.args...) } type edge struct { diff --git a/zititest/zitilab/actions/fabric_cli.go b/zititest/zitilab/actions/fabric_cli.go index de58d771c..10c3bd0c0 100644 --- a/zititest/zitilab/actions/fabric_cli.go +++ b/zititest/zitilab/actions/fabric_cli.go @@ -27,8 +27,8 @@ func Fabric(args ...string) model.Action { } } -func (a *fabric) Execute(m *model.Model) error { - _, err := cli.Exec(m, append([]string{"fabric", "-i", model.ActiveInstanceId()}, a.args...)...) +func (a *fabric) Execute(run model.Run) error { + _, err := cli.Exec(run.GetModel(), append([]string{"fabric", "-i", model.ActiveInstanceId()}, a.args...)...) return err } diff --git a/zititest/zitilab/actions/logs.go b/zititest/zitilab/actions/logs.go index 33ab3857e..a51dadb62 100644 --- a/zititest/zitilab/actions/logs.go +++ b/zititest/zitilab/actions/logs.go @@ -31,13 +31,13 @@ func Logs() model.Action { return &logs{} } -func (self *logs) Execute(m *model.Model) error { - if !m.IsBound() { +func (self *logs) Execute(run model.Run) error { + if !run.GetModel().IsBound() { return fmt.Errorf("model not bound") } snapshot := fmt.Sprintf("%d", info.NowInMilliseconds()) - for rn, r := range m.Regions { + for rn, r := range run.GetModel().Regions { for hn, h := range r.Hosts { ssh := lib.NewSshConfigFactory(h) if err := self.forHost(snapshot, rn, hn, ssh); err != nil { diff --git a/zititest/zitilab/actions/metricbeat.go b/zititest/zitilab/actions/metricbeat.go index dd32156b1..79ec78121 100644 --- a/zititest/zitilab/actions/metricbeat.go +++ b/zititest/zitilab/actions/metricbeat.go @@ -24,8 +24,8 @@ func StartMetricbeat(hostSpec, configPath, dataPath, logPath string) model.Actio } } -func (mbs *metricbeatStart) Execute(m *model.Model) error { - return m.ForEachHost(mbs.hostSpec, 24, func(c *model.Host) error { +func (mbs *metricbeatStart) Execute(run model.Run) error { + return run.GetModel().ForEachHost(mbs.hostSpec, 24, func(c *model.Host) error { ssh := lib.NewSshConfigFactory(c) cmd := fmt.Sprintf("screen -d -m nohup metricbeat --path.config %s --path.data %s --path.logs %s 2>&1 &", mbs.configPath, mbs.dataPath, mbs.logPath) diff --git a/zititest/zitilab/component_common.go b/zititest/zitilab/component_common.go new file mode 100644 index 000000000..34b030712 --- /dev/null +++ b/zititest/zitilab/component_common.go @@ -0,0 +1,70 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zitilab + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/model" + "github.com/sirupsen/logrus" + "strings" +) + +func getZitiProcessFilter(c *model.Component, zitiType string) func(string) bool { + return func(s string) bool { + return strings.Contains(s, "ziti") && + strings.Contains(s, zitiType) && + strings.Contains(s, fmt.Sprintf("--cli-agent-alias %s", c.Id)) && + !strings.Contains(s, "sudo ") + } +} + +func startZitiComponent(c *model.Component, zitiType string, version string, configName string) error { + binaryName := "ziti" + if version != "" { + binaryName += "-" + version + } + + factory := lib.NewSshConfigFactory(c.GetHost()) + + binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), binaryName) + configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s", factory.User(), configName) + logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id) + + if zitiType == "tunnel" { + + } + + useSudo := "" + if zitiType == "tunnel" || c.HasTag("tunneler") { + useSudo = "sudo" + } + + serviceCmd := fmt.Sprintf("nohup %s %s %s run --log-formatter pfxlog %s --cli-agent-alias %s > %s 2>&1 &", + useSudo, binaryPath, zitiType, configPath, c.Id, logsPath) + + value, err := lib.RemoteExec(factory, serviceCmd) + if err != nil { + return err + } + + if len(value) > 0 { + logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n")) + } + + return nil +} diff --git a/zititest/zitilab/component_controller.go b/zititest/zitilab/component_controller.go new file mode 100644 index 000000000..62a9fc154 --- /dev/null +++ b/zititest/zitilab/component_controller.go @@ -0,0 +1,136 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zitilab + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/lib/actions/host" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/pki" + "github.com/openziti/ziti/zititest/zitilab/stageziti" + "github.com/pkg/errors" + "io/fs" +) + +var _ model.ComponentType = (*ControllerType)(nil) +var _ model.ServerComponent = (*ControllerType)(nil) +var _ model.FileStagingComponent = (*ControllerType)(nil) +var _ model.ActionsComponent = (*ControllerType)(nil) + +const ( + ControllerActionInitStandalone = "initStandalone" +) + +type ControllerType struct { + ConfigSourceFS fs.FS + ConfigSource string + ConfigName string + Version string + LocalPath string +} + +func (self *ControllerType) GetActions() map[string]model.ComponentAction { + return map[string]model.ComponentAction{ + ControllerActionInitStandalone: model.ComponentActionF(self.InitStandalone), + } +} + +func (self *ControllerType) Dump() any { + return map[string]string{ + "type_id": "controller", + "config_source": self.ConfigSource, + "config_name": self.ConfigName, + "version": self.Version, + "local_path": self.LocalPath, + } +} + +func (self *ControllerType) StageFiles(r model.Run, c *model.Component) error { + configSource := self.ConfigSource + if configSource == "" { + configSource = "ctrl.yml.tmpl" + } + + configName := self.getConfigName(c) + + if err := lib.GenerateConfigForComponent(c, self.ConfigSourceFS, configSource, configName, r); err != nil { + return err + } + + if err := pki.CreateControllerCerts(r, c, c.Id); err != nil { + return err + } + + return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath) +} + +func (self *ControllerType) getConfigName(c *model.Component) string { + configName := self.ConfigName + if configName == "" { + configName = c.Id + ".yml" + } + return configName +} + +func (self *ControllerType) getProcessFilter(c *model.Component) func(string) bool { + return getZitiProcessFilter(c, "controller") +} + +func (self *ControllerType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + factory := lib.NewSshConfigFactory(c.GetHost()) + pids, err := lib.FindProcesses(factory, self.getProcessFilter(c)) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *ControllerType) Start(_ model.Run, c *model.Component) error { + return startZitiComponent(c, "controller", self.Version, self.getConfigName(c)) +} + +func (self *ControllerType) Stop(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + return lib.RemoteKillFilterF(factory, self.getProcessFilter(c)) +} + +func (self *ControllerType) InitStandalone(run model.Run, c *model.Component) error { + username := c.MustStringVariable("credentials.edge.username") + password := c.MustStringVariable("credentials.edge.password") + + if username == "" { + return errors.New("variable credentials/edge/username must be a string") + } + + if password == "" { + return errors.New("variable credentials/edge/password must be a string") + } + + factory := lib.NewSshConfigFactory(c.GetHost()) + + binaryName := "ziti" + if self.Version != "" { + binaryName += "-" + self.Version + } + + configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s", factory.User(), self.getConfigName(c)) + + tmpl := "rm -f /home/%v/fablab/ctrl.db && set -o pipefail; /home/%s/fablab/bin/%s controller --log-formatter pfxlog edge init %s -u %s -p %s 2>&1 | tee logs/controller.edge.init.log" + cmd := fmt.Sprintf(tmpl, factory.User(), factory.User(), binaryName, configPath, username, password) + return host.Exec(c.GetHost(), cmd).Execute(run) +} diff --git a/zititest/zitilab/component_echo_server.go b/zititest/zitilab/component_echo_server.go new file mode 100644 index 000000000..513402ec0 --- /dev/null +++ b/zititest/zitilab/component_echo_server.go @@ -0,0 +1,76 @@ +package zitilab + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/stageziti" + "github.com/sirupsen/logrus" + "strings" +) + +var _ model.ComponentType = (*EchoServerType)(nil) + +type EchoServerMode int + +type EchoServerType struct { + Version string + LocalPath string +} + +func (self *EchoServerType) Dump() any { + return map[string]string{ + "type_id": "echo-server", + "version": self.Version, + "local_path": self.LocalPath, + } +} + +func (self *EchoServerType) StageFiles(r model.Run, c *model.Component) error { + return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath) +} + +func (self *EchoServerType) getProcessFilter(c *model.Component) func(string) bool { + return getZitiProcessFilter(c, "echo-server") +} + +func (self *EchoServerType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + factory := lib.NewSshConfigFactory(c.GetHost()) + pids, err := lib.FindProcesses(factory, self.getProcessFilter(c)) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *EchoServerType) Start(_ model.Run, c *model.Component) error { + binaryName := "ziti" + if self.Version != "" { + binaryName += "-" + self.Version + } + + factory := lib.NewSshConfigFactory(c.GetHost()) + + binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), binaryName) + configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s.json", factory.User(), c.Id) + logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id) + + serviceCmd := fmt.Sprintf("nohup %s learn demo echo-server -i %s --cli-agent-alias %s > %s 2>&1 &", + binaryPath, configPath, c.Id, logsPath) + + value, err := lib.RemoteExec(factory, serviceCmd) + if err != nil { + return err + } + + if len(value) > 0 { + logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n")) + } + + return nil +} + +func (self *EchoServerType) Stop(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + return lib.RemoteKillFilterF(factory, self.getProcessFilter(c)) +} diff --git a/zititest/zitilab/component_iperf.go b/zititest/zitilab/component_iperf.go new file mode 100644 index 000000000..af20df11d --- /dev/null +++ b/zititest/zitilab/component_iperf.go @@ -0,0 +1,83 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zitilab + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/model" + "github.com/sirupsen/logrus" + "strings" +) + +var _ model.ComponentType = (*IPerfServerType)(nil) + +type IPerfServerType struct { + Port uint16 +} + +func (self *IPerfServerType) Dump() any { + return map[string]string{ + "type_id": "iperf-server", + "port": fmt.Sprintf("%v", self.GetPort()), + } +} + +func (self *IPerfServerType) GetPort() uint16 { + if self.Port == 0 { + return 5201 + } + return self.Port +} + +func (self *IPerfServerType) getProcessFilter(*model.Component) func(string) bool { + return func(s string) bool { + return strings.Contains(s, fmt.Sprintf("iperf3 -s -p %v", self.GetPort())) + } +} + +func (self *IPerfServerType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + factory := lib.NewSshConfigFactory(c.GetHost()) + pids, err := lib.FindProcesses(factory, self.getProcessFilter(c)) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *IPerfServerType) Start(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + + logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id) + serviceCmd := fmt.Sprintf("nohup iperf3 -s -p %v > %s 2>&1 &", self.GetPort(), logsPath) + + value, err := lib.RemoteExec(factory, serviceCmd) + if err != nil { + return err + } + + if len(value) > 0 { + logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n")) + } + + return nil +} + +func (self *IPerfServerType) Stop(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + return lib.RemoteKillFilterF(factory, self.getProcessFilter(c)) +} diff --git a/zititest/zitilab/component_router.go b/zititest/zitilab/component_router.go new file mode 100644 index 000000000..8103d1359 --- /dev/null +++ b/zititest/zitilab/component_router.go @@ -0,0 +1,200 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zitilab + +import ( + "fmt" + "github.com/michaelquigley/pfxlog" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/lib/actions/host" + "github.com/openziti/fablab/kernel/model" + zitilib_actions "github.com/openziti/ziti/zititest/zitilab/actions" + "github.com/openziti/ziti/zititest/zitilab/stageziti" + "io/fs" + "path/filepath" + "strings" +) + +var _ model.ComponentType = (*RouterType)(nil) + +const ( + RouterActionsCreateAndEnroll = "createAndEnroll" + RouterActionsReEnroll = "reEnroll" +) + +type RouterType struct { + ConfigSourceFS fs.FS + ConfigSource string + ConfigName string + Version string + LocalPath string +} + +func (self *RouterType) GetActions() map[string]model.ComponentAction { + return map[string]model.ComponentAction{ + RouterActionsCreateAndEnroll: model.ComponentActionF(self.CreateAndEnroll), + RouterActionsReEnroll: model.ComponentActionF(self.ReEnroll), + } +} + +func (self *RouterType) Dump() any { + return map[string]string{ + "type_id": "router", + "config_source": self.ConfigSource, + "config_name": self.ConfigName, + "version": self.Version, + "local_path": self.LocalPath, + } +} + +func (self *RouterType) InitializeHost(run model.Run, c *model.Component) error { + if self.isTunneler(c) { + cmds := []string{ + "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", + "sudo systemctl restart systemd-resolved", + "mkdir -p /home/ubuntu/logs", + } + return host.Exec(c.GetHost(), cmds...).Execute(run) + } + return nil +} + +func (self *RouterType) StageFiles(r model.Run, c *model.Component) error { + configSource := self.ConfigSource + if configSource == "" { + configSource = "router.yml.tmpl" + } + + configName := self.getConfigName(c) + + if err := lib.GenerateConfigForComponent(c, self.ConfigSourceFS, configSource, configName, r); err != nil { + return err + } + + return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath) +} + +func (self *RouterType) isTunneler(c *model.Component) bool { + return c.HasLocalOrAncestralTag("tunneler") +} + +func (self *RouterType) getConfigName(c *model.Component) string { + configName := self.ConfigName + if configName == "" { + configName = c.Id + ".yml" + } + return configName +} + +func (self *RouterType) getBinaryName() string { + binaryName := "ziti" + version := self.Version + if version != "" { + binaryName += "-" + version + } + return binaryName +} + +func (self *RouterType) getProcessFilter(c *model.Component) func(string) bool { + return getZitiProcessFilter(c, "router") +} + +func (self *RouterType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + factory := lib.NewSshConfigFactory(c.GetHost()) + pids, err := lib.FindProcesses(factory, self.getProcessFilter(c)) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *RouterType) Start(_ model.Run, c *model.Component) error { + return startZitiComponent(c, "router", self.Version, self.getConfigName(c)) +} + +func (self *RouterType) Stop(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + return lib.RemoteKillFilterF(factory, self.getProcessFilter(c)) +} + +func (self *RouterType) CreateAndEnroll(run model.Run, c *model.Component) error { + if err := zitilib_actions.EdgeExec(c.GetModel(), "delete", "edge-router", c.Id); err != nil { + pfxlog.Logger(). + WithError(err). + WithField("router", c.Id). + Warn("unable to delete router (may not be present") + } + + ssh := lib.NewSshConfigFactory(c.GetHost()) + + jwtFileName := filepath.Join(run.GetTmpDir(), c.Id+".jwt") + + attributes := strings.Join(c.Tags, ",") + + args := []string{"create", "edge-router", c.Id, "-j", "--jwt-output-file", jwtFileName, "-a", attributes} + + isTunneler := c.HasLocalOrAncestralTag("tunneler") + if isTunneler { + args = append(args, "--tunneler-enabled") + } + + if c.HasLocalOrAncestralTag("no-traversal") { + args = append(args, "--no-traversal") + } + + if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil { + return err + } + + if isTunneler { + if err := zitilib_actions.EdgeExec(c.GetModel(), "update", "identity", c.Id, "-a", attributes); err != nil { + return err + } + } + + remoteJwt := "/home/ubuntu/fablab/cfg/" + c.Id + ".jwt" + if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil { + return err + } + + tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%v router enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log " + cmd := fmt.Sprintf(tmpl, self.getBinaryName(), self.getConfigName(c), remoteJwt, c.Id) + + return host.Exec(c.GetHost(), cmd).Execute(run) +} + +func (self *RouterType) ReEnroll(run model.Run, c *model.Component) error { + ssh := lib.NewSshConfigFactory(c.GetHost()) + + jwtFileName := filepath.Join(model.ConfigBuild(), c.Id+".jwt") + + args := []string{"re-enroll", "edge-router", c.Id, "-j", "--jwt-output-file", jwtFileName} + + if err := zitilib_actions.EdgeExec(c.GetModel(), args...); err != nil { + return err + } + + remoteJwt := "/home/ubuntu/fablab/cfg/" + c.Id + ".jwt" + if err := lib.SendFile(ssh, jwtFileName, remoteJwt); err != nil { + return err + } + + tmpl := "set -o pipefail; /home/ubuntu/fablab/bin/%s enroll /home/ubuntu/fablab/cfg/%s -j %s 2>&1 | tee /home/ubuntu/logs/%s.router.enroll.log " + cmd := fmt.Sprintf(tmpl, self.getBinaryName(), self.getConfigName(c), remoteJwt, c.Id) + + return host.Exec(c.GetHost(), cmd).Execute(run) +} diff --git a/zititest/zitilab/component_zcat.go b/zititest/zitilab/component_zcat.go new file mode 100644 index 000000000..8608320e0 --- /dev/null +++ b/zititest/zitilab/component_zcat.go @@ -0,0 +1,49 @@ +package zitilab + +import ( + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/stageziti" + "strings" +) + +var _ model.ComponentType = (*ZCatType)(nil) + +type ZCatMode int + +type ZCatType struct { + Version string + LocalPath string +} + +func (self *ZCatType) Dump() any { + return map[string]string{ + "type_id": "zcat", + "version": self.Version, + "local_path": self.LocalPath, + } +} + +func (self *ZCatType) StageFiles(r model.Run, c *model.Component) error { + return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath) +} + +func (self *ZCatType) getProcessFilter(c *model.Component) func(string) bool { + return func(s string) bool { + return strings.Contains(s, "ziti") && strings.Contains(s, "zcat ") + } +} + +func (self *ZCatType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + factory := lib.NewSshConfigFactory(c.GetHost()) + pids, err := lib.FindProcesses(factory, self.getProcessFilter(c)) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *ZCatType) Stop(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + return lib.RemoteKillFilterF(factory, self.getProcessFilter(c)) +} diff --git a/zititest/zitilab/component_ziti_edge_tunnel.go b/zititest/zitilab/component_ziti_edge_tunnel.go new file mode 100644 index 000000000..ba831408f --- /dev/null +++ b/zititest/zitilab/component_ziti_edge_tunnel.go @@ -0,0 +1,104 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zitilab + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/stageziti" + "github.com/sirupsen/logrus" + "strings" +) + +var _ model.ComponentType = (*ZitiEdgeTunnelType)(nil) + +type ZitiEdgeTunnelType struct { + Version string + LocalPath string +} + +func (self *ZitiEdgeTunnelType) Dump() any { + return map[string]string{ + "type_id": "ziti-edge-tunnel", + "version": self.Version, + "local_path": self.LocalPath, + } +} + +func (self *ZitiEdgeTunnelType) getVersion() string { + if strings.HasPrefix(self.Version, "v") { + return self.Version[1:] + } + return self.Version +} + +func (self *ZitiEdgeTunnelType) getBinaryName() string { + binaryName := "ziti-edge-tunnel" + version := self.getVersion() + if version != "" { + binaryName += "-" + version + } + return binaryName +} + +func (self *ZitiEdgeTunnelType) StageFiles(r model.Run, c *model.Component) error { + return stageziti.StageZitiEdgeTunnelOnce(r, c, self.getVersion(), self.LocalPath) +} + +func (self *ZitiEdgeTunnelType) getProcessFilter(c *model.Component) func(string) bool { + return func(s string) bool { + return strings.Contains(s, self.getBinaryName()) && + strings.Contains(s, fmt.Sprintf("%s.json", c.Id)) && + !strings.Contains(s, "sudo ") + } +} + +func (self *ZitiEdgeTunnelType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + factory := lib.NewSshConfigFactory(c.GetHost()) + pids, err := lib.FindProcesses(factory, self.getProcessFilter(c)) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *ZitiEdgeTunnelType) Start(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + + binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), self.getBinaryName()) + configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s.json", factory.User(), c.Id) + logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id) + + serviceCmd := fmt.Sprintf("nohup sudo %s run -i %s > %s 2>&1 &", binaryPath, configPath, logsPath) + + value, err := lib.RemoteExec(factory, serviceCmd) + if err != nil { + return err + } + + if len(value) > 0 { + logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n")) + } + + return nil +} + +func (self *ZitiEdgeTunnelType) Stop(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + return lib.RemoteKillFilterF(factory, self.getProcessFilter(c)) +} diff --git a/zititest/zitilab/component_ziti_tunnel.go b/zititest/zitilab/component_ziti_tunnel.go new file mode 100644 index 000000000..154b6a6a3 --- /dev/null +++ b/zititest/zitilab/component_ziti_tunnel.go @@ -0,0 +1,131 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package zitilab + +import ( + "fmt" + "github.com/openziti/fablab/kernel/lib" + "github.com/openziti/fablab/kernel/lib/actions/host" + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/zititest/zitilab/stageziti" + "github.com/sirupsen/logrus" + "strings" +) + +var _ model.ComponentType = (*ZitiTunnelType)(nil) + +type ZitiTunnelMode int + +const ( + ZitiTunnelModeTproxy ZitiTunnelMode = 0 + ZitiTunnelModeProxy ZitiTunnelMode = 1 + ZitiTunnelModeHost ZitiTunnelMode = 2 +) + +func (self ZitiTunnelMode) String() string { + if self == ZitiTunnelModeTproxy { + return "tproxy" + } + if self == ZitiTunnelModeProxy { + return "proxy" + } + if self == ZitiTunnelModeHost { + return "host" + } + return "invalid" +} + +type ZitiTunnelType struct { + Mode ZitiTunnelMode + Version string + LocalPath string +} + +func (self *ZitiTunnelType) Dump() any { + return map[string]string{ + "type_id": "ziti-tunnel", + "version": self.Version, + "local_path": self.LocalPath, + } +} + +func (self *ZitiTunnelType) StageFiles(r model.Run, c *model.Component) error { + return stageziti.StageZitiOnce(r, c, self.Version, self.LocalPath) +} + +func (self *ZitiTunnelType) InitializeHost(run model.Run, c *model.Component) error { + cmds := []string{"mkdir -p /home/ubuntu/logs"} + if self.Mode == ZitiTunnelModeTproxy { + cmds = append(cmds, + "sudo sed -i 's/#DNS=/DNS=127.0.0.1/g' /etc/systemd/resolved.conf", + "sudo systemctl restart systemd-resolved", + ) + } + return host.Exec(c.GetHost(), cmds...).Execute(run) +} + +func (self *ZitiTunnelType) getProcessFilter(c *model.Component) func(string) bool { + return getZitiProcessFilter(c, "tunnel") +} + +func (self *ZitiTunnelType) IsRunning(_ model.Run, c *model.Component) (bool, error) { + factory := lib.NewSshConfigFactory(c.GetHost()) + pids, err := lib.FindProcesses(factory, self.getProcessFilter(c)) + if err != nil { + return false, err + } + return len(pids) > 0, nil +} + +func (self *ZitiTunnelType) Start(_ model.Run, c *model.Component) error { + binaryName := "ziti" + if self.Version != "" { + binaryName += "-" + self.Version + } + + mode := self.Mode + + factory := lib.NewSshConfigFactory(c.GetHost()) + + binaryPath := fmt.Sprintf("/home/%s/fablab/bin/%s", factory.User(), binaryName) + configPath := fmt.Sprintf("/home/%s/fablab/cfg/%s.json", factory.User(), c.Id) + logsPath := fmt.Sprintf("/home/%s/logs/%s.log", factory.User(), c.Id) + + useSudo := "" + if mode == ZitiTunnelModeTproxy { + useSudo = "sudo" + } + + serviceCmd := fmt.Sprintf("nohup %s %s tunnel %s --log-formatter pfxlog -i %s --cli-agent-alias %s > %s 2>&1 &", + useSudo, binaryPath, mode.String(), configPath, c.Id, logsPath) + + value, err := lib.RemoteExec(factory, serviceCmd) + if err != nil { + return err + } + + if len(value) > 0 { + logrus.Infof("output [%s]", strings.Trim(value, " \t\r\n")) + } + + return nil +} + +func (self *ZitiTunnelType) Stop(_ model.Run, c *model.Component) error { + factory := lib.NewSshConfigFactory(c.GetHost()) + return lib.RemoteKillSignalFilterF(factory, "-KILL", self.getProcessFilter(c)) +} diff --git a/zititest/zitilab/console/action.go b/zititest/zitilab/console/action.go index af506729a..d9863a58f 100644 --- a/zititest/zitilab/console/action.go +++ b/zititest/zitilab/console/action.go @@ -30,7 +30,7 @@ func Console() model.Action { return &console{} } -func (consoleAction *console) Execute(m *model.Model) error { +func (consoleAction *console) Execute(model.Run) error { server := NewServer() go server.Listen() diff --git a/zititest/zitilab/models/db_builder.go b/zititest/zitilab/models/db_builder.go index 2d52b1204..bfda6ae84 100644 --- a/zititest/zitilab/models/db_builder.go +++ b/zititest/zitilab/models/db_builder.go @@ -7,6 +7,7 @@ import ( "github.com/openziti/fabric/controller/db" "github.com/openziti/fabric/controller/network" "github.com/openziti/storage/boltz" + "github.com/openziti/ziti/zititest/zitilab" "github.com/pkg/errors" "go.etcd.io/bbolt" "strings" @@ -112,11 +113,9 @@ func (self *ZitiDbBuilder) Build(m *model.Model) error { region.Hosts["router_"+id] = host component := &model.Component{ - Scope: model.Scope{Tags: model.Tags{}}, - ConfigSrc: "router.yml", - ConfigName: fmt.Sprintf("router-%v.yml", er.Id), - BinaryName: "ziti router", - Host: host, + Scope: model.Scope{Tags: model.Tags{}}, + Type: &zitilab.RouterType{}, + Host: host, } host.Components[er.Id] = component diff --git a/zititest/zitilab/paths.go b/zititest/zitilab/paths.go index a444c599d..053eb1c5c 100644 --- a/zititest/zitilab/paths.go +++ b/zititest/zitilab/paths.go @@ -25,7 +25,7 @@ func ZitiRoot() string { return zitiRoot } -func DefaultZitiBinaries() model.ConfigurationStage { +func DefaultZitiBinaries() model.Stage { zitiBinaries := []string{ "ziti", "ziti-controller", diff --git a/zititest/zitilab/pki/cert.go b/zititest/zitilab/pki/cert.go new file mode 100644 index 000000000..ac145fc6a --- /dev/null +++ b/zititest/zitilab/pki/cert.go @@ -0,0 +1,153 @@ +/* + Copyright 2019 NetFoundry Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package pki + +import ( + "bytes" + "fmt" + "github.com/pkg/errors" + "os" + "path/filepath" + "strings" + + "github.com/openziti/fablab/kernel/model" + "github.com/openziti/ziti/ziti/cmd" + "github.com/sirupsen/logrus" +) + +func HasExistingCA(run model.Run, name string) (bool, error) { + return run.DirExists(filepath.Join(model.BuildKitDir, model.BuildPkiDir, name)) +} + +func EnsureCaExists(run model.Run, trustDomain string, name string) error { + if caExists, err := HasExistingCA(run, name); caExists || err != nil { + return err + } + + return GenerateCA(run, trustDomain, name) +} + +func GenerateCA(run model.Run, trustDomain string, name string) error { + var pkiOut bytes.Buffer + var pkiErr bytes.Buffer + ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr) + args := []string{"pki", "create", "ca", "--pki-root", run.GetPkiDir(), "--ca-name", name, "--ca-file", name, "--trust-domain", trustDomain, "--"} + ziticli.SetArgs(args) + logrus.Infof("%v", args) + if err := ziticli.Execute(); err != nil { + logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n")) + return errors.Wrapf(err, "error generating ca '%s'", name) + } + + return nil +} + +func EnsureIntermediateCaExists(run model.Run, caName, name string) error { + logrus.Infof("generating signing certificate [%s]", name) + + if caExists, err := HasExistingCA(run, name); caExists || err != nil { + return err + } + + var pkiOut bytes.Buffer + var pkiErr bytes.Buffer + + ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr) + + args := []string{"pki", "create", "intermediate", "--pki-root", run.GetPkiDir(), "--ca-name", caName, + "--intermediate-file", name, "--intermediate-name", name + " signing cert"} + ziticli.SetArgs(args) + logrus.Infof("%v", args) + if err := ziticli.Execute(); err != nil { + logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n")) + return fmt.Errorf("error generating key (%s)", err) + } + return nil +} + +func loadInfoFile(file string) string { + data, err := os.ReadFile(file) + if err != nil { + if os.IsNotExist(err) { + return "" + } + logrus.WithError(err).Info("unable to read cert info file '%s'", file) + } + return string(data) +} + +func EnsureServerCertExists(run model.Run, name string, ip string, spiffeId string) error { + logrus.Infof("generating server certificate [%s:%s]", name, ip) + + certFile := name + "-server" + certFilePath := filepath.Join(model.BuildKitDir, model.BuildPkiDir, name, "certs", certFile) + + certExists, err := run.FileExists(certFilePath + ".cert") + if err != nil { + return err + } + + certInfo := ip + "\n" + spiffeId + "\n" + infoFile := filepath.Join(run.GetWorkingDir(), certFilePath+".info") + + if certExists { + cachedInfo := loadInfoFile(infoFile) + if certInfo == cachedInfo { + logrus.Infof("server cert %v already up to date", name) + return nil + } + } + + var pkiOut bytes.Buffer + var pkiErr bytes.Buffer + ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr) + + args := []string{"pki", "create", "server", + "--server-name", name, + "--pki-root", run.GetPkiDir(), + "--ca-name", name, + "--server-file", certFile, + "--allow-overwrite", + "--ip", ip} + + if spiffeId != "" { + args = append(args, "--spiffe-id", spiffeId) + } + + logrus.Infof("%v", args) + ziticli.SetArgs(args) + if err := ziticli.Execute(); err != nil { + logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n")) + return fmt.Errorf("error generating server certificate (%s)", err) + } + + return os.WriteFile(infoFile, []byte(certInfo), 0600) +} + +func CreateControllerCerts(run model.Run, component *model.Component, name string) error { + trustDomain := component.GetStringVariableOr("ca.trustDomain", "ziti.test") + rootCaName := component.GetStringVariableOr("ca.rootName", "root") + if err := EnsureCaExists(run, trustDomain, rootCaName); err != nil { + return fmt.Errorf("error generating ca (%s)", err) + } + + if err := EnsureIntermediateCaExists(run, rootCaName, name); err != nil { + return errors.Wrapf(err, "error generating public identity for component [%s]", component.Id) + } + + return EnsureServerCertExists(run, name, component.Host.PublicIp, "controller/"+name) +} diff --git a/zititest/zitilab/runlevel/0_infrastructure/consul.go b/zititest/zitilab/runlevel/0_infrastructure/consul.go index eb4407683..beb72a0db 100644 --- a/zititest/zitilab/runlevel/0_infrastructure/consul.go +++ b/zititest/zitilab/runlevel/0_infrastructure/consul.go @@ -12,13 +12,13 @@ type installConsul struct { hostSpec string } -func InstallConsul(hostSpec string) model.InfrastructureStage { +func InstallConsul(hostSpec string) model.Stage { return &installConsul{ hostSpec: hostSpec, } } -func (imb *installConsul) Express(run model.Run) error { +func (imb *installConsul) Execute(run model.Run) error { return run.GetModel().ForEachHost(imb.hostSpec, 25, func(host *model.Host) error { ssh := lib.NewSshConfigFactory(host) diff --git a/zititest/zitilab/runlevel/0_infrastructure/metricbeat.go b/zititest/zitilab/runlevel/0_infrastructure/metricbeat.go index 9fa68ca09..24f3ab9ba 100644 --- a/zititest/zitilab/runlevel/0_infrastructure/metricbeat.go +++ b/zititest/zitilab/runlevel/0_infrastructure/metricbeat.go @@ -15,14 +15,14 @@ type installMetricbeat struct { version string } -func InstallMetricbeat(hostSpec, version string) model.InfrastructureStage { +func InstallMetricbeat(hostSpec, version string) model.Stage { return &installMetricbeat{ hostSpec: hostSpec, version: version, } } -func (imb *installMetricbeat) Express(run model.Run) error { +func (imb *installMetricbeat) Execute(run model.Run) error { return run.GetModel().ForEachHost(imb.hostSpec, 25, func(host *model.Host) error { ssh := lib.NewSshConfigFactory(host) diff --git a/zititest/zitilab/runlevel/1_configuration/cert.go b/zititest/zitilab/runlevel/1_configuration/cert.go deleted file mode 100644 index 8ebfc9783..000000000 --- a/zititest/zitilab/runlevel/1_configuration/cert.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - Copyright 2019 NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package zitilib_runlevel_1_configuration - -import ( - "bytes" - "fmt" - "strings" - - "github.com/openziti/fablab/kernel/model" - "github.com/openziti/ziti/ziti/cmd" - "github.com/sirupsen/logrus" -) - -func generateCa(trustDomain string) error { - if caExists, err := hasExistingCA("root"); caExists || err != nil { - return err - } - - var pkiOut bytes.Buffer - var pkiErr bytes.Buffer - ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr) - args := []string{"pki", "create", "ca", "--pki-root", model.PkiBuild(), "--ca-name", "root", "--ca-file", "root", "--trust-domain", trustDomain, "--"} - ziticli.SetArgs(args) - logrus.Infof("%v", args) - if err := ziticli.Execute(); err != nil { - logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n")) - return fmt.Errorf("error generating key (%s)", err) - } - - return nil -} - -func generateSigningCert(name string) error { - logrus.Infof("generating signing certificate [%s]", name) - - var pkiOut bytes.Buffer - var pkiErr bytes.Buffer - - ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr) - - args := []string{"pki", "create", "intermediate", "--pki-root", model.PkiBuild(), "--ca-name", "root", - "--intermediate-file", name, "--intermediate-name", name + " signing cert"} - ziticli.SetArgs(args) - logrus.Infof("%v", args) - if err := ziticli.Execute(); err != nil { - logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n")) - return fmt.Errorf("error generating key (%s)", err) - } - return nil -} - -func generateCert(name, ip, spiffeId string) error { - logrus.Infof("generating server certificate [%s:%s]", name, ip) - - var pkiOut bytes.Buffer - var pkiErr bytes.Buffer - ziticli := cmd.NewRootCommand(nil, &pkiOut, &pkiErr) - - args := []string{"pki", "create", "server", - "--server-name", name, - "--pki-root", model.PkiBuild(), - "--ca-name", name, - "--server-file", fmt.Sprintf("%s-server", name), - "--ip", ip} - - if spiffeId != "" { - args = append(args, "--spiffe-id", spiffeId) - } - - logrus.Infof("%v", args) - ziticli.SetArgs(args) - if err := ziticli.Execute(); err != nil { - logrus.Errorf("stdOut [%s], stdErr [%s]", strings.Trim(pkiOut.String(), " \t\r\n"), strings.Trim(pkiErr.String(), " \t\r\n")) - return fmt.Errorf("error generating server certificate (%s)", err) - } - - return nil -} diff --git a/zititest/zitilab/runlevel/1_configuration/fabric.go b/zititest/zitilab/runlevel/1_configuration/fabric.go deleted file mode 100644 index 56ef33ccb..000000000 --- a/zititest/zitilab/runlevel/1_configuration/fabric.go +++ /dev/null @@ -1,176 +0,0 @@ -/* - Copyright 2019 NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package zitilib_runlevel_1_configuration - -import ( - "fmt" - "github.com/openziti/fablab/kernel/model" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" - "gopkg.in/yaml.v3" - "os" - "path" - "strings" -) - -func Fabric(trustDomain string, componentSpecs ...string) model.ConfigurationStage { - if len(componentSpecs) == 0 { - componentSpecs = []string{"*"} - } - return &fabric{ - trustDomain: trustDomain, - componentSpecs: componentSpecs, - } -} - -type fabric struct { - trustDomain string - componentSpecs []string -} - -func (f *fabric) getSpiffeId(c *model.Component, id string) string { - for _, tag := range c.Tags { - if strings.HasPrefix(tag, "spiffe:") { - prefix := tag[len("spiffe:"):] - return prefix + "/" + id - } - } - return id -} - -func (f *fabric) Configure(run model.Run) error { - m := run.GetModel() - if err := generateCa(f.trustDomain); err != nil { - return fmt.Errorf("error generating ca (%s)", err) - } - - ips := map[string]string{} - - processedComponents := map[*model.Component]struct{}{} - - for _, spec := range f.componentSpecs { - for _, component := range m.SelectComponents(spec) { - if _, found := processedComponents[component]; found { - continue - } - processedComponents[component] = struct{}{} - - if component.PublicIdentity != "" { - logrus.Infof("generating public ip identity [%s/%s] on [%s/%s]", component.Id, component.PublicIdentity, component.Region().Id, component.Host.Id) - if err := generateSigningCert(component.PublicIdentity); err != nil { - return errors.Wrapf(err, "error generating public identity [%s/%s]", component.Id, component.PublicIdentity) - } - - if err := generateCert(component.PublicIdentity, component.Host.PublicIp, f.getSpiffeId(component, component.PublicIdentity)); err != nil { - return fmt.Errorf("error generating public identity [%s/%s]", component.Id, component.PublicIdentity) - } - - ips[component.GetPathId()+".public"] = component.Host.PublicIp - } - - if component.PrivateIdentity != "" { - logrus.Infof("generating private ip identity [%s/%s] on [%s/%s]", component.Id, component.PrivateIdentity, component.Region().Id, component.Host.Id) - if err := generateSigningCert(component.PrivateIdentity); err != nil { - return errors.Wrapf(err, "error generating private identity [%s/%s]", component.Id, component.PublicIdentity) - } - - if err := generateCert(component.PrivateIdentity, component.Host.PrivateIp, f.getSpiffeId(component, component.PrivateIdentity)); err != nil { - return fmt.Errorf("error generating private identity [%s/%s]", component.Id, component.PrivateIdentity) - } - ips[component.GetPathId()+".private"] = component.Host.PrivateIp - } - } - } - - return storeIps(ips) -} - -func haveIpsChanged(m *model.Model) (bool, error) { - ipFile := path.Join(model.PkiBuild(), "ips") - ips := map[string]string{} - if _, err := os.Stat(ipFile); err == nil { - ipData, err := os.ReadFile(ipFile) - if err != nil { - return false, err - } - if err = yaml.Unmarshal(ipData, &ips); err != nil { - return false, err - } - } - - for _, c := range m.SelectComponents("*") { - pubIdKey := c.GetPathId() + ".public" - if c.PublicIdentity != "" { - if ip, found := ips[pubIdKey]; found { - if ip == c.Host.PublicIp { - delete(ips, pubIdKey) - } else { - logrus.Infof("public ip for %v/%v has changed from %v -> %v. rebuilding pki", c.Id, c.PublicIdentity, c.Host.PublicIp, ip) - return true, nil - } - } else { - logrus.Infof("missing cert public identity of %v/%v. rebuilding pki", c.Id, c.PublicIdentity) - return true, nil - } - } - - privIdKey := c.GetPathId() + ".private" - if c.PrivateIdentity != "" { - if ip, found := ips[privIdKey]; found { - if ip == c.Host.PrivateIp { - delete(ips, privIdKey) - } else { - logrus.Infof("private ip for %v/%v has changed from %v -> %v. rebuilding pki", c.Id, c.PrivateIdentity, c.Host.PrivateIp, ip) - return true, nil - } - } else { - logrus.Infof("missing cert for private identity of %v/%v. rebuilding pki", c.Id, c.PrivateIdentity) - } - } - } - - return false, nil -} - -func storeIps(ips map[string]string) error { - ipFile := path.Join(model.PkiBuild(), "ips") - data, err := yaml.Marshal(ips) - if err != nil { - return err - } - return os.WriteFile(ipFile, data, 0600) -} - -func hasExisitingPki() (bool, error) { - if _, err := os.Stat(model.PkiBuild()); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return true, err - } - return true, nil -} - -func hasExistingCA(name string) (bool, error) { - if _, err := os.Stat(path.Join(model.PkiBuild(), name)); err != nil { - if os.IsNotExist(err) { - return false, nil - } - return true, err - } - return true, nil -} diff --git a/zititest/zitilab/runlevel/1_configuration/if_no_pki.go b/zititest/zitilab/runlevel/1_configuration/if_no_pki.go deleted file mode 100644 index 6cc74ea73..000000000 --- a/zititest/zitilab/runlevel/1_configuration/if_no_pki.go +++ /dev/null @@ -1,50 +0,0 @@ -/* - Copyright 2019 NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package zitilib_runlevel_1_configuration - -import ( - "fmt" - "github.com/openziti/fablab/kernel/model" - "github.com/sirupsen/logrus" -) - -func IfNoPki(stages ...model.ConfigurationStage) model.ConfigurationStage { - return &ifNoPki{stages: stages} -} - -func (self *ifNoPki) Configure(run model.Run) error { - if existing, err := hasExisitingPki(); err == nil { - if existing { - logrus.Infof("skipping configuration. existing pki system at [%s]", model.PkiBuild()) - return nil - } - } else { - return fmt.Errorf("error checking pki existence at [%s] (%s)", model.PkiBuild(), err) - } - - for _, stage := range self.stages { - if err := stage.Configure(run); err != nil { - return fmt.Errorf("error running configuration stage (%w)", err) - } - } - - return nil -} - -type ifNoPki struct { - stages []model.ConfigurationStage -} diff --git a/zititest/zitilab/runlevel/1_configuration/if_pki_needs_refresh.go b/zititest/zitilab/runlevel/1_configuration/if_pki_needs_refresh.go deleted file mode 100644 index 19c7bc4e4..000000000 --- a/zititest/zitilab/runlevel/1_configuration/if_pki_needs_refresh.go +++ /dev/null @@ -1,51 +0,0 @@ -/* - Copyright 2019 NetFoundry Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ - -package zitilib_runlevel_1_configuration - -import ( - "fmt" - "github.com/openziti/fablab/kernel/model" - "os" -) - -func IfPkiNeedsRefresh(stages ...model.ConfigurationStage) model.ConfigurationStage { - return &ifPkiNeedsRefresh{stages: stages} -} - -func (self *ifPkiNeedsRefresh) Configure(run model.Run) error { - if changed, err := haveIpsChanged(run.GetModel()); err != nil || !changed { - return err - } - - if existing, err := hasExisitingPki(); err == nil && existing { - if err = os.RemoveAll(model.PkiBuild()); err != nil { - return err - } - } - - for _, stage := range self.stages { - if err := stage.Configure(run); err != nil { - return fmt.Errorf("error running configuration stage (%w)", err) - } - } - - return nil -} - -type ifPkiNeedsRefresh struct { - stages []model.ConfigurationStage -} diff --git a/zititest/zitilab/runlevel/5_operation/circuit_metrics.go b/zititest/zitilab/runlevel/5_operation/circuit_metrics.go index 469fc67d7..93ed07568 100644 --- a/zititest/zitilab/runlevel/5_operation/circuit_metrics.go +++ b/zititest/zitilab/runlevel/5_operation/circuit_metrics.go @@ -14,7 +14,7 @@ import ( "time" ) -func CircuitMetrics(pollFreq time.Duration, closer <-chan struct{}, f func(string) string) model.OperatingStage { +func CircuitMetrics(pollFreq time.Duration, closer <-chan struct{}, f func(string) string) model.Stage { return &circuitMetrics{ closer: closer, circuits: map[string]struct{}{}, @@ -34,7 +34,7 @@ type circuitMetrics struct { idToSelectorMapper func(string) string } -func (self *circuitMetrics) Operate(run model.Run) error { +func (self *circuitMetrics) Execute(run model.Run) error { self.model = run.GetModel() bindHandler := func(binding channel.Binding) error { binding.AddReceiveHandler(int32(mgmt_pb.ContentType_StreamEventsEventType), channel.ReceiveHandlerF(self.receiveCircuitEvents)) diff --git a/zititest/zitilab/runlevel/5_operation/client_metrics.go b/zititest/zitilab/runlevel/5_operation/client_metrics.go index 9e422d245..9a482a4d7 100644 --- a/zititest/zitilab/runlevel/5_operation/client_metrics.go +++ b/zititest/zitilab/runlevel/5_operation/client_metrics.go @@ -53,7 +53,7 @@ type ClientMetrics struct { idToSelectorMapper func(string) string } -func (metrics *ClientMetrics) Activate(run model.Run) error { +func (metrics *ClientMetrics) Execute(run model.Run) error { if err := zitilib_actions.EdgeExec(run.GetModel(), "delete", "identity", "metrics-host"); err != nil { return err } diff --git a/zititest/zitilab/runlevel/5_operation/loop_dialer.go b/zititest/zitilab/runlevel/5_operation/loop_dialer.go index 964a9e9d8..1fe6402a5 100644 --- a/zititest/zitilab/runlevel/5_operation/loop_dialer.go +++ b/zititest/zitilab/runlevel/5_operation/loop_dialer.go @@ -24,7 +24,7 @@ import ( "strings" ) -func Loop3Dialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.OperatingStage { +func Loop3Dialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.Stage { return &loopDialer{ host: host, scenario: scenario, @@ -35,7 +35,7 @@ func Loop3Dialer(host *model.Host, scenario, endpoint string, joiner chan struct } } -func LoopDialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.OperatingStage { +func LoopDialer(host *model.Host, scenario, endpoint string, joiner chan struct{}, extraArgs ...string) model.Stage { return &loopDialer{ host: host, scenario: scenario, @@ -46,7 +46,7 @@ func LoopDialer(host *model.Host, scenario, endpoint string, joiner chan struct{ } } -func (self *loopDialer) Operate(run model.Run) error { +func (self *loopDialer) Execute(run model.Run) error { ssh := lib.NewSshConfigFactory(self.host) if err := lib.RemoteKill(ssh, fmt.Sprintf("ziti-fabric-test %v dialer", self.subcmd)); err != nil { return fmt.Errorf("error killing %v listeners (%w)", self.subcmd, err) diff --git a/zititest/zitilab/runlevel/5_operation/loop_listener.go b/zititest/zitilab/runlevel/5_operation/loop_listener.go index 83d93655f..efc85a087 100644 --- a/zititest/zitilab/runlevel/5_operation/loop_listener.go +++ b/zititest/zitilab/runlevel/5_operation/loop_listener.go @@ -8,7 +8,7 @@ import ( "strings" ) -func Loop3Listener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.OperatingStage { +func Loop3Listener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.Stage { return &loopListener{ host: host, joiner: joiner, @@ -18,7 +18,7 @@ func Loop3Listener(host *model.Host, joiner chan struct{}, bindAddress string, e } } -func LoopListener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.OperatingStage { +func LoopListener(host *model.Host, joiner chan struct{}, bindAddress string, extraArgs ...string) model.Stage { return &loopListener{ host: host, joiner: joiner, @@ -28,7 +28,7 @@ func LoopListener(host *model.Host, joiner chan struct{}, bindAddress string, ex } } -func (self *loopListener) Operate(run model.Run) error { +func (self *loopListener) Execute(run model.Run) error { ssh := lib.NewSshConfigFactory(self.host) if err := lib.RemoteKill(ssh, fmt.Sprintf("ziti-fabric-test %v listener", self.subcmd)); err != nil { return fmt.Errorf("error killing %v listeners (%w)", self.subcmd, err) diff --git a/zititest/zitilab/runlevel/5_operation/mesh.go b/zititest/zitilab/runlevel/5_operation/mesh.go index 4e8b30e69..14b5c9df1 100644 --- a/zititest/zitilab/runlevel/5_operation/mesh.go +++ b/zititest/zitilab/runlevel/5_operation/mesh.go @@ -26,11 +26,11 @@ import ( "time" ) -func Mesh(closer <-chan struct{}) model.OperatingStage { +func Mesh(closer <-chan struct{}) model.Stage { return &mesh{closer: closer} } -func (mesh *mesh) Operate(run model.Run) error { +func (mesh *mesh) Execute(run model.Run) error { if endpoint, id, err := dotziti.LoadIdentity(model.ActiveInstanceId()); err == nil { if address, err := transport.ParseAddress(endpoint); err == nil { dialer := channel.NewClassicDialer(id, address, nil) diff --git a/zititest/zitilab/runlevel/5_operation/model_metrics.go b/zititest/zitilab/runlevel/5_operation/model_metrics.go index 9b61e49c9..55608842b 100644 --- a/zititest/zitilab/runlevel/5_operation/model_metrics.go +++ b/zititest/zitilab/runlevel/5_operation/model_metrics.go @@ -27,13 +27,13 @@ import ( "time" ) -func ModelMetrics(closer <-chan struct{}) model.OperatingStage { +func ModelMetrics(closer <-chan struct{}) model.Stage { return ModelMetricsWithIdMapper(closer, func(id string) string { return "#" + id }) } -func ModelMetricsWithIdMapper(closer <-chan struct{}, f func(string) string) model.OperatingStage { +func ModelMetricsWithIdMapper(closer <-chan struct{}, f func(string) string) model.Stage { return &modelMetrics{ closer: closer, idToSelectorMapper: f, @@ -47,7 +47,7 @@ type modelMetrics struct { idToSelectorMapper func(string) string } -func (self *modelMetrics) Operate(run model.Run) error { +func (self *modelMetrics) Execute(run model.Run) error { self.m = run.GetModel() bindHandler := func(binding channel.Binding) error { diff --git a/zititest/zitilab/stageziti/stageziti.go b/zititest/zitilab/stageziti/stageziti.go index 87fa7a592..b2c0162d6 100644 --- a/zititest/zitilab/stageziti/stageziti.go +++ b/zititest/zitilab/stageziti/stageziti.go @@ -1,31 +1,120 @@ package stageziti import ( - "github.com/openziti/fablab/kernel/lib/runlevel/2_kitting/devkit" "github.com/openziti/fablab/kernel/model" "github.com/openziti/ziti/common/getziti" + "github.com/openziti/ziti/ziti/util" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "os" + "os/exec" + "path/filepath" ) -func FetchZitiEdgeTunnel(version string) model.ConfigurationStage { - return &ZitiEdgeTunnelFetchStage{ - Version: version, +func StageZitiOnce(run model.Run, component *model.Component, version string, source string) error { + op := "install.ziti-" + if version == "" { + op += "local" + } else { + op += version } + + return run.DoOnce(op, func() error { + return StageZiti(run, component, version, source) + }) } -type ZitiEdgeTunnelFetchStage struct { - Version string - OS string - Arch string +func StageZitiEdgeTunnelOnce(run model.Run, component *model.Component, version string, source string) error { + op := "install.ziti-edge-tunnel-" + if version == "" { + op += "local" + } else { + op += version + } + + return run.DoOnce(op, func() error { + return StageZitiEdgeTunnel(run, component, version, source) + }) } -func (self *ZitiEdgeTunnelFetchStage) Configure(run model.Run) error { - return devkit.FetchBinaries(func(binDir string) error { - if self.OS == "" { - self.OS = "linux" +func StageZiti(run model.Run, component *model.Component, version string, source string) error { + fileName := "ziti" + if version != "" { + fileName += "-" + version + } + + target := filepath.Join(run.GetBinDir(), fileName) + if version == "" || version == "latest" { + _ = os.Remove(target) + } + + if version == "" { + if source != "" { + logrus.Infof("[%s] => [%s]", source, target) + return util.CopyFile(source, target) } - if self.Arch == "" { - self.Arch = "amd64" + if envSource, found := component.GetStringVariable("ziti.path"); found { + logrus.Infof("[%s] => [%s]", envSource, target) + return util.CopyFile(envSource, target) } - return getziti.InstallZitiEdgeTunnel(self.Version, self.OS, self.Arch, binDir, false) - }).Configure(run) + if zitiPath, err := exec.LookPath("ziti"); err == nil { + logrus.Infof("[%s] => [%s]", zitiPath, target) + return util.CopyFile(zitiPath, target) + } + return errors.New("ziti binary not found in path, no path provided and no ziti.path env variable set") + } + + found, err := run.FileExists(filepath.Join(model.BuildKitDir, model.BuildBinDir, fileName)) + if err != nil { + return err + } + + if found { + logrus.Infof("%s already present, not downloading again", target) + return nil + } + logrus.Infof("%s not present, attempting to fetch", target) + + return getziti.InstallZiti(version, "linux", "amd64", run.GetBinDir(), false) +} + +func StageZitiEdgeTunnel(run model.Run, component *model.Component, version string, source string) error { + fileName := "ziti-edge-tunnel" + if version != "" { + fileName += "-" + version + } + + target := filepath.Join(run.GetBinDir(), fileName) + if version == "" || version == "latest" { + _ = os.Remove(target) + } + + if version == "" { + if source != "" { + logrus.Infof("[%s] => [%s]", source, target) + return util.CopyFile(source, target) + } + if envSource, found := component.GetStringVariable("ziti-edge-tunnel.path"); found { + logrus.Infof("[%s] => [%s]", envSource, target) + return util.CopyFile(envSource, target) + } + if zitiPath, err := exec.LookPath("ziti-edge-tunnel"); err == nil { + logrus.Infof("[%s] => [%s]", zitiPath, target) + return util.CopyFile(zitiPath, target) + } + return errors.New("ziti-edge-tunnel binary not found in path, no path provided and no ziti-edge-tunnel.path env variable set") + } + + found, err := run.FileExists(filepath.Join(model.BuildKitDir, model.BuildBinDir, fileName)) + if err != nil { + return err + } + + if found { + logrus.Infof("%s already present, not downloading again", target) + return nil + } + logrus.Infof("%s not present, attempting to fetch", target) + + return getziti.InstallZitiEdgeTunnel(version, "linux", "amd64", run.GetBinDir(), false) }